From 17f5fb1565943f23984bbecc0460fbba80dc87cc Mon Sep 17 00:00:00 2001 From: "wangzihao.wzh" Date: Tue, 2 Jun 2026 17:53:33 +0800 Subject: [PATCH] MDEV-39822 vector index returns wrong results for large M (M >= 128) The HNSW graph serializes each node's per-layer neighbor list into the neighbors blob as ......, where the count was both written (FVectorNode::save) and read (FVectorNode::load_from_record) as a single byte. But layer 0 allows up to max_neighbors(0) == 2*M neighbors, and the per-index M option is bound to the default_m sysvar whose maximum is 200, so a node can accumulate up to 400 neighbors. The count then overflows the byte (256 -> 0, 400 -> 144), which isolates or truncates nodes and breaks graph connectivity. A subsequent ORDER BY VEC_DISTANCE ... LIMIT k search can no longer navigate away from the entry point and returns far-away rows instead of the nearest ones. This is triggered whenever 2*M > 255, i.e. M >= 128. Store counts >= 255 with a 0xff escape byte followed by a 2-byte little-endian count. Counts below 255 keep the original one-byte encoding, so the on-disk format is unchanged for every index that previously worked correctly (only M >= 128 could overflow), and those indexes need no rebuild. Also harden the read path: reject a neighbor count that exceeds max_neighbors() for the layer. Otherwise a corrupt or legacy blob -- for example one written before this fix with exactly 255 neighbors stored as a literal 0xff, now misread as the escape marker -- could drive the links[] fill loop past the allocated array (heap out-of-bounds write). Co-Authored-By: Claude Opus 4.8 (1M context) --- mysql-test/main/vector_mdev_large_m.result | 34 ++++++++++++++++ mysql-test/main/vector_mdev_large_m.test | 27 +++++++++++++ sql/vector_mhnsw.cc | 47 +++++++++++++++++++++- 3 files changed, 106 insertions(+), 2 deletions(-) create mode 100644 mysql-test/main/vector_mdev_large_m.result create mode 100644 mysql-test/main/vector_mdev_large_m.test diff --git a/mysql-test/main/vector_mdev_large_m.result b/mysql-test/main/vector_mdev_large_m.result new file mode 100644 index 00000000000..fbb0300faa4 --- /dev/null +++ b/mysql-test/main/vector_mdev_large_m.result @@ -0,0 +1,34 @@ +# +# MDEV-39822: vector index returns wrong results for a large M (M >= 128). +# +# The per-layer neighbor count in the HNSW graph blob was stored in a +# single byte, but layer 0 allows up to 2*M (== 400 at M=200) neighbors, +# so the count overflowed the byte (256 -> 0, 400 -> 144). That isolated +# or truncated nodes, broke graph connectivity, and a kNN search could no +# longer reach the true nearest neighbors. +# +create table t (id int primary key, v vector(1) not null, +vector index (v) distance=euclidean m=200); +# Fill with 1-D vectors [1], [2], ... [10000]. The dense 1-D layout makes +# some layer-0 nodes exceed 255 neighbors (M=200 allows 2*M = 400). +set @save_max_recursive_iterations= @@max_recursive_iterations; +set max_recursive_iterations= 1000000; +insert into t +with recursive s as (select 1 as n union all select n + 1 from s where n < 10000) +select n, vec_fromtext(concat('[', n, ']')) from s; +set max_recursive_iterations= @save_max_recursive_iterations; +# The 10 nearest vectors to [1] must be ids 1..10. +# Before the fix this returned far-away rows instead. +select id from t order by vec_distance_euclidean(v, vec_fromtext('[1]')) limit 10; +id +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +drop table t; diff --git a/mysql-test/main/vector_mdev_large_m.test b/mysql-test/main/vector_mdev_large_m.test new file mode 100644 index 00000000000..9dec3199157 --- /dev/null +++ b/mysql-test/main/vector_mdev_large_m.test @@ -0,0 +1,27 @@ +--echo # +--echo # MDEV-39822: vector index returns wrong results for a large M (M >= 128). +--echo # +--echo # The per-layer neighbor count in the HNSW graph blob was stored in a +--echo # single byte, but layer 0 allows up to 2*M (== 400 at M=200) neighbors, +--echo # so the count overflowed the byte (256 -> 0, 400 -> 144). That isolated +--echo # or truncated nodes, broke graph connectivity, and a kNN search could no +--echo # longer reach the true nearest neighbors. +--echo # + +create table t (id int primary key, v vector(1) not null, + vector index (v) distance=euclidean m=200); + +--echo # Fill with 1-D vectors [1], [2], ... [10000]. The dense 1-D layout makes +--echo # some layer-0 nodes exceed 255 neighbors (M=200 allows 2*M = 400). +set @save_max_recursive_iterations= @@max_recursive_iterations; +set max_recursive_iterations= 1000000; +insert into t +with recursive s as (select 1 as n union all select n + 1 from s where n < 10000) +select n, vec_fromtext(concat('[', n, ']')) from s; +set max_recursive_iterations= @save_max_recursive_iterations; + +--echo # The 10 nearest vectors to [1] must be ids 1..10. +--echo # Before the fix this returned far-away rows instead. +select id from t order by vec_distance_euclidean(v, vec_fromtext('[1]')) limit 10; + +drop table t; diff --git a/sql/vector_mhnsw.cc b/sql/vector_mhnsw.cc index c480c36c7e7..deeeb071688 100644 --- a/sql/vector_mhnsw.cc +++ b/sql/vector_mhnsw.cc @@ -36,6 +36,20 @@ static constexpr float subdist_margin= 1.05f; static constexpr double subdist_stddev_threshold= 0.05; // 3σ, p>99.9% static constexpr ulonglong subdist_stddev_valid= 10000; // sufficient +/* + A node's per-layer neighbor count is serialized into the neighbors blob as a + single byte. But layer 0 allows up to max_neighbors(0) == 2*M neighbors, and + M can be as large as 200, so the count can reach 400 and overflow the byte. + Use 0xff as an escape marker followed by a 2-byte little-endian count; counts + below 255 keep the original one-byte encoding, so the on-disk format is + unchanged for any index that worked before (only M >= 128 could overflow). +*/ +static constexpr uchar neighbor_count_escape= 0xff; +static inline size_t neighbor_count_pack_len(size_t num) +{ + return num < neighbor_count_escape ? 1 : 3; +} + /* The class below can assume normal distribution and only collect M1 and M2, or go beyond that and collect M3 and M4 to account @@ -1030,7 +1044,24 @@ int FVectorNode::load_from_record(TABLE *graph) { if (unlikely(ptr >= end)) return my_errno= HA_ERR_CRASHED; + // a 0xff count byte is an escape for a 2-byte count, see save() size_t grefs= *ptr++; + if (grefs == neighbor_count_escape) + { + if (unlikely(ptr + 2 > end)) + return my_errno= HA_ERR_CRASHED; + grefs= uint2korr(ptr); + ptr+= 2; + } + /* + grefs must fit the per-layer links[] array (sized max_neighbors(i)). This + also hardens against a corrupt/legacy blob: an index built before this fix + could store a layer with exactly 255 neighbors as a literal 0xff, now + misread as the escape marker -- without this check the loop below could + write past the allocated links[] array. + */ + if (unlikely(grefs > ctx->max_neighbors(i))) + return my_errno= HA_ERR_CRASHED; if (unlikely(ptr + grefs * gref_len() > end)) return my_errno= HA_ERR_CRASHED; neighbors[i].num= grefs; @@ -1217,13 +1248,25 @@ int FVectorNode::save(TABLE *graph) size_t total_size= 0; for (size_t i=0; i <= max_layer; i++) - total_size+= 1 + gref_len() * neighbors[i].num; + total_size+= neighbor_count_pack_len(neighbors[i].num) + + gref_len() * neighbors[i].num; uchar *neighbor_blob= static_cast(my_safe_alloca(total_size)); uchar *ptr= neighbor_blob; for (size_t i= 0; i <= max_layer; i++) { - *ptr++= (uchar)(neighbors[i].num); + /* + counts >= 255 (only possible at layer 0 when 2*M > 255) don't fit in a + byte: escape with 0xff + 2-byte little-endian count, see load_from_record + */ + if (neighbors[i].num < neighbor_count_escape) + *ptr++= (uchar)(neighbors[i].num); + else + { + *ptr++= neighbor_count_escape; + int2store(ptr, (uint16)(neighbors[i].num)); + ptr+= 2; + } for (size_t j= 0; j < neighbors[i].num; j++, ptr+= gref_len()) memcpy(ptr, neighbors[i].links[j]->gref(), gref_len()); } -- 2.43.7