Fix cosine distance for very small and very large float32 vectors

distance_cosine_float and vec_normalize accumulated squared magnitudes
in float32. The square of an element below roughly 3e-23 underflowed to
zero and the square of one above roughly 2e19 overflowed to Inf, so
nonzero finite vectors got a cosine distance of +-Inf, NaN or a flat
1.0, and in a cosine vec0 table those rows ranked ahead of (or behind)
the true nearest neighbours. vec_normalize returned NULL, a zero vector
or a non-unit vector for the same inputs.

Accumulate in double instead, where every product of two float32 values
is representable. This covers vec_distance_cosine(), cosine KNN queries
and MMR reranking.

Cosine distances of ordinary vectors change in the last few significant
digits because they are now correctly rounded, so two hard-coded
expectations and the cosine MMR snapshots are updated. Only distance
values change, no orderings.

Reported upstream as asg017#324.

Author: vlasky

CHANGELOG.md

@@ -2,6 +2,15 @@
 
 All notable changes to this community fork will be documented in this file.
 
+## [Unreleased]
+
+### Fixed
+
+- **Cosine distance returned ±Inf, NaN or a wrong value for very small or very large nonzero float32 vectors** (upstream [#324](https://github.com/asg017/sqlite-vec/issues/324))
+  - Squared magnitudes were accumulated in float32, so elements below roughly 3e-23 underflowed to a zero magnitude and elements above roughly 2e19 overflowed to Inf (giving a distance of 1.0 whatever the direction). In a cosine `vec0` table the affected rows ranked ahead of (or behind) the true nearest neighbours
+  - `vec_distance_cosine()`, cosine KNN queries and MMR reranking now accumulate in double precision, as does `vec_normalize()`, which returned NULL, a zero vector or a non-unit vector for the same inputs
+  - Cosine distances of ordinary float32 vectors can differ from previous releases in the last few significant digits, as they are now correctly rounded
+
 ## [0.2.6-alpha] - 2026-08-22
 
 ### Added

sqlite-vec.c

@@ -586,13 +586,17 @@ static f32 distance_cosine_float(const void *pVect1v, const void *pVect2v,
   f32 *pVect2 = (f32 *)pVect2v;
   size_t qty = *((size_t *)qty_ptr);
 
-  f32 dot = 0;
-  f32 aMag = 0;
-  f32 bMag = 0;
+  // Accumulate in double: the square of a small (or large) f32 element
+  // underflows (or overflows) f32, which made the magnitude 0 (or Inf) and
+  // the distance +-Inf, NaN or plain wrong for nonzero finite vectors. Every
+  // f32 product is representable in double.
+  double dot = 0;
+  double aMag = 0;
+  double bMag = 0;
   for (size_t i = 0; i < qty; i++) {
-    dot += *pVect1 * *pVect2;
-    aMag += *pVect1 * *pVect1;
-    bMag += *pVect2 * *pVect2;
+    dot += (double)*pVect1 * (double)*pVect2;
+    aMag += (double)*pVect1 * (double)*pVect1;
+    bMag += (double)*pVect2 * (double)*pVect2;
     pVect1++;
     pVect2++;
   }
@@ -2054,12 +2058,14 @@ static void vec_normalize(sqlite3_context *context, int argc,
 
   f32 *v = (f32 *)vector;
 
-  f32 norm = 0;
+  // Accumulate in double so that squaring small or large f32 elements does
+  // not underflow or overflow, see distance_cosine_float().
+  double norm = 0;
   for (size_t i = 0; i < dimensions; i++) {
-    norm += v[i] * v[i];
+    norm += (double)v[i] * (double)v[i];
   }
   norm = sqrt(norm);
-  if (norm == 0.0f) {
+  if (norm == 0.0) {
     // A zero-magnitude vector cannot be normalized. Return NULL instead of
     // producing a vector of NaNs.
     sqlite3_free(out);

tests/__snapshots__/test-knn-mmr.ambr

@@ -9,19 +9,19 @@
       }),
       OrderedDict({
         'rowid': 2,
-        'distance': 0.005062814336270094,
+        'distance': 0.005062811076641083,
       }),
       OrderedDict({
         'rowid': 3,
-        'distance': 0.011511977761983871,
+        'distance': 0.011511985212564468,
       }),
       OrderedDict({
         'rowid': 4,
-        'distance': 0.020601648837327957,
+        'distance': 0.020601652562618256,
       }),
       OrderedDict({
         'rowid': 5,
-        'distance': 0.03227578103542328,
+        'distance': 0.032275766134262085,
       }),
     ]),
   })
@@ -36,19 +36,19 @@
       }),
       OrderedDict({
         'rowid': 2,
-        'distance': 0.005062814336270094,
+        'distance': 0.005062811076641083,
       }),
       OrderedDict({
         'rowid': 8,
         'distance': 1.0,
       }),
       OrderedDict({
         'rowid': 3,
-        'distance': 0.011511977761983871,
+        'distance': 0.011511985212564468,
       }),
       OrderedDict({
         'rowid': 4,
-        'distance': 0.020601648837327957,
+        'distance': 0.020601652562618256,
       }),
     ]),
   })
@@ -63,11 +63,11 @@
       }),
       OrderedDict({
         'rowid': 2,
-        'distance': 0.005062814336270094,
+        'distance': 0.005062811076641083,
       }),
       OrderedDict({
         'rowid': 3,
-        'distance': 0.02019595541059971,
+        'distance': 0.020195940509438515,
       }),
     ]),
   })
@@ -82,11 +82,11 @@
       }),
       OrderedDict({
         'rowid': 2,
-        'distance': 0.005062814336270094,
+        'distance': 0.005062811076641083,
       }),
       OrderedDict({
         'rowid': 3,
-        'distance': 0.02019595541059971,
+        'distance': 0.020195940509438515,
       }),
     ]),
   })
@@ -101,7 +101,7 @@
       }),
       OrderedDict({
         'rowid': 2,
-        'distance': 0.005062814336270094,
+        'distance': 0.005062811076641083,
       }),
       OrderedDict({
         'rowid': 5,
@@ -209,15 +209,15 @@
     'rows': list([
       OrderedDict({
         'rowid': 2,
-        'distance': 0.005062814336270094,
+        'distance': 0.005062811076641083,
       }),
       OrderedDict({
         'rowid': 5,
         'distance': 1.0,
       }),
       OrderedDict({
         'rowid': 3,
-        'distance': 0.02019595541059971,
+        'distance': 0.020195940509438515,
       }),
     ]),
   })
@@ -232,7 +232,7 @@
       }),
       OrderedDict({
         'rowid': 2,
-        'distance': 0.005062814336270094,
+        'distance': 0.005062811076641083,
       }),
       OrderedDict({
         'rowid': 5,

tests/test-loadable.py

@@ -423,7 +423,7 @@ def check(a, b, dtype: type = np.float32):
     check([1.2, 0.1], [0.4, -0.4])
     check([-1.2, -0.1], [-0.4, 0.4])
     check([1, 2, 3], [-9, -8, -7], dtype=np.int8)
-    assert vec_distance_cosine("[1.1, 1.0]", "[1.2, 1.2]") == 0.001131898257881403
+    assert vec_distance_cosine("[1.1, 1.0]", "[1.2, 1.2]") == 0.0011318627512082458
 
     vec_distance_cosine_bit = lambda *args: db.execute(
         "select vec_distance_cosine(vec_bit(?), vec_bit(?))", args
@@ -468,6 +468,29 @@ def test_vec_distance_cosine_zero_vector():
     )
 
 
+def test_vec_distance_cosine_extreme_magnitudes():
+    # squaring very small or very large f32 elements used to underflow or
+    # overflow the f32 magnitude accumulators, returning +-Inf, NaN or a wrong
+    # distance for nonzero finite vectors
+    # https://github.com/asg017/sqlite-vec/issues/324
+    vec_distance_cosine = lambda a, b: db.execute(
+        "select vec_distance_cosine(?, ?)", [_f32(a), _f32(b)]
+    ).fetchone()[0]
+
+    for scale in [1e-45, 1e-30, 1e-25, 1e-20, 1.0, 1e20, 3e38]:
+        # cosine distance is invariant under positive scaling
+        assert vec_distance_cosine([1, 0], [scale, 0]) == 0.0
+        assert vec_distance_cosine([scale, 0], [1, 0]) == 0.0
+        assert vec_distance_cosine([scale, 0], [scale, 0]) == 0.0
+        assert vec_distance_cosine([1, 0], [-scale, 0]) == 2.0
+        assert vec_distance_cosine([1, 0], [0, scale]) == 1.0
+        assert isclose(
+            vec_distance_cosine([1, 0], [scale, scale]),
+            1 - 1 / np.sqrt(2),
+            abs_tol=1e-6,
+        )
+
+
 def test_ensure_vector_match_cleanup_on_second_vector_error():
     """
     Test that ensure_vector_match properly cleans up the first vector
@@ -748,6 +771,18 @@ def test_vec_normalize():
     assert vec_normalize(_f32([0, 0, 0, 0])) is None
     assert db.execute("select vec_normalize('[0,0,0]')").fetchone()[0] is None
 
+    # small and large nonzero vectors are normalized rather than treated as
+    # zero-magnitude or returned as a zero vector, the squared magnitude used
+    # to underflow/overflow f32
+    # https://github.com/asg017/sqlite-vec/issues/324
+    for scale in [1e-45, 1e-30, 1e-25, 1e-22, 1e-20, 1e20, 3e38]:
+        assert vec_normalize(_f32([scale, 0])) == _f32([1, 0])
+        assert vec_normalize(_f32([0, -scale])) == _f32([0, -1])
+        assert np.allclose(
+            np.frombuffer(vec_normalize(_f32([scale, scale])), dtype=np.float32),
+            [1 / np.sqrt(2)] * 2,
+        )
+
 
 def test_vec_slice():
     vec_slice = lambda *args, f="?": db.execute(
@@ -2702,7 +2737,7 @@ def test_vec0_cosine_zero_vector():
         db, "select rowid, distance from v where a match '[1,0,0]' and k = 2"
     ) == [
         {"rowid": 1, "distance": 0.0},
-        {"rowid": 3, "distance": 0.006116250995546579},
+        {"rowid": 3, "distance": 0.006116265896707773},
     ]
 
     # non-cosine tables still accept zero-magnitude vectors
@@ -2713,6 +2748,44 @@ def test_vec0_cosine_zero_vector():
     ) == [{"rowid": 1, "distance": 0.0}]
 
 
+def test_vec0_cosine_extreme_magnitudes():
+    # tiny or huge nonzero vectors used to get +-Inf, NaN or wrong cosine
+    # distances, which ranked them ahead of (or behind) the true nearest
+    # neighbors
+    # https://github.com/asg017/sqlite-vec/issues/324
+    db = connect(EXT_PATH)
+    db.execute("create virtual table v using vec0(a float[2] distance_metric=cosine)")
+    db.executemany(
+        "insert into v(rowid, a) values (?, ?)",
+        [
+            (1, _f32([0.9, 0.1])),
+            (2, _f32([1e-25, 0])),
+            (3, _f32([1e-25, 1e-25])),
+            (4, _f32([-1e-25, 0])),
+            (5, _f32([1e-30, 0])),
+            (6, _f32([3e38, 3e38])),
+        ],
+    )
+
+    expected = [
+        {"rowid": 2, "distance": 0.0},
+        {"rowid": 5, "distance": 0.0},
+        {"rowid": 1, "distance": 0.006116265896707773},
+        {"rowid": 3, "distance": 0.2928932309150696},
+        {"rowid": 6, "distance": 0.2928932309150696},
+        {"rowid": 4, "distance": 2.0},
+    ]
+    for query in ([1, 0], [1e-25, 0], [3e38, 0]):
+        rows = execute_all(
+            db,
+            "select rowid, distance from v where a match ? and k = 6",
+            [_f32(query)],
+        )
+        # the order of equidistant rows is unspecified
+        assert sorted(rows, key=lambda r: (r["distance"], r["rowid"])) == expected
+        assert [r["distance"] for r in rows] == [r["distance"] for r in expected]
+
+
 def test_vec0_vacuum():
     db = connect(EXT_PATH)
     db.execute("create virtual table vec_t using vec0(a float[1]);")