From 4cb644cf295b35c213b122f01d1b2f89dadd9a94 Mon Sep 17 00:00:00 2001 From: xuanwujian Date: Sun, 12 Jul 2026 23:17:10 +0800 Subject: [PATCH] fix: prevent byte/char index mismatch panic in jaro_winkler_distance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit jaro_winkler_distance mixed character indices (from .chars().enumerate()) with byte indices (str::len() and &str slicing) when locating matched characters. Slicing a &str at a byte offset that falls inside a multi-byte UTF-8 character panics, so any non-ASCII input (e.g. jaro_winkler_distance("ab", "céd")) crashed the caller's process. Rewrite the function to operate on Vec throughout instead of byte-indexed &str, so both the matching-window logic and the score formula's length denominators use character counts consistently. This fixes the panic and also corrects the Jaro score itself for non-ASCII input (byte-length denominators would otherwise under-weight multi-byte strings even without panicking). The placeholder-based match removal (replacing a matched character rather than deleting it) is preserved so existing ASCII results are unchanged. Added regression tests for the exact repro from the issue plus CJK/accented-character cases. Fixes #1047 --- src/string/jaro_winkler_distance.rs | 73 +++++++++++++++++++++++------ 1 file changed, 58 insertions(+), 15 deletions(-) diff --git a/src/string/jaro_winkler_distance.rs b/src/string/jaro_winkler_distance.rs index e00e526e676..5ce3305e983 100644 --- a/src/string/jaro_winkler_distance.rs +++ b/src/string/jaro_winkler_distance.rs @@ -8,31 +8,41 @@ pub fn jaro_winkler_distance(str1: &str, str2: &str) -> f64 { if str1.is_empty() || str2.is_empty() { return 0.0; } - fn get_matched_characters(s1: &str, s2: &str) -> String { - let mut s2 = s2.to_string(); + + // Operate on character vectors throughout so that all indices and + // lengths are character counts, never byte offsets. This keeps the + // algorithm correct (not merely non-panicking) for arbitrary Unicode. + let chars1: Vec = str1.chars().collect(); + let chars2: Vec = str2.chars().collect(); + + fn get_matched_characters(s1: &[char], s2: &[char]) -> Vec { + let mut s2: Vec = s2.to_vec(); let mut matched: Vec = Vec::new(); let limit = std::cmp::min(s1.len(), s2.len()) / 2; - for (i, l) in s1.chars().enumerate() { + for (i, &l) in s1.iter().enumerate() { let left = std::cmp::max(0, i as i32 - limit as i32) as usize; let right = std::cmp::min(i + limit + 1, s2.len()); - if s2[left..right].contains(l) { + if s2[left..right].contains(&l) { matched.push(l); - let a = &s2[0..s2.find(l).expect("this exists")]; - let b = &s2[(s2.find(l).expect("this exists") + 1)..]; - s2 = format!("{a} {b}"); + // Mark the first occurrence as used with a placeholder so it + // can no longer be matched, while preserving the positions of + // the remaining characters (mirrors the original space insertion). + if let Some(pos) = s2.iter().position(|&c| c == l) { + s2[pos] = ' '; + } } } - matched.iter().collect::() + matched } - let matching_1 = get_matched_characters(str1, str2); - let matching_2 = get_matched_characters(str2, str1); + let matching_1 = get_matched_characters(&chars1, &chars2); + let matching_2 = get_matched_characters(&chars2, &chars1); let match_count = matching_1.len(); // transposition let transpositions = { let mut count = 0; - for (c1, c2) in matching_1.chars().zip(matching_2.chars()) { + for (c1, c2) in matching_1.iter().zip(matching_2.iter()) { if c1 != c2 { count += 1; } @@ -45,14 +55,14 @@ pub fn jaro_winkler_distance(str1: &str, str2: &str) -> f64 { return 0.0; } (1_f64 / 3_f64) - * (match_count as f64 / str1.len() as f64 - + match_count as f64 / str2.len() as f64 + * (match_count as f64 / chars1.len() as f64 + + match_count as f64 / chars2.len() as f64 + (match_count - transpositions) as f64 / match_count as f64) }; let mut prefix_len = 0.0; - let bound = std::cmp::min(std::cmp::min(str1.len(), str2.len()), 4); - for (c1, c2) in str1[..bound].chars().zip(str2[..bound].chars()) { + let bound = std::cmp::min(std::cmp::min(chars1.len(), chars2.len()), 4); + for (c1, c2) in chars1[..bound].iter().zip(chars2[..bound].iter()) { if c1 == c2 { prefix_len += 1.0; } else { @@ -81,4 +91,37 @@ mod tests { let a = jaro_winkler_distance("hello world", "HeLLo W0rlD"); assert_eq!(a, 0.6363636363636364); } + + #[test] + fn test_jaro_winkler_distance_non_ascii() { + // Regression test for issue #1047: multi-byte UTF-8 input must not + // panic and must use character counts (not byte lengths). + // "ab" and "céd" share no characters, so the distance is 0.0. + let a = jaro_winkler_distance("ab", "céd"); + assert_eq!(a, 0.0); + } + + #[test] + fn test_jaro_winkler_distance_all_multibyte_identical() { + // Two entirely non-ASCII strings that are identical. Characters, + // not bytes, must be counted (each CJK char is 3 bytes). + let a = jaro_winkler_distance("测试", "测试"); + assert_eq!(a, 1.0); + + // Identical combining-accent strings (each char is 2 bytes). + let a = jaro_winkler_distance("áé", "áé"); + assert_eq!(a, 1.0); + } + + #[test] + fn test_jaro_winkler_distance_multibyte_partial() { + // Non-identical multi-byte strings: character counting in the Jaro + // denominator matters here (bytes would over-count the length). + // "测试" and "测a": one of two characters matches. + // match_count = 1, transpositions = 0 + // jaro = (1/3) * (1/2 + 1/2 + 1/1) = 2/3 ≈ 0.6666666666666666 + // prefix = 1 ("测" matches), result = 2/3 + 0.1 * 1 * (1 - 2/3) = 0.7 + let a = jaro_winkler_distance("测试", "测a"); + assert_eq!(a, 0.7); + } }