From 3edfa3760de6a1630fec868c360b6a2bd7e523c1 Mon Sep 17 00:00:00 2001 From: adityaanikam Date: Fri, 31 Jul 2026 20:48:55 +0530 Subject: [PATCH] Fix index misalignment in char_wb ngram padding --- python/cuml/cuml/feature_extraction/_vectorizers.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/python/cuml/cuml/feature_extraction/_vectorizers.py b/python/cuml/cuml/feature_extraction/_vectorizers.py index 716b9fe631..272869ab90 100644 --- a/python/cuml/cuml/feature_extraction/_vectorizers.py +++ b/python/cuml/cuml/feature_extraction/_vectorizers.py @@ -213,11 +213,15 @@ def get_char_ngrams(self, ngram_size, str_series, doc_id_sr): tokens = str_series.str.tokenize(self.delimiter) del str_series - padding = Series(self.delimiter).repeat(len(tokens)) + # tokens keeps the original per-document index (repeated per + # token); reset both to a plain range first so the two str.cat() + # calls below align positionally instead of by that index. + tokens = tokens.reset_index(drop=True) + padding = Series(self.delimiter).repeat(len(tokens)).reset_index( + drop=True + ) tokens = tokens.str.cat(padding) - padding = padding.reset_index(drop=True) tokens = padding.str.cat(tokens) - tokens = tokens.reset_index(drop=True) ngram_sr = tokens.str.character_ngrams(n=ngram_size)