diff --git a/python/cuml/cuml/feature_extraction/_vectorizers.py b/python/cuml/cuml/feature_extraction/_vectorizers.py index 716b9fe631..272869ab90 100644 --- a/python/cuml/cuml/feature_extraction/_vectorizers.py +++ b/python/cuml/cuml/feature_extraction/_vectorizers.py @@ -213,11 +213,15 @@ def get_char_ngrams(self, ngram_size, str_series, doc_id_sr): tokens = str_series.str.tokenize(self.delimiter) del str_series - padding = Series(self.delimiter).repeat(len(tokens)) + # tokens keeps the original per-document index (repeated per + # token); reset both to a plain range first so the two str.cat() + # calls below align positionally instead of by that index. + tokens = tokens.reset_index(drop=True) + padding = Series(self.delimiter).repeat(len(tokens)).reset_index( + drop=True + ) tokens = tokens.str.cat(padding) - padding = padding.reset_index(drop=True) tokens = padding.str.cat(tokens) - tokens = tokens.reset_index(drop=True) ngram_sr = tokens.str.character_ngrams(n=ngram_size)