diff --git a/molt/utils/utils.py b/molt/utils/utils.py index 76c3ff8..b523c3b 100644 --- a/molt/utils/utils.py +++ b/molt/utils/utils.py @@ -66,9 +66,12 @@ def get_tokenizer(pretrain, model, padding_side="left", use_fast=True): # AutoProcessor wraps tokenizer + image_processor; downstream code # detects VLM via hasattr(tokenizer, "image_processor"). tokenizer = AutoProcessor.from_pretrained(pretrain, trust_remote_code=True) + # A checkpoint whose config declares vision_config but ships no complete + # processor bundle (e.g. no preprocessor_config.json) makes AutoProcessor + # fall back to returning the bare tokenizer; use it directly. + inner = getattr(tokenizer, "tokenizer", tokenizer) # AutoProcessor doesn't delegate tokenizer attributes, so set them on # the inner tokenizer and mirror the essentials back. - inner = tokenizer.tokenizer inner.padding_side = padding_side if inner.pad_token is None: inner.pad_token = inner.eos_token diff --git a/tests/unit/test_get_tokenizer.py b/tests/unit/test_get_tokenizer.py new file mode 100644 index 0000000..da0fcff --- /dev/null +++ b/tests/unit/test_get_tokenizer.py @@ -0,0 +1,54 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from types import SimpleNamespace + +from transformers import AutoProcessor + +from molt.utils.utils import get_tokenizer + + +class _BareTokenizer: + """What AutoProcessor returns for a vision_config model without a full + processor bundle: the tokenizer itself, with no .tokenizer attribute.""" + + def __init__(self): + self.padding_side = "right" + self.pad_token = None + self.pad_token_id = None + self.eos_token = "" + self.eos_token_id = 2 + + +class _Processor: + def __init__(self): + self.tokenizer = _BareTokenizer() + + +def _get(monkeypatch, returned): + # Patch the classmethod rather than the module attribute: transformers' + # lazy module re-resolves attributes on import, which bypasses module-level + # monkeypatching. + monkeypatch.setattr(AutoProcessor, "from_pretrained", classmethod(lambda cls, *args, **kwargs: returned)) + model = SimpleNamespace(is_vlm=True, config=SimpleNamespace(pad_token_id=None)) + return get_tokenizer("some/vlm", model, padding_side="left") + + +def test_vlm_processor_without_tokenizer_attribute_is_used_directly(monkeypatch): + bare = _BareTokenizer() + tokenizer = _get(monkeypatch, bare) + + assert tokenizer is bare + assert tokenizer.padding_side == "left" + assert tokenizer.pad_token == "" + assert tokenizer.pad_token_id == 2 + + +def test_vlm_processor_with_inner_tokenizer_mirrors_essentials(monkeypatch): + processor = _Processor() + tokenizer = _get(monkeypatch, processor) + + assert tokenizer is processor + assert processor.tokenizer.padding_side == "left" + assert tokenizer.pad_token == "" + assert tokenizer.pad_token_id == 2