ckip-transfomers does not work with latest version of transformers library

[ecwootten]

On a fresh install of ckip-transformers on python 3.11, the following code snippet:

from ckip_transformers.nlp import CkipWordSegmenter
ws_driver = CkipWordSegmenter(model="bert-base")

produces the following error:

cannot access local variable 'model' where it is not associated with a value
NameError: name 'init_empty_weights' is not defined

(full stack trace below).

I believe this is caused by changes made in the latest release of the transformers library (4.51.0). Since the ckip-transformers requirements.txt file only specifies transformers>=3.5.0, it is pulling in the latest release.

To work around this issue, I am currently manually down-grading transformers to the previous version (4.50.3).

Full stacktrace:

NameError: name 'init_empty_weights' is not defined
File <command-7696991796238467>, line 4
      1 #%pip install ckip-transformers
      2 #%pip install --upgrade transformers 
      3 from ckip_transformers.nlp import CkipWordSegmenter
----> 4 ws_driver = CkipWordSegmenter()
File /local_disk0/.ephemeral_nfs/envs/pythonEnv-3062e6de-230a-40b2-b107-0beb01fad653/lib/python3.11/site-packages/ckip_transformers/nlp/driver.py:53, in CkipWordSegmenter.__init__(self, model, **kwargs)
     47 def __init__(
     48     self,
     49     model: str = "bert-base",
     50     **kwargs,
     51 ):
     52     model_name = kwargs.pop("model_name", self._get_model_name(model))
---> 53     super().__init__(model_name=model_name, **kwargs)
File /local_disk0/.ephemeral_nfs/envs/pythonEnv-3062e6de-230a-40b2-b107-0beb01fad653/lib/python3.11/site-packages/ckip_transformers/nlp/util.py:66, in CkipTokenClassification.__init__(self, model_name, tokenizer_name, device)
     59 def __init__(
     60     self,
     61     model_name: str,
   (...)
     64     device: Union[int, torch.device] = -1,
     65 ):
---> 66     self.model = AutoModelForTokenClassification.from_pretrained(model_name)
     67     self.tokenizer = BertTokenizerFast.from_pretrained(tokenizer_name or model_name)
     69     # Allow passing a customized torch.device.
File /local_disk0/.ephemeral_nfs/envs/pythonEnv-3062e6de-230a-40b2-b107-0beb01fad653/lib/python3.11/site-packages/transformers/models/auto/auto_factory.py:571, in _BaseAutoModelClass.from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs)
    569     if model_class.config_class == config.sub_configs.get("text_config", None):
    570         config = config.get_text_config()
--> 571     return model_class.from_pretrained(
    572         pretrained_model_name_or_path, *model_args, config=config, **hub_kwargs, **kwargs
    573     )
    574 raise ValueError(
    575     f"Unrecognized configuration class {config.__class__} for this kind of AutoModel: {cls.__name__}.\n"
    576     f"Model type should be one of {', '.join(c.__name__ for c in cls._model_mapping.keys())}."
    577 )
File /databricks/python_shell/lib/dbruntime/huggingface_patches/transformers.py:21, in _create_patch_function.<locals>.patched_from_pretrained(cls, *args, **kwargs)
     19 call_succeeded = False
     20 try:
---> 21     model = original_method.__func__(cls, *args, **kwargs)
     22     call_succeeded = True
     23     return model
File /local_disk0/.ephemeral_nfs/envs/pythonEnv-3062e6de-230a-40b2-b107-0beb01fad653/lib/python3.11/site-packages/transformers/modeling_utils.py:279, in restore_default_torch_dtype.<locals>._wrapper(*args, **kwargs)
    277 old_dtype = torch.get_default_dtype()
    278 try:
--> 279     return func(*args, **kwargs)
    280 finally:
    281     torch.set_default_dtype(old_dtype)
File /local_disk0/.ephemeral_nfs/envs/pythonEnv-3062e6de-230a-40b2-b107-0beb01fad653/lib/python3.11/site-packages/transformers/modeling_utils.py:4333, in PreTrainedModel.from_pretrained(cls, pretrained_model_name_or_path, config, cache_dir, ignore_mismatched_sizes, force_download, local_files_only, token, revision, use_safetensors, weights_only, *model_args, **kwargs)
   4330 config.name_or_path = pretrained_model_name_or_path
   4332 # Instantiate model.
-> 4333 model_init_context = cls.get_init_context(is_quantized, _is_ds_init_called)
   4335 config = copy.deepcopy(config)  # We do not want to modify the config inplace in from_pretrained.
   4336 if not getattr(config, "_attn_implementation_autoset", False):
File /local_disk0/.ephemeral_nfs/envs/pythonEnv-3062e6de-230a-40b2-b107-0beb01fad653/lib/python3.11/site-packages/transformers/modeling_utils.py:3736, in PreTrainedModel.get_init_context(cls, is_quantized, _is_ds_init_called)
   3734         init_contexts.append(set_quantized_state())
   3735 else:
-> 3736     init_contexts = [no_init_weights(), init_empty_weights()]
   3738 return init_contexts