配置参数报错

Terence888

from transformers import AutoModel
#定义debert模型
class LLMModel(nn.Module):
    def __init__(self, cfg, config_path=None, pretrained=False):
        super().__init__()
        self.cfg = cfg
        if config_path is None:
            self.config = AutoConfig.from_pretrained(cfg.model, output_hidden_states=True)#
            self.config.hidden_dropout = 0.
            self.config.hidden_dropout_prob = 0.
            self.config.attention_dropout = 0.
            self.config.attention_probs_dropout_prob = 0.
            self.config.add_pooling_layer = False 
        else:
            self.config = torch.load(config_path)
        if pretrained:
            self.model = AutoModel.from_pretrained(cfg.model, config=self.config)
        else:
            self.model = AutoModel.from_config(self.config)

model = LLMModel(CFG, config_path=config_path,pretrained=False)

模型参数如下

DebertaConfig {
  "_name_or_path": "microsoft/deberta-xlarge",
  "add_pooling_layer": false,
  "attention_dropout": 0.0,
  "attention_probs_dropout_prob": 0.0,
  "hidden_act": "gelu",
  "hidden_dropout": 0.0,
  "hidden_dropout_prob": 0.0,
  "hidden_size": 1024,
  "initializer_range": 0.02,
  "intermediate_size": 4096,
  "layer_norm_eps": 1e-07,
  "max_position_embeddings": 512,
  "max_relative_positions": -1,
  "model_type": "deberta",
  "num_attention_heads": 16,
  "num_hidden_layers": 48,
  "output_hidden_states": true,
  "pad_token_id": 0,
  "pooler_dropout": 0,
  "pooler_hidden_act": "gelu",
  "pooler_hidden_size": 1024,
  "pos_att_type": [
    "c2p",
    "p2c"
  ],
  "position_biased_input": false,
  "relative_attention": true,
  "transformers_version": "4.45.1",
  "type_vocab_size": 0,
  "vocab_size": 50265
}

报错如下

/tmp/ipykernel_31/3171944020.py:15: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
  self.config = torch.load(config_path)
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
Cell In[42], line 1
----> 1 model = LLMModel(CFG, config_path=config_path,pretrained=False)

Cell In[40], line 19, in LLMModel.__init__(self, cfg, config_path, pretrained)
     17     self.model = AutoModel.from_pretrained(cfg.model, config=self.config)
     18 else:
---> 19     self.model = AutoModel.from_config(self.config)
     21 self.model.resize_token_embeddings(len(tokenizer))#
     22 if self.cfg.gradient_checkpointing:#使用梯度检查点技术时，模型不会保存前向传播计算的中间结果，减少占用内存

File /opt/conda/lib/python3.10/site-packages/transformers/models/auto/auto_factory.py:440, in _BaseAutoModelClass.from_config(cls, config, **kwargs)
    438 elif type(config) in cls._model_mapping.keys():
    439     model_class = _get_model_class(config, cls._model_mapping)
--> 440     return model_class._from_config(config, **kwargs)
    442 raise ValueError(
    443     f"Unrecognized configuration class {config.__class__} for this kind of AutoModel: {cls.__name__}.\n"
    444     f"Model type should be one of {', '.join(c.__name__ for c in cls._model_mapping.keys())}."
    445 )

File /opt/conda/lib/python3.10/site-packages/transformers/modeling_utils.py:1494, in PreTrainedModel._from_config(cls, config, **kwargs)
   1490     dtype_orig = cls._set_default_torch_dtype(torch_dtype)
   1492 config = copy.deepcopy(config)  # We do not want to modify the config inplace in _from_config.
-> 1494 if config._attn_implementation_internal is not None:
   1495     # In this case, the config has been created with the attn_implementation set by the user, which we
   1496     # should respect.
   1497     attn_implementation = config._attn_implementation_internal
   1498 else:

File /opt/conda/lib/python3.10/site-packages/transformers/configuration_utils.py:202, in PretrainedConfig.__getattribute__(self, key)
    200 if key != "attribute_map" and key in super().__getattribute__("attribute_map"):
    201     key = super().__getattribute__("attribute_map")[key]
--> 202 return super().__getattribute__(key)

AttributeError: 'DebertaConfig' object has no attribute '_attn_implementation_internal'

FishC

根据错误信息，问题出在`DebertaConfig`对象缺少`_attn_implementation_internal`属性。这个属性可能是某个版本的`transformers`库中引入的，而你当前使用的版本可能不支持这个属性。

以下是一些可能的解决方案：

1. **更新`transformers`库**：确保你使用的是最新版本的`transformers`库，因为新版本可能已经修复了这个问题。你可以使用以下命令更新库：

pip install --upgrade transformers

2. **检查配置文件**：确保配置文件中没有遗漏或拼写错误的属性。你可以参考`transformers`库的文档，查看`DebertaConfig`的所有可用属性。

3. **手动设置属性**：如果更新库后问题依然存在，可能需要手动在配置对象中设置缺失的属性。你可以在初始化配置对象后，手动添加该属性：

self.config._attn_implementation_internal = None  # 或者其他适当的默认值

4. **查看`transformers`库的变更日志**：有时候库的更新会引入不兼容的更改，查看变更日志可以帮助你了解是否有相关的更改。

如果以上方法都无法解决问题，建议在`transformers`的GitHub仓库中提交一个issue，寻求开发者的帮助。

以上回复来自 FishC AI 小助理，如未能正确解答您的问题，请继续追问。