训练kimivl报错 assert not self.training

[zss205]

[rank7]: File "/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
[rank7]: return self._call_impl(*args, **kwargs)
[rank7]: File "/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
[rank7]: return forward_call(*args, **kwargs)
[rank7]: File "/home/tiger/.local/lib/python3.10/site-packages/deepspeed/utils/nvtx.py", line 18, in wrapped_fn
[rank7]: ret_val = func(*args, **kwargs)
[rank7]: File "/home/tiger/.local/lib/python3.10/site-packages/deepspeed/runtime/engine.py", line 1899, in forward
[rank7]: loss = self.module(*inputs, **kwargs)
[rank7]: File "/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
[rank7]: return self._call_impl(*args, **kwargs)
[rank7]: File "/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1844, in _call_impl
[rank7]: return inner()
[rank7]: File "/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1790, in inner
[rank7]: result = forward_call(*args, **kwargs)
[rank7]: File "/home/tiger/.cache/huggingface/modules/transformers_modules/Kimi-VL-A3B-Instruct/modeling_kimi_vl.py", line 2602, in forward
[rank7]: outputs = self.language_model(
[rank7]: File "/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
[rank7]: return self._call_impl(*args, **kwargs)
[rank7]: File "/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1844, in _call_impl
[rank7]: return inner()
[rank7]: File "/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1790, in inner
[rank7]: result = forward_call(*args, **kwargs)
[rank7]: File "/home/tiger/.cache/huggingface/modules/transformers_modules/Kimi-VL-A3B-Instruct/modeling_kimi_vl.py", line 2180, in forward
[rank7]: outputs = self.model(
[rank7]: File "/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
[rank7]: return self._call_impl(*args, **kwargs)
[rank7]: File "/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1844, in _call_impl
[rank7]: return inner()
[rank7]: File "/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1790, in inner
[rank7]: result = forward_call(*args, **kwargs)
[rank7]: File "/home/tiger/.cache/huggingface/modules/transformers_modules/Kimi-VL-A3B-Instruct/modeling_kimi_vl.py", line 2049, in forward
[rank7]: layer_outputs = decoder_layer(
[rank7]: File "/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
[rank7]: return self._call_impl(*args, **kwargs)
[rank7]: File "/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1844, in _call_impl
[rank7]: return inner()
[rank7]: File "/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1790, in inner
[rank7]: result = forward_call(*args, **kwargs)
[rank7]: File "/home/tiger/.cache/huggingface/modules/transformers_modules/Kimi-VL-A3B-Instruct/modeling_kimi_vl.py", line 1795, in forward
[rank7]: hidden_states = self.mlp(hidden_states)
[rank7]: File "/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
[rank7]: return self._call_impl(*args, **kwargs)
[rank7]: File "/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1844, in _call_impl
[rank7]: return inner()
[rank7]: File "/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1790, in inner
[rank7]: result = forward_call(*args, **kwargs)
[rank7]: File "/home/tiger/.cache/huggingface/modules/transformers_modules/Kimi-VL-A3B-Instruct/modeling_kimi_vl.py", line 1094, in forward
[rank7]: topk_idx, topk_weight, aux_loss = self.gate(hidden_states)
[rank7]: File "/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
[rank7]: return self._call_impl(*args, **kwargs)
[rank7]: File "/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1844, in _call_impl
[rank7]: return inner()
[rank7]: File "/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1790, in inner
[rank7]: result = forward_call(*args, **kwargs)
[rank7]: File "/home/tiger/.cache/huggingface/modules/transformers_modules/Kimi-VL-A3B-Instruct/modeling_kimi_vl.py", line 946, in forward
[rank7]: assert not self.training
[rank7]: AssertionError
Exception in thread Thread-2 (_pin_memory_loop):

[Jintao-Huang]

需要修改config.json topk_method为greedy