fix omni aligner

examples/train/long_text/zero3.sh

@@ -1,4 +1,5 @@
 # Env: 4 * A100
+# https://github.com/modelscope/ms-swift/blob/main/examples/train/megatron/long_text.sh
 # Max Length: 16K
 # GPU Memory: 4 * 56GB, Training Speed 10s/it
 NPROC_PER_NODE=4 \

swift/llm/model/model_arch.py

@@ -471,6 +471,7 @@ def register_model_arch(model_arch: ModelKeys, *, exist_ok: bool = False) -> Non
         MLLMModelArch.qwen2_5_omni,
         language_model='thinker.model',
         vision_tower=['thinker.audio_tower', 'thinker.visual'],
+        aligner=['thinker.audio_tower.proj', 'thinker.visual.merger'],
         generator=['talker', 'token2wav'],
     ))
 

swift/llm/train/tuner.py

@@ -9,7 +9,7 @@
 from packaging import version
 from transformers import TrainingArguments
 
-from swift.llm import TrainArguments, get_model_arch
+from swift.llm import TrainArguments, deep_getattr, get_model_arch
 from swift.plugin import Tuner, extra_tuners
 from swift.tuners import Swift
 from swift.utils import (activate_parameters, find_all_linears, find_embedding, find_norm, freeze_parameters,
@@ -59,30 +59,33 @@ def get_multimodal_target_regex(
 ) -> str:
     model_arch = get_model_arch(model.model_meta.model_arch)
     modules = []
-    rejected_modules = []
     if not freeze_llm:
         modules += model_arch.language_model
     if not freeze_vit:
         modules += model_arch.vision_tower
     if not freeze_aligner:
         modules += model_arch.aligner
-    elif not freeze_vit:
-        rejected_modules += model_arch.aligner
-
     assert len(modules) > 0, f'modules: {modules}'
-    prefix_pattern = '|'.join(modules)
-    rejected_pattern = '|'.join(rejected_modules)
 
     extra_layers = []
     if include_embedding:
         extra_layers.append(nn.Embedding)
-    target_modules = []
+    res = []
     for module in modules:
-        target_modules += find_all_linears(model, model_arch, extra_layers, sub_module=module)
-    target_regex = rf'^({prefix_pattern}).*\.({"|".join(target_modules)})$'
-    if rejected_pattern:
-        target_regex = rf'(?!^({rejected_pattern}))' + target_regex
-    return target_regex
+        rejected_modules = []
+        if not freeze_vit:
+            for aligner in model_arch.aligner:
+                if aligner.startswith(f'{module}.'):
+                    rejected_modules.append(aligner)
+
+        sub_module = deep_getattr(model, module)
+        target_modules = find_all_linears(sub_module, model_arch, extra_layers)
+        target_modules = [tm for tm in target_modules if tm]
+        target_pattern = rf'.*\.({"|".join(target_modules)})' if target_modules else ''
+        rejected_pattern = rf'(?!({"|".join(rejected_modules)}))' if rejected_modules else ''
+        res.append(rf'{rejected_pattern}{module}{target_pattern}')
+
+    return rf'^({"|".join(res)})$'
 
 
 def get_target_modules(args, model) -> Union[str, List[str]]:

swift/utils/torch_utils.py

@@ -174,7 +174,7 @@ def find_layers(
     target_module_names = set()
     for name, module in sub_module.named_modules():
         if sub_module_str:
-            name = f'{sub_module_str}.{name}'
+            name = f'{sub_module_str}.{name}' if name else sub_module_str
         if cond(name, module):
             module_name_list = name.split('.')
             module_name = module_name_list.pop()