update

swift/utils/torch_utils.py

@@ -19,7 +19,7 @@
 from transformers.integrations import is_deepspeed_zero3_enabled
 from transformers.utils import is_torch_cuda_available, is_torch_mps_available, is_torch_npu_available
 
-from .env import get_dist_setting, is_dist, is_dist_ta, is_master
+from .env import get_dist_setting, is_dist, is_dist_ta, is_local_master, is_master
 from .logger import get_logger
 from .utils import deep_getattr
 
@@ -232,11 +232,19 @@ def _cond(name, module):
 @contextmanager
 def safe_ddp_context(hash_id: Optional[str], use_barrier: bool = False):
     if use_barrier and dist.is_initialized():
-        if (is_dist() or is_dist_ta()) and not is_master():
-            dist.barrier()
+        if (is_dist() or is_dist_ta()):
+            if not is_master():
+                dist.barrier()
+            if not is_local_master():
+                # Compatible with multi-machine scenarios,
+                # where each machine uses different storage hardware.
+                dist.barrier()
         yield
-        if (is_dist() or is_dist_ta()) and is_master():
-            dist.barrier()
+        if (is_dist() or is_dist_ta()):
+            if is_master():
+                dist.barrier()
+            if is_local_master():
+                dist.barrier()
     elif hash_id is not None:
         lock_dir = os.path.join(get_cache_dir(), 'lockers')
         os.makedirs(lock_dir, exist_ok=True)