From 7df821e6eaee58d252c8c945f94dba9577609f5b Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 31 Mar 2024 18:36:50 +0200 Subject: [PATCH] Deprecations fix `TORCH_NCCL_BLOCKING_WAIT` (#9448) --- ultralytics/engine/trainer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ultralytics/engine/trainer.py b/ultralytics/engine/trainer.py index b8d917ce..f344b157 100644 --- a/ultralytics/engine/trainer.py +++ b/ultralytics/engine/trainer.py @@ -210,7 +210,7 @@ class BaseTrainer: torch.cuda.set_device(RANK) self.device = torch.device("cuda", RANK) # LOGGER.info(f'DDP info: RANK {RANK}, WORLD_SIZE {world_size}, DEVICE {self.device}') - os.environ["NCCL_BLOCKING_WAIT"] = "1" # set to enforce timeout + os.environ["TORCH_NCCL_BLOCKING_WAIT"] = "1" # set to enforce timeout dist.init_process_group( "nccl" if dist.is_nccl_available() else "gloo", timeout=timedelta(seconds=10800), # 3 hours