diff --git a/ultralytics/__init__.py b/ultralytics/__init__.py index f9367776..dfcf2dbb 100644 --- a/ultralytics/__init__.py +++ b/ultralytics/__init__.py @@ -1,6 +1,6 @@ # Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license -__version__ = "8.3.74" +__version__ = "8.3.75" import os diff --git a/ultralytics/utils/callbacks/comet.py b/ultralytics/utils/callbacks/comet.py index 910e3c42..65c138bc 100644 --- a/ultralytics/utils/callbacks/comet.py +++ b/ultralytics/utils/callbacks/comet.py @@ -1,5 +1,7 @@ # Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license +from types import SimpleNamespace + from ultralytics.utils import LOGGER, RANK, SETTINGS, TESTS_RUNNING, ops from ultralytics.utils.metrics import ClassifyMetrics, DetMetrics, OBBMetrics, PoseMetrics, SegmentMetrics @@ -29,9 +31,19 @@ except (ImportError, AssertionError): comet_ml = None -def _get_comet_mode(): +def _get_comet_mode() -> str: """Returns the mode of comet set in the environment variables, defaults to 'online' if not set.""" - return os.getenv("COMET_MODE", "online") + comet_mode = os.getenv("COMET_MODE") + if comet_mode is not None: + LOGGER.warning( + "WARNING ⚠️ The COMET_MODE environment variable is deprecated. " + "Please use COMET_START_ONLINE to set the Comet experiment mode. " + "To start an offline Comet experiment, use 'export COMET_START_ONLINE=0'. " + "If COMET_START_ONLINE is not set or is set to '1', an online Comet experiment will be created." + ) + return comet_mode + + return "online" def _get_comet_model_name(): @@ -65,22 +77,24 @@ def _should_log_image_predictions(): return os.getenv("COMET_EVAL_LOG_IMAGE_PREDICTIONS", "true").lower() == "true" -def _get_experiment_type(mode, project_name): - """Return an experiment based on mode and project name.""" - if mode == "offline": - return comet_ml.OfflineExperiment(project_name=project_name) +def _resume_or_create_experiment(args: SimpleNamespace) -> None: + """ + Resumes CometML experiment or creates a new experiment based on args. - return comet_ml.Experiment(project_name=project_name) - - -def _create_experiment(args): - """Ensures that the experiment object is only created in a single process during distributed training.""" + Ensures that the experiment object is only created in a single process during distributed training. + """ if RANK not in {-1, 0}: return - try: + + # Set environment variable (if not set by the user) to configure the Comet experiment's online mode under the hood. + # IF COMET_START_ONLINE is set by the user it will override COMET_MODE value. + if os.getenv("COMET_START_ONLINE") is None: comet_mode = _get_comet_mode() + os.environ["COMET_START_ONLINE"] = "1" if comet_mode != "offline" else "0" + + try: _project_name = os.getenv("COMET_PROJECT_NAME", args.project) - experiment = _get_experiment_type(comet_mode, _project_name) + experiment = comet_ml.start(project_name=_project_name) experiment.log_parameters(vars(args)) experiment.log_others( { @@ -313,15 +327,12 @@ def _log_model(experiment, trainer): def on_pretrain_routine_start(trainer): """Creates or resumes a CometML experiment at the start of a YOLO pre-training routine.""" - experiment = comet_ml.get_global_experiment() - is_alive = getattr(experiment, "alive", False) - if not experiment or not is_alive: - _create_experiment(trainer.args) + _resume_or_create_experiment(trainer.args) def on_train_epoch_end(trainer): """Log metrics and save batch images at the end of training epochs.""" - experiment = comet_ml.get_global_experiment() + experiment = comet_ml.get_running_experiment() if not experiment: return @@ -334,7 +345,7 @@ def on_train_epoch_end(trainer): def on_fit_epoch_end(trainer): """Logs model assets at the end of each epoch.""" - experiment = comet_ml.get_global_experiment() + experiment = comet_ml.get_running_experiment() if not experiment: return @@ -362,7 +373,7 @@ def on_fit_epoch_end(trainer): def on_train_end(trainer): """Perform operations at the end of training.""" - experiment = comet_ml.get_global_experiment() + experiment = comet_ml.get_running_experiment() if not experiment: return