ultralytics 8.3.75 Comet update to new comet_ml.start() API (#19187)
Co-authored-by: UltralyticsAssistant <web@ultralytics.com> Co-authored-by: Glenn Jocher <glenn.jocher@ultralytics.com>
This commit is contained in:
parent
5edf3419f9
commit
675d370592
2 changed files with 32 additions and 21 deletions
|
|
@ -1,6 +1,6 @@
|
||||||
# Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license
|
# Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license
|
||||||
|
|
||||||
__version__ = "8.3.74"
|
__version__ = "8.3.75"
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,7 @@
|
||||||
# Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license
|
# Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license
|
||||||
|
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
from ultralytics.utils import LOGGER, RANK, SETTINGS, TESTS_RUNNING, ops
|
from ultralytics.utils import LOGGER, RANK, SETTINGS, TESTS_RUNNING, ops
|
||||||
from ultralytics.utils.metrics import ClassifyMetrics, DetMetrics, OBBMetrics, PoseMetrics, SegmentMetrics
|
from ultralytics.utils.metrics import ClassifyMetrics, DetMetrics, OBBMetrics, PoseMetrics, SegmentMetrics
|
||||||
|
|
||||||
|
|
@ -29,9 +31,19 @@ except (ImportError, AssertionError):
|
||||||
comet_ml = None
|
comet_ml = None
|
||||||
|
|
||||||
|
|
||||||
def _get_comet_mode():
|
def _get_comet_mode() -> str:
|
||||||
"""Returns the mode of comet set in the environment variables, defaults to 'online' if not set."""
|
"""Returns the mode of comet set in the environment variables, defaults to 'online' if not set."""
|
||||||
return os.getenv("COMET_MODE", "online")
|
comet_mode = os.getenv("COMET_MODE")
|
||||||
|
if comet_mode is not None:
|
||||||
|
LOGGER.warning(
|
||||||
|
"WARNING ⚠️ The COMET_MODE environment variable is deprecated. "
|
||||||
|
"Please use COMET_START_ONLINE to set the Comet experiment mode. "
|
||||||
|
"To start an offline Comet experiment, use 'export COMET_START_ONLINE=0'. "
|
||||||
|
"If COMET_START_ONLINE is not set or is set to '1', an online Comet experiment will be created."
|
||||||
|
)
|
||||||
|
return comet_mode
|
||||||
|
|
||||||
|
return "online"
|
||||||
|
|
||||||
|
|
||||||
def _get_comet_model_name():
|
def _get_comet_model_name():
|
||||||
|
|
@ -65,22 +77,24 @@ def _should_log_image_predictions():
|
||||||
return os.getenv("COMET_EVAL_LOG_IMAGE_PREDICTIONS", "true").lower() == "true"
|
return os.getenv("COMET_EVAL_LOG_IMAGE_PREDICTIONS", "true").lower() == "true"
|
||||||
|
|
||||||
|
|
||||||
def _get_experiment_type(mode, project_name):
|
def _resume_or_create_experiment(args: SimpleNamespace) -> None:
|
||||||
"""Return an experiment based on mode and project name."""
|
"""
|
||||||
if mode == "offline":
|
Resumes CometML experiment or creates a new experiment based on args.
|
||||||
return comet_ml.OfflineExperiment(project_name=project_name)
|
|
||||||
|
|
||||||
return comet_ml.Experiment(project_name=project_name)
|
Ensures that the experiment object is only created in a single process during distributed training.
|
||||||
|
"""
|
||||||
|
|
||||||
def _create_experiment(args):
|
|
||||||
"""Ensures that the experiment object is only created in a single process during distributed training."""
|
|
||||||
if RANK not in {-1, 0}:
|
if RANK not in {-1, 0}:
|
||||||
return
|
return
|
||||||
try:
|
|
||||||
|
# Set environment variable (if not set by the user) to configure the Comet experiment's online mode under the hood.
|
||||||
|
# IF COMET_START_ONLINE is set by the user it will override COMET_MODE value.
|
||||||
|
if os.getenv("COMET_START_ONLINE") is None:
|
||||||
comet_mode = _get_comet_mode()
|
comet_mode = _get_comet_mode()
|
||||||
|
os.environ["COMET_START_ONLINE"] = "1" if comet_mode != "offline" else "0"
|
||||||
|
|
||||||
|
try:
|
||||||
_project_name = os.getenv("COMET_PROJECT_NAME", args.project)
|
_project_name = os.getenv("COMET_PROJECT_NAME", args.project)
|
||||||
experiment = _get_experiment_type(comet_mode, _project_name)
|
experiment = comet_ml.start(project_name=_project_name)
|
||||||
experiment.log_parameters(vars(args))
|
experiment.log_parameters(vars(args))
|
||||||
experiment.log_others(
|
experiment.log_others(
|
||||||
{
|
{
|
||||||
|
|
@ -313,15 +327,12 @@ def _log_model(experiment, trainer):
|
||||||
|
|
||||||
def on_pretrain_routine_start(trainer):
|
def on_pretrain_routine_start(trainer):
|
||||||
"""Creates or resumes a CometML experiment at the start of a YOLO pre-training routine."""
|
"""Creates or resumes a CometML experiment at the start of a YOLO pre-training routine."""
|
||||||
experiment = comet_ml.get_global_experiment()
|
_resume_or_create_experiment(trainer.args)
|
||||||
is_alive = getattr(experiment, "alive", False)
|
|
||||||
if not experiment or not is_alive:
|
|
||||||
_create_experiment(trainer.args)
|
|
||||||
|
|
||||||
|
|
||||||
def on_train_epoch_end(trainer):
|
def on_train_epoch_end(trainer):
|
||||||
"""Log metrics and save batch images at the end of training epochs."""
|
"""Log metrics and save batch images at the end of training epochs."""
|
||||||
experiment = comet_ml.get_global_experiment()
|
experiment = comet_ml.get_running_experiment()
|
||||||
if not experiment:
|
if not experiment:
|
||||||
return
|
return
|
||||||
|
|
||||||
|
|
@ -334,7 +345,7 @@ def on_train_epoch_end(trainer):
|
||||||
|
|
||||||
def on_fit_epoch_end(trainer):
|
def on_fit_epoch_end(trainer):
|
||||||
"""Logs model assets at the end of each epoch."""
|
"""Logs model assets at the end of each epoch."""
|
||||||
experiment = comet_ml.get_global_experiment()
|
experiment = comet_ml.get_running_experiment()
|
||||||
if not experiment:
|
if not experiment:
|
||||||
return
|
return
|
||||||
|
|
||||||
|
|
@ -362,7 +373,7 @@ def on_fit_epoch_end(trainer):
|
||||||
|
|
||||||
def on_train_end(trainer):
|
def on_train_end(trainer):
|
||||||
"""Perform operations at the end of training."""
|
"""Perform operations at the end of training."""
|
||||||
experiment = comet_ml.get_global_experiment()
|
experiment = comet_ml.get_running_experiment()
|
||||||
if not experiment:
|
if not experiment:
|
||||||
return
|
return
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue