ultralytics 8.2.103 Windows Benchmarks CI (#16523)

Signed-off-by: UltralyticsAssistant <web@ultralytics.com>
Co-authored-by: UltralyticsAssistant <web@ultralytics.com>

.github/workflows/ci.yaml

@@ -98,7 +98,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-14]
+        os: [ubuntu-latest, windows-latest, macos-14]
         python-version: ["3.11"]
         model: [yolov8n]
     steps:

ultralytics/__init__.py

@@ -1,6 +1,6 @@
 # Ultralytics YOLO 🚀, AGPL-3.0 license
 
-__version__ = "8.2.102"
+__version__ = "8.2.103"
 
 
 import os

ultralytics/utils/benchmarks.py

@@ -43,36 +43,40 @@ from ultralytics.utils import ARM64, ASSETS, IS_JETSON, IS_RASPBERRYPI, LINUX, L
 from ultralytics.utils.checks import IS_PYTHON_3_12, check_requirements, check_yolo
 from ultralytics.utils.downloads import safe_download
 from ultralytics.utils.files import file_size
-from ultralytics.utils.torch_utils import select_device
+from ultralytics.utils.torch_utils import get_cpu_info, select_device
 
 
 def benchmark(
-    model=WEIGHTS_DIR / "yolov8n.pt", data=None, imgsz=160, half=False, int8=False, device="cpu", verbose=False
+    model=WEIGHTS_DIR / "yolov8n.pt",
+    data=None,
+    imgsz=160,
+    half=False,
+    int8=False,
+    device="cpu",
+    verbose=False,
+    eps=1e-3,
 ):
     """
     Benchmark a YOLO model across different formats for speed and accuracy.
 
     Args:
-        model (str | Path | optional): Path to the model file or directory. Default is
+        model (str | Path): Path to the model file or directory.
-            Path(SETTINGS['weights_dir']) / 'yolov8n.pt'.
+        data (str | None): Dataset to evaluate on, inherited from TASK2DATA if not passed.
-        data (str, optional): Dataset to evaluate on, inherited from TASK2DATA if not passed. Default is None.
+        imgsz (int): Image size for the benchmark.
-        imgsz (int, optional): Image size for the benchmark. Default is 160.
+        half (bool): Use half-precision for the model if True.
-        half (bool, optional): Use half-precision for the model if True. Default is False.
+        int8 (bool): Use int8-precision for the model if True.
-        int8 (bool, optional): Use int8-precision for the model if True. Default is False.
+        device (str): Device to run the benchmark on, either 'cpu' or 'cuda'.
-        device (str, optional): Device to run the benchmark on, either 'cpu' or 'cuda'. Default is 'cpu'.
+        verbose (bool | float): If True or a float, assert benchmarks pass with given metric.
-        verbose (bool | float | optional): If True or a float, assert benchmarks pass with given metric.
+        eps (float): Epsilon value for divide by zero prevention.
-            Default is False.
 
     Returns:
-        df (pandas.DataFrame): A pandas DataFrame with benchmark results for each format, including file size,
+        (pandas.DataFrame): A pandas DataFrame with benchmark results for each format, including file size, metric,
-            metric, and inference time.
+            and inference time.
 
-    Example:
+    Examples:
-        ```python
+        Benchmark a YOLO model with default settings:
-        from ultralytics.utils.benchmarks import benchmark
+        >>> from ultralytics.utils.benchmarks import benchmark
-
+        >>> benchmark(model="yolov8n.pt", imgsz=640)
-        benchmark(model="yolov8n.pt", imgsz=640)
-        ```
     """
     import pandas as pd  # scope for faster 'import ultralytics'
 
@@ -106,6 +110,7 @@ def benchmark(
             if i in {11}:  # Paddle
                 assert not isinstance(model, YOLOWorld), "YOLOWorldv2 Paddle exports not supported yet"
                 assert not is_end2end, "End-to-end models not supported by PaddlePaddle yet"
+                assert LINUX or MACOS, "Windows Paddle exports not supported yet"
             if i in {12}:  # NCNN
                 assert not isinstance(model, YOLOWorld), "YOLOWorldv2 NCNN exports not supported yet"
             if "cpu" in device.type:
@@ -138,7 +143,7 @@ def benchmark(
                 data=data, batch=1, imgsz=imgsz, plots=False, device=device, half=half, int8=int8, verbose=False
             )
             metric, speed = results.results_dict[key], results.speed["inference"]
-            fps = round((1000 / speed), 2)  # frames per second
+            fps = round(1000 / (speed + eps), 2)  # frames per second
             y.append([name, "✅", round(file_size(filename), 1), round(metric, 4), round(speed, 2), fps])
         except Exception as e:
             if verbose:
@@ -165,10 +170,10 @@ def benchmark(
 
 
 class RF100Benchmark:
-    """Benchmark YOLO model performance across formats for speed and accuracy."""
+    """Benchmark YOLO model performance across various formats for speed and accuracy."""
 
     def __init__(self):
-        """Function for initialization of RF100Benchmark."""
+        """Initialize the RF100Benchmark class for benchmarking YOLO model performance across various formats."""
         self.ds_names = []
         self.ds_cfg_list = []
         self.rf = None
@@ -180,6 +185,11 @@ class RF100Benchmark:
 
         Args:
             api_key (str): The API key.
+
+        Examples:
+            Set the Roboflow API key for accessing datasets:
+            >>> benchmark = RF100Benchmark()
+            >>> benchmark.set_key("your_roboflow_api_key")
         """
         check_requirements("roboflow")
         from roboflow import Roboflow
@@ -188,10 +198,15 @@ class RF100Benchmark:
 
     def parse_dataset(self, ds_link_txt="datasets_links.txt"):
         """
-        Parse dataset links and downloads datasets.
+        Parse dataset links and download datasets.
 
         Args:
-            ds_link_txt (str): Path to dataset_links file.
+            ds_link_txt (str): Path to the file containing dataset links.
+
+        Examples:
+            >>> benchmark = RF100Benchmark()
+            >>> benchmark.set_key("api_key")
+            >>> benchmark.parse_dataset("datasets_links.txt")
         """
         (shutil.rmtree("rf-100"), os.mkdir("rf-100")) if os.path.exists("rf-100") else os.mkdir("rf-100")
         os.chdir("rf-100")
@@ -217,10 +232,13 @@ class RF100Benchmark:
     @staticmethod
     def fix_yaml(path):
         """
-        Function to fix YAML train and val path.
+        Fixes the train and validation paths in a given YAML file.
 
         Args:
-            path (str): YAML file path.
+            path (str): Path to the YAML file to be fixed.
+
+        Examples:
+            >>> RF100Benchmark.fix_yaml("path/to/data.yaml")
         """
         with open(path) as file:
             yaml_data = yaml.safe_load(file)
@@ -231,13 +249,21 @@ class RF100Benchmark:
 
     def evaluate(self, yaml_path, val_log_file, eval_log_file, list_ind):
         """
-        Model evaluation on validation results.
+        Evaluate model performance on validation results.
 
         Args:
-            yaml_path (str): YAML file path.
+            yaml_path (str): Path to the YAML configuration file.
-            val_log_file (str): val_log_file path.
+            val_log_file (str): Path to the validation log file.
-            eval_log_file (str): eval_log_file path.
+            eval_log_file (str): Path to the evaluation log file.
-            list_ind (int): Index for current dataset.
+            list_ind (int): Index of the current dataset in the list.
+
+        Returns:
+            (float): The mean average precision (mAP) value for the evaluated model.
+
+        Examples:
+            Evaluate a model on a specific dataset
+            >>> benchmark = RF100Benchmark()
+            >>> benchmark.evaluate("path/to/data.yaml", "path/to/val_log.txt", "path/to/eval_log.txt", 0)
         """
         skip_symbols = ["🚀", "⚠️", "💡", "❌"]
         with open(yaml_path) as stream:
@@ -285,21 +311,23 @@ class ProfileModels:
     This class profiles the performance of different models, returning results such as model speed and FLOPs.
 
     Attributes:
-        paths (list): Paths of the models to profile.
+        paths (List[str]): Paths of the models to profile.
-        num_timed_runs (int): Number of timed runs for the profiling. Default is 100.
+        num_timed_runs (int): Number of timed runs for the profiling.
-        num_warmup_runs (int): Number of warmup runs before profiling. Default is 10.
+        num_warmup_runs (int): Number of warmup runs before profiling.
-        min_time (float): Minimum number of seconds to profile for. Default is 60.
+        min_time (float): Minimum number of seconds to profile for.
-        imgsz (int): Image size used in the models. Default is 640.
+        imgsz (int): Image size used in the models.
+        half (bool): Flag to indicate whether to use FP16 half-precision for TensorRT profiling.
+        trt (bool): Flag to indicate whether to profile using TensorRT.
+        device (torch.device): Device used for profiling.
 
     Methods:
-        profile(): Profiles the models and prints the result.
+        profile: Profiles the models and prints the result.
 
-    Example:
+    Examples:
-        ```python
+        Profile models and print results
-        from ultralytics.utils.benchmarks import ProfileModels
+        >>> from ultralytics.utils.benchmarks import ProfileModels
-
+        >>> profiler = ProfileModels(["yolov8n.yaml", "yolov8s.yaml"], imgsz=640)
-        ProfileModels(["yolov8n.yaml", "yolov8s.yaml"], imgsz=640).profile()
+        >>> profiler.profile()
-        ```
     """
 
     def __init__(
@@ -317,17 +345,23 @@ class ProfileModels:
         Initialize the ProfileModels class for profiling models.
 
         Args:
-            paths (list): List of paths of the models to be profiled.
+            paths (List[str]): List of paths of the models to be profiled.
-            num_timed_runs (int, optional): Number of timed runs for the profiling. Default is 100.
+            num_timed_runs (int): Number of timed runs for the profiling.
-            num_warmup_runs (int, optional): Number of warmup runs before the actual profiling starts. Default is 10.
+            num_warmup_runs (int): Number of warmup runs before the actual profiling starts.
-            min_time (float, optional): Minimum time in seconds for profiling a model. Default is 60.
+            min_time (float): Minimum time in seconds for profiling a model.
-            imgsz (int, optional): Size of the image used during profiling. Default is 640.
+            imgsz (int): Size of the image used during profiling.
-            half (bool, optional): Flag to indicate whether to use FP16 half-precision for TensorRT profiling.
+            half (bool): Flag to indicate whether to use FP16 half-precision for TensorRT profiling.
-            trt (bool, optional): Flag to indicate whether to profile using TensorRT. Default is True.
+            trt (bool): Flag to indicate whether to profile using TensorRT.
-            device (torch.device, optional): Device used for profiling. If None, it is determined automatically.
+            device (torch.device | None): Device used for profiling. If None, it is determined automatically.
 
         Notes:
-            FP16 'half' argument option removed for ONNX as slower on CPU than FP32
+            FP16 'half' argument option removed for ONNX as slower on CPU than FP32.
+
+        Examples:
+            Initialize and profile models
+            >>> from ultralytics.utils.benchmarks import ProfileModels
+            >>> profiler = ProfileModels(["yolov8n.yaml", "yolov8s.yaml"], imgsz=640)
+            >>> profiler.profile()
         """
         self.paths = paths
         self.num_timed_runs = num_timed_runs
@@ -339,7 +373,7 @@ class ProfileModels:
         self.device = device or torch.device(0 if torch.cuda.is_available() else "cpu")
 
     def profile(self):
-        """Logs the benchmarking results of a model, checks metrics against floor and returns the results."""
+        """Profiles YOLO models for speed and accuracy across various formats including ONNX and TensorRT."""
         files = self.get_files()
 
         if not files:
@@ -404,7 +438,7 @@ class ProfileModels:
 
     @staticmethod
     def iterative_sigma_clipping(data, sigma=2, max_iters=3):
-        """Applies an iterative sigma clipping algorithm to the given data times number of iterations."""
+        """Applies iterative sigma clipping to data to remove outliers based on specified sigma and iteration count."""
         data = np.array(data)
         for _ in range(max_iters):
             mean, std = np.mean(data), np.std(data)
@@ -415,7 +449,7 @@ class ProfileModels:
         return data
 
     def profile_tensorrt_model(self, engine_file: str, eps: float = 1e-3):
-        """Profiles the TensorRT model, measuring average run time and standard deviation among runs."""
+        """Profiles YOLO model performance with TensorRT, measuring average run time and standard deviation."""
         if not self.trt or not Path(engine_file).is_file():
             return 0.0, 0.0
 
@@ -499,7 +533,7 @@ class ProfileModels:
         return np.mean(run_times), np.std(run_times)
 
     def generate_table_row(self, model_name, t_onnx, t_engine, model_info):
-        """Generates a formatted string for a table row that includes model performance and metric details."""
+        """Generates a table row string with model performance metrics including inference times and model details."""
         layers, params, gradients, flops = model_info
         return (
             f"| {model_name:18s} | {self.imgsz} | - | {t_onnx[0]:.2f} ± {t_onnx[1]:.2f} ms | {t_engine[0]:.2f} ± "
@@ -508,7 +542,7 @@ class ProfileModels:
 
     @staticmethod
     def generate_results_dict(model_name, t_onnx, t_engine, model_info):
-        """Generates a dictionary of model details including name, parameters, GFLOPS and speed metrics."""
+        """Generates a dictionary of profiling results including model name, parameters, GFLOPs, and speed metrics."""
         layers, params, gradients, flops = model_info
         return {
             "model/name": model_name,
@@ -520,16 +554,19 @@ class ProfileModels:
 
     @staticmethod
     def print_table(table_rows):
-        """Formats and prints a comparison table for different models with given statistics and performance data."""
+        """Prints a formatted table of model profiling results, including speed and accuracy metrics."""
         gpu = torch.cuda.get_device_name(0) if torch.cuda.is_available() else "GPU"
-        header = (
+        headers = [
-            f"| Model | size<br><sup>(pixels) | mAP<sup>val<br>50-95 | Speed<br><sup>CPU ONNX<br>(ms) | "
+            "Model",
-            f"Speed<br><sup>{gpu} TensorRT<br>(ms) | params<br><sup>(M) | FLOPs<br><sup>(B) |"
+            "size<br><sup>(pixels)",
-        )
+            "mAP<sup>val<br>50-95",
-        separator = (
+            f"Speed<br><sup>CPU ({get_cpu_info()}) ONNX<br>(ms)",
-            "|-------------|---------------------|--------------------|------------------------------|"
+            f"Speed<br><sup>{gpu} TensorRT<br>(ms)",
-            "-----------------------------------|------------------|-----------------|"
+            "params<br><sup>(M)",
-        )
+            "FLOPs<br><sup>(B)",
+        ]
+        header = "|" + "|".join(f" {h} " for h in headers) + "|"
+        separator = "|" + "|".join("-" * (len(h) + 2) for h in headers) + "|"
 
         print(f"\n\n{header}")
         print(separator)