HongChuang/ultralytics-ascend
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 3

ultralytics 8.2.103 Windows Benchmarks CI (#16523)

Browse source 
Signed-off-by: UltralyticsAssistant <web@ultralytics.com>
Co-authored-by: UltralyticsAssistant <web@ultralytics.com>
This commit is contained in:
Glenn Jocher 2024-09-28 20:31:21 +02:00 • committed by GitHub
parent 54eccccd11
commit 2530755210
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 105 additions and 68 deletions
Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/ci.yaml vendored
View file

@ -98,7 +98,7 @@ jobs:
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-14]
os: [ubuntu-latest, windows-latest, macos-14]
python-version: ["3.11"]
model: [yolov8n]
steps:

2
ultralytics/__init__.py
View file

@ -1,6 +1,6 @@
# Ultralytics YOLO 🚀, AGPL-3.0 license
__version__ = "8.2.102"
__version__ = "8.2.103"
import os

169
ultralytics/utils/benchmarks.py
View file

@ -43,36 +43,40 @@ from ultralytics.utils import ARM64, ASSETS, IS_JETSON, IS_RASPBERRYPI, LINUX, L
from ultralytics.utils.checks import IS_PYTHON_3_12, check_requirements, check_yolo
from ultralytics.utils.downloads import safe_download
from ultralytics.utils.files import file_size
from ultralytics.utils.torch_utils import select_device
from ultralytics.utils.torch_utils import get_cpu_info, select_device
def benchmark(
model=WEIGHTS_DIR / "yolov8n.pt", data=None, imgsz=160, half=False, int8=False, device="cpu", verbose=False
model=WEIGHTS_DIR / "yolov8n.pt",
data=None,
imgsz=160,
half=False,
int8=False,
device="cpu",
verbose=False,
eps=1e-3,
):
"""
Benchmark a YOLO model across different formats for speed and accuracy.
Args:
model (str | Path | optional): Path to the model file or directory. Default is
Path(SETTINGS['weights_dir']) / 'yolov8n.pt'.
data (str, optional): Dataset to evaluate on, inherited from TASK2DATA if not passed. Default is None.
imgsz (int, optional): Image size for the benchmark. Default is 160.
half (bool, optional): Use half-precision for the model if True. Default is False.
int8 (bool, optional): Use int8-precision for the model if True. Default is False.
device (str, optional): Device to run the benchmark on, either 'cpu' or 'cuda'. Default is 'cpu'.
verbose (bool | float | optional): If True or a float, assert benchmarks pass with given metric.
Default is False.
model (str | Path): Path to the model file or directory.
data (str | None): Dataset to evaluate on, inherited from TASK2DATA if not passed.
imgsz (int): Image size for the benchmark.
half (bool): Use half-precision for the model if True.
int8 (bool): Use int8-precision for the model if True.
device (str): Device to run the benchmark on, either 'cpu' or 'cuda'.
verbose (bool | float): If True or a float, assert benchmarks pass with given metric.
eps (float): Epsilon value for divide by zero prevention.
Returns:
df (pandas.DataFrame): A pandas DataFrame with benchmark results for each format, including file size,
metric, and inference time.
(pandas.DataFrame): A pandas DataFrame with benchmark results for each format, including file size, metric,
and inference time.
Example:
```python
from ultralytics.utils.benchmarks import benchmark
benchmark(model="yolov8n.pt", imgsz=640)
```
Examples:
Benchmark a YOLO model with default settings:
>>> from ultralytics.utils.benchmarks import benchmark
>>> benchmark(model="yolov8n.pt", imgsz=640)
"""
import pandas as pd # scope for faster 'import ultralytics'
@ -106,6 +110,7 @@ def benchmark(
if i in {11}: # Paddle
assert not isinstance(model, YOLOWorld), "YOLOWorldv2 Paddle exports not supported yet"
assert not is_end2end, "End-to-end models not supported by PaddlePaddle yet"
assert LINUX or MACOS, "Windows Paddle exports not supported yet"
if i in {12}: # NCNN
assert not isinstance(model, YOLOWorld), "YOLOWorldv2 NCNN exports not supported yet"
if "cpu" in device.type:
@ -138,7 +143,7 @@ def benchmark(
data=data, batch=1, imgsz=imgsz, plots=False, device=device, half=half, int8=int8, verbose=False
)
metric, speed = results.results_dict[key], results.speed["inference"]
fps = round((1000 / speed), 2) # frames per second
fps = round(1000 / (speed + eps), 2) # frames per second
y.append([name, "✅", round(file_size(filename), 1), round(metric, 4), round(speed, 2), fps])
except Exception as e:
if verbose:
@ -165,10 +170,10 @@ def benchmark(
class RF100Benchmark:
"""Benchmark YOLO model performance across formats for speed and accuracy."""
"""Benchmark YOLO model performance across various formats for speed and accuracy."""
def __init__(self):
"""Function for initialization of RF100Benchmark."""
"""Initialize the RF100Benchmark class for benchmarking YOLO model performance across various formats."""
self.ds_names = []
self.ds_cfg_list = []
self.rf = None
@ -180,6 +185,11 @@ class RF100Benchmark:
Args:
api_key (str): The API key.
Examples:
Set the Roboflow API key for accessing datasets:
>>> benchmark = RF100Benchmark()
>>> benchmark.set_key("your_roboflow_api_key")
"""
check_requirements("roboflow")
from roboflow import Roboflow
@ -188,10 +198,15 @@ class RF100Benchmark:
def parse_dataset(self, ds_link_txt="datasets_links.txt"):
"""
Parse dataset links and downloads datasets.
Parse dataset links and download datasets.
Args:
ds_link_txt (str): Path to dataset_links file.
ds_link_txt (str): Path to the file containing dataset links.
Examples:
>>> benchmark = RF100Benchmark()
>>> benchmark.set_key("api_key")
>>> benchmark.parse_dataset("datasets_links.txt")
"""
(shutil.rmtree("rf-100"), os.mkdir("rf-100")) if os.path.exists("rf-100") else os.mkdir("rf-100")
os.chdir("rf-100")
@ -217,10 +232,13 @@ class RF100Benchmark:
@staticmethod
def fix_yaml(path):
"""
Function to fix YAML train and val path.
Fixes the train and validation paths in a given YAML file.
Args:
path (str): YAML file path.
path (str): Path to the YAML file to be fixed.
Examples:
>>> RF100Benchmark.fix_yaml("path/to/data.yaml")
"""
with open(path) as file:
yaml_data = yaml.safe_load(file)
@ -231,13 +249,21 @@ class RF100Benchmark:
def evaluate(self, yaml_path, val_log_file, eval_log_file, list_ind):
"""
Model evaluation on validation results.
Evaluate model performance on validation results.
Args:
yaml_path (str): YAML file path.
val_log_file (str): val_log_file path.
eval_log_file (str): eval_log_file path.
list_ind (int): Index for current dataset.
yaml_path (str): Path to the YAML configuration file.
val_log_file (str): Path to the validation log file.
eval_log_file (str): Path to the evaluation log file.
list_ind (int): Index of the current dataset in the list.
Returns:
(float): The mean average precision (mAP) value for the evaluated model.
Examples:
Evaluate a model on a specific dataset
>>> benchmark = RF100Benchmark()
>>> benchmark.evaluate("path/to/data.yaml", "path/to/val_log.txt", "path/to/eval_log.txt", 0)
"""
skip_symbols = ["🚀", "⚠️", "💡", "❌"]
with open(yaml_path) as stream:
@ -285,21 +311,23 @@ class ProfileModels:
This class profiles the performance of different models, returning results such as model speed and FLOPs.
Attributes:
paths (list): Paths of the models to profile.
num_timed_runs (int): Number of timed runs for the profiling. Default is 100.
num_warmup_runs (int): Number of warmup runs before profiling. Default is 10.
min_time (float): Minimum number of seconds to profile for. Default is 60.
imgsz (int): Image size used in the models. Default is 640.
paths (List[str]): Paths of the models to profile.
num_timed_runs (int): Number of timed runs for the profiling.
num_warmup_runs (int): Number of warmup runs before profiling.
min_time (float): Minimum number of seconds to profile for.
imgsz (int): Image size used in the models.
half (bool): Flag to indicate whether to use FP16 half-precision for TensorRT profiling.
trt (bool): Flag to indicate whether to profile using TensorRT.
device (torch.device): Device used for profiling.
Methods:
profile(): Profiles the models and prints the result.
profile: Profiles the models and prints the result.
Example:
```python
from ultralytics.utils.benchmarks import ProfileModels
ProfileModels(["yolov8n.yaml", "yolov8s.yaml"], imgsz=640).profile()
```
Examples:
Profile models and print results
>>> from ultralytics.utils.benchmarks import ProfileModels
>>> profiler = ProfileModels(["yolov8n.yaml", "yolov8s.yaml"], imgsz=640)
>>> profiler.profile()
"""
def __init__(
@ -317,17 +345,23 @@ class ProfileModels:
Initialize the ProfileModels class for profiling models.
Args:
paths (list): List of paths of the models to be profiled.
num_timed_runs (int, optional): Number of timed runs for the profiling. Default is 100.
num_warmup_runs (int, optional): Number of warmup runs before the actual profiling starts. Default is 10.
min_time (float, optional): Minimum time in seconds for profiling a model. Default is 60.
imgsz (int, optional): Size of the image used during profiling. Default is 640.
half (bool, optional): Flag to indicate whether to use FP16 half-precision for TensorRT profiling.
trt (bool, optional): Flag to indicate whether to profile using TensorRT. Default is True.
device (torch.device, optional): Device used for profiling. If None, it is determined automatically.
paths (List[str]): List of paths of the models to be profiled.
num_timed_runs (int): Number of timed runs for the profiling.
num_warmup_runs (int): Number of warmup runs before the actual profiling starts.
min_time (float): Minimum time in seconds for profiling a model.
imgsz (int): Size of the image used during profiling.
half (bool): Flag to indicate whether to use FP16 half-precision for TensorRT profiling.
trt (bool): Flag to indicate whether to profile using TensorRT.
device (torch.device | None): Device used for profiling. If None, it is determined automatically.
Notes:
FP16 'half' argument option removed for ONNX as slower on CPU than FP32
FP16 'half' argument option removed for ONNX as slower on CPU than FP32.
Examples:
Initialize and profile models
>>> from ultralytics.utils.benchmarks import ProfileModels
>>> profiler = ProfileModels(["yolov8n.yaml", "yolov8s.yaml"], imgsz=640)
>>> profiler.profile()
"""
self.paths = paths
self.num_timed_runs = num_timed_runs
@ -339,7 +373,7 @@ class ProfileModels:
self.device = device or torch.device(0 if torch.cuda.is_available() else "cpu")
def profile(self):
"""Logs the benchmarking results of a model, checks metrics against floor and returns the results."""
"""Profiles YOLO models for speed and accuracy across various formats including ONNX and TensorRT."""
files = self.get_files()
if not files:
@ -404,7 +438,7 @@ class ProfileModels:
@staticmethod
def iterative_sigma_clipping(data, sigma=2, max_iters=3):
"""Applies an iterative sigma clipping algorithm to the given data times number of iterations."""
"""Applies iterative sigma clipping to data to remove outliers based on specified sigma and iteration count."""
data = np.array(data)
for _ in range(max_iters):
mean, std = np.mean(data), np.std(data)
@ -415,7 +449,7 @@ class ProfileModels:
return data
def profile_tensorrt_model(self, engine_file: str, eps: float = 1e-3):
"""Profiles the TensorRT model, measuring average run time and standard deviation among runs."""
"""Profiles YOLO model performance with TensorRT, measuring average run time and standard deviation."""
if not self.trt or not Path(engine_file).is_file():
return 0.0, 0.0
@ -499,7 +533,7 @@ class ProfileModels:
return np.mean(run_times), np.std(run_times)
def generate_table_row(self, model_name, t_onnx, t_engine, model_info):
"""Generates a formatted string for a table row that includes model performance and metric details."""
"""Generates a table row string with model performance metrics including inference times and model details."""
layers, params, gradients, flops = model_info
return (
f"| {model_name:18s} | {self.imgsz} | - | {t_onnx[0]:.2f} ± {t_onnx[1]:.2f} ms | {t_engine[0]:.2f} ± "
@ -508,7 +542,7 @@ class ProfileModels:
@staticmethod
def generate_results_dict(model_name, t_onnx, t_engine, model_info):
"""Generates a dictionary of model details including name, parameters, GFLOPS and speed metrics."""
"""Generates a dictionary of profiling results including model name, parameters, GFLOPs, and speed metrics."""
layers, params, gradients, flops = model_info
return {
"model/name": model_name,
@ -520,16 +554,19 @@ class ProfileModels:
@staticmethod
def print_table(table_rows):
"""Formats and prints a comparison table for different models with given statistics and performance data."""
"""Prints a formatted table of model profiling results, including speed and accuracy metrics."""
gpu = torch.cuda.get_device_name(0) if torch.cuda.is_available() else "GPU"
header = (
f"| Model | size<br><sup>(pixels) | mAP<sup>val<br>50-95 | Speed<br><sup>CPU ONNX<br>(ms) | "
f"Speed<br><sup>{gpu} TensorRT<br>(ms) | params<br><sup>(M) | FLOPs<br><sup>(B) |"
)
separator = (
"|-------------|---------------------|--------------------|------------------------------|"
"-----------------------------------|------------------|-----------------|"
)
headers = [
"Model",
"size<br><sup>(pixels)",
"mAP<sup>val<br>50-95",
f"Speed<br><sup>CPU ({get_cpu_info()}) ONNX<br>(ms)",
f"Speed<br><sup>{gpu} TensorRT<br>(ms)",
"params<br><sup>(M)",
"FLOPs<br><sup>(B)",
]
header = "|" + "|".join(f" {h} " for h in headers) + "|"
separator = "|" + "|".join("-" * (len(h) + 2) for h in headers) + "|"
print(f"\n\n{header}")
print(separator)