HongChuang/ultralytics-ascend
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 3

Hold failed upload metrics and upload with next upload metrics (#8513)

Browse source 
Co-authored-by: hassaanfarooq01 <hassaanfarooq01@gmail.com>
Co-authored-by: Glenn Jocher <glenn.jocher@ultralytics.com>
This commit is contained in:
Hassaan Farooq 2024-03-01 05:08:29 +05:00 • committed by GitHub
parent 33fff69f3d
commit 31cf94e7f9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 13 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

8
ultralytics/hub/session.py
View file

@ -52,6 +52,7 @@ class HUBTrainingSession:
"heartbeat": 300.0,
} # rate limits (seconds)
self.metrics_queue = {} # holds metrics for each epoch until upload
self.metrics_upload_failed_queue = {} # holds metrics for each epoch if upload failed
self.timers = {} # holds timers in ultralytics/utils/callbacks/hub.py
# Parse input
@ -234,6 +235,9 @@ class HUBTrainingSession:
self._show_upload_progress(progress_total, response)
if HTTPStatus.OK <= response.status_code < HTTPStatus.MULTIPLE_CHOICES:
# if request related to metrics upload
if kwargs.get("metrics"):
self.metrics_upload_failed_queue = {}
return response # Success, no need to retry
if i == 0:
@ -249,6 +253,10 @@ class HUBTrainingSession:
time.sleep(2**i) # Exponential backoff for retries
# if request related to metrics upload and exceed retries
if response is None and kwargs.get("metrics"):
self.metrics_upload_failed_queue.update(kwargs.get("metrics", None))
return response
if thread: