HUB setup (#108)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Glenn Jocher <glenn.jocher@ultralytics.com>

ultralytics/hub/utils.py

@@ -0,0 +1,139 @@
+import shutil
+import threading
+import time
+import uuid
+
+import requests
+
+from ultralytics.hub.config import HUB_API_ROOT
+from ultralytics.yolo.utils import LOGGER, RANK, SETTINGS, colorstr, emojis
+
+PREFIX = colorstr('Ultralytics: ')
+HELP_MSG = 'If this issue persists please visit https://github.com/ultralytics/hub/issues for assistance.'
+
+
+def check_dataset_disk_space(url='https://github.com/ultralytics/yolov5/releases/download/v1.0/coco128.zip', sf=2.0):
+    # Check that url fits on disk with safety factor sf, i.e. require 2GB free if url size is 1GB with sf=2.0
+    gib = 1 << 30  # bytes per GiB
+    data = int(requests.head(url).headers['Content-Length']) / gib  # dataset size (GB)
+    total, used, free = (x / gib for x in shutil.disk_usage("/"))  # bytes
+    LOGGER.info(f'{PREFIX}{data:.3f} GB dataset, {free:.1f}/{total:.1f} GB free disk space')
+    if data * sf < free:
+        return True  # sufficient space
+    LOGGER.warning(f'{PREFIX}WARNING: Insufficient free disk space {free:.1f} GB < {data * sf:.3f} GB required, '
+                   f'training cancelled ❌. Please free {data * sf - free:.1f} GB additional disk space and try again.')
+    return False  # insufficient space
+
+
+def request_with_credentials(url: str) -> any:
+    """ Make a ajax request with cookies attached """
+    from google.colab import output  # noqa
+    from IPython import display  # noqa
+    display.display(
+        display.Javascript("""
+            window._hub_tmp = new Promise((resolve, reject) => {
+                const timeout = setTimeout(() => reject("Failed authenticating existing browser session"), 5000)
+                fetch("%s", {
+                    method: 'POST',
+                    credentials: 'include'
+                })
+                    .then((response) => resolve(response.json()))
+                    .then((json) => {
+                    clearTimeout(timeout);
+                    }).catch((err) => {
+                    clearTimeout(timeout);
+                    reject(err);
+                });
+            });
+            """ % url))
+    return output.eval_js("_hub_tmp")
+
+
+# Deprecated TODO: eliminate this function?
+def split_key(key: str = '') -> tuple[str, str]:
+    """
+    Verify and split a 'api_key[sep]model_id' string, sep is one of '.' or '_'
+
+    Args:
+        key (str): The model key to split. If not provided, the user will be prompted to enter it.
+
+    Returns:
+        Tuple[str, str]: A tuple containing the API key and model ID.
+    """
+
+    import getpass
+
+    error_string = emojis(f'{PREFIX}Invalid API key ⚠️\n')  # error string
+    if not key:
+        key = getpass.getpass('Enter model key: ')
+    sep = '_' if '_' in key else '.' if '.' in key else None  # separator
+    assert sep, error_string
+    api_key, model_id = key.split(sep)
+    assert len(api_key) and len(model_id), error_string
+    return api_key, model_id
+
+
+def smart_request(*args, retry=3, timeout=30, thread=True, code=-1, method="post", **kwargs):
+    """
+    Makes an HTTP request using the 'requests' library, with exponential backoff retries up to a specified timeout.
+
+    Args:
+        *args: Positional arguments to be passed to the requests function specified in method.
+        retry (int, optional): Number of retries to attempt before giving up. Default is 3.
+        timeout (int, optional): Timeout in seconds after which the function will give up retrying. Default is 30.
+        thread (bool, optional): Whether to execute the request in a separate daemon thread. Default is True.
+        code (int, optional): An identifier for the request, used for logging purposes. Default is -1.
+        method (str, optional): The HTTP method to use for the request. Choices are 'post' and 'get'. Default is 'post'.
+        **kwargs: Keyword arguments to be passed to the requests function specified in method.
+
+    Returns:
+        requests.Response: The HTTP response object. If the request is executed in a separate thread, returns None.
+    """
+    retry_codes = (408, 500)  # retry only these codes
+    methods = {'post': requests.post, 'get': requests.get}  # request methods
+
+    def fcn(*args, **kwargs):
+        t0 = time.time()
+        for i in range(retry + 1):
+            if (time.time() - t0) > timeout:
+                break
+            r = methods[method](*args, **kwargs)  # i.e. post(url, data, json, files)
+            if r.status_code == 200:
+                break
+            try:
+                m = r.json().get('message', 'No JSON message.')
+            except Exception:
+                m = 'Unable to read JSON.'
+            if i == 0:
+                if r.status_code in retry_codes:
+                    m += f' Retrying {retry}x for {timeout}s.' if retry else ''
+                elif r.status_code == 429:  # rate limit
+                    h = r.headers  # response headers
+                    m = f"Rate limit reached ({h['X-RateLimit-Remaining']}/{h['X-RateLimit-Limit']}). " \
+                        f"Please retry after {h['Retry-After']}s."
+                LOGGER.warning(f"{PREFIX}{m} {HELP_MSG} ({r.status_code} #{code})")
+                if r.status_code not in retry_codes:
+                    return r
+            time.sleep(2 ** i)  # exponential standoff
+        return r
+
+    if thread:
+        threading.Thread(target=fcn, args=args, kwargs=kwargs, daemon=True).start()
+    else:
+        return fcn(*args, **kwargs)
+
+
+def sync_analytics(cfg, enabled=False):
+    """
+   Sync analytics data if enabled in the global settings
+
+    Args:
+        cfg (DictConfig): Configuration for the task and mode.
+        enabled (bool): For debugging.
+    """
+    if SETTINGS['sync'] and RANK in {-1, 0} and enabled:
+        cfg = dict(cfg)  # convert type from DictConfig to dict
+        cfg['uuid'] = uuid.getnode()  # add the device UUID to the configuration data
+
+        # Send a request to the HUB API to sync the analytics data
+        smart_request(f'{HUB_API_ROOT}/analytics', data=cfg, headers=None, code=3, retry=0)