Start export implementation (#110)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
This commit is contained in:
Glenn Jocher 2022-12-29 14:17:14 +01:00 committed by GitHub
parent c1b38428bc
commit 92dad1c1b5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
32 changed files with 827 additions and 222 deletions

View file

@ -1,6 +1,7 @@
import contextlib
import glob
import os
import urllib
from datetime import datetime
from pathlib import Path
from zipfile import ZipFile
@ -43,7 +44,7 @@ def increment_path(path, exist_ok=False, sep='', mkdir=False):
return path
def save_yaml(file='data.yaml', data=None):
def yaml_save(file='data.yaml', data=None):
# Single-line safe yaml saving
with open(file, 'w') as f:
yaml.safe_dump({k: str(v) if isinstance(v, Path) else v for k, v in data.items()}, f, sort_keys=False)
@ -52,7 +53,7 @@ def save_yaml(file='data.yaml', data=None):
def yaml_load(file='data.yaml'):
# Single-line safe yaml loading
with open(file, errors='ignore') as f:
return yaml.safe_load(f)
return {**yaml.safe_load(f), 'yaml_file': file} # add YAML filename to dict and return
def unzip_file(file, path=None, exclude=('.DS_Store', '__MACOSX')):
@ -77,6 +78,24 @@ def file_date(path=__file__):
return f'{t.year}-{t.month}-{t.day}'
def file_size(path):
# Return file/dir size (MB)
mb = 1 << 20 # bytes to MiB (1024 ** 2)
path = Path(path)
if path.is_file():
return path.stat().st_size / mb
elif path.is_dir():
return sum(f.stat().st_size for f in path.glob('**/*') if f.is_file()) / mb
else:
return 0.0
def url2file(url):
# Convert URL to filename, i.e. https://url.com/file.txt?auth -> file.txt
url = str(Path(url)).replace(':/', '://') # Pathlib turns :// -> :/
return Path(urllib.parse.unquote(url)).name.split('?')[0] # '%2F' to '/', split https://url.com/file.txt?auth
def get_latest_run(search_dir='.'):
# Return path to most recent 'last.pt' in /runs (i.e. to --resume from)
last_list = glob.glob(f'{search_dir}/**/last*.pt', recursive=True)