mirror of
https://github.com/gsi-upm/soil
synced 2025-07-01 19:12:20 +00:00
d9947c2c52
Documentation needs some improvement The API has been simplified to only allow for ONE topology per NetworkEnvironment. This covers the main use case, and simplifies the code.
135 lines
4.0 KiB
Python
135 lines
4.0 KiB
Python
import logging
|
|
from time import time as current_time, strftime, gmtime, localtime
|
|
import os
|
|
import traceback
|
|
|
|
from functools import partial
|
|
from shutil import copyfile
|
|
from multiprocessing import Pool
|
|
|
|
from contextlib import contextmanager
|
|
|
|
logger = logging.getLogger('soil')
|
|
logger.setLevel(logging.INFO)
|
|
|
|
timeformat = "%H:%M:%S"
|
|
|
|
if os.environ.get('SOIL_VERBOSE', ''):
|
|
logformat = "[%(levelname)-5.5s][%(asctime)s][%(name)s]: %(message)s"
|
|
else:
|
|
logformat = "[%(levelname)-5.5s][%(asctime)s] %(message)s"
|
|
|
|
logFormatter = logging.Formatter(logformat, timeformat)
|
|
consoleHandler = logging.StreamHandler()
|
|
consoleHandler.setFormatter(logFormatter)
|
|
|
|
logging.basicConfig(level=logging.INFO,
|
|
handlers=[consoleHandler,])
|
|
|
|
|
|
@contextmanager
|
|
def timer(name='task', pre="", function=logger.info, to_object=None):
|
|
start = current_time()
|
|
function('{}Starting {} at {}.'.format(pre, name,
|
|
strftime("%X", gmtime(start))))
|
|
yield start
|
|
end = current_time()
|
|
function('{}Finished {} at {} in {} seconds'.format(pre, name,
|
|
strftime("%X", gmtime(end)),
|
|
str(end-start)))
|
|
if to_object:
|
|
to_object.start = start
|
|
to_object.end = end
|
|
|
|
|
|
def safe_open(path, mode='r', backup=True, **kwargs):
|
|
outdir = os.path.dirname(path)
|
|
if outdir and not os.path.exists(outdir):
|
|
os.makedirs(outdir)
|
|
if backup and 'w' in mode and os.path.exists(path):
|
|
creation = os.path.getctime(path)
|
|
stamp = strftime('%Y-%m-%d_%H.%M.%S', localtime(creation))
|
|
|
|
backup_dir = os.path.join(outdir, 'backup')
|
|
if not os.path.exists(backup_dir):
|
|
os.makedirs(backup_dir)
|
|
newpath = os.path.join(backup_dir, '{}@{}'.format(os.path.basename(path),
|
|
stamp))
|
|
copyfile(path, newpath)
|
|
return open(path, mode=mode, **kwargs)
|
|
|
|
|
|
@contextmanager
|
|
def open_or_reuse(f, *args, **kwargs):
|
|
try:
|
|
with safe_open(f, *args, **kwargs) as f:
|
|
yield f
|
|
except (AttributeError, TypeError):
|
|
yield f
|
|
|
|
def flatten_dict(d):
|
|
if not isinstance(d, dict):
|
|
return d
|
|
return dict(_flatten_dict(d))
|
|
|
|
def _flatten_dict(d, prefix=''):
|
|
if not isinstance(d, dict):
|
|
# print('END:', prefix, d)
|
|
yield prefix, d
|
|
return
|
|
if prefix:
|
|
prefix = prefix + '.'
|
|
for k, v in d.items():
|
|
# print(k, v)
|
|
res = list(_flatten_dict(v, prefix='{}{}'.format(prefix, k)))
|
|
# print('RES:', res)
|
|
yield from res
|
|
|
|
|
|
def unflatten_dict(d):
|
|
out = {}
|
|
for k, v in d.items():
|
|
target = out
|
|
if not isinstance(k, str):
|
|
target[k] = v
|
|
continue
|
|
tokens = k.split('.')
|
|
if len(tokens) < 2:
|
|
target[k] = v
|
|
continue
|
|
for token in tokens[:-1]:
|
|
if token not in target:
|
|
target[token] = {}
|
|
target = target[token]
|
|
target[tokens[-1]] = v
|
|
return out
|
|
|
|
|
|
def run_and_return_exceptions(func, *args, **kwargs):
|
|
'''
|
|
A wrapper for run_trial that catches exceptions and returns them.
|
|
It is meant for async simulations.
|
|
'''
|
|
try:
|
|
return func(*args, **kwargs)
|
|
except Exception as ex:
|
|
if ex.__cause__ is not None:
|
|
ex = ex.__cause__
|
|
ex.message = ''.join(traceback.format_exception(type(ex), ex, ex.__traceback__)[:])
|
|
return ex
|
|
|
|
|
|
def run_parallel(func, iterable, parallel=False, **kwargs):
|
|
if parallel and not os.environ.get('SOIL_DEBUG', None):
|
|
p = Pool()
|
|
wrapped_func = partial(run_and_return_exceptions,
|
|
func, **kwargs)
|
|
for i in p.imap_unordered(wrapped_func, iterable):
|
|
if isinstance(i, Exception):
|
|
logger.error('Trial failed:\n\t%s', i.message)
|
|
continue
|
|
yield i
|
|
else:
|
|
for i in iterable:
|
|
yield func(i, **kwargs)
|