1
0
mirror of https://github.com/gsi-upm/soil synced 2024-11-12 22:42:28 +00:00

WIP: exporters

This commit is contained in:
J. Fernando Sánchez 2019-04-26 19:22:45 +02:00
parent a3ea434f23
commit 9bc036d185
16 changed files with 484 additions and 282 deletions

View File

@ -3,6 +3,19 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [0.14.0]
### Added
* Loading configuration from template definitions in the yaml, in preparation for SALib support.
The definition of the variables and their possible values (i.e., a problem in SALib terms), as well as a sampler function, can be provided.
Soil uses this definition and the template to generate a set of configurations.
* Simulation group names, to link related simulations. For now, they are only used to group all simulations in the same group under the same folder.
### Changed
* `dir_path` is now the directory for resources (modules, files), and the output dir has been renamed to `outdir`
## Fixed
* Modules with custom agents/environments failed to load when they were run from outside the directory of the definition file. Modules are now loaded from the directory of the simulation file in addition to the working directory
## [0.13.8]
### Changed
* Moved TerroristNetworkModel to examples

View File

@ -1,5 +1,6 @@
---
name: simple
group: tests
dir_path: "/tmp/"
num_trials: 3
dry_run: True

29
examples/template.yml Normal file
View File

@ -0,0 +1,29 @@
---
vars:
bounds:
x1: [0, 1]
x2: [1, 2]
fixed:
x3: ["a", "b", "c"]
sampler: "SALib.sample.morris.sample"
samples: 10
template: |
group: simple
num_trials: 1
interval: 1
max_time: 2
seed: "CompleteSeed!"
dump: false
network_params:
generator: complete_graph
n: 10
network_agents:
- agent_type: CounterModel
weight: {{ x1 }}
state:
id: 0
- agent_type: AggregatedCounter
weight: {{ 1 - x1 }}
environment_params:
name: {{ x3 }}
skip_test: true

View File

@ -1,8 +1,10 @@
nxsim
nxsim>=0.1.2
simpy
networkx>=2.0
numpy
matplotlib
pyyaml
pandas
scipy
pandas>=0.23
scipy>=1.2
SALib>=1.3
Jinja2

View File

@ -15,7 +15,7 @@ from . import agents
from .simulation import *
from .environment import Environment
from .history import History
from . import utils
from . import serialization
from . import analysis
def main():
@ -44,6 +44,8 @@ def main():
help='folder to write results to. It defaults to the current directory.')
parser.add_argument('--synchronous', action='store_true',
help='Run trials serially and synchronously instead of in parallel. Defaults to false.')
parser.add_argument('-e', '--exporter', action='append',
help='Export environment and/or simulations using this exporter')
args = parser.parse_args()
@ -64,8 +66,9 @@ def main():
simulation.run_from_config(args.file,
dry_run=args.dry_run,
dump=dump,
exporters=args.exporter,
parallel=(not args.synchronous),
results_dir=args.output)
outdir=args.output)
except Exception:
if args.pdb:
pdb.post_mortem()

View File

@ -15,7 +15,7 @@ import json
from functools import wraps
from .. import utils, history
from .. import serialization, history
def as_node(agent):
@ -388,7 +388,7 @@ def serialize_type(agent_type, known_modules=[], **kwargs):
if isinstance(agent_type, str):
return agent_type
known_modules += ['soil.agents']
return utils.serialize(agent_type, known_modules=known_modules, **kwargs)[1] # Get the name of the class
return serialization.serialize(agent_type, known_modules=known_modules, **kwargs)[1] # Get the name of the class
def serialize_distribution(network_agents, known_modules=[]):
@ -409,7 +409,7 @@ def deserialize_type(agent_type, known_modules=[]):
if not isinstance(agent_type, str):
return agent_type
known = known_modules + ['soil.agents', 'soil.agents.custom' ]
agent_type = utils.deserializer(agent_type, known_modules=known)
agent_type = serialization.deserializer(agent_type, known_modules=known)
return agent_type

View File

@ -4,7 +4,7 @@ import glob
import yaml
from os.path import join
from . import utils, history
from . import serialization, history
def read_data(*args, group=False, **kwargs):
@ -56,7 +56,7 @@ def read_csv(filename, keys=None, convert_types=False, **kwargs):
def convert_row(row):
row['value'] = utils.deserialize(row['value_type'], row['value'])
row['value'] = serialization.deserialize(row['value_type'], row['value'])
return row

View File

@ -14,7 +14,7 @@ from networkx.readwrite import json_graph
import networkx as nx
import nxsim
from . import utils, agents, analysis, history
from . import serialization, agents, analysis, history
# These properties will be copied when pickling/unpickling the environment
_CONFIG_PROPS = [ 'name',
@ -22,7 +22,7 @@ _CONFIG_PROPS = [ 'name',
'default_state',
'interval',
'dry_run',
'dir_path',
'outdir',
]
class Environment(nxsim.NetworkEnvironment):
@ -44,7 +44,7 @@ class Environment(nxsim.NetworkEnvironment):
interval=1,
seed=None,
dry_run=False,
dir_path=None,
outdir=None,
topology=None,
*args, **kwargs):
self.name = name or 'UnnamedEnvironment'
@ -58,11 +58,11 @@ class Environment(nxsim.NetworkEnvironment):
self._env_agents = {}
self.dry_run = dry_run
self.interval = interval
self.dir_path = dir_path or tempfile.mkdtemp('soil-env')
self.outdir = outdir or tempfile.mkdtemp('soil-env')
if not dry_run:
self.get_path()
self._history = history.History(name=self.name if not dry_run else None,
dir_path=self.dir_path,
outdir=self.outdir,
backup=True)
# Add environment agents first, so their events get
# executed before network agents
@ -124,7 +124,7 @@ class Environment(nxsim.NetworkEnvironment):
elif agent_distribution:
agent_type, state = agents._agent_from_distribution(agent_distribution, agent_id=agent_id)
else:
utils.logger.debug('Skipping node {}'.format(agent_id))
serialization.logger.debug('Skipping node {}'.format(agent_id))
return
return self.set_agent(agent_id, agent_type, state)
@ -169,7 +169,7 @@ class Environment(nxsim.NetworkEnvironment):
def _save_state(self, now=None):
# for agent in self.agents:
# agent.save_state()
utils.logger.debug('Saving state @{}'.format(self.now))
serialization.logger.debug('Saving state @{}'.format(self.now))
self._history.save_records(self.state_to_tuples(now=now))
def save_state(self):
@ -180,7 +180,7 @@ class Environment(nxsim.NetworkEnvironment):
self._save_state()
while self.peek() != simpy.core.Infinity:
delay = max(self.peek() - self.now, self.interval)
utils.logger.debug('Step: {}'.format(self.now))
serialization.logger.debug('Step: {}'.format(self.now))
ev = self.event()
ev._ok = True
# Schedule the event with minimum priority so
@ -222,14 +222,14 @@ class Environment(nxsim.NetworkEnvironment):
'''
return self[key] if key in self else default
def get_path(self, dir_path=None):
dir_path = dir_path or self.dir_path
if not os.path.exists(dir_path):
def get_path(self, outdir=None):
outdir = outdir or self.outdir
if not os.path.exists(outdir):
try:
os.makedirs(dir_path)
os.makedirs(outdir)
except FileExistsError:
pass
return dir_path
return outdir
def get_agent(self, agent_id):
return self.G.node[agent_id]['agent']
@ -239,8 +239,8 @@ class Environment(nxsim.NetworkEnvironment):
return list(self.agents)
return [self.G.node[i]['agent'] for i in nodes]
def dump_csv(self, dir_path=None):
csv_name = os.path.join(self.get_path(dir_path),
def dump_csv(self, outdir=None):
csv_name = os.path.join(self.get_path(outdir),
'{}.environment.csv'.format(self.name))
with open(csv_name, 'w') as f:
@ -249,9 +249,9 @@ class Environment(nxsim.NetworkEnvironment):
for i in self.history_to_tuples():
cr.writerow(i)
def dump_gexf(self, dir_path=None):
def dump_gexf(self, outdir=None):
G = self.history_to_graph()
graph_path = os.path.join(self.get_path(dir_path),
graph_path = os.path.join(self.get_path(outdir),
self.name+".gexf")
# Workaround for geometric models
# See soil/soil#4
@ -262,7 +262,7 @@ class Environment(nxsim.NetworkEnvironment):
nx.write_gexf(G, graph_path, version="1.2draft")
def dump(self, dir_path=None, formats=None):
def dump(self, outdir=None, formats=None):
if not formats:
return
functions = {
@ -271,7 +271,7 @@ class Environment(nxsim.NetworkEnvironment):
}
for f in formats:
if f in functions:
functions[f](dir_path)
functions[f](outdir)
else:
raise ValueError('Unknown format: {}'.format(f))
@ -356,7 +356,7 @@ class Environment(nxsim.NetworkEnvironment):
def log_stats(self):
stats = self.stats()
utils.logger.info('Environment stats: \n{}'.format(yaml.dump(stats, default_flow_style=False)))
serialization.logger.info('Environment stats: \n{}'.format(yaml.dump(stats, default_flow_style=False)))
def __getstate__(self):
state = {}

43
soil/exporters.py Normal file
View File

@ -0,0 +1,43 @@
from .serialization import deserialize
import os
import time
def for_sim(simulation, names, dir_path=None):
exporters = []
for name in names:
mod = deserialize(name, known_modules=['soil.exporters'])
exporters.append(mod(simulation))
return exporters
class Base:
def __init__(self, simulation):
self.sim = simulation
def start(self):
pass
def end(self):
pass
def env(self):
pass
class Dummy(Base):
def start(self):
with open(os.path.join(self.sim.outdir, 'dummy')) as f:
f.write('simulation started @ {}'.format(time.time()))
def env(self, env):
with open(os.path.join(self.sim.outdir, 'dummy-trial-{}'.format(env.name))) as f:
for i in env.history_to_tuples():
f.write(','.join(i))
def end(self):
with open(os.path.join(self.sim.outdir, 'dummy')) as f:
f.write('simulation ended @ {}'.format(time.time()))

View File

@ -9,7 +9,7 @@ logger = logging.getLogger(__name__)
from collections import UserDict, namedtuple
from . import utils
from . import serialization
class History:
@ -17,9 +17,9 @@ class History:
Store and retrieve values from a sqlite database.
"""
def __init__(self, db_path=None, name=None, dir_path=None, backup=False):
def __init__(self, db_path=None, name=None, outdir=None, backup=False):
if db_path is None and name:
db_path = os.path.join(dir_path or os.getcwd(),
db_path = os.path.join(outdir or os.getcwd(),
'{}.db.sqlite'.format(name))
if db_path:
if backup and os.path.exists(db_path):
@ -94,9 +94,9 @@ class History:
if key not in self._dtypes:
self.read_types()
if key not in self._dtypes:
name = utils.name(value)
serializer = utils.serializer(name)
deserializer = utils.deserializer(name)
name = serialization.name(value)
serializer = serialization.serializer(name)
deserializer = serialization.deserializer(name)
self._dtypes[key] = (name, serializer, deserializer)
with self.db:
self.db.execute("replace into value_types (key, value_type) values (?, ?)", (key, name))
@ -135,8 +135,8 @@ class History:
with self.db:
res = self.db.execute("select key, value_type from value_types ").fetchall()
for k, v in res:
serializer = utils.serializer(v)
deserializer = utils.deserializer(v)
serializer = serialization.serializer(v)
deserializer = serialization.deserializer(v)
self._dtypes[k] = (v, serializer, deserializer)
def __getitem__(self, key):

199
soil/serialization.py Normal file
View File

@ -0,0 +1,199 @@
import os
import logging
import ast
import sys
import yaml
import importlib
from glob import glob
from random import random
from copy import deepcopy
from itertools import product, chain
from jinja2 import Template
import networkx as nx
logger = logging.getLogger('soil')
logger.setLevel(logging.INFO)
def load_network(network_params, dir_path=None):
if network_params is None:
return nx.Graph()
path = network_params.get('path', None)
if path:
if dir_path and not os.path.isabs(path):
path = os.path.join(dir_path, path)
extension = os.path.splitext(path)[1][1:]
kwargs = {}
if extension == 'gexf':
kwargs['version'] = '1.2draft'
kwargs['node_type'] = int
try:
method = getattr(nx.readwrite, 'read_' + extension)
except AttributeError:
raise AttributeError('Unknown format')
return method(path, **kwargs)
net_args = network_params.copy()
net_gen = net_args.pop('generator')
if dir_path not in sys.path:
sys.path.append(dir_path)
method = deserializer(net_gen,
known_modules=['networkx.generators',])
return method(**net_args)
def load_file(infile):
with open(infile, 'r') as f:
return list(chain.from_iterable(map(expand_template, load_string(f))))
def load_string(string):
yield from yaml.load_all(string)
def expand_template(config):
if 'template' not in config:
yield config
return
if 'vars' not in config:
raise ValueError(('You must provide a definition of variables'
' for the template.'))
template = Template(config['template'])
sampler_name = config.get('sampler', 'SALib.sample.morris.sample')
n_samples = int(config.get('samples', 100))
sampler = deserializer(sampler_name)
bounds = config['vars']['bounds']
problem = {
'num_vars': len(bounds),
'names': list(bounds.keys()),
'bounds': list(v for v in bounds.values())
}
samples = sampler(problem, n_samples)
lists = config['vars'].get('lists', {})
names = list(lists.keys())
values = list(lists.values())
combs = list(product(*values))
allnames = names + problem['names']
allvalues = [(list(i[0])+list(i[1])) for i in product(combs, samples)]
params = list(map(lambda x: dict(zip(allnames, x)), allvalues))
blank_str = template.render({k: 0 for k in allnames})
blank = list(load_string(blank_str))
if len(blank) > 1:
raise ValueError('Templates must not return more than one configuration')
if 'name' in blank[0]:
raise ValueError('Templates cannot be named, use group instead')
confs = []
for ps in params:
string = template.render(ps)
for c in load_string(string):
yield c
def load_files(*patterns, **kwargs):
for pattern in patterns:
for i in glob(pattern, **kwargs):
for config in load_file(i):
path = os.path.abspath(i)
if 'dir_path' not in config:
config['dir_path'] = os.path.dirname(path)
yield config, path
def load_config(config):
if isinstance(config, dict):
yield config, None
else:
yield from load_files(config)
builtins = importlib.import_module('builtins')
def name(value, known_modules=[]):
'''Return a name that can be imported, to serialize/deserialize an object'''
if value is None:
return 'None'
if not isinstance(value, type): # Get the class name first
value = type(value)
tname = value.__name__
if hasattr(builtins, tname):
return tname
modname = value.__module__
if modname == '__main__':
return tname
if known_modules and modname in known_modules:
return tname
for kmod in known_modules:
if not kmod:
continue
module = importlib.import_module(kmod)
if hasattr(module, tname):
return tname
return '{}.{}'.format(modname, tname)
def serializer(type_):
if type_ != 'str' and hasattr(builtins, type_):
return repr
return lambda x: x
def serialize(v, known_modules=[]):
'''Get a text representation of an object.'''
tname = name(v, known_modules=known_modules)
func = serializer(tname)
return func(v), tname
def deserializer(type_, known_modules=[]):
if type(type_) != str: # Already deserialized
return type_
if type_ == 'str':
return lambda x='': x
if type_ == 'None':
return lambda x=None: None
if hasattr(builtins, type_): # Check if it's a builtin type
cls = getattr(builtins, type_)
return lambda x=None: ast.literal_eval(x) if x is not None else cls()
# Otherwise, see if we can find the module and the class
modules = known_modules or []
options = []
for mod in modules:
if mod:
options.append((mod, type_))
if '.' in type_: # Fully qualified module
module, type_ = type_.rsplit(".", 1)
options.append ((module, type_))
errors = []
for modname, tname in options:
try:
module = importlib.import_module(modname)
cls = getattr(module, tname)
return getattr(cls, 'deserialize', cls)
except (ModuleNotFoundError, AttributeError) as ex:
errors.append((modname, tname, ex))
raise Exception('Could not find type {}. Tried: {}'.format(type_, errors))
def deserialize(type_, value=None, **kwargs):
'''Get an object from a text representation'''
if not isinstance(type_, str):
return type_
des = deserializer(type_, **kwargs)
if value is None:
return des
return des(value)

View File

@ -13,9 +13,10 @@ import pickle
from nxsim import NetworkSimulation
from . import utils, basestring, agents
from . import serialization, utils, basestring, agents
from .environment import Environment
from .utils import logger
from .serialization import logger
from .exporters import for_sim as exporters_for_sim
class Simulation(NetworkSimulation):
@ -50,6 +51,8 @@ class Simulation(NetworkSimulation):
---------
name : str, optional
name of the Simulation
group : str, optional
a group name can be used to link simulations
topology : networkx.Graph instance, optional
network_params : dict
parameters used to create a topology with networkx, if no topology is given
@ -60,8 +63,10 @@ class Simulation(NetworkSimulation):
states : list, optional
List of initial states corresponding to the nodes in the topology. Basic form is a list of integers
whose value indicates the state
dir_path : str, optional
Directory path where to save pickled objects
dir_path: str, optional
Directory path to load simulation assets (files, modules...)
outdir : str, optional
Directory path to save simulation results
seed : str, optional
Seed to use for the random generator
num_trials : int, optional
@ -80,38 +85,45 @@ class Simulation(NetworkSimulation):
"""
def __init__(self, name=None, topology=None, network_params=None,
def __init__(self, name=None, group=None, topology=None, network_params=None,
network_agents=None, agent_type=None, states=None,
default_state=None, interval=1, dump=None, dry_run=False,
dir_path=None, num_trials=1, max_time=100,
load_module=None, seed=None,
outdir=None, num_trials=1, max_time=100,
load_module=None, seed=None, dir_path=None,
environment_agents=None, environment_params=None,
environment_class=None, **kwargs):
self.seed = str(seed) or str(time.time())
self.load_module = load_module
self.network_params = network_params
self.name = name or 'UnnamedSimulation'
self.name = name or 'Unnamed_' + time.strftime("%Y-%m-%d_%H:%M:%S")
self.group = group or None
self.num_trials = num_trials
self.max_time = max_time
self.default_state = default_state or {}
if not outdir:
outdir = os.path.join(os.getcwd(),
'soil_output')
self.outdir = os.path.join(outdir,
self.group or '',
self.name)
self.dir_path = dir_path or os.getcwd()
self.interval = interval
self.dump = dump
self.dry_run = dry_run
sys.path += [self.dir_path, os.getcwd()]
sys.path += list(x for x in [self.outdir, os.getcwd(), self.dir_path] if x not in sys.path)
if topology is None:
topology = utils.load_network(network_params,
dir_path=self.dir_path)
topology = serialization.load_network(network_params,
dir_path=self.dir_path)
elif isinstance(topology, basestring) or isinstance(topology, dict):
topology = json_graph.node_link_graph(topology)
self.topology = nx.Graph(topology)
self.environment_params = environment_params or {}
self.environment_class = utils.deserialize(environment_class,
self.environment_class = serialization.deserialize(environment_class,
known_modules=['soil.environment', ]) or Environment
environment_agents = environment_agents or []
@ -130,34 +142,51 @@ class Simulation(NetworkSimulation):
return self.run(*args, **kwargs)
def run(self, *args, **kwargs):
return list(self.run_simulation_gen(*args, **kwargs))
return list(self._run_simulation_gen(*args, **kwargs))
def run_simulation_gen(self, *args, parallel=False, dry_run=False,
**kwargs):
p = Pool()
def _run_sync_or_async(self, parallel=False, *args, **kwargs):
if parallel:
p = Pool()
func = partial(self.run_trial_exceptions,
*args,
**kwargs)
for i in p.imap_unordered(func, range(self.num_trials)):
if isinstance(i, Exception):
logger.error('Trial failed:\n\t{}'.format(i.message))
continue
yield i
else:
for i in range(self.num_trials):
yield self.run_trial(i,
*args,
**kwargs)
def _run_simulation_gen(self, *args, parallel=False, dry_run=False,
exporters=None, **kwargs):
exporters = exporters_for_sim(self,
exporters or [])
with utils.timer('simulation {}'.format(self.name)):
if parallel:
func = partial(self.run_trial_exceptions, dry_run=dry_run or self.dry_run,
return_env=True,
**kwargs)
for i in p.imap_unordered(func, range(self.num_trials)):
if isinstance(i, Exception):
logger.error('Trial failed:\n\t{}'.format(i.message))
continue
yield i
else:
for i in range(self.num_trials):
yield self.run_trial(i, dry_run = dry_run or self.dry_run, **kwargs)
if not (dry_run or self.dry_run):
logger.info('Dumping results to {}'.format(self.dir_path))
self.dump_pickle(self.dir_path)
self.dump_yaml(self.dir_path)
logger.info('Dumping results to {}'.format(self.outdir))
self.dump_pickle(self.outdir)
self.dump_yaml(self.outdir)
else:
logger.info('NOT dumping results')
for exporter in exporters:
exporter.start()
for env in self._run_sync_or_async(*args, parallel=parallel,
dry_run=dry_run, **kwargs):
for exporter in exporters:
exporter.env(env)
yield env
for exporter in exporters:
exporter.end()
def get_env(self, trial_id = 0, **kwargs):
opts=self.environment_params.copy()
env_name='{}_trial_{}'.format(self.name, trial_id)
opts = self.environment_params.copy()
env_name = '{}_trial_{}'.format(self.name, trial_id)
opts.update({
'name': env_name,
'topology': self.topology.copy(),
@ -169,13 +198,13 @@ class Simulation(NetworkSimulation):
'states': self.states,
'default_state': self.default_state,
'environment_agents': self.environment_agents,
'dir_path': self.dir_path,
'outdir': self.outdir,
})
opts.update(kwargs)
env=self.environment_class(**opts)
env = self.environment_class(**opts)
return env
def run_trial(self, trial_id = 0, until = None, return_env = True, **opts):
def run_trial(self, trial_id=0, until=None, dry_run=False, **opts):
"""Run a single trial of the simulation
Parameters
@ -183,16 +212,16 @@ class Simulation(NetworkSimulation):
trial_id : int
"""
# Set-up trial environment and graph
until=until or self.max_time
env=self.get_env(trial_id = trial_id, **opts)
until = until or self.max_time
env = self.get_env(trial_id = trial_id, **opts)
dry_run = self.dry_run or dry_run
# Set up agents on nodes
with utils.timer('Simulation {} trial {}'.format(self.name, trial_id)):
env.run(until)
if self.dump and not self.dry_run:
if self.dump and not dry_run:
with utils.timer('Dumping simulation {} trial {}'.format(self.name, trial_id)):
env.dump(formats = self.dump)
if return_env:
return env
return env
def run_trial_exceptions(self, *args, **kwargs):
'''
A wrapper for run_trial that catches exceptions and returns them.
@ -211,22 +240,22 @@ class Simulation(NetworkSimulation):
def to_yaml(self):
return yaml.dump(self.to_dict())
def dump_yaml(self, dir_path = None, file_name = None):
dir_path=dir_path or self.dir_path
if not os.path.exists(dir_path):
os.makedirs(dir_path)
def dump_yaml(self, outdir = None, file_name = None):
outdir = outdir or self.outdir
if not os.path.exists(outdir):
os.makedirs(outdir)
if not file_name:
file_name=os.path.join(dir_path,
file_name=os.path.join(outdir,
'{}.dumped.yml'.format(self.name))
with open(file_name, 'w') as f:
f.write(self.to_yaml())
def dump_pickle(self, dir_path = None, pickle_name = None):
dir_path=dir_path or self.dir_path
if not os.path.exists(dir_path):
os.makedirs(dir_path)
def dump_pickle(self, outdir = None, pickle_name = None):
outdir = outdir or self.outdir
if not os.path.exists(outdir):
os.makedirs(outdir)
if not pickle_name:
pickle_name=os.path.join(dir_path,
pickle_name=os.path.join(outdir,
'{}.simulation.pickle'.format(self.name))
with open(pickle_name, 'wb') as f:
pickle.dump(self, f)
@ -235,14 +264,14 @@ class Simulation(NetworkSimulation):
state={}
for k, v in self.__dict__.items():
if k[0] != '_':
state[k]=v
state['topology']=json_graph.node_link_data(self.topology)
state['network_agents']=agents.serialize_distribution(self.network_agents,
known_modules = [])
state['environment_agents']=agents.serialize_distribution(self.environment_agents,
known_modules = [])
state['environment_class']=utils.serialize(self.environment_class,
known_modules=['soil.environment'])[1] # func, name
state[k] = v
state['topology'] = json_graph.node_link_data(self.topology)
state['network_agents'] = agents.serialize_distribution(self.network_agents,
known_modules = [])
state['environment_agents'] = agents.serialize_distribution(self.environment_agents,
known_modules = [])
state['environment_class'] = serialization.serialize(self.environment_class,
known_modules=['soil.environment'])[1] # func, name
if state['load_module'] is None:
del state['load_module']
return state
@ -250,19 +279,28 @@ class Simulation(NetworkSimulation):
def __setstate__(self, state):
self.__dict__ = state
self.load_module = getattr(self, 'load_module', None)
if self.outdir not in sys.path:
sys.path += [self.outdir, os.getcwd()]
if self.dir_path not in sys.path:
sys.path += [self.dir_path, os.getcwd()]
self.topology = json_graph.node_link_graph(state['topology'])
self.network_agents = agents.calculate_distribution(agents._convert_agent_types(self.network_agents))
self.environment_agents = agents._convert_agent_types(self.environment_agents,
known_modules=[self.load_module])
self.environment_class = utils.deserialize(self.environment_class,
self.environment_class = serialization.deserialize(self.environment_class,
known_modules=[self.load_module, 'soil.environment', ]) # func, name
return state
def from_config(config):
config = list(utils.load_config(config))
def all_from_config(config):
configs = list(serialization.load_config(config))
for config, _ in configs:
sim = Simulation(**config)
yield sim
def from_config(conf_or_path):
config = list(serialization.load_config(conf_or_path))
if len(config) > 1:
raise AttributeError('Provide only one configuration')
config = config[0][0]
@ -270,10 +308,10 @@ def from_config(config):
return sim
def run_from_config(*configs, results_dir='soil_output', dump=None, timestamp=False, **kwargs):
def run_from_config(*configs, outdir=None, dump=None, timestamp=False, **kwargs):
for config_def in configs:
# logger.info("Found {} config(s)".format(len(ls)))
for config, _ in utils.load_config(config_def):
for config, path in serialization.load_config(config_def):
name = config.get('name', 'unnamed')
logger.info("Using config(s): {name}".format(name=name))
@ -282,9 +320,12 @@ def run_from_config(*configs, results_dir='soil_output', dump=None, timestamp=Fa
time.strftime("%Y-%m-%d_%H:%M:%S"))
else:
sim_folder = name
dir_path = os.path.join(results_dir, sim_folder)
if dump is not None:
config['dump'] = dump
sim = Simulation(dir_path=dir_path, **config)
logger.info('Dumping results to {} : {}'.format(sim.dir_path, sim.dump))
dir_path = config.pop('dir_path', os.path.dirname(path))
outdir = config.pop('outdir', outdir)
sim = Simulation(dir_path=dir_path,
outdir=outdir,
**config)
logger.info('Dumping results to {} : {}'.format(sim.outdir, sim.dump))
sim.run_simulation(**kwargs)

View File

@ -1,72 +1,12 @@
import os
import ast
import sys
import yaml
import logging
import importlib
import time
from glob import glob
from random import random
from copy import deepcopy
import networkx as nx
from contextlib import contextmanager
logger = logging.getLogger('soil')
logger.setLevel(logging.INFO)
def load_network(network_params, dir_path=None):
if network_params is None:
return nx.Graph()
path = network_params.get('path', None)
if path:
if dir_path and not os.path.isabs(path):
path = os.path.join(dir_path, path)
extension = os.path.splitext(path)[1][1:]
kwargs = {}
if extension == 'gexf':
kwargs['version'] = '1.2draft'
kwargs['node_type'] = int
try:
method = getattr(nx.readwrite, 'read_' + extension)
except AttributeError:
raise AttributeError('Unknown format')
return method(path, **kwargs)
net_args = network_params.copy()
net_gen = net_args.pop('generator')
if dir_path not in sys.path:
sys.path.append(dir_path)
method = deserializer(net_gen,
known_modules=['networkx.generators',])
return method(**net_args)
def load_file(infile):
with open(infile, 'r') as f:
return list(yaml.load_all(f))
def load_files(*patterns):
for pattern in patterns:
for i in glob(pattern):
for config in load_file(i):
yield config, os.path.abspath(i)
def load_config(config):
if isinstance(config, dict):
yield config, None
else:
yield from load_files(config)
@contextmanager
def timer(name='task', pre="", function=logger.info, to_object=None):
start = time.time()
@ -80,83 +20,3 @@ def timer(name='task', pre="", function=logger.info, to_object=None):
if to_object:
to_object.start = start
to_object.end = end
builtins = importlib.import_module('builtins')
def name(value, known_modules=[]):
'''Return a name that can be imported, to serialize/deserialize an object'''
if value is None:
return 'None'
if not isinstance(value, type): # Get the class name first
value = type(value)
tname = value.__name__
if hasattr(builtins, tname):
return tname
modname = value.__module__
if modname == '__main__':
return tname
if known_modules and modname in known_modules:
return tname
for kmod in known_modules:
if not kmod:
continue
module = importlib.import_module(kmod)
if hasattr(module, tname):
return tname
return '{}.{}'.format(modname, tname)
def serializer(type_):
if type_ != 'str' and hasattr(builtins, type_):
return repr
return lambda x: x
def serialize(v, known_modules=[]):
'''Get a text representation of an object.'''
tname = name(v, known_modules=known_modules)
func = serializer(tname)
return func(v), tname
def deserializer(type_, known_modules=[]):
if type(type_) != str: # Already deserialized
return type_
if type_ == 'str':
return lambda x='': x
if type_ == 'None':
return lambda x=None: None
if hasattr(builtins, type_): # Check if it's a builtin type
cls = getattr(builtins, type_)
return lambda x=None: ast.literal_eval(x) if x is not None else cls()
# Otherwise, see if we can find the module and the class
modules = known_modules or []
options = []
for mod in modules:
if mod:
options.append((mod, type_))
if '.' in type_: # Fully qualified module
module, type_ = type_.rsplit(".", 1)
options.append ((module, type_))
errors = []
for modname, tname in options:
try:
module = importlib.import_module(modname)
cls = getattr(module, tname)
return getattr(cls, 'deserialize', cls)
except (ModuleNotFoundError, AttributeError) as ex:
errors.append((modname, tname, ex))
raise Exception('Could not find type {}. Tried: {}'.format(type_, errors))
def deserialize(type_, value=None, **kwargs):
'''Get an object from a text representation'''
if not isinstance(type_, str):
return type_
des = deserializer(type_, **kwargs)
if value is None:
return des
return des(value)

View File

@ -180,7 +180,7 @@ class SocketHandler(tornado.websocket.WebSocketHandler):
with self.logging(self.simulation_name):
try:
config = dict(**self.config)
config['dir_path'] = os.path.join(self.application.dir_path, config['name'])
config['outdir'] = os.path.join(self.application.outdir, config['name'])
config['dump'] = self.application.dump
self.trials = yield self.nonblocking(config)
@ -232,12 +232,12 @@ class ModularServer(tornado.web.Application):
settings = {'debug': True,
'template_path': ROOT + '/templates'}
def __init__(self, dump=False, dir_path='output', name='SOIL', verbose=True, *args, **kwargs):
def __init__(self, dump=False, outdir='output', name='SOIL', verbose=True, *args, **kwargs):
self.verbose = verbose
self.name = name
self.dump = dump
self.dir_path = dir_path
self.outdir = outdir
# Initializing the application itself:
super().__init__(self.handlers, **self.settings)

View File

@ -2,7 +2,7 @@ from unittest import TestCase
import os
from os.path import join
from soil import utils, simulation
from soil import serialization, simulation
ROOT = os.path.abspath(os.path.dirname(__file__))
EXAMPLES = join(ROOT, '..', 'examples')
@ -17,31 +17,32 @@ class TestExamples(TestCase):
def make_example_test(path, config):
def wrapped(self):
root = os.getcwd()
os.chdir(os.path.dirname(path))
s = simulation.from_config(config)
iterations = s.max_time * s.num_trials
if iterations > 1000:
s.max_time = 100
s.num_trials = 1
if config.get('skip_test', False) and not FORCE_TESTS:
self.skipTest('Example ignored.')
envs = s.run_simulation(dry_run=True)
assert envs
for env in envs:
assert env
try:
n = config['network_params']['n']
assert len(list(env.network_agents)) == n
assert env.now > 2 # It has run
assert env.now <= config['max_time'] # But not further than allowed
except KeyError:
pass
os.chdir(root)
for s in simulation.all_from_config(path):
iterations = s.max_time * s.num_trials
if iterations > 1000:
s.max_time = 100
s.num_trials = 1
if config.get('skip_test', False) and not FORCE_TESTS:
self.skipTest('Example ignored.')
envs = s.run_simulation(dry_run=True)
assert envs
for env in envs:
assert env
try:
n = config['network_params']['n']
assert len(list(env.network_agents)) == n
assert env.now > 2 # It has run
assert env.now <= config['max_time'] # But not further than allowed
except KeyError:
pass
return wrapped
def add_example_tests():
for config, path in utils.load_config(join(EXAMPLES, '**', '*.yml')):
for config, path in serialization.load_files(
join(EXAMPLES, '*', '*.yml'),
join(EXAMPLES, '*.yml'),
):
p = make_example_test(path=path, config=config)
fname = os.path.basename(path)
p.__name__ = 'test_example_file_%s' % fname

View File

@ -7,7 +7,8 @@ import networkx as nx
from functools import partial
from os.path import join
from soil import simulation, Environment, agents, utils, history
from soil import (simulation, Environment, agents, serialization,
history, utils)
ROOT = os.path.abspath(os.path.dirname(__file__))
@ -32,7 +33,7 @@ class TestMain(TestCase):
'path': join(ROOT, 'test.gexf')
}
}
G = utils.load_network(config['network_params'])
G = serialization.load_network(config['network_params'])
assert G
assert len(G) == 2
with self.assertRaises(AttributeError):
@ -42,7 +43,7 @@ class TestMain(TestCase):
'path': join(ROOT, 'unknown.extension')
}
}
G = utils.load_network(config['network_params'])
G = serialization.load_network(config['network_params'])
print(G)
def test_generate_barabasi(self):
@ -57,10 +58,10 @@ class TestMain(TestCase):
}
}
with self.assertRaises(TypeError):
G = utils.load_network(config['network_params'])
G = serialization.load_network(config['network_params'])
config['network_params']['n'] = 100
config['network_params']['m'] = 10
G = utils.load_network(config['network_params'])
G = serialization.load_network(config['network_params'])
assert len(G) == 100
def test_empty_simulation(self):
@ -153,7 +154,7 @@ class TestMain(TestCase):
def test_torvalds_example(self):
"""A complete example from a documentation should work."""
config = utils.load_file(join(EXAMPLES, 'torvalds.yml'))[0]
config = serialization.load_file(join(EXAMPLES, 'torvalds.yml'))[0]
config['network_params']['path'] = join(EXAMPLES,
config['network_params']['path'])
s = simulation.from_config(config)
@ -180,7 +181,7 @@ class TestMain(TestCase):
should be equivalent to the configuration file used
"""
with utils.timer('loading'):
config = utils.load_file(join(EXAMPLES, 'complete.yml'))[0]
config = serialization.load_file(join(EXAMPLES, 'complete.yml'))[0]
s = simulation.from_config(config)
s.dry_run = True
with utils.timer('serializing'):
@ -189,6 +190,7 @@ class TestMain(TestCase):
recovered = yaml.load(serial)
with utils.timer('deleting'):
del recovered['topology']
del recovered['outdir']
assert config == recovered
def test_configuration_changes(self):
@ -196,13 +198,14 @@ class TestMain(TestCase):
The configuration should not change after running
the simulation.
"""
config = utils.load_file(join(EXAMPLES, 'complete.yml'))[0]
config = serialization.load_file(join(EXAMPLES, 'complete.yml'))[0]
s = simulation.from_config(config)
s.dry_run = True
for i in range(5):
s.run_simulation(dry_run=True)
nconfig = s.to_dict()
del nconfig['topology']
del nconfig['outdir']
assert config == nconfig
def test_row_conversion(self):
@ -245,11 +248,11 @@ class TestMain(TestCase):
assert ('finish', 10, None) in values
def test_serialize_class(self):
ser, name = utils.serialize(agents.BaseAgent)
ser, name = serialization.serialize(agents.BaseAgent)
assert name == 'soil.agents.BaseAgent'
assert ser == agents.BaseAgent
ser, name = utils.serialize(CustomAgent)
ser, name = serialization.serialize(CustomAgent)
assert name == 'test_main.CustomAgent'
assert ser == CustomAgent
pickle.dumps(ser)
@ -257,9 +260,9 @@ class TestMain(TestCase):
def test_serialize_builtin_types(self):
for i in [1, None, True, False, {}, [], list(), dict()]:
ser, name = utils.serialize(i)
ser, name = serialization.serialize(i)
assert type(ser) == str
des = utils.deserialize(name, ser)
des = serialization.deserialize(name, ser)
assert i == des
def test_serialize_agent_type(self):
@ -336,3 +339,10 @@ class TestMain(TestCase):
assert len(a3.subgraph(limit_neighbors=True)) == 1
assert len(a3.subgraph(limit_neighbors=True, center=False)) == 0
assert len(a3.subgraph(agent_type=agents.NetworkAgent)) == 3
def test_templates(self):
'''Loading a template should result in several configs'''
configs = serialization.load_file(join(EXAMPLES, 'template.yml'))
assert len(configs) > 0