1
0
mirror of https://github.com/gsi-upm/soil synced 2024-11-21 18:52:28 +00:00
This commit is contained in:
J. Fernando Sánchez 2022-04-04 16:00:54 +02:00
parent 1a8313e4f6
commit 6f7481769e
9 changed files with 467 additions and 362 deletions

View File

@ -36,13 +36,13 @@ def main():
parser.add_argument('--module', '-m', type=str,
help='file containing the code of any custom agents.')
parser.add_argument('--dry-run', '--dry', action='store_true',
help='Do not store the results of the simulation.')
help='Do not store the results of the simulation to disk, show in terminal instead.')
parser.add_argument('--pdb', action='store_true',
help='Use a pdb console in case of exception.')
parser.add_argument('--graph', '-g', action='store_true',
help='Dump GEXF graph. Defaults to false.')
help='Dump each trial\'s network topology as a GEXF graph. Defaults to false.')
parser.add_argument('--csv', action='store_true',
help='Dump history in CSV format. Defaults to false.')
help='Dump all data collected in CSV format. Defaults to false.')
parser.add_argument('--level', type=str,
help='Logging level')
parser.add_argument('--output', '-o', type=str, default="soil_output",

View File

@ -26,7 +26,14 @@ class DeadAgent(Exception):
class BaseAgent(Agent):
"""
A special Agent that keeps track of its state history.
A special type of Mesa Agent that:
* Can be used as a dictionary to access its state.
* Has logging built-in
* Can be given default arguments through a defaults class attribute,
which will be used on construction to initialize each agent's state
Any attribute that is not preceded by an underscore (`_`) will also be added to its state.
"""
defaults = {}
@ -61,6 +68,9 @@ class BaseAgent(Agent):
for (k, v) in kwargs.items():
setattr(self, k, v)
for (k, v) in getattr(self, 'defaults', {}).items():
if not hasattr(self, k) or getattr(self, k) is None:
setattr(self, k, v)
# TODO: refactor to clean up mesa compatibility
@property
@ -79,7 +89,6 @@ class BaseAgent(Agent):
def state(self):
'''
Return the agent itself, which behaves as a dictionary.
Changes made to `agent.state` will be reflected in the history.
This method shouldn't be used, but is kept here for backwards compatibility.
'''

251
soil/config.py Normal file
View File

@ -0,0 +1,251 @@
import yaml
import os
import sys
import networkx as nx
import collections.abc
from . import serialization, utils, basestring, agents
class Config(collections.abc.Mapping):
"""
1) agent type can be specified by name or by class.
2) instead of just one type, a network agents distribution can be used.
The distribution specifies the weight (or probability) of each
agent type in the topology. This is an example distribution: ::
[
{'agent_type': 'agent_type_1',
'weight': 0.2,
'state': {
'id': 0
}
},
{'agent_type': 'agent_type_2',
'weight': 0.8,
'state': {
'id': 1
}
}
]
In this example, 20% of the nodes will be marked as type
'agent_type_1'.
3) if no initial state is given, each node's state will be set
to `{'id': 0}`.
Parameters
---------
name : str, optional
name of the Simulation
group : str, optional
a group name can be used to link simulations
topology (optional): networkx.Graph instance or Node-Link topology as a dict or string (will be loaded with `json_graph.node_link_graph(topology`).
network_params : dict
parameters used to create a topology with networkx, if no topology is given
network_agents : dict
definition of agents to populate the topology with
agent_type : NetworkAgent subclass, optional
Default type of NetworkAgent to use for nodes not specified in network_agents
states : list, optional
List of initial states corresponding to the nodes in the topology. Basic form is a list of integers
whose value indicates the state
dir_path: str, optional
Directory path to load simulation assets (files, modules...)
seed : str, optional
Seed to use for the random generator
num_trials : int, optional
Number of independent simulation runs
max_time : int, optional
Maximum step/time for each simulation
environment_params : dict, optional
Dictionary of globally-shared environmental parameters
environment_agents: dict, optional
Similar to network_agents. Distribution of Agents that control the environment
environment_class: soil.environment.Environment subclass, optional
Class for the environment. It defailts to soil.environment.Environment
"""
__slots__ = 'name', 'agent_type', 'group', 'network_agents', 'environment_agents', 'states', 'default_state', 'interval', 'network_params', 'seed', 'num_trials', 'max_time', 'topology', 'schedule', 'initial_time', 'environment_params', 'environment_class', 'dir_path', '_added_to_path'
def __init__(self, name=None,
group=None,
agent_type='BaseAgent',
network_agents=None,
environment_agents=None,
states=None,
default_state=None,
interval=1,
network_params=None,
seed=None,
num_trials=1,
max_time=None,
topology=None,
schedule=None,
initial_time=0,
environment_params={},
environment_class='soil.Environment',
dir_path=None):
self.network_params = network_params
self.name = name or 'Unnamed'
self.seed = str(seed or name)
self.group = group or ''
self.num_trials = num_trials
self.max_time = max_time
self.default_state = default_state or {}
self.dir_path = dir_path or os.getcwd()
self.interval = interval
self._added_to_path = list(x for x in [os.getcwd(), self.dir_path] if x not in sys.path)
sys.path += self._added_to_path
self.topology = topology
self.schedule = schedule
self.initial_time = initial_time
self.environment_class = environment_class
self.environment_params = dict(environment_params)
#TODO: Check agent distro vs fixed agents
self.environment_agents = environment_agents or []
self.agent_type = agent_type
self.network_agents = network_agents or {}
self.states = states or {}
def validate(self):
agents._validate_states(self.states,
self._topology)
def restore_path(self):
for added in self._added_to_path:
sys.path.remove(added)
def to_yaml(self):
return yaml.dump(self.to_dict())
def dump_yaml(self, f=None, outdir=None):
if not f and not outdir:
raise ValueError('specify a file or an output directory')
if not f:
f = os.path.join(outdir, '{}.dumped.yml'.format(self.name))
with utils.open_or_reuse(f, 'w') as f:
f.write(self.to_yaml())
def to_yaml(self):
return yaml.dump(self.to_dict())
# TODO: See note on getstate
def to_dict(self):
return self.__getstate__()
def dump_yaml(self, f=None, outdir=None):
if not f and not outdir:
raise ValueError('specify a file or an output directory')
if not f:
f = os.path.join(outdir, '{}.dumped.yml'.format(self.name))
with utils.open_or_reuse(f, 'w') as f:
f.write(self.to_yaml())
def __getitem__(self, key):
return getattr(self, key)
def __iter__(self):
return (k for k in self.__slots__ if k[0] != '_')
def __len__(self):
return len(self.__slots__)
def dump_pickle(self, f=None, outdir=None):
if not outdir and not f:
raise ValueError('specify a file or an output directory')
if not f:
f = os.path.join(outdir,
'{}.simulation.pickle'.format(self.name))
with utils.open_or_reuse(f, 'wb') as f:
pickle.dump(self, f)
# TODO: remove this. A config should be sendable regardless. Non-pickable objects could be computed via properties and the like
# def __getstate__(self):
# state={}
# for k, v in self.__dict__.items():
# if k[0] != '_':
# state[k] = v
# state['topology'] = json_graph.node_link_data(self.topology)
# state['network_agents'] = agents.serialize_definition(self.network_agents,
# known_modules = [])
# state['environment_agents'] = agents.serialize_definition(self.environment_agents,
# known_modules = [])
# state['environment_class'] = serialization.serialize(self.environment_class,
# known_modules=['soil.environment'])[1] # func, name
# if state['load_module'] is None:
# del state['load_module']
# return state
# # TODO: remove, same as __getstate__
# def __setstate__(self, state):
# self.__dict__ = state
# self.load_module = getattr(self, 'load_module', None)
# if self.dir_path not in sys.path:
# sys.path += [self.dir_path, os.getcwd()]
# self.topology = json_graph.node_link_graph(state['topology'])
# self.network_agents = agents.calculate_distribution(agents._convert_agent_types(self.network_agents))
# self.environment_agents = agents._convert_agent_types(self.environment_agents,
# known_modules=[self.load_module])
# self.environment_class = serialization.deserialize(self.environment_class,
# known_modules=[self.load_module,
# 'soil.environment', ]) # func, name
class CalculatedConfig(Config):
def __init__(self, config):
"""
Returns a configuration object that replaces some "plain" attributes (e.g., `environment_class` string) into
a Python object (`soil.environment.Environment` class).
"""
self._config = config
values = dict(config)
values['environment_class'] = self._environment_class()
values['environment_agents'] = self._environment_agents()
values['topology'] = self._topology()
values['network_agents'] = self._network_agents()
values['agent_type'] = serialization.deserialize(self.agent_type, known_modules=['soil.agents'])
return values
def _topology(self):
topology = self._config.topology
if topology is None:
topology = serialization.load_network(self._config.network_params,
dir_path=self._config.dir_path)
elif isinstance(topology, basestring) or isinstance(topology, dict):
topology = json_graph.node_link_graph(topology)
return nx.Graph(topology)
def _environment_class(self):
return serialization.deserialize(self._config.environment_class,
known_modules=['soil.environment', ]) or Environment
def _environment_agents(self):
return agents._convert_agent_types(self._config.environment_agents)
def _network_agents(self):
distro = agents.calculate_distribution(self._config.network_agents,
self._config.agent_type)
return agents._convert_agent_types(distro)
def _environment_class(self):
return serialization.deserialize(self._config.environment_class,
known_modules=['soil.environment', ]) # func, name

View File

@ -8,19 +8,17 @@ class SoilDataCollector(MDC):
# Populate model and env reporters so they have a key per
# So they can be shown in the web interface
self.environment = environment
raise NotImplementedError()
@property
def model_vars(self):
pass
raise NotImplementedError()
@model_vars.setter
def model_vars(self, value):
pass
raise NotImplementedError()
@property
def agent_reporters(self):
self.model._history._
pass
raise NotImplementedError()

View File

@ -1,23 +1,20 @@
from __future__ import annotations
import os
import sqlite3
import csv
import math
import random
import yaml
import tempfile
import logging
import pandas as pd
from time import time as current_time
from copy import deepcopy
from networkx.readwrite import json_graph
import networkx as nx
from tsih import History, NoHistory, Record, Key
import networkx as nx
from mesa import Model
from . import serialization, agents, analysis, utils, time
from tsih import Record
from . import serialization, agents, analysis, utils, time, config
# These properties will be copied when pickling/unpickling the environment
_CONFIG_PROPS = [ 'name',
@ -49,7 +46,6 @@ class Environment(Model):
schedule=None,
initial_time=0,
environment_params=None,
history=False,
dir_path=None,
**kwargs):
@ -76,20 +72,11 @@ class Environment(Model):
topology = nx.Graph()
self.G = nx.Graph(topology)
self.environment_params = environment_params or {}
self.environment_params.update(kwargs)
self._env_agents = {}
self.interval = interval
if history:
history = History
else:
history = NoHistory
self._history = history(name=self.name,
backup=True)
self['SEED'] = seed
if network_agents:
@ -106,6 +93,19 @@ class Environment(Model):
self.logger = utils.logger.getChild(self.name)
@staticmethod
def from_config(conf: config.Config, trial_id, **kwargs) -> Environment:
'''Create an environment for a trial of the simulation'''
conf = config.Config(conf, **kwargs)
conf.seed = '{}_{}'.format(conf.seed, trial_id)
conf.name = '{}_trial_{}'.format(conf.name, trial_id).replace('.', '-')
opts = conf.environment_params.copy()
opts.update(conf)
opts.update(kwargs)
env = serialization.deserialize(conf.environment_class)(**opts)
return env
@property
def now(self):
if self.schedule:
@ -212,11 +212,14 @@ class Environment(Model):
return self.logger.log(level, message, extra=extra)
def step(self):
'''
Advance one step in the simulation, and update the data collection and scheduler appropriately
'''
super().step()
self.schedule.step()
def run(self, until, *args, **kwargs):
self._save_state()
until = until or float('inf')
while self.schedule.next_time < until:
self.step()
@ -252,14 +255,16 @@ class Environment(Model):
def get(self, key, default=None):
'''
Get the value of an environment attribute in a
given point in the simulation (history).
If key is an attribute name, this method returns
the current value.
To get values at other times, use a
:meth: `soil.history.Key` tuple.
Get the value of an environment attribute.
Return `default` if the value is not set.
'''
return self[key] if key in self else default
return self.environment_params.get(key, default)
def __getitem__(self, key):
return self.environment_params.get(key)
def __setitem__(self, key, value):
return self.environment_params.__setitem__(key, value)
def get_agent(self, agent_id):
return self.G.nodes[agent_id]['agent']
@ -269,112 +274,31 @@ class Environment(Model):
return self.agents
return (self.G.nodes[i]['agent'] for i in nodes)
def dump_csv(self, f):
with utils.open_or_reuse(f, 'w') as f:
cr = csv.writer(f)
cr.writerow(('agent_id', 't_step', 'key', 'value'))
for i in self.history_to_tuples():
cr.writerow(i)
def dump_gexf(self, f):
G = self.history_to_graph()
# Workaround for geometric models
# See soil/soil#4
for node in G.nodes():
if 'pos' in G.nodes[node]:
G.nodes[node]['viz'] = {"position": {"x": G.nodes[node]['pos'][0], "y": G.nodes[node]['pos'][1], "z": 0.0}}
del (G.nodes[node]['pos'])
nx.write_gexf(G, f, version="1.2draft")
def dump(self, *args, formats=None, **kwargs):
if not formats:
return
functions = {
'csv': self.dump_csv,
'gexf': self.dump_gexf
}
for f in formats:
if f in functions:
functions[f](*args, **kwargs)
else:
raise ValueError('Unknown format: {}'.format(f))
def df(self):
return self._history[None, None, None].df()
def dump_sqlite(self, f):
return self._history.dump(f)
def state_to_tuples(self, now=None):
def _agent_to_tuples(self, agent, now=None):
if now is None:
now = self.now
for k, v in agent.state.items():
yield Record(dict_id=agent.id,
t_step=now,
key=k,
value=v)
def state_to_tuples(self, agent_id=None, now=None):
if now is None:
now = self.now
if agent_id:
agent = self.get_agent(agent_id)
yield from self._agent_to_tuples(agent, now)
return
for k, v in self.environment_params.items():
yield Record(dict_id='env',
t_step=now,
key=k,
value=v)
for agent in self.agents:
for k, v in agent.state.items():
yield Record(dict_id=agent.id,
t_step=now,
key=k,
value=v)
def history_to_tuples(self, agent_id=None):
if isinstance(self._history, NoHistory):
tuples = self.state_to_tuples()
else:
tuples = self._history.to_tuples()
if agent_id is None:
return tuples
return filter(lambda x: str(x[0]) == str(agent_id), tuples)
def history_to_graph(self):
G = nx.Graph(self.G)
for agent in self.network_agents:
attributes = {'agent': str(agent.__class__)}
lastattributes = {}
spells = []
lastvisible = False
laststep = None
history = sorted(list(self.history_to_tuples(agent_id=agent.id)))
if not history:
continue
for _, t_step, attribute, value in history:
if attribute == 'visible':
nowvisible = value
if nowvisible and not lastvisible:
laststep = t_step
if not nowvisible and lastvisible:
spells.append((laststep, t_step))
lastvisible = nowvisible
continue
key = 'attr_' + attribute
if key not in attributes:
attributes[key] = list()
if key not in lastattributes:
lastattributes[key] = (value, t_step)
elif lastattributes[key][0] != value:
last_value, laststep = lastattributes[key]
commit_value = (last_value, laststep, t_step)
if key not in attributes:
attributes[key] = list()
attributes[key].append(commit_value)
lastattributes[key] = (value, t_step)
for k, v in lastattributes.items():
attributes[k].append((v[0], v[1], None))
if lastvisible:
spells.append((laststep, None))
if spells:
G.add_node(agent.id, spells=spells, **attributes)
else:
G.add_node(agent.id, **attributes)
return G
yield from self._agent_to_tuples(agent, now)
def __getstate__(self):
state = {}
@ -382,7 +306,6 @@ class Environment(Model):
state[prop] = self.__dict__[prop]
state['G'] = json_graph.node_link_data(self.G)
state['environment_agents'] = self._env_agents
state['history'] = self._history
state['schedule'] = self.schedule
return state
@ -391,7 +314,6 @@ class Environment(Model):
self.__dict__[prop] = state[prop]
self._env_agents = state['environment_agents']
self.G = json_graph.node_link_graph(state['G'])
self._history = state['history']
# self._env = None
self.schedule = state['schedule']
self._queue = []

View File

@ -48,20 +48,24 @@ class Exporter:
self.simulation = simulation
outdir = outdir or os.path.join(os.getcwd(), 'soil_output')
self.outdir = os.path.join(outdir,
simulation.group or '',
simulation.name)
simulation.config.group or '',
simulation.config.name)
self.dry_run = dry_run
self.copy_to = copy_to
def start(self):
def sim_start(self):
'''Method to call when the simulation starts'''
pass
def end(self, stats):
def sim_end(self, stats):
'''Method to call when the simulation ends'''
pass
def trial(self, env, stats):
def trial_start(self, env):
'''Method to call when a trial start'''
pass
def trial_end(self, env, stats):
'''Method to call when a trial ends'''
pass
@ -80,21 +84,21 @@ class Exporter:
class default(Exporter):
'''Default exporter. Writes sqlite results, as well as the simulation YAML'''
def start(self):
def sim_start(self):
if not self.dry_run:
logger.info('Dumping results to %s', self.outdir)
self.simulation.dump_yaml(outdir=self.outdir)
else:
logger.info('NOT dumping results')
def trial(self, env, stats):
def trial_start(self, env, stats):
if not self.dry_run:
with timer('Dumping simulation {} trial {}'.format(self.simulation.name,
env.name)):
with self.output('{}.sqlite'.format(env.name), mode='wb') as f:
env.dump_sqlite(f)
def end(self, stats):
def sim_end(self, stats):
with timer('Dumping simulation {}\'s stats'.format(self.simulation.name)):
with self.output('{}.sqlite'.format(self.simulation.name), mode='wb') as f:
self.simulation.dump_sqlite(f)
@ -102,15 +106,14 @@ class default(Exporter):
class csv(Exporter):
'''Export the state of each environment (and its agents) in a separate CSV file'''
def trial(self, env, stats):
def trial_end(self, env, stats):
with timer('[CSV] Dumping simulation {} trial {} @ dir {}'.format(self.simulation.name,
env.name,
self.outdir)):
with self.output('{}.csv'.format(env.name)) as f:
env.dump_csv(f)
with self.output('{}.stats.csv'.format(env.name)) as f:
with self.output('{}.stats.{}.csv'.format(env.name, stats.name)) as f:
statwriter = csvlib.writer(f, delimiter='\t', quotechar='"', quoting=csvlib.QUOTE_ALL)
for stat in stats:
@ -118,7 +121,7 @@ class csv(Exporter):
class gexf(Exporter):
def trial(self, env, stats):
def trial_end(self, env, stats):
if self.dry_run:
logger.info('Not dumping GEXF in dry_run mode')
return
@ -126,22 +129,32 @@ class gexf(Exporter):
with timer('[GEXF] Dumping simulation {} trial {}'.format(self.simulation.name,
env.name)):
with self.output('{}.gexf'.format(env.name), mode='wb') as f:
env.dump_gexf(f)
self.dump_gexf(env, f)
def dump_gexf(self, env, f):
G = env.history_to_graph()
# Workaround for geometric models
# See soil/soil#4
for node in G.nodes():
if 'pos' in G.nodes[node]:
G.nodes[node]['viz'] = {"position": {"x": G.nodes[node]['pos'][0], "y": G.nodes[node]['pos'][1], "z": 0.0}}
del (G.nodes[node]['pos'])
nx.write_gexf(G, f, version="1.2draft")
class dummy(Exporter):
def start(self):
def sim_start(self):
with self.output('dummy', 'w') as f:
f.write('simulation started @ {}\n'.format(current_time()))
def trial(self, env, stats):
def trial_end(self, env, stats):
with self.output('dummy', 'w') as f:
for i in env.history_to_tuples():
for i in stats:
f.write(','.join(map(str, i)))
f.write('\n')
def sim(self, stats):
def sim_end(self, stats):
with self.output('dummy', 'a') as f:
f.write('simulation ended @ {}\n'.format(current_time()))
@ -149,10 +162,57 @@ class dummy(Exporter):
class graphdrawing(Exporter):
def trial(self, env, stats):
def trial_end(self, env, stats):
# Outside effects
f = plt.figure()
nx.draw(env.G, node_size=10, width=0.2, pos=nx.spring_layout(env.G, scale=100), ax=f.add_subplot(111))
with open('graph-{}.png'.format(env.name)) as f:
f.savefig(f)
'''
Convert an environment into a NetworkX graph
'''
def env_to_graph(env, history=None):
G = nx.Graph(env.G)
for agent in env.network_agents:
attributes = {'agent': str(agent.__class__)}
lastattributes = {}
spells = []
lastvisible = False
laststep = None
if not history:
history = sorted(list(env.state_to_tuples()))
for _, t_step, attribute, value in history:
if attribute == 'visible':
nowvisible = value
if nowvisible and not lastvisible:
laststep = t_step
if not nowvisible and lastvisible:
spells.append((laststep, t_step))
lastvisible = nowvisible
continue
key = 'attr_' + attribute
if key not in attributes:
attributes[key] = list()
if key not in lastattributes:
lastattributes[key] = (value, t_step)
elif lastattributes[key][0] != value:
last_value, laststep = lastattributes[key]
commit_value = (last_value, laststep, t_step)
if key not in attributes:
attributes[key] = list()
attributes[key].append(commit_value)
lastattributes[key] = (value, t_step)
for k, v in lastattributes.items():
attributes[k].append((v[0], v[1], None))
if lastvisible:
spells.append((laststep, None))
if spells:
G.add_node(agent.id, spells=spells, **attributes)
else:
G.add_node(agent.id, **attributes)
return G

View File

@ -2,6 +2,7 @@ import os
import logging
import ast
import sys
import re
import importlib
from glob import glob
from itertools import product, chain
@ -18,6 +19,9 @@ logger = logging.getLogger('soil')
def load_network(network_params, dir_path=None):
G = nx.Graph()
if not network_params:
return G
if 'path' in network_params:
path = network_params['path']
if dir_path and not os.path.isabs(path):
@ -169,6 +173,9 @@ def serialize(v, known_modules=[]):
func = serializer(tname)
return func(v), tname
IS_CLASS = re.compile(r"<class '(.*)'>")
def deserializer(type_, known_modules=[]):
if type(type_) != str: # Already deserialized
return type_
@ -179,6 +186,13 @@ def deserializer(type_, known_modules=[]):
if hasattr(builtins, type_): # Check if it's a builtin type
cls = getattr(builtins, type_)
return lambda x=None: ast.literal_eval(x) if x is not None else cls()
match = IS_CLASS.match(type_)
if match:
modname, tname = match.group(1).rsplit(".", 1)
module = importlib.import_module(modname)
cls = getattr(module, tname)
return getattr(cls, 'deserialize', cls)
# Otherwise, see if we can find the module and the class
modules = known_modules or []
options = []
@ -189,7 +203,7 @@ def deserializer(type_, known_modules=[]):
if '.' in type_: # Fully qualified module
module, type_ = type_.rsplit(".", 1)
options.append ((module, type_))
options.append((module, type_))
errors = []
for modname, tname in options:
@ -213,10 +227,10 @@ def deserialize(type_, value=None, **kwargs):
def deserialize_all(names, *args, known_modules=['soil'], **kwargs):
'''Return the set of exporters for a simulation, given the exporter names'''
exporters = []
'''Return the list of deserialized objects'''
objects = []
for name in names:
mod = deserialize(name, known_modules=known_modules)
exporters.append(mod(*args, **kwargs))
return exporters
objects.append(mod(*args, **kwargs))
return objects

View File

@ -10,8 +10,6 @@ import networkx as nx
from networkx.readwrite import json_graph
from multiprocessing import Pool
from functools import partial
from tsih import History
import pickle
from . import serialization, utils, basestring, agents
@ -20,127 +18,34 @@ from .utils import logger
from .exporters import default
from .stats import defaultStats
from .config import Config
#TODO: change documentation for simulation
class Simulation:
"""
Similar to nsim.NetworkSimulation with three main differences:
1) agent type can be specified by name or by class.
2) instead of just one type, a network agents distribution can be used.
The distribution specifies the weight (or probability) of each
agent type in the topology. This is an example distribution: ::
[
{'agent_type': 'agent_type_1',
'weight': 0.2,
'state': {
'id': 0
}
},
{'agent_type': 'agent_type_2',
'weight': 0.8,
'state': {
'id': 1
}
}
]
In this example, 20% of the nodes will be marked as type
'agent_type_1'.
3) if no initial state is given, each node's state will be set
to `{'id': 0}`.
Parameters
---------
name : str, optional
config (optional): :class:`config.Config`
name of the Simulation
group : str, optional
a group name can be used to link simulations
topology : networkx.Graph instance, optional
network_params : dict
parameters used to create a topology with networkx, if no topology is given
network_agents : dict
definition of agents to populate the topology with
agent_type : NetworkAgent subclass, optional
Default type of NetworkAgent to use for nodes not specified in network_agents
states : list, optional
List of initial states corresponding to the nodes in the topology. Basic form is a list of integers
whose value indicates the state
dir_path: str, optional
Directory path to load simulation assets (files, modules...)
seed : str, optional
Seed to use for the random generator
num_trials : int, optional
Number of independent simulation runs
max_time : int, optional
Time how long the simulation should run
environment_params : dict, optional
Dictionary of globally-shared environmental parameters
environment_agents: dict, optional
Similar to network_agents. Distribution of Agents that control the environment
environment_class: soil.environment.Environment subclass, optional
Class for the environment. It defailts to soil.environment.Environment
load_module : str, module name, deprecated
If specified, soil will load the content of this module under 'soil.agents.custom'
history: tsih.History subclass, optional
Class to use to store the history of the simulation (and environments). It defailts to tsih.History
If set to True, tsih.History will be used. If set to False or None, tsih.NoHistory will be used.
kwargs: parameters to use to initialize a new configuration, if one has not been provided.
"""
def __init__(self, name=None, group=None, topology=None, network_params=None,
network_agents=None, agent_type=None, states=None,
default_state=None, interval=1, num_trials=1,
max_time=100, load_module=None, seed=None,
dir_path=None, environment_agents=None,
environment_params=None, environment_class=None,
history=History, **kwargs):
def __init__(self, config=None,
**kwargs):
self.load_module = load_module
self.network_params = network_params
self.name = name or 'Unnamed'
self.seed = str(seed or name)
self._id = '{}_{}'.format(self.name, strftime("%Y-%m-%d_%H.%M.%S"))
self.group = group or ''
self.num_trials = num_trials
self.max_time = max_time
self.default_state = default_state or {}
self.dir_path = dir_path or os.getcwd()
self.interval = interval
if bool(config) == bool(kwargs):
raise ValueError("Specify either a configuration or the parameters to initialize a configuration")
sys.path += list(x for x in [os.getcwd(), self.dir_path] if x not in sys.path)
if kwargs:
config = Config(**kwargs)
if topology is None:
topology = serialization.load_network(network_params,
dir_path=self.dir_path)
elif isinstance(topology, basestring) or isinstance(topology, dict):
topology = json_graph.node_link_graph(topology)
self.topology = nx.Graph(topology)
self.config = config
self.environment_params = environment_params or {}
self.environment_class = serialization.deserialize(environment_class,
known_modules=['soil.environment', ]) or Environment
environment_agents = environment_agents or []
self.environment_agents = agents._convert_agent_types(environment_agents,
known_modules=[self.load_module])
distro = agents.calculate_distribution(network_agents,
agent_type)
self.network_agents = agents._convert_agent_types(distro,
known_modules=[self.load_module])
self.states = agents._validate_states(states,
self.topology)
if history == True:
history = History
elif not history:
history = NoHistory
self._history = history(name=self.name,
backup=False)
@property
def name(self) -> str:
return self.config.name
def run_simulation(self, *args, **kwargs):
return self.run(*args, **kwargs)
@ -153,13 +58,13 @@ class Simulation:
if parallel and not os.environ.get('SENPY_DEBUG', None):
p = Pool()
func = partial(self.run_trial_exceptions, **kwargs)
for i in p.imap_unordered(func, range(self.num_trials)):
for i in p.imap_unordered(func, range(self.config.num_trials)):
if isinstance(i, Exception):
logger.error('Trial failed:\n\t%s', i.message)
continue
yield i
else:
for i in range(self.num_trials):
for i in range(self.config.num_trials):
yield self.run_trial(trial_id=i,
**kwargs)
@ -179,50 +84,47 @@ class Simulation:
outdir=outdir,
**exporter_params)
stats = serialization.deserialize_all(simulation=self,
names=stats,
known_modules=['soil.stats',],
**stats_params)
names=stats,
known_modules=['soil.stats',],
**stats_params)
with utils.timer('simulation {}'.format(self.name)):
with utils.timer('simulation {}'.format(self.config.name)):
for stat in stats:
stat.start()
stat.sim_start()
for exporter in exporters:
exporter.start()
for env in self._run_sync_or_async(parallel=parallel,
log_level=log_level,
**kwargs):
collected = list(stat.trial(env) for stat in stats)
for exporter in exporters:
exporter.trial_start(env)
saved = self.save_stats(collected, t_step=env.now, trial_id=env.name)
collected = list(stat.trial_end(env) for stat in stats)
saved = self._update_stats(collected, t_step=env.now, trial_id=env.name)
for exporter in exporters:
exporter.trial(env, saved)
exporter.trial_end(env, saved)
yield env
collected = list(stat.end() for stat in stats)
saved = self.save_stats(collected)
saved = self._update_stats(collected)
for exporter in exporters:
exporter.end(saved)
exporter.sim_end(saved)
def save_stats(self, collection, **kwargs):
def _update_stats(self, collection, **kwargs):
stats = dict(kwargs)
for stat in collection:
stats.update(stat)
self._history.save_stats(utils.flatten_dict(stats))
return stats
def get_stats(self, **kwargs):
return self._history.get_stats(**kwargs)
def log_stats(self, stats):
logger.info('Stats: \n{}'.format(yaml.dump(stats, default_flow_style=False)))
def get_env(self, trial_id=0, **kwargs):
'''Create an environment for a trial of the simulation'''
@ -246,18 +148,20 @@ class Simulation:
env = self.environment_class(**opts)
return env
def run_trial(self, trial_id=0, until=None, log_level=logging.INFO, **opts):
def run_trial(self, trial_id=None, until=None, log_level=logging.INFO, **opts):
"""
Run a single trial of the simulation
"""
trial_id = trial_id if trial_id is not None else current_time()
if log_level:
logger.setLevel(log_level)
# Set-up trial environment and graph
until = until or self.max_time
env = self.get_env(trial_id=trial_id, **opts)
until = until or self.config.max_time
env = Environment.from_config(self.config, trial_id=trial_id)
# Set up agents on nodes
with utils.timer('Simulation {} trial {}'.format(self.name, trial_id)):
with utils.timer('Simulation {} trial {}'.format(self.config.name, trial_id)):
env.run(until)
return env
@ -274,64 +178,6 @@ class Simulation:
ex.message = ''.join(traceback.format_exception(type(ex), ex, ex.__traceback__)[:])
return ex
def to_dict(self):
return self.__getstate__()
def to_yaml(self):
return yaml.dump(self.to_dict())
def dump_yaml(self, f=None, outdir=None):
if not f and not outdir:
raise ValueError('specify a file or an output directory')
if not f:
f = os.path.join(outdir, '{}.dumped.yml'.format(self.name))
with utils.open_or_reuse(f, 'w') as f:
f.write(self.to_yaml())
def dump_pickle(self, f=None, outdir=None):
if not outdir and not f:
raise ValueError('specify a file or an output directory')
if not f:
f = os.path.join(outdir,
'{}.simulation.pickle'.format(self.name))
with utils.open_or_reuse(f, 'wb') as f:
pickle.dump(self, f)
def dump_sqlite(self, f):
return self._history.dump(f)
def __getstate__(self):
state={}
for k, v in self.__dict__.items():
if k[0] != '_':
state[k] = v
state['topology'] = json_graph.node_link_data(self.topology)
state['network_agents'] = agents.serialize_definition(self.network_agents,
known_modules = [])
state['environment_agents'] = agents.serialize_definition(self.environment_agents,
known_modules = [])
state['environment_class'] = serialization.serialize(self.environment_class,
known_modules=['soil.environment'])[1] # func, name
if state['load_module'] is None:
del state['load_module']
return state
def __setstate__(self, state):
self.__dict__ = state
self.load_module = getattr(self, 'load_module', None)
if self.dir_path not in sys.path:
sys.path += [self.dir_path, os.getcwd()]
self.topology = json_graph.node_link_graph(state['topology'])
self.network_agents = agents.calculate_distribution(agents._convert_agent_types(self.network_agents))
self.environment_agents = agents._convert_agent_types(self.environment_agents,
known_modules=[self.load_module])
self.environment_class = serialization.deserialize(self.environment_class,
known_modules=[self.load_module, 'soil.environment', ]) # func, name
def all_from_config(config):
configs = list(serialization.load_config(config))

View File

@ -8,18 +8,23 @@ class Stats:
if you don't plan to implement all the methods.
'''
def __init__(self, simulation):
def __init__(self, simulation, name=None):
self.name = name or type(self).__name__
self.simulation = simulation
def start(self):
def sim_start(self):
'''Method to call when the simulation starts'''
pass
def end(self):
def sim_end(self):
'''Method to call when the simulation ends'''
return {}
def trial(self, env):
def trial_start(self, env):
'''Method to call when a trial starts'''
return {}
def trial_end(self, env):
'''Method to call when a trial ends'''
return {}
@ -30,12 +35,12 @@ class distribution(Stats):
the mean value, and its deviation.
'''
def start(self):
def sim_start(self):
self.means = []
self.counts = []
def trial(self, env):
df = env.df()
def trial_end(self, env):
df = pd.DataFrame(env.state_to_tuples())
df = df.drop('SEED', axis=1)
ix = df.index[-1]
attrs = df.columns.get_level_values(0)
@ -60,7 +65,7 @@ class distribution(Stats):
return stats
def end(self):
def sim_end(self):
dfm = pd.DataFrame(self.means, columns=['metric', 'key', 'value'])
dfc = pd.DataFrame(self.counts, columns=['metric', 'key', 'value', 'count'])
@ -87,7 +92,7 @@ class distribution(Stats):
class defaultStats(Stats):
def trial(self, env):
def trial_end(self, env):
c = Counter()
c.update(a.__class__.__name__ for a in env.network_agents)