WIP: exporters

soil/__init__.py

@@ -15,7 +15,7 @@ from . import agents
 from .simulation import *
 from .environment import Environment
 from .history import History
-from . import utils
+from . import serialization
 from . import analysis
 
 def main():
@@ -44,6 +44,8 @@ def main():
                         help='folder to write results to. It defaults to the current directory.')
     parser.add_argument('--synchronous', action='store_true',
                         help='Run trials serially and synchronously instead of in parallel. Defaults to false.')
+    parser.add_argument('-e', '--exporter', action='append',
+                        help='Export environment and/or simulations using this exporter')
 
     args = parser.parse_args()
 
@@ -64,8 +66,9 @@ def main():
         simulation.run_from_config(args.file,
                                    dry_run=args.dry_run,
                                    dump=dump,
+                                   exporters=args.exporter,
                                    parallel=(not args.synchronous),
-                                   results_dir=args.output)
+                                   outdir=args.output)
     except Exception:
         if args.pdb:
             pdb.post_mortem()

soil/agents/__init__.py

@@ -15,7 +15,7 @@ import json
 
 from functools import wraps
 
-from .. import utils, history
+from .. import serialization, history
 
 
 def as_node(agent):
@@ -388,7 +388,7 @@ def serialize_type(agent_type, known_modules=[], **kwargs):
     if isinstance(agent_type, str):
         return agent_type
     known_modules += ['soil.agents']
-    return utils.serialize(agent_type, known_modules=known_modules, **kwargs)[1] # Get the name of the class
+    return serialization.serialize(agent_type, known_modules=known_modules, **kwargs)[1] # Get the name of the class
 
 
 def serialize_distribution(network_agents, known_modules=[]):
@@ -409,7 +409,7 @@ def deserialize_type(agent_type, known_modules=[]):
     if not isinstance(agent_type, str):
         return agent_type
     known = known_modules + ['soil.agents', 'soil.agents.custom' ]
-    agent_type = utils.deserializer(agent_type, known_modules=known)
+    agent_type = serialization.deserializer(agent_type, known_modules=known)
     return agent_type
 
 

soil/analysis.py

@@ -4,7 +4,7 @@ import glob
 import yaml
 from os.path import join
 
-from . import utils, history
+from . import serialization, history
 
 
 def read_data(*args, group=False, **kwargs):
@@ -56,7 +56,7 @@ def read_csv(filename, keys=None, convert_types=False, **kwargs):
 
 
 def convert_row(row):
-    row['value'] = utils.deserialize(row['value_type'], row['value'])
+    row['value'] = serialization.deserialize(row['value_type'], row['value'])
     return row
 
 

soil/environment.py

@@ -14,7 +14,7 @@ from networkx.readwrite import json_graph
 import networkx as nx
 import nxsim
 
-from . import utils, agents, analysis, history
+from . import serialization, agents, analysis, history
 
 # These properties will be copied when pickling/unpickling the environment
 _CONFIG_PROPS = [ 'name',
@@ -22,7 +22,7 @@ _CONFIG_PROPS = [ 'name',
                  'default_state',
                  'interval',
                  'dry_run',
-                 'dir_path',
+                 'outdir',
                  ]
 
 class Environment(nxsim.NetworkEnvironment):
@@ -44,7 +44,7 @@ class Environment(nxsim.NetworkEnvironment):
                  interval=1,
                  seed=None,
                  dry_run=False,
-                 dir_path=None,
+                 outdir=None,
                  topology=None,
                  *args, **kwargs):
         self.name = name or 'UnnamedEnvironment'
@@ -58,11 +58,11 @@ class Environment(nxsim.NetworkEnvironment):
         self._env_agents = {}
         self.dry_run = dry_run
         self.interval = interval
-        self.dir_path = dir_path or tempfile.mkdtemp('soil-env')
+        self.outdir = outdir or tempfile.mkdtemp('soil-env')
         if not dry_run:
             self.get_path()
         self._history = history.History(name=self.name if not dry_run else None,
-                                        dir_path=self.dir_path,
+                                        outdir=self.outdir,
                                         backup=True)
         # Add environment agents first, so their events get
         # executed before network agents
@@ -124,7 +124,7 @@ class Environment(nxsim.NetworkEnvironment):
         elif agent_distribution:
             agent_type, state = agents._agent_from_distribution(agent_distribution, agent_id=agent_id)
         else:
-            utils.logger.debug('Skipping node {}'.format(agent_id))
+            serialization.logger.debug('Skipping node {}'.format(agent_id))
             return
         return self.set_agent(agent_id, agent_type, state)
 
@@ -169,7 +169,7 @@ class Environment(nxsim.NetworkEnvironment):
     def _save_state(self, now=None):
         # for agent in self.agents:
         #     agent.save_state()
-        utils.logger.debug('Saving state @{}'.format(self.now))
+        serialization.logger.debug('Saving state @{}'.format(self.now))
         self._history.save_records(self.state_to_tuples(now=now))
 
     def save_state(self):
@@ -180,7 +180,7 @@ class Environment(nxsim.NetworkEnvironment):
         self._save_state()
         while self.peek() != simpy.core.Infinity:
             delay = max(self.peek() - self.now, self.interval)
-            utils.logger.debug('Step: {}'.format(self.now))
+            serialization.logger.debug('Step: {}'.format(self.now))
             ev = self.event()
             ev._ok = True
             # Schedule the event with minimum priority so
@@ -222,14 +222,14 @@ class Environment(nxsim.NetworkEnvironment):
         '''
         return self[key] if key in self else default
 
-    def get_path(self, dir_path=None):
-        dir_path = dir_path or self.dir_path
-        if not os.path.exists(dir_path):
+    def get_path(self, outdir=None):
+        outdir = outdir or self.outdir
+        if not os.path.exists(outdir):
             try:
-                os.makedirs(dir_path)
+                os.makedirs(outdir)
             except FileExistsError:
                 pass
-        return dir_path
+        return outdir
 
     def get_agent(self, agent_id):
         return self.G.node[agent_id]['agent']
@@ -239,8 +239,8 @@ class Environment(nxsim.NetworkEnvironment):
             return list(self.agents)
         return [self.G.node[i]['agent'] for i in nodes]
 
-    def dump_csv(self, dir_path=None):
-        csv_name = os.path.join(self.get_path(dir_path),
+    def dump_csv(self, outdir=None):
+        csv_name = os.path.join(self.get_path(outdir),
                                 '{}.environment.csv'.format(self.name))
 
         with open(csv_name, 'w') as f:
@@ -249,9 +249,9 @@ class Environment(nxsim.NetworkEnvironment):
             for i in self.history_to_tuples():
                 cr.writerow(i)
 
-    def dump_gexf(self, dir_path=None):
+    def dump_gexf(self, outdir=None):
         G = self.history_to_graph()
-        graph_path = os.path.join(self.get_path(dir_path),
+        graph_path = os.path.join(self.get_path(outdir),
                                   self.name+".gexf")
         # Workaround for geometric models
         # See soil/soil#4
@@ -262,7 +262,7 @@ class Environment(nxsim.NetworkEnvironment):
 
         nx.write_gexf(G, graph_path, version="1.2draft")
 
-    def dump(self, dir_path=None, formats=None):
+    def dump(self, outdir=None, formats=None):
         if not formats:
             return
         functions = {
@@ -271,7 +271,7 @@ class Environment(nxsim.NetworkEnvironment):
         }
         for f in formats:
             if f in functions:
-                functions[f](dir_path)
+                functions[f](outdir)
             else:
                 raise ValueError('Unknown format: {}'.format(f))
 
@@ -356,7 +356,7 @@ class Environment(nxsim.NetworkEnvironment):
 
     def log_stats(self):
         stats = self.stats()
-        utils.logger.info('Environment stats: \n{}'.format(yaml.dump(stats, default_flow_style=False)))
+        serialization.logger.info('Environment stats: \n{}'.format(yaml.dump(stats, default_flow_style=False)))
     
     def __getstate__(self):
         state = {}

soil/exporters.py

@@ -0,0 +1,43 @@
+from .serialization import deserialize
+import os
+import time
+
+
+def for_sim(simulation, names, dir_path=None):
+    exporters = []
+    for name in names:
+        mod = deserialize(name, known_modules=['soil.exporters'])
+        exporters.append(mod(simulation))
+    return exporters
+
+
+class Base:
+
+    def __init__(self, simulation):
+        self.sim = simulation
+
+    def start(self):
+        pass
+
+    def end(self):
+        pass
+
+    def env(self):
+        pass
+
+
+class Dummy(Base):
+
+    def start(self):
+        with open(os.path.join(self.sim.outdir, 'dummy')) as f:
+            f.write('simulation started @ {}'.format(time.time()))
+
+    def env(self, env):
+        with open(os.path.join(self.sim.outdir, 'dummy-trial-{}'.format(env.name))) as f:
+            for i in env.history_to_tuples():
+                f.write(','.join(i))
+
+
+    def end(self):
+        with open(os.path.join(self.sim.outdir, 'dummy')) as f:
+            f.write('simulation ended @ {}'.format(time.time()))

soil/history.py

@@ -9,7 +9,7 @@ logger = logging.getLogger(__name__)
 
 from collections import UserDict, namedtuple
 
-from . import utils
+from . import serialization
 
 
 class History:
@@ -17,9 +17,9 @@ class History:
     Store and retrieve values from a sqlite database.
     """
 
-    def __init__(self, db_path=None, name=None, dir_path=None, backup=False):
+    def __init__(self, db_path=None, name=None, outdir=None, backup=False):
         if db_path is None and name:
-            db_path = os.path.join(dir_path or os.getcwd(),
+            db_path = os.path.join(outdir or os.getcwd(),
                                    '{}.db.sqlite'.format(name))
         if db_path:
             if backup and os.path.exists(db_path):
@@ -94,9 +94,9 @@ class History:
         if key not in self._dtypes:
             self.read_types()
             if key not in self._dtypes:
-                name = utils.name(value)
-                serializer = utils.serializer(name)
-                deserializer = utils.deserializer(name)
+                name = serialization.name(value)
+                serializer = serialization.serializer(name)
+                deserializer = serialization.deserializer(name)
                 self._dtypes[key] = (name, serializer, deserializer)
                 with self.db:
                     self.db.execute("replace into value_types (key, value_type) values (?, ?)", (key, name))
@@ -135,8 +135,8 @@ class History:
         with self.db:
             res = self.db.execute("select key, value_type from value_types ").fetchall()
         for k, v in res:
-            serializer = utils.serializer(v)
-            deserializer = utils.deserializer(v)
+            serializer = serialization.serializer(v)
+            deserializer = serialization.deserializer(v)
             self._dtypes[k] = (v, serializer, deserializer)
 
     def __getitem__(self, key):

soil/serialization.py

@@ -0,0 +1,199 @@
+import os
+import logging
+import ast
+import sys
+import yaml
+import importlib
+from glob import glob
+from random import random
+from copy import deepcopy
+from itertools import product, chain
+
+from jinja2 import Template
+
+import networkx as nx
+
+logger = logging.getLogger('soil')
+logger.setLevel(logging.INFO)
+
+
+def load_network(network_params, dir_path=None):
+    if network_params is None:
+        return nx.Graph()
+    path = network_params.get('path', None)
+    if path:
+        if dir_path and not os.path.isabs(path):
+            path = os.path.join(dir_path, path)
+        extension = os.path.splitext(path)[1][1:]
+        kwargs = {}
+        if extension == 'gexf':
+            kwargs['version'] = '1.2draft'
+            kwargs['node_type'] = int
+        try:
+            method = getattr(nx.readwrite, 'read_' + extension)
+        except AttributeError:
+            raise AttributeError('Unknown format')
+        return method(path, **kwargs)
+
+    net_args = network_params.copy()
+    net_gen = net_args.pop('generator')
+
+    if dir_path not in sys.path:
+        sys.path.append(dir_path)
+
+    method = deserializer(net_gen,
+                          known_modules=['networkx.generators',])
+
+    return method(**net_args)
+
+
+def load_file(infile):
+    with open(infile, 'r') as f:
+        return list(chain.from_iterable(map(expand_template, load_string(f))))
+
+def load_string(string):
+    yield from yaml.load_all(string)
+
+
+def expand_template(config):
+    if 'template' not in config:
+        yield config
+        return
+    if 'vars' not in config:
+        raise ValueError(('You must provide a definition of variables'
+                          ' for the template.'))
+
+    template = Template(config['template'])
+
+    sampler_name = config.get('sampler', 'SALib.sample.morris.sample')
+    n_samples = int(config.get('samples', 100))
+    sampler = deserializer(sampler_name)
+    bounds = config['vars']['bounds']
+
+    problem = {
+        'num_vars': len(bounds),
+        'names': list(bounds.keys()),
+        'bounds': list(v for v in bounds.values())
+    }
+    samples = sampler(problem, n_samples)
+
+    lists = config['vars'].get('lists', {})
+    names = list(lists.keys())
+    values = list(lists.values())
+    combs = list(product(*values))
+
+    allnames = names + problem['names']
+    allvalues = [(list(i[0])+list(i[1])) for i in product(combs, samples)]
+    params = list(map(lambda x: dict(zip(allnames, x)), allvalues))
+
+
+    blank_str = template.render({k: 0 for k in allnames})
+    blank = list(load_string(blank_str))
+    if len(blank) > 1:
+        raise ValueError('Templates must not return more than one configuration')
+
+    if 'name' in blank[0]:
+        raise ValueError('Templates cannot be named, use group instead')
+
+    confs = []
+    for ps in params:
+        string = template.render(ps)
+        for c in load_string(string):
+            yield c
+
+
+def load_files(*patterns, **kwargs):
+    for pattern in patterns:
+        for i in glob(pattern, **kwargs):
+            for config in load_file(i):
+                path = os.path.abspath(i)
+                if 'dir_path' not in config:
+                    config['dir_path'] = os.path.dirname(path)
+                yield config, path
+
+
+def load_config(config):
+    if isinstance(config, dict):
+        yield config, None
+    else:
+        yield from load_files(config)
+
+
+builtins = importlib.import_module('builtins')
+
+def name(value, known_modules=[]):
+    '''Return a name that can be imported, to serialize/deserialize an object'''
+    if value is None:
+        return 'None'
+    if not isinstance(value, type):  # Get the class name first
+        value = type(value)
+    tname = value.__name__
+    if hasattr(builtins, tname):
+        return tname
+    modname = value.__module__
+    if modname == '__main__':
+        return tname
+    if known_modules and modname in known_modules:
+        return tname
+    for kmod in known_modules:
+        if not kmod:
+            continue
+        module = importlib.import_module(kmod)
+        if hasattr(module, tname):
+            return tname
+    return '{}.{}'.format(modname, tname)
+
+
+def serializer(type_):
+    if type_ != 'str' and hasattr(builtins, type_):
+        return repr
+    return lambda x: x
+
+
+def serialize(v, known_modules=[]):
+    '''Get a text representation of an object.'''
+    tname = name(v, known_modules=known_modules)
+    func = serializer(tname)
+    return func(v), tname
+
+def deserializer(type_, known_modules=[]):
+    if type(type_) != str:  # Already deserialized
+        return type_
+    if type_ == 'str':
+        return lambda x='': x
+    if type_ == 'None':
+        return lambda x=None: None
+    if hasattr(builtins, type_):  # Check if it's a builtin type
+        cls = getattr(builtins, type_)
+        return lambda x=None: ast.literal_eval(x) if x is not None else cls()
+    # Otherwise, see if we can find the module and the class
+    modules = known_modules or []
+    options = []
+
+    for mod in modules:
+        if mod:
+            options.append((mod, type_))
+
+    if '.' in type_:  # Fully qualified module
+        module, type_ = type_.rsplit(".", 1)
+        options.append ((module, type_))
+
+    errors = []
+    for modname, tname in options:
+        try:
+            module = importlib.import_module(modname)
+            cls = getattr(module, tname)
+            return getattr(cls, 'deserialize', cls)
+        except (ModuleNotFoundError, AttributeError) as ex:
+            errors.append((modname, tname, ex))
+    raise Exception('Could not find type {}. Tried: {}'.format(type_, errors))
+
+
+def deserialize(type_, value=None, **kwargs):
+    '''Get an object from a text representation'''
+    if not isinstance(type_, str):
+        return type_
+    des = deserializer(type_, **kwargs)
+    if value is None:
+        return des
+    return des(value)

soil/simulation.py

@@ -13,9 +13,10 @@ import pickle
 
 from nxsim import NetworkSimulation
 
-from . import utils, basestring, agents
+from . import serialization, utils, basestring, agents
 from .environment import Environment
-from .utils import logger
+from .serialization import logger
+from .exporters import for_sim as exporters_for_sim
 
 
 class Simulation(NetworkSimulation):
@@ -50,6 +51,8 @@ class Simulation(NetworkSimulation):
     ---------
     name : str, optional
         name of the Simulation
+    group : str, optional
+        a group name can be used to link simulations
     topology : networkx.Graph instance, optional
     network_params : dict
         parameters used to create a topology with networkx, if no topology is given
@@ -60,8 +63,10 @@ class Simulation(NetworkSimulation):
     states : list, optional
         List of initial states corresponding to the nodes in the topology. Basic form is a list of integers
         whose value indicates the state
-    dir_path : str, optional
-        Directory path where to save pickled objects
+    dir_path: str, optional
+        Directory path to load simulation assets (files, modules...)
+    outdir : str, optional
+        Directory path to save simulation results
     seed : str, optional
         Seed to use for the random generator
     num_trials : int, optional
@@ -80,38 +85,45 @@ class Simulation(NetworkSimulation):
 
     """
 
-    def __init__(self, name=None, topology=None, network_params=None,
+    def __init__(self, name=None, group=None, topology=None, network_params=None,
                  network_agents=None, agent_type=None, states=None,
                  default_state=None, interval=1, dump=None, dry_run=False,
-                 dir_path=None, num_trials=1, max_time=100,
-                 load_module=None, seed=None,
+                 outdir=None, num_trials=1, max_time=100,
+                 load_module=None, seed=None, dir_path=None,
                  environment_agents=None, environment_params=None,
                  environment_class=None, **kwargs):
 
         self.seed = str(seed) or str(time.time())
         self.load_module = load_module
         self.network_params = network_params
-        self.name = name or 'UnnamedSimulation'
+        self.name = name or 'Unnamed_' + time.strftime("%Y-%m-%d_%H:%M:%S")
+        self.group = group or None
         self.num_trials = num_trials
         self.max_time = max_time
         self.default_state = default_state or {}
+        if not outdir:
+            outdir = os.path.join(os.getcwd(),
+                                       'soil_output')
+        self.outdir = os.path.join(outdir,
+                                   self.group or '',
+                                   self.name)
         self.dir_path = dir_path or os.getcwd()
         self.interval = interval
         self.dump = dump
         self.dry_run = dry_run
 
-        sys.path += [self.dir_path, os.getcwd()]
+        sys.path += list(x for x in [self.outdir, os.getcwd(), self.dir_path] if x not in sys.path)
 
         if topology is None:
-            topology = utils.load_network(network_params,
-                                          dir_path=self.dir_path)
+            topology = serialization.load_network(network_params,
+                                                  dir_path=self.dir_path)
         elif isinstance(topology, basestring) or isinstance(topology, dict):
             topology = json_graph.node_link_graph(topology)
         self.topology = nx.Graph(topology)
 
 
         self.environment_params = environment_params or {}
-        self.environment_class = utils.deserialize(environment_class,
+        self.environment_class = serialization.deserialize(environment_class,
                                                    known_modules=['soil.environment', ]) or Environment
 
         environment_agents = environment_agents or []
@@ -130,34 +142,51 @@ class Simulation(NetworkSimulation):
         return self.run(*args, **kwargs)
 
     def run(self, *args, **kwargs):
-        return list(self.run_simulation_gen(*args, **kwargs))
+        return list(self._run_simulation_gen(*args, **kwargs))
 
-    def run_simulation_gen(self, *args, parallel=False, dry_run=False,
-                           **kwargs):
-        p = Pool()
+    def _run_sync_or_async(self, parallel=False, *args, **kwargs):
+        if parallel:
+            p = Pool()
+            func = partial(self.run_trial_exceptions,
+                           *args,
+                           **kwargs)
+            for i in p.imap_unordered(func, range(self.num_trials)):
+                if isinstance(i, Exception):
+                    logger.error('Trial failed:\n\t{}'.format(i.message))
+                    continue
+                yield i
+        else:
+            for i in range(self.num_trials):
+                yield self.run_trial(i,
+                                     *args,
+                                     **kwargs)
+
+    def _run_simulation_gen(self, *args, parallel=False, dry_run=False,
+                            exporters=None, **kwargs):
+        exporters = exporters_for_sim(self,
+                                      exporters or [])
         with utils.timer('simulation {}'.format(self.name)):
-            if parallel:
-                func = partial(self.run_trial_exceptions, dry_run=dry_run or self.dry_run,
-                                                         return_env=True,
-                                                         **kwargs)
-                for i in p.imap_unordered(func, range(self.num_trials)):
-                    if isinstance(i, Exception):
-                        logger.error('Trial failed:\n\t{}'.format(i.message))
-                        continue
-                    yield i
-            else:
-                for i in range(self.num_trials):
-                    yield self.run_trial(i, dry_run = dry_run or self.dry_run, **kwargs)
             if not (dry_run or self.dry_run):
-                logger.info('Dumping results to {}'.format(self.dir_path))
-                self.dump_pickle(self.dir_path)
-                self.dump_yaml(self.dir_path)
+                logger.info('Dumping results to {}'.format(self.outdir))
+                self.dump_pickle(self.outdir)
+                self.dump_yaml(self.outdir)
             else:
                 logger.info('NOT dumping results')
+            for exporter in exporters:
+                exporter.start()
+
+            for env in self._run_sync_or_async(*args, parallel=parallel,
+                                               dry_run=dry_run, **kwargs):
+                for exporter in exporters:
+                    exporter.env(env)
+                yield env
+
+            for exporter in exporters:
+                exporter.end()
 
     def get_env(self, trial_id = 0, **kwargs):
-        opts=self.environment_params.copy()
-        env_name='{}_trial_{}'.format(self.name, trial_id)
+        opts = self.environment_params.copy()
+        env_name = '{}_trial_{}'.format(self.name, trial_id)
         opts.update({
             'name': env_name,
             'topology': self.topology.copy(),
@@ -169,13 +198,13 @@ class Simulation(NetworkSimulation):
             'states': self.states,
             'default_state': self.default_state,
             'environment_agents': self.environment_agents,
-            'dir_path': self.dir_path,
+            'outdir': self.outdir,
         })
         opts.update(kwargs)
-        env=self.environment_class(**opts)
+        env = self.environment_class(**opts)
         return env
 
-    def run_trial(self, trial_id = 0, until = None, return_env = True, **opts):
+    def run_trial(self, trial_id=0, until=None, dry_run=False, **opts):
         """Run a single trial of the simulation
 
         Parameters
@@ -183,16 +212,16 @@ class Simulation(NetworkSimulation):
         trial_id : int
         """
         # Set-up trial environment and graph
-        until=until or self.max_time
-        env=self.get_env(trial_id = trial_id, **opts)
+        until = until or self.max_time
+        env = self.get_env(trial_id = trial_id, **opts)
+        dry_run = self.dry_run or dry_run
         # Set up agents on nodes
         with utils.timer('Simulation {} trial {}'.format(self.name, trial_id)):
             env.run(until)
-        if self.dump and not self.dry_run:
+        if self.dump and not dry_run:
             with utils.timer('Dumping simulation {} trial {}'.format(self.name, trial_id)):
                 env.dump(formats = self.dump)
-        if return_env:
-            return env
+        return env
     def run_trial_exceptions(self, *args, **kwargs):
         '''
         A wrapper for run_trial that catches exceptions and returns them.
@@ -211,22 +240,22 @@ class Simulation(NetworkSimulation):
     def to_yaml(self):
         return yaml.dump(self.to_dict())
 
-    def dump_yaml(self, dir_path = None, file_name = None):
-        dir_path=dir_path or self.dir_path
-        if not os.path.exists(dir_path):
-            os.makedirs(dir_path)
+    def dump_yaml(self, outdir = None, file_name = None):
+        outdir = outdir or self.outdir
+        if not os.path.exists(outdir):
+            os.makedirs(outdir)
         if not file_name:
-            file_name=os.path.join(dir_path,
+            file_name=os.path.join(outdir,
                                      '{}.dumped.yml'.format(self.name))
         with open(file_name, 'w') as f:
             f.write(self.to_yaml())
 
-    def dump_pickle(self, dir_path = None, pickle_name = None):
-        dir_path=dir_path or self.dir_path
-        if not os.path.exists(dir_path):
-            os.makedirs(dir_path)
+    def dump_pickle(self, outdir = None, pickle_name = None):
+        outdir = outdir or self.outdir
+        if not os.path.exists(outdir):
+            os.makedirs(outdir)
         if not pickle_name:
-            pickle_name=os.path.join(dir_path,
+            pickle_name=os.path.join(outdir,
                                        '{}.simulation.pickle'.format(self.name))
         with open(pickle_name, 'wb') as f:
             pickle.dump(self, f)
@@ -235,14 +264,14 @@ class Simulation(NetworkSimulation):
         state={}
         for k, v in self.__dict__.items():
             if k[0] != '_':
-                state[k]=v
-        state['topology']=json_graph.node_link_data(self.topology)
-        state['network_agents']=agents.serialize_distribution(self.network_agents,
-                                                                known_modules = [])
-        state['environment_agents']=agents.serialize_distribution(self.environment_agents,
-                                                                    known_modules = [])
-        state['environment_class']=utils.serialize(self.environment_class,
-                                                     known_modules=['soil.environment'])[1]  # func, name
+                state[k] = v
+                state['topology'] = json_graph.node_link_data(self.topology)
+                state['network_agents'] = agents.serialize_distribution(self.network_agents,
+                                                                        known_modules = [])
+                state['environment_agents'] = agents.serialize_distribution(self.environment_agents,
+                                                                            known_modules = [])
+                state['environment_class'] = serialization.serialize(self.environment_class,
+                                                                     known_modules=['soil.environment'])[1]  # func, name
         if state['load_module'] is None:
             del state['load_module']
         return state
@@ -250,19 +279,28 @@ class Simulation(NetworkSimulation):
     def __setstate__(self, state):
         self.__dict__ = state
         self.load_module = getattr(self, 'load_module', None)
+        if self.outdir not in sys.path:
+            sys.path += [self.outdir, os.getcwd()]
         if self.dir_path not in sys.path:
             sys.path += [self.dir_path, os.getcwd()]
         self.topology = json_graph.node_link_graph(state['topology'])
         self.network_agents = agents.calculate_distribution(agents._convert_agent_types(self.network_agents))
         self.environment_agents = agents._convert_agent_types(self.environment_agents,
                                                               known_modules=[self.load_module])
-        self.environment_class = utils.deserialize(self.environment_class,
+        self.environment_class = serialization.deserialize(self.environment_class,
                                                    known_modules=[self.load_module, 'soil.environment', ])  # func, name
         return state
 
 
-def from_config(config):
-    config = list(utils.load_config(config))
+def all_from_config(config):
+    configs = list(serialization.load_config(config))
+    for config, _ in configs:
+        sim = Simulation(**config)
+        yield sim
+
+
+def from_config(conf_or_path):
+    config = list(serialization.load_config(conf_or_path))
     if len(config) > 1:
         raise AttributeError('Provide only one configuration')
     config = config[0][0]
@@ -270,10 +308,10 @@ def from_config(config):
     return sim
 
 
-def run_from_config(*configs, results_dir='soil_output', dump=None, timestamp=False,  **kwargs):
+def run_from_config(*configs, outdir=None, dump=None, timestamp=False,  **kwargs):
     for config_def in configs:
         # logger.info("Found {} config(s)".format(len(ls)))
-        for config, _ in utils.load_config(config_def):
+        for config, path in serialization.load_config(config_def):
             name = config.get('name', 'unnamed')
             logger.info("Using config(s): {name}".format(name=name))
 
@@ -282,9 +320,12 @@ def run_from_config(*configs, results_dir='soil_output', dump=None, timestamp=Fa
                                             time.strftime("%Y-%m-%d_%H:%M:%S"))
             else:
                 sim_folder = name
-            dir_path = os.path.join(results_dir, sim_folder)
             if dump is not None:
                 config['dump'] = dump
-            sim = Simulation(dir_path=dir_path, **config)
-            logger.info('Dumping results to {} : {}'.format(sim.dir_path, sim.dump))
+            dir_path = config.pop('dir_path', os.path.dirname(path))
+            outdir = config.pop('outdir', outdir)
+            sim = Simulation(dir_path=dir_path,
+                             outdir=outdir,
+                             **config)
+            logger.info('Dumping results to {} : {}'.format(sim.outdir, sim.dump))
             sim.run_simulation(**kwargs)

soil/utils.py

@@ -1,72 +1,12 @@
-import os
-import ast
-import sys
-import yaml
 import logging
-import importlib
 import time
-from glob import glob
-from random import random
-from copy import deepcopy
-
-import networkx as nx
 
 from contextlib import contextmanager
 
-
 logger = logging.getLogger('soil')
 logger.setLevel(logging.INFO)
 
 
-def load_network(network_params, dir_path=None):
-    if network_params is None:
-        return nx.Graph()
-    path = network_params.get('path', None)
-    if path:
-        if dir_path and not os.path.isabs(path):
-            path = os.path.join(dir_path, path)
-        extension = os.path.splitext(path)[1][1:]
-        kwargs = {}
-        if extension == 'gexf':
-            kwargs['version'] = '1.2draft'
-            kwargs['node_type'] = int
-        try:
-            method = getattr(nx.readwrite, 'read_' + extension)
-        except AttributeError:
-            raise AttributeError('Unknown format')
-        return method(path, **kwargs)
-
-    net_args = network_params.copy()
-    net_gen = net_args.pop('generator')
-
-    if dir_path not in sys.path:
-        sys.path.append(dir_path)
-
-    method = deserializer(net_gen,
-                          known_modules=['networkx.generators',])
-
-    return method(**net_args)
-
-
-def load_file(infile):
-    with open(infile, 'r') as f:
-        return list(yaml.load_all(f))
-
-
-def load_files(*patterns):
-    for pattern in patterns:
-        for i in glob(pattern):
-            for config in load_file(i):
-                yield config, os.path.abspath(i)
-
-
-def load_config(config):
-    if isinstance(config, dict):
-        yield config, None
-    else:
-        yield from load_files(config)
-
-
 @contextmanager
 def timer(name='task', pre="", function=logger.info, to_object=None):
     start = time.time()
@@ -80,83 +20,3 @@ def timer(name='task', pre="", function=logger.info, to_object=None):
     if to_object:
         to_object.start = start
         to_object.end = end
-
-
-builtins = importlib.import_module('builtins')
-
-def name(value, known_modules=[]):
-    '''Return a name that can be imported, to serialize/deserialize an object'''
-    if value is None:
-        return 'None'
-    if not isinstance(value, type):  # Get the class name first
-        value = type(value)
-    tname = value.__name__
-    if hasattr(builtins, tname):
-        return tname
-    modname = value.__module__
-    if modname == '__main__':
-        return tname
-    if known_modules and modname in known_modules:
-        return tname
-    for kmod in known_modules:
-        if not kmod:
-            continue
-        module = importlib.import_module(kmod)
-        if hasattr(module, tname):
-            return tname
-    return '{}.{}'.format(modname, tname)
-
-
-def serializer(type_):
-    if type_ != 'str' and hasattr(builtins, type_):
-        return repr
-    return lambda x: x
-
-
-def serialize(v, known_modules=[]):
-    '''Get a text representation of an object.'''
-    tname = name(v, known_modules=known_modules)
-    func = serializer(tname)
-    return func(v), tname
-
-def deserializer(type_, known_modules=[]):
-    if type(type_) != str:  # Already deserialized
-        return type_
-    if type_ == 'str':
-        return lambda x='': x
-    if type_ == 'None':
-        return lambda x=None: None
-    if hasattr(builtins, type_):  # Check if it's a builtin type
-        cls = getattr(builtins, type_)
-        return lambda x=None: ast.literal_eval(x) if x is not None else cls()
-    # Otherwise, see if we can find the module and the class
-    modules = known_modules or []
-    options = []
-
-    for mod in modules:
-        if mod:
-            options.append((mod, type_))
-
-    if '.' in type_:  # Fully qualified module
-        module, type_ = type_.rsplit(".", 1)
-        options.append ((module, type_))
-
-    errors = []
-    for modname, tname in options:
-        try:
-            module = importlib.import_module(modname)
-            cls = getattr(module, tname)
-            return getattr(cls, 'deserialize', cls)
-        except (ModuleNotFoundError, AttributeError) as ex:
-            errors.append((modname, tname, ex))
-    raise Exception('Could not find type {}. Tried: {}'.format(type_, errors))
-
-
-def deserialize(type_, value=None, **kwargs):
-    '''Get an object from a text representation'''
-    if not isinstance(type_, str):
-        return type_
-    des = deserializer(type_, **kwargs)
-    if value is None:
-        return des
-    return des(value)

soil/web/__init__.py

@@ -180,7 +180,7 @@ class SocketHandler(tornado.websocket.WebSocketHandler):
         with self.logging(self.simulation_name):
             try:
                 config = dict(**self.config)
-                config['dir_path'] = os.path.join(self.application.dir_path, config['name'])
+                config['outdir'] = os.path.join(self.application.outdir, config['name'])
                 config['dump'] = self.application.dump
                 self.trials = yield self.nonblocking(config)
 
@@ -232,12 +232,12 @@ class ModularServer(tornado.web.Application):
     settings = {'debug': True,
                 'template_path': ROOT + '/templates'}
 
-    def __init__(self, dump=False, dir_path='output', name='SOIL', verbose=True, *args, **kwargs):
+    def __init__(self, dump=False, outdir='output', name='SOIL', verbose=True, *args, **kwargs):
         
         self.verbose = verbose
         self.name = name
         self.dump = dump
-        self.dir_path = dir_path
+        self.outdir = outdir
 
         # Initializing the application itself:
         super().__init__(self.handlers, **self.settings)