Seed before env initialization

Fixes #6

.dockerignore

@@ -1,2 +1,5 @@
 **/soil_output
 .*
+**/__pycache__
+__pycache__
+*.pyc

CHANGELOG.md

@@ -3,6 +3,38 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.14.9]
+### Changed
+* Seed random before environment initialization
+## [0.14.8]
+### Fixed
+* Invalid directory names in Windows gsi-upm/soil#5
+## [0.14.7]
+### Changed
+* Minor change to traceback handling in async simulations
+### Fixed
+* Incomplete example in the docs (example.yml) caused an exception
+## [0.14.6]
+### Fixed
+* Bug with newer versions of networkx (0.24) where the Graph.node attribute has been removed. We have updated our calls, but the code in nxsim is not under our control, so we have pinned the networkx version until that issue is solved.
+### Changed
+* Explicit yaml.SafeLoader to avoid deprecation warnings when using yaml.load. It should not break any existing setups, but we could move to the FullLoader in the future if needed.
+
+## [0.14.4]
+### Fixed
+* Bug in `agent.get_agents()` when `state_id` is passed as a string. The tests have been modified accordingly.
+## [0.14.3]
+### Fixed
+* Incompatibility with py3.3-3.6 due to ModuleNotFoundError and TypeError in DryRunner
+## [0.14.2]
+### Fixed
+* Output path for exporters is now soil_output
+### Changed 
+* CSV output to stdout in dry_run mode
+## [0.14.1]
+### Changed
+* Exporter names in lower case
+* Add default exporter in runs
 ## [0.14.0]
 ### Added
 * Loading configuration from template definitions in the yaml, in preparation for SALib support.

README.md

@@ -30,5 +30,5 @@ If you use Soil in your research, don't forget to cite this paper:
 
 @Copyright GSI - Universidad Politécnica de Madrid 2017
 
-[![SOIL](logo_gsi.png)](https://www.gsi.dit.upm.es)
+[![SOIL](logo_gsi.png)](https://www.gsi.upm.es)
 

docs/configuration.rst

@@ -8,32 +8,8 @@ The advantage of a configuration file is that it is a clean declarative descript
 Simulation configuration files can be formatted in ``json`` or ``yaml`` and they define all the parameters of a simulation.
 Here's an example (``example.yml``).
 
-.. code:: yaml
+.. literalinclude:: example.yml
-          
+   :language: yaml
-    ---
-    name: MyExampleSimulation
-    max_time: 50
-    num_trials: 3
-    interval: 2
-    network_params:
-        generator: barabasi_albert_graph
-        n: 100
-        m: 2
-    network_agents:
-        - agent_type: SISaModel
-           weight: 1
-            state:
-            id: content
-        - agent_type: SISaModel
-            weight: 1
-            state:
-            id: discontent
-        - agent_type: SISaModel
-            weight: 8
-            state:
-            id: neutral
-    environment_params:
-        prob_infect: 0.075
 
 
 This example configuration will run three trials (``num_trials``) of a simulation containing a randomly generated network (``network_params``).

docs/example.yml

@@ -0,0 +1,35 @@
+---
+name: MyExampleSimulation
+max_time: 50
+num_trials: 3
+interval: 2
+network_params:
+    generator: barabasi_albert_graph
+    n: 100
+    m: 2
+network_agents:
+    - agent_type: SISaModel
+      weight: 1
+      state:
+        id: content
+    - agent_type: SISaModel
+      weight: 1
+      state:
+        id: discontent
+    - agent_type: SISaModel
+      weight: 8
+      state:
+        id: neutral
+environment_params:
+    prob_infect: 0.075
+    neutral_discontent_spon_prob: 0.1
+    neutral_discontent_infected_prob: 0.3
+    neutral_content_spon_prob: 0.3
+    neutral_content_infected_prob: 0.4
+    discontent_neutral: 0.5
+    discontent_content: 0.5
+    variance_d_c: 0.2
+    content_discontent: 0.2
+    variance_c_d: 0.2
+    content_neutral: 0.2
+    standard_variance: 1

docs/installation.rst

@@ -14,11 +14,11 @@ Now test that it worked by running the command line tool
 
    soil --help
 
-Or using soil programmatically:
+Or, if you're using using soil programmatically:
 
 .. code:: python
 
    import soil
    print(soil.__version__)
 
-The latest version can be installed through `GitLab <https://lab.cluster.gsi.dit.upm.es/soil/soil.git>`_.
+The latest version can be installed through `GitLab <https://lab.gsi.upm.es/soil/soil.git>`_ or `GitHub <https://github.com/gsi-upm/soil>`_.

docs/soil_tutorial.rst

@@ -323,7 +323,7 @@ Let's run our simulation:
 
 .. code:: ipython3
 
-    soil.simulation.run_from_config(config, dump=False)
+    soil.simulation.run_from_config(config)
 
 
 .. parsed-literal::

examples/pubcrawl/pubcrawl.py

@@ -59,7 +59,7 @@ class Patron(FSM):
         2) Look for a bar where the agent and other agents in the same group can get in.
         3) While in the bar, patrons only drink, until they get drunk and taken home.
     '''
-    level = logging.INFO
+    level = logging.DEBUG
 
     defaults = {
         'pub': None,
@@ -113,7 +113,8 @@ class Patron(FSM):
     @state
     def at_home(self):
         '''The end'''
-        self.debug('Life sucks. I\'m home!')
+        others = self.get_agents(state_id=Patron.at_home.id, limit_neighbors=True)
+        self.debug('I\'m home. Just like {} of my friends'.format(len(others)))
     
     def drink(self):
         self['pints'] += 1

requirements.txt

@@ -1,10 +1,10 @@
 nxsim>=0.1.2
 simpy
-networkx>=2.0
+networkx>=2.0,<2.4
 numpy
 matplotlib
 pyyaml>=5.1
 pandas>=0.23
-scipy==1.2.1 # scipy 1.3.0rc1 is not compatible with salib
+scipy>=1.3
 SALib>=1.3
 Jinja2

soil/VERSION

@@ -1 +1 @@
-0.13.8
+0.14.9

soil/__init__.py

@@ -57,11 +57,11 @@ def main():
     logging.info('Loading config file: {}'.format(args.file))
 
     try:
-        exporters = list(args.exporter or [])
+        exporters = list(args.exporter or ['default', ])
         if args.csv:
-            exporters.append('CSV')
+            exporters.append('csv')
         if args.graph:
-            exporters.append('Gexf')
+            exporters.append('gexf')
         exp_params = {}
         if args.dry_run:
             exp_params['copy_to'] = sys.stdout

soil/agents/__init__.py

@@ -476,11 +476,8 @@ class Geo(NetworkAgent):
 
 def select(agents, state_id=None, agent_type=None, ignore=None, iterator=False, **kwargs):
 
-    if state_id is not None:
+    if state_id is not None and not isinstance(state_id, (tuple, list)):
-        try:
+        state_id = tuple([state_id])
-            state_id = tuple(state_id)
-        except TypeError:
-            state_id = tuple([state_id])
     if agent_type is not None:
         try:
             agent_type = tuple(agent_type)

soil/analysis.py

@@ -20,7 +20,7 @@ def _read_data(pattern, *args, from_csv=False, process_args=None, **kwargs):
         process_args = {}
     for folder in glob.glob(pattern):
         config_file = glob.glob(join(folder, '*.yml'))[0]
-        config = yaml.load(open(config_file))
+        config = yaml.load(open(config_file), Loader=yaml.SafeLoader)
         df = None
         if from_csv:
             for trial_data in sorted(glob.glob(join(folder,
@@ -133,7 +133,7 @@ def get_count(df, *keys):
 def get_value(df, *keys, aggfunc='sum'):
     if keys:
         df = df[list(keys)]
-    return df.groupby(axis=1, level=0).agg(aggfunc, axis=1)
+    return df.groupby(axis=1, level=0).agg(aggfunc)
 
 
 def plot_all(*args, **kwargs):

soil/environment.py

@@ -44,6 +44,8 @@ class Environment(nxsim.NetworkEnvironment):
                  topology=None,
                  *args, **kwargs):
         self.name = name or 'UnnamedEnvironment'
+        seed = seed or time.time()
+        random.seed(seed)
         if isinstance(states, list):
             states = dict(enumerate(states))
         self.states = deepcopy(states) if states else {}
@@ -55,12 +57,11 @@ class Environment(nxsim.NetworkEnvironment):
         self.interval = interval
         self._history = history.History(name=self.name,
                                         backup=True)
+        self['SEED'] = seed
         # Add environment agents first, so their events get
         # executed before network agents
         self.environment_agents = environment_agents or []
         self.network_agents = network_agents or []
-        self['SEED'] = seed or time.time()
-        random.seed(self['SEED'])
 
     @property
     def agents(self):
@@ -87,7 +88,7 @@ class Environment(nxsim.NetworkEnvironment):
     @property
     def network_agents(self):
         for i in self.G.nodes():
-            node = self.G.node[i]
+            node = self.G.nodes[i]
             if 'agent' in node:
                 yield node['agent']
 
@@ -212,12 +213,12 @@ class Environment(nxsim.NetworkEnvironment):
         return self[key] if key in self else default
 
     def get_agent(self, agent_id):
-        return self.G.node[agent_id]['agent']
+        return self.G.nodes[agent_id]['agent']
 
     def get_agents(self, nodes=None):
         if nodes is None:
             return list(self.agents)
-        return [self.G.node[i]['agent'] for i in nodes]
+        return [self.G.nodes[i]['agent'] for i in nodes]
 
     def dump_csv(self, f):
         with utils.open_or_reuse(f, 'w') as f:
@@ -231,9 +232,9 @@ class Environment(nxsim.NetworkEnvironment):
         # Workaround for geometric models
         # See soil/soil#4
         for node in G.nodes():
-            if 'pos' in G.node[node]:
+            if 'pos' in G.nodes[node]:
-                G.node[node]['viz'] = {"position": {"x": G.node[node]['pos'][0], "y": G.node[node]['pos'][1], "z": 0.0}}
+                G.nodes[node]['viz'] = {"position": {"x": G.nodes[node]['pos'][0], "y": G.nodes[node]['pos'][1], "z": 0.0}}
-                del (G.node[node]['pos'])
+                del (G.nodes[node]['pos'])
 
         nx.write_gexf(G, f, version="1.2draft")
 

soil/exporters.py

@@ -50,7 +50,7 @@ class Exporter:
 
     def __init__(self, simulation, outdir=None, dry_run=None, copy_to=None):
         self.sim = simulation
-        outdir = outdir or os.getcwd()
+        outdir = outdir or os.path.join(os.getcwd(), 'soil_output')
         self.outdir = os.path.join(outdir,
                                    simulation.group or '',
                                    simulation.name)
@@ -78,8 +78,8 @@ class Exporter:
         return open_or_reuse(f, mode=mode, **kwargs)
 
 
-class Default(Exporter):
+class default(Exporter):
-    '''Default exporter. Writes CSV and sqlite results, as well as the simulation YAML'''
+    '''Default exporter. Writes sqlite results, as well as the simulation YAML'''
 
     def start(self):
         if not self.dry_run:
@@ -96,25 +96,29 @@ class Default(Exporter):
                     env.dump_sqlite(f)
 
 
-class CSV(Exporter):
+class csv(Exporter):
+    '''Export the state of each environment (and its agents) in a separate CSV file'''
     def trial_end(self, env):
-        if not self.dry_run:
+        with timer('[CSV] Dumping simulation {} trial {} @ dir {}'.format(self.sim.name,
-            with timer('[CSV] Dumping simulation {} trial {}'.format(self.sim.name,
+                                                                          env.name,
-                                                               env.name)):
+                                                                          self.outdir)):
-                with self.output('{}.csv'.format(env.name)) as f:
+            with self.output('{}.csv'.format(env.name)) as f:
-                    env.dump_csv(f)
+                env.dump_csv(f)
 
 
-class Gexf(Exporter):
+class gexf(Exporter):
     def trial_end(self, env):
-        if not self.dry_run:
+        if self.dry_run:
-            with timer('[CSV] Dumping simulation {} trial {}'.format(self.sim.name,
+            logger.info('Not dumping GEXF in dry_run mode')
-                                                                     env.name)):
+            return
-                with self.output('{}.gexf'.format(env.name), mode='wb') as f:
+
-                    env.dump_gexf(f)
+        with timer('[GEXF] Dumping simulation {} trial {}'.format(self.sim.name,
+                                                                  env.name)):
+            with self.output('{}.gexf'.format(env.name), mode='wb') as f:
+                env.dump_gexf(f)
 
 
-class Dummy(Exporter):
+class dummy(Exporter):
 
     def start(self):
         with self.output('dummy', 'w') as f:
@@ -131,7 +135,7 @@ class Dummy(Exporter):
             f.write('simulation ended @ {}\n'.format(time.time()))
 
 
-class Distribution(Exporter):
+class distribution(Exporter):
     '''
     Write the distribution of agent states at the end of each trial,
     the mean value, and its deviation.
@@ -165,7 +169,7 @@ class Distribution(Exporter):
         with self.output('metrics.csv') as f:
             dfm.to_csv(f)
 
-class GraphDrawing(Exporter):
+class graphdrawing(Exporter):
 
     def trial_end(self, env):
         # Outside effects

soil/history.py

@@ -11,6 +11,7 @@ logger = logging.getLogger(__name__)
 from collections import UserDict, namedtuple
 
 from . import serialization
+from .utils import open_or_reuse
 
 
 class History:
@@ -236,7 +237,7 @@ class History:
 
     def dump(self, f):
         self._close()
-        for line in open(self.db_path, 'rb'):
+        for line in open_or_reuse(self.db_path, 'rb'):
             f.write(line)
 
 

soil/serialization.py

@@ -186,7 +186,7 @@ def deserializer(type_, known_modules=[]):
             module = importlib.import_module(modname)
             cls = getattr(module, tname)
             return getattr(cls, 'deserialize', cls)
-        except (ModuleNotFoundError, AttributeError) as ex:
+        except (ImportError, AttributeError) as ex:
             errors.append((modname, tname, ex))
     raise Exception('Could not find type {}. Tried: {}'.format(type_, errors))
 

soil/simulation.py

@@ -94,7 +94,7 @@ class Simulation(NetworkSimulation):
         self.seed = str(seed) or str(time.time())
         self.load_module = load_module
         self.network_params = network_params
-        self.name = name or 'Unnamed_' + time.strftime("%Y-%m-%d_%H:%M:%S")
+        self.name = name or 'Unnamed_' + time.strftime("%Y-%m-%d_%H.%M.%S")
         self.group = group or None
         self.num_trials = num_trials
         self.max_time = max_time
@@ -153,11 +153,11 @@ class Simulation(NetworkSimulation):
                                      **kwargs)
 
     def _run_simulation_gen(self, *args, parallel=False, dry_run=False,
-                            exporters=None, outdir=None, exporter_params={}, **kwargs):
+                            exporters=['default', ], outdir=None, exporter_params={}, **kwargs):
         logger.info('Using exporters: %s', exporters or [])
         logger.info('Output directory: %s', outdir)
         exporters = exporters_for_sim(self,
-                                      exporters or [],
+                                      exporters,
                                       dry_run=dry_run,
                                       outdir=outdir,
                                       **exporter_params)
@@ -216,9 +216,10 @@ class Simulation(NetworkSimulation):
         try:
             return self.run_trial(*args, **kwargs)
         except Exception as ex:
-            c = ex.__cause__
+            if ex.__cause__ is not None:
-            c.message = ''.join(traceback.format_exception(type(c), c, c.__traceback__)[:])
+                ex = ex.__cause__
-            return c
+            ex.message = ''.join(traceback.format_exception(type(ex), ex, ex.__traceback__)[:])
+            return ex
 
     def to_dict(self):
         return self.__getstate__()

soil/utils.py

@@ -2,6 +2,8 @@ import logging
 import time
 import os
 
+from shutil import copyfile
+
 from contextlib import contextmanager
 
 logger = logging.getLogger('soil')
@@ -23,15 +25,26 @@ def timer(name='task', pre="", function=logger.info, to_object=None):
         to_object.end = end
 
 
-def safe_open(path, *args, **kwargs):
+def safe_open(path, mode='r', backup=True, **kwargs):
     outdir = os.path.dirname(path)
     if outdir and not os.path.exists(outdir):
         os.makedirs(outdir)
-    return open(path, *args, **kwargs)
+    if backup and 'w' in mode and os.path.exists(path):
+        creation = os.path.getctime(path)
+        stamp = time.strftime('%Y-%m-%d_%H.%M', time.localtime(creation))
+
+        backup_dir = os.path.join(outdir, stamp)
+        if not os.path.exists(backup_dir):
+            os.makedirs(backup_dir)
+        newpath = os.path.join(backup_dir, os.path.basename(path))
+        if os.path.exists(newpath):
+            newpath = '{}@{}'.format(newpath, time.time())
+        copyfile(path, newpath)
+    return open(path, mode=mode, **kwargs)
 
 
 def open_or_reuse(f, *args, **kwargs):
     try:
         return safe_open(f, *args, **kwargs)
-    except TypeError:
+    except (AttributeError, TypeError):
         return f

soil/web/__init__.py

@@ -118,9 +118,9 @@ class SocketHandler(tornado.websocket.WebSocketHandler):
         elif msg['type'] == 'download_gexf':
             G = self.trials[ int(msg['data']) ].history_to_graph()
             for node in G.nodes():
-                if 'pos' in G.node[node]:
+                if 'pos' in G.nodes[node]:
-                    G.node[node]['viz'] = {"position": {"x": G.node[node]['pos'][0], "y": G.node[node]['pos'][1], "z": 0.0}}
+                    G.nodes[node]['viz'] = {"position": {"x": G.nodes[node]['pos'][0], "y": G.nodes[node]['pos'][1], "z": 0.0}}
-                    del (G.node[node]['pos'])
+                    del (G.nodes[node]['pos'])
             writer = nx.readwrite.gexf.GEXFWriter(version='1.2draft')
             writer.add_graph(G)
             self.write_message({'type': 'download_gexf',
@@ -130,9 +130,9 @@ class SocketHandler(tornado.websocket.WebSocketHandler):
         elif msg['type'] == 'download_json':
             G = self.trials[ int(msg['data']) ].history_to_graph()
             for node in G.nodes():
-                if 'pos' in G.node[node]:
+                if 'pos' in G.nodes[node]:
-                    G.node[node]['viz'] = {"position": {"x": G.node[node]['pos'][0], "y": G.node[node]['pos'][1], "z": 0.0}}
+                    G.nodes[node]['viz'] = {"position": {"x": G.nodes[node]['pos'][0], "y": G.nodes[node]['pos'][1], "z": 0.0}}
-                    del (G.node[node]['pos'])
+                    del (G.nodes[node]['pos'])
             self.write_message({'type': 'download_json',
                 'filename': self.config['name'] + '_trial_' + str(msg['data']),
                 'data': nx.node_link_data(G) })

tests/test_exporters.py

@@ -60,7 +60,7 @@ class Exporters(TestCase):
         }
         output = io.StringIO()
         s = simulation.from_config(config)
-        s.run_simulation(exporters=[exporters.Distribution], dry_run=True, exporter_params={'copy_to': output})
+        s.run_simulation(exporters=[exporters.distribution], dry_run=True, exporter_params={'copy_to': output})
         result = output.getvalue()
         assert 'count' in result
         assert 'SEED,Noneexporter_sim_trial_3,1,,1,1,1,1' in result
@@ -83,10 +83,10 @@ class Exporters(TestCase):
         s = simulation.from_config(config)
         tmpdir = tempfile.mkdtemp()
         envs = s.run_simulation(exporters=[
-            exporters.Default,
+            exporters.default,
-            exporters.CSV,
+            exporters.csv,
-            exporters.Gexf,
+            exporters.gexf,
-            exporters.Distribution,
+            exporters.distribution,
         ],
                                 outdir=tmpdir,
                                 exporter_params={'copy_to': output})

tests/test_main.py

@@ -16,10 +16,15 @@ ROOT = os.path.abspath(os.path.dirname(__file__))
 EXAMPLES = join(ROOT, '..', 'examples')
 
 
-class CustomAgent(agents.BaseAgent):
+class CustomAgent(agents.FSM):
-    def step(self):
+    @agents.default_state
-        self.state['neighbors'] = self.count_agents(state_id=0,
+    @agents.state
+    def normal(self):
+        self.state['neighbors'] = self.count_agents(state_id='normal',
                                                     limit_neighbors=True)
+    @agents.state
+    def unreachable(self):
+        return
 
 class TestMain(TestCase):
 
@@ -134,8 +139,7 @@ class TestMain(TestCase):
             },
             'network_agents': [{
                 'agent_type': CustomAgent,
-                'weight': 1,
+                'weight': 1
-                'state': {'id': 0}
 
             }],
             'max_time': 10,
@@ -145,6 +149,9 @@ class TestMain(TestCase):
         s = simulation.from_config(config)
         env = s.run_simulation(dry_run=True)[0]
         assert env.get_agent(0).state['neighbors'] == 1
+        assert env.get_agent(0).state['neighbors'] == 1
+        assert env.get_agent(1).count_agents(state_id='normal') == 2
+        assert env.get_agent(1).count_agents(state_id='normal', limit_neighbors=True) == 1
 
     def test_torvalds_example(self):
         """A complete example from a documentation should work."""
@@ -179,7 +186,7 @@ class TestMain(TestCase):
         with utils.timer('serializing'):
             serial = s.to_yaml()
         with utils.timer('recovering'):
-            recovered = yaml.load(serial)
+            recovered = yaml.load(serial, Loader=yaml.SafeLoader)
         with utils.timer('deleting'):
             del recovered['topology']
         assert config == recovered
@@ -233,7 +240,7 @@ class TestMain(TestCase):
         env[0, 0, 'testvalue'] = 'start'
         env[0, 10, 'testvalue'] = 'finish'
         nG = env.history_to_graph()
-        values = nG.node[0]['attr_testvalue']
+        values = nG.nodes[0]['attr_testvalue']
         assert ('start', 0, 10) in values
         assert ('finish', 10, None) in values