Fix incompatibility and bug in get_agents

1.3.0rc1 is not compatible with salib

CHANGELOG.md

@@ -3,6 +3,21 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.14.4]
+### Fixed
+* Bug in `agent.get_agents()` when `state_id` is passed as a string. The tests have been modified accordingly.
+## [0.14.3]
+### Fixed
+* Incompatibility with py3.3-3.6 due to ModuleNotFoundError and TypeError in DryRunner
+## [0.14.2]
+### Fixed
+* Output path for exporters is now soil_output
+### Changed 
+* CSV output to stdout in dry_run mode
+## [0.14.1]
+### Changed
+* Exporter names in lower case
+* Add default exporter in runs
 ## [0.14.0]
 ### Added
 * Loading configuration from template definitions in the yaml, in preparation for SALib support.

Makefile

@@ -1,4 +1,7 @@
-test:
+quick-test:
 	docker-compose exec dev python -m pytest -s -v
 
-.PHONY: test
+test:
+	docker run -t -v $$PWD:/usr/src/app -w /usr/src/app python:3.7 python setup.py test
+
+.PHONY: test

docs/soil_tutorial.rst

@@ -323,7 +323,7 @@ Let's run our simulation:
 
 .. code:: ipython3
 
-    soil.simulation.run_from_config(config, dump=False)
+    soil.simulation.run_from_config(config)
 
 
 .. parsed-literal::

examples/pubcrawl/pubcrawl.py

@@ -59,7 +59,7 @@ class Patron(FSM):
         2) Look for a bar where the agent and other agents in the same group can get in.
         3) While in the bar, patrons only drink, until they get drunk and taken home.
     '''
-    level = logging.INFO
+    level = logging.DEBUG
 
     defaults = {
         'pub': None,
@@ -113,7 +113,8 @@ class Patron(FSM):
     @state
     def at_home(self):
         '''The end'''
-        self.debug('Life sucks. I\'m home!')
+        others = self.get_agents(state_id=Patron.at_home.id, limit_neighbors=True)
+        self.debug('I\'m home. Just like {} of my friends'.format(len(others)))
     
     def drink(self):
         self['pints'] += 1

requirements.txt

@@ -3,8 +3,8 @@ simpy
 networkx>=2.0
 numpy
 matplotlib
-pyyaml
+pyyaml>=5.1
 pandas>=0.23
-scipy>=1.2
+scipy==1.2.1 # scipy 1.3.0rc1 is not compatible with salib
 SALib>=1.3
 Jinja2

soil/VERSION

@@ -1 +1 @@
-0.13.8
+0.14.4

soil/__init__.py

@@ -57,11 +57,11 @@ def main():
     logging.info('Loading config file: {}'.format(args.file))
 
     try:
-        exporters = list(args.exporter or [])
+        exporters = list(args.exporter or ['default', ])
         if args.csv:
-            exporters.append('CSV')
+            exporters.append('csv')
         if args.graph:
-            exporters.append('Gexf')
+            exporters.append('gexf')
         exp_params = {}
         if args.dry_run:
             exp_params['copy_to'] = sys.stdout

soil/agents/__init__.py

@@ -171,7 +171,7 @@ class BaseAgent(nxsim.BaseAgent):
 
     def info(self, *args, **kwargs):
         return self.log(*args, level=logging.INFO, **kwargs)
-    
+
     def __getstate__(self):
         '''
         Serializing an agent will lose all its running information (you cannot
@@ -476,11 +476,8 @@ class Geo(NetworkAgent):
 
 def select(agents, state_id=None, agent_type=None, ignore=None, iterator=False, **kwargs):
 
-    if state_id is not None:
+    if state_id is not None and not isinstance(state_id, (tuple, list)):
-        try:
+        state_id = tuple([state_id])
-            state_id = tuple(state_id)
-        except TypeError:
-            state_id = tuple([state_id])
     if agent_type is not None:
         try:
             agent_type = tuple(agent_type)

soil/exporters.py

@@ -21,6 +21,7 @@ def for_sim(simulation, names, *args, **kwargs):
         exporters.append(mod(simulation, *args, **kwargs))
     return exporters
 
+
 class DryRunner(BytesIO):
     def __init__(self, fname, *args, copy_to=None, **kwargs):
         super().__init__(*args, **kwargs)
@@ -49,7 +50,7 @@ class Exporter:
 
     def __init__(self, simulation, outdir=None, dry_run=None, copy_to=None):
         self.sim = simulation
-        outdir = outdir or os.getcwd()
+        outdir = outdir or os.path.join(os.getcwd(), 'soil_output')
         self.outdir = os.path.join(outdir,
                                    simulation.group or '',
                                    simulation.name)
@@ -77,8 +78,8 @@ class Exporter:
         return open_or_reuse(f, mode=mode, **kwargs)
 
 
-class Default(Exporter):
+class default(Exporter):
-    '''Default exporter. Writes CSV and sqlite results, as well as the simulation YAML'''
+    '''Default exporter. Writes sqlite results, as well as the simulation YAML'''
 
     def start(self):
         if not self.dry_run:
@@ -95,25 +96,29 @@ class Default(Exporter):
                     env.dump_sqlite(f)
 
 
-class CSV(Exporter):
+class csv(Exporter):
+    '''Export the state of each environment (and its agents) in a separate CSV file'''
     def trial_end(self, env):
-        if not self.dry_run:
+        with timer('[CSV] Dumping simulation {} trial {} @ dir {}'.format(self.sim.name,
-            with timer('[CSV] Dumping simulation {} trial {}'.format(self.sim.name,
+                                                                          env.name,
-                                                               env.name)):
+                                                                          self.outdir)):
-                with self.output('{}.csv'.format(env.name)) as f:
+            with self.output('{}.csv'.format(env.name)) as f:
-                    env.dump_csv(f)
+                env.dump_csv(f)
 
 
-class Gexf(Exporter):
+class gexf(Exporter):
     def trial_end(self, env):
-        if not self.dry_run:
+        if self.dry_run:
-            with timer('[CSV] Dumping simulation {} trial {}'.format(self.sim.name,
+            logger.info('Not dumping GEXF in dry_run mode')
-                                                                     env.name)):
+            return
-                with self.output('{}.gexf'.format(env.name), mode='wb') as f:
+
-                    env.dump_gexf(f)
+        with timer('[GEXF] Dumping simulation {} trial {}'.format(self.sim.name,
+                                                                  env.name)):
+            with self.output('{}.gexf'.format(env.name), mode='wb') as f:
+                env.dump_gexf(f)
 
 
-class Dummy(Exporter):
+class dummy(Exporter):
 
     def start(self):
         with self.output('dummy', 'w') as f:
@@ -130,7 +135,7 @@ class Dummy(Exporter):
             f.write('simulation ended @ {}\n'.format(time.time()))
 
 
-class Distribution(Exporter):
+class distribution(Exporter):
     '''
     Write the distribution of agent states at the end of each trial,
     the mean value, and its deviation.
@@ -164,7 +169,7 @@ class Distribution(Exporter):
         with self.output('metrics.csv') as f:
             dfm.to_csv(f)
 
-class GraphDrawing(Exporter):
+class graphdrawing(Exporter):
 
     def trial_end(self, env):
         # Outside effects

soil/history.py

@@ -11,6 +11,7 @@ logger = logging.getLogger(__name__)
 from collections import UserDict, namedtuple
 
 from . import serialization
+from .utils import open_or_reuse
 
 
 class History:
@@ -236,7 +237,7 @@ class History:
 
     def dump(self, f):
         self._close()
-        for line in open(self.db_path, 'rb'):
+        for line in open_or_reuse(self.db_path, 'rb'):
             f.write(line)
 
 

soil/serialization.py

@@ -55,7 +55,7 @@ def load_file(infile):
 
 
 def load_string(string):
-    yield from yaml.load_all(string)
+    yield from yaml.load_all(string, Loader=yaml.FullLoader)
 
 
 def expand_template(config):
@@ -186,7 +186,7 @@ def deserializer(type_, known_modules=[]):
             module = importlib.import_module(modname)
             cls = getattr(module, tname)
             return getattr(cls, 'deserialize', cls)
-        except (ModuleNotFoundError, AttributeError) as ex:
+        except (ImportError, AttributeError) as ex:
             errors.append((modname, tname, ex))
     raise Exception('Could not find type {}. Tried: {}'.format(type_, errors))
 

soil/simulation.py

@@ -153,11 +153,11 @@ class Simulation(NetworkSimulation):
                                      **kwargs)
 
     def _run_simulation_gen(self, *args, parallel=False, dry_run=False,
-                            exporters=None, outdir=None, exporter_params={}, **kwargs):
+                            exporters=['default', ], outdir=None, exporter_params={}, **kwargs):
         logger.info('Using exporters: %s', exporters or [])
         logger.info('Output directory: %s', outdir)
         exporters = exporters_for_sim(self,
-                                      exporters or [],
+                                      exporters,
                                       dry_run=dry_run,
                                       outdir=outdir,
                                       **exporter_params)

soil/utils.py

@@ -2,6 +2,8 @@ import logging
 import time
 import os
 
+from shutil import copyfile
+
 from contextlib import contextmanager
 
 logger = logging.getLogger('soil')
@@ -23,15 +25,26 @@ def timer(name='task', pre="", function=logger.info, to_object=None):
         to_object.end = end
 
 
-def safe_open(path, *args, **kwargs):
+def safe_open(path, mode='r', backup=True, **kwargs):
     outdir = os.path.dirname(path)
     if outdir and not os.path.exists(outdir):
         os.makedirs(outdir)
-    return open(path, *args, **kwargs)
+    if backup and 'w' in mode and os.path.exists(path):
+        creation = os.path.getctime(path)
+        stamp = time.strftime('%Y-%m-%d_%H:%M', time.localtime(creation))
+
+        backup_dir = os.path.join(outdir, stamp)
+        if not os.path.exists(backup_dir):
+            os.makedirs(backup_dir)
+        newpath = os.path.join(backup_dir, os.path.basename(path))
+        if os.path.exists(newpath):
+            newpath = '{}@{}'.format(newpath, time.time())
+        copyfile(path, newpath)
+    return open(path, mode=mode, **kwargs)
 
 
 def open_or_reuse(f, *args, **kwargs):
     try:
         return safe_open(f, *args, **kwargs)
-    except TypeError:
+    except (AttributeError, TypeError):
         return f

tests/test_exporters.py

@@ -0,0 +1,110 @@
+import os
+import io
+import tempfile
+import shutil
+from time import time
+
+from unittest import TestCase
+from soil import exporters
+from soil.utils import safe_open
+from soil import simulation
+
+
+class Dummy(exporters.Exporter):
+    started = False
+    trials = 0
+    ended = False
+    total_time = 0
+
+    def start(self):
+        self.__class__.started = True
+
+    def trial_end(self, env):
+        assert env
+        self.__class__.trials += 1
+        self.__class__.total_time += env.now
+
+    def end(self):
+        self.__class__.ended = True
+
+
+class Exporters(TestCase):
+    def test_basic(self):
+        config = {
+            'name': 'exporter_sim',
+            'network_params': {},
+            'agent_type': 'CounterModel',
+            'max_time': 2,
+            'num_trials': 5,
+            'environment_params': {}
+        }
+        s = simulation.from_config(config)
+        s.run_simulation(exporters=[Dummy], dry_run=True)
+        assert Dummy.started
+        assert Dummy.ended
+        assert Dummy.trials == 5
+        assert Dummy.total_time == 2*5
+
+    def test_distribution(self):
+        '''The distribution exporter should write the number of agents in each state'''
+        config = {
+            'name': 'exporter_sim',
+            'network_params': {
+                'generator': 'complete_graph',
+                'n': 4
+            },
+            'agent_type': 'CounterModel',
+            'max_time': 2,
+            'num_trials': 5,
+            'environment_params': {}
+        }
+        output = io.StringIO()
+        s = simulation.from_config(config)
+        s.run_simulation(exporters=[exporters.distribution], dry_run=True, exporter_params={'copy_to': output})
+        result = output.getvalue()
+        assert 'count' in result
+        assert 'SEED,Noneexporter_sim_trial_3,1,,1,1,1,1' in result
+
+    def test_writing(self):
+        '''Try to write CSV, GEXF, sqlite and YAML (without dry_run)'''
+        n_trials = 5
+        config = {
+            'name': 'exporter_sim',
+            'network_params': {
+                'generator': 'complete_graph',
+                'n': 4
+            },
+            'agent_type': 'CounterModel',
+            'max_time': 2,
+            'num_trials': n_trials,
+            'environment_params': {}
+        }
+        output = io.StringIO()
+        s = simulation.from_config(config)
+        tmpdir = tempfile.mkdtemp()
+        envs = s.run_simulation(exporters=[
+            exporters.default,
+            exporters.csv,
+            exporters.gexf,
+            exporters.distribution,
+        ],
+                                outdir=tmpdir,
+                                exporter_params={'copy_to': output})
+        result = output.getvalue()
+
+        simdir = os.path.join(tmpdir, s.group or '', s.name)
+        with open(os.path.join(simdir, '{}.dumped.yml'.format(s.name))) as f:
+            result = f.read()
+            assert result
+
+        try:
+            for e in envs:
+                with open(os.path.join(simdir, '{}.gexf'.format(e.name))) as f:
+                    result = f.read()
+                    assert result
+
+                with open(os.path.join(simdir, '{}.csv'.format(e.name))) as f:
+                    result = f.read()
+                    assert result
+        finally:
+            shutil.rmtree(tmpdir)

tests/test_main.py

@@ -1,6 +1,7 @@
 from unittest import TestCase
 
 import os
+import io
 import yaml
 import pickle
 import networkx as nx
@@ -15,10 +16,15 @@ ROOT = os.path.abspath(os.path.dirname(__file__))
 EXAMPLES = join(ROOT, '..', 'examples')
 
 
-class CustomAgent(agents.BaseAgent):
+class CustomAgent(agents.FSM):
-    def step(self):
+    @agents.default_state
-        self.state['neighbors'] = self.count_agents(state_id=0,
+    @agents.state
+    def normal(self):
+        self.state['neighbors'] = self.count_agents(state_id='normal',
                                                     limit_neighbors=True)
+    @agents.state
+    def unreachable(self):
+        return
 
 class TestMain(TestCase):
 
@@ -133,8 +139,7 @@ class TestMain(TestCase):
             },
             'network_agents': [{
                 'agent_type': CustomAgent,
-                'weight': 1,
+                'weight': 1
-                'state': {'id': 0}
 
             }],
             'max_time': 10,
@@ -144,6 +149,9 @@ class TestMain(TestCase):
         s = simulation.from_config(config)
         env = s.run_simulation(dry_run=True)[0]
         assert env.get_agent(0).state['neighbors'] == 1
+        assert env.get_agent(0).state['neighbors'] == 1
+        assert env.get_agent(1).count_agents(state_id='normal') == 2
+        assert env.get_agent(1).count_agents(state_id='normal', limit_neighbors=True) == 1
 
     def test_torvalds_example(self):
         """A complete example from a documentation should work."""
@@ -217,7 +225,8 @@ class TestMain(TestCase):
         """
         G = nx.random_geometric_graph(20, 0.1)
         env = Environment(topology=G)
-        env.dump_gexf('/tmp/dump-gexf/prueba.gexf')
+        f = io.BytesIO()
+        env.dump_gexf(f)
 
     def test_save_graph(self):
         '''