Fix gephi representation. Add sqlite

examples/complete.yml

@@ -4,6 +4,8 @@ dir_path: "/tmp/"
 num_trials: 3
 max_time: 100
 interval: 1
+seed: "CompleteSeed!"
+dump: false
 network_params:
   generator: complete_graph
   n: 10
@@ -21,4 +23,4 @@ default_state:
   incidents: 0
 states:
   - name: 'The first node'
-  - name: 'The second node'
+  - name: 'The second node'

examples/rabbits/rabbit_agents.py

@@ -1,10 +1,10 @@
-import logging
 from soil.agents import NetworkAgent, FSM, state, default_state, BaseAgent
 from enum import Enum
 from random import random, choice
 from itertools import islice
+import logging
+import math
 
-logger = logging.getLogger(__name__)
 
 class Genders(Enum):
     male = 'male'
@@ -13,6 +13,8 @@ class Genders(Enum):
 
 class RabbitModel(NetworkAgent, FSM):
 
+    level = logging.INFO
+
     defaults = {
         'age': 0,
         'gender': Genders.male.value,
@@ -24,7 +26,7 @@ class RabbitModel(NetworkAgent, FSM):
     life_expectancy = 365 * 3
     gestation = 33
     pregnancy = -1
-    max_females = 10
+    max_females = 2
 
     @default_state
     @state
@@ -57,7 +59,7 @@ class RabbitModel(NetworkAgent, FSM):
         whom['pregnancy'] = 0
         whom['mate'] = self.id
         whom.set_state(whom.pregnant)
-        logger.debug('{} impregnating: {}. {}'.format(self.id, whom.id, whom.state))
+        self.debug('{} impregnating: {}. {}'.format(self.id, whom.id, whom.state))
 
     @state
     def pregnant(self):
@@ -66,38 +68,52 @@ class RabbitModel(NetworkAgent, FSM):
             return self.dead
 
         self['pregnancy'] += 1
-        logger.debug('Pregnancy: {}'.format(self['pregnancy']))
+        self.debug('Pregnancy: {}'.format(self['pregnancy']))
         if self['pregnancy'] >= self.gestation:
-
-            state = {}
-            state['gender'] = choice(list(Genders)).value
-            child = self.env.add_node(self.__class__, state)
-            self.env.add_edge(self.id, child.id)
-            self.env.add_edge(self['mate'], child.id)
-            # self.add_edge()
-            logger.info("A rabbit has been born: {}. Total: {}".format(child.id, len(self.global_topology.nodes)))
-            self['offspring'] += 1
-            self.env.get_agent(self['mate'])['offspring'] += 1
-            del self['mate']
-            self['pregnancy'] = -1
-            return self.fertile
+            number_of_babies = int(8+4*random())
+            for i in range(number_of_babies):
+                state = {}
+                state['gender'] = choice(list(Genders)).value
+                child = self.env.add_node(self.__class__, state)
+                self.env.add_edge(self.id, child.id)
+                self.env.add_edge(self['mate'], child.id)
+                # self.add_edge()
+                self.debug('A BABY IS COMING TO LIFE')
+                self.env['rabbits_alive'] = self.env.get('rabbits_alive', 0)+1
+                self.debug('Rabbits alive: {}'.format(self.env['rabbits_alive']))
+                self['offspring'] += 1
+                self.env.get_agent(self['mate'])['offspring'] += 1
+                del self['mate']
+                self['pregnancy'] = -1
+                return self.fertile
 
     @state
     def dead(self):
-        logger.info('Agent {} is dying'.format(self.id))
+        self.info('Agent {} is dying'.format(self.id))
         if 'pregnancy' in self and self['pregnancy'] > -1:
-            logger.info('A mother has died carrying a baby!: {}!'.format(self.state))
+            self.info('A mother has died carrying a baby!!')
         self.die()
         return
 
 
 class RandomAccident(BaseAgent):
 
+    level = logging.INFO
+
     def step(self):
-        logger.debug('Killing some rabbits!')
-        prob_death = self.env.get('prob_death', -1)
+        rabbits_total = self.global_topology.number_of_nodes()
+        rabbits_alive = self.env.get('rabbits_alive', rabbits_total)
+        prob_death = self.env.get('prob_death', 1e-100)*math.log(max(1, rabbits_alive))
+        self.debug('Killing some rabbits with prob={}!'.format(prob_death))
         for i in self.env.network_agents:
+            if i.state['id'] == i.dead.id:
+                continue
             r = random()
             if r < prob_death:
-                logger.info('I killed a rabbit: {}'.format(i.id))
+                self.debug('I killed a rabbit: {}'.format(i.id))
+                rabbits_alive = self.env['rabbits_alive'] = rabbits_alive -1
+                self.log('Rabbits alive: {}'.format(self.env['rabbits_alive']))
                 i.set_state(i.dead)
+        self.log('Rabbits alive: {}/{}'.format(rabbits_alive, rabbits_total))
+        if self.count_agents(state_id=RabbitModel.dead.id) == self.global_topology.number_of_nodes():
+            self.die()

examples/rabbits/rabbits.yml

@@ -1,14 +1,16 @@
 ---
-load_module: custom_agents
-name: custom_agent_example
-max_time: 2500
+load_module: rabbit_agents
+name: rabbits_example
+max_time: 1500
 interval: 1
-seed: MySimulationSeed
+seed: MySeed
 agent_type: RabbitModel
 environment_agents:
     - agent_type: RandomAccident
+environment_params:
+  prob_death: 0.0001
 default_state:
-  mating_prob: 1
+  mating_prob: 0.01
 topology:
   nodes:
     - id: 1

soil/__init__.py

@@ -11,11 +11,8 @@ try:
 except NameError:
     basestring = str
 
-from . import agents
-from . import simulation
-from . import environment
+logging.basicConfig()#format=FORMAT)
 from . import utils
-from . import settings
 
 
 def main():
@@ -42,9 +39,8 @@ def main():
         sys.path.append(os.getcwd())
         importlib.import_module(args.module)
 
-    logging.basicConfig(level=logging.INFO)
-    logger = logging.getLogger(__name__)
-    logger.info('Loading config file: {}'.format(args.file, args.output))
+    logging.info('Loading config file: {}'.format(args.file, args.output))
+
     try:
         simulation.run_from_config(args.file, dump=(not args.dry_run), results_dir=args.output)
     except Exception as ex:

soil/agents/CounterModel.py

@@ -9,8 +9,8 @@ class CounterModel(NetworkAgent):
 
     def step(self):
         # Outside effects
-        total = len(self.get_all_agents())
-        neighbors = len(self.get_neighboring_agents())
+        total = len(list(self.get_all_agents()))
+        neighbors = len(list(self.get_neighboring_agents()))
         self.state['times'] = self.state.get('times', 0) + 1
         self.state['neighbors'] = neighbors
         self.state['total'] = total
@@ -24,8 +24,8 @@ class AggregatedCounter(NetworkAgent):
 
     def step(self):
         # Outside effects
-        total = len(self.get_all_agents())
-        neighbors = len(self.get_neighboring_agents())
+        total = len(list(self.get_all_agents()))
+        neighbors = len(list(self.get_neighboring_agents()))
         self.state['times'] = self.state.get('times', 0) + 1
         self.state['neighbors'] = self.state.get('neighbors', 0) + neighbors
         self.state['total'] = self.state.get('total', 0) + total

soil/agents/__init__.py

@@ -6,11 +6,13 @@
     
 
 import nxsim
+import logging
 from collections import OrderedDict
 from copy import deepcopy
 from functools import partial
 import json
 
+
 from functools import wraps
 
 
@@ -37,11 +39,16 @@ class BaseAgent(nxsim.BaseAgent, metaclass=MetaAgent):
         state.update(kwargs.pop('state', {}))
         kwargs['state'] = state
         super().__init__(**kwargs)
+        if not hasattr(self, 'level'):
+            self.level = logging.DEBUG
+        self.logger = logging.getLogger('Agent-{}'.format(self.id))
+        self.logger.setLevel(self.level)
+
 
     def __getitem__(self, key):
         if isinstance(key, tuple):
             k, t_step = key
-            return self.env[t_step, self.id, k]
+            return self.env[self.id, t_step, k]
         return self.state.get(key, None)
 
     def __delitem__(self, key):
@@ -78,7 +85,7 @@ class BaseAgent(nxsim.BaseAgent, metaclass=MetaAgent):
         pass
 
     def to_json(self):
-        return json.dumps(self._history)
+        return json.dumps(self.state)
 
     def count_agents(self, state_id=None, limit_neighbors=False):
         if limit_neighbors:
@@ -108,6 +115,20 @@ class BaseAgent(nxsim.BaseAgent, metaclass=MetaAgent):
 
         return filter(matches_all, agents)
 
+    def log(self, message, level=logging.INFO, **kwargs):
+        message = "\t@{:>5}:\t{}".format(self.now, message)
+        for k, v in kwargs:
+            message += " {k}={v} ".format(k, v)
+        extra = {}
+        extra['now'] = self.now
+        extra['id'] = self.id
+        return self.logger.log(level, message, extra=extra)
+
+    def debug(self, *args, **kwargs):
+        return self.log(*args, level=logging.DEBUG, **kwargs)
+
+    def info(self, *args, **kwargs):
+        return self.log(*args, level=logging.INFO, **kwargs)
 
 class NetworkAgent(BaseAgent, nxsim.BaseNetworkAgent):
 

soil/environment.py

@@ -1,10 +1,11 @@
 import os
+import sqlite3
 import time
-import csv
 import weakref
+import csv
 import random
+import simpy
 from copy import deepcopy
-from functools import partial
 
 import networkx as nx
 import nxsim
@@ -22,15 +23,19 @@ class SoilEnvironment(nxsim.NetworkEnvironment):
                  interval=1,
                  seed=None,
                  dump=False,
+                 simulation=None,
                  *args, **kwargs):
         self.name = name or 'UnnamedEnvironment'
-        self.states = deepcopy(states) or {}
+        if isinstance(states, list):
+            states = dict(enumerate(states))
+        self.states = deepcopy(states) if states else {}
         self.default_state = deepcopy(default_state) or {}
+        self.sim = weakref.ref(simulation)
+        if 'topology' not in kwargs and simulation:
+            kwargs['topology'] = self.sim().topology.copy()
         super().__init__(*args, **kwargs)
         self._env_agents = {}
-        self._history = {}
         self.interval = interval
-        self.logger = None
         self.dump = dump
         # Add environment agents first, so their events get
         # executed before network agents
@@ -39,6 +44,17 @@ class SoilEnvironment(nxsim.NetworkEnvironment):
         self.environment_agents = environment_agents or []
         self.network_agents = network_agents or []
         self.process(self.save_state())
+        if self.dump:
+            self._db_path = os.path.join(self.get_path(), 'db.sqlite')
+        else:
+            self._db_path = ":memory:"
+        self.create_db(self._db_path)
+
+    def create_db(self, db_path=None):
+        db_path = db_path or self._db_path
+        self._db = sqlite3.connect(db_path)
+        with self._db:
+            self._db.execute('''CREATE TABLE IF NOT EXISTS history (agent_id text, t_step int, key text, value text, value_type text)''')
 
     @property
     def agents(self):
@@ -48,7 +64,7 @@ class SoilEnvironment(nxsim.NetworkEnvironment):
     @property
     def environment_agents(self):
         for ref in self._env_agents.values():
-            yield ref()
+            yield ref
 
     @environment_agents.setter
     def environment_agents(self, environment_agents):
@@ -60,7 +76,7 @@ class SoilEnvironment(nxsim.NetworkEnvironment):
             kwargs['agent_id'] = kwargs.get('agent_id', atype.__name__)
             kwargs['state'] = kwargs.get('state', {})
             a = atype(environment=self, **kwargs)
-            self._env_agents[a.id] = weakref.ref(a)
+            self._env_agents[a.id] = a
 
     @property
     def network_agents(self):
@@ -106,42 +122,72 @@ class SoilEnvironment(nxsim.NetworkEnvironment):
         super().run(*args, **kwargs)
         self._save_state()
 
-    def _save_state(self):
+    def _save_state(self, now=None):
         # for agent in self.agents:
         #     agent.save_state()
-        nowd = self._history[self.now] = {}
-        nowd['env'] = deepcopy(self.environment_params)
-        for agent in self.agents:
-            nowd[agent.id] = deepcopy(agent.state)
+        with self._db:
+            self._db.executemany("insert into history(agent_id, t_step, key, value, value_type) values (?, ?, ?, ?, ?)", self.state_to_tuples(now=now))
 
     def save_state(self):
-        while True:
+        while self.peek() != simpy.core.Infinity:
+            utils.logger.info('Step: {}'.format(self.now))
             ev = self.event()
             ev._ok = True
             # Schedule the event with minimum priority so
             # that it executes after all agents are done
-            self.schedule(ev, -1, self.interval)
+            self.schedule(ev, -1, self.peek())
             yield ev
             self._save_state()
 
     def __getitem__(self, key):
         if isinstance(key, tuple):
-            t_step, agent_id, k = key
+            values = {"agent_id": key[0],
+                      "t_step": key[1],
+                      "key": key[2],
+                      "value": None,
+                      "value_type": None
+            }
 
-            def key_or_dict(d, k, nfunc):
-                if k is None:
-                    if d is None:
-                        return {}
-                    return {k: nfunc(v) for k, v in d.items()}
-                if k in d:
-                    return nfunc(d[k])
-                return {}
+            fields = list(k for k, v in values.items() if v is None)
+            conditions = " and ".join("{}='{}'".format(k, v) for k, v in values.items() if v is not None)
+
+            query = """SELECT {fields} from history""".format(fields=",".join(fields))
+            if conditions:
+                query = """{query} where {conditions}""".format(query=query,
+                                                                conditions=conditions)
+            with self._db:
+                rows = self._db.execute(query).fetchall()
+
+            utils.logger.debug(rows)
+            results = self.rows_to_dict(rows)
+            return results
 
-            f1 = partial(key_or_dict, k=k, nfunc=lambda x: x)
-            f2 = partial(key_or_dict, k=agent_id, nfunc=f1)
-            return key_or_dict(self._history, t_step, f2)
         return self.environment_params[key]
 
+    def rows_to_dict(self, rows):
+        if len(rows) < 1:
+            return None
+
+        level = len(rows[0])-2
+
+        if level == 0:
+            if len(rows) != 1:
+                raise ValueError('Cannot convert {} to dictionaries'.format(rows))
+            value, value_type = rows[0]
+            return utils.convert(value, value_type)
+
+        results = {}
+        for row in rows:
+            item = results
+            for i in range(level-1):
+                key = row[i]
+                if key not in item:
+                    item[key] = {}
+                item = item[key]
+            key, value, value_type = row[level-1:]
+            item[key] = utils.convert(value, value_type)
+        return results
+
     def __setitem__(self, key, value):
         self.environment_params[key] = value
 
@@ -179,23 +225,34 @@ class SoilEnvironment(nxsim.NetworkEnvironment):
                                   self.name+".gexf")
         nx.write_gexf(G, graph_path, version="1.2draft")
 
+    def state_to_tuples(self, now=None):
+        if now is None:
+            now = self.now
+        for k, v in self.environment_params.items():
+            yield 'env', now, k, v, type(v).__name__
+        for agent in self.agents:
+            for k, v in agent.state.items():
+                yield agent.id, now, k, v, type(v).__name__
+
     def history_to_tuples(self):
-        for tstep, states in self._history.items():
-            for a_id, state in states.items():
-                for attribute, value in state.items():
-                    yield (a_id, tstep, attribute, value)
+        with self._db:
+            res = self._db.execute("select agent_id, t_step, key, value from history ").fetchall()
+        yield from res
 
     def history_to_graph(self):
         G = nx.Graph(self.G)
 
-        for agent in self.agents:
+        for agent in self.network_agents:
 
             attributes = {'agent': str(agent.__class__)}
             lastattributes = {}
             spells = []
             lastvisible = False
             laststep = None
-            for t_step, state in reversed(list(self[None, agent.id, None].items())):
+            history = self[agent.id, None, None]
+            if not history:
+                continue
+            for t_step, state in reversed(sorted(list(history.items()))):
                 for attribute, value in state.items():
                     if attribute == 'visible':
                         nowvisible = state[attribute]
@@ -206,15 +263,20 @@ class SoilEnvironment(nxsim.NetworkEnvironment):
 
                         lastvisible = nowvisible
                     else:
-                        if attribute not in lastattributes or lastattributes[attribute][0] != value:
-                            laststep = lastattributes.get(attribute,
-                                                          (None, None))[1]
-                            value = (state[attribute], t_step, laststep)
-                            key = 'attr_' + attribute
+                        key = 'attr_' + attribute
+                        if key not in attributes:
+                            attributes[key] = list()
+                        if key not in lastattributes:
+                            lastattributes[key] = (state[attribute], t_step)
+                        elif lastattributes[key][0] != value:
+                            last_value, laststep = lastattributes[key]
+                            value = (last_value, t_step, laststep)
                             if key not in attributes:
                                 attributes[key] = list()
                             attributes[key].append(value)
-                            lastattributes[attribute] = (state[attribute], t_step)
+                            lastattributes[key] = (state[attribute], t_step)
+            for k, v in lastattributes.items():
+                attributes[k].append((v[0], 0, v[1]))
             if lastvisible:
                 spells.append((laststep, None))
             if spells:

soil/simulation.py

@@ -1,10 +1,8 @@
-import weakref
 import os
 import time
 import imp
 import sys
 import yaml
-import logging
 import networkx as nx
 from networkx.readwrite import json_graph
 
@@ -15,8 +13,8 @@ import pickle
 from nxsim import NetworkSimulation
 
 from . import agents, utils, environment, basestring
+from .utils import logger
 
-logger = logging.getLogger(__name__)
 
 class SoilSimulation(NetworkSimulation):
     """
@@ -86,7 +84,7 @@ class SoilSimulation(NetworkSimulation):
         self.network_agents = self._convert_agent_types(distro)
 
         self.states = self.validate_states(states,
-                                           topology)
+                                           self.topology)
 
     def calculate_distribution(self,
                                network_agents=None,
@@ -178,9 +176,8 @@ class SoilSimulation(NetworkSimulation):
                                           states=self.states,
                                           default_state=self.default_state,
                                           environment_agents=self.environment_agents,
+                                          simulation=self,
                                           **self.environment_params)
-
-        env.sim = weakref.ref(self)
         # Set up agents on nodes
         logger.info('\tRunning')
         with utils.timer('trial'):

soil/utils.py

@@ -10,10 +10,15 @@ import networkx as nx
 
 from contextlib import contextmanager
 
+
 logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+logger.addHandler(logging.StreamHandler())
 
 
 def load_network(network_params, dir_path=None):
+    if network_params is None:
+        return nx.Graph()
     path = network_params.get('path', None)
     if path:
         if dir_path and not os.path.isabs(path):
@@ -73,9 +78,23 @@ def agent_from_distribution(distribution, value=-1):
     for d in distribution:
         threshold = d['threshold']
         if value >= threshold[0] and value < threshold[1]:
-            state = None
+            state = {}
             if 'state' in d:
                 state = deepcopy(d['state'])
             return d['agent_type'], state
 
     raise Exception('Distribution for value {} not found in: {}'.format(value, distribution))
+
+
+def convert(value, type_):
+    import importlib
+    try:
+        # Check if it's a builtin type
+        module = importlib.import_module('builtins')
+        cls = getattr(module, type_)
+    except AttributeError:
+        # if not, separate module and class
+        module, type_ = type_.rsplit(".", 1)
+        module = importlib.import_module(module)
+        cls = getattr(module, type_)
+    return cls(value)

tests/test_main.py

@@ -5,7 +5,7 @@ import yaml
 from functools import partial
 
 from os.path import join
-from soil import simulation, agents, utils
+from soil import simulation, environment, agents, utils
 
 
 ROOT = os.path.abspath(os.path.dirname(__file__))
@@ -198,6 +198,26 @@ class TestMain(TestCase):
         """
         Make sure all examples in the examples folder are correct
         """
+        pass
+
+    def test_row_conversion(self):
+        sim = simulation.SoilSimulation()
+        env = environment.SoilEnvironment(simulation=sim)
+        env['test'] = 'test_value'
+        env._save_state(now=0)
+
+        res = list(env.history_to_tuples())
+        assert len(res) == len(env.environment_params)
+        assert ('env', 0, 'test', 'test_value') in res
+
+        env['test'] = 'second_value'
+        env._save_state(now=1)
+        res = list(env.history_to_tuples())
+
+        assert env['env', 0, 'test' ] == 'test_value'
+        assert env['env', 1, 'test' ] == 'second_value'
+
+
 
 
 def make_example_test(path, config):