From 78364d89d531b53861903de68313c41b3232c200 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=2E=20Fernando=20S=C3=A1nchez?= Date: Tue, 17 Oct 2017 19:48:56 +0200 Subject: [PATCH] Fix gephi representation. Add sqlite --- examples/complete.yml | 4 +- .../rabbit_agents.py} | 62 +++++--- examples/{custom.yml => rabbits/rabbits.yml} | 12 +- soil/__init__.py | 10 +- soil/agents/CounterModel.py | 8 +- soil/agents/__init__.py | 25 +++- soil/environment.py | 138 +++++++++++++----- soil/simulation.py | 9 +- soil/utils.py | 21 ++- tests/test_main.py | 22 ++- 10 files changed, 223 insertions(+), 88 deletions(-) rename examples/{custom_agents.py => rabbits/rabbit_agents.py} (50%) rename examples/{custom.yml => rabbits/rabbits.yml} (64%) diff --git a/examples/complete.yml b/examples/complete.yml index 7b52842..60b6639 100644 --- a/examples/complete.yml +++ b/examples/complete.yml @@ -4,6 +4,8 @@ dir_path: "/tmp/" num_trials: 3 max_time: 100 interval: 1 +seed: "CompleteSeed!" +dump: false network_params: generator: complete_graph n: 10 @@ -21,4 +23,4 @@ default_state: incidents: 0 states: - name: 'The first node' - - name: 'The second node' \ No newline at end of file + - name: 'The second node' diff --git a/examples/custom_agents.py b/examples/rabbits/rabbit_agents.py similarity index 50% rename from examples/custom_agents.py rename to examples/rabbits/rabbit_agents.py index dcf8e2b..51977e1 100644 --- a/examples/custom_agents.py +++ b/examples/rabbits/rabbit_agents.py @@ -1,10 +1,10 @@ -import logging from soil.agents import NetworkAgent, FSM, state, default_state, BaseAgent from enum import Enum from random import random, choice from itertools import islice +import logging +import math -logger = logging.getLogger(__name__) class Genders(Enum): male = 'male' @@ -13,6 +13,8 @@ class Genders(Enum): class RabbitModel(NetworkAgent, FSM): + level = logging.INFO + defaults = { 'age': 0, 'gender': Genders.male.value, @@ -24,7 +26,7 @@ class RabbitModel(NetworkAgent, FSM): life_expectancy = 365 * 3 gestation = 33 pregnancy = -1 - max_females = 10 + max_females = 2 @default_state @state @@ -57,7 +59,7 @@ class RabbitModel(NetworkAgent, FSM): whom['pregnancy'] = 0 whom['mate'] = self.id whom.set_state(whom.pregnant) - logger.debug('{} impregnating: {}. {}'.format(self.id, whom.id, whom.state)) + self.debug('{} impregnating: {}. {}'.format(self.id, whom.id, whom.state)) @state def pregnant(self): @@ -66,38 +68,52 @@ class RabbitModel(NetworkAgent, FSM): return self.dead self['pregnancy'] += 1 - logger.debug('Pregnancy: {}'.format(self['pregnancy'])) + self.debug('Pregnancy: {}'.format(self['pregnancy'])) if self['pregnancy'] >= self.gestation: - - state = {} - state['gender'] = choice(list(Genders)).value - child = self.env.add_node(self.__class__, state) - self.env.add_edge(self.id, child.id) - self.env.add_edge(self['mate'], child.id) - # self.add_edge() - logger.info("A rabbit has been born: {}. Total: {}".format(child.id, len(self.global_topology.nodes))) - self['offspring'] += 1 - self.env.get_agent(self['mate'])['offspring'] += 1 - del self['mate'] - self['pregnancy'] = -1 - return self.fertile + number_of_babies = int(8+4*random()) + for i in range(number_of_babies): + state = {} + state['gender'] = choice(list(Genders)).value + child = self.env.add_node(self.__class__, state) + self.env.add_edge(self.id, child.id) + self.env.add_edge(self['mate'], child.id) + # self.add_edge() + self.debug('A BABY IS COMING TO LIFE') + self.env['rabbits_alive'] = self.env.get('rabbits_alive', 0)+1 + self.debug('Rabbits alive: {}'.format(self.env['rabbits_alive'])) + self['offspring'] += 1 + self.env.get_agent(self['mate'])['offspring'] += 1 + del self['mate'] + self['pregnancy'] = -1 + return self.fertile @state def dead(self): - logger.info('Agent {} is dying'.format(self.id)) + self.info('Agent {} is dying'.format(self.id)) if 'pregnancy' in self and self['pregnancy'] > -1: - logger.info('A mother has died carrying a baby!: {}!'.format(self.state)) + self.info('A mother has died carrying a baby!!') self.die() return class RandomAccident(BaseAgent): + level = logging.INFO + def step(self): - logger.debug('Killing some rabbits!') - prob_death = self.env.get('prob_death', -1) + rabbits_total = self.global_topology.number_of_nodes() + rabbits_alive = self.env.get('rabbits_alive', rabbits_total) + prob_death = self.env.get('prob_death', 1e-100)*math.log(max(1, rabbits_alive)) + self.debug('Killing some rabbits with prob={}!'.format(prob_death)) for i in self.env.network_agents: + if i.state['id'] == i.dead.id: + continue r = random() if r < prob_death: - logger.info('I killed a rabbit: {}'.format(i.id)) + self.debug('I killed a rabbit: {}'.format(i.id)) + rabbits_alive = self.env['rabbits_alive'] = rabbits_alive -1 + self.log('Rabbits alive: {}'.format(self.env['rabbits_alive'])) i.set_state(i.dead) + self.log('Rabbits alive: {}/{}'.format(rabbits_alive, rabbits_total)) + if self.count_agents(state_id=RabbitModel.dead.id) == self.global_topology.number_of_nodes(): + self.die() diff --git a/examples/custom.yml b/examples/rabbits/rabbits.yml similarity index 64% rename from examples/custom.yml rename to examples/rabbits/rabbits.yml index 403ad8e..a2a89db 100644 --- a/examples/custom.yml +++ b/examples/rabbits/rabbits.yml @@ -1,14 +1,16 @@ --- -load_module: custom_agents -name: custom_agent_example -max_time: 2500 +load_module: rabbit_agents +name: rabbits_example +max_time: 1500 interval: 1 -seed: MySimulationSeed +seed: MySeed agent_type: RabbitModel environment_agents: - agent_type: RandomAccident +environment_params: + prob_death: 0.0001 default_state: - mating_prob: 1 + mating_prob: 0.01 topology: nodes: - id: 1 diff --git a/soil/__init__.py b/soil/__init__.py index f916204..fc0732b 100644 --- a/soil/__init__.py +++ b/soil/__init__.py @@ -11,11 +11,8 @@ try: except NameError: basestring = str -from . import agents -from . import simulation -from . import environment +logging.basicConfig()#format=FORMAT) from . import utils -from . import settings def main(): @@ -42,9 +39,8 @@ def main(): sys.path.append(os.getcwd()) importlib.import_module(args.module) - logging.basicConfig(level=logging.INFO) - logger = logging.getLogger(__name__) - logger.info('Loading config file: {}'.format(args.file, args.output)) + logging.info('Loading config file: {}'.format(args.file, args.output)) + try: simulation.run_from_config(args.file, dump=(not args.dry_run), results_dir=args.output) except Exception as ex: diff --git a/soil/agents/CounterModel.py b/soil/agents/CounterModel.py index 186ed45..851f3c2 100644 --- a/soil/agents/CounterModel.py +++ b/soil/agents/CounterModel.py @@ -9,8 +9,8 @@ class CounterModel(NetworkAgent): def step(self): # Outside effects - total = len(self.get_all_agents()) - neighbors = len(self.get_neighboring_agents()) + total = len(list(self.get_all_agents())) + neighbors = len(list(self.get_neighboring_agents())) self.state['times'] = self.state.get('times', 0) + 1 self.state['neighbors'] = neighbors self.state['total'] = total @@ -24,8 +24,8 @@ class AggregatedCounter(NetworkAgent): def step(self): # Outside effects - total = len(self.get_all_agents()) - neighbors = len(self.get_neighboring_agents()) + total = len(list(self.get_all_agents())) + neighbors = len(list(self.get_neighboring_agents())) self.state['times'] = self.state.get('times', 0) + 1 self.state['neighbors'] = self.state.get('neighbors', 0) + neighbors self.state['total'] = self.state.get('total', 0) + total diff --git a/soil/agents/__init__.py b/soil/agents/__init__.py index 0460870..84d2e15 100644 --- a/soil/agents/__init__.py +++ b/soil/agents/__init__.py @@ -6,11 +6,13 @@ import nxsim +import logging from collections import OrderedDict from copy import deepcopy from functools import partial import json + from functools import wraps @@ -37,11 +39,16 @@ class BaseAgent(nxsim.BaseAgent, metaclass=MetaAgent): state.update(kwargs.pop('state', {})) kwargs['state'] = state super().__init__(**kwargs) + if not hasattr(self, 'level'): + self.level = logging.DEBUG + self.logger = logging.getLogger('Agent-{}'.format(self.id)) + self.logger.setLevel(self.level) + def __getitem__(self, key): if isinstance(key, tuple): k, t_step = key - return self.env[t_step, self.id, k] + return self.env[self.id, t_step, k] return self.state.get(key, None) def __delitem__(self, key): @@ -78,7 +85,7 @@ class BaseAgent(nxsim.BaseAgent, metaclass=MetaAgent): pass def to_json(self): - return json.dumps(self._history) + return json.dumps(self.state) def count_agents(self, state_id=None, limit_neighbors=False): if limit_neighbors: @@ -108,6 +115,20 @@ class BaseAgent(nxsim.BaseAgent, metaclass=MetaAgent): return filter(matches_all, agents) + def log(self, message, level=logging.INFO, **kwargs): + message = "\t@{:>5}:\t{}".format(self.now, message) + for k, v in kwargs: + message += " {k}={v} ".format(k, v) + extra = {} + extra['now'] = self.now + extra['id'] = self.id + return self.logger.log(level, message, extra=extra) + + def debug(self, *args, **kwargs): + return self.log(*args, level=logging.DEBUG, **kwargs) + + def info(self, *args, **kwargs): + return self.log(*args, level=logging.INFO, **kwargs) class NetworkAgent(BaseAgent, nxsim.BaseNetworkAgent): diff --git a/soil/environment.py b/soil/environment.py index 51b89ec..9fb06c4 100644 --- a/soil/environment.py +++ b/soil/environment.py @@ -1,10 +1,11 @@ import os +import sqlite3 import time -import csv import weakref +import csv import random +import simpy from copy import deepcopy -from functools import partial import networkx as nx import nxsim @@ -22,15 +23,19 @@ class SoilEnvironment(nxsim.NetworkEnvironment): interval=1, seed=None, dump=False, + simulation=None, *args, **kwargs): self.name = name or 'UnnamedEnvironment' - self.states = deepcopy(states) or {} + if isinstance(states, list): + states = dict(enumerate(states)) + self.states = deepcopy(states) if states else {} self.default_state = deepcopy(default_state) or {} + self.sim = weakref.ref(simulation) + if 'topology' not in kwargs and simulation: + kwargs['topology'] = self.sim().topology.copy() super().__init__(*args, **kwargs) self._env_agents = {} - self._history = {} self.interval = interval - self.logger = None self.dump = dump # Add environment agents first, so their events get # executed before network agents @@ -39,6 +44,17 @@ class SoilEnvironment(nxsim.NetworkEnvironment): self.environment_agents = environment_agents or [] self.network_agents = network_agents or [] self.process(self.save_state()) + if self.dump: + self._db_path = os.path.join(self.get_path(), 'db.sqlite') + else: + self._db_path = ":memory:" + self.create_db(self._db_path) + + def create_db(self, db_path=None): + db_path = db_path or self._db_path + self._db = sqlite3.connect(db_path) + with self._db: + self._db.execute('''CREATE TABLE IF NOT EXISTS history (agent_id text, t_step int, key text, value text, value_type text)''') @property def agents(self): @@ -48,7 +64,7 @@ class SoilEnvironment(nxsim.NetworkEnvironment): @property def environment_agents(self): for ref in self._env_agents.values(): - yield ref() + yield ref @environment_agents.setter def environment_agents(self, environment_agents): @@ -60,7 +76,7 @@ class SoilEnvironment(nxsim.NetworkEnvironment): kwargs['agent_id'] = kwargs.get('agent_id', atype.__name__) kwargs['state'] = kwargs.get('state', {}) a = atype(environment=self, **kwargs) - self._env_agents[a.id] = weakref.ref(a) + self._env_agents[a.id] = a @property def network_agents(self): @@ -106,42 +122,72 @@ class SoilEnvironment(nxsim.NetworkEnvironment): super().run(*args, **kwargs) self._save_state() - def _save_state(self): + def _save_state(self, now=None): # for agent in self.agents: # agent.save_state() - nowd = self._history[self.now] = {} - nowd['env'] = deepcopy(self.environment_params) - for agent in self.agents: - nowd[agent.id] = deepcopy(agent.state) + with self._db: + self._db.executemany("insert into history(agent_id, t_step, key, value, value_type) values (?, ?, ?, ?, ?)", self.state_to_tuples(now=now)) def save_state(self): - while True: + while self.peek() != simpy.core.Infinity: + utils.logger.info('Step: {}'.format(self.now)) ev = self.event() ev._ok = True # Schedule the event with minimum priority so # that it executes after all agents are done - self.schedule(ev, -1, self.interval) + self.schedule(ev, -1, self.peek()) yield ev self._save_state() def __getitem__(self, key): if isinstance(key, tuple): - t_step, agent_id, k = key + values = {"agent_id": key[0], + "t_step": key[1], + "key": key[2], + "value": None, + "value_type": None + } - def key_or_dict(d, k, nfunc): - if k is None: - if d is None: - return {} - return {k: nfunc(v) for k, v in d.items()} - if k in d: - return nfunc(d[k]) - return {} + fields = list(k for k, v in values.items() if v is None) + conditions = " and ".join("{}='{}'".format(k, v) for k, v in values.items() if v is not None) + + query = """SELECT {fields} from history""".format(fields=",".join(fields)) + if conditions: + query = """{query} where {conditions}""".format(query=query, + conditions=conditions) + with self._db: + rows = self._db.execute(query).fetchall() + + utils.logger.debug(rows) + results = self.rows_to_dict(rows) + return results - f1 = partial(key_or_dict, k=k, nfunc=lambda x: x) - f2 = partial(key_or_dict, k=agent_id, nfunc=f1) - return key_or_dict(self._history, t_step, f2) return self.environment_params[key] + def rows_to_dict(self, rows): + if len(rows) < 1: + return None + + level = len(rows[0])-2 + + if level == 0: + if len(rows) != 1: + raise ValueError('Cannot convert {} to dictionaries'.format(rows)) + value, value_type = rows[0] + return utils.convert(value, value_type) + + results = {} + for row in rows: + item = results + for i in range(level-1): + key = row[i] + if key not in item: + item[key] = {} + item = item[key] + key, value, value_type = row[level-1:] + item[key] = utils.convert(value, value_type) + return results + def __setitem__(self, key, value): self.environment_params[key] = value @@ -179,23 +225,34 @@ class SoilEnvironment(nxsim.NetworkEnvironment): self.name+".gexf") nx.write_gexf(G, graph_path, version="1.2draft") + def state_to_tuples(self, now=None): + if now is None: + now = self.now + for k, v in self.environment_params.items(): + yield 'env', now, k, v, type(v).__name__ + for agent in self.agents: + for k, v in agent.state.items(): + yield agent.id, now, k, v, type(v).__name__ + def history_to_tuples(self): - for tstep, states in self._history.items(): - for a_id, state in states.items(): - for attribute, value in state.items(): - yield (a_id, tstep, attribute, value) + with self._db: + res = self._db.execute("select agent_id, t_step, key, value from history ").fetchall() + yield from res def history_to_graph(self): G = nx.Graph(self.G) - for agent in self.agents: + for agent in self.network_agents: attributes = {'agent': str(agent.__class__)} lastattributes = {} spells = [] lastvisible = False laststep = None - for t_step, state in reversed(list(self[None, agent.id, None].items())): + history = self[agent.id, None, None] + if not history: + continue + for t_step, state in reversed(sorted(list(history.items()))): for attribute, value in state.items(): if attribute == 'visible': nowvisible = state[attribute] @@ -206,15 +263,20 @@ class SoilEnvironment(nxsim.NetworkEnvironment): lastvisible = nowvisible else: - if attribute not in lastattributes or lastattributes[attribute][0] != value: - laststep = lastattributes.get(attribute, - (None, None))[1] - value = (state[attribute], t_step, laststep) - key = 'attr_' + attribute + key = 'attr_' + attribute + if key not in attributes: + attributes[key] = list() + if key not in lastattributes: + lastattributes[key] = (state[attribute], t_step) + elif lastattributes[key][0] != value: + last_value, laststep = lastattributes[key] + value = (last_value, t_step, laststep) if key not in attributes: attributes[key] = list() attributes[key].append(value) - lastattributes[attribute] = (state[attribute], t_step) + lastattributes[key] = (state[attribute], t_step) + for k, v in lastattributes.items(): + attributes[k].append((v[0], 0, v[1])) if lastvisible: spells.append((laststep, None)) if spells: diff --git a/soil/simulation.py b/soil/simulation.py index 2b06ebe..8a9285c 100644 --- a/soil/simulation.py +++ b/soil/simulation.py @@ -1,10 +1,8 @@ -import weakref import os import time import imp import sys import yaml -import logging import networkx as nx from networkx.readwrite import json_graph @@ -15,8 +13,8 @@ import pickle from nxsim import NetworkSimulation from . import agents, utils, environment, basestring +from .utils import logger -logger = logging.getLogger(__name__) class SoilSimulation(NetworkSimulation): """ @@ -86,7 +84,7 @@ class SoilSimulation(NetworkSimulation): self.network_agents = self._convert_agent_types(distro) self.states = self.validate_states(states, - topology) + self.topology) def calculate_distribution(self, network_agents=None, @@ -178,9 +176,8 @@ class SoilSimulation(NetworkSimulation): states=self.states, default_state=self.default_state, environment_agents=self.environment_agents, + simulation=self, **self.environment_params) - - env.sim = weakref.ref(self) # Set up agents on nodes logger.info('\tRunning') with utils.timer('trial'): diff --git a/soil/utils.py b/soil/utils.py index be1f7a2..4c5f4fb 100644 --- a/soil/utils.py +++ b/soil/utils.py @@ -10,10 +10,15 @@ import networkx as nx from contextlib import contextmanager + logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) +logger.addHandler(logging.StreamHandler()) def load_network(network_params, dir_path=None): + if network_params is None: + return nx.Graph() path = network_params.get('path', None) if path: if dir_path and not os.path.isabs(path): @@ -73,9 +78,23 @@ def agent_from_distribution(distribution, value=-1): for d in distribution: threshold = d['threshold'] if value >= threshold[0] and value < threshold[1]: - state = None + state = {} if 'state' in d: state = deepcopy(d['state']) return d['agent_type'], state raise Exception('Distribution for value {} not found in: {}'.format(value, distribution)) + + +def convert(value, type_): + import importlib + try: + # Check if it's a builtin type + module = importlib.import_module('builtins') + cls = getattr(module, type_) + except AttributeError: + # if not, separate module and class + module, type_ = type_.rsplit(".", 1) + module = importlib.import_module(module) + cls = getattr(module, type_) + return cls(value) diff --git a/tests/test_main.py b/tests/test_main.py index d315f3f..55d1c9b 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -5,7 +5,7 @@ import yaml from functools import partial from os.path import join -from soil import simulation, agents, utils +from soil import simulation, environment, agents, utils ROOT = os.path.abspath(os.path.dirname(__file__)) @@ -198,6 +198,26 @@ class TestMain(TestCase): """ Make sure all examples in the examples folder are correct """ + pass + + def test_row_conversion(self): + sim = simulation.SoilSimulation() + env = environment.SoilEnvironment(simulation=sim) + env['test'] = 'test_value' + env._save_state(now=0) + + res = list(env.history_to_tuples()) + assert len(res) == len(env.environment_params) + assert ('env', 0, 'test', 'test_value') in res + + env['test'] = 'second_value' + env._save_state(now=1) + res = list(env.history_to_tuples()) + + assert env['env', 0, 'test' ] == 'test_value' + assert env['env', 1, 'test' ] == 'second_value' + + def make_example_test(path, config):