balkian/soil
1
0
Fork 0
mirror of https://github.com/gsi-upm/soil synced 2025-08-23 19:52:19 +00:00
Code Issues Releases Wiki Activity

WIP: mesa compatibility

Browse Source
This commit is contained in:
J. Fernando Sánchez
2021-10-14 17:37:06 +02:00
parent e860bdb922
commit 5d7e57675a
32 changed files with 963 additions and 524 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
soil/__init__.py
View File

@@ -11,6 +11,7 @@ try:
except NameError:
basestring = str
from .agents import *
from . import agents
from .simulation import *
from .environment import Environment
@@ -18,6 +19,7 @@ from .history import History
from . import serialization
from . import analysis
from .utils import logger
from .time import *
def main():
import argparse

41
soil/agents/BassModel.py
View File

@@ -1,40 +1,31 @@
import random
from . import BaseAgent
from . import FSM, state, default_state
class BassModel(BaseAgent):
class BassModel(FSM):
"""
Settings:
innovation_prob
imitation_prob
"""
def __init__(self, environment, agent_id, state, **kwargs):
super().__init__(environment=environment, agent_id=agent_id, state=state)
env_params = environment.environment_params
self.state['sentimentCorrelation'] = 0
sentimentCorrelation = 0
def step(self):
self.behaviour()
def behaviour(self):
# Outside effects
if random.random() < self['innovation_prob']:
if self.state['id'] == 0:
self.state['id'] = 1
self.state['sentimentCorrelation'] = 1
else:
pass
return
# Imitation effects
if self.state['id'] == 0:
aware_neighbors = self.get_neighboring_agents(state_id=1)
@default_state
@state
def innovation(self):
if random.random() < self.innovation_prob:
self.sentimentCorrelation = 1
return self.aware
else:
aware_neighbors = self.get_neighboring_agents(state_id=self.aware.id)
num_neighbors_aware = len(aware_neighbors)
if random.random() < (self['imitation_prob']*num_neighbors_aware):
self.state['id'] = 1
self.state['sentimentCorrelation'] = 1
self.sentimentCorrelation = 1
return self.aware
else:
pass
@state
def aware(self):
self.die()

39
soil/agents/BigMarketModel.py
View File

@@ -1,8 +1,8 @@
import random
from . import BaseAgent
from . import FSM, state, default_state
class BigMarketModel(BaseAgent):
class BigMarketModel(FSM):
"""
Settings:
Names:
@@ -19,34 +19,25 @@ class BigMarketModel(BaseAgent):
sentiment_about [Array]
"""
def __init__(self, environment=None, agent_id=0, state=()):
super().__init__(environment=environment, agent_id=agent_id, state=state)
self.enterprises = environment.environment_params['enterprises']
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.enterprises = self.env.environment_params['enterprises']
self.type = ""
self.number_of_enterprises = len(environment.environment_params['enterprises'])
if self.id < self.number_of_enterprises: # Enterprises
self.state['id'] = self.id
if self.id < len(self.enterprises): # Enterprises
self.set_state(self.enterprise.id)
self.type = "Enterprise"
self.tweet_probability = environment.environment_params['tweet_probability_enterprises'][self.id]
else: # normal users
self.state['id'] = self.number_of_enterprises
self.type = "User"
self.set_state(self.user.id)
self.tweet_probability = environment.environment_params['tweet_probability_users']
self.tweet_relevant_probability = environment.environment_params['tweet_relevant_probability']
self.tweet_probability_about = environment.environment_params['tweet_probability_about'] # List
self.sentiment_about = environment.environment_params['sentiment_about'] # List
def step(self):
if self.id < self.number_of_enterprises: # Enterprise
self.enterpriseBehaviour()
else: # Usuario
self.userBehaviour()
for i in range(self.number_of_enterprises): # So that it never is set to 0 if there are not changes (logs)
self.attrs['sentiment_enterprise_%s'% self.enterprises[i]] = self.sentiment_about[i]
def enterpriseBehaviour(self):
@state
def enterprise(self):
if random.random() < self.tweet_probability: # Tweets
aware_neighbors = self.get_neighboring_agents(state_id=self.number_of_enterprises) # Nodes neighbour users
@@ -64,12 +55,12 @@ class BigMarketModel(BaseAgent):
x.attrs['sentiment_enterprise_%s'% self.enterprises[self.id]] = x.sentiment_about[self.id]
def userBehaviour(self):
@state
def user(self):
if random.random() < self.tweet_probability: # Tweets
if random.random() < self.tweet_relevant_probability: # Tweets something relevant
# Tweet probability per enterprise
for i in range(self.number_of_enterprises):
for i in range(len(self.enterprises)):
random_num = random.random()
if random_num < self.tweet_probability_about[i]:
# The condition is fulfilled, sentiments are evaluated towards that enterprise
@@ -82,8 +73,10 @@ class BigMarketModel(BaseAgent):
else:
# POSITIVO
self.userTweets("positive",i)
for i in range(len(self.enterprises)): # So that it never is set to 0 if there are not changes (logs)
self.attrs['sentiment_enterprise_%s'% self.enterprises[i]] = self.sentiment_about[i]
def userTweets(self,sentiment,enterprise):
def userTweets(self, sentiment,enterprise):
aware_neighbors = self.get_neighboring_agents(state_id=self.number_of_enterprises) # Nodes neighbours users
for x in aware_neighbors:
if sentiment == "positive":

20
soil/agents/Geo.py Normal file
View File

@@ -0,0 +1,20 @@
from scipy.spatial import cKDTree as KDTree
from . import NetworkAgent
class Geo(NetworkAgent):
'''In this type of network, nodes have a "pos" attribute.'''
def geo_search(self, radius, node=None, center=False, **kwargs):
'''Get a list of nodes whose coordinates are closer than *radius* to *node*.'''
node = as_node(node if node is not None else self)
G = self.subgraph(**kwargs)
pos = nx.get_node_attributes(G, 'pos')
if not pos:
return []
nodes, coords = list(zip(*pos.items()))
kdtree = KDTree(coords) # Cannot provide generator.
indices = kdtree.query_ball_point(pos[node], radius)
return [nodes[i] for i in indices if center or (nodes[i] != node)]

8
soil/agents/IndependentCascadeModel.py
View File

@@ -10,10 +10,10 @@ class IndependentCascadeModel(BaseAgent):
imitation_prob
"""
def __init__(self, environment=None, agent_id=0, state=()):
super().__init__(environment=environment, agent_id=agent_id, state=state)
self.innovation_prob = environment.environment_params['innovation_prob']
self.imitation_prob = environment.environment_params['imitation_prob']
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.innovation_prob = self.env.environment_params['innovation_prob']
self.imitation_prob = self.env.environment_params['imitation_prob']
self.state['time_awareness'] = 0
self.state['sentimentCorrelation'] = 0

8
soil/agents/ModelM2.py
View File

@@ -21,8 +21,8 @@ class SpreadModelM2(BaseAgent):
prob_generate_anti_rumor
"""
def __init__(self, environment=None, agent_id=0, state=()):
super().__init__(environment=environment, agent_id=agent_id, state=state)
def __init__(self, model=None, unique_id=0, state=()):
super().__init__(model=environment, unique_id=unique_id, state=state)
self.prob_neutral_making_denier = np.random.normal(environment.environment_params['prob_neutral_making_denier'],
environment.environment_params['standard_variance'])
@@ -123,8 +123,8 @@ class ControlModelM2(BaseAgent):
"""
def __init__(self, environment=None, agent_id=0, state=()):
super().__init__(environment=environment, agent_id=agent_id, state=state)
def __init__(self, model=None, unique_id=0, state=()):
super().__init__(model=environment, unique_id=unique_id, state=state)
self.prob_neutral_making_denier = np.random.normal(environment.environment_params['prob_neutral_making_denier'],
environment.environment_params['standard_variance'])

4
soil/agents/SISaModel.py
View File

@@ -29,8 +29,8 @@ class SISaModel(FSM):
standard_variance
"""
def __init__(self, environment, agent_id=0, state=()):
super().__init__(environment=environment, agent_id=agent_id, state=state)
def __init__(self, environment, unique_id=0, state=()):
super().__init__(model=environment, unique_id=unique_id, state=state)
self.neutral_discontent_spon_prob = np.random.normal(self.env['neutral_discontent_spon_prob'],
self.env['standard_variance'])

4
soil/agents/SentimentCorrelationModel.py
View File

@@ -16,8 +16,8 @@ class SentimentCorrelationModel(BaseAgent):
disgust_prob
"""
def __init__(self, environment, agent_id=0, state=()):
super().__init__(environment=environment, agent_id=agent_id, state=state)
def __init__(self, environment, unique_id=0, state=()):
super().__init__(model=environment, unique_id=unique_id, state=state)
self.outside_effects_prob = environment.environment_params['outside_effects_prob']
self.anger_prob = environment.environment_params['anger_prob']
self.joy_prob = environment.environment_params['joy_prob']

318
soil/agents/__init__.py
View File

@@ -1,21 +1,15 @@
# networkStatus = {} # Dict that will contain the status of every agent in the network
# sentimentCorrelationNodeArray = []
# for x in range(0, settings.network_params["number_of_nodes"]):
# sentimentCorrelationNodeArray.append({'id': x})
# Initialize agent states. Let's assume everyone is normal.
import logging
from collections import OrderedDict
from collections import OrderedDict, defaultdict
from copy import deepcopy
from functools import partial
from scipy.spatial import cKDTree as KDTree
import json
import simpy
import networkx as nx
from functools import wraps
from .. import serialization, history, utils
from .. import serialization, history, utils, time
from mesa import Agent
def as_node(agent):
@@ -24,39 +18,51 @@ def as_node(agent):
return agent
class BaseAgent:
class BaseAgent(Agent):
"""
A special simpy BaseAgent that keeps track of its state history.
A special Agent that keeps track of its state history.
"""
defaults = {}
def __init__(self, environment, agent_id, state=None,
name=None, interval=None):
def __init__(self,
unique_id,
model,
state=None,
name=None,
interval=None):
# Check for REQUIRED arguments
assert environment is not None, TypeError('__init__ missing 1 required keyword argument: \'environment\'. '
'Cannot be NoneType.')
# Initialize agent parameters
self.id = agent_id
self.name = name or '{}[{}]'.format(type(self).__name__, self.id)
# Register agent to environment
self.env = environment
if isinstance(unique_id, Agent):
raise Exception()
super().__init__(unique_id=unique_id, model=model)
self.name = name or '{}[{}]'.format(type(self).__name__, self.unique_id)
self._neighbors = None
self.alive = True
real_state = deepcopy(self.defaults)
real_state.update(state or {})
self.state = real_state
self.interval = interval
self.logger = logging.getLogger(self.env.name).getChild(self.name)
self.interval = interval or self.get('interval', getattr(self.model, 'interval', 1))
self.logger = logging.getLogger(self.model.name).getChild(self.name)
if hasattr(self, 'level'):
self.logger.setLevel(self.level)
# initialize every time an instance of the agent is created
self.action = self.env.process(self.run())
# TODO: refactor to clean up mesa compatibility
@property
def id(self):
return self.unique_id
@property
def env(self):
return self.model
@env.setter
def env(self, model):
self.model = model
@property
def state(self):
@@ -76,17 +82,17 @@ class BaseAgent:
@property
def environment_params(self):
return self.env.environment_params
return self.model.environment_params
@environment_params.setter
def environment_params(self, value):
self.env.environment_params = value
self.model.environment_params = value
def __getitem__(self, key):
if isinstance(key, tuple):
key, t_step = key
k = history.Key(key=key, t_step=t_step, agent_id=self.id)
return self.env[k]
return self.model[k]
return self._state.get(key, None)
def __delitem__(self, key):
@@ -100,7 +106,7 @@ class BaseAgent:
k = history.Key(t_step=self.now,
agent_id=self.id,
key=key)
self.env[k] = value
self.model[k] = value
def items(self):
return self._state.items()
@@ -111,29 +117,33 @@ class BaseAgent:
@property
def now(self):
try:
return self.env.now
return self.model.now
except AttributeError:
# No environment
return None
def run(self):
if self.interval is not None:
interval = self.interval
elif 'interval' in self:
interval = self['interval']
else:
interval = self.env.interval
while self.alive:
res = self.step()
yield res or self.env.timeout(interval)
def die(self, remove=False):
self.alive = False
if remove:
self.remove_node(self.id)
def step(self):
return
if not self.alive:
return time.When('inf')
return super().step() or time.Delta(self.interval)
def log(self, message, *args, level=logging.INFO, **kwargs):
if not self.logger.isEnabledFor(level):
return
message = message + " ".join(str(i) for i in args)
message = " @{:>3}: {}".format(self.now, message)
for k, v in kwargs:
message += " {k}={v} ".format(k, v)
extra = {}
extra['now'] = self.now
extra['unique_id'] = self.unique_id
extra['agent_name'] = self.name
return self.logger.log(level, message, extra=extra)
def debug(self, *args, **kwargs):
return self.log(*args, level=logging.DEBUG, **kwargs)
@@ -149,7 +159,7 @@ class BaseAgent:
'''
state = {}
state['id'] = self.id
state['environment'] = self.env
state['environment'] = self.model
state['_state'] = self._state
return state
@@ -157,19 +167,19 @@ class BaseAgent:
'''
Get back a serialized agent and try to re-compose it
'''
self.id = state['id']
self.state_id = state['id']
self._state = state['_state']
self.env = state['environment']
self.model = state['environment']
class NetworkAgent(BaseAgent):
@property
def topology(self):
return self.env.G
return self.model.G
@property
def G(self):
return self.env.G
return self.model.G
def count_agents(self, **kwargs):
return len(list(self.get_agents(**kwargs)))
@@ -182,37 +192,26 @@ class NetworkAgent(BaseAgent):
def get_agents(self, agents=None, limit_neighbors=False, **kwargs):
if limit_neighbors:
agents = self.topology.neighbors(self.id)
agents = self.topology.neighbors(self.unique_id)
agents = self.env.get_agents(agents)
agents = self.model.get_agents(agents)
return select(agents, **kwargs)
def log(self, message, *args, level=logging.INFO, **kwargs):
message = message + " ".join(str(i) for i in args)
message = " @{:>3}: {}".format(self.now, message)
for k, v in kwargs:
message += " {k}={v} ".format(k, v)
extra = {}
extra['now'] = self.now
extra['agent_id'] = self.id
extra['agent_name'] = self.name
return self.logger.log(level, message, extra=extra)
def subgraph(self, center=True, **kwargs):
include = [self] if center else []
return self.topology.subgraph(n.id for n in self.get_agents(**kwargs)+include)
return self.topology.subgraph(n.unique_id for n in self.get_agents(**kwargs)+include)
def remove_node(self, agent_id):
self.topology.remove_node(agent_id)
def remove_node(self, unique_id):
self.topology.remove_node(unique_id)
def add_edge(self, other, edge_attr_dict=None, *edge_attrs):
# return super(NetworkAgent, self).add_edge(node1=self.id, node2=other, **kwargs)
if self.id not in self.topology.nodes(data=False):
raise ValueError('{} not in list of existing agents in the network'.format(self.id))
if self.unique_id not in self.topology.nodes(data=False):
raise ValueError('{} not in list of existing agents in the network'.format(self.unique_id))
if other not in self.topology.nodes(data=False):
raise ValueError('{} not in list of existing agents in the network'.format(other))
self.topology.add_edge(self.id, other, edge_attr_dict=edge_attr_dict, *edge_attrs)
self.topology.add_edge(self.unique_id, other, edge_attr_dict=edge_attr_dict, *edge_attrs)
def ego_search(self, steps=1, center=False, node=None, **kwargs):
@@ -223,17 +222,17 @@ class NetworkAgent(BaseAgent):
def degree(self, node, force=False):
node = as_node(node)
if force or (not hasattr(self.env, '_degree')) or getattr(self.env, '_last_step', 0) < self.now:
self.env._degree = nx.degree_centrality(self.topology)
self.env._last_step = self.now
return self.env._degree[node]
if force or (not hasattr(self.model, '_degree')) or getattr(self.model, '_last_step', 0) < self.now:
self.model._degree = nx.degree_centrality(self.topology)
self.model._last_step = self.now
return self.model._degree[node]
def betweenness(self, node, force=False):
node = as_node(node)
if force or (not hasattr(self.env, '_betweenness')) or getattr(self.env, '_last_step', 0) < self.now:
self.env._betweenness = nx.betweenness_centrality(self.topology)
self.env._last_step = self.now
return self.env._betweenness[node]
if force or (not hasattr(self.model, '_betweenness')) or getattr(self.model, '_last_step', 0) < self.now:
self.model._betweenness = nx.betweenness_centrality(self.topology)
self.model._last_step = self.now
return self.model._betweenness[node]
def state(name=None):
@@ -301,36 +300,29 @@ class FSM(NetworkAgent, metaclass=MetaFSM):
super(FSM, self).__init__(*args, **kwargs)
if 'id' not in self.state:
if not self.default_state:
raise ValueError('No default state specified for {}'.format(self.id))
raise ValueError('No default state specified for {}'.format(self.unique_id))
self['id'] = self.default_state.id
self._next_change = simpy.core.Infinity
self._next_state = self.state
self.set_state(self.state['id'])
def step(self):
if self._next_change < self.now:
next_state = self._next_state
self._next_change = simpy.core.Infinity
self['id'] = next_state
elif 'id' in self.state:
next_state = self['id']
elif self.default_state:
next_state = self.default_state.id
else:
raise Exception('{} has no valid state id or default state'.format(self))
if next_state not in self.states:
raise Exception('{} is not a valid id for {}'.format(next_state, self))
return self.states[next_state](self)
def next_state(self, state):
self._next_change = self.now
self._next_state = state
self.debug(f'Agent {self.unique_id} @ state {self["id"]}')
interval = super().step()
if 'id' not in self:
if 'id' in self.state:
self.set_state(self['state_id'])
elif self.default_state:
self.set_state(self.default_state.id)
else:
raise Exception('{} has no valid state id or default state'.format(self))
return self.states[self['id']](self) or interval
def set_state(self, state):
if hasattr(state, 'id'):
state = state.id
if state not in self.states:
raise ValueError('{} is not a valid state'.format(state))
self['id'] = state
self['state_id'] = state
return state
@@ -349,9 +341,6 @@ def prob(prob=1):
return r < prob
STATIC_THRESHOLD = (-1, -1)
def calculate_distribution(network_agents=None,
agent_type=None):
'''
@@ -379,7 +368,7 @@ def calculate_distribution(network_agents=None,
'agent_type_1'.
'''
if network_agents:
network_agents = deepcopy(network_agents)
network_agents = [deepcopy(agent) for agent in network_agents if not hasattr(agent, 'id')]
elif agent_type:
network_agents = [{'agent_type': agent_type}]
else:
@@ -394,7 +383,6 @@ def calculate_distribution(network_agents=None,
acc = 0
for v in network_agents:
if 'ids' in v:
v['threshold'] = STATIC_THRESHOLD
continue
upper = acc + (v['weight']/total)
v['threshold'] = [acc, upper]
@@ -409,7 +397,7 @@ def serialize_type(agent_type, known_modules=[], **kwargs):
return serialization.serialize(agent_type, known_modules=known_modules, **kwargs)[1] # Get the name of the class
def serialize_distribution(network_agents, known_modules=[]):
def serialize_definition(network_agents, known_modules=[]):
'''
When serializing an agent distribution, remove the thresholds, in order
to avoid cluttering the YAML definition file.
@@ -431,7 +419,7 @@ def deserialize_type(agent_type, known_modules=[]):
return agent_type
def deserialize_distribution(ind, **kwargs):
def deserialize_definition(ind, **kwargs):
d = deepcopy(ind)
for v in d:
v['agent_type'] = deserialize_type(v['agent_type'], **kwargs)
@@ -452,44 +440,84 @@ def _validate_states(states, topology):
def _convert_agent_types(ind, to_string=False, **kwargs):
'''Convenience method to allow specifying agents by class or class name.'''
if to_string:
return serialize_distribution(ind, **kwargs)
return deserialize_distribution(ind, **kwargs)
return serialize_definition(ind, **kwargs)
return deserialize_definition(ind, **kwargs)
def _agent_from_distribution(distribution, value=-1, agent_id=None):
def _agent_from_definition(definition, value=-1, unique_id=None):
"""Used in the initialization of agents given an agent distribution."""
if value < 0:
value = random.random()
for d in sorted(distribution, key=lambda x: x['threshold']):
threshold = d['threshold']
for d in sorted(definition, key=lambda x: x.get('threshold')):
threshold = d.get('threshold', (-1, -1))
# Check if the definition matches by id (first) or by threshold
if not ((agent_id is not None and threshold == STATIC_THRESHOLD and agent_id in d['ids']) or \
(value >= threshold[0] and value < threshold[1])):
continue
state = {}
if 'state' in d:
state = deepcopy(d['state'])
return d['agent_type'], state
if (unique_id is not None and unique_id in d.get('ids', [])) or \
(value >= threshold[0] and value < threshold[1]):
state = {}
if 'state' in d:
state = deepcopy(d['state'])
return d['agent_type'], state
raise Exception('Distribution for value {} not found in: {}'.format(value, distribution))
raise Exception('Definition for value {} not found in: {}'.format(value, definition))
class Geo(NetworkAgent):
'''In this type of network, nodes have a "pos" attribute.'''
def _definition_to_dict(definition, size=None, default_state=None):
state = default_state or {}
agents = {}
remaining = {}
if size:
for ix in range(size):
remaining[ix] = copy(state)
else:
remaining = defaultdict(lambda x: copy(state))
def geo_search(self, radius, node=None, center=False, **kwargs):
'''Get a list of nodes whose coordinates are closer than *radius* to *node*.'''
node = as_node(node if node is not None else self)
distro = sorted([item for item in definition if 'weight' in item])
G = self.subgraph(**kwargs)
ix = 0
def init_agent(item, id=ix):
while id in agents:
id += 1
pos = nx.get_node_attributes(G, 'pos')
if not pos:
return []
nodes, coords = list(zip(*pos.items()))
kdtree = KDTree(coords) # Cannot provide generator.
indices = kdtree.query_ball_point(pos[node], radius)
return [nodes[i] for i in indices if center or (nodes[i] != node)]
agent = remaining[id]
agent['state'].update(copy(item.get('state', {})))
agents[id] = agent
del remaining[id]
return agent
for item in definition:
if 'ids' in item:
ids = item['ids']
del item['ids']
for id in ids:
agent = init_agent(item, id)
for item in definition:
if 'number' in item:
times = item['number']
del item['number']
for times in range(times):
if size:
ix = random.choice(remaining.keys())
agent = init_agent(item, id)
else:
agent = init_agent(item)
if not size:
return agents
if len(remaining) < 0:
raise Exception('Invalid definition. Too many agents to add')
total_weight = float(sum(s['weight'] for s in distro))
unit = size / total_weight
for item in distro:
times = unit * item['weight']
del item['weight']
for times in range(times):
ix = random.choice(remaining.keys())
agent = init_agent(item, id)
return agents
def select(agents, state_id=None, agent_type=None, ignore=None, iterator=False, **kwargs):
@@ -502,22 +530,21 @@ def select(agents, state_id=None, agent_type=None, ignore=None, iterator=False,
except TypeError:
agent_type = tuple([agent_type])
def matches_all(agent):
if state_id is not None:
if agent.state.get('id', None) not in state_id:
return False
if agent_type is not None:
if not isinstance(agent, agent_type):
return False
state = agent.state
for k, v in kwargs.items():
if state.get(k, None) != v:
return False
return True
checks = []
f = agents
f = filter(matches_all, agents)
if ignore:
f = filter(lambda x: x not in ignore, f)
if state_id is not None:
f = filter(lambda agent: agent.state.get('id', None) in state_id, f)
if agent_type is not None:
f = filter(lambda agent: isinstance(agent, agent_type), f)
for k, v in kwargs.items():
f = filter(lambda agent: agent.state.get(k, None) == v, f)
if iterator:
return f
return list(f)
@@ -530,3 +557,10 @@ from .ModelM2 import *
from .SentimentCorrelationModel import *
from .SISaModel import *
from .CounterModel import *
try:
import scipy
from .Geo import Geo
except ImportError:
import sys
print('Could not load the Geo Agent, scipy is not installed', file=sys.stderr)

33
soil/analysis.py
View File

@@ -61,7 +61,12 @@ def convert_row(row):
def convert_types_slow(df):
'''This is a slow operation.'''
'''
Go over every column in a dataframe and convert it to the type determined by the `get_types`
function.
This is a slow operation.
'''
dtypes = get_types(df)
for k, v in dtypes.items():
t = df[df['key']==k]
@@ -102,6 +107,9 @@ def process(df, **kwargs):
def get_types(df):
'''
Get the value type for every key stored in a raw history dataframe.
'''
dtypes = df.groupby(by=['key'])['value_type'].unique()
return {k:v[0] for k,v in dtypes.iteritems()}
@@ -126,8 +134,14 @@ def process_one(df, *keys, columns=['key', 'agent_id'], values='value',
def get_count(df, *keys):
'''
For every t_step and key, get the value count.
The result is a dataframe with `t_step` as index, an a multiindex column based on `key` and the values found for each `key`.
'''
if keys:
df = df[list(keys)]
df.columns = df.columns.remove_unused_levels()
counts = pd.DataFrame()
for key in df.columns.levels[0]:
g = df[[key]].apply(pd.Series.value_counts, axis=1).fillna(0)
@@ -137,10 +151,25 @@ def get_count(df, *keys):
return counts
def get_majority(df, *keys):
'''
For every t_step and key, get the value of the majority of agents
The result is a dataframe with `t_step` as index, and columns based on `key`.
'''
df = get_count(df, *keys)
return df.stack(level=0).idxmax(axis=1).unstack()
def get_value(df, *keys, aggfunc='sum'):
'''
For every t_step and key, get the value of *numeric columns*, aggregated using a specific function.
'''
if keys:
df = df[list(keys)]
return df.groupby(axis=1, level=0).agg(aggfunc)
df.columns = df.columns.remove_unused_levels()
df = df.select_dtypes('number')
return df.groupby(level='key', axis=1).agg(aggfunc)
def plot_all(*args, plot_args={}, **kwargs):

26
soil/datacollection.py Normal file
View File

@@ -0,0 +1,26 @@
from mesa import DataCollector as MDC
class SoilDataCollector(MDC):
def __init__(self, environment, *args, **kwargs):
super().__init__(*args, **kwargs)
# Populate model and env reporters so they have a key per
# So they can be shown in the web interface
self.environment = environment
@property
def model_vars(self):
pass
@model_vars.setter
def model_vars(self, value):
pass
@property
def agent_reporters(self):
self.model._history._
pass

116
soil/environment.py
View File

@@ -1,28 +1,29 @@
import os
import sqlite3
import time
import csv
import math
import random
import simpy
import yaml
import tempfile
import pandas as pd
from time import time as current_time
from copy import deepcopy
from networkx.readwrite import json_graph
import networkx as nx
import simpy
from . import serialization, agents, analysis, history, utils
from mesa import Model
from . import serialization, agents, analysis, history, utils, time
# These properties will be copied when pickling/unpickling the environment
_CONFIG_PROPS = [ 'name',
'states',
'default_state',
'interval',
'states',
'default_state',
'interval',
]
class Environment(simpy.Environment):
class Environment(Model):
"""
The environment is key in a simulation. It contains the network topology,
a reference to network and environment agents, as well as the environment
@@ -39,25 +40,41 @@ class Environment(simpy.Environment):
states=None,
default_state=None,
interval=1,
network_params=None,
seed=None,
topology=None,
schedule=None,
initial_time=0,
**environment_params):
environment_params=None,
dir_path=None,
**kwargs):
super().__init__()
self.schedule = schedule
if schedule is None:
self.schedule = time.TimedActivation()
self.name = name or 'UnnamedEnvironment'
seed = seed or time.time()
seed = seed or current_time()
random.seed(seed)
if isinstance(states, list):
states = dict(enumerate(states))
self.states = deepcopy(states) if states else {}
self.default_state = deepcopy(default_state) or {}
if topology is None:
network_params = network_params or {}
topology = serialization.load_network(network_params,
dir_path=dir_path)
if not topology:
topology = nx.Graph()
self.G = nx.Graph(topology)
super().__init__(initial_time=initial_time)
self.environment_params = environment_params
self.environment_params = environment_params or {}
self.environment_params.update(kwargs)
self._env_agents = {}
self.interval = interval
@@ -66,8 +83,26 @@ class Environment(simpy.Environment):
self['SEED'] = seed
# Add environment agents first, so their events get
# executed before network agents
self.environment_agents = environment_agents or []
self.network_agents = network_agents or []
if network_agents:
distro = agents.calculate_distribution(network_agents)
self.network_agents = agents._convert_agent_types(distro)
else:
self.network_agents = []
environment_agents = environment_agents or []
if environment_agents:
distro = agents.calculate_distribution(environment_agents)
environment_agents = agents._convert_agent_types(distro)
self.environment_agents = environment_agents
@property
def now(self):
if self.schedule:
return self.schedule.time
raise Exception('The environment has not been scheduled, so it has no sense of time')
@property
def agents(self):
@@ -81,15 +116,9 @@ class Environment(simpy.Environment):
@environment_agents.setter
def environment_agents(self, environment_agents):
# Set up environmental agent
self._env_agents = {}
for item in environment_agents:
kwargs = deepcopy(item)
atype = kwargs.pop('agent_type')
kwargs['agent_id'] = kwargs.get('agent_id', atype.__name__)
kwargs['state'] = kwargs.get('state', {})
a = atype(environment=self, **kwargs)
self._env_agents[a.id] = a
self._environment_agents = environment_agents
self._env_agents = agents._definition_to_dict(definition=environment_agents)
@property
def network_agents(self):
@@ -102,9 +131,9 @@ class Environment(simpy.Environment):
def network_agents(self, network_agents):
self._network_agents = network_agents
for ix in self.G.nodes():
self.init_agent(ix, agent_distribution=network_agents)
self.init_agent(ix, agent_definitions=network_agents)
def init_agent(self, agent_id, agent_distribution):
def init_agent(self, agent_id, agent_definitions):
node = self.G.nodes[agent_id]
init = False
state = dict(node)
@@ -119,8 +148,8 @@ class Environment(simpy.Environment):
if agent_type:
agent_type = agents.deserialize_type(agent_type)
elif agent_distribution:
agent_type, state = agents._agent_from_distribution(agent_distribution, agent_id=agent_id)
elif agent_definitions:
agent_type, state = agents._agent_from_definition(agent_definitions, unique_id=agent_id)
else:
serialization.logger.debug('Skipping node {}'.format(agent_id))
return
@@ -136,8 +165,8 @@ class Environment(simpy.Environment):
a = None
if agent_type:
state = defstate
a = agent_type(environment=self,
agent_id=agent_id,
a = agent_type(model=self,
unique_id=agent_id,
state=state)
node['agent'] = a
return a
@@ -159,30 +188,18 @@ class Environment(simpy.Environment):
def run(self, until, *args, **kwargs):
self._save_state()
super().run(until, *args, **kwargs)
for agent in self.agents:
self.schedule.add(agent)
while self.schedule.next_time <= until and not math.isinf(self.schedule.next_time):
self.schedule.step(until=until)
utils.logger.debug(f'Simulation step {self.schedule.time}/{until}. Next: {self.schedule.next_time}')
self._history.flush_cache()
def _save_state(self, now=None):
serialization.logger.debug('Saving state @{}'.format(self.now))
self._history.save_records(self.state_to_tuples(now=now))
def save_state(self):
'''
:DEPRECATED:
Periodically save the state of the environment and the agents.
'''
self._save_state()
while self.peek() != simpy.core.Infinity:
delay = max(self.peek() - self.now, self.interval)
serialization.logger.debug('Step: {}'.format(self.now))
ev = self.event()
ev._ok = True
# Schedule the event with minimum priority so
# that it executes before all agents
self.schedule(ev, -999, delay)
yield ev
self._save_state()
def __getitem__(self, key):
if isinstance(key, tuple):
self._history.flush_cache()
@@ -329,7 +346,7 @@ class Environment(simpy.Environment):
state['G'] = json_graph.node_link_data(self.G)
state['environment_agents'] = self._env_agents
state['history'] = self._history
state['_now'] = self._now
state['schedule'] = self.schedule
return state
def __setstate__(self, state):
@@ -338,7 +355,8 @@ class Environment(simpy.Environment):
self._env_agents = state['environment_agents']
self.G = json_graph.node_link_graph(state['G'])
self._history = state['history']
self._now = state['_now']
# self._env = None
self.schedule = state['schedule']
self._queue = []

17
soil/history.py
View File

@@ -52,7 +52,7 @@ class History:
with self.db:
logger.debug('Creating database {}'.format(self.db_path))
self.db.execute('''CREATE TABLE IF NOT EXISTS history (agent_id text, t_step int, key text, value text)''')
self.db.execute('''CREATE TABLE IF NOT EXISTS history (agent_id text, t_step real, key text, value text)''')
self.db.execute('''CREATE TABLE IF NOT EXISTS value_types (key text, value_type text)''')
self.db.execute('''CREATE TABLE IF NOT EXISTS stats (trial_id text)''')
self.db.execute('''CREATE UNIQUE INDEX IF NOT EXISTS idx_history ON history (agent_id, t_step, key);''')
@@ -103,7 +103,7 @@ class History:
dtype = 'real'
int(value)
dtype = 'int'
except ValueError:
except (ValueError, OverflowError):
pass
self.db.execute('ALTER TABLE stats ADD "{}" "{}"'.format(column, dtype))
self._stats_columns.append(column)
@@ -167,6 +167,7 @@ class History:
with self.db:
self.db.execute("replace into value_types (key, value_type) values (?, ?)", (key, name))
value = self._dtypes[key][1](value)
self._tups.append(Record(agent_id=agent_id,
t_step=t_step,
key=key,
@@ -183,9 +184,9 @@ class History:
raise Exception('DB in readonly mode')
logger.debug('Flushing cache {}'.format(self.db_path))
with self.db:
for rec in self._tups:
self.db.execute("replace into history(agent_id, t_step, key, value) values (?, ?, ?, ?)", (rec.agent_id, rec.t_step, rec.key, rec.value))
self._tups = list()
self.db.executemany("replace into history(agent_id, t_step, key, value) values (?, ?, ?, ?)", self._tups)
# (rec.agent_id, rec.t_step, rec.key, rec.value))
self._tups.clear()
def to_tuples(self):
self.flush_cache()
@@ -209,6 +210,7 @@ class History:
self._dtypes[k] = (v, serializer, deserializer)
def __getitem__(self, key):
# raise NotImplementedError()
self.flush_cache()
key = Key(*key)
agent_ids = [key.agent_id] if key.agent_id is not None else []
@@ -223,7 +225,7 @@ class History:
return r.value()
return r
def read_sql(self, keys=None, agent_ids=None, t_steps=None, convert_types=False, limit=-1):
def read_sql(self, keys=None, agent_ids=None, not_agent_ids=None, t_steps=None, convert_types=False, limit=-1):
self._read_types()
@@ -233,7 +235,8 @@ class History:
return ",".join(map(lambda x: "\'{}\'".format(x), v))
filters = [("key in ({})".format(escape_and_join(keys)), keys),
("agent_id in ({})".format(escape_and_join(agent_ids)), agent_ids)
("agent_id in ({})".format(escape_and_join(agent_ids)), agent_ids),
("agent_id not in ({})".format(escape_and_join(not_agent_ids)), not_agent_ids)
]
filters = list(k[0] for k in filters if k[1])

4
soil/serialization.py
View File

@@ -13,7 +13,6 @@ from jinja2 import Template
logger = logging.getLogger('soil')
logger.setLevel(logging.INFO)
def load_network(network_params, dir_path=None):
@@ -51,6 +50,9 @@ def load_network(network_params, dir_path=None):
def load_file(infile):
folder = os.path.dirname(infile)
if folder not in sys.path:
sys.path.append(folder)
with open(infile, 'r') as f:
return list(chain.from_iterable(map(expand_template, load_string(f))))

13
soil/simulation.py
View File

@@ -143,7 +143,7 @@ class Simulation:
return list(self.run_gen(*args, **kwargs))
def _run_sync_or_async(self, parallel=False, *args, **kwargs):
if parallel:
if parallel and not os.environ.get('SENPY_DEBUG', None):
p = Pool()
func = partial(self.run_trial_exceptions,
*args,
@@ -226,12 +226,14 @@ class Simulation:
opts.update({
'name': trial_id,
'topology': self.topology.copy(),
'network_params': self.network_params,
'seed': '{}_trial_{}'.format(self.seed, trial_id),
'initial_time': 0,
'interval': self.interval,
'network_agents': self.network_agents,
'initial_time': 0,
'states': self.states,
'dir_path': self.dir_path,
'default_state': self.default_state,
'environment_agents': self.environment_agents,
})
@@ -304,10 +306,10 @@ class Simulation:
if k[0] != '_':
state[k] = v
state['topology'] = json_graph.node_link_data(self.topology)
state['network_agents'] = agents.serialize_distribution(self.network_agents,
known_modules = [])
state['environment_agents'] = agents.serialize_distribution(self.environment_agents,
known_modules = [])
state['network_agents'] = agents.serialize_definition(self.network_agents,
known_modules = [])
state['environment_agents'] = agents.serialize_definition(self.environment_agents,
known_modules = [])
state['environment_class'] = serialization.serialize(self.environment_class,
known_modules=['soil.environment'])[1] # func, name
if state['load_module'] is None:
@@ -325,7 +327,6 @@ class Simulation:
known_modules=[self.load_module])
self.environment_class = serialization.deserialize(self.environment_class,
known_modules=[self.load_module, 'soil.environment', ]) # func, name
return state
def all_from_config(config):

84
soil/time.py Normal file
View File

@@ -0,0 +1,84 @@
from mesa.time import BaseScheduler
from queue import Empty
from heapq import heappush, heappop
import math
from .utils import logger
from mesa import Agent
class When:
def __init__(self, time):
self._time = float(time)
def abs(self, time):
return self._time
class Delta:
def __init__(self, delta):
self._delta = delta
def abs(self, time):
return time + self._delta
class TimedActivation(BaseScheduler):
"""A scheduler which activates each agent when the agent requests.
In each activation, each agent will update its 'next_time'.
"""
def __init__(self, *args, **kwargs):
super().__init__(self)
self._queue = []
self.next_time = 0
def add(self, agent: Agent):
if agent.unique_id not in self._agents:
heappush(self._queue, (self.time, agent.unique_id))
super().add(agent)
def step(self, until: float =float('inf')) -> None:
"""
Executes agents in order, one at a time. After each step,
an agent will signal when it wants to be scheduled next.
"""
when = None
agent_id = None
unsched = []
until = until or float('inf')
if not self._queue:
self.time = until
self.next_time = float('inf')
return
(when, agent_id) = self._queue[0]
if until and when > until:
self.time = until
self.next_time = when
return
self.time = when
next_time = float("inf")
while when == self.time:
heappop(self._queue)
logger.debug(f'Stepping agent {agent_id}')
when = (self._agents[agent_id].step() or Delta(1)).abs(self.time)
heappush(self._queue, (when, agent_id))
if when < next_time:
next_time = when
if not self._queue or self._queue[0][0] > self.time:
agent_id = None
break
else:
(when, agent_id) = self._queue[0]
if when and when < self.time:
raise Exception("Invalid scheduling time")
self.next_time = next_time
self.steps += 1

4
soil/utils.py
View File

@@ -7,8 +7,8 @@ from shutil import copyfile
from contextlib import contextmanager
logger = logging.getLogger('soil')
logging.basicConfig()
logger.setLevel(logging.INFO)
# logging.basicConfig()
# logger.setLevel(logging.INFO)
@contextmanager

5
soil/visualization.py Normal file
View File

@@ -0,0 +1,5 @@
from mesa.visualization.UserParam import UserSettableParameter
class UserSettableParameter(UserSettableParameter):
def __str__(self):
return self.value