balkian/soil
1
0
Fork 0
mirror of https://github.com/gsi-upm/soil synced 2024-11-21 18:52:28 +00:00
Code Issues Releases Wiki Activity

WIP: mesa compatibility

Browse Source
This commit is contained in:
J. Fernando Sánchez 2021-10-14 17:37:06 +02:00
parent e860bdb922
commit 5d7e57675a
32 changed files with 963 additions and 524 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
CHANGELOG.md
View File

@ -3,6 +3,27 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [Unreleased]
### Added
* [WIP] Integration with MESA
* `not_agent_ids` paramter to get sql in history
### Changed
* `soil.Environment` now also inherits from `mesa.Model`
* `soil.Agent` now also inherits from `mesa.Agent`
* `soil.time` to replace `simpy` events, delays, duration, etc.
### Removed
* `simpy` dependency and compatibility. Each agent used to be a simpy generator, but that made debugging and error handling more complex. That has been replaced by a scheduler within the `soil.Environment` class, similar to how `mesa` does it.
### TODO:
* agent_id -> unique_id?
* mesa has Agent.model and soil has Agent.env
* Environments.agents and mesa.Agent.agents are not the same. env is a property, and it only takes into account network and environment agents. Might rename environment_agents to other_agents or sth like that
* soil.History should mimic a mesa.datacollector :/
* soil.Simulation *could* mimic a mesa.batchrunner
* DONE include scheduler in environment
* DONE environment inherits from `mesa.Model`
## [0.15.2]
### Fixed
* Pass the right known_modules and parameters to stats discovery in simulation

6
docs/soil_tutorial.rst
View File

@ -47,12 +47,6 @@ There are three main elements in a soil simulation:
- The environment. It assigns agents to nodes in the network, and
stores the environment parameters (shared state for all agents).
Soil is based on ``simpy``, which is an event-based network simulation
library. Soil provides several abstractions over events to make
developing agents easier. This means you can use events (timeouts,
delays) in soil, but for the most part we will assume your models will
be step-based.
Modeling behaviour
------------------

21
examples/mesa/mesa.yml Normal file
View File

@ -0,0 +1,21 @@
---
name: mesa_sim
group: tests
dir_path: "/tmp"
num_trials: 3
max_time: 100
interval: 1
seed: '1'
network_params:
generator: social_wealth.graph_generator
n: 5
network_agents:
- agent_type: social_wealth.SocialMoneyAgent
weight: 1
environment_class: social_wealth.MoneyEnv
environment_params:
num_mesa_agents: 5
mesa_agent_type: social_wealth.MoneyAgent
N: 10
width: 50
height: 50

106
examples/mesa/server.py Normal file
View File

@ -0,0 +1,106 @@
from mesa.visualization.ModularVisualization import ModularServer
from soil.visualization import UserSettableParameter
from mesa.visualization.modules import ChartModule, NetworkModule, CanvasGrid
from social_wealth import MoneyEnv, graph_generator, SocialMoneyAgent
class MyNetwork(NetworkModule):
def render(self, model):
return self.portrayal_method(model)
def network_portrayal(env):
# The model ensures there is 0 or 1 agent per node
portrayal = dict()
portrayal["nodes"] = [
{
"id": agent_id,
"size": env.get_agent(agent_id).wealth,
# "color": "#CC0000" if not agents or agents[0].wealth == 0 else "#007959",
"color": "#CC0000",
"label": f"{agent_id}: {env.get_agent(agent_id).wealth}",
}
for (agent_id) in env.G.nodes
]
# import pdb;pdb.set_trace()
portrayal["edges"] = [
{"id": edge_id, "source": source, "target": target, "color": "#000000"}
for edge_id, (source, target) in enumerate(env.G.edges)
]
return portrayal
def gridPortrayal(agent):
"""
This function is registered with the visualization server to be called
each tick to indicate how to draw the agent in its current state.
:param agent: the agent in the simulation
:return: the portrayal dictionary
"""
color = max(10, min(agent.wealth*10, 100))
return {
"Shape": "rect",
"w": 1,
"h": 1,
"Filled": "true",
"Layer": 0,
"Label": agent.unique_id,
"Text": agent.unique_id,
"x": agent.pos[0],
"y": agent.pos[1],
"Color": f"rgba(31, 10, 255, 0.{color})"
}
grid = MyNetwork(network_portrayal, 500, 500, library="sigma")
chart = ChartModule(
[{"Label": "Gini", "Color": "Black"}], data_collector_name="datacollector"
)
model_params = {
"N": UserSettableParameter(
"slider",
"N",
1,
1,
10,
1,
description="Choose how many agents to include in the model",
),
"network_agents": [{"agent_type": SocialMoneyAgent}],
"height": UserSettableParameter(
"slider",
"height",
5,
5,
10,
1,
description="Grid height",
),
"width": UserSettableParameter(
"slider",
"width",
5,
5,
10,
1,
description="Grid width",
),
"network_params": {
'generator': graph_generator
},
}
canvas_element = CanvasGrid(gridPortrayal, model_params["width"].value, model_params["height"].value, 500, 500)
server = ModularServer(
MoneyEnv, [grid, chart, canvas_element], "Money Model", model_params
)
server.port = 8521
server.launch(open_browser=False)

134
examples/mesa/social_wealth.py Normal file
View File

@ -0,0 +1,134 @@
'''
This is an example that adds soil agents and environment in a normal
mesa workflow.
'''
from mesa import Agent as MesaAgent
from mesa.space import MultiGrid
# from mesa.time import RandomActivation
from mesa.datacollection import DataCollector
from mesa.batchrunner import BatchRunner
import networkx as nx
from soil import NetworkAgent, Environment
def compute_gini(model):
agent_wealths = [agent.wealth for agent in model.agents]
x = sorted(agent_wealths)
N = len(list(model.agents))
B = sum( xi * (N-i) for i,xi in enumerate(x) ) / (N*sum(x))
return (1 + (1/N) - 2*B)
class MoneyAgent(MesaAgent):
"""
A MESA agent with fixed initial wealth.
It will only share wealth with neighbors based on grid proximity
"""
def __init__(self, unique_id, model):
super().__init__(unique_id=unique_id, model=model)
self.wealth = 1
def move(self):
possible_steps = self.model.grid.get_neighborhood(
self.pos,
moore=True,
include_center=False)
print(self.pos, possible_steps)
new_position = self.random.choice(possible_steps)
print(self.pos, new_position)
self.model.grid.move_agent(self, new_position)
def give_money(self):
cellmates = self.model.grid.get_cell_list_contents([self.pos])
if len(cellmates) > 1:
other = self.random.choice(cellmates)
other.wealth += 1
self.wealth -= 1
def step(self):
self.info("Crying wolf", self.pos)
self.move()
if self.wealth > 0:
self.give_money()
class SocialMoneyAgent(NetworkAgent, MoneyAgent):
wealth = 1
def give_money(self):
cellmates = set(self.model.grid.get_cell_list_contents([self.pos]))
friends = set(self.get_neighboring_agents())
self.info("Trying to give money")
self.debug("Cellmates: ", cellmates)
self.debug("Friends: ", friends)
nearby_friends = list(cellmates & friends)
if len(nearby_friends):
other = self.random.choice(nearby_friends)
other.wealth += 1
self.wealth -= 1
class MoneyEnv(Environment):
"""A model with some number of agents."""
def __init__(self, N, width, height, *args, network_params, **kwargs):
self.initialized = True
# import pdb;pdb.set_trace()
network_params['n'] = N
super().__init__(*args, network_params=network_params, **kwargs)
self.grid = MultiGrid(width, height, False)
# self.schedule = RandomActivation(self)
self.running = True
# Create agents
for agent in self.agents:
self.schedule.add(agent)
# a = MoneyAgent(i, self)
# self.schedule.add(a)
# Add the agent to a random grid cell
x = self.random.randrange(self.grid.width)
y = self.random.randrange(self.grid.height)
self.grid.place_agent(agent, (x, y))
self.datacollector = DataCollector(
model_reporters={"Gini": compute_gini},
agent_reporters={"Wealth": "wealth"})
def step(self):
super().step()
self.datacollector.collect(self)
self.schedule.step()
def graph_generator(n=5):
G = nx.Graph()
for ix in range(n):
G.add_edge(0, ix)
return G
if __name__ == '__main__':
G = graph_generator()
fixed_params = {"topology": G,
"width": 10,
"network_agents": [{"agent_type": SocialMoneyAgent,
'weight': 1}],
"height": 10}
variable_params = {"N": range(10, 100, 10)}
batch_run = BatchRunner(MoneyEnv,
variable_parameters=variable_params,
fixed_parameters=fixed_params,
iterations=5,
max_steps=100,
model_reporters={"Gini": compute_gini})
batch_run.run_all()
run_data = batch_run.get_model_vars_dataframe()
run_data.head()
print(run_data.Gini)

83
examples/mesa/wealth.py Normal file
View File

@ -0,0 +1,83 @@
from mesa import Agent, Model
from mesa.space import MultiGrid
from mesa.time import RandomActivation
from mesa.datacollection import DataCollector
from mesa.batchrunner import BatchRunner
def compute_gini(model):
agent_wealths = [agent.wealth for agent in model.schedule.agents]
x = sorted(agent_wealths)
N = model.num_agents
B = sum( xi * (N-i) for i,xi in enumerate(x) ) / (N*sum(x))
return (1 + (1/N) - 2*B)
class MoneyAgent(Agent):
""" An agent with fixed initial wealth."""
def __init__(self, unique_id, model):
super().__init__(unique_id, model)
self.wealth = 1
def move(self):
possible_steps = self.model.grid.get_neighborhood(
self.pos,
moore=True,
include_center=False)
new_position = self.random.choice(possible_steps)
self.model.grid.move_agent(self, new_position)
def give_money(self):
cellmates = self.model.grid.get_cell_list_contents([self.pos])
if len(cellmates) > 1:
other = self.random.choice(cellmates)
other.wealth += 1
self.wealth -= 1
def step(self):
self.move()
if self.wealth > 0:
self.give_money()
class MoneyModel(Model):
"""A model with some number of agents."""
def __init__(self, N, width, height):
self.num_agents = N
self.grid = MultiGrid(width, height, True)
self.schedule = RandomActivation(self)
self.running = True
# Create agents
for i in range(self.num_agents):
a = MoneyAgent(i, self)
self.schedule.add(a)
# Add the agent to a random grid cell
x = self.random.randrange(self.grid.width)
y = self.random.randrange(self.grid.height)
self.grid.place_agent(a, (x, y))
self.datacollector = DataCollector(
model_reporters={"Gini": compute_gini},
agent_reporters={"Wealth": "wealth"})
def step(self):
self.datacollector.collect(self)
self.schedule.step()
if __name__ == '__main__':
fixed_params = {"width": 10,
"height": 10}
variable_params = {"N": range(10, 500, 10)}
batch_run = BatchRunner(MoneyModel,
variable_params,
fixed_params,
iterations=5,
max_steps=100,
model_reporters={"Gini": compute_gini})
batch_run.run_all()
run_data = batch_run.get_model_vars_dataframe()
run_data.head()
print(run_data.Gini)

50
examples/terrorism/TerroristNetworkModel.py
View File

@ -18,12 +18,12 @@ class TerroristSpreadModel(FSM, Geo):
prob_interaction
"""
def __init__(self, environment=None, agent_id=0, state=()):
super().__init__(environment=environment, agent_id=agent_id, state=state)
def __init__(self, model=None, unique_id=0, state=()):
super().__init__(model=model, unique_id=unique_id, state=state)
self.information_spread_intensity = environment.environment_params['information_spread_intensity']
self.terrorist_additional_influence = environment.environment_params['terrorist_additional_influence']
self.prob_interaction = environment.environment_params['prob_interaction']
self.information_spread_intensity = model.environment_params['information_spread_intensity']
self.terrorist_additional_influence = model.environment_params['terrorist_additional_influence']
self.prob_interaction = model.environment_params['prob_interaction']
if self['id'] == self.civilian.id: # Civilian
self.mean_belief = random.uniform(0.00, 0.5)
@ -34,10 +34,10 @@ class TerroristSpreadModel(FSM, Geo):
else:
raise Exception('Invalid state id: {}'.format(self['id']))
if 'min_vulnerability' in environment.environment_params:
self.vulnerability = random.uniform( environment.environment_params['min_vulnerability'], environment.environment_params['max_vulnerability'] )
if 'min_vulnerability' in model.environment_params:
self.vulnerability = random.uniform( model.environment_params['min_vulnerability'], model.environment_params['max_vulnerability'] )
else :
self.vulnerability = random.uniform( 0, environment.environment_params['max_vulnerability'] )
self.vulnerability = random.uniform( 0, model.environment_params['max_vulnerability'] )
@state
@ -93,11 +93,11 @@ class TrainingAreaModel(FSM, Geo):
Requires TerroristSpreadModel.
"""
def __init__(self, environment=None, agent_id=0, state=()):
super().__init__(environment=environment, agent_id=agent_id, state=state)
self.training_influence = environment.environment_params['training_influence']
if 'min_vulnerability' in environment.environment_params:
self.min_vulnerability = environment.environment_params['min_vulnerability']
def __init__(self, model=None, unique_id=0, state=()):
super().__init__(model=model, unique_id=unique_id, state=state)
self.training_influence = model.environment_params['training_influence']
if 'min_vulnerability' in model.environment_params:
self.min_vulnerability = model.environment_params['min_vulnerability']
else: self.min_vulnerability = 0
@default_state
@ -120,13 +120,13 @@ class HavenModel(FSM, Geo):
Requires TerroristSpreadModel.
"""
def __init__(self, environment=None, agent_id=0, state=()):
super().__init__(environment=environment, agent_id=agent_id, state=state)
self.haven_influence = environment.environment_params['haven_influence']
if 'min_vulnerability' in environment.environment_params:
self.min_vulnerability = environment.environment_params['min_vulnerability']
def __init__(self, model=None, unique_id=0, state=()):
super().__init__(model=model, unique_id=unique_id, state=state)
self.haven_influence = model.environment_params['haven_influence']
if 'min_vulnerability' in model.environment_params:
self.min_vulnerability = model.environment_params['min_vulnerability']
else: self.min_vulnerability = 0
self.max_vulnerability = environment.environment_params['max_vulnerability']
self.max_vulnerability = model.environment_params['max_vulnerability']
def get_occupants(self, **kwargs):
return self.get_neighboring_agents(agent_type=TerroristSpreadModel, **kwargs)
@ -162,13 +162,13 @@ class TerroristNetworkModel(TerroristSpreadModel):
weight_link_distance
"""
def __init__(self, environment=None, agent_id=0, state=()):
super().__init__(environment=environment, agent_id=agent_id, state=state)
def __init__(self, model=None, unique_id=0, state=()):
super().__init__(model=model, unique_id=unique_id, state=state)
self.vision_range = environment.environment_params['vision_range']
self.sphere_influence = environment.environment_params['sphere_influence']
self.weight_social_distance = environment.environment_params['weight_social_distance']
self.weight_link_distance = environment.environment_params['weight_link_distance']
self.vision_range = model.environment_params['vision_range']
self.sphere_influence = model.environment_params['sphere_influence']
self.weight_social_distance = model.environment_params['weight_social_distance']
self.weight_link_distance = model.environment_params['weight_link_distance']
@state
def terrorist(self):

3
requirements.txt
View File

@ -1,9 +1,8 @@
simpy>=4.0
networkx>=2.5
numpy
matplotlib
pyyaml>=5.1
pandas>=0.23
scipy>=1.3
SALib>=1.3
Jinja2
Mesa>=0.8

11
setup.py
View File

@ -16,6 +16,12 @@ def parse_requirements(filename):
install_reqs = parse_requirements("requirements.txt")
test_reqs = parse_requirements("test-requirements.txt")
extras_require={
'mesa': ['mesa>=0.8.9'],
'geo': ['scipy>=1.3'],
'web': ['tornado']
}
extras_require['all'] = [dep for package in extras_require.values() for dep in package]
setup(
@ -40,10 +46,7 @@ setup(
'Operating System :: POSIX',
'Programming Language :: Python :: 3'],
install_requires=install_reqs,
extras_require={
'web': ['tornado']
},
extras_require=extras_require,
tests_require=test_reqs,
setup_requires=['pytest-runner', ],
include_package_data=True,

2
soil/__init__.py
View File

@ -11,6 +11,7 @@ try:
except NameError:
basestring = str
from .agents import *
from . import agents
from .simulation import *
from .environment import Environment
@ -18,6 +19,7 @@ from .history import History
from . import serialization
from . import analysis
from .utils import logger
from .time import *
def main():
import argparse

41
soil/agents/BassModel.py
View File

@ -1,40 +1,31 @@
import random
from . import BaseAgent
from . import FSM, state, default_state
class BassModel(BaseAgent):
class BassModel(FSM):
"""
Settings:
innovation_prob
imitation_prob
"""
def __init__(self, environment, agent_id, state, **kwargs):
super().__init__(environment=environment, agent_id=agent_id, state=state)
env_params = environment.environment_params
self.state['sentimentCorrelation'] = 0
sentimentCorrelation = 0
def step(self):
self.behaviour()
def behaviour(self):
# Outside effects
if random.random() < self['innovation_prob']:
if self.state['id'] == 0:
self.state['id'] = 1
self.state['sentimentCorrelation'] = 1
else:
pass
return
# Imitation effects
if self.state['id'] == 0:
aware_neighbors = self.get_neighboring_agents(state_id=1)
@default_state
@state
def innovation(self):
if random.random() < self.innovation_prob:
self.sentimentCorrelation = 1
return self.aware
else:
aware_neighbors = self.get_neighboring_agents(state_id=self.aware.id)
num_neighbors_aware = len(aware_neighbors)
if random.random() < (self['imitation_prob']*num_neighbors_aware):
self.state['id'] = 1
self.state['sentimentCorrelation'] = 1
self.sentimentCorrelation = 1
return self.aware
else:
pass
@state
def aware(self):
self.die()

39
soil/agents/BigMarketModel.py
View File

@ -1,8 +1,8 @@
import random
from . import BaseAgent
from . import FSM, state, default_state
class BigMarketModel(BaseAgent):
class BigMarketModel(FSM):
"""
Settings:
Names:
@ -19,34 +19,25 @@ class BigMarketModel(BaseAgent):
sentiment_about [Array]
"""
def __init__(self, environment=None, agent_id=0, state=()):
super().__init__(environment=environment, agent_id=agent_id, state=state)
self.enterprises = environment.environment_params['enterprises']
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.enterprises = self.env.environment_params['enterprises']
self.type = ""
self.number_of_enterprises = len(environment.environment_params['enterprises'])
if self.id < self.number_of_enterprises: # Enterprises
self.state['id'] = self.id
if self.id < len(self.enterprises): # Enterprises
self.set_state(self.enterprise.id)
self.type = "Enterprise"
self.tweet_probability = environment.environment_params['tweet_probability_enterprises'][self.id]
else: # normal users
self.state['id'] = self.number_of_enterprises
self.type = "User"
self.set_state(self.user.id)
self.tweet_probability = environment.environment_params['tweet_probability_users']
self.tweet_relevant_probability = environment.environment_params['tweet_relevant_probability']
self.tweet_probability_about = environment.environment_params['tweet_probability_about'] # List
self.sentiment_about = environment.environment_params['sentiment_about'] # List
def step(self):
if self.id < self.number_of_enterprises: # Enterprise
self.enterpriseBehaviour()
else: # Usuario
self.userBehaviour()
for i in range(self.number_of_enterprises): # So that it never is set to 0 if there are not changes (logs)
self.attrs['sentiment_enterprise_%s'% self.enterprises[i]] = self.sentiment_about[i]
def enterpriseBehaviour(self):
@state
def enterprise(self):
if random.random() < self.tweet_probability: # Tweets
aware_neighbors = self.get_neighboring_agents(state_id=self.number_of_enterprises) # Nodes neighbour users
@ -64,12 +55,12 @@ class BigMarketModel(BaseAgent):
x.attrs['sentiment_enterprise_%s'% self.enterprises[self.id]] = x.sentiment_about[self.id]
def userBehaviour(self):
@state
def user(self):
if random.random() < self.tweet_probability: # Tweets
if random.random() < self.tweet_relevant_probability: # Tweets something relevant
# Tweet probability per enterprise
for i in range(self.number_of_enterprises):
for i in range(len(self.enterprises)):
random_num = random.random()
if random_num < self.tweet_probability_about[i]:
# The condition is fulfilled, sentiments are evaluated towards that enterprise
@ -82,8 +73,10 @@ class BigMarketModel(BaseAgent):
else:
# POSITIVO
self.userTweets("positive",i)
for i in range(len(self.enterprises)): # So that it never is set to 0 if there are not changes (logs)
self.attrs['sentiment_enterprise_%s'% self.enterprises[i]] = self.sentiment_about[i]
def userTweets(self,sentiment,enterprise):
def userTweets(self, sentiment,enterprise):
aware_neighbors = self.get_neighboring_agents(state_id=self.number_of_enterprises) # Nodes neighbours users
for x in aware_neighbors:
if sentiment == "positive":

20
soil/agents/Geo.py Normal file
View File

@ -0,0 +1,20 @@
from scipy.spatial import cKDTree as KDTree
from . import NetworkAgent
class Geo(NetworkAgent):
'''In this type of network, nodes have a "pos" attribute.'''
def geo_search(self, radius, node=None, center=False, **kwargs):
'''Get a list of nodes whose coordinates are closer than *radius* to *node*.'''
node = as_node(node if node is not None else self)
G = self.subgraph(**kwargs)
pos = nx.get_node_attributes(G, 'pos')
if not pos:
return []
nodes, coords = list(zip(*pos.items()))
kdtree = KDTree(coords) # Cannot provide generator.
indices = kdtree.query_ball_point(pos[node], radius)
return [nodes[i] for i in indices if center or (nodes[i] != node)]

8
soil/agents/IndependentCascadeModel.py
View File

@ -10,10 +10,10 @@ class IndependentCascadeModel(BaseAgent):
imitation_prob
"""
def __init__(self, environment=None, agent_id=0, state=()):
super().__init__(environment=environment, agent_id=agent_id, state=state)
self.innovation_prob = environment.environment_params['innovation_prob']
self.imitation_prob = environment.environment_params['imitation_prob']
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.innovation_prob = self.env.environment_params['innovation_prob']
self.imitation_prob = self.env.environment_params['imitation_prob']
self.state['time_awareness'] = 0
self.state['sentimentCorrelation'] = 0

8
soil/agents/ModelM2.py
View File

@ -21,8 +21,8 @@ class SpreadModelM2(BaseAgent):
prob_generate_anti_rumor
"""
def __init__(self, environment=None, agent_id=0, state=()):
super().__init__(environment=environment, agent_id=agent_id, state=state)
def __init__(self, model=None, unique_id=0, state=()):
super().__init__(model=environment, unique_id=unique_id, state=state)
self.prob_neutral_making_denier = np.random.normal(environment.environment_params['prob_neutral_making_denier'],
environment.environment_params['standard_variance'])
@ -123,8 +123,8 @@ class ControlModelM2(BaseAgent):
"""
def __init__(self, environment=None, agent_id=0, state=()):
super().__init__(environment=environment, agent_id=agent_id, state=state)
def __init__(self, model=None, unique_id=0, state=()):
super().__init__(model=environment, unique_id=unique_id, state=state)
self.prob_neutral_making_denier = np.random.normal(environment.environment_params['prob_neutral_making_denier'],
environment.environment_params['standard_variance'])

4
soil/agents/SISaModel.py
View File

@ -29,8 +29,8 @@ class SISaModel(FSM):
standard_variance
"""
def __init__(self, environment, agent_id=0, state=()):
super().__init__(environment=environment, agent_id=agent_id, state=state)
def __init__(self, environment, unique_id=0, state=()):
super().__init__(model=environment, unique_id=unique_id, state=state)
self.neutral_discontent_spon_prob = np.random.normal(self.env['neutral_discontent_spon_prob'],
self.env['standard_variance'])

4
soil/agents/SentimentCorrelationModel.py
View File

@ -16,8 +16,8 @@ class SentimentCorrelationModel(BaseAgent):
disgust_prob
"""
def __init__(self, environment, agent_id=0, state=()):
super().__init__(environment=environment, agent_id=agent_id, state=state)
def __init__(self, environment, unique_id=0, state=()):
super().__init__(model=environment, unique_id=unique_id, state=state)
self.outside_effects_prob = environment.environment_params['outside_effects_prob']
self.anger_prob = environment.environment_params['anger_prob']
self.joy_prob = environment.environment_params['joy_prob']

318
soil/agents/__init__.py
View File

@ -1,21 +1,15 @@
# networkStatus = {} # Dict that will contain the status of every agent in the network
# sentimentCorrelationNodeArray = []
# for x in range(0, settings.network_params["number_of_nodes"]):
# sentimentCorrelationNodeArray.append({'id': x})
# Initialize agent states. Let's assume everyone is normal.
import logging
from collections import OrderedDict
from collections import OrderedDict, defaultdict
from copy import deepcopy
from functools import partial
from scipy.spatial import cKDTree as KDTree
import json
import simpy
import networkx as nx
from functools import wraps
from .. import serialization, history, utils
from .. import serialization, history, utils, time
from mesa import Agent
def as_node(agent):
@ -24,39 +18,51 @@ def as_node(agent):
return agent
class BaseAgent:
class BaseAgent(Agent):
"""
A special simpy BaseAgent that keeps track of its state history.
A special Agent that keeps track of its state history.
"""
defaults = {}
def __init__(self, environment, agent_id, state=None,
name=None, interval=None):
def __init__(self,
unique_id,
model,
state=None,
name=None,
interval=None):
# Check for REQUIRED arguments
assert environment is not None, TypeError('__init__ missing 1 required keyword argument: \'environment\'. '
'Cannot be NoneType.')
# Initialize agent parameters
self.id = agent_id
self.name = name or '{}[{}]'.format(type(self).__name__, self.id)
# Register agent to environment
self.env = environment
if isinstance(unique_id, Agent):
raise Exception()
super().__init__(unique_id=unique_id, model=model)
self.name = name or '{}[{}]'.format(type(self).__name__, self.unique_id)
self._neighbors = None
self.alive = True
real_state = deepcopy(self.defaults)
real_state.update(state or {})
self.state = real_state
self.interval = interval
self.logger = logging.getLogger(self.env.name).getChild(self.name)
self.interval = interval or self.get('interval', getattr(self.model, 'interval', 1))
self.logger = logging.getLogger(self.model.name).getChild(self.name)
if hasattr(self, 'level'):
self.logger.setLevel(self.level)
# initialize every time an instance of the agent is created
self.action = self.env.process(self.run())
# TODO: refactor to clean up mesa compatibility
@property
def id(self):
return self.unique_id
@property
def env(self):
return self.model
@env.setter
def env(self, model):
self.model = model
@property
def state(self):
@ -76,17 +82,17 @@ class BaseAgent:
@property
def environment_params(self):
return self.env.environment_params
return self.model.environment_params
@environment_params.setter
def environment_params(self, value):
self.env.environment_params = value
self.model.environment_params = value
def __getitem__(self, key):
if isinstance(key, tuple):
key, t_step = key
k = history.Key(key=key, t_step=t_step, agent_id=self.id)
return self.env[k]
return self.model[k]
return self._state.get(key, None)
def __delitem__(self, key):
@ -100,7 +106,7 @@ class BaseAgent:
k = history.Key(t_step=self.now,
agent_id=self.id,
key=key)
self.env[k] = value
self.model[k] = value
def items(self):
return self._state.items()
@ -111,29 +117,33 @@ class BaseAgent:
@property
def now(self):
try:
return self.env.now
return self.model.now
except AttributeError:
# No environment
return None
def run(self):
if self.interval is not None:
interval = self.interval
elif 'interval' in self:
interval = self['interval']
else:
interval = self.env.interval
while self.alive:
res = self.step()
yield res or self.env.timeout(interval)
def die(self, remove=False):
self.alive = False
if remove:
self.remove_node(self.id)
def step(self):
return
if not self.alive:
return time.When('inf')
return super().step() or time.Delta(self.interval)
def log(self, message, *args, level=logging.INFO, **kwargs):
if not self.logger.isEnabledFor(level):
return
message = message + " ".join(str(i) for i in args)
message = " @{:>3}: {}".format(self.now, message)
for k, v in kwargs:
message += " {k}={v} ".format(k, v)
extra = {}
extra['now'] = self.now
extra['unique_id'] = self.unique_id
extra['agent_name'] = self.name
return self.logger.log(level, message, extra=extra)
def debug(self, *args, **kwargs):
return self.log(*args, level=logging.DEBUG, **kwargs)
@ -149,7 +159,7 @@ class BaseAgent:
'''
state = {}
state['id'] = self.id
state['environment'] = self.env
state['environment'] = self.model
state['_state'] = self._state
return state
@ -157,19 +167,19 @@ class BaseAgent:
'''
Get back a serialized agent and try to re-compose it
'''
self.id = state['id']
self.state_id = state['id']
self._state = state['_state']
self.env = state['environment']
self.model = state['environment']
class NetworkAgent(BaseAgent):
@property
def topology(self):
return self.env.G
return self.model.G
@property
def G(self):
return self.env.G
return self.model.G
def count_agents(self, **kwargs):
return len(list(self.get_agents(**kwargs)))
@ -182,37 +192,26 @@ class NetworkAgent(BaseAgent):
def get_agents(self, agents=None, limit_neighbors=False, **kwargs):
if limit_neighbors:
agents = self.topology.neighbors(self.id)
agents = self.topology.neighbors(self.unique_id)
agents = self.env.get_agents(agents)
agents = self.model.get_agents(agents)
return select(agents, **kwargs)
def log(self, message, *args, level=logging.INFO, **kwargs):
message = message + " ".join(str(i) for i in args)
message = " @{:>3}: {}".format(self.now, message)
for k, v in kwargs:
message += " {k}={v} ".format(k, v)
extra = {}
extra['now'] = self.now
extra['agent_id'] = self.id
extra['agent_name'] = self.name
return self.logger.log(level, message, extra=extra)
def subgraph(self, center=True, **kwargs):
include = [self] if center else []
return self.topology.subgraph(n.id for n in self.get_agents(**kwargs)+include)
return self.topology.subgraph(n.unique_id for n in self.get_agents(**kwargs)+include)
def remove_node(self, agent_id):
self.topology.remove_node(agent_id)
def remove_node(self, unique_id):
self.topology.remove_node(unique_id)
def add_edge(self, other, edge_attr_dict=None, *edge_attrs):
# return super(NetworkAgent, self).add_edge(node1=self.id, node2=other, **kwargs)
if self.id not in self.topology.nodes(data=False):
raise ValueError('{} not in list of existing agents in the network'.format(self.id))
if self.unique_id not in self.topology.nodes(data=False):
raise ValueError('{} not in list of existing agents in the network'.format(self.unique_id))
if other not in self.topology.nodes(data=False):
raise ValueError('{} not in list of existing agents in the network'.format(other))
self.topology.add_edge(self.id, other, edge_attr_dict=edge_attr_dict, *edge_attrs)
self.topology.add_edge(self.unique_id, other, edge_attr_dict=edge_attr_dict, *edge_attrs)
def ego_search(self, steps=1, center=False, node=None, **kwargs):
@ -223,17 +222,17 @@ class NetworkAgent(BaseAgent):
def degree(self, node, force=False):
node = as_node(node)
if force or (not hasattr(self.env, '_degree')) or getattr(self.env, '_last_step', 0) < self.now:
self.env._degree = nx.degree_centrality(self.topology)
self.env._last_step = self.now
return self.env._degree[node]
if force or (not hasattr(self.model, '_degree')) or getattr(self.model, '_last_step', 0) < self.now:
self.model._degree = nx.degree_centrality(self.topology)
self.model._last_step = self.now
return self.model._degree[node]
def betweenness(self, node, force=False):
node = as_node(node)
if force or (not hasattr(self.env, '_betweenness')) or getattr(self.env, '_last_step', 0) < self.now:
self.env._betweenness = nx.betweenness_centrality(self.topology)
self.env._last_step = self.now
return self.env._betweenness[node]
if force or (not hasattr(self.model, '_betweenness')) or getattr(self.model, '_last_step', 0) < self.now:
self.model._betweenness = nx.betweenness_centrality(self.topology)
self.model._last_step = self.now
return self.model._betweenness[node]
def state(name=None):
@ -301,36 +300,29 @@ class FSM(NetworkAgent, metaclass=MetaFSM):
super(FSM, self).__init__(*args, **kwargs)
if 'id' not in self.state:
if not self.default_state:
raise ValueError('No default state specified for {}'.format(self.id))
raise ValueError('No default state specified for {}'.format(self.unique_id))
self['id'] = self.default_state.id
self._next_change = simpy.core.Infinity
self._next_state = self.state
self.set_state(self.state['id'])
def step(self):
if self._next_change < self.now:
next_state = self._next_state
self._next_change = simpy.core.Infinity
self['id'] = next_state
elif 'id' in self.state:
next_state = self['id']
elif self.default_state:
next_state = self.default_state.id
else:
raise Exception('{} has no valid state id or default state'.format(self))
if next_state not in self.states:
raise Exception('{} is not a valid id for {}'.format(next_state, self))
return self.states[next_state](self)
def next_state(self, state):
self._next_change = self.now
self._next_state = state
self.debug(f'Agent {self.unique_id} @ state {self["id"]}')
interval = super().step()
if 'id' not in self:
if 'id' in self.state:
self.set_state(self['state_id'])
elif self.default_state:
self.set_state(self.default_state.id)
else:
raise Exception('{} has no valid state id or default state'.format(self))
return self.states[self['id']](self) or interval
def set_state(self, state):
if hasattr(state, 'id'):
state = state.id
if state not in self.states:
raise ValueError('{} is not a valid state'.format(state))
self['id'] = state
self['state_id'] = state
return state
@ -349,9 +341,6 @@ def prob(prob=1):
return r < prob
STATIC_THRESHOLD = (-1, -1)
def calculate_distribution(network_agents=None,
agent_type=None):
'''
@ -379,7 +368,7 @@ def calculate_distribution(network_agents=None,
'agent_type_1'.
'''
if network_agents:
network_agents = deepcopy(network_agents)
network_agents = [deepcopy(agent) for agent in network_agents if not hasattr(agent, 'id')]
elif agent_type:
network_agents = [{'agent_type': agent_type}]
else:
@ -394,7 +383,6 @@ def calculate_distribution(network_agents=None,
acc = 0
for v in network_agents:
if 'ids' in v:
v['threshold'] = STATIC_THRESHOLD
continue
upper = acc + (v['weight']/total)
v['threshold'] = [acc, upper]
@ -409,7 +397,7 @@ def serialize_type(agent_type, known_modules=[], **kwargs):
return serialization.serialize(agent_type, known_modules=known_modules, **kwargs)[1] # Get the name of the class
def serialize_distribution(network_agents, known_modules=[]):
def serialize_definition(network_agents, known_modules=[]):
'''
When serializing an agent distribution, remove the thresholds, in order
to avoid cluttering the YAML definition file.
@ -431,7 +419,7 @@ def deserialize_type(agent_type, known_modules=[]):
return agent_type
def deserialize_distribution(ind, **kwargs):
def deserialize_definition(ind, **kwargs):
d = deepcopy(ind)
for v in d:
v['agent_type'] = deserialize_type(v['agent_type'], **kwargs)
@ -452,44 +440,84 @@ def _validate_states(states, topology):
def _convert_agent_types(ind, to_string=False, **kwargs):
'''Convenience method to allow specifying agents by class or class name.'''
if to_string:
return serialize_distribution(ind, **kwargs)
return deserialize_distribution(ind, **kwargs)
return serialize_definition(ind, **kwargs)
return deserialize_definition(ind, **kwargs)
def _agent_from_distribution(distribution, value=-1, agent_id=None):
def _agent_from_definition(definition, value=-1, unique_id=None):
"""Used in the initialization of agents given an agent distribution."""
if value < 0:
value = random.random()
for d in sorted(distribution, key=lambda x: x['threshold']):
threshold = d['threshold']
for d in sorted(definition, key=lambda x: x.get('threshold')):
threshold = d.get('threshold', (-1, -1))
# Check if the definition matches by id (first) or by threshold
if not ((agent_id is not None and threshold == STATIC_THRESHOLD and agent_id in d['ids']) or \
(value >= threshold[0] and value < threshold[1])):
continue
state = {}
if 'state' in d:
state = deepcopy(d['state'])
return d['agent_type'], state
if (unique_id is not None and unique_id in d.get('ids', [])) or \
(value >= threshold[0] and value < threshold[1]):
state = {}
if 'state' in d:
state = deepcopy(d['state'])
return d['agent_type'], state
raise Exception('Distribution for value {} not found in: {}'.format(value, distribution))
raise Exception('Definition for value {} not found in: {}'.format(value, definition))
class Geo(NetworkAgent):
'''In this type of network, nodes have a "pos" attribute.'''
def _definition_to_dict(definition, size=None, default_state=None):
state = default_state or {}
agents = {}
remaining = {}
if size:
for ix in range(size):
remaining[ix] = copy(state)
else:
remaining = defaultdict(lambda x: copy(state))
def geo_search(self, radius, node=None, center=False, **kwargs):
'''Get a list of nodes whose coordinates are closer than *radius* to *node*.'''
node = as_node(node if node is not None else self)
distro = sorted([item for item in definition if 'weight' in item])
G = self.subgraph(**kwargs)
ix = 0
def init_agent(item, id=ix):
while id in agents:
id += 1
pos = nx.get_node_attributes(G, 'pos')
if not pos:
return []
nodes, coords = list(zip(*pos.items()))
kdtree = KDTree(coords) # Cannot provide generator.
indices = kdtree.query_ball_point(pos[node], radius)
return [nodes[i] for i in indices if center or (nodes[i] != node)]
agent = remaining[id]
agent['state'].update(copy(item.get('state', {})))
agents[id] = agent
del remaining[id]
return agent
for item in definition:
if 'ids' in item:
ids = item['ids']
del item['ids']
for id in ids:
agent = init_agent(item, id)
for item in definition:
if 'number' in item:
times = item['number']
del item['number']
for times in range(times):
if size:
ix = random.choice(remaining.keys())
agent = init_agent(item, id)
else:
agent = init_agent(item)
if not size:
return agents
if len(remaining) < 0:
raise Exception('Invalid definition. Too many agents to add')
total_weight = float(sum(s['weight'] for s in distro))
unit = size / total_weight
for item in distro:
times = unit * item['weight']
del item['weight']
for times in range(times):
ix = random.choice(remaining.keys())
agent = init_agent(item, id)
return agents
def select(agents, state_id=None, agent_type=None, ignore=None, iterator=False, **kwargs):
@ -502,22 +530,21 @@ def select(agents, state_id=None, agent_type=None, ignore=None, iterator=False,
except TypeError:
agent_type = tuple([agent_type])
def matches_all(agent):
if state_id is not None:
if agent.state.get('id', None) not in state_id:
return False
if agent_type is not None:
if not isinstance(agent, agent_type):
return False
state = agent.state
for k, v in kwargs.items():
if state.get(k, None) != v:
return False
return True
checks = []
f = agents
f = filter(matches_all, agents)
if ignore:
f = filter(lambda x: x not in ignore, f)
if state_id is not None:
f = filter(lambda agent: agent.state.get('id', None) in state_id, f)
if agent_type is not None:
f = filter(lambda agent: isinstance(agent, agent_type), f)
for k, v in kwargs.items():
f = filter(lambda agent: agent.state.get(k, None) == v, f)
if iterator:
return f
return list(f)
@ -530,3 +557,10 @@ from .ModelM2 import *
from .SentimentCorrelationModel import *
from .SISaModel import *
from .CounterModel import *
try:
import scipy
from .Geo import Geo
except ImportError:
import sys
print('Could not load the Geo Agent, scipy is not installed', file=sys.stderr)

33
soil/analysis.py
View File

@ -61,7 +61,12 @@ def convert_row(row):
def convert_types_slow(df):
'''This is a slow operation.'''
'''
Go over every column in a dataframe and convert it to the type determined by the `get_types`
function.
This is a slow operation.
'''
dtypes = get_types(df)
for k, v in dtypes.items():
t = df[df['key']==k]
@ -102,6 +107,9 @@ def process(df, **kwargs):
def get_types(df):
'''
Get the value type for every key stored in a raw history dataframe.
'''
dtypes = df.groupby(by=['key'])['value_type'].unique()
return {k:v[0] for k,v in dtypes.iteritems()}
@ -126,8 +134,14 @@ def process_one(df, *keys, columns=['key', 'agent_id'], values='value',
def get_count(df, *keys):
'''
For every t_step and key, get the value count.
The result is a dataframe with `t_step` as index, an a multiindex column based on `key` and the values found for each `key`.
'''
if keys:
df = df[list(keys)]
df.columns = df.columns.remove_unused_levels()
counts = pd.DataFrame()
for key in df.columns.levels[0]:
g = df[[key]].apply(pd.Series.value_counts, axis=1).fillna(0)
@ -137,10 +151,25 @@ def get_count(df, *keys):
return counts
def get_majority(df, *keys):
'''
For every t_step and key, get the value of the majority of agents
The result is a dataframe with `t_step` as index, and columns based on `key`.
'''
df = get_count(df, *keys)
return df.stack(level=0).idxmax(axis=1).unstack()
def get_value(df, *keys, aggfunc='sum'):
'''
For every t_step and key, get the value of *numeric columns*, aggregated using a specific function.
'''
if keys:
df = df[list(keys)]
return df.groupby(axis=1, level=0).agg(aggfunc)
df.columns = df.columns.remove_unused_levels()
df = df.select_dtypes('number')
return df.groupby(level='key', axis=1).agg(aggfunc)
def plot_all(*args, plot_args={}, **kwargs):

26
soil/datacollection.py Normal file
View File

@ -0,0 +1,26 @@
from mesa import DataCollector as MDC
class SoilDataCollector(MDC):
def __init__(self, environment, *args, **kwargs):
super().__init__(*args, **kwargs)
# Populate model and env reporters so they have a key per
# So they can be shown in the web interface
self.environment = environment
@property
def model_vars(self):
pass
@model_vars.setter
def model_vars(self, value):
pass
@property
def agent_reporters(self):
self.model._history._
pass

116
soil/environment.py
View File

@ -1,28 +1,29 @@
import os
import sqlite3
import time
import csv
import math
import random
import simpy
import yaml
import tempfile
import pandas as pd
from time import time as current_time
from copy import deepcopy
from networkx.readwrite import json_graph
import networkx as nx
import simpy
from . import serialization, agents, analysis, history, utils
from mesa import Model
from . import serialization, agents, analysis, history, utils, time
# These properties will be copied when pickling/unpickling the environment
_CONFIG_PROPS = [ 'name',
'states',
'default_state',
'interval',
'states',
'default_state',
'interval',
]
class Environment(simpy.Environment):
class Environment(Model):
"""
The environment is key in a simulation. It contains the network topology,
a reference to network and environment agents, as well as the environment
@ -39,25 +40,41 @@ class Environment(simpy.Environment):
states=None,
default_state=None,
interval=1,
network_params=None,
seed=None,
topology=None,
schedule=None,
initial_time=0,
**environment_params):
environment_params=None,
dir_path=None,
**kwargs):
super().__init__()
self.schedule = schedule
if schedule is None:
self.schedule = time.TimedActivation()
self.name = name or 'UnnamedEnvironment'
seed = seed or time.time()
seed = seed or current_time()
random.seed(seed)
if isinstance(states, list):
states = dict(enumerate(states))
self.states = deepcopy(states) if states else {}
self.default_state = deepcopy(default_state) or {}
if topology is None:
network_params = network_params or {}
topology = serialization.load_network(network_params,
dir_path=dir_path)
if not topology:
topology = nx.Graph()
self.G = nx.Graph(topology)
super().__init__(initial_time=initial_time)
self.environment_params = environment_params
self.environment_params = environment_params or {}
self.environment_params.update(kwargs)
self._env_agents = {}
self.interval = interval
@ -66,8 +83,26 @@ class Environment(simpy.Environment):
self['SEED'] = seed
# Add environment agents first, so their events get
# executed before network agents
self.environment_agents = environment_agents or []
self.network_agents = network_agents or []
if network_agents:
distro = agents.calculate_distribution(network_agents)
self.network_agents = agents._convert_agent_types(distro)
else:
self.network_agents = []
environment_agents = environment_agents or []
if environment_agents:
distro = agents.calculate_distribution(environment_agents)
environment_agents = agents._convert_agent_types(distro)
self.environment_agents = environment_agents
@property
def now(self):
if self.schedule:
return self.schedule.time
raise Exception('The environment has not been scheduled, so it has no sense of time')
@property
def agents(self):
@ -81,15 +116,9 @@ class Environment(simpy.Environment):
@environment_agents.setter
def environment_agents(self, environment_agents):
# Set up environmental agent
self._env_agents = {}
for item in environment_agents:
kwargs = deepcopy(item)
atype = kwargs.pop('agent_type')
kwargs['agent_id'] = kwargs.get('agent_id', atype.__name__)
kwargs['state'] = kwargs.get('state', {})
a = atype(environment=self, **kwargs)
self._env_agents[a.id] = a
self._environment_agents = environment_agents
self._env_agents = agents._definition_to_dict(definition=environment_agents)
@property
def network_agents(self):
@ -102,9 +131,9 @@ class Environment(simpy.Environment):
def network_agents(self, network_agents):
self._network_agents = network_agents
for ix in self.G.nodes():
self.init_agent(ix, agent_distribution=network_agents)
self.init_agent(ix, agent_definitions=network_agents)
def init_agent(self, agent_id, agent_distribution):
def init_agent(self, agent_id, agent_definitions):
node = self.G.nodes[agent_id]
init = False
state = dict(node)
@ -119,8 +148,8 @@ class Environment(simpy.Environment):
if agent_type:
agent_type = agents.deserialize_type(agent_type)
elif agent_distribution:
agent_type, state = agents._agent_from_distribution(agent_distribution, agent_id=agent_id)
elif agent_definitions:
agent_type, state = agents._agent_from_definition(agent_definitions, unique_id=agent_id)
else:
serialization.logger.debug('Skipping node {}'.format(agent_id))
return
@ -136,8 +165,8 @@ class Environment(simpy.Environment):
a = None
if agent_type:
state = defstate
a = agent_type(environment=self,
agent_id=agent_id,
a = agent_type(model=self,
unique_id=agent_id,
state=state)
node['agent'] = a
return a
@ -159,30 +188,18 @@ class Environment(simpy.Environment):
def run(self, until, *args, **kwargs):
self._save_state()
super().run(until, *args, **kwargs)
for agent in self.agents:
self.schedule.add(agent)
while self.schedule.next_time <= until and not math.isinf(self.schedule.next_time):
self.schedule.step(until=until)
utils.logger.debug(f'Simulation step {self.schedule.time}/{until}. Next: {self.schedule.next_time}')
self._history.flush_cache()
def _save_state(self, now=None):
serialization.logger.debug('Saving state @{}'.format(self.now))
self._history.save_records(self.state_to_tuples(now=now))
def save_state(self):
'''
:DEPRECATED:
Periodically save the state of the environment and the agents.
'''
self._save_state()
while self.peek() != simpy.core.Infinity:
delay = max(self.peek() - self.now, self.interval)
serialization.logger.debug('Step: {}'.format(self.now))
ev = self.event()
ev._ok = True
# Schedule the event with minimum priority so
# that it executes before all agents
self.schedule(ev, -999, delay)
yield ev
self._save_state()
def __getitem__(self, key):
if isinstance(key, tuple):
self._history.flush_cache()
@ -329,7 +346,7 @@ class Environment(simpy.Environment):
state['G'] = json_graph.node_link_data(self.G)
state['environment_agents'] = self._env_agents
state['history'] = self._history
state['_now'] = self._now
state['schedule'] = self.schedule
return state
def __setstate__(self, state):
@ -338,7 +355,8 @@ class Environment(simpy.Environment):
self._env_agents = state['environment_agents']
self.G = json_graph.node_link_graph(state['G'])
self._history = state['history']
self._now = state['_now']
# self._env = None
self.schedule = state['schedule']
self._queue = []

17
soil/history.py
View File

@ -52,7 +52,7 @@ class History:
with self.db:
logger.debug('Creating database {}'.format(self.db_path))
self.db.execute('''CREATE TABLE IF NOT EXISTS history (agent_id text, t_step int, key text, value text)''')
self.db.execute('''CREATE TABLE IF NOT EXISTS history (agent_id text, t_step real, key text, value text)''')
self.db.execute('''CREATE TABLE IF NOT EXISTS value_types (key text, value_type text)''')
self.db.execute('''CREATE TABLE IF NOT EXISTS stats (trial_id text)''')
self.db.execute('''CREATE UNIQUE INDEX IF NOT EXISTS idx_history ON history (agent_id, t_step, key);''')
@ -103,7 +103,7 @@ class History:
dtype = 'real'
int(value)
dtype = 'int'
except ValueError:
except (ValueError, OverflowError):
pass
self.db.execute('ALTER TABLE stats ADD "{}" "{}"'.format(column, dtype))
self._stats_columns.append(column)
@ -167,6 +167,7 @@ class History:
with self.db:
self.db.execute("replace into value_types (key, value_type) values (?, ?)", (key, name))
value = self._dtypes[key][1](value)
self._tups.append(Record(agent_id=agent_id,
t_step=t_step,
key=key,
@ -183,9 +184,9 @@ class History:
raise Exception('DB in readonly mode')
logger.debug('Flushing cache {}'.format(self.db_path))
with self.db:
for rec in self._tups:
self.db.execute("replace into history(agent_id, t_step, key, value) values (?, ?, ?, ?)", (rec.agent_id, rec.t_step, rec.key, rec.value))
self._tups = list()
self.db.executemany("replace into history(agent_id, t_step, key, value) values (?, ?, ?, ?)", self._tups)
# (rec.agent_id, rec.t_step, rec.key, rec.value))
self._tups.clear()
def to_tuples(self):
self.flush_cache()
@ -209,6 +210,7 @@ class History:
self._dtypes[k] = (v, serializer, deserializer)
def __getitem__(self, key):
# raise NotImplementedError()
self.flush_cache()
key = Key(*key)
agent_ids = [key.agent_id] if key.agent_id is not None else []
@ -223,7 +225,7 @@ class History:
return r.value()
return r
def read_sql(self, keys=None, agent_ids=None, t_steps=None, convert_types=False, limit=-1):
def read_sql(self, keys=None, agent_ids=None, not_agent_ids=None, t_steps=None, convert_types=False, limit=-1):
self._read_types()
@ -233,7 +235,8 @@ class History:
return ",".join(map(lambda x: "\'{}\'".format(x), v))
filters = [("key in ({})".format(escape_and_join(keys)), keys),
("agent_id in ({})".format(escape_and_join(agent_ids)), agent_ids)
("agent_id in ({})".format(escape_and_join(agent_ids)), agent_ids),
("agent_id not in ({})".format(escape_and_join(not_agent_ids)), not_agent_ids)
]
filters = list(k[0] for k in filters if k[1])

4
soil/serialization.py
View File

@ -13,7 +13,6 @@ from jinja2 import Template
logger = logging.getLogger('soil')
logger.setLevel(logging.INFO)
def load_network(network_params, dir_path=None):
@ -51,6 +50,9 @@ def load_network(network_params, dir_path=None):
def load_file(infile):
folder = os.path.dirname(infile)
if folder not in sys.path:
sys.path.append(folder)
with open(infile, 'r') as f:
return list(chain.from_iterable(map(expand_template, load_string(f))))

13
soil/simulation.py
View File

@ -143,7 +143,7 @@ class Simulation:
return list(self.run_gen(*args, **kwargs))
def _run_sync_or_async(self, parallel=False, *args, **kwargs):
if parallel:
if parallel and not os.environ.get('SENPY_DEBUG', None):
p = Pool()
func = partial(self.run_trial_exceptions,
*args,
@ -226,12 +226,14 @@ class Simulation:
opts.update({
'name': trial_id,
'topology': self.topology.copy(),
'network_params': self.network_params,
'seed': '{}_trial_{}'.format(self.seed, trial_id),
'initial_time': 0,
'interval': self.interval,
'network_agents': self.network_agents,
'initial_time': 0,
'states': self.states,
'dir_path': self.dir_path,
'default_state': self.default_state,
'environment_agents': self.environment_agents,
})
@ -304,10 +306,10 @@ class Simulation:
if k[0] != '_':
state[k] = v
state['topology'] = json_graph.node_link_data(self.topology)
state['network_agents'] = agents.serialize_distribution(self.network_agents,
known_modules = [])
state['environment_agents'] = agents.serialize_distribution(self.environment_agents,
known_modules = [])
state['network_agents'] = agents.serialize_definition(self.network_agents,
known_modules = [])
state['environment_agents'] = agents.serialize_definition(self.environment_agents,
known_modules = [])
state['environment_class'] = serialization.serialize(self.environment_class,
known_modules=['soil.environment'])[1] # func, name
if state['load_module'] is None:
@ -325,7 +327,6 @@ class Simulation:
known_modules=[self.load_module])
self.environment_class = serialization.deserialize(self.environment_class,
known_modules=[self.load_module, 'soil.environment', ]) # func, name
return state
def all_from_config(config):

84
soil/time.py Normal file
View File

@ -0,0 +1,84 @@
from mesa.time import BaseScheduler
from queue import Empty
from heapq import heappush, heappop
import math
from .utils import logger
from mesa import Agent
class When:
def __init__(self, time):
self._time = float(time)
def abs(self, time):
return self._time
class Delta:
def __init__(self, delta):
self._delta = delta
def abs(self, time):
return time + self._delta
class TimedActivation(BaseScheduler):
"""A scheduler which activates each agent when the agent requests.
In each activation, each agent will update its 'next_time'.
"""
def __init__(self, *args, **kwargs):
super().__init__(self)
self._queue = []
self.next_time = 0
def add(self, agent: Agent):
if agent.unique_id not in self._agents:
heappush(self._queue, (self.time, agent.unique_id))
super().add(agent)
def step(self, until: float =float('inf')) -> None:
"""
Executes agents in order, one at a time. After each step,
an agent will signal when it wants to be scheduled next.
"""
when = None
agent_id = None
unsched = []
until = until or float('inf')
if not self._queue:
self.time = until
self.next_time = float('inf')
return
(when, agent_id) = self._queue[0]
if until and when > until:
self.time = until
self.next_time = when
return
self.time = when
next_time = float("inf")
while when == self.time:
heappop(self._queue)
logger.debug(f'Stepping agent {agent_id}')
when = (self._agents[agent_id].step() or Delta(1)).abs(self.time)
heappush(self._queue, (when, agent_id))
if when < next_time:
next_time = when
if not self._queue or self._queue[0][0] > self.time:
agent_id = None
break
else:
(when, agent_id) = self._queue[0]
if when and when < self.time:
raise Exception("Invalid scheduling time")
self.next_time = next_time
self.steps += 1

4
soil/utils.py
View File

@ -7,8 +7,8 @@ from shutil import copyfile
from contextlib import contextmanager
logger = logging.getLogger('soil')
logging.basicConfig()
logger.setLevel(logging.INFO)
# logging.basicConfig()
# logger.setLevel(logging.INFO)
@contextmanager

5
soil/visualization.py Normal file
View File

@ -0,0 +1,5 @@
from mesa.visualization.UserParam import UserSettableParameter
class UserSettableParameter(UserSettableParameter):
def __str__(self):
return self.value

5
test-requirements.txt
View File

@ -1 +1,4 @@
pytest
pytest
mesa>=0.8.9
scipy>=1.3
tornado

7
tests/test_analysis.py
View File

@ -21,11 +21,13 @@ class Ping(agents.FSM):
@agents.default_state
@agents.state
def even(self):
self.debug(f'Even {self["count"]}')
self['count'] += 1
return self.odd
@agents.state
def odd(self):
self.debug(f'Odd {self["count"]}')
self['count'] += 1
return self.even
@ -82,8 +84,7 @@ class TestAnalysis(TestCase):
import numpy as np
res_mean = analysis.get_value(df, 'count', aggfunc=np.mean)
assert res_mean['count'].iloc[0] == 1
res_total = analysis.get_value(df)
assert res_mean['count'].iloc[15] == (16+8)/2
res_total = analysis.get_majority(df)
res_total['SEED'].iloc[0] == self.env['SEED']

203
tests/test_history.py
View File

@ -1,203 +0,0 @@
from unittest import TestCase
import os
import shutil
from glob import glob
from soil import history
from soil import utils
ROOT = os.path.abspath(os.path.dirname(__file__))
DBROOT = os.path.join(ROOT, 'testdb')
class TestHistory(TestCase):
def setUp(self):
if not os.path.exists(DBROOT):
os.makedirs(DBROOT)
def tearDown(self):
if os.path.exists(DBROOT):
shutil.rmtree(DBROOT)
def test_history(self):
"""
"""
tuples = (
('a_0', 0, 'id', 'h'),
('a_0', 1, 'id', 'e'),
('a_0', 2, 'id', 'l'),
('a_0', 3, 'id', 'l'),
('a_0', 4, 'id', 'o'),
('a_1', 0, 'id', 'v'),
('a_1', 1, 'id', 'a'),
('a_1', 2, 'id', 'l'),
('a_1', 3, 'id', 'u'),
('a_1', 4, 'id', 'e'),
('env', 1, 'prob', 1),
('env', 3, 'prob', 2),
('env', 5, 'prob', 3),
('a_2', 7, 'finished', True),
)
h = history.History()
h.save_tuples(tuples)
# assert h['env', 0, 'prob'] == 0
for i in range(1, 7):
assert h['env', i, 'prob'] == ((i-1)//2)+1
for i, k in zip(range(5), 'hello'):
assert h['a_0', i, 'id'] == k
for record, value in zip(h['a_0', None, 'id'], 'hello'):
t_step, val = record
assert val == value
for i, k in zip(range(5), 'value'):
assert h['a_1', i, 'id'] == k
for i in range(5, 8):
assert h['a_1', i, 'id'] == 'e'
for i in range(7):
assert h['a_2', i, 'finished'] == False
assert h['a_2', 7, 'finished']
def test_history_gen(self):
"""
"""
tuples = (
('a_1', 0, 'id', 'v'),
('a_1', 1, 'id', 'a'),
('a_1', 2, 'id', 'l'),
('a_1', 3, 'id', 'u'),
('a_1', 4, 'id', 'e'),
('env', 1, 'prob', 1),
('env', 2, 'prob', 2),
('env', 3, 'prob', 3),
('a_2', 7, 'finished', True),
)
h = history.History()
h.save_tuples(tuples)
for t_step, key, value in h['env', None, None]:
assert t_step == value
assert key == 'prob'
records = list(h[None, 7, None])
assert len(records) == 3
for i in records:
agent_id, key, value = i
if agent_id == 'a_1':
assert key == 'id'
assert value == 'e'
elif agent_id == 'a_2':
assert key == 'finished'
assert value
else:
assert key == 'prob'
assert value == 3
records = h['a_1', 7, None]
assert records['id'] == 'e'
def test_history_file(self):
"""
History should be saved to a file
"""
tuples = (
('a_1', 0, 'id', 'v'),
('a_1', 1, 'id', 'a'),
('a_1', 2, 'id', 'l'),
('a_1', 3, 'id', 'u'),
('a_1', 4, 'id', 'e'),
('env', 1, 'prob', 1),
('env', 2, 'prob', 2),
('env', 3, 'prob', 3),
('a_2', 7, 'finished', True),
)
db_path = os.path.join(DBROOT, 'test')
h = history.History(db_path=db_path)
h.save_tuples(tuples)
h.flush_cache()
assert os.path.exists(db_path)
# Recover the data
recovered = history.History(db_path=db_path)
assert recovered['a_1', 0, 'id'] == 'v'
assert recovered['a_1', 4, 'id'] == 'e'
# Using backup=True should create a backup copy, and initialize an empty history
newhistory = history.History(db_path=db_path, backup=True)
backuppaths = glob(db_path + '.backup*.sqlite')
assert len(backuppaths) == 1
backuppath = backuppaths[0]
assert newhistory.db_path == h.db_path
assert os.path.exists(backuppath)
assert len(newhistory[None, None, None]) == 0
def test_history_tuples(self):
"""
The data recovered should be equal to the one recorded.
"""
tuples = (
('a_1', 0, 'id', 'v'),
('a_1', 1, 'id', 'a'),
('a_1', 2, 'id', 'l'),
('a_1', 3, 'id', 'u'),
('a_1', 4, 'id', 'e'),
('env', 1, 'prob', 1),
('env', 2, 'prob', 2),
('env', 3, 'prob', 3),
('a_2', 7, 'finished', True),
)
h = history.History()
h.save_tuples(tuples)
recovered = list(h.to_tuples())
assert recovered
for i in recovered:
assert i in tuples
def test_stats(self):
"""
The data recovered should be equal to the one recorded.
"""
tuples = (
('a_1', 0, 'id', 'v'),
('a_1', 1, 'id', 'a'),
('a_1', 2, 'id', 'l'),
('a_1', 3, 'id', 'u'),
('a_1', 4, 'id', 'e'),
('env', 1, 'prob', 1),
('env', 2, 'prob', 2),
('env', 3, 'prob', 3),
('a_2', 7, 'finished', True),
)
stat_tuples = [
{'num_infected': 5, 'runtime': 0.2},
{'num_infected': 5, 'runtime': 0.2},
{'new': '40'},
]
h = history.History()
h.save_tuples(tuples)
for stat in stat_tuples:
h.save_stats(stat)
recovered = h.get_stats()
assert recovered
assert recovered[0]['num_infected'] == 5
assert recovered[1]['runtime'] == 0.2
assert recovered[2]['new'] == '40'
def test_unflatten(self):
ex = {'count.neighbors.3': 4,
'count.times.2': 4,
'count.total.4': 4,
'mean.neighbors': 3,
'mean.times': 2,
'mean.total': 4,
't_step': 2,
'trial_id': 'exporter_sim_trial_1605817956-4475424'}
res = utils.unflatten_dict(ex)
assert 'count' in res
assert 'mean' in res
assert 't_step' in res
assert 'trial_id' in res

22
tests/test_main.py
View File

@ -126,7 +126,7 @@ class TestMain(TestCase):
env = s.run_simulation(dry_run=True)[0]
for agent in env.network_agents:
last = 0
assert len(agent[None, None]) == 10
assert len(agent[None, None]) == 11
for step, total in sorted(agent['total', None]):
assert total == last + 2
last = total
@ -198,11 +198,11 @@ class TestMain(TestCase):
"""
config = serialization.load_file(join(EXAMPLES, 'complete.yml'))[0]
s = simulation.from_config(config)
for i in range(5):
s.run_simulation(dry_run=True)
nconfig = s.to_dict()
del nconfig['topology']
assert config == nconfig
s.run_simulation(dry_run=True)
nconfig = s.to_dict()
del nconfig['topology']
assert config == nconfig
def test_row_conversion(self):
env = Environment()
@ -211,7 +211,7 @@ class TestMain(TestCase):
res = list(env.history_to_tuples())
assert len(res) == len(env.environment_params)
env._now = 1
env.schedule.time = 1
env['test'] = 'second_value'
res = list(env.history_to_tuples())
@ -281,7 +281,7 @@ class TestMain(TestCase):
'weight': 2
},
]
converted = agents.deserialize_distribution(agent_distro)
converted = agents.deserialize_definition(agent_distro)
assert converted[0]['agent_type'] == agents.CounterModel
assert converted[1]['agent_type'] == CustomAgent
pickle.dumps(converted)
@ -297,14 +297,14 @@ class TestMain(TestCase):
'weight': 2
},
]
converted = agents.serialize_distribution(agent_distro)
converted = agents.serialize_definition(agent_distro)
assert converted[0]['agent_type'] == 'CounterModel'
assert converted[1]['agent_type'] == 'test_main.CustomAgent'
pickle.dumps(converted)
def test_pickle_agent_environment(self):
env = Environment(name='Test')
a = agents.BaseAgent(environment=env, agent_id=25)
a = agents.BaseAgent(model=env, unique_id=25)
a['key'] = 'test'
@ -345,7 +345,7 @@ class TestMain(TestCase):
def test_until(self):
config = {
'name': 'exporter_sim',
'name': 'until_sim',
'network_params': {},
'agent_type': 'CounterModel',
'max_time': 2,

69
tests/test_mesa.py Normal file
View File

@ -0,0 +1,69 @@
'''
Mesa-SOIL integration tests
We have to test that:
- Mesa agents can be used in SOIL
- Simplified soil agents can be used in mesa simulations
- Mesa and soil agents can interact in a simulation
- Mesa visualizations work with SOIL simulations
'''
from mesa import Agent, Model
from mesa.time import RandomActivation
from mesa.space import MultiGrid
class MoneyAgent(Agent):
""" An agent with fixed initial wealth."""
def __init__(self, unique_id, model):
super().__init__(unique_id, model)
self.wealth = 1
def step(self):
self.move()
if self.wealth > 0:
self.give_money()
def give_money(self):
cellmates = self.model.grid.get_cell_list_contents([self.pos])
if len(cellmates) > 1:
other = self.random.choice(cellmates)
other.wealth += 1
self.wealth -= 1
def move(self):
possible_steps = self.model.grid.get_neighborhood(
self.pos,
moore=True,
include_center=False)
new_position = self.random.choice(possible_steps)
self.model.grid.move_agent(self, new_position)
class MoneyModel(Model):
"""A model with some number of agents."""
def __init__(self, N, width, height):
self.num_agents = N
self.grid = MultiGrid(width, height, True)
self.schedule = RandomActivation(self)
# Create agents
for i in range(self.num_agents):
a = MoneyAgent(i, self)
self.schedule.add(a)
# Add the agent to a random grid cell
x = self.random.randrange(self.grid.width)
y = self.random.randrange(self.grid.height)
self.grid.place_agent(a, (x, y))
def step(self):
'''Advance the model by one step.'''
self.schedule.step()
# model = MoneyModel(10)
# for i in range(10):
# model.step()
# agent_wealth = [a.wealth for a in model.schedule.agents]