Initial benchmarking

benchmarks/noop/_config.py

@@ -0,0 +1,25 @@
+import os
+
+NUM_AGENTS = int(os.environ.get('NUM_AGENTS', 100))
+NUM_ITERS = int(os.environ.get('NUM_ITERS', 10))
+MAX_STEPS = int(os.environ.get('MAX_STEPS', 1000))
+
+
+def run_sim(model, **kwargs):
+    from soil import Simulation
+    opts = dict(model=model,
+                dump=False,
+                num_processes=1,
+                parameters={'num_agents': NUM_AGENTS},
+                max_steps=MAX_STEPS,
+                iterations=NUM_ITERS)
+    opts.update(kwargs)
+    res = Simulation(**opts).run()
+
+    total = sum(a.num_calls for e in res for a in e.schedule.agents)
+    expected = NUM_AGENTS * NUM_ITERS * MAX_STEPS
+    print(total)
+    print(expected)
+
+    assert total == expected
+    return res

benchmarks/noop/mesa_batchrunner.py

@@ -0,0 +1,44 @@
+from mesa import batch_run, DataCollector, Agent, Model
+from mesa.time import RandomActivation
+
+
+class NoopAgent(Agent):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.num_calls = 0
+
+    def step(self):
+        # import pdb;pdb.set_trace()
+        self.num_calls += 1 
+
+
+class NoopModel(Model):
+    def __init__(self, N):
+        super().__init__()
+        self.schedule = RandomActivation(self)
+        for i in range(N):
+            self.schedule.add(NoopAgent(self.next_id(), self))
+        self.datacollector = DataCollector(model_reporters={"num_agents": lambda m: m.schedule.get_agent_count(),
+                                                            "time": lambda m: m.schedule.time},
+                                           agent_reporters={"num_calls": "num_calls"})
+        self.datacollector.collect(self)
+
+    def step(self):
+        self.schedule.step()
+        self.datacollector.collect(self)
+
+
+if __name__ == "__main__":
+    from _config import *
+
+    res = batch_run(model_cls=NoopModel,
+                    max_steps=MAX_STEPS,
+                    iterations=NUM_ITERS,
+                    number_processes=1,
+                    parameters={'N': NUM_AGENTS})
+    total = sum(s["num_calls"] for s in res)
+    total_agents = sum(s["num_agents"] for s in res)
+    assert len(res) == NUM_AGENTS * NUM_ITERS
+    assert total == NUM_AGENTS * NUM_ITERS * MAX_STEPS
+    assert total_agents == NUM_AGENTS * NUM_AGENTS * NUM_ITERS
+

benchmarks/noop/mesa_simulation.py

@@ -0,0 +1,38 @@
+from mesa import batch_run, DataCollector, Agent, Model
+from mesa.time import RandomActivation
+from soil import Simulation
+from _config import *
+
+
+class NoopAgent(Agent):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.num_calls = 0
+
+    def step(self):
+        # import pdb;pdb.set_trace()
+        self.num_calls += 1 
+
+
+class NoopModel(Model):
+    def __init__(self, num_agents, *args, **kwargs):
+        super().__init__()
+        self.schedule = RandomActivation(self)
+        for i in range(num_agents):
+            self.schedule.add(NoopAgent(self.next_id(), self))
+        self.datacollector = DataCollector(model_reporters={"num_agents": lambda m: m.schedule.get_agent_count(),
+                                                            "time": lambda m: m.schedule.time},
+                                           agent_reporters={"num_calls": "num_calls"})
+        self.datacollector.collect(self)
+
+    def step(self):
+        self.schedule.step()
+        self.datacollector.collect(self)
+
+
+def run():
+    run_sim(model=NoopModel)
+
+
+if __name__ == "__main__":
+    run()

benchmarks/noop/noop-bench.csv

@@ -0,0 +1,3 @@
+command,mean,stddev,median,user,system,min,max,parameter_sim
+python mesa1_batchrunner.py,1.2932078178200002,0.05649377020829272,1.2705532802200001,1.25902256,0.27242284,1.22210926572,1.40867459172,mesa1_batchrunner
+python mesa1_simulation.py,1.81112963812,0.015491072368938567,1.81342524572,1.8594407599999996,0.8005329399999999,1.78538603972,1.84176361172,mesa1_simulation

benchmarks/noop/soil_async.py

@@ -0,0 +1,24 @@
+from soil import BaseAgent, Environment, Simulation
+
+
+class NoopAgent(BaseAgent):
+    num_calls = 0
+
+    async def step(self):
+        while True:
+            self.num_calls += 1
+            await self.delay()
+
+
+class NoopEnvironment(Environment):
+    num_agents = 100
+
+    def init(self):
+        self.add_agents(NoopAgent, k=self.num_agents)
+        self.add_agent_reporter("num_calls")
+
+
+if __name__ == "__main__":
+    from _config import *
+
+    run_sim(model=NoopEnvironment)

benchmarks/noop/soil_async_pqueue.py

@@ -0,0 +1,25 @@
+from soil import BaseAgent, Environment, Simulation, PQueueActivation
+
+
+class NoopAgent(BaseAgent):
+    num_calls = 0
+
+    async def step(self):
+        while True:
+            self.num_calls += 1
+            await self.delay()
+
+
+class NoopEnvironment(Environment):
+    num_agents = 100
+    schedule_class = PQueueActivation
+
+    def init(self):
+        self.add_agents(NoopAgent, k=self.num_agents)
+        self.add_agent_reporter("num_calls")
+
+
+if __name__ == "__main__":
+    from _config import *
+
+    run_sim(model=NoopEnvironment)

benchmarks/noop/soil_gens.py

@@ -0,0 +1,24 @@
+from soil import BaseAgent, Environment, Simulation
+
+
+class NoopAgent(BaseAgent):
+    num_calls = 0
+
+    def step(self):
+        while True:
+            self.num_calls += 1
+            yield self.delay()
+
+
+class NoopEnvironment(Environment):
+    num_agents = 100
+
+    def init(self):
+        self.add_agents(NoopAgent, k=self.num_agents)
+        self.add_agent_reporter("num_calls")
+
+
+if __name__ == "__main__":
+    from _config import *
+
+    run_sim(model=NoopEnvironment)

benchmarks/noop/soil_gens_pqueue.py

@@ -0,0 +1,25 @@
+from soil import BaseAgent, Environment, Simulation, PQueueActivation
+
+
+class NoopAgent(BaseAgent):
+    num_calls = 0
+
+    def step(self):
+        while True:
+            self.num_calls += 1
+            yield self.delay()
+
+
+class NoopEnvironment(Environment):
+    num_agents = 100
+    schedule_class = PQueueActivation
+
+    def init(self):
+        self.add_agents(NoopAgent, k=self.num_agents)
+        self.add_agent_reporter("num_calls")
+
+
+if __name__ == "__main__":
+    from _config import *
+
+    run_sim(model=NoopEnvironment)

benchmarks/noop/soil_step.py

@@ -0,0 +1,21 @@
+from soil import BaseAgent, Environment, Simulation
+
+
+class NoopAgent(BaseAgent):
+    num_calls = 0
+
+    def step(self):
+        self.num_calls += 1
+
+class NoopEnvironment(Environment):
+    num_agents = 100
+
+    def init(self):
+        self.add_agents(NoopAgent, k=self.num_agents)
+        self.add_agent_reporter("num_calls")
+
+
+if __name__ == "__main__":
+    from _config import *
+
+    run_sim(model=NoopEnvironment)

benchmarks/noop/soil_step_pqueue.py

@@ -0,0 +1,22 @@
+from soil import BaseAgent, Environment, Simulation, PQueueActivation
+
+
+class NoopAgent(BaseAgent):
+    num_calls = 0
+
+    def step(self):
+        self.num_calls += 1
+
+class NoopEnvironment(Environment):
+    num_agents = 100
+    schedule_class = PQueueActivation
+
+    def init(self):
+        self.add_agents(NoopAgent, k=self.num_agents)
+        self.add_agent_reporter("num_calls")
+
+
+if __name__ == "__main__":
+    from _config import *
+
+    run_sim(model=NoopEnvironment)

benchmarks/noop/soilent_async.py

@@ -0,0 +1,29 @@
+from soil import Agent, Environment, Simulation
+from soilent import Scheduler
+
+
+class NoopAgent(Agent):
+    num_calls = 0
+
+    async def step(self):
+        while True:
+            self.num_calls += 1
+            # yield self.delay(1)
+            await self.delay()
+
+
+class NoopEnvironment(Environment):
+    num_agents = 100
+    schedule_class = Scheduler
+
+    def init(self):
+        self.add_agents(NoopAgent, k=self.num_agents)
+        self.add_agent_reporter("num_calls")
+
+
+if __name__ == "__main__":
+    from _config import *
+
+    res = run_sim(model=NoopEnvironment)
+    for r in res:
+        assert isinstance(r.schedule, Scheduler)

benchmarks/noop/soilent_async_pqueue.py

@@ -0,0 +1,27 @@
+from soil import Agent, Environment
+from soilent import PQueueScheduler
+
+
+class NoopAgent(Agent):
+    num_calls = 0
+
+    async def step(self):
+        while True:
+            self.num_calls += 1
+            await self.delay()
+
+class NoopEnvironment(Environment):
+    num_agents = 100
+    schedule_class = PQueueScheduler
+
+    def init(self):
+        self.add_agents(NoopAgent, k=self.num_agents)
+        self.add_agent_reporter("num_calls")
+
+
+if __name__ == "__main__":
+    from _config import *
+
+    res = run_sim(model=NoopEnvironment)
+    for r in res:
+        assert isinstance(r.schedule, PQueueScheduler)

benchmarks/noop/soilent_gens.py

@@ -0,0 +1,28 @@
+from soil import Agent, Environment, Simulation
+from soilent import Scheduler
+
+
+class NoopAgent(Agent):
+    num_calls = 0
+
+    def step(self):
+        while True:
+            self.num_calls += 1
+            # yield self.delay(1)
+            yield self.delay()
+
+class NoopEnvironment(Environment):
+    num_agents = 100
+    schedule_class = Scheduler
+
+    def init(self):
+        self.add_agents(NoopAgent, k=self.num_agents)
+        self.add_agent_reporter("num_calls")
+
+
+if __name__ == "__main__":
+    from _config import *
+
+    res = run_sim(model=NoopEnvironment)
+    for r in res:
+        assert isinstance(r.schedule, Scheduler)

benchmarks/noop/soilent_gens_pqueue.py

@@ -0,0 +1,28 @@
+from soil import Agent, Environment
+from soilent import PQueueScheduler
+
+
+class NoopAgent(Agent):
+    num_calls = 0
+
+    def step(self):
+        while True:
+            self.num_calls += 1
+            # yield self.delay(1)
+            yield
+
+class NoopEnvironment(Environment):
+    num_agents = 100
+    schedule_class = PQueueScheduler
+
+    def init(self):
+        self.add_agents(NoopAgent, k=self.num_agents)
+        self.add_agent_reporter("num_calls")
+
+
+if __name__ == "__main__":
+    from _config import *
+
+    res = run_sim(model=NoopEnvironment)
+    for r in res:
+        assert isinstance(r.schedule, PQueueScheduler)

benchmarks/noop/soilent_step.py

@@ -0,0 +1,24 @@
+from soil import BaseAgent, Environment, Simulation
+from soilent import Scheduler
+
+
+class NoopAgent(BaseAgent):
+    num_calls = 0
+
+    def step(self):
+        self.num_calls += 1
+
+class NoopEnvironment(Environment):
+    num_agents = 100
+    schedule_class = Scheduler
+
+    def init(self):
+        self.add_agents(NoopAgent, k=self.num_agents)
+        self.add_agent_reporter("num_calls")
+
+
+if __name__ == "__main__":
+    from _config import *
+    res = run_sim(model=NoopEnvironment)
+    for r in res:
+        assert isinstance(r.schedule, Scheduler)

benchmarks/noop/soilent_step_pqueue.py

@@ -0,0 +1,24 @@
+from soil import BaseAgent, Environment, Simulation
+from soilent import PQueueScheduler
+
+
+class NoopAgent(BaseAgent):
+    num_calls = 0
+
+    def step(self):
+        self.num_calls += 1
+
+class NoopEnvironment(Environment):
+    num_agents = 100
+    schedule_class = PQueueScheduler
+
+    def init(self):
+        self.add_agents(NoopAgent, k=self.num_agents)
+        self.add_agent_reporter("num_calls")
+
+
+if __name__ == "__main__":
+    from _config import *
+    res = run_sim(model=NoopEnvironment)
+    for r in res:
+        assert isinstance(r.schedule, PQueueScheduler)