Update tutorial and fix plotting bug

* Improve text
* Move to docs
* Autogenerate with sphinx
* Fix naming issue `environment.run` (double name)
* Add to tests

README.md

@@ -4,7 +4,7 @@
 Soil is an extensible and user-friendly Agent-based Social Simulator for Social Networks.
 Learn how to run your own simulations with our [documentation](http://soilsim.readthedocs.io).
 
-Follow our [tutorial](examples/tutorial/soil_tutorial.ipynb) to develop your own agent models.
+Follow our [tutorial](docs/tutorial/soil_tutorial.ipynb) to develop your own agent models.
 
 > **Warning**
 > Soil 1.0 introduced many fundamental changes. Check the [documention on how to update your simulations to work with newer versions](docs/notes_v1.0.rst)

docs/conf.py

@@ -31,7 +31,10 @@
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = ['IPython.sphinxext.ipython_console_highlighting']
+extensions = [
+    "IPython.sphinxext.ipython_console_highlighting",
+    "nbsphinx",
+]
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
@@ -64,7 +67,7 @@ release = '0.1'
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = None
+language = "en"
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
@@ -152,6 +155,3 @@ texinfo_documents = [
      author, 'Soil', 'One line description of project.',
      'Miscellaneous'),
 ]
-
-
-

docs/index.rst

@@ -2,19 +2,20 @@ Welcome to Soil's documentation!
 ================================
 
 Soil is an opinionated Agent-based Social Simulator in Python focused on Social Networks.
+To get started developing your own simulations and agent behaviors, check out our :doc:`Tutorial <soil_tutorial>` and the `examples on GitHub <https://github.com/gsi-upm/soil/tree/master/examples>`.
+
+Soil can be installed through pip (see more details in the :doc:`installation` page):.
 
 .. image:: soil.png
   :width: 80%
   :align: center
 
-Soil can be installed through pip (see more details in the :doc:`installation` page):
 
 .. code:: bash
 
     pip install soil
 
 
-To get started developing your own simulations and agent behaviors, check out our :doc:`Tutorial <soil_tutorial>` and the `examples on GitHub <https://github.com/gsi-upm/soil/tree/master/examples>.
 
 If you use Soil in your research, do not forget to cite this paper:
 
@@ -46,7 +47,8 @@ If you use Soil in your research, do not forget to cite this paper:
    :caption: Learn more about soil:
 
    installation
-   Tutorial <soil_tutorial>
+   Tutorial <tutorial/soil_tutorial>
+   notes_v1.0
 
 ..
 

docs/notes_v1.0.rst

@@ -1,3 +1,6 @@
+Upgrading to Soil 1.0
+---------------------
+
 What are the main changes in version 1.0?
 #########################################
 

docs/requirements.txt

@@ -1 +1,2 @@
 ipython>=7.31.1
+nbsphinx==0.9.1

setup.py

@@ -17,9 +17,9 @@ def parse_requirements(filename):
 install_reqs = parse_requirements("requirements.txt")
 test_reqs = parse_requirements("test-requirements.txt")
 extras_require={
-    'mesa': ['mesa>=0.8.9'],
     'geo': ['scipy>=1.3'],
-    'web': ['tornado']
+    'web': ['tornado'],
+    'ipython': ['ipython==8.12', 'nbformat==5.8'],
 }
 extras_require['all'] = [dep for package in extras_require.values() for dep in package]
 

soil/analysis.py

@@ -1,12 +1,11 @@
 import os
+import sys
 import sqlalchemy
 import pandas as pd
 from collections import namedtuple
 
 def plot(env, agent_df=None, model_df=None, steps=False, ignore=["agent_count", ]):
     """Plot the model dataframe and agent dataframe together."""
-    if agent_df is None:
-        agent_df = env.agent_df()
     if model_df is None:
         model_df = env.model_df()
     ignore = list(ignore)
@@ -16,9 +15,17 @@ def plot(env, agent_df=None, model_df=None, steps=False, ignore=["agent_count",
         ignore.append("time")
 
     ax = model_df.drop(ignore, axis='columns').plot();
+    if agent_df is None:
+        try:
+            agent_df = env.agent_df()
+        except UserWarning:
+            print("No agent dataframe provided and no agent reporters found. Skipping agent plot.", file=sys.stderr)
+            return
     if not agent_df.empty:
         agent_df.unstack().apply(lambda x: x.value_counts(),
-                                 axis=1).fillna(0).plot(ax=ax, secondary_y=True);
+                                 axis=1).fillna(0).plot(ax=ax,
+                                                        secondary_y=True)
+
 
 Results = namedtuple("Results", ["config", "parameters", "env", "agents"])
 #TODO implement reading from CSV and SQLITE

soil/environment.py

@@ -216,10 +216,11 @@ class BaseEnvironment(Model):
 
     @classmethod
     def run(cls, *,
+            name=None,
             iterations=1,
             num_processes=1, **kwargs):
         from .simulation import Simulation
-        return Simulation(name=cls.__name__,
+        return Simulation(name=name or cls.__name__,
                           model=cls, iterations=iterations,
                           num_processes=num_processes, **kwargs).run()
 

soil/serialization.py

@@ -8,6 +8,8 @@ import importlib.machinery, importlib.util
 from glob import glob
 from itertools import product, chain
 
+from contextlib import contextmanager
+
 import yaml
 import networkx as nx
 
@@ -110,12 +112,12 @@ KNOWN_MODULES = {
 
 MODULE_FILES = {}
 
-def add_source_file(file):
+def _add_source_file(file):
     """Add a file to the list of known modules"""
     file = os.path.abspath(file)
     if file in MODULE_FILES:
         logger.warning(f"File {file} already added as module {MODULE_FILES[file]}. Reloading")
-        remove_source_file(file)
+        _remove_source_file(file)
     modname = f"imported_module_{len(MODULE_FILES)}"
     loader = importlib.machinery.SourceFileLoader(modname, file)
     spec = importlib.util.spec_from_loader(loader.name, loader)
@@ -124,7 +126,7 @@ def add_source_file(file):
     MODULE_FILES[file] = modname
     KNOWN_MODULES[modname] = my_module
 
-def remove_source_file(file):
+def _remove_source_file(file):
     """Remove a file from the list of known modules"""
     file = os.path.abspath(file)
     modname = None
@@ -134,6 +136,18 @@ def remove_source_file(file):
     except KeyError as ex:
         raise ValueError(f"File {file} had not been added as a module: {ex}")
 
+
+@contextmanager
+def with_source(file=None):
+    """Add a file to the list of known modules, and remove it afterwards"""
+    if file:
+        _add_source_file(file)
+    try:
+        yield
+    finally:
+        if file:
+            _remove_source_file(file)
+
 def get_module(modname):
     """Get a module from the list of known modules"""
     if modname not in KNOWN_MODULES or KNOWN_MODULES[modname] is None:

soil/simulation.py

@@ -119,28 +119,23 @@ class Simulation:
         self.logger = logger.getChild(self.name)
         self.logger.setLevel(self.level)
 
-        if self.source_file:
+        if self.source_file and (not os.path.isabs(self.source_file)):
-            source_file = self.source_file
+            self.source_file = os.path.abspath(os.path.join(self.dir_path, self.source_file))
-            if not os.path.isabs(source_file):
+        with serialization.with_source(self.source_file):
-                source_file = os.path.abspath(os.path.join(self.dir_path, source_file))
-            serialization.add_source_file(source_file)
-            self.source_file = source_file
 
-        if isinstance(self.model, str):
+            if isinstance(self.model, str):
-            self.model = serialization.deserialize(self.model)
+                self.model = serialization.deserialize(self.model)
 
-        def deserialize_reporters(reporters):
+            def deserialize_reporters(reporters):
-            for (k, v) in reporters.items():
+                for (k, v) in reporters.items():
-                if isinstance(v, str) and v.startswith("py:"):
+                    if isinstance(v, str) and v.startswith("py:"):
-                    reporters[k] = serialization.deserialize(v.split(":", 1)[1])
+                        reporters[k] = serialization.deserialize(v.split(":", 1)[1])
-            return reporters
+                return reporters
 
-        self.agent_reporters = deserialize_reporters(self.agent_reporters)
+            self.agent_reporters = deserialize_reporters(self.agent_reporters)
-        self.model_reporters = deserialize_reporters(self.model_reporters)
+            self.model_reporters = deserialize_reporters(self.model_reporters)
-        self.tables = deserialize_reporters(self.tables)
+            self.tables = deserialize_reporters(self.tables)
-        if self.source_file:
+            self.id = f"{self.name}_{current_time()}"
-            serialization.remove_source_file(self.source_file)
-        self.id = f"{self.name}_{current_time()}"
 
     def run(self, **kwargs):
         """Run the simulation and return the list of resulting environments"""
@@ -217,10 +212,7 @@ class Simulation:
     ):
         """Run the simulation and yield the resulting environments."""
 
-        try:
+        with serialization.with_source(self.source_file):
-            if self.source_file:
-                serialization.add_source_file(self.source_file)
-
             with utils.timer(f"running for config {params}"):
                 if self.dry_run:
                     def func(*args, **kwargs):
@@ -237,9 +229,6 @@ class Simulation:
                         continue
 
                     yield env
-        finally:
-            if self.source_file:
-                serialization.remove_source_file(self.source_file)
 
     def _get_env(self, iteration_id, params):
         """Create an environment for a iteration of the simulation"""

test-requirements.txt

@@ -2,3 +2,6 @@ pytest
 pytest-profiling
 scipy>=1.3
 tornado
+nbconvert==7.3.1
+nbformat==5.8.0
+jupyter==1.0.0

tests/test_ipython.py

@@ -0,0 +1,18 @@
+from unittest import TestCase
+import os
+import nbformat
+from nbconvert.preprocessors import ExecutePreprocessor
+
+ROOT = os.path.abspath(os.path.dirname(__file__))
+
+class TestNotebooks(TestCase):
+    def test_tutorial(self):
+        notebook = os.path.join(ROOT, "../docs/tutorial/soil_tutorial.ipynb")
+        with open(notebook) as f:
+            nb = nbformat.read(f, as_version=4)
+            ep = ExecutePreprocessor(timeout=60000)
+            try:
+                assert ep.preprocess(nb) is not None, f"Got empty notebook for {notebook}"
+            except Exception:
+                assert False, f"Failed executing {notebook}"
+