Change entrypoint build stage

Multithreading needs pickling to work.
Pickling/unpickling didn't work in some situations, like when the
environment_agents parameter was left blank.
This was due to two reasons:

1) agents and history didn't have a setstate method, and some of their
attributes cannot be pickled (generators, sqlite connection)
2) the environment was adding generators (agents) to its state.

This fixes the situation by restricting the keys that the environment exports
when it pickles, and by adding the set/getstate methods in agents.

The resulting pickles should contain enough information to inspect
them (history, state values, etc), but very limited.

.gitlab-ci.yml

@@ -1,6 +1,4 @@
-image: python:3.7
-
-steps:
+stages:
   - build
   - test
 
@@ -8,7 +6,9 @@ build:
   stage: build
   image:
     name: gcr.io/kaniko-project/executor:debug
-    entrypoint: [""]
+    entrypoint: ["/busybox/sh"]
+  tags:
+    - docker
   script:
     - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json
     - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --destination $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG
@@ -17,5 +17,9 @@ build:
 
 
 test:
+  tags:
+    - docker
+  image: python:3.7
+  stage: test
   script:
-    python setup.py test
+    - python setup.py test

docs/quickstart.rst

@@ -16,15 +16,14 @@ The configuration includes things such as number of agents to use and their type
 
 Soil includes several agent classes in the ``soil.agents`` module, and we will use them in this quickstart.
 If you are interested in developing your own agents classes, see :doc:`soil_tutorial`.
-The configuration is the following:
+
+Configuration
+=============
+To get you started, we will use this configuration (:download:`download the file <quickstart.yml>` directly):
 
 .. literalinclude:: quickstart.yml
    :language: yaml
 
-Configuration
-=============
-
-You may :download:`download the file <quickstart.yml>` directly.
 The agent type used, SISa, is a very simple model.
 It only has three states (neutral, content and discontent),
 Its parameters are the probabilities to change from one state to another, either spontaneously or because of contagion from neighboring agents.
@@ -79,16 +78,16 @@ Change some of the parameters, such as the number of agents, the probability of
 Soil also includes a web server that allows you to upload your simulations, change parameters, and visualize the results, including a timeline of the network.
 To make it work, you have to install soil like this:
 
-```
-pip install soil[web]
-```
+.. code::
+
+  pip install soil[web]
 
 Once installed, the soil web UI can be run in two ways:
 
-```
-soil-web
+.. code::
 
-OR
+  soil-web
 
-python -m soil.web
-```
+  # OR
+
+  python -m soil.web

soil/VERSION

@@ -1 +1 @@
-0.13.0
+0.13.4

soil/__init__.py

@@ -11,8 +11,6 @@ try:
 except NameError:
     basestring = str
 
-logging.basicConfig()
-
 from . import agents
 from .simulation import *
 from .environment import Environment
@@ -23,6 +21,9 @@ def main():
     import argparse
     from . import simulation
 
+    logging.basicConfig(level=logging.INFO)
+    logging.info('Running SOIL version: {}'.format(__version__))
+
     parser = argparse.ArgumentParser(description='Run a SOIL simulation')
     parser.add_argument('file', type=str,
                         nargs="?",
@@ -62,7 +63,7 @@ def main():
         simulation.run_from_config(args.file,
                                    dry_run=args.dry_run,
                                    dump=dump,
-                                   parallel=(not args.synchronous and not args.pdb),
+                                   parallel=(not args.synchronous),
                                    results_dir=args.output)
     except Exception:
         if args.pdb:

soil/agents/__init__.py

@@ -24,7 +24,7 @@ class BaseAgent(nxsim.BaseAgent):
 
     defaults = {}
 
-    def __init__(self, environment, agent_id=None, state=None,
+    def __init__(self, environment, agent_id, state=None,
                  name='network_process', interval=None, **state_params):
         # Check for REQUIRED arguments
         assert environment is not None, TypeError('__init__ missing 1 required keyword argument: \'environment\'. '
@@ -34,10 +34,6 @@ class BaseAgent(nxsim.BaseAgent):
         self.name = name
         self.state_params = state_params
 
-        # Global parameters
-        self.global_topology = environment.G
-        self.environment_params = environment.environment_params
-
         # Register agent to environment
         self.env = environment
 
@@ -73,6 +69,18 @@ class BaseAgent(nxsim.BaseAgent):
         for k, v in value.items():
             self[k] = v
 
+    @property
+    def global_topology(self):
+        return self.env.G
+    
+    @property
+    def environment_params(self):
+        return self.env.environment_params
+    
+    @environment_params.setter
+    def environment_params(self, value):
+        self.env.environment_params = value
+
     def __getitem__(self, key):
         if isinstance(key, tuple):
             key, t_step = key
@@ -126,9 +134,6 @@ class BaseAgent(nxsim.BaseAgent):
     def step(self):
         pass
 
-    def to_json(self):
-        return json.dumps(self.state)
-
     def count_agents(self, state_id=None, limit_neighbors=False):
         if limit_neighbors:
             agents = self.global_topology.neighbors(self.id)
@@ -182,6 +187,26 @@ class BaseAgent(nxsim.BaseAgent):
 
     def info(self, *args, **kwargs):
         return self.log(*args, level=logging.INFO, **kwargs)
+    
+    def __getstate__(self):
+        '''
+        Serializing an agent will lose all its running information (you cannot
+        serialize an iterator), but it keeps the state and link to the environment,
+        so it can be used for inspection and dumping to a file
+        '''
+        state = {}
+        state['id'] = self.id
+        state['environment'] = self.env
+        state['_state'] = self._state
+        return state
+
+    def __setstate__(self, state):
+        '''
+        Get back a serialized agent and try to re-compose it
+        '''
+        self.id = state['id']
+        self._state = state['_state']
+        self.env = state['environment']
 
 
 def state(func):
@@ -336,7 +361,7 @@ def serialize_distribution(network_agents, known_modules=[]):
     When serializing an agent distribution, remove the thresholds, in order
     to avoid cluttering the YAML definition file.
     '''
-    d = deepcopy(network_agents)
+    d = deepcopy(list(network_agents))
     for v in d:
         if 'threshold' in v:
             del v['threshold']

soil/environment.py

@@ -14,6 +14,14 @@ import nxsim
 
 from . import utils, agents, analysis, history
 
+# These properties will be copied when pickling/unpickling the environment
+_CONFIG_PROPS = [ 'name',
+                 'states',
+                 'default_state',
+                 'interval',
+                 'dry_run',
+                 'dir_path',
+                 ]
 
 class Environment(nxsim.NetworkEnvironment):
     """
@@ -318,21 +326,22 @@ class Environment(nxsim.NetworkEnvironment):
                 G.add_node(agent.id, **attributes)
 
         return G
-
+    
     def __getstate__(self):
-        state = self.__dict__.copy()
+        state = {}
+        for prop in _CONFIG_PROPS:
+            state[prop] = self.__dict__[prop]
         state['G'] = json_graph.node_link_data(self.G)
-        state['network_agents'] = agents._serialize_distribution(self.network_agents)
-        state['environment_agents'] = agents._convert_agent_types(self.environment_agents,
-                                                                 to_string=True)
+        state['environment_agents'] = self._env_agents
+        state['history'] = self._history
         return state
 
     def __setstate__(self, state):
-        self.__dict__ = state
+        for prop in _CONFIG_PROPS:
+            self.__dict__[prop] = state[prop]
+        self._env_agents = state['environment_agents']
         self.G = json_graph.node_link_graph(state['G'])
-        self.network_agents = self.calculate_distribution(self._convert_agent_types(self.network_agents))
-        self.environment_agents = self._convert_agent_types(self.environment_agents)
-        return state
+        self._history = state['history']
 
 
 SoilEnvironment = Environment

soil/history.py

@@ -38,7 +38,7 @@ class History:
     def db(self):
         try:
             self._db.cursor()
-        except sqlite3.ProgrammingError:
+        except (sqlite3.ProgrammingError, AttributeError):
             self.db = None  # Reset the database
         return self._db
 
@@ -207,6 +207,16 @@ class History:
         if t_steps:
             df_p = df_p.reindex(t_steps, method='ffill')
         return df_p.ffill()
+    
+    def __getstate__(self):
+        state = dict(**self.__dict__)
+        del state['_db']
+        del state['_dtypes']
+        return state
+    
+    def __setstate__(self, state):
+        self.__dict__ = state
+        self._dtypes = {}
 
 
 class Records():

soil/simulation.py

@@ -201,7 +201,7 @@ class Simulation(NetworkSimulation):
             return self.run_trial(*args, **kwargs)
         except Exception as ex:
             c = ex.__cause__
-            c.message = ''.join(traceback.format_tb(c.__traceback__)[3:])
+            c.message = ''.join(traceback.format_exception(type(c), c, c.__traceback__)[:])
             return c
 
     def to_dict(self):

soil/web/__init__.py

@@ -19,7 +19,7 @@ from xml.etree.ElementTree import tostring
 from tornado.concurrent import run_on_executor
 from concurrent.futures import ThreadPoolExecutor
 
-from ..simulation import SoilSimulation
+from ..simulation import Simulation
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
 
@@ -168,7 +168,7 @@ class SocketHandler(tornado.websocket.WebSocketHandler):
 
     @run_on_executor
     def nonblocking(self, config):
-        simulation = SoilSimulation(**config)
+        simulation = Simulation(**config)
         return simulation.run()
 
     @tornado.gen.coroutine

tests/test_main.py

@@ -2,6 +2,7 @@ from unittest import TestCase
 
 import os
 import yaml
+import pickle
 import networkx as nx
 from functools import partial
 
@@ -248,12 +249,10 @@ class TestMain(TestCase):
         assert name == 'soil.agents.BaseAgent'
         assert ser == agents.BaseAgent
 
-        class CustomAgent(agents.BaseAgent):
-            pass
-
         ser, name = utils.serialize(CustomAgent)
         assert name == 'test_main.CustomAgent'
         assert ser == CustomAgent
+        pickle.dumps(ser)
 
     def test_serialize_builtin_types(self):
 
@@ -269,7 +268,8 @@ class TestMain(TestCase):
         assert ser == 'test_main.CustomAgent'
         ser = agents.serialize_type(agents.BaseAgent)
         assert ser == 'BaseAgent'
-
+        pickle.dumps(ser)
+    
     def test_deserialize_agent_distribution(self):
         agent_distro = [
             {
@@ -284,6 +284,7 @@ class TestMain(TestCase):
         converted = agents.deserialize_distribution(agent_distro)
         assert converted[0]['agent_type'] == agents.CounterModel
         assert converted[1]['agent_type'] == CustomAgent
+        pickle.dumps(converted)
 
     def test_serialize_agent_distribution(self):
         agent_distro = [
@@ -299,6 +300,20 @@ class TestMain(TestCase):
         converted = agents.serialize_distribution(agent_distro)
         assert converted[0]['agent_type'] == 'CounterModel'
         assert converted[1]['agent_type'] == 'test_main.CustomAgent'
+        pickle.dumps(converted)
+
+    def test_pickle_agent_environment(self):
+        env = Environment(name='Test')
+        a = agents.BaseAgent(environment=env, agent_id=25)
+
+        a['key'] = 'test'
+
+        pickled = pickle.dumps(a)
+        recovered = pickle.loads(pickled)
+
+        assert recovered.env.name == 'Test'
+        assert recovered['key'] == 'test'
+        assert recovered['key', 0] == 'test'
 
     def test_history(self):
         '''Test storing in and retrieving from history (sqlite)'''