merge visualization branch

The web server is included as a submodule.
The dependencies for the web (tornado) are not installed by default, but they
can be installed as an extra:

```
pip install soil[web]
```

Once installed, the soil web can be used like this:

```
soil-web

OR

python -m soil.web
```

There are other minor changes:

* History re-connects to the sqlite database if it is used from a different
thread.
* Environment accepts additional parameters (so it can run simulations with
`visualization_params` or any other in the future).
* The simulator class is no longer necessary
* Logging is done in the same thread, and the simulation is run in a separate
one. This had to be done because it was creating some problems with tornado not
being able to find the current thread during logs, which caused hundreds of
repeated lines in the web "console".
* The player is slightly modified in this version. I noticed that when the
  visualization was playing, if you clicked somewhere it would change for a
  second, and then go back to the previous place. The code for the playback
  seemed too complex, especially speed control, so I rewrote some parts. I
  might've introduced new bugs.

Dockerfile

@@ -1,3 +1,5 @@
 FROM python:3.4-onbuild
 
+RUN pip install '.[web]'
+
 ENTRYPOINT ["python", "-m", "soil"]

docker-compose.yml

@@ -6,3 +6,5 @@ services:
       - .:/usr/src/app
     tty: true
     entrypoint: /bin/bash
+    ports:
+      - '8001:8001'

setup.py

@@ -40,10 +40,15 @@ setup(
         'Operating System :: POSIX',
         'Programming Language :: Python :: 3'],
     install_requires=install_reqs,
+    extras_require={
+        'web': ['tornado']
+
+    },
     tests_require=test_reqs,
     setup_requires=['pytest-runner', ],
     include_package_data=True,
     entry_points={
         'console_scripts':
-        ['soil = soil.__init__:main']
+        ['soil = soil.__init__:main',
+        'soil-web = soil.web.__init__:main']
     })

soil/VERSION

@@ -1 +1 @@
-0.11.3
+0.12.0

soil/history.py

@@ -23,16 +23,14 @@ class History:
             if backup and os.path.exists(db_path):
                 newname = db_path + '.backup{}.sqlite'.format(time.time())
                 os.rename(db_path, newname)
-        self._db_path = db_path
+        self.db_path = db_path
-        if isinstance(db_path, str):
+
-            self._db = sqlite3.connect(db_path)
+        self.db = db_path
-        else:
-            self._db = db_path
 
-        with self._db:
+        with self.db:
-            self._db.execute('''CREATE TABLE IF NOT EXISTS history (agent_id text, t_step int, key text, value text text)''')
+            self.db.execute('''CREATE TABLE IF NOT EXISTS history (agent_id text, t_step int, key text, value text text)''')
-            self._db.execute('''CREATE TABLE IF NOT EXISTS value_types (key text, value_type text)''')
+            self.db.execute('''CREATE TABLE IF NOT EXISTS value_types (key text, value_type text)''')
-            self._db.execute('''CREATE UNIQUE INDEX IF NOT EXISTS idx_history ON history (agent_id, t_step, key);''')
+            self.db.execute('''CREATE UNIQUE INDEX IF NOT EXISTS idx_history ON history (agent_id, t_step, key);''')
         self._dtypes = {}
         self._tups = []
 
@@ -41,6 +39,22 @@ class History:
         if key not in self._dtypes:
             self.read_types()
         return self._dtypes[key]
+    
+    @property
+    def db(self):
+        try:
+            self._db.cursor()
+        except sqlite3.ProgrammingError:
+            self.db = None  # Reset the database
+        return self._db
+
+    @db.setter
+    def db(self, db_path=None):
+        db_path = db_path or self.db_path
+        if isinstance(db_path, str):
+            self._db = sqlite3.connect(db_path)
+        else:
+            self._db = db_path
 
     @property
     def dtypes(self):
@@ -50,7 +64,7 @@ class History:
         self.save_records(Record(*tup) for tup in tuples)
 
     def save_records(self, records):
-        with self._db:
+        with self.db:
             for rec in records:
                 if not isinstance(rec, Record):
                     rec = Record(*rec)
@@ -59,8 +73,8 @@ class History:
                     serializer = utils.serializer(name)
                     deserializer = utils.deserializer(name)
                     self._dtypes[rec.key] = (name, serializer, deserializer)
-                    self._db.execute("replace into value_types (key, value_type) values (?, ?)", (rec.key, name))
+                    self.db.execute("replace into value_types (key, value_type) values (?, ?)", (rec.key, name))
-                self._db.execute("replace into history(agent_id, t_step, key, value) values (?, ?, ?, ?)", (rec.agent_id, rec.t_step, rec.key, rec.value))
+                self.db.execute("replace into history(agent_id, t_step, key, value) values (?, ?, ?, ?)", (rec.agent_id, rec.t_step, rec.key, rec.value))
 
     def save_record(self, *args, **kwargs):
         self._tups.append(Record(*args, **kwargs))
@@ -77,16 +91,16 @@ class History:
 
     def to_tuples(self):
             self.flush_cache()
-            with self._db:
+            with self.db:
-                res = self._db.execute("select agent_id, t_step, key, value from history ").fetchall()
+                res = self.db.execute("select agent_id, t_step, key, value from history ").fetchall()
             for r in res:
                 agent_id, t_step, key, value = r
                 _, _ , des = self.conversors(key)
                 yield agent_id, t_step, key, des(value)
 
     def read_types(self):
-            with self._db:
+            with self.db:
-                res = self._db.execute("select key, value_type from value_types ").fetchall()
+                res = self.db.execute("select key, value_type from value_types ").fetchall()
             for k, v in res:
                 serializer = utils.serializer(v)
                 deserializer = utils.deserializer(v)
@@ -143,7 +157,7 @@ class History:
                h1.key      = h2.key       and
                h1.t_step   = h2.t_step
             '''.format(condition=condition)
-            last_df = pd.read_sql_query(last_query, self._db)
+            last_df = pd.read_sql_query(last_query, self.db)
 
             filters.append("t_step >= '{}' and t_step <= '{}'".format(min_step, max(t_steps)))
 
@@ -151,7 +165,7 @@ class History:
         if filters:
             condition = 'where {} '.format(' and '.join(filters))
         query = 'select * from history {} limit {}'.format(condition, limit)
-        df = pd.read_sql_query(query, self._db)
+        df = pd.read_sql_query(query, self.db)
         if last_df is not None:
             df = pd.concat([df, last_df])
 

soil/simulation.py

@@ -49,7 +49,7 @@ class SoilSimulation(NetworkSimulation):
                  default_state=None, interval=1, dump=None, dry_run=False,
                  dir_path=None, num_trials=1, max_time=100,
                  agent_module=None, load_module=None, seed=None,
-                 environment_agents=None, environment_params=None):
+                 environment_agents=None, environment_params=None, **kwargs):
 
         if topology is None:
             topology = utils.load_network(network_params,

soil/web/__init__.py

@@ -1,8 +1,10 @@
 import io
+import threading
+import asyncio
 import logging
 import networkx as nx
 import os
-import threading
+import sys
 import tornado.ioloop
 import tornado.web
 import tornado.websocket
@@ -14,9 +16,20 @@ from contextlib import contextmanager
 from time import sleep
 from xml.etree.ElementTree import tostring
 
+from tornado.concurrent import run_on_executor
+from concurrent.futures import ThreadPoolExecutor
+
+from ..simulation import SoilSimulation
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
 
+ROOT = os.path.abspath(os.path.dirname(__file__))
+
+MAX_WORKERS = 4
+LOGGING_INTERVAL = 0.5
+
+# Workaround to let Soil load the required modules
+sys.path.append(ROOT)
 
 class PageHandler(tornado.web.RequestHandler):
     """ Handler for the HTML template which holds the visualization. """
@@ -28,6 +41,7 @@ class PageHandler(tornado.web.RequestHandler):
 
 class SocketHandler(tornado.websocket.WebSocketHandler):
     """ Handler for websocket. """
+    executor = ThreadPoolExecutor(max_workers=MAX_WORKERS)
 
     def open(self):
         if self.application.verbose:
@@ -55,9 +69,10 @@ class SocketHandler(tornado.websocket.WebSocketHandler):
                 self.write_message({'type': 'error',
                     'error': error})
                 return
-            else:
+
-                self.config = self.config[0]
+            self.config = self.config[0]
-                self.send_log('INFO.' + self.application.simulator.name, 'Using config: {name}'.format(name=self.config['name']))
+            self.send_log('INFO.' + self.simulation_name,
+                          'Using config: {name}'.format(name=self.config['name']))
 
             if 'visualization_params' in self.config:
                 self.write_message({'type': 'visualization_params',
@@ -91,17 +106,17 @@ class SocketHandler(tornado.websocket.WebSocketHandler):
                 logger.info('Trial {} requested!'.format(msg['data']))
             self.send_log('INFO.' + __name__, 'Trial {} requested!'.format(msg['data']))
             self.write_message({'type': 'get_trial',
-                'data': self.get_trial( int(msg['data']) ) })
+                                'data': self.get_trial(int(msg['data']))})
 
         elif msg['type'] == 'run_simulation':
             if self.application.verbose:
                 logger.info('Running new simulation for {name}'.format(name=self.config['name']))
-            self.send_log('INFO.' + self.application.simulator.name, 'Running new simulation for {name}'.format(name=self.config['name']))
+            self.send_log('INFO.' + self.simulation_name, 'Running new simulation for {name}'.format(name=self.config['name']))
             self.config['environment_params'] = msg['data']
             self.run_simulation()
 
         elif msg['type'] == 'download_gexf':
-            G = self.simulation[ int(msg['data']) ].history_to_graph()
+            G = self.trials[ int(msg['data']) ].history_to_graph()
             for node in G.nodes():
                 if 'pos' in G.node[node]:
                     G.node[node]['viz'] = {"position": {"x": G.node[node]['pos'][0], "y": G.node[node]['pos'][1], "z": 0.0}}
@@ -113,7 +128,7 @@ class SocketHandler(tornado.websocket.WebSocketHandler):
                 'data': tostring(writer.xml).decode(writer.encoding) })
 
         elif msg['type'] == 'download_json':
-            G = self.simulation[ int(msg['data']) ].history_to_graph()
+            G = self.trials[ int(msg['data']) ].history_to_graph()
             for node in G.nodes():
                 if 'pos' in G.node[node]:
                     G.node[node]['viz'] = {"position": {"x": G.node[node]['pos'][0], "y": G.node[node]['pos'][1], "z": 0.0}}
@@ -130,13 +145,13 @@ class SocketHandler(tornado.websocket.WebSocketHandler):
         try:
             if (not self.log_capture_string.closed and self.log_capture_string.getvalue()):
                 for i in range(len(self.log_capture_string.getvalue().split('\n')) - 1):
-                    self.send_log('INFO.' + self.application.simulator.name, self.log_capture_string.getvalue().split('\n')[i])
+                    self.send_log('INFO.' + self.simulation_name, self.log_capture_string.getvalue().split('\n')[i])
                 self.log_capture_string.truncate(0)
                 self.log_capture_string.seek(0)
         finally:
             if self.capture_logging:
-                thread = threading.Timer(0.01, self.update_logging)
+                tornado.ioloop.IOLoop.current().call_later(LOGGING_INTERVAL, self.update_logging)
-                thread.start()
+
 
     def on_close(self):
         if self.application.verbose:
@@ -144,29 +159,45 @@ class SocketHandler(tornado.websocket.WebSocketHandler):
 
     def send_log(self, logger, logging):
         self.write_message({'type': 'log',
-            'logger': logger,
+                            'logger': logger,
-            'logging': logging })
+                            'logging': logging})
+
+    @property
+    def simulation_name(self):
+        return self.config.get('name', 'NoSimulationRunning')
+
+    @run_on_executor
+    def nonblocking(self, config):
+        simulation = SoilSimulation(**config)
+        return simulation.run()
 
+    @tornado.gen.coroutine
     def run_simulation(self):
         # Run simulation and capture logs
+        logger.info('Running simulation!')
         if 'visualization_params' in self.config:
             del self.config['visualization_params']
-        with self.logging(self.application.simulator.name):
+        with self.logging(self.simulation_name):
             try:
-                self.simulation = self.application.simulator.run(self.config)
+                config = dict(**self.config)
-                trials = []
+                config['dir_path'] = os.path.join(self.application.dir_path, config['name'])
-                for i in range(self.config['num_trials']):
+                config['dump'] = self.application.dump
-                    trials.append('{}_trial_{}'.format(self.name, i))
+                self.trials = yield self.nonblocking(config)
+
                 self.write_message({'type': 'trials',
-                    'data': trials })
+                    'data': list(trial.name for trial in self.trials) })
-            except:
+            except Exception as ex:
-                error = 'Something went wrong. Please, try again.'
+                error = 'Something went wrong:\n\t{}'.format(ex)
+                logging.info(error)
                 self.write_message({'type': 'error',
-                    'error': error})
+                                    'error': error})
-                self.send_log('ERROR.' + self.application.simulator.name, error)
+                self.send_log('ERROR.' + self.simulation_name, error)
 
     def get_trial(self, trial):
-        G = self.simulation[trial].history_to_graph()
+        logger.info('Available trials: %s ' % len(self.trials))
+        logger.info('Ask for : %s' % trial)
+        trial = self.trials[trial]
+        G = trial.history_to_graph()
         return nx.node_link_data(G)
 
     @contextmanager
@@ -193,19 +224,20 @@ class ModularServer(tornado.web.Application):
     page_handler = (r'/', PageHandler)
     socket_handler = (r'/ws', SocketHandler)
     static_handler = (r'/(.*)', tornado.web.StaticFileHandler,
-                      {'path': 'templates'})
+                      {'path': os.path.join(ROOT, 'static')})
     local_handler = (r'/local/(.*)', tornado.web.StaticFileHandler,
                      {'path': ''})
 
     handlers = [page_handler, socket_handler, static_handler, local_handler]
     settings = {'debug': True,
-                'template_path': os.path.dirname(__file__) + '/templates'}
+                'template_path': ROOT + '/templates'}
 
-    def __init__(self, simulator, name='SOIL', verbose=True, *args, **kwargs):
+    def __init__(self, dump=False, dir_path='output', name='SOIL', verbose=True, *args, **kwargs):
         
         self.verbose = verbose
         self.name = name
-        self.simulator = simulator
+        self.dump = dump
+        self.dir_path = dir_path
 
         # Initializing the application itself:
         super().__init__(self.handlers, **self.settings)
@@ -220,3 +252,23 @@ class ModularServer(tornado.web.Application):
         self.listen(self.port)
         # webbrowser.open(url)
         tornado.ioloop.IOLoop.instance().start()
+
+
+def run(*args, **kwargs):
+    asyncio.set_event_loop(asyncio.new_event_loop())
+    server = ModularServer(*args, **kwargs)
+    server.launch()
+
+
+def main():
+    import argparse
+
+    parser = argparse.ArgumentParser(description='Visualization of a Graph Model')
+
+    parser.add_argument('--name', '-n', nargs=1, default='SOIL', help='name of the simulation')
+    parser.add_argument('--dump', '-d', help='dumping results in folder output', action='store_true')
+    parser.add_argument('--port', '-p', nargs=1, default=8001, help='port for launching the server')
+    parser.add_argument('--verbose', '-v', help='verbose mode', action='store_true')
+    args = parser.parse_args()
+
+    run(name=args.name, port=(args.port[0] if isinstance(args.port, list) else args.port), verbose=args.verbose)

soil/web/__main__.py

@@ -0,0 +1,5 @@
+from . import main
+
+
+if __name__ == "__main__":
+    main()

soil/web/simulator.py

@@ -1,36 +0,0 @@
-import os
-import networkx as nx
-from soil.simulation import SoilSimulation
-
-
-class Simulator():
-    """ Simulator for running simulations. Using SOIL."""
-
-    def __init__(self, dump=False, dir_path='output'):
-        self.name = 'soil'
-        self.dump = dump
-        self.dir_path = dir_path
-
-    def run(self, config):
-        name = config['name']
-        print('Using config(s): {name}'.format(name=name))
-
-        sim = SoilSimulation(**config)
-        sim.dir_path = os.path.join(self.dir_path, name)
-        sim.dump = self.dump
-
-        print('Dumping results to {} : {}'.format(sim.dir_path, sim.dump))
-        
-        simulation_results = sim.run_simulation()
-
-        # G = simulation_results[0].history_to_graph()
-        # for node in G.nodes():
-        #     if 'pos' in G.node[node]:
-        #         G.node[node]['viz'] = {"position": {"x": G.node[node]['pos'][0], "y": G.node[node]['pos'][1], "z": 0.0}}
-        #         del (G.node[node]['pos'])
-        # nx.write_gexf(G, 'test.gexf', version='1.2draft')
-
-        return simulation_results
-
-    def reset(self):
-        pass

soil/web/static/js/socket.js

@@ -239,17 +239,19 @@ var reset_configuration = function() {
     $('#download_json').off();
 }
 
+var slider;
+
 var set_timeline = function(graph) {
     // 'Timeline' slider
     var [min, max] = get_limits(graph);
 
-    var stepUnix = (max - min) / 200;
+    var stepUnix = 1;
     var minUnix  = (min !== Math.min()) ? min : 0;
     var maxUnix  = (max !== Math.max()) ? max : minUnix + 20;
 
     slider = d3.slider();
     d3.select('#slider3').attr('width', width).call(
-        slider.axis(true).min(minUnix).max(maxUnix).step(stepUnix).value(maxUnix)
+        slider.axis(true).min(minUnix).max(maxUnix).step(stepUnix).value(minUnix)
         .on('slide', function(evt, value) {
             self.GraphVisualization.update_graph($('.config-item #properties').val(), value, function() {
                 update_statistics_table();
@@ -281,65 +283,64 @@ var set_timeline = function(graph) {
 
     // Button 'Play'
     $('button#button_play').on('click', function() {
+        play();
 
-        $('button#button_play').addClass('pressed').prop("disabled", true);
-        $('#speed-slider').slider('disable');
-        slider.step( 1 / speed );
-        
-        if (slider.value() >= maxUnix) {
-            slider.value(minUnix);
-            self.GraphVisualization.update_graph($('.config-item #properties').val(), slider.value(), function() {
-                update_statistics_table();
-            });
-            setTimeout(player, 1000);
-        } else {
-            player();
-        }
-
-        var i = slider.value();
-        function player() {
-            clearInterval(play);
-            play = setInterval(function() {
-                self.GraphVisualization.update_graph($('.config-item #properties').val(), slider.value(), function() {
-                    update_statistics_table();
-                });
-
-                if (slider.value() + slider.step() >= maxUnix) {
-                    slider.value(maxUnix);
-                    slider.step(stepUnix);
-                    clearInterval(play);
-                    $('button#button_play').removeClass('pressed').prop("disabled", false);
-                    $('#speed-slider').slider('enable');
-                } else {
-                    slider.value(i);
-                    i += slider.step();
-                }
-
-            }, 5);
-        }
     });
 
     // Button 'Pause'
     $('button#button_pause').on('click', function() {
-        clearInterval(play);
+        stop();
-        slider.step(stepUnix);
         $('button#button_play').removeClass('pressed').prop("disabled", false);
-        $('#speed-slider').slider('enable');
     });
 
     // Button 'Zoom to Fit'
     $('button#button_zoomFit').click(function() { self.GraphVisualization.fit(); });
 }
 
+var player;
+
+function play(){
+    $('button#button_play').addClass('pressed').prop("disabled", true);
+
+    if (slider.value() >= slider.max()) {
+        slider.value(slider.min());
+    }
+
+    var FRAME_INTERVAL = 100;
+    var speed_ratio = FRAME_INTERVAL / 1000 // speed=1 => 1 step per second
+    
+    nextStep = function() {
+        newvalue = Math.min(slider.value() + speed*speed_ratio, slider.max());
+        console.log("new time value", newvalue);
+        slider.value(newvalue);
+
+        self.GraphVisualization.update_graph($('.config-item #properties').val(), slider.value(), function () {
+            update_statistics_table();
+        });
+
+        if (newvalue < slider.max()) {
+            player = setTimeout(nextStep, FRAME_INTERVAL);
+        } else {
+            $('button#button_play').removeClass('pressed').prop("disabled", false);
+        }
+    }
+
+    player = setTimeout(nextStep, FRAME_INTERVAL);
+}
+
+function stop() {
+    clearTimeout(player);
+}
+
 var reset_timeline = function() {
     // 'Timeline' slider
     $('#slider3').html('');
 
     // 'Speed' slider
-    $('#speed-slider').slider('disable').slider('setValue', 1000);
+    // $('#speed-slider').slider('disable').slider('setValue', 1000);
 
     // Buttons
-    clearInterval(play);
+    stop();
     $('button#button_play').off().removeClass('pressed').prop("disabled", false);
     $('button#button_pause').off();
     $('button#button_zoomFit').off();

soil/web/templates/index.html

@@ -41,7 +41,7 @@
 
     var width   = window.innerWidth * 0.75,
         height  = window.innerHeight * 3 / 5,
-        speed   = 1000,
+        speed   = 1,
         play,
         slider;
 
@@ -255,7 +255,7 @@
                             <th class="text-right max">max</th>
                         </tr></tbody></table>
                         <div class="speed-slider">
-                            <input id="speed-slider" type="text" data-slider-min="1" data-slider-max="1000" data-slider-step="0.01" data-slider-value="1000" data-slider-tooltip="hide" data-slider-reversed="true" data-slider-enabled="false" data-slider-scale="logarithmic"/> 
+                            <input id="speed-slider" type="text" data-slider-min="0.1" data-slider-max="100" data-slider-step="0.1" data-slider-value="1" data-slider-tooltip="hide" data-slider-enabled="false" data-slider-scale="logarithmic"/> 
                         </div>
                     </div>
                     <hr />