Clean up exporters

CHANGELOG.md

@@ -3,6 +3,15 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.14.2]
+### Fixed
+* Output path for exporters is now soil_output
+### Changed 
+* CSV output to stdout in dry_run mode
+## [0.14.1]
+### Changed
+* Exporter names in lower case
+* Add default exporter in runs
 ## [0.14.0]
 ### Added
 * Loading configuration from template definitions in the yaml, in preparation for SALib support.

docs/soil_tutorial.rst

@@ -323,7 +323,7 @@ Let's run our simulation:
 
 .. code:: ipython3
 
-    soil.simulation.run_from_config(config, dump=False)
+    soil.simulation.run_from_config(config)
 
 
 .. parsed-literal::

soil/VERSION

@@ -1 +1 @@
-0.14.0
+0.14.2

soil/__init__.py

@@ -57,11 +57,11 @@ def main():
     logging.info('Loading config file: {}'.format(args.file))
 
     try:
-        exporters = list(args.exporter or [])
+        exporters = list(args.exporter or ['default', ])
         if args.csv:
-            exporters.append('CSV')
+            exporters.append('csv')
         if args.graph:
-            exporters.append('Gexf')
+            exporters.append('gexf')
         exp_params = {}
         if args.dry_run:
             exp_params['copy_to'] = sys.stdout

soil/exporters.py

@@ -50,7 +50,7 @@ class Exporter:
 
     def __init__(self, simulation, outdir=None, dry_run=None, copy_to=None):
         self.sim = simulation
-        outdir = outdir or os.getcwd()
+        outdir = outdir or os.path.join(os.getcwd(), 'soil_output')
         self.outdir = os.path.join(outdir,
                                    simulation.group or '',
                                    simulation.name)
@@ -78,8 +78,8 @@ class Exporter:
         return open_or_reuse(f, mode=mode, **kwargs)
 
 
-class Default(Exporter):
+class default(Exporter):
-    '''Default exporter. Writes CSV and sqlite results, as well as the simulation YAML'''
+    '''Default exporter. Writes sqlite results, as well as the simulation YAML'''
 
     def start(self):
         if not self.dry_run:
@@ -96,25 +96,29 @@ class Default(Exporter):
                     env.dump_sqlite(f)
 
 
-class CSV(Exporter):
+class csv(Exporter):
+    '''Export the state of each environment (and its agents) in a separate CSV file'''
     def trial_end(self, env):
-        if not self.dry_run:
+        with timer('[CSV] Dumping simulation {} trial {} @ dir {}'.format(self.sim.name,
-            with timer('[CSV] Dumping simulation {} trial {}'.format(self.sim.name,
+                                                                          env.name,
-                                                               env.name)):
+                                                                          self.outdir)):
-                with self.output('{}.csv'.format(env.name)) as f:
+            with self.output('{}.csv'.format(env.name)) as f:
-                    env.dump_csv(f)
+                env.dump_csv(f)
 
 
-class Gexf(Exporter):
+class gexf(Exporter):
     def trial_end(self, env):
-        if not self.dry_run:
+        if self.dry_run:
-            with timer('[CSV] Dumping simulation {} trial {}'.format(self.sim.name,
+            logger.info('Not dumping GEXF in dry_run mode')
-                                                                     env.name)):
+            return
-                with self.output('{}.gexf'.format(env.name), mode='wb') as f:
+
-                    env.dump_gexf(f)
+        with timer('[GEXF] Dumping simulation {} trial {}'.format(self.sim.name,
+                                                                  env.name)):
+            with self.output('{}.gexf'.format(env.name), mode='wb') as f:
+                env.dump_gexf(f)
 
 
-class Dummy(Exporter):
+class dummy(Exporter):
 
     def start(self):
         with self.output('dummy', 'w') as f:
@@ -131,7 +135,7 @@ class Dummy(Exporter):
             f.write('simulation ended @ {}\n'.format(time.time()))
 
 
-class Distribution(Exporter):
+class distribution(Exporter):
     '''
     Write the distribution of agent states at the end of each trial,
     the mean value, and its deviation.
@@ -165,7 +169,7 @@ class Distribution(Exporter):
         with self.output('metrics.csv') as f:
             dfm.to_csv(f)
 
-class GraphDrawing(Exporter):
+class graphdrawing(Exporter):
 
     def trial_end(self, env):
         # Outside effects

soil/history.py

@@ -11,6 +11,7 @@ logger = logging.getLogger(__name__)
 from collections import UserDict, namedtuple
 
 from . import serialization
+from .utils import open_or_reuse
 
 
 class History:
@@ -236,7 +237,7 @@ class History:
 
     def dump(self, f):
         self._close()
-        for line in open(self.db_path, 'rb'):
+        for line in open_or_reuse(self.db_path, 'rb'):
             f.write(line)
 
 

soil/simulation.py

@@ -153,11 +153,11 @@ class Simulation(NetworkSimulation):
                                      **kwargs)
 
     def _run_simulation_gen(self, *args, parallel=False, dry_run=False,
-                            exporters=None, outdir=None, exporter_params={}, **kwargs):
+                            exporters=['default', ], outdir=None, exporter_params={}, **kwargs):
         logger.info('Using exporters: %s', exporters or [])
         logger.info('Output directory: %s', outdir)
         exporters = exporters_for_sim(self,
-                                      exporters or [],
+                                      exporters,
                                       dry_run=dry_run,
                                       outdir=outdir,
                                       **exporter_params)

soil/utils.py

@@ -2,6 +2,8 @@ import logging
 import time
 import os
 
+from shutil import copyfile
+
 from contextlib import contextmanager
 
 logger = logging.getLogger('soil')
@@ -23,11 +25,22 @@ def timer(name='task', pre="", function=logger.info, to_object=None):
         to_object.end = end
 
 
-def safe_open(path, *args, **kwargs):
+def safe_open(path, mode='r', backup=True, **kwargs):
     outdir = os.path.dirname(path)
     if outdir and not os.path.exists(outdir):
         os.makedirs(outdir)
-    return open(path, *args, **kwargs)
+    if backup and 'w' in mode and os.path.exists(path):
+        creation = os.path.getctime(path)
+        stamp = time.strftime('%Y-%m-%d_%H:%M', time.localtime(creation))
+
+        backup_dir = os.path.join(outdir, stamp)
+        if not os.path.exists(backup_dir):
+            os.makedirs(backup_dir)
+        newpath = os.path.join(backup_dir, os.path.basename(path))
+        if os.path.exists(newpath):
+            newpath = '{}@{}'.format(newpath, time.time())
+        copyfile(path, newpath)
+    return open(path, mode=mode, **kwargs)
 
 
 def open_or_reuse(f, *args, **kwargs):

tests/test_exporters.py

@@ -60,7 +60,7 @@ class Exporters(TestCase):
         }
         output = io.StringIO()
         s = simulation.from_config(config)
-        s.run_simulation(exporters=[exporters.Distribution], dry_run=True, exporter_params={'copy_to': output})
+        s.run_simulation(exporters=[exporters.distribution], dry_run=True, exporter_params={'copy_to': output})
         result = output.getvalue()
         assert 'count' in result
         assert 'SEED,Noneexporter_sim_trial_3,1,,1,1,1,1' in result
@@ -83,10 +83,10 @@ class Exporters(TestCase):
         s = simulation.from_config(config)
         tmpdir = tempfile.mkdtemp()
         envs = s.run_simulation(exporters=[
-            exporters.Default,
+            exporters.default,
-            exporters.CSV,
+            exporters.csv,
-            exporters.Gexf,
+            exporters.gexf,
-            exporters.Distribution,
+            exporters.distribution,
         ],
                                 outdir=tmpdir,
                                 exporter_params={'copy_to': output})