From 97835b3d106a7a7d3b75c70240320bac4c85b204 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=2E=20Fernando=20S=C3=A1nchez?= Date: Fri, 3 May 2019 13:17:27 +0200 Subject: [PATCH] Clean up exporters --- CHANGELOG.md | 9 +++++++++ docs/soil_tutorial.rst | 2 +- soil/VERSION | 2 +- soil/__init__.py | 6 +++--- soil/exporters.py | 40 ++++++++++++++++++++++------------------ soil/history.py | 3 ++- soil/simulation.py | 4 ++-- soil/utils.py | 17 +++++++++++++++-- tests/test_exporters.py | 10 +++++----- 9 files changed, 60 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c11473d..ca98951 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,15 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.14.2] +### Fixed +* Output path for exporters is now soil_output +### Changed +* CSV output to stdout in dry_run mode +## [0.14.1] +### Changed +* Exporter names in lower case +* Add default exporter in runs ## [0.14.0] ### Added * Loading configuration from template definitions in the yaml, in preparation for SALib support. diff --git a/docs/soil_tutorial.rst b/docs/soil_tutorial.rst index d20d992..6e7247b 100644 --- a/docs/soil_tutorial.rst +++ b/docs/soil_tutorial.rst @@ -323,7 +323,7 @@ Let's run our simulation: .. code:: ipython3 - soil.simulation.run_from_config(config, dump=False) + soil.simulation.run_from_config(config) .. parsed-literal:: diff --git a/soil/VERSION b/soil/VERSION index a803cc2..e867cc2 100644 --- a/soil/VERSION +++ b/soil/VERSION @@ -1 +1 @@ -0.14.0 +0.14.2 diff --git a/soil/__init__.py b/soil/__init__.py index 191ec9d..8ea4da1 100644 --- a/soil/__init__.py +++ b/soil/__init__.py @@ -57,11 +57,11 @@ def main(): logging.info('Loading config file: {}'.format(args.file)) try: - exporters = list(args.exporter or []) + exporters = list(args.exporter or ['default', ]) if args.csv: - exporters.append('CSV') + exporters.append('csv') if args.graph: - exporters.append('Gexf') + exporters.append('gexf') exp_params = {} if args.dry_run: exp_params['copy_to'] = sys.stdout diff --git a/soil/exporters.py b/soil/exporters.py index 3ba16de..ba31f90 100644 --- a/soil/exporters.py +++ b/soil/exporters.py @@ -50,7 +50,7 @@ class Exporter: def __init__(self, simulation, outdir=None, dry_run=None, copy_to=None): self.sim = simulation - outdir = outdir or os.getcwd() + outdir = outdir or os.path.join(os.getcwd(), 'soil_output') self.outdir = os.path.join(outdir, simulation.group or '', simulation.name) @@ -78,8 +78,8 @@ class Exporter: return open_or_reuse(f, mode=mode, **kwargs) -class Default(Exporter): - '''Default exporter. Writes CSV and sqlite results, as well as the simulation YAML''' +class default(Exporter): + '''Default exporter. Writes sqlite results, as well as the simulation YAML''' def start(self): if not self.dry_run: @@ -96,25 +96,29 @@ class Default(Exporter): env.dump_sqlite(f) -class CSV(Exporter): +class csv(Exporter): + '''Export the state of each environment (and its agents) in a separate CSV file''' def trial_end(self, env): - if not self.dry_run: - with timer('[CSV] Dumping simulation {} trial {}'.format(self.sim.name, - env.name)): - with self.output('{}.csv'.format(env.name)) as f: - env.dump_csv(f) + with timer('[CSV] Dumping simulation {} trial {} @ dir {}'.format(self.sim.name, + env.name, + self.outdir)): + with self.output('{}.csv'.format(env.name)) as f: + env.dump_csv(f) -class Gexf(Exporter): +class gexf(Exporter): def trial_end(self, env): - if not self.dry_run: - with timer('[CSV] Dumping simulation {} trial {}'.format(self.sim.name, - env.name)): - with self.output('{}.gexf'.format(env.name), mode='wb') as f: - env.dump_gexf(f) + if self.dry_run: + logger.info('Not dumping GEXF in dry_run mode') + return + + with timer('[GEXF] Dumping simulation {} trial {}'.format(self.sim.name, + env.name)): + with self.output('{}.gexf'.format(env.name), mode='wb') as f: + env.dump_gexf(f) -class Dummy(Exporter): +class dummy(Exporter): def start(self): with self.output('dummy', 'w') as f: @@ -131,7 +135,7 @@ class Dummy(Exporter): f.write('simulation ended @ {}\n'.format(time.time())) -class Distribution(Exporter): +class distribution(Exporter): ''' Write the distribution of agent states at the end of each trial, the mean value, and its deviation. @@ -165,7 +169,7 @@ class Distribution(Exporter): with self.output('metrics.csv') as f: dfm.to_csv(f) -class GraphDrawing(Exporter): +class graphdrawing(Exporter): def trial_end(self, env): # Outside effects diff --git a/soil/history.py b/soil/history.py index 41282f0..0532ee9 100644 --- a/soil/history.py +++ b/soil/history.py @@ -11,6 +11,7 @@ logger = logging.getLogger(__name__) from collections import UserDict, namedtuple from . import serialization +from .utils import open_or_reuse class History: @@ -236,7 +237,7 @@ class History: def dump(self, f): self._close() - for line in open(self.db_path, 'rb'): + for line in open_or_reuse(self.db_path, 'rb'): f.write(line) diff --git a/soil/simulation.py b/soil/simulation.py index daade10..6dd9c18 100644 --- a/soil/simulation.py +++ b/soil/simulation.py @@ -153,11 +153,11 @@ class Simulation(NetworkSimulation): **kwargs) def _run_simulation_gen(self, *args, parallel=False, dry_run=False, - exporters=None, outdir=None, exporter_params={}, **kwargs): + exporters=['default', ], outdir=None, exporter_params={}, **kwargs): logger.info('Using exporters: %s', exporters or []) logger.info('Output directory: %s', outdir) exporters = exporters_for_sim(self, - exporters or [], + exporters, dry_run=dry_run, outdir=outdir, **exporter_params) diff --git a/soil/utils.py b/soil/utils.py index 5fa67aa..a292a50 100644 --- a/soil/utils.py +++ b/soil/utils.py @@ -2,6 +2,8 @@ import logging import time import os +from shutil import copyfile + from contextlib import contextmanager logger = logging.getLogger('soil') @@ -23,11 +25,22 @@ def timer(name='task', pre="", function=logger.info, to_object=None): to_object.end = end -def safe_open(path, *args, **kwargs): +def safe_open(path, mode='r', backup=True, **kwargs): outdir = os.path.dirname(path) if outdir and not os.path.exists(outdir): os.makedirs(outdir) - return open(path, *args, **kwargs) + if backup and 'w' in mode and os.path.exists(path): + creation = os.path.getctime(path) + stamp = time.strftime('%Y-%m-%d_%H:%M', time.localtime(creation)) + + backup_dir = os.path.join(outdir, stamp) + if not os.path.exists(backup_dir): + os.makedirs(backup_dir) + newpath = os.path.join(backup_dir, os.path.basename(path)) + if os.path.exists(newpath): + newpath = '{}@{}'.format(newpath, time.time()) + copyfile(path, newpath) + return open(path, mode=mode, **kwargs) def open_or_reuse(f, *args, **kwargs): diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 95a0ebf..a1354dc 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -60,7 +60,7 @@ class Exporters(TestCase): } output = io.StringIO() s = simulation.from_config(config) - s.run_simulation(exporters=[exporters.Distribution], dry_run=True, exporter_params={'copy_to': output}) + s.run_simulation(exporters=[exporters.distribution], dry_run=True, exporter_params={'copy_to': output}) result = output.getvalue() assert 'count' in result assert 'SEED,Noneexporter_sim_trial_3,1,,1,1,1,1' in result @@ -83,10 +83,10 @@ class Exporters(TestCase): s = simulation.from_config(config) tmpdir = tempfile.mkdtemp() envs = s.run_simulation(exporters=[ - exporters.Default, - exporters.CSV, - exporters.Gexf, - exporters.Distribution, + exporters.default, + exporters.csv, + exporters.gexf, + exporters.distribution, ], outdir=tmpdir, exporter_params={'copy_to': output})