1
0
mirror of https://github.com/gsi-upm/soil synced 2024-11-13 06:52:28 +00:00

Clean up exporters

This commit is contained in:
J. Fernando Sánchez 2019-05-03 13:17:27 +02:00
parent b0add8552e
commit 97835b3d10
9 changed files with 60 additions and 33 deletions

View File

@ -3,6 +3,15 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [0.14.2]
### Fixed
* Output path for exporters is now soil_output
### Changed
* CSV output to stdout in dry_run mode
## [0.14.1]
### Changed
* Exporter names in lower case
* Add default exporter in runs
## [0.14.0]
### Added
* Loading configuration from template definitions in the yaml, in preparation for SALib support.

View File

@ -323,7 +323,7 @@ Let's run our simulation:
.. code:: ipython3
soil.simulation.run_from_config(config, dump=False)
soil.simulation.run_from_config(config)
.. parsed-literal::

View File

@ -1 +1 @@
0.14.0
0.14.2

View File

@ -57,11 +57,11 @@ def main():
logging.info('Loading config file: {}'.format(args.file))
try:
exporters = list(args.exporter or [])
exporters = list(args.exporter or ['default', ])
if args.csv:
exporters.append('CSV')
exporters.append('csv')
if args.graph:
exporters.append('Gexf')
exporters.append('gexf')
exp_params = {}
if args.dry_run:
exp_params['copy_to'] = sys.stdout

View File

@ -50,7 +50,7 @@ class Exporter:
def __init__(self, simulation, outdir=None, dry_run=None, copy_to=None):
self.sim = simulation
outdir = outdir or os.getcwd()
outdir = outdir or os.path.join(os.getcwd(), 'soil_output')
self.outdir = os.path.join(outdir,
simulation.group or '',
simulation.name)
@ -78,8 +78,8 @@ class Exporter:
return open_or_reuse(f, mode=mode, **kwargs)
class Default(Exporter):
'''Default exporter. Writes CSV and sqlite results, as well as the simulation YAML'''
class default(Exporter):
'''Default exporter. Writes sqlite results, as well as the simulation YAML'''
def start(self):
if not self.dry_run:
@ -96,25 +96,29 @@ class Default(Exporter):
env.dump_sqlite(f)
class CSV(Exporter):
class csv(Exporter):
'''Export the state of each environment (and its agents) in a separate CSV file'''
def trial_end(self, env):
if not self.dry_run:
with timer('[CSV] Dumping simulation {} trial {}'.format(self.sim.name,
env.name)):
with self.output('{}.csv'.format(env.name)) as f:
env.dump_csv(f)
with timer('[CSV] Dumping simulation {} trial {} @ dir {}'.format(self.sim.name,
env.name,
self.outdir)):
with self.output('{}.csv'.format(env.name)) as f:
env.dump_csv(f)
class Gexf(Exporter):
class gexf(Exporter):
def trial_end(self, env):
if not self.dry_run:
with timer('[CSV] Dumping simulation {} trial {}'.format(self.sim.name,
env.name)):
with self.output('{}.gexf'.format(env.name), mode='wb') as f:
env.dump_gexf(f)
if self.dry_run:
logger.info('Not dumping GEXF in dry_run mode')
return
with timer('[GEXF] Dumping simulation {} trial {}'.format(self.sim.name,
env.name)):
with self.output('{}.gexf'.format(env.name), mode='wb') as f:
env.dump_gexf(f)
class Dummy(Exporter):
class dummy(Exporter):
def start(self):
with self.output('dummy', 'w') as f:
@ -131,7 +135,7 @@ class Dummy(Exporter):
f.write('simulation ended @ {}\n'.format(time.time()))
class Distribution(Exporter):
class distribution(Exporter):
'''
Write the distribution of agent states at the end of each trial,
the mean value, and its deviation.
@ -165,7 +169,7 @@ class Distribution(Exporter):
with self.output('metrics.csv') as f:
dfm.to_csv(f)
class GraphDrawing(Exporter):
class graphdrawing(Exporter):
def trial_end(self, env):
# Outside effects

View File

@ -11,6 +11,7 @@ logger = logging.getLogger(__name__)
from collections import UserDict, namedtuple
from . import serialization
from .utils import open_or_reuse
class History:
@ -236,7 +237,7 @@ class History:
def dump(self, f):
self._close()
for line in open(self.db_path, 'rb'):
for line in open_or_reuse(self.db_path, 'rb'):
f.write(line)

View File

@ -153,11 +153,11 @@ class Simulation(NetworkSimulation):
**kwargs)
def _run_simulation_gen(self, *args, parallel=False, dry_run=False,
exporters=None, outdir=None, exporter_params={}, **kwargs):
exporters=['default', ], outdir=None, exporter_params={}, **kwargs):
logger.info('Using exporters: %s', exporters or [])
logger.info('Output directory: %s', outdir)
exporters = exporters_for_sim(self,
exporters or [],
exporters,
dry_run=dry_run,
outdir=outdir,
**exporter_params)

View File

@ -2,6 +2,8 @@ import logging
import time
import os
from shutil import copyfile
from contextlib import contextmanager
logger = logging.getLogger('soil')
@ -23,11 +25,22 @@ def timer(name='task', pre="", function=logger.info, to_object=None):
to_object.end = end
def safe_open(path, *args, **kwargs):
def safe_open(path, mode='r', backup=True, **kwargs):
outdir = os.path.dirname(path)
if outdir and not os.path.exists(outdir):
os.makedirs(outdir)
return open(path, *args, **kwargs)
if backup and 'w' in mode and os.path.exists(path):
creation = os.path.getctime(path)
stamp = time.strftime('%Y-%m-%d_%H:%M', time.localtime(creation))
backup_dir = os.path.join(outdir, stamp)
if not os.path.exists(backup_dir):
os.makedirs(backup_dir)
newpath = os.path.join(backup_dir, os.path.basename(path))
if os.path.exists(newpath):
newpath = '{}@{}'.format(newpath, time.time())
copyfile(path, newpath)
return open(path, mode=mode, **kwargs)
def open_or_reuse(f, *args, **kwargs):

View File

@ -60,7 +60,7 @@ class Exporters(TestCase):
}
output = io.StringIO()
s = simulation.from_config(config)
s.run_simulation(exporters=[exporters.Distribution], dry_run=True, exporter_params={'copy_to': output})
s.run_simulation(exporters=[exporters.distribution], dry_run=True, exporter_params={'copy_to': output})
result = output.getvalue()
assert 'count' in result
assert 'SEED,Noneexporter_sim_trial_3,1,,1,1,1,1' in result
@ -83,10 +83,10 @@ class Exporters(TestCase):
s = simulation.from_config(config)
tmpdir = tempfile.mkdtemp()
envs = s.run_simulation(exporters=[
exporters.Default,
exporters.CSV,
exporters.Gexf,
exporters.Distribution,
exporters.default,
exporters.csv,
exporters.gexf,
exporters.distribution,
],
outdir=tmpdir,
exporter_params={'copy_to': output})