diff --git a/bluepyopt/tests/test_l5pc.py b/bluepyopt/tests/test_l5pc.py index 1d3acb5..8b382fd 100644 --- a/bluepyopt/tests/test_l5pc.py +++ b/bluepyopt/tests/test_l5pc.py @@ -1,7 +1,9 @@ """Test l5pc example""" +import json import os import sys + from contextlib import contextmanager if sys.version_info[0] < 3: from StringIO import StringIO @@ -55,8 +57,6 @@ def load_from_json(filename): """Load structure from json""" - import json - with open(filename) as json_file: return json.load(json_file) @@ -64,8 +64,6 @@ def load_from_json(filename): def dump_to_json(content, filename): """Dump structure to json""" - import json - with open(filename, 'w') as json_file: return json.dump(content, json_file, indent=4, separators=(',', ': ')) @@ -77,9 +75,6 @@ def test_import(): import l5pc_evaluator # NOQA import opt_l5pc # NOQA - # Delete the optimisation inside the module - del opt_l5pc.opt - class TestL5PCModel(object): diff --git a/examples/l5pc/benchmark/start.sh b/examples/l5pc/benchmark/start.sh index 1019bbd..f59d397 100755 --- a/examples/l5pc/benchmark/start.sh +++ b/examples/l5pc/benchmark/start.sh @@ -1,29 +1,39 @@ #!/bin/bash set -e -set -x +set -x + +PWD=$(pwd) +LOGS=$PWD/logs +mkdir -p $LOGS cd .. OFFSPRING_SIZE=100 MAX_NGEN=100 -export L5PCBENCHMARK_USEIPYP=1 -export IPYTHONDIR="`pwd`/.ipython" -export IPYTHON_PROFILE=benchmark.${SLURM_JOBID} -# ipython profile create --profile=${IPYTHON_PROFILE} +export IPYTHONDIR=${PWD}/.ipython +export IPYTHON_PROFILE=benchmark.${SLURM_JOBID} + ipcontroller --init --ip='*' --sqlitedb --profile=${IPYTHON_PROFILE} & sleep 10 -srun ipengine --profile=${IPYTHON_PROFILE} & +srun --output="${LOGS}/engine_%j_%2t.out" ipengine --profile=${IPYTHON_PROFILE} & +sleep 10 CHECKPOINTS_DIR="checkpoints/run.${SLURM_JOBID}" mkdir -p ${CHECKPOINTS_DIR} pids="" -for seed in `seq 1 4` -do - BLUEPYOPT_SEED=${seed} python opt_l5pc.py --offspring_size=${OFFSPRING_SIZE} --max_ngen=${MAX_NGEN} --start --checkpoint "${CHECKPOINTS_DIR}/seed${seed}.pkl" & - pids="${pids} $!" +for seed in {1..4}; do + python opt_l5pc.py \ + -vv \ + --offspring_size=${OFFSPRING_SIZE} \ + --max_ngen=${MAX_NGEN} \ + --seed=${seed} \ + --ipyparallel \ + --start \ + --checkpoint "${CHECKPOINTS_DIR}/seed${seed}.pkl" & + pids+="$! " done wait $pids diff --git a/examples/l5pc/opt_l5pc.py b/examples/l5pc/opt_l5pc.py old mode 100644 new mode 100755 index b1d61d6..c5ece29 --- a/examples/l5pc/opt_l5pc.py +++ b/examples/l5pc/opt_l5pc.py @@ -1,4 +1,9 @@ -"""Run simple cell optimisation""" +#!/usr/bin/env python +"""Run simple cell optimisation + +This optimisation is based on L5PC optimisations developed by Etay Hay in the +context of the BlueBrain project +""" """ Copyright (c) 2016, EPFL/Blue Brain Project @@ -19,55 +24,65 @@ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. """ -""" -This optimisation is based on L5PC optimisations developed by Etay Hay in the -context of the BlueBrain project -""" -# pylint: disable=R0914, W0403 +import argparse +import logging import os import sys +import textwrap -import argparse -# pylint: disable=R0914 -import logging -logging.basicConfig(stream=sys.stdout) -logger = logging.getLogger() -logger.setLevel(logging.DEBUG) +from datetime import datetime import bluepyopt +import l5pc_evaluator + +logger = logging.getLogger() + # TODO store definition dicts in json # TODO add functionality to read settings of every object from config format -import l5pc_evaluator -evaluator = l5pc_evaluator.create() +def create_optimizer(args): + '''returns configured bluepyopt.optimisations.DEAPOptimisation''' + if args.ipyparallel or os.getenv('L5PCBENCHMARK_USEIPYP'): + from ipyparallel import Client + rc = Client(profile=os.getenv('IPYTHON_PROFILE')) + logger.debug('Using ipyparallel with %d engines', len(rc)) -def evaluate(parameter_array): - """Global evaluate function""" + lview = rc.load_balanced_view() - return evaluator.evaluate(parameter_array) + def mapper(func, it): + start_time = datetime.now() + ret = lview.map_sync(func, it) + logger.debug('Generation took %s', datetime.now() - start_time) + return ret -if os.getenv('L5PCBENCHMARK_USEIPYP') == '1': - from ipyparallel import Client - rc = Client(profile=os.getenv('IPYTHON_PROFILE')) - lview = rc.load_balanced_view() + map_function = mapper + else: + map_function = None - map_function = lview.map_sync -else: - map_function = None + evaluator = l5pc_evaluator.create() + seed = os.getenv('BLUEPYOPT_SEED', args.seed) + opt = bluepyopt.optimisations.DEAPOptimisation( + evaluator=evaluator, + map_function=map_function, + seed=seed) -opt = bluepyopt.optimisations.DEAPOptimisation( - evaluator=evaluator, - map_function=map_function, - seed=os.getenv('BLUEPYOPT_SEED')) + return opt -def main(): - """Main""" - parser = argparse.ArgumentParser(description='L5PC example') +def get_parser(): + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description='L5PC example', + epilog=textwrap.dedent('''\ +The folling environment variables are considered: + L5PCBENCHMARK_USEIPYP: if set, will use ipyparallel + IPYTHON_PROFILE: if set, used as the path to the ipython profile + BLUEPYOPT_SEED: The seed used for initial randomization + ''')) parser.add_argument('--start', action="store_true") parser.add_argument('--continu', action="store_false", default=False) parser.add_argument('--checkpoint', required=False, default=None, @@ -81,20 +96,37 @@ def main(): parser.add_argument('--analyse', action="store_true") parser.add_argument('--compile', action="store_true") parser.add_argument('--hocanalyse', action="store_true") + parser.add_argument('--seed', type=int, default=42, + help='Seed to use for optimization') + parser.add_argument('--ipyparallel', action="store_true", default=False, + help='Use ipyparallel') parser.add_argument( '--diversity', help='plot the diversity of parameters from checkpoint pickle file') + parser.add_argument('-v', '--verbose', action='count', dest='verbose', + default=0, help='-v for INFO, -vv for DEBUG') + + return parser + - args = parser.parse_args() +def main(): # pylint: disable=too-many-statements + """Main""" + args = get_parser().parse_args() + + if args.verbose > 2: + sys.exit('cannot be more verbose than -vv') + logging.basicConfig(level=(logging.WARNING, + logging.INFO, + logging.DEBUG)[args.verbose], + stream=sys.stdout) + + opt = create_optimizer(args) if args.compile: logger.debug('Doing compile') import commands commands.getstatusoutput('cd mechanisms/; nrnivmodl; cd ..') - # TODO store definition dicts in json - # TODO add functionality to read settings of every object from config format - if args.hocanalyse: logger.debug('Doing hocanalyse') try: @@ -156,11 +188,9 @@ def main(): elif args.hocanalyse: logger.debug('Continuing hocanalyse') + import matplotlib.pyplot as plt import l5pc_analysis - # _, axes_obj = plt.subplots(n_of_rows, n_of_cols, facecolor='white') - # axes = numpy.ravel(axes_obj) - import matplotlib.pyplot as plt fig_release = plt.figure(figsize=(10, 10), facecolor='white') box = { @@ -194,5 +224,6 @@ def main(): fig_diversity.savefig('figures/l5pc_diversity.eps') plt.show() + if __name__ == '__main__': main()