From 47c3eb5d5ebd11e74f7363f6312b505ac8dcbefc Mon Sep 17 00:00:00 2001 From: Daniel Nachbaur Date: Tue, 30 May 2017 12:19:54 +0200 Subject: [PATCH 1/2] Improve synapse attribute reading performance Smarter skipping for unmerged synapse file case, especially in the case of nrn_extra. Performance increase for reading conductances of all synapses for one cell from ~900ms to 5ms on the release O1 circuit. Reading again takes around 500 microseconds. --- brion/synapse.cpp | 33 +++++++++++++++++++++------------ doc/Changelog.md | 4 +++- 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/brion/synapse.cpp b/brion/synapse.cpp index c6a842b..ef2fe88 100644 --- a/brion/synapse.cpp +++ b/brion/synapse.cpp @@ -26,10 +26,8 @@ #include #include -#include #include #include -#include #include @@ -37,6 +35,7 @@ #include #include +#include namespace brion { @@ -84,7 +83,6 @@ bool _openDataset(const H5::H5File& file, const std::string& name, } }; namespace fs = boost::filesystem; -using boost::lexical_cast; /** Access a single synapse file (nrn*.h5 or nrn*.h5. */ class SynapseFile : public boost::noncopyable @@ -108,8 +106,8 @@ class SynapseFile : public boost::noncopyable Dataset dataset; const std::string& datasetName = _file.getObjnameByIdx(0); if (!detail::_openDataset(_file, datasetName, dataset)) - LBTHROW(std::runtime_error("Cannot open dataset in synapse file " + - source)); + LBTHROW(std::runtime_error("Cannot open dataset " + datasetName + + " in synapse file " + source)); _numAttributes = dataset.dims[1]; if (_numAttributes != SYNAPSE_ALL && @@ -166,7 +164,7 @@ class SynapseFile : public boost::noncopyable lunchbox::ScopedWrite mutex(detail::_hdf5Lock); size_t numSynapses = 0; - BOOST_FOREACH (const uint32_t gid, gids) + for (const uint32_t gid : gids) { Dataset dataset; if (!_openDataset(gid, dataset)) @@ -223,14 +221,25 @@ class Synapse : public boost::noncopyable } catch (const std::runtime_error&) { - LBINFO << "No merged synapse file found at " << source << std::endl; - const fs::path sourcePath(source); const fs::path dir = sourcePath.parent_path(); const std::string filename = sourcePath.filename().generic_string(); - _createIndex(dir, filename); + // OPT: check if we have at least one unmerged file before fetching + // all the filenames. Much faster in case there is no merged and/or + // unmerged file at all. + if (!boost::filesystem::exists(fs::path(source + ".0"))) + { + throw std::runtime_error( + "No merged or unmerged synapse file found: " + source); + } + + LBWARN << "Only unmerged synapse files found for " << source + << "; consider using merged files for better performance." + << std::endl; + _findCandidateFiles(dir, filename); + _createIndex(dir, filename); } } @@ -245,7 +254,7 @@ class Synapse : public boost::noncopyable size_t getNumSynapses(const GIDSet& gids) const { size_t numSynapses = 0; - BOOST_FOREACH (const uint32_t gid, gids) + for (const uint32_t gid : gids) { if (!_findFile(gid)) continue; @@ -284,7 +293,7 @@ class Synapse : public boost::noncopyable } private: - typedef boost::unordered_map GidFileMap; + typedef std::unordered_map GidFileMap; mutable SynapseFile* _file; mutable uint32_t _gid; // current or 0 for all @@ -384,7 +393,7 @@ class Synapse : public boost::noncopyable SilenceHDF5 silence; // this trial-and-error is the 'fastest' path found - BOOST_FOREACH (const std::string& candidate, _fileNames) + for (const std::string& candidate : _fileNames) { try { diff --git a/doc/Changelog.md b/doc/Changelog.md index dafc16d..92bec9e 100644 --- a/doc/Changelog.md +++ b/doc/Changelog.md @@ -1,8 +1,10 @@ Changelog {#Changelog} ========= -# Release 2.0.0 (24-May-2017) +# Release 2.0.0 (30-May-2017) +* [157](https://github.com/BlueBrain/Brion/pull/157): + Improve synapse attribute reading performance * [154](https://github.com/BlueBrain/Brion/pull/154): - Improved help and command line inverface for spikeConverter and compartmentConverter. From df34a49fc3a34513452d6928d37690c182564725 Mon Sep 17 00:00:00 2001 From: Daniel Nachbaur Date: Tue, 30 May 2017 17:52:37 +0200 Subject: [PATCH 2/2] Add Synapse.gid() python wrapping --- brain/python/synapses.cpp | 2 ++ doc/Changelog.md | 3 ++- tests/brain/python/synapses.py | 4 ++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/brain/python/synapses.cpp b/brain/python/synapses.cpp index 773bdc8..8b05506 100644 --- a/brain/python/synapses.cpp +++ b/brain/python/synapses.cpp @@ -95,6 +95,8 @@ const auto selfarg = bp::arg( "self" ); bp::class_< SynapseWrapper >( "Synapse", DOXY_CLASS( brain::Synapse ), bp::no_init ) + .def( "gid", &Synapse::getGID, ( selfarg ), + DOXY_FN( brain::Synapse::getGID )) .def( "pre_gid", &Synapse::getPresynapticGID, ( selfarg ), DOXY_FN( brain::Synapse::getPresynapticGID )) .def( "pre_section", &Synapse::getPresynapticSectionID, ( selfarg ), diff --git a/doc/Changelog.md b/doc/Changelog.md index 92bec9e..c9ca523 100644 --- a/doc/Changelog.md +++ b/doc/Changelog.md @@ -4,7 +4,8 @@ Changelog {#Changelog} # Release 2.0.0 (30-May-2017) * [157](https://github.com/BlueBrain/Brion/pull/157): - Improve synapse attribute reading performance + - Add Synapse.gid() python wrapping + - Improve synapse attribute reading performance * [154](https://github.com/BlueBrain/Brion/pull/154): - Improved help and command line inverface for spikeConverter and compartmentConverter. diff --git a/tests/brain/python/synapses.py b/tests/brain/python/synapses.py index 9ff1cf6..d215692 100644 --- a/tests/brain/python/synapses.py +++ b/tests/brain/python/synapses.py @@ -103,6 +103,10 @@ def setUp(self): self.synapse = self.synapses[0] def test_synapse(self): + # This data is not available in the test dataset, so we only check the + # function exists + self.assertRaises(RuntimeError, lambda: self.synapse.gid()) + pre_gid = self.synapse.pre_gid() pre_section_id = self.synapse.pre_section() pre_segment_id = self.synapse.pre_segment()