From 08cc9d28817bd7b05f8b064b5317738d4d05cc25 Mon Sep 17 00:00:00 2001 From: Stefan Eilemann Date: Fri, 23 Dec 2016 15:34:39 +0100 Subject: [PATCH 1/3] Implement loadNeuron() for map and bin reports --- apps/compartmentConverter.cpp | 13 ++++--- brion/compartmentReport.cpp | 13 +++++++ brion/compartmentReport.h | 17 ++++++-- brion/compartmentReportPlugin.h | 6 ++- brion/plugin/compartmentReportBinary.cpp | 63 +++++++++++++++++++++++++----- brion/plugin/compartmentReportBinary.h | 5 ++- brion/plugin/compartmentReportMap.cpp | 67 +++++++++++++++++++++++++++++++- brion/plugin/compartmentReportMap.h | 2 + tests/compartmentReport.cpp | 22 ++++++++++- 9 files changed, 185 insertions(+), 23 deletions(-) diff --git a/apps/compartmentConverter.cpp b/apps/compartmentConverter.cpp index f9ceba1..b3dd93c 100644 --- a/apps/compartmentConverter.cpp +++ b/apps/compartmentConverter.cpp @@ -131,21 +131,22 @@ int main( const int argc, char** argv ) brion::CompartmentReport in( inURI, brion::MODE_READ ); float loadTime = clock.getTimef(); + const float start = in.getStartTime(); + const float step = in.getTimestep(); + float end = in.getEndTime(); + if( vm.count( "dump" )) { std::cout << "Compartment report " << inURI << ":" << std::endl - << " Time: " << in.getStartTime() << ".." - << in.getEndTime() << " / " << in.getTimestep() - << " " << in.getTimeUnit() << std::endl + << " " << (end - start) / step << " frames: " + << start << ".." << end << " / " << step << " " + << in.getTimeUnit() << std::endl << " " << in.getGIDs().size() << " neurons" << std::endl << " " << in.getFrameSize() << " compartments" << std::endl; return EXIT_SUCCESS; } - const float start = in.getStartTime(); - const float step = in.getTimestep(); - float end = in.getEndTime(); const float maxEnd = start + maxFrames * step; end = std::min( end, maxEnd ); const brion::CompartmentCounts& counts = in.getCompartmentCounts(); diff --git a/brion/compartmentReport.cpp b/brion/compartmentReport.cpp index 9be9930..e596ff9 100644 --- a/brion/compartmentReport.cpp +++ b/brion/compartmentReport.cpp @@ -120,6 +120,19 @@ floatsPtr CompartmentReport::loadFrame( const float timestamp ) const return _impl->plugin->loadFrame( timestamp ); } +size_t CompartmentReport::getNeuronSize( const uint32_t gid ) const +{ + const auto& gids = getGIDs(); + const size_t index = std::distance( gids.begin(), gids.find( gid )); + const size_t nTimesteps = (getEndTime() - getStartTime( )) / getTimestep(); + return getNumCompartments( index ) * nTimesteps; +} + +floatsPtr CompartmentReport::loadNeuron( const uint32_t gid ) const +{ + return _impl->plugin->loadNeuron( gid ); +} + void CompartmentReport::setBufferSize( const size_t size ) { _impl->plugin->setBufferSize( size ); diff --git a/brion/compartmentReport.h b/brion/compartmentReport.h index 40076a6..0330ea6 100644 --- a/brion/compartmentReport.h +++ b/brion/compartmentReport.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, EPFL/Blue Brain Project +/* Copyright (c) 2013-2017, EPFL/Blue Brain Project * Daniel Nachbaur * * This file is part of Brion @@ -122,17 +122,28 @@ class CompartmentReport : public boost::noncopyable /** @return the time unit of the report. @version 1.0 */ BRION_API const std::string& getTimeUnit() const; - /** @return the size of a loaded report frame. @version 1.0 */ + /** @return the number of values of a loaded report frame. @version 1.0 */ BRION_API size_t getFrameSize() const; /** Load report values at the given time stamp. * * @param timestamp the time stamp of interest - * @return the report values if found at timestamp, 0 otherwise + * @return the report values if found at timestamp, nullptr otherwise * @version 1.0 */ BRION_API floatsPtr loadFrame( float timestamp ) const; + /** @return the number of values of the given neuron report. @version 1.0 */ + BRION_API size_t getNeuronSize( uint32_t gid ) const; + + /** Load report values for the given neuron. + * + * @param gid the cell identifier + * @return the report values if cell is found, nullptr otherwise + * @version 1.0 + */ + BRION_API floatsPtr loadNeuron( uint32_t gid ) const; + /** Set the size of the stream buffer for loaded frames. * * Configures the number of simulation frame buffers for stream readers. diff --git a/brion/compartmentReportPlugin.h b/brion/compartmentReportPlugin.h index 6dc0bfa..dfb2eed 100644 --- a/brion/compartmentReportPlugin.h +++ b/brion/compartmentReportPlugin.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2015, EPFL/Blue Brain Project +/* Copyright (c) 2013-2017, EPFL/Blue Brain Project * Daniel Nachbaur * * This file is part of Brion @@ -131,6 +131,10 @@ class CompartmentReportPlugin : public boost::noncopyable /** @copydoc brion::CompartmentReport::loadFrame */ virtual floatsPtr loadFrame( float timestamp ) const = 0; + /** @copydoc brion::CompartmentReport::loadNeuron */ + virtual floatsPtr loadNeuron( uint32_t ) const + { throw std::runtime_error( "loadNeuron() not implemented" ); } + /** @copydoc brion::CompartmentReport::updateMapping */ virtual void updateMapping( const GIDSet& gids ) = 0; diff --git a/brion/plugin/compartmentReportBinary.cpp b/brion/plugin/compartmentReportBinary.cpp index e3c80dd..0c24c55 100644 --- a/brion/plugin/compartmentReportBinary.cpp +++ b/brion/plugin/compartmentReportBinary.cpp @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2015, EPFL/Blue Brain Project +/* Copyright (c) 2013-2017, EPFL/Blue Brain Project * Daniel Nachbaur * * This file is part of Brion @@ -147,7 +147,7 @@ CompartmentReportBinary::CompartmentReportBinary( , _timestep( 0 ) , _file() , _header() - , _comps( 0 ) + , _subNumCompartments( 0 ) , _subtarget( false ) { if( initData.getAccessMode() != MODE_READ ) @@ -202,7 +202,7 @@ const CompartmentCounts& CompartmentReportBinary::getCompartmentCounts() const size_t CompartmentReportBinary::getFrameSize() const { - return _subtarget ? _comps : _header.numCompartments; + return _subtarget ? _subNumCompartments : _header.numCompartments; } floatsPtr CompartmentReportBinary::loadFrame( const float timestamp ) const @@ -212,7 +212,6 @@ floatsPtr CompartmentReportBinary::loadFrame( const float timestamp ) const return floatsPtr(); const size_t frameNumber = _getFrameNumber( timestamp ); - const size_t frameOffset = _header.dataBlockOffset + _header.numCompartments * sizeof(float) * frameNumber; @@ -231,10 +230,10 @@ floatsPtr CompartmentReportBinary::loadFrame( const float timestamp ) const return buffer; } - if( _comps == 0 ) + if( _subNumCompartments == 0 ) return floatsPtr(); - floatsPtr buffer( new floats( _comps )); + floatsPtr buffer( new floats( _subNumCompartments )); const float* const source = (const float*)(ptr + frameOffset); const SectionOffsets& offsets = getOffsets(); const CompartmentCounts& compCounts = getCompartmentCounts(); @@ -255,9 +254,55 @@ floatsPtr CompartmentReportBinary::loadFrame( const float timestamp ) const if( _header.byteswap ) { #pragma omp parallel for - for( ssize_t i = 0; i < ssize_t( _comps ); ++i ) + for( ssize_t i = 0; i < ssize_t( _subNumCompartments ); ++i ) + lunchbox::byteswap( (*buffer)[i] ); + } + return buffer; +} + +floatsPtr CompartmentReportBinary::loadNeuron( const uint32_t gid ) const +{ + const uint8_t* const bytePtr = _file.getAddress< const uint8_t >(); + if( !bytePtr || _offsets[_subtarget].empty( )) + return floatsPtr(); + + const size_t index = std::distance( _gids.begin(), _gids.find( gid )); + if( index >= _gids.size( )) + return floatsPtr(); + + const float* const ptr = (const float*)(bytePtr + _header.dataBlockOffset); + + const size_t frameSize = _header.numCompartments; + const size_t nFrames = (_endTime - _startTime) / _timestep; + const size_t nCompartments = getNumCompartments( index ); + const size_t nValues = nFrames * nCompartments; + floatsPtr buffer( new floats( nValues )); + + const SectionOffsets& offsets = _offsets[0]; + const CompartmentCounts& compCounts = getCompartmentCounts(); + for( size_t i = 0; i < nFrames; ++i ) + { + const size_t srcOffset = i * frameSize; + size_t dstOffset = i * nCompartments; + for( size_t j = 0; j < offsets[index].size(); ++j ) + { + const uint16_t numCompartments = compCounts[index][j]; + const uint64_t sourceOffset = offsets[index][j]; + + ::memcpy( buffer->data() + dstOffset, + ptr + srcOffset + sourceOffset, + numCompartments * sizeof( float )); + dstOffset += numCompartments; + } + } + + if( _header.byteswap ) + { +#pragma omp parallel for + for( ssize_t i = 0; i < ssize_t( nValues ); ++i ) lunchbox::byteswap( (*buffer)[i] ); } + return buffer; } @@ -292,7 +337,7 @@ void CompartmentReportBinary::updateMapping( const GIDSet& gids ) // then build conversion mapping from original to subtarget size_t cellIndex = 0; - _comps = 0; + _subNumCompartments = 0; for( GIDSetCIter i = _gids.begin(); i != _gids.end(); ++i ) { const size_t index = gidIndex[*i]; @@ -309,7 +354,7 @@ void CompartmentReportBinary::updateMapping( const GIDSet& gids ) const uint16_t compCount = _counts[0][index][sid]; sectionOffsets[sid] = _offsets[0][index][sid]; sectionCounts[sid] = compCount; - _comps += compCount; + _subNumCompartments += compCount; } ++cellIndex; } diff --git a/brion/plugin/compartmentReportBinary.h b/brion/plugin/compartmentReportBinary.h index b174a27..8fd0d53 100644 --- a/brion/plugin/compartmentReportBinary.h +++ b/brion/plugin/compartmentReportBinary.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2015, EPFL/Blue Brain Project +/* Copyright (c) 2013-2017, EPFL/Blue Brain Project * Daniel Nachbaur * * This file is part of Brion @@ -79,6 +79,7 @@ class CompartmentReportBinary : public CompartmentReportCommon size_t getFrameSize() const final; floatsPtr loadFrame( float timestamp ) const final; + floatsPtr loadNeuron( const uint32_t gid ) const final; void updateMapping( const GIDSet& gids ) final; void writeHeader( float startTime, float endTime, @@ -112,7 +113,7 @@ class CompartmentReportBinary : public CompartmentReportCommon SectionOffsets _conversionOffsets; - size_t _comps; // subtarget + size_t _subNumCompartments; // subtarget GIDSet _originalGIDs; bool _subtarget; diff --git a/brion/plugin/compartmentReportMap.cpp b/brion/plugin/compartmentReportMap.cpp index ff8a150..78366cf 100644 --- a/brion/plugin/compartmentReportMap.cpp +++ b/brion/plugin/compartmentReportMap.cpp @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2016, EPFL/Blue Brain Project +/* Copyright (c) 2014-2017, EPFL/Blue Brain Project * Stefan.Eilemann@epfl.ch * * This file is part of Brion @@ -435,5 +435,70 @@ floatsPtr CompartmentReportMap::loadFrame( const float time ) const return floatsPtr(); } +floatsPtr CompartmentReportMap::loadNeuron( const uint32_t gid ) const +{ + if( !_readable ) + return floatsPtr(); + + const size_t index = std::distance( _gids.begin(), _gids.find( gid )); + if( index >= _gids.size( )) + return floatsPtr(); + + const std::string& scope = _uri + std::to_string( gid ) + "_"; + const size_t nFrames = (_header.endTime - _header.startTime) / + _header.timestep; + const size_t nCompartments = getNumCompartments( index ); + + floatsPtr buffer( new floats( nFrames * nCompartments )); + float* const ptr = buffer->data(); + + std::unordered_map< std::string, size_t > offsetMap; + Strings keys; + keys.reserve( nFrames ); + + for( size_t i = 0; i < nFrames; ++i ) + { + keys.push_back( scope + std::to_string( i )); + offsetMap.emplace( keys.back(), i * nCompartments ); + } + +#ifdef BRION_USE_OPENMP + lunchbox::a_ssize_t taken; + omp_set_num_threads( _stores.size( )); +#else + size_t taken = 0; +#endif +#pragma omp parallel + { +#ifdef BRION_USE_OPENMP + auto& store = _stores[ omp_get_thread_num( )]; + const size_t start = float( omp_get_thread_num( )) * + float( keys.size( )) / float( omp_get_num_threads( )); + const size_t end = float( omp_get_thread_num() + 1 ) * + float( keys.size( )) / float( omp_get_num_threads( )); + const Strings subKeys( keys.begin() + start, keys.begin() + end ); +#else + const Strings& subKeys = keys; +#endif + + const auto takeValue = [ ptr, &offsetMap, &taken ] + ( const std::string& key, char* data, const size_t size ) + { + ::memcpy( ptr + offsetMap[ key ], data, size ); + std::free( data ); + ++taken; + }; + + store.takeValues( subKeys, takeValue ); + } + + if( size_t( taken ) == nFrames ) + return buffer; + + LBWARN << "Missing " << nFrames - taken << " of " << nFrames + << " time steps of gid " << gid << std::endl; + return floatsPtr(); +} + } } diff --git a/brion/plugin/compartmentReportMap.h b/brion/plugin/compartmentReportMap.h index 9efcb8a..fc4fed1 100644 --- a/brion/plugin/compartmentReportMap.h +++ b/brion/plugin/compartmentReportMap.h @@ -51,6 +51,8 @@ class CompartmentReportMap : public CompartmentReportCommon { return _counts; } floatsPtr loadFrame( float timestamp ) const final; + floatsPtr loadNeuron( uint32_t gid ) const final; + void updateMapping( const GIDSet& gids ) final; void writeHeader( float startTime, float endTime, diff --git a/tests/compartmentReport.cpp b/tests/compartmentReport.cpp index a2e9a93..373ea78 100644 --- a/tests/compartmentReport.cpp +++ b/tests/compartmentReport.cpp @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, EPFL/Blue Brain Project +/* Copyright (c) 2013-2017, EPFL/Blue Brain Project * Daniel Nachbaur * * This file is part of Brion @@ -221,6 +221,26 @@ void test_compare( const brion::URI& uri1, const brion::URI& uri2 ) } } + try + { + // cross-check second GID's voltage report + const uint32_t gid = *( ++report1.getGIDs().begin( )); + brion::floatsPtr frame1 = report1.loadNeuron( gid ); + brion::floatsPtr frame2 = report2.loadNeuron( gid ); + const size_t size = report1.getNeuronSize( gid ); + + BOOST_CHECK( frame1 ); + BOOST_CHECK( frame2 ); + BOOST_CHECK( size > 0 ); + BOOST_CHECK_EQUAL( size, report2.getNeuronSize( gid )); + + for( size_t i = 0; i < size; ++i ) + { + BOOST_CHECK_EQUAL( (*frame1)[i], (*frame2)[i] ); + } + } + catch( const std::runtime_error& ) {} // loadNeuron(gid) is optional, ignore + brion::GIDSet gids; gids.insert( 394 ); report1.updateMapping( gids ); From c213ec038069f5a3991c261c761e19cd973dd2d1 Mon Sep 17 00:00:00 2001 From: Stefan Eilemann Date: Tue, 3 Jan 2017 16:25:47 +0100 Subject: [PATCH 2/3] compartment converter: allow limiting output GIDs --- apps/compartmentConverter.cpp | 10 ++++++++++ brion/plugin/compartmentReportMap.cpp | 1 + doc/Changelog.md | 2 ++ 3 files changed, 13 insertions(+) diff --git a/apps/compartmentConverter.cpp b/apps/compartmentConverter.cpp index b3dd93c..bd38467 100644 --- a/apps/compartmentConverter.cpp +++ b/apps/compartmentConverter.cpp @@ -79,6 +79,8 @@ int main( const int argc, char** argv ) ( "output,o", po::value< std::string >(), "Output report URI" ) ( "maxFrames,m", po::value< size_t >(), "Convert at most the given number of frames" ) + ( "gids,g", po::value< std::vector< uint32_t >>()->multitoken(), + "List of whitespace-separated sub-GIDs to convert" ) ( "compare,c", "Compare written report with input" ) ( "dump,d", "Dump input report information (no output conversion)" ); po::variables_map vm; @@ -147,6 +149,14 @@ int main( const int argc, char** argv ) return EXIT_SUCCESS; } + if( vm.count( "gids" )) + { + brion::GIDSet gids; + for( const auto gid : vm["gids"].as< std::vector< uint32_t >>( )) + gids.emplace( gid ); + in.updateMapping( gids ); + } + const float maxEnd = start + maxFrames * step; end = std::min( end, maxEnd ); const brion::CompartmentCounts& counts = in.getCompartmentCounts(); diff --git a/brion/plugin/compartmentReportMap.cpp b/brion/plugin/compartmentReportMap.cpp index 78366cf..5560e9d 100644 --- a/brion/plugin/compartmentReportMap.cpp +++ b/brion/plugin/compartmentReportMap.cpp @@ -478,6 +478,7 @@ floatsPtr CompartmentReportMap::loadNeuron( const uint32_t gid ) const float( keys.size( )) / float( omp_get_num_threads( )); const Strings subKeys( keys.begin() + start, keys.begin() + end ); #else + auto& store = _stores.front(); const Strings& subKeys = keys; #endif diff --git a/doc/Changelog.md b/doc/Changelog.md index ff7f778..771df37 100644 --- a/doc/Changelog.md +++ b/doc/Changelog.md @@ -3,6 +3,8 @@ Changelog {#Changelog} # git master +* [122](https://github.com/BlueBrain/Brion/pull/122): + Add loadNeuron for binary and map compartment reports * [121](https://github.com/BlueBrain/Brion/pull/121): Performance optimizations for map and binary compartment reports, fix sub target loading of map compartment reports From 153a7ef8746f89ba5b39f437b8143549c0758db9 Mon Sep 17 00:00:00 2001 From: Stefan Eilemann Date: Thu, 5 Jan 2017 15:59:03 +0100 Subject: [PATCH 3/3] CR #122 --- apps/compartmentConverter.cpp | 2 +- brion/compartmentReport.cpp | 10 ++-- brion/compartmentReport.h | 20 ++++++-- brion/compartmentReportPlugin.h | 33 ++++++++------ brion/plugin/compartmentReportBinary.cpp | 9 ++-- brion/plugin/compartmentReportBinary.h | 2 +- brion/plugin/compartmentReportMap.cpp | 78 +++++++++++--------------------- brion/plugin/compartmentReportMap.h | 4 ++ doc/Changelog.md | 2 +- tests/compartmentReport.cpp | 1 + 10 files changed, 78 insertions(+), 83 deletions(-) diff --git a/apps/compartmentConverter.cpp b/apps/compartmentConverter.cpp index bd38467..c877c96 100644 --- a/apps/compartmentConverter.cpp +++ b/apps/compartmentConverter.cpp @@ -80,7 +80,7 @@ int main( const int argc, char** argv ) ( "maxFrames,m", po::value< size_t >(), "Convert at most the given number of frames" ) ( "gids,g", po::value< std::vector< uint32_t >>()->multitoken(), - "List of whitespace-separated sub-GIDs to convert" ) + "List of whitespace separated GIDs to convert" ) ( "compare,c", "Compare written report with input" ) ( "dump,d", "Dump input report information (no output conversion)" ); po::variables_map vm; diff --git a/brion/compartmentReport.cpp b/brion/compartmentReport.cpp index e596ff9..3a0ea54 100644 --- a/brion/compartmentReport.cpp +++ b/brion/compartmentReport.cpp @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, EPFL/Blue Brain Project +/* Copyright (c) 2013-2017, EPFL/Blue Brain Project * Daniel Nachbaur * * This file is part of Brion @@ -62,6 +62,11 @@ const GIDSet& CompartmentReport::getGIDs() const return _impl->plugin->getGIDs(); } +size_t CompartmentReport::getIndex( const uint32_t gid ) const +{ + return _impl->plugin->getIndex( gid ); +} + const SectionOffsets& CompartmentReport::getOffsets() const { return _impl->plugin->getOffsets(); @@ -122,8 +127,7 @@ floatsPtr CompartmentReport::loadFrame( const float timestamp ) const size_t CompartmentReport::getNeuronSize( const uint32_t gid ) const { - const auto& gids = getGIDs(); - const size_t index = std::distance( gids.begin(), gids.find( gid )); + const size_t index = getIndex( gid ); const size_t nTimesteps = (getEndTime() - getStartTime( )) / getTimestep(); return getNumCompartments( index ) * nTimesteps; } diff --git a/brion/compartmentReport.h b/brion/compartmentReport.h index 0330ea6..1fa7d2a 100644 --- a/brion/compartmentReport.h +++ b/brion/compartmentReport.h @@ -73,6 +73,13 @@ class CompartmentReport : public boost::noncopyable /** @return the current considered GIDs. @version 1.0 */ BRION_API const GIDSet& getGIDs() const; + /** + * @return the index of the given gid. + * @version 1.10 + * @throw std::runtime_error if the gid is not mapped + */ + BRION_API size_t getIndex( const uint32_t gid ) const; + /** Get the current mapping of each section of each neuron in each * simulation frame buffer. * @@ -133,14 +140,19 @@ class CompartmentReport : public boost::noncopyable */ BRION_API floatsPtr loadFrame( float timestamp ) const; - /** @return the number of values of the given neuron report. @version 1.0 */ + /** + * @param gid the neuron report to be loaded. + * @return the number of values of the given neuron report. + * @throw std::runtime_error if gid is not mapped. + * @version 1.10 + */ BRION_API size_t getNeuronSize( uint32_t gid ) const; /** Load report values for the given neuron. * - * @param gid the cell identifier - * @return the report values if cell is found, nullptr otherwise - * @version 1.0 + * @param gid the neuron identifier + * @return the report values if neuron is found, nullptr otherwise + * @version 1.10 */ BRION_API floatsPtr loadNeuron( uint32_t gid ) const; diff --git a/brion/compartmentReportPlugin.h b/brion/compartmentReportPlugin.h index dfb2eed..0065aa0 100644 --- a/brion/compartmentReportPlugin.h +++ b/brion/compartmentReportPlugin.h @@ -24,6 +24,7 @@ #include #include +#include #include namespace brion @@ -52,11 +53,7 @@ class CompartmentReportInitData : public PluginInitData , _gids( gids ) {} - /** - * - * @return Returns the gids. - * @version 1.4 - */ + /** @return Returns the gids. @version 1.4 */ const GIDSet& getGids() const { return _gids; } private: @@ -140,10 +137,7 @@ class CompartmentReportPlugin : public boost::noncopyable /** @copydoc brion::CompartmentReport::setBufferSize */ virtual void setBufferSize( size_t size ) - { - // To keep doxygen happy - (void)size; - } + { /* To keep doxygen happy */ (void)size; } /** @copydoc brion::CompartmentReport::clearBuffer */ virtual void clearBuffer() {} @@ -153,21 +147,30 @@ class CompartmentReportPlugin : public boost::noncopyable /** @copydoc brion::CompartmentReport::writeHeader */ virtual void writeHeader( float startTime, float endTime, - float timestep, - const std::string& dunit, - const std::string& tunit ) = 0; + float timestep, const std::string& dunit, + const std::string& tunit ) = 0; /** @copydoc brion::CompartmentReport::writeCompartments */ - virtual bool writeCompartments( uint32_t gid, - const uint16_ts& counts ) = 0; + virtual bool writeCompartments( uint32_t gid, const uint16_ts& counts ) = 0; /** @copydoc brion::CompartmentReport::writeFrame */ virtual bool writeFrame( uint32_t gid, const floats& voltages, - float timestamp ) = 0; + float timestamp ) = 0; /** @copydoc brion::CompartmentReport::flush */ virtual bool flush() = 0; //@} + + /** @copydoc brion::CompartmentReport::getIndex */ + size_t getIndex( const uint32_t gid ) const + { + const auto& gids = getGIDs(); + const size_t index = std::distance( gids.begin(), gids.find( gid )); + if( index >= gids.size( )) + LBTHROW( std::runtime_error( "Gid " + std::to_string( gid ) + + " not in report" )); + return index; + } }; } diff --git a/brion/plugin/compartmentReportBinary.cpp b/brion/plugin/compartmentReportBinary.cpp index 0c24c55..5df2406 100644 --- a/brion/plugin/compartmentReportBinary.cpp +++ b/brion/plugin/compartmentReportBinary.cpp @@ -266,10 +266,7 @@ floatsPtr CompartmentReportBinary::loadNeuron( const uint32_t gid ) const if( !bytePtr || _offsets[_subtarget].empty( )) return floatsPtr(); - const size_t index = std::distance( _gids.begin(), _gids.find( gid )); - if( index >= _gids.size( )) - return floatsPtr(); - + const size_t index = getIndex( gid ); const float* const ptr = (const float*)(bytePtr + _header.dataBlockOffset); const size_t frameSize = _header.numCompartments; @@ -282,7 +279,7 @@ floatsPtr CompartmentReportBinary::loadNeuron( const uint32_t gid ) const const CompartmentCounts& compCounts = getCompartmentCounts(); for( size_t i = 0; i < nFrames; ++i ) { - const size_t srcOffset = i * frameSize; + const size_t frameOffset = i * frameSize; size_t dstOffset = i * nCompartments; for( size_t j = 0; j < offsets[index].size(); ++j ) { @@ -290,7 +287,7 @@ floatsPtr CompartmentReportBinary::loadNeuron( const uint32_t gid ) const const uint64_t sourceOffset = offsets[index][j]; ::memcpy( buffer->data() + dstOffset, - ptr + srcOffset + sourceOffset, + ptr + frameOffset + sourceOffset, numCompartments * sizeof( float )); dstOffset += numCompartments; } diff --git a/brion/plugin/compartmentReportBinary.h b/brion/plugin/compartmentReportBinary.h index 8fd0d53..282c77c 100644 --- a/brion/plugin/compartmentReportBinary.h +++ b/brion/plugin/compartmentReportBinary.h @@ -113,7 +113,7 @@ class CompartmentReportBinary : public CompartmentReportCommon SectionOffsets _conversionOffsets; - size_t _subNumCompartments; // subtarget + size_t _subNumCompartments; GIDSet _originalGIDs; bool _subtarget; diff --git a/brion/plugin/compartmentReportMap.cpp b/brion/plugin/compartmentReportMap.cpp index 5560e9d..6a05eda 100644 --- a/brion/plugin/compartmentReportMap.cpp +++ b/brion/plugin/compartmentReportMap.cpp @@ -372,7 +372,7 @@ floatsPtr CompartmentReportMap::loadFrame( const float time ) const const std::string index = std::string( "_" ) + std::to_string( _getFrameNumber( time )); - std::unordered_map< std::string, size_t > offsetMap; + OffsetMap offsetMap; size_t offset = 0; Strings keys; @@ -394,44 +394,8 @@ floatsPtr CompartmentReportMap::loadFrame( const float time ) const } floatsPtr buffer( new floats( getFrameSize( ))); - float* const ptr = buffer->data(); - -#ifdef BRION_USE_OPENMP - lunchbox::a_ssize_t taken; - omp_set_num_threads( _stores.size( )); -#else - size_t taken = 0; -#endif -#pragma omp parallel - { -#ifdef BRION_USE_OPENMP - auto& store = _stores[ omp_get_thread_num( )]; - const size_t start = float( omp_get_thread_num( )) * - float( keys.size( )) / float( omp_get_num_threads( )); - const size_t end = float( omp_get_thread_num() + 1 ) * - float( keys.size( )) / float( omp_get_num_threads( )); - const Strings subKeys( keys.begin() + start, keys.begin() + end ); -#else - auto& store = _stores.front(); - const Strings& subKeys = keys; -#endif - - const auto takeValue = [ ptr, &offsetMap, &taken ] - ( const std::string& key, char* data, const size_t size ) - { - ::memcpy( ptr + offsetMap[ key ], data, size ); - std::free( data ); - ++taken; - }; - - store.takeValues( subKeys, takeValue ); - } - - if( size_t( taken ) == _gids.size( )) + if( _load( buffer, keys, offsetMap )) return buffer; - - LBWARN << "Missing " << _gids.size() - taken << " of " << _gids.size() - << " gids in report frame at " << time << " ms" << std::endl; return floatsPtr(); } @@ -440,20 +404,16 @@ floatsPtr CompartmentReportMap::loadNeuron( const uint32_t gid ) const if( !_readable ) return floatsPtr(); - const size_t index = std::distance( _gids.begin(), _gids.find( gid )); - if( index >= _gids.size( )) - return floatsPtr(); - + const size_t index = getIndex( gid ); const std::string& scope = _uri + std::to_string( gid ) + "_"; const size_t nFrames = (_header.endTime - _header.startTime) / _header.timestep; const size_t nCompartments = getNumCompartments( index ); floatsPtr buffer( new floats( nFrames * nCompartments )); - float* const ptr = buffer->data(); - std::unordered_map< std::string, size_t > offsetMap; Strings keys; + OffsetMap offsetMap; keys.reserve( nFrames ); for( size_t i = 0; i < nFrames; ++i ) @@ -462,6 +422,16 @@ floatsPtr CompartmentReportMap::loadNeuron( const uint32_t gid ) const offsetMap.emplace( keys.back(), i * nCompartments ); } + if( _load( buffer, keys, offsetMap )) + return buffer; + return floatsPtr(); +} + +bool CompartmentReportMap::_load( floatsPtr buffer, const Strings& keys, + const OffsetMap& offsets ) const +{ + float* const ptr = buffer->data(); + #ifdef BRION_USE_OPENMP lunchbox::a_ssize_t taken; omp_set_num_threads( _stores.size( )); @@ -482,23 +452,27 @@ floatsPtr CompartmentReportMap::loadNeuron( const uint32_t gid ) const const Strings& subKeys = keys; #endif - const auto takeValue = [ ptr, &offsetMap, &taken ] + const auto takeValue = [ ptr, &offsets, &taken ] ( const std::string& key, char* data, const size_t size ) { - ::memcpy( ptr + offsetMap[ key ], data, size ); + const auto i = offsets.find( key ); + if( i != offsets.end( )) + { + ::memcpy( ptr + i->second, data, size ); + ++taken; + } std::free( data ); - ++taken; }; store.takeValues( subKeys, takeValue ); } - if( size_t( taken ) == nFrames ) - return buffer; + if( size_t( taken ) == keys.size( )) + return true; - LBWARN << "Missing " << nFrames - taken << " of " << nFrames - << " time steps of gid " << gid << std::endl; - return floatsPtr(); + LBWARN << "Missing " << keys.size() - taken << " of " << keys.size() + << " values in report frame" << std::endl; + return false; } } diff --git a/brion/plugin/compartmentReportMap.h b/brion/plugin/compartmentReportMap.h index fc4fed1..8b0a1dc 100644 --- a/brion/plugin/compartmentReportMap.h +++ b/brion/plugin/compartmentReportMap.h @@ -106,9 +106,13 @@ class CompartmentReportMap : public CompartmentReportCommon bool _readable; + using OffsetMap = std::unordered_map< std::string, size_t >; + void _clear(); bool _loadHeader(); bool _flushHeader(); + bool _load( floatsPtr buffer, const Strings& keys, + const OffsetMap& offsets) const; }; } diff --git a/doc/Changelog.md b/doc/Changelog.md index 771df37..491afd9 100644 --- a/doc/Changelog.md +++ b/doc/Changelog.md @@ -4,7 +4,7 @@ Changelog {#Changelog} # git master * [122](https://github.com/BlueBrain/Brion/pull/122): - Add loadNeuron for binary and map compartment reports + Support loading of individual neurons for binary and map compartment reports * [121](https://github.com/BlueBrain/Brion/pull/121): Performance optimizations for map and binary compartment reports, fix sub target loading of map compartment reports diff --git a/tests/compartmentReport.cpp b/tests/compartmentReport.cpp index 373ea78..f2437c7 100644 --- a/tests/compartmentReport.cpp +++ b/tests/compartmentReport.cpp @@ -232,6 +232,7 @@ void test_compare( const brion::URI& uri1, const brion::URI& uri2 ) BOOST_CHECK( frame1 ); BOOST_CHECK( frame2 ); BOOST_CHECK( size > 0 ); + BOOST_CHECK_CLOSE( (*frame1)[2017], -65.1365891f, 0.000001f ); BOOST_CHECK_EQUAL( size, report2.getNeuronSize( gid )); for( size_t i = 0; i < size; ++i )