diff --git a/apps/compartmentConverter.cpp b/apps/compartmentConverter.cpp index f9ceba1..c877c96 100644 --- a/apps/compartmentConverter.cpp +++ b/apps/compartmentConverter.cpp @@ -79,6 +79,8 @@ int main( const int argc, char** argv ) ( "output,o", po::value< std::string >(), "Output report URI" ) ( "maxFrames,m", po::value< size_t >(), "Convert at most the given number of frames" ) + ( "gids,g", po::value< std::vector< uint32_t >>()->multitoken(), + "List of whitespace separated GIDs to convert" ) ( "compare,c", "Compare written report with input" ) ( "dump,d", "Dump input report information (no output conversion)" ); po::variables_map vm; @@ -131,21 +133,30 @@ int main( const int argc, char** argv ) brion::CompartmentReport in( inURI, brion::MODE_READ ); float loadTime = clock.getTimef(); + const float start = in.getStartTime(); + const float step = in.getTimestep(); + float end = in.getEndTime(); + if( vm.count( "dump" )) { std::cout << "Compartment report " << inURI << ":" << std::endl - << " Time: " << in.getStartTime() << ".." - << in.getEndTime() << " / " << in.getTimestep() - << " " << in.getTimeUnit() << std::endl + << " " << (end - start) / step << " frames: " + << start << ".." << end << " / " << step << " " + << in.getTimeUnit() << std::endl << " " << in.getGIDs().size() << " neurons" << std::endl << " " << in.getFrameSize() << " compartments" << std::endl; return EXIT_SUCCESS; } - const float start = in.getStartTime(); - const float step = in.getTimestep(); - float end = in.getEndTime(); + if( vm.count( "gids" )) + { + brion::GIDSet gids; + for( const auto gid : vm["gids"].as< std::vector< uint32_t >>( )) + gids.emplace( gid ); + in.updateMapping( gids ); + } + const float maxEnd = start + maxFrames * step; end = std::min( end, maxEnd ); const brion::CompartmentCounts& counts = in.getCompartmentCounts(); diff --git a/brion/compartmentReport.cpp b/brion/compartmentReport.cpp index 9be9930..3a0ea54 100644 --- a/brion/compartmentReport.cpp +++ b/brion/compartmentReport.cpp @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, EPFL/Blue Brain Project +/* Copyright (c) 2013-2017, EPFL/Blue Brain Project * Daniel Nachbaur * * This file is part of Brion @@ -62,6 +62,11 @@ const GIDSet& CompartmentReport::getGIDs() const return _impl->plugin->getGIDs(); } +size_t CompartmentReport::getIndex( const uint32_t gid ) const +{ + return _impl->plugin->getIndex( gid ); +} + const SectionOffsets& CompartmentReport::getOffsets() const { return _impl->plugin->getOffsets(); @@ -120,6 +125,18 @@ floatsPtr CompartmentReport::loadFrame( const float timestamp ) const return _impl->plugin->loadFrame( timestamp ); } +size_t CompartmentReport::getNeuronSize( const uint32_t gid ) const +{ + const size_t index = getIndex( gid ); + const size_t nTimesteps = (getEndTime() - getStartTime( )) / getTimestep(); + return getNumCompartments( index ) * nTimesteps; +} + +floatsPtr CompartmentReport::loadNeuron( const uint32_t gid ) const +{ + return _impl->plugin->loadNeuron( gid ); +} + void CompartmentReport::setBufferSize( const size_t size ) { _impl->plugin->setBufferSize( size ); diff --git a/brion/compartmentReport.h b/brion/compartmentReport.h index 40076a6..1fa7d2a 100644 --- a/brion/compartmentReport.h +++ b/brion/compartmentReport.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, EPFL/Blue Brain Project +/* Copyright (c) 2013-2017, EPFL/Blue Brain Project * Daniel Nachbaur * * This file is part of Brion @@ -73,6 +73,13 @@ class CompartmentReport : public boost::noncopyable /** @return the current considered GIDs. @version 1.0 */ BRION_API const GIDSet& getGIDs() const; + /** + * @return the index of the given gid. + * @version 1.10 + * @throw std::runtime_error if the gid is not mapped + */ + BRION_API size_t getIndex( const uint32_t gid ) const; + /** Get the current mapping of each section of each neuron in each * simulation frame buffer. * @@ -122,17 +129,33 @@ class CompartmentReport : public boost::noncopyable /** @return the time unit of the report. @version 1.0 */ BRION_API const std::string& getTimeUnit() const; - /** @return the size of a loaded report frame. @version 1.0 */ + /** @return the number of values of a loaded report frame. @version 1.0 */ BRION_API size_t getFrameSize() const; /** Load report values at the given time stamp. * * @param timestamp the time stamp of interest - * @return the report values if found at timestamp, 0 otherwise + * @return the report values if found at timestamp, nullptr otherwise * @version 1.0 */ BRION_API floatsPtr loadFrame( float timestamp ) const; + /** + * @param gid the neuron report to be loaded. + * @return the number of values of the given neuron report. + * @throw std::runtime_error if gid is not mapped. + * @version 1.10 + */ + BRION_API size_t getNeuronSize( uint32_t gid ) const; + + /** Load report values for the given neuron. + * + * @param gid the neuron identifier + * @return the report values if neuron is found, nullptr otherwise + * @version 1.10 + */ + BRION_API floatsPtr loadNeuron( uint32_t gid ) const; + /** Set the size of the stream buffer for loaded frames. * * Configures the number of simulation frame buffers for stream readers. diff --git a/brion/compartmentReportPlugin.h b/brion/compartmentReportPlugin.h index 6dc0bfa..0065aa0 100644 --- a/brion/compartmentReportPlugin.h +++ b/brion/compartmentReportPlugin.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2015, EPFL/Blue Brain Project +/* Copyright (c) 2013-2017, EPFL/Blue Brain Project * Daniel Nachbaur * * This file is part of Brion @@ -24,6 +24,7 @@ #include #include +#include #include namespace brion @@ -52,11 +53,7 @@ class CompartmentReportInitData : public PluginInitData , _gids( gids ) {} - /** - * - * @return Returns the gids. - * @version 1.4 - */ + /** @return Returns the gids. @version 1.4 */ const GIDSet& getGids() const { return _gids; } private: @@ -131,15 +128,16 @@ class CompartmentReportPlugin : public boost::noncopyable /** @copydoc brion::CompartmentReport::loadFrame */ virtual floatsPtr loadFrame( float timestamp ) const = 0; + /** @copydoc brion::CompartmentReport::loadNeuron */ + virtual floatsPtr loadNeuron( uint32_t ) const + { throw std::runtime_error( "loadNeuron() not implemented" ); } + /** @copydoc brion::CompartmentReport::updateMapping */ virtual void updateMapping( const GIDSet& gids ) = 0; /** @copydoc brion::CompartmentReport::setBufferSize */ virtual void setBufferSize( size_t size ) - { - // To keep doxygen happy - (void)size; - } + { /* To keep doxygen happy */ (void)size; } /** @copydoc brion::CompartmentReport::clearBuffer */ virtual void clearBuffer() {} @@ -149,21 +147,30 @@ class CompartmentReportPlugin : public boost::noncopyable /** @copydoc brion::CompartmentReport::writeHeader */ virtual void writeHeader( float startTime, float endTime, - float timestep, - const std::string& dunit, - const std::string& tunit ) = 0; + float timestep, const std::string& dunit, + const std::string& tunit ) = 0; /** @copydoc brion::CompartmentReport::writeCompartments */ - virtual bool writeCompartments( uint32_t gid, - const uint16_ts& counts ) = 0; + virtual bool writeCompartments( uint32_t gid, const uint16_ts& counts ) = 0; /** @copydoc brion::CompartmentReport::writeFrame */ virtual bool writeFrame( uint32_t gid, const floats& voltages, - float timestamp ) = 0; + float timestamp ) = 0; /** @copydoc brion::CompartmentReport::flush */ virtual bool flush() = 0; //@} + + /** @copydoc brion::CompartmentReport::getIndex */ + size_t getIndex( const uint32_t gid ) const + { + const auto& gids = getGIDs(); + const size_t index = std::distance( gids.begin(), gids.find( gid )); + if( index >= gids.size( )) + LBTHROW( std::runtime_error( "Gid " + std::to_string( gid ) + + " not in report" )); + return index; + } }; } diff --git a/brion/plugin/compartmentReportBinary.cpp b/brion/plugin/compartmentReportBinary.cpp index e3c80dd..5df2406 100644 --- a/brion/plugin/compartmentReportBinary.cpp +++ b/brion/plugin/compartmentReportBinary.cpp @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2015, EPFL/Blue Brain Project +/* Copyright (c) 2013-2017, EPFL/Blue Brain Project * Daniel Nachbaur * * This file is part of Brion @@ -147,7 +147,7 @@ CompartmentReportBinary::CompartmentReportBinary( , _timestep( 0 ) , _file() , _header() - , _comps( 0 ) + , _subNumCompartments( 0 ) , _subtarget( false ) { if( initData.getAccessMode() != MODE_READ ) @@ -202,7 +202,7 @@ const CompartmentCounts& CompartmentReportBinary::getCompartmentCounts() const size_t CompartmentReportBinary::getFrameSize() const { - return _subtarget ? _comps : _header.numCompartments; + return _subtarget ? _subNumCompartments : _header.numCompartments; } floatsPtr CompartmentReportBinary::loadFrame( const float timestamp ) const @@ -212,7 +212,6 @@ floatsPtr CompartmentReportBinary::loadFrame( const float timestamp ) const return floatsPtr(); const size_t frameNumber = _getFrameNumber( timestamp ); - const size_t frameOffset = _header.dataBlockOffset + _header.numCompartments * sizeof(float) * frameNumber; @@ -231,10 +230,10 @@ floatsPtr CompartmentReportBinary::loadFrame( const float timestamp ) const return buffer; } - if( _comps == 0 ) + if( _subNumCompartments == 0 ) return floatsPtr(); - floatsPtr buffer( new floats( _comps )); + floatsPtr buffer( new floats( _subNumCompartments )); const float* const source = (const float*)(ptr + frameOffset); const SectionOffsets& offsets = getOffsets(); const CompartmentCounts& compCounts = getCompartmentCounts(); @@ -255,9 +254,52 @@ floatsPtr CompartmentReportBinary::loadFrame( const float timestamp ) const if( _header.byteswap ) { #pragma omp parallel for - for( ssize_t i = 0; i < ssize_t( _comps ); ++i ) + for( ssize_t i = 0; i < ssize_t( _subNumCompartments ); ++i ) + lunchbox::byteswap( (*buffer)[i] ); + } + return buffer; +} + +floatsPtr CompartmentReportBinary::loadNeuron( const uint32_t gid ) const +{ + const uint8_t* const bytePtr = _file.getAddress< const uint8_t >(); + if( !bytePtr || _offsets[_subtarget].empty( )) + return floatsPtr(); + + const size_t index = getIndex( gid ); + const float* const ptr = (const float*)(bytePtr + _header.dataBlockOffset); + + const size_t frameSize = _header.numCompartments; + const size_t nFrames = (_endTime - _startTime) / _timestep; + const size_t nCompartments = getNumCompartments( index ); + const size_t nValues = nFrames * nCompartments; + floatsPtr buffer( new floats( nValues )); + + const SectionOffsets& offsets = _offsets[0]; + const CompartmentCounts& compCounts = getCompartmentCounts(); + for( size_t i = 0; i < nFrames; ++i ) + { + const size_t frameOffset = i * frameSize; + size_t dstOffset = i * nCompartments; + for( size_t j = 0; j < offsets[index].size(); ++j ) + { + const uint16_t numCompartments = compCounts[index][j]; + const uint64_t sourceOffset = offsets[index][j]; + + ::memcpy( buffer->data() + dstOffset, + ptr + frameOffset + sourceOffset, + numCompartments * sizeof( float )); + dstOffset += numCompartments; + } + } + + if( _header.byteswap ) + { +#pragma omp parallel for + for( ssize_t i = 0; i < ssize_t( nValues ); ++i ) lunchbox::byteswap( (*buffer)[i] ); } + return buffer; } @@ -292,7 +334,7 @@ void CompartmentReportBinary::updateMapping( const GIDSet& gids ) // then build conversion mapping from original to subtarget size_t cellIndex = 0; - _comps = 0; + _subNumCompartments = 0; for( GIDSetCIter i = _gids.begin(); i != _gids.end(); ++i ) { const size_t index = gidIndex[*i]; @@ -309,7 +351,7 @@ void CompartmentReportBinary::updateMapping( const GIDSet& gids ) const uint16_t compCount = _counts[0][index][sid]; sectionOffsets[sid] = _offsets[0][index][sid]; sectionCounts[sid] = compCount; - _comps += compCount; + _subNumCompartments += compCount; } ++cellIndex; } diff --git a/brion/plugin/compartmentReportBinary.h b/brion/plugin/compartmentReportBinary.h index b174a27..282c77c 100644 --- a/brion/plugin/compartmentReportBinary.h +++ b/brion/plugin/compartmentReportBinary.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2015, EPFL/Blue Brain Project +/* Copyright (c) 2013-2017, EPFL/Blue Brain Project * Daniel Nachbaur * * This file is part of Brion @@ -79,6 +79,7 @@ class CompartmentReportBinary : public CompartmentReportCommon size_t getFrameSize() const final; floatsPtr loadFrame( float timestamp ) const final; + floatsPtr loadNeuron( const uint32_t gid ) const final; void updateMapping( const GIDSet& gids ) final; void writeHeader( float startTime, float endTime, @@ -112,7 +113,7 @@ class CompartmentReportBinary : public CompartmentReportCommon SectionOffsets _conversionOffsets; - size_t _comps; // subtarget + size_t _subNumCompartments; GIDSet _originalGIDs; bool _subtarget; diff --git a/brion/plugin/compartmentReportMap.cpp b/brion/plugin/compartmentReportMap.cpp index ff8a150..6a05eda 100644 --- a/brion/plugin/compartmentReportMap.cpp +++ b/brion/plugin/compartmentReportMap.cpp @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2016, EPFL/Blue Brain Project +/* Copyright (c) 2014-2017, EPFL/Blue Brain Project * Stefan.Eilemann@epfl.ch * * This file is part of Brion @@ -372,7 +372,7 @@ floatsPtr CompartmentReportMap::loadFrame( const float time ) const const std::string index = std::string( "_" ) + std::to_string( _getFrameNumber( time )); - std::unordered_map< std::string, size_t > offsetMap; + OffsetMap offsetMap; size_t offset = 0; Strings keys; @@ -394,6 +394,42 @@ floatsPtr CompartmentReportMap::loadFrame( const float time ) const } floatsPtr buffer( new floats( getFrameSize( ))); + if( _load( buffer, keys, offsetMap )) + return buffer; + return floatsPtr(); +} + +floatsPtr CompartmentReportMap::loadNeuron( const uint32_t gid ) const +{ + if( !_readable ) + return floatsPtr(); + + const size_t index = getIndex( gid ); + const std::string& scope = _uri + std::to_string( gid ) + "_"; + const size_t nFrames = (_header.endTime - _header.startTime) / + _header.timestep; + const size_t nCompartments = getNumCompartments( index ); + + floatsPtr buffer( new floats( nFrames * nCompartments )); + + Strings keys; + OffsetMap offsetMap; + keys.reserve( nFrames ); + + for( size_t i = 0; i < nFrames; ++i ) + { + keys.push_back( scope + std::to_string( i )); + offsetMap.emplace( keys.back(), i * nCompartments ); + } + + if( _load( buffer, keys, offsetMap )) + return buffer; + return floatsPtr(); +} + +bool CompartmentReportMap::_load( floatsPtr buffer, const Strings& keys, + const OffsetMap& offsets ) const +{ float* const ptr = buffer->data(); #ifdef BRION_USE_OPENMP @@ -416,23 +452,27 @@ floatsPtr CompartmentReportMap::loadFrame( const float time ) const const Strings& subKeys = keys; #endif - const auto takeValue = [ ptr, &offsetMap, &taken ] + const auto takeValue = [ ptr, &offsets, &taken ] ( const std::string& key, char* data, const size_t size ) { - ::memcpy( ptr + offsetMap[ key ], data, size ); + const auto i = offsets.find( key ); + if( i != offsets.end( )) + { + ::memcpy( ptr + i->second, data, size ); + ++taken; + } std::free( data ); - ++taken; }; store.takeValues( subKeys, takeValue ); } - if( size_t( taken ) == _gids.size( )) - return buffer; + if( size_t( taken ) == keys.size( )) + return true; - LBWARN << "Missing " << _gids.size() - taken << " of " << _gids.size() - << " gids in report frame at " << time << " ms" << std::endl; - return floatsPtr(); + LBWARN << "Missing " << keys.size() - taken << " of " << keys.size() + << " values in report frame" << std::endl; + return false; } } diff --git a/brion/plugin/compartmentReportMap.h b/brion/plugin/compartmentReportMap.h index 9efcb8a..8b0a1dc 100644 --- a/brion/plugin/compartmentReportMap.h +++ b/brion/plugin/compartmentReportMap.h @@ -51,6 +51,8 @@ class CompartmentReportMap : public CompartmentReportCommon { return _counts; } floatsPtr loadFrame( float timestamp ) const final; + floatsPtr loadNeuron( uint32_t gid ) const final; + void updateMapping( const GIDSet& gids ) final; void writeHeader( float startTime, float endTime, @@ -104,9 +106,13 @@ class CompartmentReportMap : public CompartmentReportCommon bool _readable; + using OffsetMap = std::unordered_map< std::string, size_t >; + void _clear(); bool _loadHeader(); bool _flushHeader(); + bool _load( floatsPtr buffer, const Strings& keys, + const OffsetMap& offsets) const; }; } diff --git a/doc/Changelog.md b/doc/Changelog.md index ff7f778..491afd9 100644 --- a/doc/Changelog.md +++ b/doc/Changelog.md @@ -3,6 +3,8 @@ Changelog {#Changelog} # git master +* [122](https://github.com/BlueBrain/Brion/pull/122): + Support loading of individual neurons for binary and map compartment reports * [121](https://github.com/BlueBrain/Brion/pull/121): Performance optimizations for map and binary compartment reports, fix sub target loading of map compartment reports diff --git a/tests/compartmentReport.cpp b/tests/compartmentReport.cpp index a2e9a93..f2437c7 100644 --- a/tests/compartmentReport.cpp +++ b/tests/compartmentReport.cpp @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, EPFL/Blue Brain Project +/* Copyright (c) 2013-2017, EPFL/Blue Brain Project * Daniel Nachbaur * * This file is part of Brion @@ -221,6 +221,27 @@ void test_compare( const brion::URI& uri1, const brion::URI& uri2 ) } } + try + { + // cross-check second GID's voltage report + const uint32_t gid = *( ++report1.getGIDs().begin( )); + brion::floatsPtr frame1 = report1.loadNeuron( gid ); + brion::floatsPtr frame2 = report2.loadNeuron( gid ); + const size_t size = report1.getNeuronSize( gid ); + + BOOST_CHECK( frame1 ); + BOOST_CHECK( frame2 ); + BOOST_CHECK( size > 0 ); + BOOST_CHECK_CLOSE( (*frame1)[2017], -65.1365891f, 0.000001f ); + BOOST_CHECK_EQUAL( size, report2.getNeuronSize( gid )); + + for( size_t i = 0; i < size; ++i ) + { + BOOST_CHECK_EQUAL( (*frame1)[i], (*frame2)[i] ); + } + } + catch( const std::runtime_error& ) {} // loadNeuron(gid) is optional, ignore + brion::GIDSet gids; gids.insert( 394 ); report1.updateMapping( gids );