diff --git a/.gitsubprojects b/.gitsubprojects index 9f23d9f..44accfb 100644 --- a/.gitsubprojects +++ b/.gitsubprojects @@ -1,6 +1,6 @@ # -*- mode: cmake -*- -git_subproject(Servus https://github.com/HBPVIS/Servus.git 001588d) -git_subproject(Lunchbox https://github.com/Eyescale/Lunchbox.git 5a37f93) -git_subproject(Keyv https://github.com/BlueBrain/Keyv.git 7ef03d0) -git_subproject(vmmlib https://github.com/Eyescale/vmmlib.git caa86d8) -git_subproject(MVDTool https://github.com/BlueBrain/MVDTool.git e7f9418) +git_subproject(Servus https://github.com/HBPVIS/Servus.git 85e715b) +git_subproject(Lunchbox https://github.com/Eyescale/Lunchbox.git 907cc47) +git_subproject(Keyv https://github.com/BlueBrain/Keyv.git cf63449) +git_subproject(vmmlib https://github.com/Eyescale/vmmlib.git 8795629) +git_subproject(MVDTool https://github.com/BlueBrain/MVDTool.git 4e2f0f4) diff --git a/.travis.yml b/.travis.yml index 332a163..1afc57e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,13 +3,6 @@ notifications: on_success: never language: cpp sudo: false -cache: - ccache: true - pip: true - directories: - - /usr/local -before_cache: - - brew cleanup os: - osx env: diff --git a/apps/compartmentConverter.cpp b/apps/compartmentConverter.cpp index 988afbb..f9ceba1 100644 --- a/apps/compartmentConverter.cpp +++ b/apps/compartmentConverter.cpp @@ -23,12 +23,8 @@ # include #endif -#include #include #include -#ifdef BRION_USE_OPENMP -# include -#endif namespace po = boost::program_options; using boost::lexical_cast; @@ -84,12 +80,7 @@ int main( const int argc, char** argv ) ( "maxFrames,m", po::value< size_t >(), "Convert at most the given number of frames" ) ( "compare,c", "Compare written report with input" ) - ( "dump,d", "Dump input report information (no output conversion)" ) -#ifdef BRION_USE_OPENMP - ( "threads,t", po::value< unsigned >()->default_value( 1 ), - "Number of threads to use" ) -#endif - ; + ( "dump,d", "Dump input report information (no output conversion)" ); po::variables_map vm; try @@ -97,7 +88,7 @@ int main( const int argc, char** argv ) po::store( po::parse_command_line( argc, argv, options ), vm ); po::notify( vm ); } - catch( const boost::program_options::error& e ) + catch( const po::error& e ) { std::cerr << "Command line parse error: " << e.what() << std::endl << options << std::endl; @@ -117,13 +108,6 @@ int main( const int argc, char** argv ) return EXIT_SUCCESS; } -#ifdef BRION_USE_OPENMP - omp_set_num_threads( vm[ "threads" ].as< unsigned >( )); - const unsigned nThreads = omp_get_num_threads(); -#else - const unsigned nThreads = 1; -#endif - std::string input; if( vm.count( "input" )) input = vm["input"].as< std::string >(); @@ -177,7 +161,7 @@ int main( const int argc, char** argv ) { size_t index = 0; - BOOST_FOREACH( const uint32_t gid, gids ) + for( const uint32_t gid : gids ) to.writeCompartments( gid, counts[ index++ ] ); } @@ -185,38 +169,42 @@ int main( const int argc, char** argv ) float writeTime = clock.getTimef(); const size_t nFrames = (end - start) / step; - boost::progress_display progress( nFrames / nThreads ); + boost::progress_display progress( nFrames ); -#pragma omp parallel for private(clock) for( size_t i = 0; i < nFrames; ++i ) { const float t = start + i * step; clock.reset(); brion::floatsPtr data = in.loadFrame( t ); loadTime += clock.getTimef(); + if( !data ) + { + LBERROR << "Can't load frame at " << t << " ms" << std::endl; + ::exit( EXIT_FAILURE ); + } const brion::floats& voltages = *data.get(); const brion::SectionOffsets& offsets = in.getOffsets(); size_t index = 0; clock.reset(); - BOOST_FOREACH( const uint32_t gid, gids ) + for( const uint32_t gid : gids ) { brion::floats cellVoltages; cellVoltages.reserve( in.getNumCompartments( index )); for( size_t j = 0; j < offsets[index].size(); ++j ) + { + const auto offset = offsets[index][j]; for( size_t k = 0; k < counts[index][j]; ++k ) - cellVoltages.push_back( voltages[ offsets[index][j]+k] ); + cellVoltages.emplace_back( voltages[ offset + k ] ); + } to.writeFrame( gid, cellVoltages, t ); ++index; } writeTime += clock.getTimef(); -#ifdef BRION_USE_OPENMP - if( omp_get_thread_num() == 0 ) -#endif - ++progress; + ++progress; } clock.reset(); @@ -225,7 +213,7 @@ int main( const int argc, char** argv ) std::cout << "Converted " << inURI << " to " << outURI << " (in " << size_t( loadTime ) << " out " << size_t( writeTime ) - << " ms) using " << nThreads << " threads" << std::endl; + << " ms)" << std::endl; if( vm.count( "compare" )) { diff --git a/brain/circuit.cpp b/brain/circuit.cpp index b646c50..78cac44 100644 --- a/brain/circuit.cpp +++ b/brain/circuit.cpp @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, EPFL/Blue Brain Project +/* Copyright (c) 2013-2017, EPFL/Blue Brain Project * Juan Hernando * Adrien.Devresse@epfl.ch * Daniel.Nachbaur@epfl.ch @@ -24,6 +24,7 @@ #include "detail/circuit.h" #include "synapsesStream.h" +#include namespace brain { diff --git a/brain/neuron/morphologyImpl.cpp b/brain/neuron/morphologyImpl.cpp index fc529df..736d15d 100644 --- a/brain/neuron/morphologyImpl.cpp +++ b/brain/neuron/morphologyImpl.cpp @@ -1,5 +1,5 @@ -/* Copyright (c) 2013-2016, EPFL/Blue Brain Project +/* Copyright (c) 2013-2017, EPFL/Blue Brain Project * Juan Hernando * * This file is part of Brion @@ -37,7 +37,7 @@ namespace { template< typename T > void _serializeArray( unsigned char*& dst, - const boost::shared_ptr< std::vector< T > >& src ) + const std::shared_ptr< std::vector< T >>& src ) { const size_t arraySize = src->size(); *reinterpret_cast< size_t* >( dst ) = arraySize; @@ -47,7 +47,7 @@ void _serializeArray( unsigned char*& dst, } template< typename T > -void _deserializeArray( boost::shared_ptr< std::vector< T > >& dst, +void _deserializeArray( std::shared_ptr< std::vector< T >>& dst, const unsigned char*& src ) { const size_t arraySize = *reinterpret_cast< const size_t* >( src ); diff --git a/brain/types.h b/brain/types.h index b7007f2..a25d14d 100644 --- a/brain/types.h +++ b/brain/types.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, EPFL/Blue Brain Project +/* Copyright (c) 2013-2017, EPFL/Blue Brain Project * Juan Hernando * * This file is part of Brion @@ -69,8 +69,8 @@ using brion::size_ts; typedef std::vector< Matrix4f > Matrix4fs; typedef std::vector< Quaternionf > Quaternionfs; -typedef boost::shared_ptr< SpikeReportReader > SpikeReportReaderPtr; -typedef boost::shared_ptr< SpikeReportWriter > SpikeReportWriterPtr; +typedef std::shared_ptr< SpikeReportReader > SpikeReportReaderPtr; +typedef std::shared_ptr< SpikeReportWriter > SpikeReportWriterPtr; /** * The GID of a synapse is the a tuple of two numbers: diff --git a/brion/blueConfig.cpp b/brion/blueConfig.cpp index 052838a..d152b8f 100644 --- a/brion/blueConfig.cpp +++ b/brion/blueConfig.cpp @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, EPFL/Blue Brain Project +/* Copyright (c) 2013-2017, EPFL/Blue Brain Project * Daniel Nachbaur * * This file is part of Brion @@ -22,6 +22,7 @@ #include "constants.h" #include "target.h" +#include #include #include #include diff --git a/brion/plugin/compartmentReportBinary.cpp b/brion/plugin/compartmentReportBinary.cpp index c2e16ad..e3c80dd 100644 --- a/brion/plugin/compartmentReportBinary.cpp +++ b/brion/plugin/compartmentReportBinary.cpp @@ -216,43 +216,49 @@ floatsPtr CompartmentReportBinary::loadFrame( const float timestamp ) const const size_t frameOffset = _header.dataBlockOffset + _header.numCompartments * sizeof(float) * frameNumber; - floatsPtr buffer( new floats( _header.numCompartments )); - memcpy( buffer->data(), ptr + frameOffset, - _header.numCompartments * sizeof(float) ); - - if( _header.byteswap ) + if( !_subtarget ) { -#pragma omp parallel for - for( int32_t i = 0; i < _header.numCompartments; ++i ) - lunchbox::byteswap( (*buffer)[i] ); - } + floatsPtr buffer( new floats( _header.numCompartments )); + memcpy( buffer->data(), ptr + frameOffset, + _header.numCompartments * sizeof(float) ); - if( !_subtarget ) + if( _header.byteswap ) + { +#pragma omp parallel for + for( int32_t i = 0; i < _header.numCompartments; ++i ) + lunchbox::byteswap( (*buffer)[i] ); + } return buffer; + } if( _comps == 0 ) return floatsPtr(); - floatsPtr subBuffer( new floats( _comps )); - size_t cellIndex = 0; - for( GIDSetCIter i = _gids.begin(); i != _gids.end(); ++i ) - { - const SectionOffsets& offsets = getOffsets(); - const CompartmentCounts& compCounts = getCompartmentCounts(); + floatsPtr buffer( new floats( _comps )); + const float* const source = (const float*)(ptr + frameOffset); + const SectionOffsets& offsets = getOffsets(); + const CompartmentCounts& compCounts = getCompartmentCounts(); - for( size_t j = 0; j < offsets[cellIndex].size(); ++j ) + for( size_t i = 0; i < _gids.size(); ++i ) + { + for( size_t j = 0; j < offsets[i].size(); ++j ) { - const uint16_t numCompartments = compCounts[cellIndex][j]; - const uint64_t sourceOffset = _conversionOffsets[cellIndex][j]; - const uint64_t targetOffset = offsets[cellIndex][j]; + const uint16_t numCompartments = compCounts[i][j]; + const uint64_t sourceOffset = _conversionOffsets[i][j]; + const uint64_t targetOffset = offsets[i][j]; for( uint16_t k = 0; k < numCompartments; ++k ) - (*subBuffer)[k + targetOffset] = (*buffer)[k + sourceOffset]; + (*buffer)[k + targetOffset] = source[k + sourceOffset]; } - ++cellIndex; } - return subBuffer; + if( _header.byteswap ) + { +#pragma omp parallel for + for( ssize_t i = 0; i < ssize_t( _comps ); ++i ) + lunchbox::byteswap( (*buffer)[i] ); + } + return buffer; } void CompartmentReportBinary::updateMapping( const GIDSet& gids ) @@ -263,24 +269,14 @@ void CompartmentReportBinary::updateMapping( const GIDSet& gids ) if( !_subtarget ) return; - GIDSet intersection; - std::set_intersection( _gids.begin(), _gids.end(), - _originalGIDs.begin(), _originalGIDs.end(), - std::inserter( intersection, intersection.begin( ))); + GIDSet intersection = _computeIntersection( _originalGIDs, _gids ); + if( intersection.empty( )) + { + LBTHROW( std::runtime_error( "CompartmentReportBinary::updateMapping:" + " GIDs out of range" )); + } if( intersection != _gids ) { - LBWARN << "Requested GIDs [" << *_gids.begin() << ":" <<*_gids.rbegin() - << "] are not a subset of the GIDs in the report [" - << *_originalGIDs.begin() << ":" << *_originalGIDs.rbegin(); - if( intersection.empty( )) - { - LBWARN << " with no GIDs in common" << std::endl; - LBTHROW( std::runtime_error("CompartmentReportBinary::updateMapping" - ": GIDs out of range" )); - } - - LBWARN << "], using intersection [" << *intersection.begin() << ":" - << *intersection.rbegin() << "]" << std::endl; updateMapping( intersection ); return; } diff --git a/brion/plugin/compartmentReportCommon.cpp b/brion/plugin/compartmentReportCommon.cpp index c0fc0f2..b47153b 100644 --- a/brion/plugin/compartmentReportCommon.cpp +++ b/brion/plugin/compartmentReportCommon.cpp @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2015, EPFL/Blue Brain Project +/* Copyright (c) 2013-2017, EPFL/Blue Brain Project * Daniel Nachbaur * * This file is part of Brion @@ -18,6 +18,7 @@ */ #include "compartmentReportCommon.h" +#include namespace brion { @@ -58,5 +59,27 @@ size_t CompartmentReportCommon::_getFrameNumber( float timestamp ) const return frame; } +GIDSet CompartmentReportCommon::_computeIntersection( const GIDSet& all, + const GIDSet& subset ) +{ + GIDSet intersection; + std::set_intersection( subset.begin(), subset.end(), all.begin(), all.end(), + std::inserter( intersection, intersection.begin( ))); + if( intersection != subset || intersection.empty( )) + { + LBWARN << "Requested " << subset.size() << " GIDs [" << *subset.begin() + << ":" <<*subset.rbegin() << "] are not a subset of the " + << all.size() << " GIDs in the report [" << *all.begin() << ":" + << *all.rbegin(); + if( intersection.empty( )) + LBWARN << " with no GIDs in common" << std::endl; + else + LBWARN << "], using intersection size " << intersection.size() + << " [" << *intersection.begin() << ":" + << *intersection.rbegin() << "]" << std::endl; + } + return intersection; +} + } } diff --git a/brion/plugin/compartmentReportCommon.h b/brion/plugin/compartmentReportCommon.h index fcd051c..368f6cb 100644 --- a/brion/plugin/compartmentReportCommon.h +++ b/brion/plugin/compartmentReportCommon.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2015, EPFL/Blue Brain Project +/* Copyright (c) 2013-2017, EPFL/Blue Brain Project * Daniel Nachbaur * * This file is part of Brion @@ -38,12 +38,12 @@ class CompartmentReportCommon : public CompartmentReportPlugin size_t getNumCompartments( size_t index ) const final; protected: - void _cacheNeuronCompartmentCounts( const GIDSet& gids ); size_t _getFrameNumber( float timestamp ) const; + static GIDSet _computeIntersection( const GIDSet& all, + const GIDSet& subset ); private: - size_ts _neuronCompartments; }; diff --git a/brion/plugin/compartmentReportHDF5.cpp b/brion/plugin/compartmentReportHDF5.cpp index a5cfac1..d5cc5d0 100644 --- a/brion/plugin/compartmentReportHDF5.cpp +++ b/brion/plugin/compartmentReportHDF5.cpp @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, EPFL/Blue Brain Project +/* Copyright (c) 2013-2017, EPFL/Blue Brain Project * Daniel Nachbaur * * This file is part of Brion @@ -441,7 +441,7 @@ bool CompartmentReportHDF5::writeCompartments( const uint32_t gid, const size_t compCount = std::accumulate( counts.begin(), counts.end(), 0 ); LBASSERT( !counts.empty( )); - LBASSERTINFO( compCount > 0, gid ); + LBASSERTINFO( compCount > 0, "No compartments for GID " << gid ); H5::DataSet dataset = _createDataset( gid, compCount ); const size_t sections = counts.size(); diff --git a/brion/plugin/compartmentReportMap.cpp b/brion/plugin/compartmentReportMap.cpp index 0e57ca8..ff8a150 100644 --- a/brion/plugin/compartmentReportMap.cpp +++ b/brion/plugin/compartmentReportMap.cpp @@ -19,6 +19,7 @@ #include "compartmentReportMap.h" +#include #include #include #include @@ -51,18 +52,11 @@ const std::string tunitKey( "tunit" ); const uint32_t _version = 3; // Increase with each change in a k/v pair const uint32_t _magic = 0xdb; const size_t _queueDepth = 32768; // async queue depth, heuristic from benchmark -#define ASYNC_IO - -std::string _getScope( const URI& uri ) -{ - return uri.findQuery( "name" )->second + "_" + - uri.findQuery( "target" )->second + "_"; -} - -template< class T > std::string toString( const T& value ) -{ - return boost::lexical_cast< std::string >( value ); -} +#ifdef BRION_USE_OPENMP +const size_t _nThreads = omp_get_num_procs(); +#else +const size_t _nThreads = 1; +#endif } namespace plugin @@ -74,27 +68,31 @@ namespace CompartmentReportMap::CompartmentReportMap( const CompartmentReportInitData& initData ) - : _uri( initData.getURI( )) - , _store( initData.getURI( )) + : _uri( std::to_string( initData.getURI( )) + "_" ) , _readable( false ) { - _store.setQueueDepth( _queueDepth ); - if( _uri.findQuery( "name" ) == _uri.queryEnd( )) - _uri.addQuery( "name", "default" ); - if( _uri.findQuery( "target" ) == _uri.queryEnd( )) - _uri.addQuery( "target", "allCompartments" ); + // have at least one store + const auto& uri = initData.getURI(); + _stores.emplace_back( keyv::Map( uri )); + _stores.back().setQueueDepth( _queueDepth ); + if( uri.getScheme() == "memcached" ) // parallelize loading with memcached + while( _stores.size() < _nThreads ) + { + _stores.emplace_back( keyv::Map( uri )); + _stores.back().setQueueDepth( _queueDepth ); + } const int accessMode = initData.getAccessMode(); if(( accessMode & MODE_READ ) && !_loadHeader( )) LBTHROW( std::runtime_error( "Incomplete or missing report at " + - _uri.getPath( ))); + std::to_string( uri ))); if( accessMode == MODE_WRITE || accessMode == MODE_READWRITE ) { if( _loadHeader( )) LBTHROW( std::runtime_error( "Cannot overwrite existing report at "+ - _uri.getPath( ))); + std::to_string( uri ))); _clear(); // reset after loading header } @@ -115,7 +113,8 @@ bool CompartmentReportMap::handles(const CompartmentReportInitData& initData ) void CompartmentReportMap::_clear() { _readable = false; - _store.setByteswap( false ); + for( auto& store : _stores ) + store.setByteswap( false ); _header = Header(); _gids.clear(); _offsets.clear(); @@ -163,8 +162,8 @@ bool CompartmentReportMap::writeCompartments( const uint32_t gid, { LBASSERTINFO( !counts.empty(), gid ); _gids.insert( gid ); - return _store.insert( _getScope( _uri ) + countsKey + toString( gid ), - counts ); + return _stores.front().insert( + _uri + countsKey + std::to_string( gid ), counts ); } bool CompartmentReportMap::writeFrame( const uint32_t gid, @@ -174,25 +173,27 @@ bool CompartmentReportMap::writeFrame( const uint32_t gid, if( !_flushHeader( )) return false; - const std::string& scope = _getScope( _uri ); #ifndef NDEBUG - const uint16_ts& counts = _store.getVector< uint16_t >( scope + countsKey + - toString( gid )); + const uint16_ts& counts = + _stores.front().getVector< uint16_t >( _uri + countsKey + + std::to_string( gid )); const size_t size = std::accumulate( counts.begin(), counts.end(), 0 ); LBASSERTINFO( size == voltages.size(), "gid " << gid << " should have " << size << " voltages not " << voltages.size( )); #endif const size_t index = _getFrameNumber( time ); - const std::string& key = scope + toString( gid ) + "_" + toString( index ); - return _store.insert( key, voltages ); + const std::string& key = _uri + std::to_string( gid ) + "_" + + std::to_string( index ); + return _stores.front().insert( key, voltages ); } bool CompartmentReportMap::flush() { if( !_flushHeader( )) return false; - _store.flush(); + for( auto& store : _stores ) + store.flush(); return true; } @@ -206,11 +207,11 @@ bool CompartmentReportMap::_flushHeader() _header.nGIDs = uint32_t(_gids.size( )); - const std::string& scope = _getScope( _uri ); - if( !_store.insert( scope + headerKey, _header ) || - !_store.insert( scope + gidsKey, _gids ) || - !_store.insert( scope + dunitKey, _dunit ) || - !_store.insert( scope + tunitKey, _tunit )) + auto& store = _stores.front(); + if( !store.insert( _uri + headerKey, _header ) || + !store.insert( _uri + gidsKey, _gids ) || + !store.insert( _uri + dunitKey, _dunit ) || + !store.insert( _uri + tunitKey, _tunit )) { return false; } @@ -221,17 +222,21 @@ bool CompartmentReportMap::_flushHeader() bool CompartmentReportMap::_loadHeader() { + GIDSet gids; + gids.swap( _gids ); // keep requested gids _clear(); - const std::string& scope = _getScope( _uri ); + _gids.swap( gids ); // restore after clear try { - _header = _store.get< Header >( scope + headerKey ); + auto& store = _stores.front(); + _header = store.get< Header >( _uri + headerKey ); const bool byteswap = ( _header.magic != _magic ); if( byteswap ) { lunchbox::byteswap( _header ); - _store.setByteswap( true ); + for( auto& store_ : _stores ) + store_.setByteswap( true ); } LBASSERT( _header.magic == _magic ); @@ -239,6 +244,7 @@ bool CompartmentReportMap::_loadHeader() { LBWARN << "report header has wrong magic " << _header.magic << " instead of " << _magic << std::endl; + _clear(); return false; } @@ -246,6 +252,7 @@ bool CompartmentReportMap::_loadHeader() { LBWARN << "report has version " << _header.version << ", can only read version " << _version << std::endl; + _clear(); return false; } LBASSERTINFO( _header.endTime - _header.startTime >= _header.timestep, @@ -253,42 +260,69 @@ bool CompartmentReportMap::_loadHeader() _header.endTime << "/" << _header.timestep ); const bool loadGIDs = _gids.empty(); -#ifdef ASYNC_IO - _store.fetch( scope + dunitKey ); - _store.fetch( scope + tunitKey ); + store.fetch( _uri + dunitKey ); + store.fetch( _uri + tunitKey ); if( loadGIDs ) - _store.fetch( scope + gidsKey, _header.nGIDs * sizeof( uint32_t )); + store.fetch( _uri + gidsKey, _header.nGIDs * sizeof( uint32_t )); for( const uint32_t gid : _gids ) - _store.fetch( scope + countsKey + toString( gid )); -#endif + store.fetch( _uri + countsKey + std::to_string( gid )); - _dunit = _store[ scope + dunitKey ]; - _tunit = _store[ scope + tunitKey ]; + _dunit = store[ _uri + dunitKey ]; + _tunit = store[ _uri + tunitKey ]; if( loadGIDs ) - _gids = _store.getSet< uint32_t >( scope + gidsKey ); + _gids = store.getSet< uint32_t >( _uri + gidsKey ); - _readable = true; -#ifdef ASYNC_IO - if( loadGIDs ) - for( const uint32_t gid : _gids ) - _store.fetch( scope + countsKey + toString( gid )); -#endif + if( _gids.empty( )) + { + LBWARN << "No gids for report" << std::endl; + _clear(); + return false; + } + + Strings keys; + keys.reserve( _gids.size( )); + std::unordered_map< std::string, uint32_t > gidMap; + for( const uint32_t gid : _gids ) + { + keys.push_back( _uri + countsKey + std::to_string( gid )); + gidMap.emplace( keys.back(), gid ); + } + + size_t taken = 0; + const auto takeValue = [&] ( const std::string& key, + char* data, const size_t size ) + { + const uint32_t gid = gidMap[ key ]; + _cellCounts[ gid ] = std::vector< uint16_t >( + reinterpret_cast< const uint16_t* >( data ), + reinterpret_cast< const uint16_t* >( data + size )); + + std::free( data ); + ++taken; + }; + + store.takeValues( keys, takeValue ); + if( taken != _gids.size( )) + { + LBWARN << "Missing mapping " << _gids.size() - taken << " of " + << _gids.size() << " gids" << std::endl; + _clear(); + return false; + } uint64_t offset = 0; for( const uint32_t gid : _gids ) { - _cellCounts[ gid ] = _store.getVector< uint16_t >( scope + - countsKey + - toString( gid )); - _counts.push_back( _cellCounts[ gid ] ); + const auto& cellCounts = _cellCounts[ gid ]; + _counts.push_back( cellCounts ); - const size_t nSections = _cellCounts[ gid ].size(); + const size_t nSections = cellCounts.size(); _offsets.push_back( uint64_ts( nSections, std::numeric_limits::max( ))); for( size_t i = 0; i < nSections; ++i ) { - const uint16_t numCompartments = _cellCounts[ gid ][ i ]; + const uint16_t numCompartments = cellCounts[ i ]; if( numCompartments == 0 ) continue; @@ -296,13 +330,17 @@ bool CompartmentReportMap::_loadHeader() offset += numCompartments; } - _totalCompartments += std::accumulate( _cellCounts[ gid ].begin(), - _cellCounts[ gid ].end(), 0); + _totalCompartments += std::accumulate( cellCounts.begin(), + cellCounts.end(), 0 ); } + + _readable = true; + return true; } catch( const std::runtime_error& e ) { LBWARN << "Missing header information: " << e.what() << std::endl; + _clear(); return false; } @@ -314,7 +352,14 @@ void CompartmentReportMap::updateMapping( const GIDSet& gids ) if( _gids == gids && !gids.empty() && _readable ) return; - _gids = gids; + const auto all = _stores.front().getSet< uint32_t >( _uri + gidsKey ); + const auto& subset = gids.empty() ? all : gids; + + _gids = _computeIntersection( all, subset ); + + if( _gids.empty( )) + LBTHROW( std::runtime_error( "CompartmentReportMap::updateMapping:" + " GIDs out of range" )); if( !_loadHeader( )) LBTHROW( std::runtime_error( "Incomplete data source" )); } @@ -324,14 +369,19 @@ floatsPtr CompartmentReportMap::loadFrame( const float time ) const if( !_readable ) return floatsPtr(); - const std::string& scope = _getScope( _uri ); - floatsPtr buffer( new floats( getFrameSize( ))); - floats::iterator iter = buffer->begin(); - const size_t index = _getFrameNumber( time ); - std::map< uint32_t, size_t > sizeMap; + const std::string index = std::string( "_" ) + + std::to_string( _getFrameNumber( time )); + + std::unordered_map< std::string, size_t > offsetMap; + size_t offset = 0; + + Strings keys; + keys.reserve( _gids.size( )); for( const uint32_t gid : _gids ) { + keys.push_back( _uri + std::to_string( gid ) + index ); + offsetMap.emplace( keys.back(), offset ); const CellCompartments::const_iterator i = _cellCounts.find( gid ); if( i == _cellCounts.end( )) { @@ -339,38 +389,49 @@ floatsPtr CompartmentReportMap::loadFrame( const float time ) const return floatsPtr(); } const size_t size = std::accumulate( i->second.begin(), - i->second.end(), 0 ) * - sizeof( float ); - sizeMap[ gid ] = size; - -#ifdef ASYNC_IO - _store.fetch( scope + toString( gid ) + "_" + toString( index ), size ); -#endif + i->second.end(), 0 ); + offset += size; } - for( const uint32_t gid : _gids ) + floatsPtr buffer( new floats( getFrameSize( ))); + float* const ptr = buffer->data(); + +#ifdef BRION_USE_OPENMP + lunchbox::a_ssize_t taken; + omp_set_num_threads( _stores.size( )); +#else + size_t taken = 0; +#endif +#pragma omp parallel { - LBASSERTINFO( iter < buffer->end(), buffer->size() << " gid " << gid ); - const std::string& cellData = _store[ scope + toString( gid ) + "_" + - toString( index ) ]; - LBASSERT( !cellData.empty( )); - if( cellData.empty( )) +#ifdef BRION_USE_OPENMP + auto& store = _stores[ omp_get_thread_num( )]; + const size_t start = float( omp_get_thread_num( )) * + float( keys.size( )) / float( omp_get_num_threads( )); + const size_t end = float( omp_get_thread_num() + 1 ) * + float( keys.size( )) / float( omp_get_num_threads( )); + const Strings subKeys( keys.begin() + start, keys.begin() + end ); +#else + auto& store = _stores.front(); + const Strings& subKeys = keys; +#endif + + const auto takeValue = [ ptr, &offsetMap, &taken ] + ( const std::string& key, char* data, const size_t size ) { - LBWARN << "Missing data for gid " << toString( gid ) << std::endl; - return floatsPtr(); - } - LBASSERTINFO( sizeMap[ gid ] == cellData.size(), - sizeMap[ gid ] << " != " << cellData.size( )); + ::memcpy( ptr + offsetMap[ key ], data, size ); + std::free( data ); + ++taken; + }; - ::memcpy( &(*iter), cellData.data(), cellData.size( )); - iter += cellData.size() / sizeof( float ); - if( iter > buffer->end( )) - return floatsPtr(); + store.takeValues( subKeys, takeValue ); } - if( iter == buffer->end( )) + + if( size_t( taken ) == _gids.size( )) return buffer; - LBASSERT( iter == buffer->end( )); + LBWARN << "Missing " << _gids.size() - taken << " of " << _gids.size() + << " gids in report frame at " << time << " ms" << std::endl; return floatsPtr(); } diff --git a/brion/plugin/compartmentReportMap.h b/brion/plugin/compartmentReportMap.h index 062a06d..9efcb8a 100644 --- a/brion/plugin/compartmentReportMap.h +++ b/brion/plugin/compartmentReportMap.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2016, EPFL/Blue Brain Project +/* Copyright (c) 2014-2017, EPFL/Blue Brain Project * Stefan.Eilemann@epfl.ch * * This file is part of Brion @@ -38,13 +38,10 @@ class CompartmentReportMap : public CompartmentReportCommon static bool handles( const CompartmentReportInitData& initData ); float getStartTime() const final { return _header.startTime; } - float getEndTime() const final { return _header.endTime; } - float getTimestep() const final { return _header.timestep; } const std::string& getDataUnit() const final { return _dunit; } - const std::string& getTimeUnit() const final { return _tunit; } const brion::GIDSet& getGIDs() const final { return _gids; } @@ -76,8 +73,8 @@ class CompartmentReportMap : public CompartmentReportCommon }; private: - URI _uri; - keyv::Map _store; + const std::string _uri; + std::vector< keyv::Map > _stores; Header _header; diff --git a/brion/plugin/morphologySWC.cpp b/brion/plugin/morphologySWC.cpp index 204e121..9c3819d 100644 --- a/brion/plugin/morphologySWC.cpp +++ b/brion/plugin/morphologySWC.cpp @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, EPFL/Blue Brain Project +/* Copyright (c) 2013-2017, EPFL/Blue Brain Project * Juan Hernando * Daniel Nachbaur * @@ -27,6 +27,7 @@ #include #include #include +#include namespace brion { diff --git a/brion/plugin/spikeReportFile.cpp b/brion/plugin/spikeReportFile.cpp index 2f15cbf..8081b72 100644 --- a/brion/plugin/spikeReportFile.cpp +++ b/brion/plugin/spikeReportFile.cpp @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, EPFL/Blue Brain Project +/* Copyright (c) 2013-2017, EPFL/Blue Brain Project * Raphael Dumusc * * This file is part of Brion diff --git a/brion/plugin/spikeReportFile.h b/brion/plugin/spikeReportFile.h index 057d909..1e5a4be 100644 --- a/brion/plugin/spikeReportFile.h +++ b/brion/plugin/spikeReportFile.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2015, EPFL/Blue Brain Project +/* Copyright (c) 2013-2017, EPFL/Blue Brain Project * Raphael Dumusc * * This file is part of Brion @@ -24,6 +24,7 @@ #include "spikeReportTypes.h" #include +#include #include namespace brion diff --git a/brion/plugin/spikeReportNEST.cpp b/brion/plugin/spikeReportNEST.cpp index 2200beb..c708135 100644 --- a/brion/plugin/spikeReportNEST.cpp +++ b/brion/plugin/spikeReportNEST.cpp @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, EPFL/Blue Brain Project +/* Copyright (c) 2013-2017, EPFL/Blue Brain Project * Raphael Dumusc * * This file is part of Brion @@ -21,6 +21,7 @@ #include "spikeReportNEST.h" #include "../pluginInitData.h" +#include #include #include #include diff --git a/brion/target.cpp b/brion/target.cpp index 1c48ff4..c7e52d5 100644 --- a/brion/target.cpp +++ b/brion/target.cpp @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, EPFL/Blue Brain Project +/* Copyright (c) 2013-2017, EPFL/Blue Brain Project * Daniel Nachbaur * * This file is part of Brion @@ -21,6 +21,7 @@ #include #include +#include #include #include #include diff --git a/brion/types.h b/brion/types.h index 3674ea3..df71841 100644 --- a/brion/types.h +++ b/brion/types.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, EPFL/Blue Brain Project +/* Copyright (c) 2013-2017, EPFL/Blue Brain Project * Daniel Nachbaur * * This file is part of Brion @@ -22,9 +22,7 @@ #include -#include #include -#include #include #include #include @@ -72,16 +70,16 @@ typedef std::vector< Vector3d > Vector3ds; typedef std::vector< Vector4d > Vector4ds; typedef std::vector< SectionType > SectionTypes; typedef std::vector< Target > Targets; -typedef boost::shared_ptr< int32_ts > int32_tsPtr; -typedef boost::shared_ptr< uint16_ts > uint16_tsPtr; -typedef boost::shared_ptr< uint32_ts > uint32_tsPtr; -typedef boost::shared_ptr< floats > floatsPtr; -typedef boost::shared_ptr< Vector2is > Vector2isPtr; -typedef boost::shared_ptr< Vector3fs > Vector3fsPtr; -typedef boost::shared_ptr< Vector4fs > Vector4fsPtr; -typedef boost::shared_ptr< Vector3ds > Vector3dsPtr; -typedef boost::shared_ptr< Vector4ds > Vector4dsPtr; -typedef boost::shared_ptr< SectionTypes > SectionTypesPtr; +typedef std::shared_ptr< int32_ts > int32_tsPtr; +typedef std::shared_ptr< uint16_ts > uint16_tsPtr; +typedef std::shared_ptr< uint32_ts > uint32_tsPtr; +typedef std::shared_ptr< floats > floatsPtr; +typedef std::shared_ptr< Vector2is > Vector2isPtr; +typedef std::shared_ptr< Vector3fs > Vector3fsPtr; +typedef std::shared_ptr< Vector4fs > Vector4fsPtr; +typedef std::shared_ptr< Vector3ds > Vector3dsPtr; +typedef std::shared_ptr< Vector4ds > Vector4dsPtr; +typedef std::shared_ptr< SectionTypes > SectionTypesPtr; /** Ordered set of GIDs of neurons. */ typedef std::set< uint32_t > GIDSet; diff --git a/doc/Changelog.md b/doc/Changelog.md index 6d77dc9..ff7f778 100644 --- a/doc/Changelog.md +++ b/doc/Changelog.md @@ -3,6 +3,9 @@ Changelog {#Changelog} # git master +* [121](https://github.com/BlueBrain/Brion/pull/121): + Performance optimizations for map and binary compartment reports, fix + sub target loading of map compartment reports * [119](https://github.com/BlueBrain/Brion/pull/119): Optimized brain::Circuit::loadMorphologies for slow stat'ing filesystems (GPFS) diff --git a/tests/brain/circuit.cpp b/tests/brain/circuit.cpp index 1dc08ac..916e004 100644 --- a/tests/brain/circuit.cpp +++ b/tests/brain/circuit.cpp @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2015, EPFL/Blue Brain Project +/* Copyright (c) 2013-2017, EPFL/Blue Brain Project * Daniel Nachbaur * * This file is part of Brion @@ -22,6 +22,7 @@ #include #define BOOST_TEST_MODULE Circuit +#include #include #include #include diff --git a/tests/compartmentReport.cpp b/tests/compartmentReport.cpp index a4e879b..a2e9a93 100644 --- a/tests/compartmentReport.cpp +++ b/tests/compartmentReport.cpp @@ -172,8 +172,8 @@ void test_compare( const brion::URI& uri1, const brion::URI& uri2 ) { std::cout << "Compare " << uri1 << " == " << uri2 << std::endl; - const brion::CompartmentReport report1( uri1, brion::MODE_READ ); - const brion::CompartmentReport report2( uri2, brion::MODE_READ ); + brion::CompartmentReport report1( uri1, brion::MODE_READ ); + brion::CompartmentReport report2( uri2, brion::MODE_READ ); BOOST_CHECK_EQUAL( report1.getStartTime(), report2.getStartTime( )); BOOST_CHECK_EQUAL( report1.getEndTime(), report2.getEndTime( )); @@ -220,6 +220,16 @@ void test_compare( const brion::URI& uri1, const brion::URI& uri2 ) BOOST_CHECK_EQUAL( (*frame1)[j], (*frame2)[j] ); } } + + brion::GIDSet gids; + gids.insert( 394 ); + report1.updateMapping( gids ); + report2.updateMapping( gids ); + + BOOST_CHECK_EQUAL( report1.getGIDs().size(), 1 ); + BOOST_CHECK_EQUAL( *report1.getGIDs().begin(), 394 ); + BOOST_CHECK_EQUAL( report2.getGIDs().size(), 1 ); + BOOST_CHECK_EQUAL( *report2.getGIDs().begin(), 394 ); } /** @return false if no "from" or "to" report plugin was found, true otherwise*/ @@ -331,6 +341,7 @@ BOOST_AUTO_TEST_CASE( test_convert_and_compare ) uris.push_back( brion::URI( temp.string() + ".h5" )); uris.push_back( brion::URI( "leveldb://" )); + uris.push_back( brion::URI( "memcached:///briontest" )); while( !uris.empty( )) {