diff --git a/CMakeLists.txt b/CMakeLists.txt index 86604ea..c9fea45 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,7 +13,7 @@ include(GitExternal) include(SubProject) set(VERSION_MAJOR "1") -set(VERSION_MINOR "5") +set(VERSION_MINOR "6") set(VERSION_PATCH "0") set(VERSION_ABI 5) diff --git a/brion/synapse.cpp b/brion/synapse.cpp index f713145..5f40f62 100644 --- a/brion/synapse.cpp +++ b/brion/synapse.cpp @@ -22,8 +22,10 @@ #include "detail/silenceHDF5.h" #include +#include #include #include +#include #include #include #include @@ -39,11 +41,14 @@ struct Dataset H5::DataSpace dataspace; hsize_t dims[2]; }; +namespace fs = boost::filesystem; +using boost::lexical_cast; -class Synapse : public boost::noncopyable +/** Access a single synapse file (nrn*.h5 or nrn*.h5. */ +class SynapseFile : public boost::noncopyable { public: - explicit Synapse( const std::string& source ) + explicit SynapseFile( const std::string& source ) { lunchbox::ScopedWrite mutex( detail::_hdf5Lock ); @@ -72,7 +77,7 @@ class Synapse : public boost::noncopyable } } - ~Synapse() + ~SynapseFile() { lunchbox::ScopedWrite mutex( detail::_hdf5Lock ); @@ -166,9 +171,166 @@ class Synapse : public boost::noncopyable return true; } + SynapseMatrix read( const uint32_t gid, const uint32_t attributes ) const + { + switch( _numAttributes ) + { + case SYNAPSE_ALL: + return read< SYNAPSE_ALL >( gid, attributes ); + case SYNAPSE_POSITION_ALL: + return read< SYNAPSE_POSITION_ALL >( gid, attributes ); + default: + LBERROR << "Synapse file " << _file.getFileName() + << " has unknown number of attributes: " << _numAttributes + << std::endl; + return SynapseMatrix(); + } + } + +private: H5::H5File _file; size_t _numAttributes; }; + +/** Implement the logic to read a merged .h5 or individual .h5. files */ +class Synapse : public boost::noncopyable +{ +public: + explicit Synapse( const std::string& source ) + : _file( 0 ) + , _gid( 0 ) + { + try + { + _file = new SynapseFile( source ); + } + catch( const std::runtime_error& ) + { + // try to open in individual files + const fs::path dir = fs::path( source ).parent_path(); + const fs::path filename = fs::path( source ).filename(); + const boost::regex filter( filename.string() + "\\.[0-9]+$" ); + + fs::directory_iterator end; + for( fs::directory_iterator i( dir ); i != end; ++i ) + { + const fs::path candidate = i->path().filename(); + boost::smatch match; + + if( !boost::filesystem::is_regular_file( i->status( )) || + !boost::regex_match( candidate.string(), match, filter )) + { + continue; + } + + _unmappedFiles.push_back( i->path().string( )); + } + + if( _unmappedFiles.empty( )) + { + LBTHROW( std::runtime_error( "Could not find synapse files " + + dir.string() + "/" + + filename.string() + ".[0-9]+" )); + } + } + } + + ~Synapse() + { + delete _file; + } + + SynapseMatrix read( const uint32_t gid, const uint32_t attributes ) const + { + if( _findFile( gid )) + return _file->read( gid, attributes ); + return SynapseMatrix(); + } + + size_t getNumSynapses( const GIDSet& gids ) const + { + size_t numSynapses = 0; + BOOST_FOREACH( const uint32_t gid, gids ) + { + if( !_findFile( gid )) + continue; + + GIDSet set; + set.insert( gid ); + numSynapses += _file->getNumSynapses( set ); + } + return numSynapses; + } + +private: + typedef std::map< uint32_t, std::string > GidFileMap; + + mutable SynapseFile* _file; + mutable uint32_t _gid; // current or 0 for all + mutable Strings _unmappedFiles; + mutable GidFileMap _fileMap; + + bool _findFile( const uint32_t gid ) const + { + if( _file && ( _gid == gid || _gid == 0 )) + return true; + + const std::string& filename = _findFilename( gid ); + if( filename.empty( )) + return false; + + delete _file; + _file = new SynapseFile( filename ); + _gid = gid; + return true; + } + + std::string _findFilename( const uint32_t gid ) const + { + while( _fileMap[ gid ].empty( )) + { + if( _unmappedFiles.empty( )) + return std::string(); + + const std::string candidate = _unmappedFiles.back(); + _unmappedFiles.pop_back(); + + lunchbox::ScopedWrite mutex( detail::_hdf5Lock ); + H5::H5File file; + try + { + SilenceHDF5 silence; + file.openFile( candidate, H5F_ACC_RDONLY ); + } + catch( const H5::Exception& exc ) + { + LBINFO << "Could not open synapse file " << candidate << ": " + << exc.getDetailMsg() << std::endl; + continue; + } + + const size_t size = file.getNumObjs(); + for( size_t i = 0; i < size; ++i ) + { + const std::string& name = file.getObjnameByIdx( i ); + const boost::regex filter( "^a[0-9]+$" ); + boost::smatch match; + + if( boost::regex_match( name, match, filter )) + { + std::string string = match.str(); + string.erase( 0, 1 ); // remove the 'a' + const uint32_t cGID = lexical_cast( string ); + _fileMap[ cGID ] = candidate; + } + } + file.close(); + } + + return _fileMap[ gid ]; + } +}; + } Synapse::Synapse( const std::string& source ) @@ -184,18 +346,7 @@ Synapse::~Synapse() SynapseMatrix Synapse::read( const uint32_t gid, const uint32_t attributes ) const { - switch( _impl->_numAttributes ) - { - case SYNAPSE_ALL: - return _impl->read< SYNAPSE_ALL >( gid, attributes ); - case SYNAPSE_POSITION_ALL: - return _impl->read< SYNAPSE_POSITION_ALL >( gid, attributes ); - default: - LBERROR << "Synapse file " << _impl->_file.getFileName() - << " has unknown number of attributes: " - << _impl->_numAttributes << std::endl; - return SynapseMatrix(); - } + return _impl->read( gid, attributes ); } size_t Synapse::getNumSynapses( const GIDSet& gids ) const diff --git a/brion/target.cpp b/brion/target.cpp index d5a608c..61c1ac6 100644 --- a/brion/target.cpp +++ b/brion/target.cpp @@ -38,6 +38,8 @@ inline brion::TargetType lexical_cast( const std::string& s ) } } +using boost::lexical_cast; + namespace brion { namespace detail @@ -68,7 +70,7 @@ class Target const std::string& name = *i++; std::string content = *i++; - const TargetType type = boost::lexical_cast< TargetType >( typeStr); + const TargetType type = lexical_cast< TargetType >( typeStr ); _targetNames[type].push_back( name ); boost::trim( content ); if( content.empty( )) @@ -107,6 +109,10 @@ class Target }; } +Target::Target( const Target& from ) + : _impl( new detail::Target( *from._impl )) +{} + Target::Target( const std::string& source ) : _impl( new detail::Target( source )) { @@ -117,6 +123,17 @@ Target::~Target() delete _impl; } +Target& Target::operator = ( const Target& rhs ) +{ + if( this == &rhs ) + return *this; + + delete _impl; + _impl = new detail::Target( *rhs._impl ); + return *this; +} + + const Strings& Target::getTargetNames( const TargetType type ) const { return _impl->getTargetNames( type ); @@ -127,6 +144,36 @@ const Strings& Target::get( const std::string& name ) const return _impl->get( name ); } +namespace +{ +void _parse( const Targets& targets, const std::string& name, GIDSet& gids ) +{ + BOOST_FOREACH( const Target& target, targets ) + { + const brion::Strings& values = target.get( name ); + BOOST_FOREACH( const std::string& value, values ) + { + try + { + gids.insert( lexical_cast< uint32_t >( value.substr( 1 ))); + } + catch( ... ) + { + if( value != name ) + _parse( targets, value, gids ); + } + } + } +} +} + +GIDSet Target::parse( const Targets& targets, const std::string& name ) +{ + brion::GIDSet gids; + _parse( targets, name, gids ); + return gids; +} + std::ostream& operator << ( std::ostream& os, const Target& target ) { const Strings& targetNames = target.getTargetNames( brion::TARGET_CELL ); diff --git a/brion/target.h b/brion/target.h index e1b2b2f..58718ff 100644 --- a/brion/target.h +++ b/brion/target.h @@ -21,7 +21,6 @@ #define BRION_TARGET #include -#include namespace brion { @@ -33,12 +32,18 @@ namespace detail { class Target; } * Following RAII, this class is ready to use after the creation and will ensure * release of resources upon destruction. */ -class Target : public boost::noncopyable +class Target { public: + /** Copy-construct a target file. @version 1.6 */ + Target( const Target& from ); + /** Close target file. @version 1.0 */ ~Target(); + /** Assign a different target. @version 1.6 */ + Target& operator = ( const Target& rhs ); + /** @name Read API */ //@{ /** Open the given source to a target file for reading. @@ -64,12 +69,25 @@ class Target : public boost::noncopyable * @version 1.0 */ const Strings& get( const std::string& name ) const; + + /** + * Parse a given target into a GID set. + * + * All given targets are searched for the given name. If found, the named + * target is recursively resolved to a GID set. + * + * @param targets the targets to parse + * @param name the target name to parse + * @return the set of cell identifiers parsed + * @version 1.6 + */ + static GIDSet parse( const Targets& targets, const std::string& name ); //@} private: friend std::ostream& operator << ( std::ostream&, const Target& ); - detail::Target* const _impl; + detail::Target* _impl; }; /** Stream out content of target file. */ diff --git a/brion/types.h b/brion/types.h index 82a7fdb..6ff8f5e 100644 --- a/brion/types.h +++ b/brion/types.h @@ -76,7 +76,7 @@ typedef std::vector< Vector4f > Vector4fs; typedef std::vector< Vector3d > Vector3ds; typedef std::vector< Vector4d > Vector4ds; typedef std::vector< SectionType > SectionTypes; - +typedef std::vector< Target > Targets; typedef boost::shared_ptr< int32_ts > int32_tsPtr; typedef boost::shared_ptr< uint16_ts > uint16_tsPtr; typedef boost::shared_ptr< uint32_ts > uint32_tsPtr; diff --git a/doc/Changelog.md b/doc/Changelog.md index 75667d4..27d4894 100644 --- a/doc/Changelog.md +++ b/doc/Changelog.md @@ -1,6 +1,13 @@ Changelog {#Changelog} ========= +## git master {master) + +* [#12](https://github.com/BlueBrain/Brion/pull/12): + Add brion::Target::parse() to resolve a given target name +* [#12](https://github.com/BlueBrain/Brion/pull/12): + Extended brion::Synapse to also support non-merged synapse files + ## Release 1.5.0 (2015-07-07) {changelog_1_5_0} * Add RESTING_VOLTAGE constant. diff --git a/tests/synapse.cpp b/tests/synapse.cpp index 1febb06..f2c8c8f 100644 --- a/tests/synapse.cpp +++ b/tests/synapse.cpp @@ -1,5 +1,7 @@ + /* Copyright (c) 2013-2015, EPFL/Blue Brain Project * Daniel Nachbaur + * Stefan.Eilemann@epfl.ch * * This file is part of Brion * @@ -53,7 +55,7 @@ lunchbox::SpinLock testLock; BOOST_CHECK_GT( ( L ), ( R ) ); \ } -BOOST_AUTO_TEST_CASE( test_invalid_open ) +BOOST_AUTO_TEST_CASE( invalid_open ) { BOOST_CHECK_THROW( brion::Synapse( "/bla" ), std::runtime_error ); BOOST_CHECK_THROW( brion::Synapse( "bla" ), std::runtime_error ); @@ -67,25 +69,24 @@ BOOST_AUTO_TEST_CASE( test_invalid_open ) BOOST_CHECK_THROW( brion::Synapse( path.string( )), std::runtime_error ); } -BOOST_AUTO_TEST_CASE( test_invalid_read ) +BOOST_AUTO_TEST_CASE( invalid_read ) { boost::filesystem::path path( BBP_TESTDATA ); path /= "circuitBuilding_1000neurons/Functionalizer_output/nrn.h5"; - brion::Synapse synapseFile( path.string( )); + const brion::Synapse synapseFile( path.string( )); const brion::SynapseMatrix& data = synapseFile.read( 0, brion::SYNAPSE_ALL_ATTRIBUTES ); BOOST_CHECK_EQUAL( data.shape()[0], 0 ); BOOST_CHECK_EQUAL( data.shape()[1], 0 ); } -BOOST_AUTO_TEST_CASE( test_read ) +BOOST_AUTO_TEST_CASE( read_attributes ) { boost::filesystem::path path( BBP_TESTDATA ); path /= "circuitBuilding_1000neurons/Functionalizer_output/nrn.h5"; - brion::Synapse synapseFile( path.string( )); - + const brion::Synapse synapseFile( path.string( )); const brion::SynapseMatrix& empty = synapseFile.read( 1, brion::SYNAPSE_NO_ATTRIBUTES ); BOOST_CHECK_EQUAL( empty.shape()[0], 0 ); @@ -93,7 +94,6 @@ BOOST_AUTO_TEST_CASE( test_read ) const brion::SynapseMatrix& data = synapseFile.read( 1, brion::SYNAPSE_ALL_ATTRIBUTES ); - std::cout << data << std::endl; BOOST_CHECK_EQUAL( data.shape()[0], 77 ); // 77 synapses for GID 1 BOOST_CHECK_EQUAL( data.shape()[1], 19 ); // 19 (==all) synapse attributes BOOST_CHECK_EQUAL( data[0][0], 10 ); @@ -107,7 +107,6 @@ BOOST_AUTO_TEST_CASE( test_read ) const brion::SynapseMatrix& data2 = synapseFile.read( 4, brion::SYNAPSE_DELAY ); - std::cout << data2 << std::endl; BOOST_CHECK_EQUAL( data2.shape()[0], 41 ); // 41 synapses for GID 4 BOOST_CHECK_EQUAL( data2.shape()[1], 1 ); // 1 synapse attributes BOOST_CHECK_CLOSE( data2[0][0], 1.46838176f, .0003f ); @@ -115,12 +114,12 @@ BOOST_AUTO_TEST_CASE( test_read ) BOOST_CHECK_CLOSE( data2[9][0], 2.21976233f, .0003f ); } -BOOST_AUTO_TEST_CASE( test_parallel_read ) +BOOST_AUTO_TEST_CASE( parallel_read ) { boost::filesystem::path path( BBP_TESTDATA ); path /= "circuitBuilding_1000neurons/Functionalizer_output/nrn.h5"; - brion::Synapse synapseFile( path.string( )); + const brion::Synapse synapseFile( path.string( )); brion::uint32_ts connectedNeurons( 100 ); brion::GIDSet gids; @@ -142,13 +141,12 @@ BOOST_AUTO_TEST_CASE( test_parallel_read ) } } -BOOST_AUTO_TEST_CASE( test_read_positions ) +BOOST_AUTO_TEST_CASE( read_positions ) { boost::filesystem::path path( BBP_TESTDATA ); - path /= "circuitBuilding_1000neurons/Functionalizer_output/nrn_positions.h5"; - - brion::Synapse synapseFile( path.string( )); + path /="circuitBuilding_1000neurons/Functionalizer_output/nrn_positions.h5"; + const brion::Synapse synapseFile( path.string( )); const brion::SynapseMatrix& empty = synapseFile.read( 1, brion::SYNAPSE_POSITION_NO_ATTRIBUTES ); BOOST_CHECK_EQUAL( empty.shape()[0], 0 ); @@ -156,7 +154,6 @@ BOOST_AUTO_TEST_CASE( test_read_positions ) const brion::SynapseMatrix& data = synapseFile.read( 1, brion::SYNAPSE_POSITION ); - std::cout << data << std::endl; BOOST_CHECK_EQUAL( data.shape()[0], 77 ); // 77 synapses for GID 1 BOOST_CHECK_EQUAL( data.shape()[1], 13 ); // 19 (==all) synapse attributes BOOST_CHECK_EQUAL( data[0][0], 10 ); @@ -170,7 +167,6 @@ BOOST_AUTO_TEST_CASE( test_read_positions ) const brion::SynapseMatrix& data2 = synapseFile.read( 4, brion::SYNAPSE_POSTSYNAPTIC_SURFACE_Y ); - std::cout << data2 << std::endl; BOOST_CHECK_EQUAL( data2.shape()[0], 41 ); // 41 synapses for GID 4 BOOST_CHECK_EQUAL( data2.shape()[1], 1 ); // 1 synapse attribute BOOST_CHECK_CLOSE( data2[0][0], 2029.24304f, .0003f ); @@ -178,12 +174,12 @@ BOOST_AUTO_TEST_CASE( test_read_positions ) BOOST_CHECK_CLOSE( data2[9][0], 2001.01599f, .0003f ); } -BOOST_AUTO_TEST_CASE( test_getNumSynapses ) +BOOST_AUTO_TEST_CASE( getNumSynapses ) { boost::filesystem::path path( BBP_TESTDATA ); path /= "circuitBuilding_1000neurons/Functionalizer_output/nrn.h5"; - brion::Synapse synapseFile( path.string( )); + const brion::Synapse synapseFile( path.string( )); brion::GIDSet gids; size_t numSynapses = synapseFile.getNumSynapses( gids ); @@ -203,11 +199,11 @@ BOOST_AUTO_TEST_CASE( test_getNumSynapses ) BOOST_CHECK_EQUAL( numSynapses, 1172 ); } -BOOST_AUTO_TEST_CASE( test_perf ) +BOOST_AUTO_TEST_CASE( perf ) { boost::filesystem::path path( BBP_TESTDATA ); path /= "circuitBuilding_1000neurons/Functionalizer_output/nrn.h5"; - brion::Synapse synapseFile( path.string( )); + const brion::Synapse synapseFile( path.string( )); brion::GIDSet gids; for( uint32_t i = 1; i <= 1000; ++ i) @@ -247,3 +243,33 @@ BOOST_AUTO_TEST_CASE( test_perf ) << " synapses for " << gids.size() << " cells took: " << duration.total_milliseconds() << " ms." << std::endl; } + +BOOST_AUTO_TEST_CASE( read_unmerged ) +{ + boost::filesystem::path path( BBP_TESTDATA ); + path /= "local/unmergedSynapses/nrn.h5"; + + const brion::Synapse synapseFile( path.string( )); + const brion::SynapseMatrix& data = synapseFile.read( 1, + brion::SYNAPSE_ALL_ATTRIBUTES ); + BOOST_CHECK_EQUAL( data.shape()[0], 376 ); // synapses for GID 1 + BOOST_CHECK_EQUAL( data.shape()[1], 19 ); // 19 (==all) synapse attributes + BOOST_CHECK_EQUAL( data[0][0], 6 ); + BOOST_CHECK_EQUAL( data[1][0], 6 ); + BOOST_CHECK_EQUAL( data[2][0], 11 ); + BOOST_CHECK_EQUAL( data[3][0], 11 ); + BOOST_CHECK_EQUAL( data[4][0], 12 ); + BOOST_CHECK_EQUAL( data[5][0], 12 ); + BOOST_CHECK_EQUAL( data[6][0], 20 ); + + brion::GIDSet gids; + gids.insert( 1 ); + size_t numSynapses = synapseFile.getNumSynapses( gids ); + BOOST_CHECK_EQUAL( numSynapses, 376 ); + + for( uint32_t i = 2; i <= 10; ++i ) + gids.insert( i ); + + numSynapses = synapseFile.getNumSynapses( gids ); + BOOST_CHECK_EQUAL( numSynapses, 2903 ); +} diff --git a/tests/target.cpp b/tests/target.cpp index 8f16524..647da36 100644 --- a/tests/target.cpp +++ b/tests/target.cpp @@ -1,5 +1,6 @@ /* Copyright (c) 2013-2015, EPFL/Blue Brain Project * Daniel Nachbaur + * Stefan.Eilemann@epfl.ch * * This file is part of Brion * @@ -36,7 +37,7 @@ #include -BOOST_AUTO_TEST_CASE( test_invalid_open ) +BOOST_AUTO_TEST_CASE( invalid_open ) { BOOST_CHECK_THROW( brion::Target( "blub" ), std::runtime_error ); @@ -45,7 +46,7 @@ BOOST_AUTO_TEST_CASE( test_invalid_open ) BOOST_CHECK_THROW( brion::Target( path.string( )), std::runtime_error ); } -BOOST_AUTO_TEST_CASE( test_read ) +BOOST_AUTO_TEST_CASE( get ) { boost::filesystem::path path( BBP_TESTDATA ); path /= "local/circuits/18.10.10_600cell/ncsFunctionalCompare/start.target"; @@ -68,3 +69,13 @@ BOOST_AUTO_TEST_CASE( test_read ) BOOST_CHECK_EQUAL( layer4Target[10], "a279" ); BOOST_CHECK_EQUAL( layer4Target[42], "a311" ); } + +BOOST_AUTO_TEST_CASE( parse ) +{ + boost::filesystem::path path( BBP_TESTDATA ); + path /= "local/circuits/18.10.10_600cell/ncsFunctionalCompare/start.target"; + const brion::Target target( path.string( )); + const brion::GIDSet& column = brion::Target::parse( + brion::Targets( 1, target ), "Column" ); + BOOST_CHECK_EQUAL( column.size(), 600 ); +}