From 138598315ea9374faa220b7ff1e0d4b4220b99bb Mon Sep 17 00:00:00 2001 From: Mohamed-Ghaith Kaabi Date: Wed, 28 Jun 2017 11:00:57 +0200 Subject: [PATCH 1/4] VIZTM-732 : brain.CompartmentReport improvements --- CMakeLists.txt | 2 +- brain/compartmentReport.h | 12 ++++++++++++ brain/compartmentReportView.cpp | 30 ++++++++++++++++++++++++++++++ brain/compartmentReportView.h | 16 ++++++++++++++++ brain/detail/compartmentReport.h | 4 ++++ brain/python/compartmentReport.cpp | 14 ++++++++++++++ brion/compartmentReport.cpp | 5 +++++ brion/compartmentReport.h | 3 +++ brion/compartmentReportPlugin.h | 3 +++ brion/plugin/compartmentReportCommon.cpp | 7 +++++++ brion/plugin/compartmentReportCommon.h | 1 + tests/brain/python/compartmentReport.py | 15 +++++++++++++++ 12 files changed, 111 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 81f54e3..b9f50d7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,7 +5,7 @@ # cmake_minimum_required(VERSION 3.1 FATAL_ERROR) -project(Brion VERSION 2.0.0) +project(Brion VERSION 2.1.0) set(Brion_VERSION_ABI 9) list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/CMake diff --git a/brain/compartmentReport.h b/brain/compartmentReport.h index f3fccdf..6c200ca 100644 --- a/brain/compartmentReport.h +++ b/brain/compartmentReport.h @@ -54,6 +54,18 @@ struct CompartmentReportMetaData /** The data unit of the report */ std::string dataUnit; + + /** The cell count of the report */ + size_t cellCount; + + /** The gids in the report */ + brion::GIDSet gids; + + /** The total frame count in the report */ + size_t frameCount; + + /** The total compartment count in the report */ + size_t compartmentCount; }; /** diff --git a/brain/compartmentReportView.cpp b/brain/compartmentReportView.cpp index c8a6b78..c066a8f 100644 --- a/brain/compartmentReportView.cpp +++ b/brain/compartmentReportView.cpp @@ -99,6 +99,36 @@ std::future CompartmentReportView::load(double start, double end) return _impl->report->loadFrames(start, end); } +std::future CompartmentReportView::load(double start, double end, + double stride) +{ + if (end <= start) + throw std::logic_error("Invalid interval"); + + if (stride < _impl->report->getTimestep()) + throw std::logic_error("Invalid stride"); + + start = std::max(start, _impl->report->getStartTime()); + end = std::min(end, _impl->report->getEndTime()); + + auto task = [this, start, end, stride] { + + brion::Frames frames; + frames.timeStamps.reset(new brion::doubles); + frames.data.reset(new floats); + for (double t = start; t < end; t += stride) + { + auto frame = load(t).get(); + frames.timeStamps->push_back(frame.timestamp); + std::copy(frame.data->begin(), frame.data->end(), + std::back_inserter(*frames.data)); + } + return frames; + }; + + return _impl->readerImpl->threadPool.post(task); +} + std::future CompartmentReportView::loadAll() { return load(_impl->report->getStartTime(), _impl->report->getEndTime()); diff --git a/brain/compartmentReportView.h b/brain/compartmentReportView.h index e807bc4..ff3c856 100644 --- a/brain/compartmentReportView.h +++ b/brain/compartmentReportView.h @@ -78,6 +78,22 @@ class CompartmentReportView */ BRAIN_API std::future load(double start, double end); + /** Load frames between start and end time stamps. + * + * @param start the start time stamp + * @param end the end time stamp + * @param stride the time stride + * @return the frames overlapped by the given time window, spaced by a given + * stride. The start time + * doesn't need to be aligned with the stride and the time + * interval is open on the right. The result may be empty if the + * time window falls out of the report window. + * @throw std::logic_error if invalid interval or stride < timeStep + * @version 2.1 + */ + BRAIN_API std::future load(double start, double end, + double stride); + /** Load all the frames. * This is equivalent to call load(starTime, endTime) * @version 2.0 diff --git a/brain/detail/compartmentReport.h b/brain/detail/compartmentReport.h index b773ec5..aa1c74d 100644 --- a/brain/detail/compartmentReport.h +++ b/brain/detail/compartmentReport.h @@ -42,6 +42,10 @@ struct CompartmentReportReader metaData.timeStep = report.getTimestep(); metaData.timeUnit = report.getTimeUnit(); metaData.dataUnit = report.getDataUnit(); + metaData.gids = report.getGIDs(); + metaData.cellCount = metaData.gids.size(); + metaData.compartmentCount = report.getFrameSize(); + metaData.frameCount = report.getFrameCount(); } const brion::URI uri; diff --git a/brain/python/compartmentReport.cpp b/brain/python/compartmentReport.cpp index 78cbfd7..0dd55de 100644 --- a/brain/python/compartmentReport.cpp +++ b/brain/python/compartmentReport.cpp @@ -90,6 +90,10 @@ bp::object CompartmentReport_getMetaData(const CompartmentReport& reader) dict["time_step"] = md.timeStep; dict["time_unit"] = md.timeUnit; dict["data_unit"] = md.dataUnit; + dict["num_cells"] = md.cellCount; + dict["num_compartments"] = md.compartmentCount; + dict["num_frames"] = md.frameCount; + dict["gids"] = toNumpy(toVector(md.gids)); return dict; } @@ -117,6 +121,13 @@ bp::object CompartmentReportView_load(CompartmentReportView& view, return framesToTuple(view.load(start, end).get()); } +bp::object CompartmentReportView_load2(CompartmentReportView& view, + const double start, const double end, + const double stride) +{ + return framesToTuple(view.load(start, end, stride).get()); +} + bp::object CompartmentReportView_loadAll(CompartmentReportView& view) { return framesToTuple(view.loadAll().get()); @@ -195,6 +206,9 @@ bp::class_( .def("load", CompartmentReportView_load, (selfarg, bp::arg("start"), bp::arg("end")), DOXY_FN(brain::CompartmentReportView::load(double,double))) + .def("load", CompartmentReportView_load2, + (selfarg, bp::arg("start"), bp::arg("end"), bp::arg("stride")), + DOXY_FN(brain::CompartmentReportView::load(double,double,double))) .def("load_all", CompartmentReportView_loadAll, (selfarg), DOXY_FN(brain::CompartmentReportView::loadAll)); } diff --git a/brion/compartmentReport.cpp b/brion/compartmentReport.cpp index 21e13c9..87116be 100644 --- a/brion/compartmentReport.cpp +++ b/brion/compartmentReport.cpp @@ -119,6 +119,11 @@ size_t CompartmentReport::getFrameSize() const return _impl->plugin->getFrameSize(); } +size_t CompartmentReport::getFrameCount() const +{ + return _impl->plugin->getFrameCount(); +} + size_t CompartmentReport::getBufferSize() const { return _impl->plugin->getBufferSize(); diff --git a/brion/compartmentReport.h b/brion/compartmentReport.h index 40224e0..6ea74fc 100644 --- a/brion/compartmentReport.h +++ b/brion/compartmentReport.h @@ -140,6 +140,9 @@ class CompartmentReport : public boost::noncopyable /** @return the number of values of a loaded report frame. @version 1.0 */ BRION_API size_t getFrameSize() const; + /** @return the total frame count in the report. @version 2.1 */ + BRION_API size_t getFrameCount() const; + /** Load report values at the given time stamp. * * @param timestamp the time stamp of interest diff --git a/brion/compartmentReportPlugin.h b/brion/compartmentReportPlugin.h index abd23de..91f5410 100644 --- a/brion/compartmentReportPlugin.h +++ b/brion/compartmentReportPlugin.h @@ -123,6 +123,9 @@ class CompartmentReportPlugin : public boost::noncopyable /** @copydoc brion::CompartmentReport::getFrameSize */ virtual size_t getFrameSize() const = 0; + /** @copydoc brion::CompartmentReport::getFrameCount */ + virtual size_t getFrameCount() const = 0; + /** @copydoc brion::CompartmentReport::loadFrame */ virtual floatsPtr loadFrame(double timestamp) const = 0; diff --git a/brion/plugin/compartmentReportCommon.cpp b/brion/plugin/compartmentReportCommon.cpp index 79e23ab..fcb6d36 100644 --- a/brion/plugin/compartmentReportCommon.cpp +++ b/brion/plugin/compartmentReportCommon.cpp @@ -61,6 +61,13 @@ size_t CompartmentReportCommon::_getFrameNumber(double timestamp) const return size_t(timestamp / step); } +size_t CompartmentReportCommon::getFrameCount() const +{ + if (getStartTime() < getEndTime()) + return _getFrameNumber(getEndTime()) + 1; + return 0; +} + GIDSet CompartmentReportCommon::_computeIntersection(const GIDSet& all, const GIDSet& subset) { diff --git a/brion/plugin/compartmentReportCommon.h b/brion/plugin/compartmentReportCommon.h index 78afb19..402c104 100644 --- a/brion/plugin/compartmentReportCommon.h +++ b/brion/plugin/compartmentReportCommon.h @@ -37,6 +37,7 @@ class CompartmentReportCommon : public CompartmentReportPlugin floatsPtr loadFrame(double timestamp) const final; Frames loadFrames(double start, double end) const final; + size_t getFrameCount() const final; protected: void _cacheNeuronCompartmentCounts(const GIDSet& gids); diff --git a/tests/brain/python/compartmentReport.py b/tests/brain/python/compartmentReport.py index 618153f..94acaf9 100644 --- a/tests/brain/python/compartmentReport.py +++ b/tests/brain/python/compartmentReport.py @@ -37,6 +37,10 @@ def test_metadata(self): assert(metadata['start_time'] == 0.0) assert(metadata['end_time'] == 10.0) assert(numpy.isclose(metadata['time_step'], 0.1)) + assert(metadata['num_compartments'] == 600) + assert(metadata['num_frames'] == 100) + assert(metadata['num_cells'] == 600) + assert((metadata['gids'] == numpy.arange(1, 601, 1)).all()) def test_create_view(self): @@ -60,6 +64,7 @@ def test_frames(self): assert(numpy.isclose(frames, [[-65., -65., -65.], [-65.14350891, -65.29447937, -65.44480133]]).all()) + #### load(start,end) timestamps, frames = view.load(0.05,0.25) # This window overlaps frames [0, 0.1), [0.1, 0.2), [0.2, 03) assert(len(timestamps) == 3) @@ -69,6 +74,15 @@ def test_frames(self): # falling on a different frame, so we get two frames. assert(len(timestamps) == 2) assert(frames.shape == (2, 3)) + + + ### load(start,end,stride) + timestamps, frames = view.load(0.0,1.0,0.2) + assert(len(timestamps) == 5) + assert(numpy.isclose(timestamps,[ 0.0, 0.2, 0.4, 0.6, 0.8]).all()) + assert(frames.shape == (5,3)) + + #### load_all() timestamps, frames = view.load_all() assert(len(timestamps) == 100) timestamp, frame = view.load(0.1) @@ -84,6 +98,7 @@ def test_frames(self): assert(frame.shape == (3,)) assert((frame == [-65.14350891113281, -65.29447937011719, -65.4448013305664]).all()) + # single neuron view = self.report.create_view({1}) From 6b19548b4642f291c14d3616ff2280c5644ea684 Mon Sep 17 00:00:00 2001 From: Mohamed-Ghaith Kaabi Date: Fri, 14 Jul 2017 13:56:21 +0200 Subject: [PATCH 2/4] VIZTM-732 : brain.CompartmentReport improvements. CR1 --- brain/compartmentReportView.cpp | 28 +++++++++++++++++++++++----- brain/compartmentReportView.h | 13 ++++++------- brain/python/compartmentReport.cpp | 6 +++--- tests/brain/compartmentReport.cpp | 28 ++++++++++++++++++++++++++++ tests/brain/python/compartmentReport.py | 24 +++++++++++++++++++++--- 5 files changed, 81 insertions(+), 18 deletions(-) diff --git a/brain/compartmentReportView.cpp b/brain/compartmentReportView.cpp index c066a8f..234d122 100644 --- a/brain/compartmentReportView.cpp +++ b/brain/compartmentReportView.cpp @@ -100,29 +100,47 @@ std::future CompartmentReportView::load(double start, double end) } std::future CompartmentReportView::load(double start, double end, - double stride) + const double step) { + const double reportTimeStep = _impl->report->getTimestep(); + const double reportStartTime = _impl->report->getStartTime(); + if (end <= start) throw std::logic_error("Invalid interval"); - if (stride < _impl->report->getTimestep()) - throw std::logic_error("Invalid stride"); + if (step < reportTimeStep || step <= 0.) + throw std::logic_error("Invalid step"); + + // check step is multiple of timestep + if (fmod(step, reportTimeStep) != 0.0) + throw std::logic_error( + "Step should be amultiple of the report time step"); + /// check with Juan start = std::max(start, _impl->report->getStartTime()); + size_t frameIndex = (start - reportStartTime) / reportTimeStep; + start = (frameIndex + 0.5) * reportTimeStep + reportStartTime; + end = std::min(end, _impl->report->getEndTime()); - auto task = [this, start, end, stride] { + auto task = [this, start, end, step] { brion::Frames frames; frames.timeStamps.reset(new brion::doubles); frames.data.reset(new floats); - for (double t = start; t < end; t += stride) + + double t = start; + uint32_t i = 0; + while (t < end) { auto frame = load(t).get(); frames.timeStamps->push_back(frame.timestamp); std::copy(frame.data->begin(), frame.data->end(), std::back_inserter(*frames.data)); + ++i; + t = start + i * step; } + return frames; }; diff --git a/brain/compartmentReportView.h b/brain/compartmentReportView.h index ff3c856..d4399d8 100644 --- a/brain/compartmentReportView.h +++ b/brain/compartmentReportView.h @@ -80,19 +80,18 @@ class CompartmentReportView /** Load frames between start and end time stamps. * - * @param start the start time stamp + * @param start the start time stamp with a time step * @param end the end time stamp - * @param stride the time stride + * @param step the time step * @return the frames overlapped by the given time window, spaced by a given - * stride. The start time - * doesn't need to be aligned with the stride and the time - * interval is open on the right. The result may be empty if the - * time window falls out of the report window. + * stride. The start time doesn't need to be aligned with the stride + * and the time interval is open on the right. The result may be + * empty if the time window falls out of the report window. * @throw std::logic_error if invalid interval or stride < timeStep * @version 2.1 */ BRAIN_API std::future load(double start, double end, - double stride); + double step); /** Load all the frames. * This is equivalent to call load(starTime, endTime) diff --git a/brain/python/compartmentReport.cpp b/brain/python/compartmentReport.cpp index 0dd55de..b9e1850 100644 --- a/brain/python/compartmentReport.cpp +++ b/brain/python/compartmentReport.cpp @@ -90,9 +90,9 @@ bp::object CompartmentReport_getMetaData(const CompartmentReport& reader) dict["time_step"] = md.timeStep; dict["time_unit"] = md.timeUnit; dict["data_unit"] = md.dataUnit; - dict["num_cells"] = md.cellCount; - dict["num_compartments"] = md.compartmentCount; - dict["num_frames"] = md.frameCount; + dict["cell_count"] = md.cellCount; + dict["compartment_count"] = md.compartmentCount; + dict["frame_count"] = md.frameCount; dict["gids"] = toNumpy(toVector(md.gids)); return dict; diff --git a/tests/brain/compartmentReport.cpp b/tests/brain/compartmentReport.cpp index 16c5d7d..07c5d4d 100644 --- a/tests/brain/compartmentReport.cpp +++ b/tests/brain/compartmentReport.cpp @@ -237,6 +237,34 @@ void testReadRange(const char* relativePath) BOOST_CHECK_EQUAL((*frames.timeStamps)[2], start + 2 * step); } +void testReadStep(const char* relativePath) +{ + boost::filesystem::path path(BBP_TESTDATA); + path /= relativePath; + + brion::GIDSet gids; + gids.insert(394); + gids.insert(400); + + brain::CompartmentReport report(brion::URI(path.string())); + auto view = report.createView(gids); + + const double start = report.getMetaData().startTime; + const double step = report.getMetaData().timeStep; + + auto frames = view.load(start, start + step * 4, step * 2).get(); + BOOST_REQUIRE_EQUAL(frames.timeStamps->size(), 2); + + BOOST_CHECK_THROW(view.load(start, start - step * 4, step), + std::logic_error); + BOOST_CHECK_THROW(view.load(start, start + step * 4, -1.), + std::logic_error); + BOOST_CHECK_THROW(view.load(start, start + step * 4, step * .5), + std::logic_error); + BOOST_CHECK_THROW(view.load(start, start + step * 4, step * 1.5), + std::logic_error); +} + BOOST_AUTO_TEST_CASE(read_binary) { testRead("local/simulations/may17_2011/Control/allCompartments.bbp"); diff --git a/tests/brain/python/compartmentReport.py b/tests/brain/python/compartmentReport.py index 94acaf9..5a9ad0a 100644 --- a/tests/brain/python/compartmentReport.py +++ b/tests/brain/python/compartmentReport.py @@ -25,6 +25,24 @@ from brain import * report_path = brain.test.root_data_path + "/local/simulations/may17_2011/Control/voltage.h5" +all_compartments_report_path = brain.test.root_data_path + "/local/simulations/may17_2011/Control/allCompartments.h5" + + +class TestMetaData(unittest.TestCase): + def setUp(self): + self.report = CompartmentReport(all_compartments_report_path) + + def test_metadata(self): + metadata = self.report.metadata + assert(metadata['data_unit'] == 'mV') + assert(metadata['time_unit'] == 'ms') + assert(metadata['start_time'] == 0.0) + assert(metadata['end_time'] == 10.0) + assert(numpy.isclose(metadata['time_step'], 0.1)) + assert(metadata['compartment_count'] == 20360) + assert(metadata['frame_count'] == 100) + assert(metadata['cell_count'] == 35) + class TestReader(unittest.TestCase): def setUp(self): @@ -37,9 +55,9 @@ def test_metadata(self): assert(metadata['start_time'] == 0.0) assert(metadata['end_time'] == 10.0) assert(numpy.isclose(metadata['time_step'], 0.1)) - assert(metadata['num_compartments'] == 600) - assert(metadata['num_frames'] == 100) - assert(metadata['num_cells'] == 600) + assert(metadata['compartment_count'] == 600) + assert(metadata['frame_count'] == 100) + assert(metadata['cell_count'] == 600) assert((metadata['gids'] == numpy.arange(1, 601, 1)).all()) From 32517110fb24ff8bc5e367736ce2c36df549efb7 Mon Sep 17 00:00:00 2001 From: Mohamed-Ghaith Kaabi Date: Fri, 14 Jul 2017 14:07:44 +0200 Subject: [PATCH 3/4] VIZTM-732 : brain.CompartmentReport improvements. CR1 --- brain/compartmentReport.h | 5 +++++ brain/compartmentReportView.cpp | 2 +- brain/compartmentReportView.h | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/brain/compartmentReport.h b/brain/compartmentReport.h index 6c200ca..7c222f3 100644 --- a/brain/compartmentReport.h +++ b/brain/compartmentReport.h @@ -89,8 +89,13 @@ class CompartmentReport BRAIN_API CompartmentReport(const URI& uri); BRAIN_API ~CompartmentReport(); +#ifndef DOXYGEN_TO_BREATHE /** @return the metadata of the report */ BRAIN_API const CompartmentReportMetaData& getMetaData() const; +#else + /** @return the metadata of the report */ + dict getMetaData() const; +#endif /** * Create a view of a subset of neurons. An empty gid diff --git a/brain/compartmentReportView.cpp b/brain/compartmentReportView.cpp index 234d122..f570cef 100644 --- a/brain/compartmentReportView.cpp +++ b/brain/compartmentReportView.cpp @@ -114,7 +114,7 @@ std::future CompartmentReportView::load(double start, double end, // check step is multiple of timestep if (fmod(step, reportTimeStep) != 0.0) throw std::logic_error( - "Step should be amultiple of the report time step"); + "Step should be a multiple of the report time step"); /// check with Juan start = std::max(start, _impl->report->getStartTime()); diff --git a/brain/compartmentReportView.h b/brain/compartmentReportView.h index d4399d8..8753e6a 100644 --- a/brain/compartmentReportView.h +++ b/brain/compartmentReportView.h @@ -84,7 +84,7 @@ class CompartmentReportView * @param end the end time stamp * @param step the time step * @return the frames overlapped by the given time window, spaced by a given - * stride. The start time doesn't need to be aligned with the stride + * step. The start time doesn't need to be aligned with the step * and the time interval is open on the right. The result may be * empty if the time window falls out of the report window. * @throw std::logic_error if invalid interval or stride < timeStep From 7de9169edc0abab0f92fa239029d32cab72cb2d4 Mon Sep 17 00:00:00 2001 From: Mohamed-Ghaith Kaabi Date: Fri, 14 Jul 2017 16:13:30 +0200 Subject: [PATCH 4/4] VIZTM-732 : brain.CompartmentReport improvements. CR2 --- brain/compartmentReport.h | 13 ++++++++++++- brain/compartmentReportView.cpp | 6 ++++-- brain/compartmentReportView.h | 3 ++- tests/brain/python/compartmentReport.py | 2 +- 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/brain/compartmentReport.h b/brain/compartmentReport.h index 7c222f3..de10362 100644 --- a/brain/compartmentReport.h +++ b/brain/compartmentReport.h @@ -93,7 +93,18 @@ class CompartmentReport /** @return the metadata of the report */ BRAIN_API const CompartmentReportMetaData& getMetaData() const; #else - /** @return the metadata of the report */ + /** @return the metadata of the report + * - *start_time* (Numeric) : The start time of the report + * - *end_time* (Numeric) : The end time of the report + * - *time_step* (Numeric) : The sampling time interval of the report + * - *time_unit* (String) : The time unit of the report + * - *data_unit* (String) : The data unit of the report + * - *cell_count* (Numeric) : The cell count of the report + * - *compartment_count* (Numeric) : The total compartment count in the + * report + * - *frame_count* (Numeric) : The total frame count in the report + * - *gids* (Vector) : The gids in the report + */ dict getMetaData() const; #endif diff --git a/brain/compartmentReportView.cpp b/brain/compartmentReportView.cpp index f570cef..f7c8869 100644 --- a/brain/compartmentReportView.cpp +++ b/brain/compartmentReportView.cpp @@ -112,11 +112,13 @@ std::future CompartmentReportView::load(double start, double end, throw std::logic_error("Invalid step"); // check step is multiple of timestep - if (fmod(step, reportTimeStep) != 0.0) + if (fmod(step, reportTimeStep) > std::numeric_limits::epsilon()) throw std::logic_error( "Step should be a multiple of the report time step"); - /// check with Juan + // Making sure the timestamps we are going to request always fall in the + // middle of a frame. For that we snap start to the beginning of the frame + // it's contained and then we add half the time step. start = std::max(start, _impl->report->getStartTime()); size_t frameIndex = (start - reportStartTime) / reportTimeStep; start = (frameIndex + 0.5) * reportTimeStep + reportStartTime; diff --git a/brain/compartmentReportView.h b/brain/compartmentReportView.h index 8753e6a..0ab8d25 100644 --- a/brain/compartmentReportView.h +++ b/brain/compartmentReportView.h @@ -87,7 +87,8 @@ class CompartmentReportView * step. The start time doesn't need to be aligned with the step * and the time interval is open on the right. The result may be * empty if the time window falls out of the report window. - * @throw std::logic_error if invalid interval or stride < timeStep + * @throw std::logic_error if invalid interval or step < timeStep or step is + * not a multiple of timeStep * @version 2.1 */ BRAIN_API std::future load(double start, double end, diff --git a/tests/brain/python/compartmentReport.py b/tests/brain/python/compartmentReport.py index 5a9ad0a..dd01649 100644 --- a/tests/brain/python/compartmentReport.py +++ b/tests/brain/python/compartmentReport.py @@ -94,7 +94,7 @@ def test_frames(self): assert(frames.shape == (2, 3)) - ### load(start,end,stride) + ### load(start,end,step) timestamps, frames = view.load(0.0,1.0,0.2) assert(len(timestamps) == 5) assert(numpy.isclose(timestamps,[ 0.0, 0.2, 0.4, 0.6, 0.8]).all())