From 9ddbac64c81ffcb7c643f75883413a4d7734d3aa Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Thu, 4 Sep 2014 14:12:46 +0200 Subject: [PATCH 01/99] First incompomplete skectch of apossible cartesian communicator. --- include/boost/mpi/cartesian_communicator.hpp | 125 +++++++++++++++++++++++++++ src/cartesian_communicator.cpp | 27 ++++++ test/cartesian_topology_test.cpp | 33 +++++++ 3 files changed, 185 insertions(+) create mode 100644 include/boost/mpi/cartesian_communicator.hpp create mode 100644 src/cartesian_communicator.cpp create mode 100644 test/cartesian_topology_test.cpp diff --git a/include/boost/mpi/cartesian_communicator.hpp b/include/boost/mpi/cartesian_communicator.hpp new file mode 100644 index 0000000..d3da227 --- /dev/null +++ b/include/boost/mpi/cartesian_communicator.hpp @@ -0,0 +1,125 @@ +// Copyright (C) 2007 Trustees of Indiana University + +// Authors: Alain Miniussi + +// Use, modification and distribution is subject to the Boost Software +// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +/** @file cartesian_communicator.hpp + * + * This header defines facilities to support MPI communicators with + * cartesian topologies. + * If known at compiled time, the dimension of the implied grid + * can be statically enforced, through the templatized communicator + * class. Otherwise, a non template, dynamic, base class is provided. + * + */ +#ifndef BOOST_MPI_CARTESIAN_COMMUNICATOR_HPP +#define BOOST_MPI_CARTESIAN_COMMUNICATOR_HPP + +#include + +#include +#include + +// Headers required to implement cartesian topologies +#include +#include + +namespace boost { namespace mpi { + +/** + * @brief An MPI communicator with a cartesian topology. + * + * A @c cartesian_communicator is a communicator whose topology is + * expressed as a grid. Cartesian communicators have the same + * functionality as (intra)communicators, but also allow one to query + * the relationships among processes and the properties of the grid. + */ +class BOOST_MPI_DECL cartesian_communicator : public communicator +{ + friend class communicator; + + /** + * INTERNAL ONLY + * + * Construct a cartesian communicator given a shared pointer to the + * underlying MPI_Comm (which must have a cartesian topology). + * This operation is used for "casting" from a communicator to + * a cartesian communicator. + */ + explicit cartesian_communicator(const shared_ptr& comm_ptr) + : communicator() + { + this->comm_ptr = comm_ptr; + BOOST_ASSERT(has_cartesian_topology()); + } + +public: + /** + * Build a new Boost.MPI cartesian communicator based on the MPI + * communicator @p comm with cartesian topology. + * + * @p comm may be any valid MPI communicator. If @p comm is + * MPI_COMM_NULL, an empty communicator (that cannot be used for + * communication) is created and the @p kind parameter is + * ignored. Otherwise, the @p kind parameter determines how the + * Boost.MPI communicator will be related to @p comm: + * + * - If @p kind is @c comm_duplicate, duplicate @c comm to create + * a new communicator. This new communicator will be freed when + * the Boost.MPI communicator (and all copies of it) is + * destroyed. This option is only permitted if the underlying MPI + * implementation supports MPI 2.0; duplication of + * intercommunicators is not available in MPI 1.x. + * + * - If @p kind is @c comm_take_ownership, take ownership of @c + * comm. It will be freed automatically when all of the Boost.MPI + * communicators go out of scope. + * + * - If @p kind is @c comm_attach, this Boost.MPI communicator + * will reference the existing MPI communicator @p comm but will + * not free @p comm when the Boost.MPI communicator goes out of + * scope. This option should only be used when the communicator is + * managed by the user. + */ + cartesian_communicator(const MPI_Comm& comm, comm_create_kind kind) + : communicator(comm, kind) + { + BOOST_ASSERT(has_cartesian_topology()); + } + + /** + * Create a new communicator whose topology is described by the + * given cartesian. The indices of the vertices in the cartesian will be + * assumed to be the ranks of the processes within the + * communicator. There may be fewer vertices in the cartesian than + * there are processes in the communicator; in this case, the + * resulting communicator will be a NULL communicator. + * + * @param comm The communicator that the new, cartesian communicator + * will be based on. + * + * @param cartesian Any type that meets the requirements of the + * Incidence Cartesian and Vertex List Cartesian concepts from the Boost Cartesian + * Library. This structure of this cartesian will become the topology + * of the communicator that is returned. + * + * @param reorder Whether MPI is permitted to re-order the process + * ranks within the returned communicator, to better optimize + * communication. If false, the ranks of each process in the + * returned process will match precisely the rank of that process + * within the original communicator. + */ + cartesian_communicator(const communicator& comm, + const std::vector& dims, + const std::vector& periodic, + bool reorder = false); +}; + +} } // end namespace boost::mpi + + + +#endif // BOOST_MPI_CARTESIAN_COMMUNICATOR_HPP diff --git a/src/cartesian_communicator.cpp b/src/cartesian_communicator.cpp new file mode 100644 index 0000000..b46efda --- /dev/null +++ b/src/cartesian_communicator.cpp @@ -0,0 +1,27 @@ +// Copyright (C) 2014 Alain Miniussi. + +// Use, modification and distribution is subject to the Boost Software +// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) +#include + +namespace boost { namespace mpi { + +cartesian_communicator::cartesian_communicator(const communicator& comm, + const std::vector& dims, + const std::vector& periodic, + bool reorder ) + : communicator() +{ + BOOST_ASSERT(dims.size() == periodic.size()); + MPI_Comm newcomm; + std::vector p(periodic.begin(), periodic.end()); + BOOST_MPI_CHECK_RESULT(MPI_Cart_create, + ((MPI_Comm)comm, dims.size(), + const_cast(dims.data()), p.data(), + int(reorder), &newcomm)); + if(newcomm != MPI_COMM_NULL) { + comm_ptr.reset(new MPI_Comm(newcomm), comm_free()); + } +} +} } // end namespace boost::mpi diff --git a/test/cartesian_topology_test.cpp b/test/cartesian_topology_test.cpp new file mode 100644 index 0000000..6b62a58 --- /dev/null +++ b/test/cartesian_topology_test.cpp @@ -0,0 +1,33 @@ +// Copyright (C) 2007 Trustees of Indiana University + +// Authors: Alain Miniussi + +// Use, modification and distribution is subject to the Boost Software +// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +#include + +#include +#include +#include + +#include + +namespace mpi = boost::mpi; + +int test_main(int argc, char* argv[]) +{ + mpi::environment env(argc, argv); + + mpi::communicator world; + std::vector dims(3); + std::vector periodic(3); + dims[0] = 2; dims[1] = 3; dims[2] = 4; + periodic[0] = true; periodic[1] = false; periodic[2] = true; + + mpi::cartesian_communicator cc(world, dims, periodic, true); + BOOST_CHECK(cc.has_cartesian_topology()); + + return 0; +} From a1c6e750cb3e6a2fea21f255f8930511bb9784b3 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Thu, 11 Sep 2014 14:46:22 +0200 Subject: [PATCH 02/99] Added first tentative implementation of cartesian_communicator. The API should be complete although primitive. No documentation yet. --- build/Jamfile.v2 | 1 + include/boost/mpi/cartesian_communicator.hpp | 81 ++++++++++++++++--- src/cartesian_communicator.cpp | 95 ++++++++++++++++++++-- test/Jamfile.v2 | 3 +- test/cartesian_topology_test.cpp | 114 +++++++++++++++++++++++++-- 5 files changed, 272 insertions(+), 22 deletions(-) diff --git a/build/Jamfile.v2 b/build/Jamfile.v2 index 6a48582..c6138b7 100644 --- a/build/Jamfile.v2 +++ b/build/Jamfile.v2 @@ -43,6 +43,7 @@ rule tag ( name : type ? : property-set ) lib boost_mpi : broadcast.cpp + cartesian_communicator.cpp communicator.cpp computation_tree.cpp content_oarchive.cpp diff --git a/include/boost/mpi/cartesian_communicator.hpp b/include/boost/mpi/cartesian_communicator.hpp index d3da227..a722648 100644 --- a/include/boost/mpi/cartesian_communicator.hpp +++ b/include/boost/mpi/cartesian_communicator.hpp @@ -1,10 +1,10 @@ -// Copyright (C) 2007 Trustees of Indiana University -// Authors: Alain Miniussi +// Copyright Alain Miniussi 2014. +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) -// Use, modification and distribution is subject to the Boost Software -// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at -// http://www.boost.org/LICENSE_1_0.txt) +// Authors: Alain Miniussi /** @file cartesian_communicator.hpp * @@ -29,6 +29,8 @@ namespace boost { namespace mpi { +class cartesian_topology; + /** * @brief An MPI communicator with a cartesian topology. * @@ -101,10 +103,12 @@ class BOOST_MPI_DECL cartesian_communicator : public communicator * @param comm The communicator that the new, cartesian communicator * will be based on. * - * @param cartesian Any type that meets the requirements of the - * Incidence Cartesian and Vertex List Cartesian concepts from the Boost Cartesian - * Library. This structure of this cartesian will become the topology - * of the communicator that is returned. + * @param dims the dimension of the new communicator. The size indicate + * the number of dimension. Some value can be set to zero, in which case + * the corresponding dimension value is left to the system. + * + * @param periodic must be the same size as dims. Each value indicate if + * the corresponding dimension is cyclic. * * @param reorder Whether MPI is permitted to re-order the process * ranks within the returned communicator, to better optimize @@ -116,6 +120,65 @@ class BOOST_MPI_DECL cartesian_communicator : public communicator const std::vector& dims, const std::vector& periodic, bool reorder = false); + + /** + * Create a new cartesian communicator whose topology is a subset of + * an existing cartesian cimmunicator. + * @param comm the original communicator. + * @param keep and array containiing the dimension to keep from the existing + * communicator. + */ + cartesian_communicator(const cartesian_communicator& comm, + const std::vector& keep ); + + using communicator::rank; + + /** + * Retrive the number of dimension of the underlying toppology. + */ + int ndims() const; + + /** + * Return the rank of the process at the given coordinates. + * @param coords the coordinates. the size must match the communicator's topology. + */ + int rank(std::vector const& coords) const; + /** + * Provides the coordinates of the process with the given rank. + * @param rk the ranks in this communicator. + * @param cbuf a buffer were to store the coordinates. + * @returns a reference to cbuf. + */ + std::vector& coords(int rk, std::vector& cbuf) const; + /** + * Provides the coordinates of the process with the given rank. + * @param rk the ranks in this communicator. + * @returns the coordinates. + */ + std::vector coords(int rk) const; + /** + * Retrieve the topology. + * + */ + void topology( std::vector& dims, + std::vector& periodic, + std::vector& coords ) const; +}; + +std::vector& cartesian_dimensions(int sz, std::vector& dims); + +inline +std::vector& cartesian_dimensions(communicator const& comm, std::vector& dims) { + return cartesian_dimensions(comm.size(), dims); +} + +class BOOST_MPI_DECL cartesian_topology { + public: + cartesian_topology(cartesian_communicator const& comm); + + private: + cartesian_communicator const& comm_ref; + }; } } // end namespace boost::mpi diff --git a/src/cartesian_communicator.cpp b/src/cartesian_communicator.cpp index b46efda..03d5652 100644 --- a/src/cartesian_communicator.cpp +++ b/src/cartesian_communicator.cpp @@ -1,8 +1,13 @@ -// Copyright (C) 2014 Alain Miniussi. -// Use, modification and distribution is subject to the Boost Software -// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at -// http://www.boost.org/LICENSE_1_0.txt) +// Copyright Alain Miniussi 2014. +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +// Authors: Alain Miniussi + +#include + #include namespace boost { namespace mpi { @@ -15,13 +20,93 @@ cartesian_communicator::cartesian_communicator(const communicator& comm, { BOOST_ASSERT(dims.size() == periodic.size()); MPI_Comm newcomm; + std::vector local_dims = dims; + // Fill the gaps, if any + if (std::count(local_dims.begin(), local_dims.end(), 0) > 0) { + cartesian_dimensions(comm, local_dims); + } std::vector p(periodic.begin(), periodic.end()); BOOST_MPI_CHECK_RESULT(MPI_Cart_create, ((MPI_Comm)comm, dims.size(), - const_cast(dims.data()), p.data(), + local_dims.data(), p.data(), int(reorder), &newcomm)); if(newcomm != MPI_COMM_NULL) { comm_ptr.reset(new MPI_Comm(newcomm), comm_free()); } } + +cartesian_communicator::cartesian_communicator(const cartesian_communicator& comm, + const std::vector& keep ) + : communicator() +{ + int max_dims = comm.ndims(); + BOOST_ASSERT(keep.size() <= max_dims); + std::vector bitset(max_dims, int(false)); + for(int i = 0; i < keep.size(); ++i) { + BOOST_ASSERT(keep[i] < max_dims); + bitset[keep[i]] = true; + } + + MPI_Comm newcomm; + BOOST_MPI_CHECK_RESULT(MPI_Cart_sub, + ((MPI_Comm)comm, bitset.data(), &newcomm)); + if(newcomm != MPI_COMM_NULL) { + comm_ptr.reset(new MPI_Comm(newcomm), comm_free()); + } +} + +int +cartesian_communicator::ndims() const { + int n = -1; + BOOST_MPI_CHECK_RESULT(MPI_Cartdim_get, + (MPI_Comm(*this), &n)); + return n; +} + +int +cartesian_communicator::rank(std::vector const& coords ) const { + int r = -1; + BOOST_ASSERT(coords.size() == ndims()); + BOOST_MPI_CHECK_RESULT(MPI_Cart_rank, + (MPI_Comm(*this), const_cast&>(coords).data(), + &r)); + return r; +} + +std::vector& +cartesian_communicator::coords(int rk, std::vector& cbuf) const { + cbuf.resize(ndims()); + BOOST_MPI_CHECK_RESULT(MPI_Cart_coords, + (MPI_Comm(*this), rk, cbuf.size(), cbuf.data() )); + return cbuf; +} + +std::vector +cartesian_communicator::coords(int rk) const { + std::vector coords; + this->coords(rk, coords); + return coords; +} + +void +cartesian_communicator::topology( std::vector& dims, + std::vector& periodic, + std::vector& coords ) const { + int ndims = this->ndims(); + dims.resize(ndims); + periodic.resize(ndims); + coords.resize(ndims); + std::vector cperiods(ndims); + BOOST_MPI_CHECK_RESULT(MPI_Cart_get, + (MPI_Comm(*this), ndims, dims.data(), cperiods.data(), coords.data())); + std::copy(cperiods.begin(), cperiods.end(), periodic.begin()); +} + +std::vector& +cartesian_dimensions(int sz, std::vector& dims) { + BOOST_MPI_CHECK_RESULT(MPI_Dims_create, + (sz, dims.size(), dims.data())); + return dims; +} + } } // end namespace boost::mpi diff --git a/test/Jamfile.v2 b/test/Jamfile.v2 index 7ee205b..d5fabb4 100644 --- a/test/Jamfile.v2 +++ b/test/Jamfile.v2 @@ -29,7 +29,7 @@ test-suite mpi [ mpi-test mt_init_test-serialized : mt_init_test.cpp : "serialized" : 1 4 ] [ mpi-test mt_init_test-multiple : mt_init_test.cpp : "multiple" : 1 4 ] # Note: Microsoft MPI fails nonblocking_test on 1 processor - [ mpi-test nonblocking_test ] +# [ mpi-test nonblocking_test ] [ mpi-test reduce_test ] [ mpi-test ring_test : : : 2 3 4 7 8 13 17 ] [ mpi-test scan_test ] @@ -37,6 +37,7 @@ test-suite mpi # Note: Microsoft MPI fails all skeleton-content tests [ mpi-test skeleton_content_test : : : 2 3 4 7 8 13 17 ] [ mpi-test graph_topology_test : : : 2 7 13 ] + [ mpi-test cartesian_topology_test : : : 24 ] [ mpi-test pointer_test : : : 2 ] [ mpi-test groups_test ] ; diff --git a/test/cartesian_topology_test.cpp b/test/cartesian_topology_test.cpp index 6b62a58..4143713 100644 --- a/test/cartesian_topology_test.cpp +++ b/test/cartesian_topology_test.cpp @@ -1,14 +1,20 @@ -// Copyright (C) 2007 Trustees of Indiana University -// Authors: Alain Miniussi +// Copyright Alain Miniussi 2014. +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) -// Use, modification and distribution is subject to the Boost Software -// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at -// http://www.boost.org/LICENSE_1_0.txt) +// Authors: Alain Miniussi #include +#include +#include +#include +#include +#include #include +#include #include #include @@ -16,6 +22,79 @@ namespace mpi = boost::mpi; +struct bvecmin { + std::vector operator()(std::vector const& v1, + std::vector const& v2) const { + BOOST_ASSERT(v1.size() == v2.size()); + std::vector res(v1.size()); + std::transform(v1.begin(), v1.end(), v2.begin(), res.begin(), + std::logical_and()); + return res; + } +}; + +std::string topology_description( std::vector const& dims, + std::vector const& periodic ) { + std::ostringstream out; + for(int i = 0; i < dims.size(); ++i) { + out << "(" << dims[i] << ','; + if (periodic[i]) { + out << "cyclic"; + } else { + out << "bounded"; + } + out << ") "; + } + out << std::flush; + return out.str(); +} + +// Check that everyone agrees on the coordinates +void test_coordinates_consistency( mpi::cartesian_communicator const& cc, + std::vector const& coords ) +{ + for(int p = 0; p < cc.size(); ++p) { + std::vector min(cc.ndims()); + std::vector local(cc.coords(p)); + mpi::reduce(cc, local.data(), local.size(), + min.data(), mpi::minimum(), p); + if (p == cc.rank()) { + BOOST_CHECK(std::equal(coords.begin(), coords.end(), + min.begin())); + std::ostringstream out; + out << "proc " << p << " at ("; + std::copy(min.begin(), min.end(), std::ostream_iterator(out, " ")); + out << ")\n"; + std::cout << out.str(); + } + } +} + +void test_topology_consistency( mpi::cartesian_communicator const& cc) +{ + std::vector idims(cc.ndims()); + std::vector iperiodic(cc.ndims()); + std::vector odims(cc.ndims()); + std::vector operiodic; + std::vector coords(cc.ndims()); + cc.topology(idims, iperiodic, coords); + + // Check that everyone agrees on the dimensions + mpi::all_reduce(cc, + &(idims[0]), idims.size(), &(odims[0]), + mpi::minimum()); + BOOST_CHECK(std::equal(idims.begin(), idims.end(), + odims.begin())); + // Check that everyone agree on the periodicities + mpi::all_reduce(cc, iperiodic, operiodic, bvecmin()); + BOOST_CHECK(std::equal(iperiodic.begin(), iperiodic.end(), + operiodic.begin())); + if (cc.rank() == 0) { + std::cout << topology_description(odims, operiodic) << '\n'; + } + test_coordinates_consistency( cc, coords ); +} + int test_main(int argc, char* argv[]) { mpi::environment env(argc, argv); @@ -23,11 +102,32 @@ int test_main(int argc, char* argv[]) mpi::communicator world; std::vector dims(3); std::vector periodic(3); - dims[0] = 2; dims[1] = 3; dims[2] = 4; + if (world.size() == 24) { + dims[0] = 2; dims[1] = 3; dims[2] = 4; + } else { + dims[0] = 0; dims[1] = 0; dims[2] = 0; + } periodic[0] = true; periodic[1] = false; periodic[2] = true; mpi::cartesian_communicator cc(world, dims, periodic, true); BOOST_CHECK(cc.has_cartesian_topology()); - + BOOST_CHECK(cc.ndims() == 3); + for( int r = 0; r < cc.size(); ++r) { + cc.barrier(); + if (r == cc.rank()) { + std::vector coords = cc.coords(r); + std::cout << "Process of cartesian rank " << cc.rank() + << " and global rank " << world.rank() + << " has coordinates ("; + std::copy(coords.begin(), coords.end(), std::ostream_iterator(std::cout,",")); + std::cout << ")\n"; + } + } + test_topology_consistency(cc); + std::vector sub02; + sub02.push_back(0); + sub02.push_back(2); + mpi::cartesian_communicator cc02(cc, sub02); + test_topology_consistency(cc02); return 0; } From 0f13d9a20cf941fb9e7fa10fe068f3f4f16696ff Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Fri, 12 Sep 2014 18:38:45 +0200 Subject: [PATCH 03/99] Improved cartesian topology description. --- doc/Jamfile.v2 | 1 + include/boost/mpi/cartesian_communicator.hpp | 116 ++++++++++++++++++++------- src/cartesian_communicator.cpp | 68 +++++++++++----- test/cartesian_topology_test.cpp | 61 +++++--------- 4 files changed, 157 insertions(+), 89 deletions(-) diff --git a/doc/Jamfile.v2 b/doc/Jamfile.v2 index 96469b7..dd1cd5b 100644 --- a/doc/Jamfile.v2 +++ b/doc/Jamfile.v2 @@ -11,6 +11,7 @@ doxygen mpi_autodoc : [ glob ../../../boost/mpi.hpp ../../../boost/mpi/allocator.hpp + ../../../boost/mpi/cartesian_communicator.hpp ../../../boost/mpi/collectives.hpp ../../../boost/mpi/collectives_fwd.hpp ../../../boost/mpi/communicator.hpp diff --git a/include/boost/mpi/cartesian_communicator.hpp b/include/boost/mpi/cartesian_communicator.hpp index a722648..1606e4e 100644 --- a/include/boost/mpi/cartesian_communicator.hpp +++ b/include/boost/mpi/cartesian_communicator.hpp @@ -22,6 +22,7 @@ #include #include +#include // Headers required to implement cartesian topologies #include @@ -29,7 +30,66 @@ namespace boost { namespace mpi { -class cartesian_topology; +struct cartesian_dimension { + + int size; + bool periodic; + cartesian_dimension(int sz = 0, bool p = false) : size(sz), periodic(p) {} +private: + friend class boost::serialization::access; + template + void serialize(Archive & ar, const unsigned int version) + { + ar & size & periodic; + } + +}; + +template <> +struct is_mpi_datatype : mpl::true_ { }; + +bool +operator==(cartesian_dimension const& d1, cartesian_dimension const& d2) { + return d1.size == d2.size && d1.periodic == d2.periodic; +} + +bool +operator!=(cartesian_dimension const& d1, cartesian_dimension const& d2) { + return !(d1 == d2); +} + +std::ostream& operator<<(std::ostream& out, cartesian_dimension const& d); + +class BOOST_MPI_DECL cartesian_topology + : private std::vector { + friend class cartesian_communicator; + typedef std::vector super; + public: + using super::operator[]; + using super::size; + using super::begin; + using super::end; + using super::swap; + + cartesian_topology(int sz) + : super(sz) {} + + template + explicit cartesian_topology(array const& dims) + : super(NDIM) { + std::copy(dims.begin(), dims.end(), begin()); + } + + template + cartesian_topology(DimIter dim_iter, PerIter period_iter, int ndim) + : super(ndim) { + for(int i = 0; i < ndim; ++i) { + (*this)[i] = cartesian_dimension(*dim_iter++, *period_iter++); + } + } + + void split(std::vector& dims, std::vector& periodics) const; +}; /** * @brief An MPI communicator with a cartesian topology. @@ -103,23 +163,19 @@ class BOOST_MPI_DECL cartesian_communicator : public communicator * @param comm The communicator that the new, cartesian communicator * will be based on. * - * @param dims the dimension of the new communicator. The size indicate - * the number of dimension. Some value can be set to zero, in which case + * @param dims the cartesian dimension of the new communicator. The size indicate + * the number of dimension. Some dimensions be set to zero, in which case * the corresponding dimension value is left to the system. * - * @param periodic must be the same size as dims. Each value indicate if - * the corresponding dimension is cyclic. - * * @param reorder Whether MPI is permitted to re-order the process * ranks within the returned communicator, to better optimize * communication. If false, the ranks of each process in the * returned process will match precisely the rank of that process * within the original communicator. */ - cartesian_communicator(const communicator& comm, - const std::vector& dims, - const std::vector& periodic, - bool reorder = false); + cartesian_communicator(const communicator& comm, + const cartesian_topology& dims, + bool reorder = false); /** * Create a new cartesian communicator whose topology is a subset of @@ -142,10 +198,10 @@ class BOOST_MPI_DECL cartesian_communicator : public communicator * Return the rank of the process at the given coordinates. * @param coords the coordinates. the size must match the communicator's topology. */ - int rank(std::vector const& coords) const; + int rank(const std::vector& coords) const; /** - * Provides the coordinates of the process with the given rank. - * @param rk the ranks in this communicator. + * Provides the coordinates of a process with the given rank. + * @param rk the rank in this communicator. * @param cbuf a buffer were to store the coordinates. * @returns a reference to cbuf. */ @@ -160,29 +216,31 @@ class BOOST_MPI_DECL cartesian_communicator : public communicator * Retrieve the topology. * */ - void topology( std::vector& dims, - std::vector& periodic, - std::vector& coords ) const; + void topology( cartesian_topology& dims, std::vector& coords ) const; }; -std::vector& cartesian_dimensions(int sz, std::vector& dims); +/** + * Given en number of processes, and a partially filled sequence + * of dimension, try to complete the dimension sequence. + * @param nb_proc the numer of mpi processes.fill a sequence of dimension. + * @param dims a sequence of positive or null dimensions. Non zero dimension + * will be left untouched. + */ +std::vector& cartesian_dimensions(int nb_proc, std::vector& dims); +/** + * Given en communicator and a partially filled sequence + * of dimension, try to complete the dimension sequence to produce an acceptable + * cartesian topology. + * @param comm the prospective parent communicator. + * @param dims a sequence of positive or null dimensions. Non zero dimension + * will be left untouched. + */ inline -std::vector& cartesian_dimensions(communicator const& comm, std::vector& dims) { +std::vector& cartesian_dimensions(const communicator& comm, std::vector& dims) { return cartesian_dimensions(comm.size(), dims); } -class BOOST_MPI_DECL cartesian_topology { - public: - cartesian_topology(cartesian_communicator const& comm); - - private: - cartesian_communicator const& comm_ref; - -}; - } } // end namespace boost::mpi - - #endif // BOOST_MPI_CARTESIAN_COMMUNICATOR_HPP diff --git a/src/cartesian_communicator.cpp b/src/cartesian_communicator.cpp index 03d5652..d019128 100644 --- a/src/cartesian_communicator.cpp +++ b/src/cartesian_communicator.cpp @@ -12,23 +12,37 @@ namespace boost { namespace mpi { -cartesian_communicator::cartesian_communicator(const communicator& comm, - const std::vector& dims, - const std::vector& periodic, - bool reorder ) +std::ostream& +operator<<(std::ostream& out, cartesian_dimension const& d) { + out << '(' << d.size << ','; + if (d.periodic) { + out << "periodic"; + } else { + out << "bounded"; + } + out << ')'; + return out; +} + +cartesian_communicator::cartesian_communicator(const communicator& comm, + const cartesian_topology& topology, + bool reorder ) : communicator() { - BOOST_ASSERT(dims.size() == periodic.size()); - MPI_Comm newcomm; - std::vector local_dims = dims; + std::vector dims(topology.size()); + std::vector periodic(topology.size()); + for(int i = 0; i < topology.size(); ++i) { + dims[i] = topology[i].size; + periodic[i] = topology[i].periodic; + } // Fill the gaps, if any - if (std::count(local_dims.begin(), local_dims.end(), 0) > 0) { - cartesian_dimensions(comm, local_dims); + if (std::count(dims.begin(), dims.end(), 0) > 0) { + cartesian_dimensions(comm, dims); } - std::vector p(periodic.begin(), periodic.end()); + MPI_Comm newcomm; BOOST_MPI_CHECK_RESULT(MPI_Cart_create, ((MPI_Comm)comm, dims.size(), - local_dims.data(), p.data(), + dims.data(), periodic.data(), int(reorder), &newcomm)); if(newcomm != MPI_COMM_NULL) { comm_ptr.reset(new MPI_Comm(newcomm), comm_free()); @@ -64,7 +78,7 @@ cartesian_communicator::ndims() const { } int -cartesian_communicator::rank(std::vector const& coords ) const { +cartesian_communicator::rank(const std::vector& coords ) const { int r = -1; BOOST_ASSERT(coords.size() == ndims()); BOOST_MPI_CHECK_RESULT(MPI_Cart_rank, @@ -72,7 +86,7 @@ cartesian_communicator::rank(std::vector const& coords ) const { &r)); return r; } - + std::vector& cartesian_communicator::coords(int rk, std::vector& cbuf) const { cbuf.resize(ndims()); @@ -80,7 +94,7 @@ cartesian_communicator::coords(int rk, std::vector& cbuf) const { (MPI_Comm(*this), rk, cbuf.size(), cbuf.data() )); return cbuf; } - + std::vector cartesian_communicator::coords(int rk) const { std::vector coords; @@ -89,17 +103,29 @@ cartesian_communicator::coords(int rk) const { } void -cartesian_communicator::topology( std::vector& dims, - std::vector& periodic, - std::vector& coords ) const { +cartesian_communicator::topology( cartesian_topology& topo, + std::vector& coords ) const { int ndims = this->ndims(); - dims.resize(ndims); - periodic.resize(ndims); + topo.resize(ndims); coords.resize(ndims); + std::vector cdims(ndims); std::vector cperiods(ndims); BOOST_MPI_CHECK_RESULT(MPI_Cart_get, - (MPI_Comm(*this), ndims, dims.data(), cperiods.data(), coords.data())); - std::copy(cperiods.begin(), cperiods.end(), periodic.begin()); + (MPI_Comm(*this), ndims, cdims.data(), cperiods.data(), coords.data())); + cartesian_topology res(cdims.begin(), cperiods.begin(), ndims); + topo.swap(res); +} + +void +cartesian_topology::split(std::vector& dims, std::vector& periodics) const { + int ndims = size(); + dims.resize(ndims); + periodics.resize(ndims); + for(int i = 0; i < ndims; ++i) { + cartesian_dimension const& d = (*this)[i]; + dims[i] = d.size; + periodics[i] = d.periodic; + } } std::vector& diff --git a/test/cartesian_topology_test.cpp b/test/cartesian_topology_test.cpp index 4143713..ba52b8d 100644 --- a/test/cartesian_topology_test.cpp +++ b/test/cartesian_topology_test.cpp @@ -22,29 +22,18 @@ namespace mpi = boost::mpi; -struct bvecmin { - std::vector operator()(std::vector const& v1, - std::vector const& v2) const { - BOOST_ASSERT(v1.size() == v2.size()); - std::vector res(v1.size()); - std::transform(v1.begin(), v1.end(), v2.begin(), res.begin(), - std::logical_and()); - return res; +struct topo_minimum { + mpi::cartesian_dimension + operator()(mpi::cartesian_dimension const& d1, + mpi::cartesian_dimension const& d2 ) const { + return mpi::cartesian_dimension(std::min(d1.size, d2.size), + d1.periodic && d2.periodic); } }; -std::string topology_description( std::vector const& dims, - std::vector const& periodic ) { +std::string topology_description( mpi::cartesian_topology const& topo ) { std::ostringstream out; - for(int i = 0; i < dims.size(); ++i) { - out << "(" << dims[i] << ','; - if (periodic[i]) { - out << "cyclic"; - } else { - out << "bounded"; - } - out << ") "; - } + std::copy(topo.begin(), topo.end(), std::ostream_iterator(out, " ")); out << std::flush; return out.str(); } @@ -72,25 +61,19 @@ void test_coordinates_consistency( mpi::cartesian_communicator const& cc, void test_topology_consistency( mpi::cartesian_communicator const& cc) { - std::vector idims(cc.ndims()); - std::vector iperiodic(cc.ndims()); - std::vector odims(cc.ndims()); - std::vector operiodic; + mpi::cartesian_topology itopo(cc.ndims()); + mpi::cartesian_topology otopo(cc.ndims()); std::vector coords(cc.ndims()); - cc.topology(idims, iperiodic, coords); + cc.topology(itopo, coords); // Check that everyone agrees on the dimensions mpi::all_reduce(cc, - &(idims[0]), idims.size(), &(odims[0]), - mpi::minimum()); - BOOST_CHECK(std::equal(idims.begin(), idims.end(), - odims.begin())); - // Check that everyone agree on the periodicities - mpi::all_reduce(cc, iperiodic, operiodic, bvecmin()); - BOOST_CHECK(std::equal(iperiodic.begin(), iperiodic.end(), - operiodic.begin())); + &(itopo[0]), itopo.size(), &(otopo[0]), + topo_minimum()); + BOOST_CHECK(std::equal(itopo.begin(), itopo.end(), + otopo.begin())); if (cc.rank() == 0) { - std::cout << topology_description(odims, operiodic) << '\n'; + std::cout << topology_description(otopo) << '\n'; } test_coordinates_consistency( cc, coords ); } @@ -100,16 +83,16 @@ int test_main(int argc, char* argv[]) mpi::environment env(argc, argv); mpi::communicator world; - std::vector dims(3); - std::vector periodic(3); + mpi::cartesian_topology topo(3); + if (world.size() == 24) { - dims[0] = 2; dims[1] = 3; dims[2] = 4; + topo[0].size = 2; topo[1].size = 3; topo[2].size = 4; } else { - dims[0] = 0; dims[1] = 0; dims[2] = 0; + topo[0].size = 0; topo[1].size = 3; topo[2].size = 0; } - periodic[0] = true; periodic[1] = false; periodic[2] = true; + topo[0].periodic = true; topo[1].periodic = false; topo[2].periodic = true; - mpi::cartesian_communicator cc(world, dims, periodic, true); + mpi::cartesian_communicator cc(world, topo, true); BOOST_CHECK(cc.has_cartesian_topology()); BOOST_CHECK(cc.ndims() == 3); for( int r = 0; r < cc.size(); ++r) { From 99acdb85fd1d17f529656cfb07be5a1f7d087c14 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Mon, 15 Sep 2014 15:18:28 +0200 Subject: [PATCH 04/99] added some documentation --- include/boost/mpi/cartesian_communicator.hpp | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/include/boost/mpi/cartesian_communicator.hpp b/include/boost/mpi/cartesian_communicator.hpp index 1606e4e..d0a9fa9 100644 --- a/include/boost/mpi/cartesian_communicator.hpp +++ b/include/boost/mpi/cartesian_communicator.hpp @@ -30,11 +30,17 @@ namespace boost { namespace mpi { +/** + * Specify the size and periodicity of the grid in a single dimension. + */ struct cartesian_dimension { - + /** The size of the grid n this dimension. */ int size; + /** Is the grid periodic in this dimension. */ bool periodic; + cartesian_dimension(int sz = 0, bool p = false) : size(sz), periodic(p) {} + private: friend class boost::serialization::access; template @@ -60,6 +66,12 @@ operator!=(cartesian_dimension const& d1, cartesian_dimension const& d2) { std::ostream& operator<<(std::ostream& out, cartesian_dimension const& d); +/** + * @brief Describe the topology of a cartesian grid. + * + * Behave mostly like a sequence of @cartesian_dimension with the notable + * exception that its size is fixed. + */ class BOOST_MPI_DECL cartesian_topology : private std::vector { friend class cartesian_communicator; @@ -87,7 +99,9 @@ class BOOST_MPI_DECL cartesian_topology (*this)[i] = cartesian_dimension(*dim_iter++, *period_iter++); } } - + /** + * Split the topology in two sequences of sizes and periodicities. + */ void split(std::vector& dims, std::vector& periodics) const; }; From 2096fa08faffe9bb6d007ac5c47b12ac08f95611 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Mon, 15 Sep 2014 16:43:31 +0200 Subject: [PATCH 05/99] added mapping documentation for MPI_Cart_create, MPI_Dims_create MPI_Cartdim_get. --- doc/mpi.qbk | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/mpi.qbk b/doc/mpi.qbk index f722933..b381836 100644 --- a/doc/mpi.qbk +++ b/doc/mpi.qbk @@ -1644,9 +1644,10 @@ algorithms. [[`MPI_CART`] [unnecessary; use [memberref boost::mpi::communicator::has_cartesian_topology `communicator::has_cartesian_topology`]]] [[[@http://www.mpi-forum.org/docs/mpi-1.1/mpi-11-html/node133.html#Node133 - `MPI_Cart_create`]] [unsupported]] + `MPI_Cart_create`]] [[classref boost::mpi::cartesian_communicator `cartesian_communicator`] + constructor]] [[[@http://www.mpi-forum.org/docs/mpi-1.1/mpi-11-html/node134.html#Node134 - `MPI_Dims_create`]] [unsupported]] + `MPI_Dims_create`]] [[funcref boost::mpi::cartesian_dimensions `cartesian_dimensions`]]] [[[@http://www.mpi-forum.org/docs/mpi-1.1/mpi-11-html/node135.html#Node135 `MPI_Graph_create`]] [[memberref boost::mpi::communicator::with_graph_topology @@ -1664,7 +1665,7 @@ algorithms. `MPI_Graph_get`]] [[funcref boost::mpi::vertices `vertices`], [funcref boost::mpi::edges `edges`]]] [[[@http://www.mpi-forum.org/docs/mpi-1.1/mpi-11-html/node136.html#Node136 - `MPI_Cartdim_get`]] [unsupported]] + `MPI_Cartdim_get`]] [[memberref boost::mpi::cartesian_communicator::ndims `cartesian_communicator::ndims` ]]] [[[@http://www.mpi-forum.org/docs/mpi-1.1/mpi-11-html/node136.html#Node136 `MPI_Cart_get`]] [unsupported]] [[[@http://www.mpi-forum.org/docs/mpi-1.1/mpi-11-html/node136.html#Node136 From a5ab336f84559edcb88500f9523d97d3c9bac741 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Mon, 15 Sep 2014 16:48:55 +0200 Subject: [PATCH 06/99] Added documentation for C API mapping --- doc/mpi.qbk | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/mpi.qbk b/doc/mpi.qbk index b381836..8726c08 100644 --- a/doc/mpi.qbk +++ b/doc/mpi.qbk @@ -1667,11 +1667,11 @@ algorithms. [[[@http://www.mpi-forum.org/docs/mpi-1.1/mpi-11-html/node136.html#Node136 `MPI_Cartdim_get`]] [[memberref boost::mpi::cartesian_communicator::ndims `cartesian_communicator::ndims` ]]] [[[@http://www.mpi-forum.org/docs/mpi-1.1/mpi-11-html/node136.html#Node136 - `MPI_Cart_get`]] [unsupported]] + `MPI_Cart_get`]] [[memberref boost::mpi::cartesian_communicator::topology `cartesian_communicator::topology` ]]] [[[@http://www.mpi-forum.org/docs/mpi-1.1/mpi-11-html/node136.html#Node136 - `MPI_Cart_rank`]] [unsupported]] + `MPI_Cart_rank`]] [[memberref boost::mpi::cartesian_communicator::rank `cartesian_communicator::rank` ]]] [[[@http://www.mpi-forum.org/docs/mpi-1.1/mpi-11-html/node136.html#Node136 - `MPI_Cart_coords`]] [unsupported]] + `MPI_Cart_coords`]] [[memberref boost::mpi::cartesian_communicator::coords `cartesian_communicator::coords` ]]] [[[@http://www.mpi-forum.org/docs/mpi-1.1/mpi-11-html/node136.html#Node136 `MPI_Graph_neighbors_count`]] [[funcref boost::mpi::out_degree `out_degree`]]] @@ -1681,7 +1681,8 @@ algorithms. [[[@http://www.mpi-forum.org/docs/mpi-1.1/mpi-11-html/node137.html#Node137 `MPI_Cart_shift`]] [unsupported]] [[[@http://www.mpi-forum.org/docs/mpi-1.1/mpi-11-html/node138.html#Node138 - `MPI_Cart_sub`]] [unsupported]] + `MPI_Cart_sub`]] [[classref boost::mpi::cartesian_communicator `cartesian_communicator`] + constructor]] [[[@http://www.mpi-forum.org/docs/mpi-1.1/mpi-11-html/node139.html#Node139 `MPI_Cart_map`]] [unsupported]] [[[@http://www.mpi-forum.org/docs/mpi-1.1/mpi-11-html/node139.html#Node139 From cbbb9a450f47c003de75bac4ecc5d152926216f4 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Mon, 15 Sep 2014 17:02:51 +0200 Subject: [PATCH 07/99] Added down cast functions for communicator to cartesian communicator conversion. --- include/boost/mpi/cartesian_communicator.hpp | 2 ++ include/boost/mpi/communicator.hpp | 26 ++++++++++++++++++++++++++ src/communicator.cpp | 23 ++++++++++++++++++++--- 3 files changed, 48 insertions(+), 3 deletions(-) diff --git a/include/boost/mpi/cartesian_communicator.hpp b/include/boost/mpi/cartesian_communicator.hpp index d0a9fa9..ef03bea 100644 --- a/include/boost/mpi/cartesian_communicator.hpp +++ b/include/boost/mpi/cartesian_communicator.hpp @@ -54,11 +54,13 @@ struct cartesian_dimension { template <> struct is_mpi_datatype : mpl::true_ { }; +inline bool operator==(cartesian_dimension const& d1, cartesian_dimension const& d2) { return d1.size == d2.size && d1.periodic == d2.periodic; } +inline bool operator!=(cartesian_dimension const& d1, cartesian_dimension const& d2) { return !(d1 == d2); diff --git a/include/boost/mpi/communicator.hpp b/include/boost/mpi/communicator.hpp index 65de3a4..eda5908 100644 --- a/include/boost/mpi/communicator.hpp +++ b/include/boost/mpi/communicator.hpp @@ -113,6 +113,14 @@ class intercommunicator; class graph_communicator; /** + * INTERNAL ONLY + * + * Forward declaration of @c cartesian_communicator needed for the "cast" + * from a communicator to a cartesian communicator. + */ +class cartesian_communicator; + +/** * @brief A communicator that permits communication and * synchronization among a set of processes. * @@ -821,6 +829,24 @@ class BOOST_MPI_DECL communicator /** * Determines whether this communicator has a Cartesian topology. */ + bool has_graph_topology() const; + + /** + * Determine if the communicator has a cartesian topology and, if so, + * return that @c cartesian_communicator. Even though the communicators + * have different types, they refer to the same underlying + * communication space and can be used interchangeably for + * communication. + * + * @returns an @c optional containing the cartesian communicator, if this + * communicator does in fact have a cartesian topology. Otherwise, returns + * an empty @c optional. + */ + optional as_cartesian_communicator() const; + + /** + * Determines whether this communicator has a Cartesian topology. + */ bool has_cartesian_topology() const; #if 0 diff --git a/src/communicator.cpp b/src/communicator.cpp index 82caac6..3dc27b0 100644 --- a/src/communicator.cpp +++ b/src/communicator.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -160,14 +161,21 @@ optional communicator::as_intercommunicator() const return optional(); } -optional communicator::as_graph_communicator() const +bool communicator::has_graph_topology() const { int status; BOOST_MPI_CHECK_RESULT(MPI_Topo_test, ((MPI_Comm)*this, &status)); - if (status == MPI_GRAPH) + + return status == MPI_GRAPH; +} + +optional communicator::as_graph_communicator() const +{ + if (has_graph_topology()) { return graph_communicator(comm_ptr); - else + } else { return optional(); + } } bool communicator::has_cartesian_topology() const @@ -178,6 +186,15 @@ bool communicator::has_cartesian_topology() const return status == MPI_CART; } +optional communicator::as_cartesian_communicator() const +{ + if (has_cartesian_topology()) { + return cartesian_communicator(comm_ptr); + } else { + return optional(); + } +} + void communicator::abort(int errcode) const { BOOST_MPI_CHECK_RESULT(MPI_Abort, (MPI_Comm(*this), errcode)); From 00a49f55c93ef29690427cb1b70576887847e8fb Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Mon, 15 Sep 2014 17:17:45 +0200 Subject: [PATCH 08/99] doxygen typo --- include/boost/mpi/cartesian_communicator.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/boost/mpi/cartesian_communicator.hpp b/include/boost/mpi/cartesian_communicator.hpp index ef03bea..0ded619 100644 --- a/include/boost/mpi/cartesian_communicator.hpp +++ b/include/boost/mpi/cartesian_communicator.hpp @@ -71,7 +71,7 @@ std::ostream& operator<<(std::ostream& out, cartesian_dimension const& d); /** * @brief Describe the topology of a cartesian grid. * - * Behave mostly like a sequence of @cartesian_dimension with the notable + * Behave mostly like a sequence of @c cartesian_dimension with the notable * exception that its size is fixed. */ class BOOST_MPI_DECL cartesian_topology From 243fa971c43c3f1fe4f08c858e10987aec496df6 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Mon, 6 Oct 2014 18:05:44 +0200 Subject: [PATCH 09/99] Added topology pretty printing Removed a few "unsigned int" comparuson warnings with g++ Splitted bug test_main function --- include/boost/mpi/cartesian_communicator.hpp | 15 ++++++ src/cartesian_communicator.cpp | 40 ++++++++++++--- test/cartesian_topology_test.cpp | 75 +++++++++++++++++++++------- 3 files changed, 105 insertions(+), 25 deletions(-) diff --git a/include/boost/mpi/cartesian_communicator.hpp b/include/boost/mpi/cartesian_communicator.hpp index 0ded619..0535073 100644 --- a/include/boost/mpi/cartesian_communicator.hpp +++ b/include/boost/mpi/cartesian_communicator.hpp @@ -23,6 +23,7 @@ #include #include #include +#include // Headers required to implement cartesian topologies #include @@ -66,6 +67,9 @@ operator!=(cartesian_dimension const& d1, cartesian_dimension const& d2) { return !(d1 == d2); } +/** + * @brief Pretty printing of a cartesian dimension (size, periodic) + */ std::ostream& operator<<(std::ostream& out, cartesian_dimension const& d); /** @@ -108,6 +112,11 @@ class BOOST_MPI_DECL cartesian_topology }; /** + * @brief Pretty printing of a cartesian topology + */ +std::ostream& operator<<(std::ostream& out, cartesian_topology const& t); + +/** * @brief An MPI communicator with a cartesian topology. * * A @c cartesian_communicator is a communicator whose topology is @@ -216,6 +225,12 @@ class BOOST_MPI_DECL cartesian_communicator : public communicator */ int rank(const std::vector& coords) const; /** + * Return the rank of the source and targetdestination process through a shift. + * @param dim the dimension in which the shift takes place. 0 <= dim <= ndim(). + * @param disp the shift displacement, can be positive (upward) or negative (downward). + */ + std::pair shifted_ranks(int dim, int disp) const; + /** * Provides the coordinates of a process with the given rank. * @param rk the rank in this communicator. * @param cbuf a buffer were to store the coordinates. diff --git a/src/cartesian_communicator.cpp b/src/cartesian_communicator.cpp index d019128..e5c877a 100644 --- a/src/cartesian_communicator.cpp +++ b/src/cartesian_communicator.cpp @@ -24,6 +24,20 @@ operator<<(std::ostream& out, cartesian_dimension const& d) { return out; } +std::ostream& +operator<<(std::ostream& out, cartesian_topology const& topo) { + out << '{'; + int const sz = topo.size(); + for (int i = 0; i < sz; ++i) { + out << topo[i]; + if ( i < (sz-1) ) { + out << ','; + } + } + out << '}'; + return out; +} + cartesian_communicator::cartesian_communicator(const communicator& comm, const cartesian_topology& topology, bool reorder ) @@ -31,7 +45,8 @@ cartesian_communicator::cartesian_communicator(const communicator& comm, { std::vector dims(topology.size()); std::vector periodic(topology.size()); - for(int i = 0; i < topology.size(); ++i) { + int tsz = topology.size(); + for(int i = 0; i < tsz; ++i) { dims[i] = topology[i].size; periodic[i] = topology[i].periodic; } @@ -46,6 +61,8 @@ cartesian_communicator::cartesian_communicator(const communicator& comm, int(reorder), &newcomm)); if(newcomm != MPI_COMM_NULL) { comm_ptr.reset(new MPI_Comm(newcomm), comm_free()); + } else { + comm_ptr.reset(MPI_COMM_NULL); } } @@ -53,10 +70,11 @@ cartesian_communicator::cartesian_communicator(const cartesian_communicator& com const std::vector& keep ) : communicator() { - int max_dims = comm.ndims(); - BOOST_ASSERT(keep.size() <= max_dims); + int const max_dims = comm.ndims(); + int const nbkept = keep.size(); + BOOST_ASSERT(nbkept <= max_dims); std::vector bitset(max_dims, int(false)); - for(int i = 0; i < keep.size(); ++i) { + for(int i = 0; i < nbkept; ++i) { BOOST_ASSERT(keep[i] < max_dims); bitset[keep[i]] = true; } @@ -80,13 +98,23 @@ cartesian_communicator::ndims() const { int cartesian_communicator::rank(const std::vector& coords ) const { int r = -1; - BOOST_ASSERT(coords.size() == ndims()); + BOOST_ASSERT(int(coords.size()) == ndims()); BOOST_MPI_CHECK_RESULT(MPI_Cart_rank, (MPI_Comm(*this), const_cast&>(coords).data(), &r)); return r; } - + +std::pair +cartesian_communicator::shifted_ranks(int dim, int disp) const { + std::pair r(-1,-1); + assert((0 <= dim && dim < ndims()) || (std::abort(), false)); + BOOST_ASSERT(0 <= dim && dim < ndims()); + BOOST_MPI_CHECK_RESULT(MPI_Cart_shift, + (MPI_Comm(*this), dim, disp, &(r.first), &(r.second))); + return r; +} + std::vector& cartesian_communicator::coords(int rk, std::vector& cbuf) const { cbuf.resize(ndims()); diff --git a/test/cartesian_topology_test.cpp b/test/cartesian_topology_test.cpp index ba52b8d..d67f40f 100644 --- a/test/cartesian_topology_test.cpp +++ b/test/cartesian_topology_test.cpp @@ -59,6 +59,34 @@ void test_coordinates_consistency( mpi::cartesian_communicator const& cc, } } +void test_shifted_coords( mpi::cartesian_communicator const& cc, int pos, mpi::cartesian_dimension desc, int dim ) +{ + for (int i = -(desc.size); i < desc.size; ++i) { + std::pair rks = cc.shifted_ranks(pos, i); + int src = cc.coords(rks.first)[dim]; + int dst = cc.coords(rks.second)[dim]; + if (pos == (dim/2)) { + std::ostringstream out; + out << "Rank " << cc.rank() << ", dim. " << dim << ", pos " << pos << ", in " << desc << ' '; + out << "shifted pos: " << src << ", " << dst << '\n'; + std::cout << out.str(); + } + } +} + +void test_shifted_coords( mpi::cartesian_communicator const& cc, std::vector const& coords, mpi::cartesian_topology const& topo ) +{ + if (cc.rank() == 0) { + std::cout << "Testing shifts with topology " << topo << '\n'; + } + for(int i = 0; i < cc.ndims(); ++i) { + if (cc.rank() == 0) { + std::cout << " for dimension " << i << ' ' << topo[i] << '\n'; + } + test_shifted_coords( cc, coords[i], topo[i], i ); + } +} + void test_topology_consistency( mpi::cartesian_communicator const& cc) { mpi::cartesian_topology itopo(cc.ndims()); @@ -78,23 +106,11 @@ void test_topology_consistency( mpi::cartesian_communicator const& cc) test_coordinates_consistency( cc, coords ); } -int test_main(int argc, char* argv[]) +void test_cartesian_topology( mpi::communicator const& world, mpi::cartesian_topology const& topo) { - mpi::environment env(argc, argv); - - mpi::communicator world; - mpi::cartesian_topology topo(3); - - if (world.size() == 24) { - topo[0].size = 2; topo[1].size = 3; topo[2].size = 4; - } else { - topo[0].size = 0; topo[1].size = 3; topo[2].size = 0; - } - topo[0].periodic = true; topo[1].periodic = false; topo[2].periodic = true; - mpi::cartesian_communicator cc(world, topo, true); BOOST_CHECK(cc.has_cartesian_topology()); - BOOST_CHECK(cc.ndims() == 3); + BOOST_CHECK(cc.ndims() == int(topo.size())); for( int r = 0; r < cc.size(); ++r) { cc.barrier(); if (r == cc.rank()) { @@ -107,10 +123,31 @@ int test_main(int argc, char* argv[]) } } test_topology_consistency(cc); - std::vector sub02; - sub02.push_back(0); - sub02.push_back(2); - mpi::cartesian_communicator cc02(cc, sub02); - test_topology_consistency(cc02); + std::vector even; + for(int i = 0; i < cc.ndims(); i += 2) { + even.push_back(i); + } + mpi::cartesian_communicator cce(cc, even); + test_topology_consistency(cce); + test_shifted_coords( cce, cce.coords(cce.rank()), topo ); +} + +int test_main(int argc, char* argv[]) +{ + mpi::environment env(argc, argv); + + mpi::communicator world; + int const ndim = world.size() >= 24 ? 3 : 2; + mpi::cartesian_topology topo(ndim); + typedef mpi::cartesian_dimension cd; + if (topo.size() == 3) { + topo[0] = cd(2,true); topo[1] = cd(3,false); topo[2] = cd(4, true); + } else if (world.size() >= 6) { + topo[0] = cd(0,true); topo[1] = cd(3, false); + } else { + topo[0] = cd(0,true); topo[1] = cd(0, false); + } + test_cartesian_topology( world, topo); + return 0; } From 44f50e6d0a7658f9f26badd37f37c6b6beeb2db0 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Tue, 7 Oct 2014 10:23:25 +0200 Subject: [PATCH 10/99] indentation --- src/communicator.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/communicator.cpp b/src/communicator.cpp index f773b1f..adf82a6 100644 --- a/src/communicator.cpp +++ b/src/communicator.cpp @@ -173,7 +173,7 @@ optional communicator::as_graph_communicator() const { if (has_graph_topology()) { return graph_communicator(comm_ptr); - } else { + } else { return optional(); } } From 911209c554f88810cf31eaea9c21f64c5d8f210c Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Fri, 10 Oct 2014 16:38:44 +0200 Subject: [PATCH 11/99] checkpointing before transfert --- include/boost/mpi/cartesian_communicator.hpp | 7 ++++++- src/cartesian_communicator.cpp | 22 +++++++++++++++++--- src/communicator.cpp | 1 + test/cartesian_topology_test.cpp | 30 ++++++++++++++++++++-------- 4 files changed, 48 insertions(+), 12 deletions(-) diff --git a/include/boost/mpi/cartesian_communicator.hpp b/include/boost/mpi/cartesian_communicator.hpp index 0535073..2b9d623 100644 --- a/include/boost/mpi/cartesian_communicator.hpp +++ b/include/boost/mpi/cartesian_communicator.hpp @@ -244,10 +244,15 @@ class BOOST_MPI_DECL cartesian_communicator : public communicator */ std::vector coords(int rk) const; /** - * Retrieve the topology. + * Retrieve the topology and coordinates of this process in the grid. * */ void topology( cartesian_topology& dims, std::vector& coords ) const; + /** + * Retrieve the topology of the grid. + * + */ + cartesian_topology topology() const; }; /** diff --git a/src/cartesian_communicator.cpp b/src/cartesian_communicator.cpp index afb6541..147b41a 100644 --- a/src/cartesian_communicator.cpp +++ b/src/cartesian_communicator.cpp @@ -41,7 +41,7 @@ operator<<(std::ostream& out, cartesian_topology const& topo) { cartesian_communicator::cartesian_communicator(const communicator& comm, const cartesian_topology& topology, bool reorder ) - : communicator() + : communicator(MPI_COMM_NULL, comm_attach) { std::vector dims(topology.size()); std::vector periodic(topology.size()); @@ -66,7 +66,7 @@ cartesian_communicator::cartesian_communicator(const communicator& comm, cartesian_communicator::cartesian_communicator(const cartesian_communicator& comm, const std::vector& keep ) - : communicator() + : communicator(MPI_COMM_NULL, comm_attach) { int const max_dims = comm.ndims(); int const nbkept = keep.size(); @@ -142,6 +142,13 @@ cartesian_communicator::topology( cartesian_topology& topo, topo.swap(res); } +cartesian_topology +cartesian_communicator::topology() const { + cartesian_topology topo(ndims()); + std::vector coords; + topology(topo, coords); + return topo; +} void cartesian_topology::split(std::vector& dims, std::vector& periodics) const { int ndims = size(); @@ -156,8 +163,17 @@ cartesian_topology::split(std::vector& dims, std::vector& periodics) std::vector& cartesian_dimensions(int sz, std::vector& dims) { + int min = 1; + int const dimsz = dims.size(); + for(int i = 0; i < dimsz; ++i) { + if (dims[i] > 0) { + min *= dims[i]; + } + } + int leftover = sz % min; + BOOST_MPI_CHECK_RESULT(MPI_Dims_create, - (sz, dims.size(), dims.data())); + (sz-leftover, dims.size(), dims.data())); return dims; } diff --git a/src/communicator.cpp b/src/communicator.cpp index e703949..a172edd 100644 --- a/src/communicator.cpp +++ b/src/communicator.cpp @@ -206,6 +206,7 @@ optional communicator::as_cartesian_communicator() const void communicator::abort(int errcode) const { BOOST_MPI_CHECK_RESULT(MPI_Abort, (MPI_Comm(*this), errcode)); + std::abort(); } /************************************************************* diff --git a/test/cartesian_topology_test.cpp b/test/cartesian_topology_test.cpp index 61cfea8..893c9a1 100644 --- a/test/cartesian_topology_test.cpp +++ b/test/cartesian_topology_test.cpp @@ -74,8 +74,10 @@ void test_shifted_coords( mpi::cartesian_communicator const& cc, int pos, mpi:: } } -void test_shifted_coords( mpi::cartesian_communicator const& cc, std::vector const& coords, mpi::cartesian_topology const& topo ) +void test_shifted_coords( mpi::cartesian_communicator const& cc) { + std::vector coords; mpi::cartesian_topology topo(cc.ndims()); + cc.topology(topo, coords); if (cc.rank() == 0) { std::cout << "Testing shifts with topology " << topo << '\n'; } @@ -106,17 +108,14 @@ void test_topology_consistency( mpi::cartesian_communicator const& cc) test_coordinates_consistency( cc, coords ); } -void test_cartesian_topology( mpi::communicator const& world, mpi::cartesian_topology const& topo) +void test_cartesian_topology( mpi::cartesian_communicator const& cc) { - mpi::cartesian_communicator cc(world, topo, true); BOOST_CHECK(cc.has_cartesian_topology()); - BOOST_CHECK(cc.ndims() == int(topo.size())); for( int r = 0; r < cc.size(); ++r) { cc.barrier(); if (r == cc.rank()) { std::vector coords = cc.coords(r); std::cout << "Process of cartesian rank " << cc.rank() - << " and global rank " << world.rank() << " has coordinates ("; std::copy(coords.begin(), coords.end(), std::ostream_iterator(std::cout,",")); std::cout << ")\n"; @@ -129,7 +128,24 @@ void test_cartesian_topology( mpi::communicator const& world, mpi::cartesian_top } mpi::cartesian_communicator cce(cc, even); test_topology_consistency(cce); - test_shifted_coords( cce, cce.coords(cce.rank()), topo ); + test_shifted_coords(cce); +} + +void test_cartesian_topology( mpi::communicator const& world, mpi::cartesian_topology const& topo) +{ + mpi::cartesian_communicator cc(world, topo, true); + if (cc) { + BOOST_CHECK(cc.has_cartesian_topology()); + BOOST_CHECK(cc.ndims() == int(topo.size())); + if (cc.rank() == 0) { + std::cout << "Asked topology " << topo << ", got " << cc.topology() << '\n'; + } + test_cartesian_topology(cc); + } else { + std::ostringstream out; + out << world.rank() << " was left outside the cartesian grid\n"; + std::cout << out.str(); + } } int test_main(int argc, char* argv[]) @@ -138,8 +154,6 @@ int test_main(int argc, char* argv[]) mpi::communicator world; int const ndim = world.size() >= 24 ? 3 : 2; - //std::cout << "Say something:" << std::flush; - //std::cin.get(); mpi::cartesian_topology topo(ndim); typedef mpi::cartesian_dimension cd; if (topo.size() == 3) { From d184ba0ffdf45e14c841671834cdaced246a4824 Mon Sep 17 00:00:00 2001 From: bavier Date: Wed, 16 Dec 2015 17:55:39 -0600 Subject: [PATCH 12/99] doc: Fix typos --- doc/mpi.qbk | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/doc/mpi.qbk b/doc/mpi.qbk index 5268f3d..8f69adc 100644 --- a/doc/mpi.qbk +++ b/doc/mpi.qbk @@ -78,7 +78,7 @@ supports: * MPI Datatypes: Boost.MPI can build MPI data types for user-defined types using the _Serialization_ library. * Separating structure from content: Boost.MPI can transfer the shape - (or "skeleton") of complexc data structures (lists, maps, + (or "skeleton") of complex data structures (lists, maps, etc.) and then separately transfer their content. This facility optimizes for cases where the data within a large, static data structure needs to be transmitted many times. @@ -193,7 +193,7 @@ First, you need to scan the =include/boost/mpi/config.hpp= file and check if som settings needs to be modified for your MPI implementation or preferences. In particular, the [macroref BOOST_MPI_HOMOGENEOUS] macro, that you will need to comment out -if you plan tu run on an heterogeneous set of macines. See the [link mpi.homogeneous_machines optimization] notes below. +if you plan to run on an heterogeneous set of machines. See the [link mpi.homogeneous_machines optimization] notes below. Most MPI implementations requires specific compilation and link options. In order to mask theses options to the user, most MPI implementations provides @@ -230,10 +230,10 @@ The directive will find the wrapper and deduce the options to use. using mpi : /opt/mpi/bullxmpi/1.2.8.3/bin/mpicc ; ] -* [*If your wrapper is really excentric] +* [*If your wrapper is really eccentric] or does not exist at all (it happens), you need to -provide the compilation and build options to the build environement using `jam` directives. +provide the compilation and build options to the build environment using `jam` directives. For example, the following could be used for a specific Intel MPI implementation: [pre @@ -248,9 +248,9 @@ using mpi : mpiicc : rt ; ] -To do that, you need to guess the libraries and include directories associated with your environement. -You can refer to the your specific MPI environement documentation. -Most of the time thoug, your wrapper have an option that provide that information, it usually starts with `--show`: +To do that, you need to guess the libraries and include directories associated with your environment. +You can refer to the your specific MPI environment documentation. +Most of the time though, your wrapper have an option that provide that information, it usually starts with `--show`: [pre $ mpiicc -show icc -I/softs/intel//impi/5.0.3.048/intel64/include -L/softs/intel//impi/5.0.3.048/intel64/lib/release_mt -L/softs/intel//impi/5.0.3.048/intel64/lib -Xlinker --enable-new-dtags -Xlinker -rpath -Xlinker /softs/intel//impi/5.0.3.048/intel64/lib/release_mt -Xlinker -rpath -Xlinker /softs/intel//impi/5.0.3.048/intel64/lib -Xlinker -rpath -Xlinker /opt/intel/mpi-rt/5.0/intel64/lib/release_mt -Xlinker -rpath -Xlinker /opt/intel/mpi-rt/5.0/intel64/lib -lmpifort -lmpi -lmpigi -ldl -lrt -lpthread @@ -284,7 +284,7 @@ for instance, the test harness will execute, e.g., mpirun -np 4 all_gather_test ] -Some implementations provides alternative launcher that can be more convenient. For exemple, Intel's MPI provides the `mpiexec.hydra`: +Some implementations provides alternative launcher that can be more convenient. For example, Intel's MPI provides the `mpiexec.hydra`: [pre $mpiexec.hydra -np 4 all_gather_test @@ -301,7 +301,7 @@ using mpi : mpiicc : [endsect] [section:installation Build and Install] -To build the whole Boost ditribution: +To build the whole Boost distribution: [pre $cd $./b2 install @@ -689,7 +689,7 @@ fixed only when the type of the parameter it stores is fixed: The redundant copy elimination optimization can only be applied when the shape of the data type is completely fixed. Variable-length types (e.g., strings, linked lists) and types that store pointers cannot use -the optimiation, but Boost.MPI will be unable to detect this error at +the optimization, but Boost.MPI will be unable to detect this error at compile time. Attempting to perform this optimization when it is not correct will likely result in segmentation faults and other strange program behavior. @@ -979,8 +979,8 @@ the `random_min.cpp` example: In that example we provide both input and output values, requiring twice as much space, which can be a problem depending on the size of the transmitted data. -If there is no need to preserve the input value, the ouput value -can be omitted. In that case the input value will be overriden with +If there is no need to preserve the input value, the output value +can be omitted. In that case the input value will be overridden with the output value and Boost.MPI is able, in some situation, to implement the operation with a more space efficient solution (using the `MPI_IN_PLACE` flag of the MPI C mapping), as in the following example (`in_place_global_min.cpp`): @@ -1206,7 +1206,7 @@ To obtain optimal performance for small fixed-length data types not containing any pointers it is very important to mark them using the type traits of Boost.MPI and Boost.Serialization. -It was alredy discussed that fixed length types containing no pointers can be +It was already discussed that fixed length types containing no pointers can be using as [classref boost::mpi::is_mpi_datatype `is_mpi_datatype`], e.g.: @@ -1423,7 +1423,7 @@ supported by compiler]] Boost.MPI does not provide direct wrappers to the MPI derived datatypes functionality. Instead, Boost.MPI relies on the _Serialization_ library to construct MPI datatypes for user-defined -classe. The section on [link mpi.user_data_types user-defined +classes. The section on [link mpi.user_data_types user-defined data types] describes this mechanism, which is used for types that marked as "MPI datatypes" using [classref boost::mpi::is_mpi_datatype `is_mpi_datatype`]. @@ -1561,7 +1561,7 @@ boost::mpi::communicator::barrier `communicator::barrier`]]] [[[@http://www.mpi-forum.org/docs/mpi-20-html/node145.htm#Node145 `MPI_IN_PLACE`]] [supported implicitly by [funcref boost::mpi::all_reduce -`all_reduce` by omiting the ouput value]]] +`all_reduce` by omitting the output value]]] ] Boost.MPI uses function objects to specify how reductions should occur @@ -1839,7 +1839,7 @@ Timer library]. ] MPI startup and shutdown are managed by the construction and -descruction of the Boost.MPI [classref boost::mpi::environment +destruction of the Boost.MPI [classref boost::mpi::environment `environment`] class. [table Startup/shutdown facilities @@ -2054,7 +2054,7 @@ The skeleton/content mechanism is a structured way to exploit the interaction between custom-built MPI datatypes and `MPI_BOTTOM`, to eliminate extra buffer copies. -[section:python_compatbility C++/Python MPI Compatibility] +[section:python_compatibility C++/Python MPI Compatibility] Boost.MPI is a C++ library whose facilities have been exposed to Python via the Boost.Python library. Since the Boost.MPI Python bindings are build directly on top of the C++ library, and nearly every feature of @@ -2137,7 +2137,7 @@ and the C MPI library. [section:threading Threads] -There are an increasing number of hybrid parrallel applications that mix +There are an increasing number of hybrid parallel applications that mix distributed and shared memory parallelism. To know how to support that model, one need to know what level of threading support is guaranteed by the MPI implementation. There are 4 ordered level of possible threading support described @@ -2146,8 +2146,8 @@ At the lowest level, you should not use threads at all, at the highest level, an thread can perform MPI call. If you want to use multi-threading in your MPI application, you should indicate -in the environment constructor your preffered threading support. Then probe the -one the librarie did provide, and decide what you can do with it (it could be +in the environment constructor your preferred threading support. Then probe the +one the library did provide, and decide what you can do with it (it could be nothing, then aborting is a valid option): #include @@ -2185,7 +2185,7 @@ raw MPI. We ran five different variants of the NetPIPE benchmark: communication. # MPI (Datatypes): NetPIPE modified to use a derived datatype (which - itself contains a single `MPI_BYTE`) rathan than a fundamental + itself contains a single `MPI_BYTE`) rather than a fundamental datatype. # Boost.MPI (Datatypes): NetPIPE modified to use a user-defined type From c70e7ff2438783a1ac6690c089b586b1431b15a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Hunold?= Date: Sun, 24 Apr 2016 21:59:36 +0200 Subject: [PATCH 13/99] Fix: adapt to renaming of serialization::array_wrapper (#30) Fix: adapt to renaming of serialization::array_wrapper Tested that both graph_parallel and MPI work fine with El Capitan and clang, thanks for the patch! --- include/boost/mpi/communicator.hpp | 4 ++-- include/boost/mpi/detail/binary_buffer_iprimitive.hpp | 4 ++-- include/boost/mpi/detail/binary_buffer_oprimitive.hpp | 4 ++-- include/boost/mpi/detail/forward_iprimitive.hpp | 2 +- include/boost/mpi/detail/forward_oprimitive.hpp | 2 +- include/boost/mpi/detail/ignore_iprimitive.hpp | 2 +- include/boost/mpi/detail/ignore_oprimitive.hpp | 2 +- include/boost/mpi/detail/mpi_datatype_primitive.hpp | 2 +- include/boost/mpi/detail/packed_iprimitive.hpp | 4 ++-- include/boost/mpi/detail/packed_oprimitive.hpp | 2 +- 10 files changed, 14 insertions(+), 14 deletions(-) diff --git a/include/boost/mpi/communicator.hpp b/include/boost/mpi/communicator.hpp index fcef086..46f7375 100644 --- a/include/boost/mpi/communicator.hpp +++ b/include/boost/mpi/communicator.hpp @@ -1257,7 +1257,7 @@ communicator::array_recv_impl(int source, int tag, T* values, int n, ia >> count; // Deserialize the data in the message - boost::serialization::array arr(values, count > n? n : count); + boost::serialization::array_wrapper arr(values, count > n? n : count); ia >> arr; if (count > n) { @@ -1459,7 +1459,7 @@ namespace detail { ia >> count; // Deserialize the data in the message - boost::serialization::array arr(values, count > n? n : count); + boost::serialization::array_wrapper arr(values, count > n? n : count); ia >> arr; if (count > n) { diff --git a/include/boost/mpi/detail/binary_buffer_iprimitive.hpp b/include/boost/mpi/detail/binary_buffer_iprimitive.hpp index f499d0d..388cd44 100644 --- a/include/boost/mpi/detail/binary_buffer_iprimitive.hpp +++ b/include/boost/mpi/detail/binary_buffer_iprimitive.hpp @@ -66,7 +66,7 @@ class BOOST_MPI_DECL binary_buffer_iprimitive // fast saving of arrays of fundamental types template - void load_array(serialization::array const& x, unsigned int /* file_version */) + void load_array(serialization::array_wrapper const& x, unsigned int /* file_version */) { BOOST_MPL_ASSERT((serialization::is_bitwise_serializable::type>)); if (x.count()) @@ -76,7 +76,7 @@ class BOOST_MPI_DECL binary_buffer_iprimitive typedef serialization::is_bitwise_serializable use_array_optimization; template - void load(serialization::array const& x) + void load(serialization::array_wrapper const& x) { load_array(x,0u); } diff --git a/include/boost/mpi/detail/binary_buffer_oprimitive.hpp b/include/boost/mpi/detail/binary_buffer_oprimitive.hpp index facf360..1de441d 100644 --- a/include/boost/mpi/detail/binary_buffer_oprimitive.hpp +++ b/include/boost/mpi/detail/binary_buffer_oprimitive.hpp @@ -55,7 +55,7 @@ class BOOST_MPI_DECL binary_buffer_oprimitive // fast saving of arrays template - void save_array(serialization::array const& x, unsigned int /* file_version */) + void save_array(serialization::array_wrapper const& x, unsigned int /* file_version */) { BOOST_MPL_ASSERT((serialization::is_bitwise_serializable::type>)); @@ -64,7 +64,7 @@ class BOOST_MPI_DECL binary_buffer_oprimitive } template - void save(serialization::array const& x) + void save(serialization::array_wrapper const& x) { save_array(x,0u); } diff --git a/include/boost/mpi/detail/forward_iprimitive.hpp b/include/boost/mpi/detail/forward_iprimitive.hpp index 44e7100..1d1f2fd 100644 --- a/include/boost/mpi/detail/forward_iprimitive.hpp +++ b/include/boost/mpi/detail/forward_iprimitive.hpp @@ -42,7 +42,7 @@ class forward_iprimitive /// loading of arrays is forwarded to the implementation archive template - void load_array(serialization::array & x, unsigned int file_version ) + void load_array(serialization::array_wrappe & x, unsigned int file_version ) { implementation_archive.load_array(x,file_version); } diff --git a/include/boost/mpi/detail/forward_oprimitive.hpp b/include/boost/mpi/detail/forward_oprimitive.hpp index cd232ae..78f7ca0 100644 --- a/include/boost/mpi/detail/forward_oprimitive.hpp +++ b/include/boost/mpi/detail/forward_oprimitive.hpp @@ -43,7 +43,7 @@ class forward_oprimitive /// saving of arrays is forwarded to the implementation archive template - void save_array(serialization::array const& x, unsigned int file_version ) + void save_array(serialization::array_wrapper const& x, unsigned int file_version ) { implementation_archive.save_array(x,file_version); } diff --git a/include/boost/mpi/detail/ignore_iprimitive.hpp b/include/boost/mpi/detail/ignore_iprimitive.hpp index eb3d2b7..151ed0b 100644 --- a/include/boost/mpi/detail/ignore_iprimitive.hpp +++ b/include/boost/mpi/detail/ignore_iprimitive.hpp @@ -37,7 +37,7 @@ class ignore_iprimitive /// don't do anything when loading arrays template - void load_array(serialization::array &, unsigned int ) + void load_array(serialization::array_wrapper &, unsigned int ) {} typedef is_mpi_datatype use_array_optimization; diff --git a/include/boost/mpi/detail/ignore_oprimitive.hpp b/include/boost/mpi/detail/ignore_oprimitive.hpp index 23375ca..4f2994b 100644 --- a/include/boost/mpi/detail/ignore_oprimitive.hpp +++ b/include/boost/mpi/detail/ignore_oprimitive.hpp @@ -36,7 +36,7 @@ class ignore_oprimitive /// don't do anything when saving arrays template - void save_array(serialization::array const&, unsigned int ) + void save_array(serialization::array_wrapper const&, unsigned int ) { } diff --git a/include/boost/mpi/detail/mpi_datatype_primitive.hpp b/include/boost/mpi/detail/mpi_datatype_primitive.hpp index 0a60785..c230055 100644 --- a/include/boost/mpi/detail/mpi_datatype_primitive.hpp +++ b/include/boost/mpi/detail/mpi_datatype_primitive.hpp @@ -63,7 +63,7 @@ class mpi_datatype_primitive // fast saving of arrays of MPI types template - void save_array(serialization::array const& x, unsigned int /* version */) + void save_array(serialization::array_wrapper const& x, unsigned int /* version */) { if (x.count()) save_impl(x.address(), boost::mpi::get_mpi_datatype(*x.address()), x.count()); diff --git a/include/boost/mpi/detail/packed_iprimitive.hpp b/include/boost/mpi/detail/packed_iprimitive.hpp index bb471a7..7080cbf 100644 --- a/include/boost/mpi/detail/packed_iprimitive.hpp +++ b/include/boost/mpi/detail/packed_iprimitive.hpp @@ -64,7 +64,7 @@ class BOOST_MPI_DECL packed_iprimitive // fast saving of arrays of fundamental types template - void load_array(serialization::array const& x, unsigned int /* file_version */) + void load_array(serialization::array_wrapper const& x, unsigned int /* file_version */) { if (x.count()) load_impl(x.address(), get_mpi_datatype(*x.address()), x.count()); @@ -72,7 +72,7 @@ class BOOST_MPI_DECL packed_iprimitive /* template - void load(serialization::array const& x) + void load(serialization::array_wrapper const& x) { load_array(x,0u); } diff --git a/include/boost/mpi/detail/packed_oprimitive.hpp b/include/boost/mpi/detail/packed_oprimitive.hpp index 5ac6835..5b6b3b2 100644 --- a/include/boost/mpi/detail/packed_oprimitive.hpp +++ b/include/boost/mpi/detail/packed_oprimitive.hpp @@ -54,7 +54,7 @@ class BOOST_MPI_DECL packed_oprimitive // fast saving of arrays template - void save_array(serialization::array const& x, unsigned int /* file_version */) + void save_array(serialization::array_wrapper const& x, unsigned int /* file_version */) { if (x.count()) save_impl(x.address(), get_mpi_datatype(*x.address()), x.count()); From e4be4411c8864e7339fdb1f612b7e150f17eaeee Mon Sep 17 00:00:00 2001 From: "K. Noel Belcourt" Date: Mon, 25 Apr 2016 13:15:25 -0600 Subject: [PATCH 14/99] Fix typo with rename to array_wrapper. --- include/boost/mpi/detail/forward_iprimitive.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/boost/mpi/detail/forward_iprimitive.hpp b/include/boost/mpi/detail/forward_iprimitive.hpp index 1d1f2fd..33d39fd 100644 --- a/include/boost/mpi/detail/forward_iprimitive.hpp +++ b/include/boost/mpi/detail/forward_iprimitive.hpp @@ -42,7 +42,7 @@ class forward_iprimitive /// loading of arrays is forwarded to the implementation archive template - void load_array(serialization::array_wrappe & x, unsigned int file_version ) + void load_array(serialization::array_wrapper & x, unsigned int file_version ) { implementation_archive.load_array(x,file_version); } From 1ae5eec15caddc5073c922975cb662d7c0a34467 Mon Sep 17 00:00:00 2001 From: "K. Noel Belcourt" Date: Sun, 1 May 2016 16:38:00 -0600 Subject: [PATCH 15/99] Fix deprecation warning by testing MPI_VERSION >= 2. Newer MPI uses MPI_Comm_set_errhandler and MPI_Comm_get_attr. --- src/environment.cpp | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/environment.cpp b/src/environment.cpp index c4e80b4..c1bf9e2 100644 --- a/src/environment.cpp +++ b/src/environment.cpp @@ -71,7 +71,11 @@ environment::environment(bool abort_on_exception) i_initialized = true; } +#if (2 <= MPI_VERSION) + MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN); +#else MPI_Errhandler_set(MPI_COMM_WORLD, MPI_ERRORS_RETURN); +#endif } environment::environment(threading::level mt_level, bool abort_on_exception) @@ -86,7 +90,11 @@ environment::environment(threading::level mt_level, bool abort_on_exception) i_initialized = true; } +#if (2 <= MPI_VERSION) + MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN); +#else MPI_Errhandler_set(MPI_COMM_WORLD, MPI_ERRORS_RETURN); +#endif } #endif @@ -99,7 +107,11 @@ environment::environment(int& argc, char** &argv, bool abort_on_exception) i_initialized = true; } +#if (2 <= MPI_VERSION) + MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN); +#else MPI_Errhandler_set(MPI_COMM_WORLD, MPI_ERRORS_RETURN); +#endif } environment::environment(int& argc, char** &argv, threading::level mt_level, @@ -115,7 +127,11 @@ environment::environment(int& argc, char** &argv, threading::level mt_level, i_initialized = true; } +#if (2 <= MPI_VERSION) + MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN); +#else MPI_Errhandler_set(MPI_COMM_WORLD, MPI_ERRORS_RETURN); +#endif } environment::~environment() @@ -154,8 +170,13 @@ int environment::max_tag() int* max_tag_value; int found = 0; +#if (2 <= MPI_VERSION) + BOOST_MPI_CHECK_RESULT(MPI_Comm_get_attr, + (MPI_COMM_WORLD, MPI_TAG_UB, &max_tag_value, &found)); +#else BOOST_MPI_CHECK_RESULT(MPI_Attr_get, (MPI_COMM_WORLD, MPI_TAG_UB, &max_tag_value, &found)); +#endif assert(found != 0); return *max_tag_value - num_reserved_tags; } @@ -170,8 +191,13 @@ optional environment::host_rank() int* host; int found = 0; +#if (2 <= MPI_VERSION) + BOOST_MPI_CHECK_RESULT(MPI_Comm_get_attr, + (MPI_COMM_WORLD, MPI_HOST, &host, &found)); +#else BOOST_MPI_CHECK_RESULT(MPI_Attr_get, (MPI_COMM_WORLD, MPI_HOST, &host, &found)); +#endif if (!found || *host == MPI_PROC_NULL) return optional(); else @@ -183,8 +209,13 @@ optional environment::io_rank() int* io; int found = 0; +#if (2 <= MPI_VERSION) + BOOST_MPI_CHECK_RESULT(MPI_Comm_get_attr, + (MPI_COMM_WORLD, MPI_IO, &io, &found)); +#else BOOST_MPI_CHECK_RESULT(MPI_Attr_get, (MPI_COMM_WORLD, MPI_IO, &io, &found)); +#endif if (!found || *io == MPI_PROC_NULL) return optional(); else From 004df1037e91595f1996ffc5ef32584f78b8ed3e Mon Sep 17 00:00:00 2001 From: Jonathan Wakely Date: Mon, 23 May 2016 18:33:11 +0100 Subject: [PATCH 16/99] Fix spelling in docs and config.hpp (#35) --- doc/mpi.qbk | 2 +- include/boost/mpi/config.hpp | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/mpi.qbk b/doc/mpi.qbk index 8f69adc..ec37f89 100644 --- a/doc/mpi.qbk +++ b/doc/mpi.qbk @@ -161,7 +161,7 @@ appears to be working with a C++ compiler and we're ready to move on. [section:config Configure and Build] -[section:bjam Build Environement] +[section:bjam Build Environment] As the rest of Boost, Boost.MPI uses version 2 of the [@http://www.boost.org/doc/html/bbv2.html Boost.Build] system for diff --git a/include/boost/mpi/config.hpp b/include/boost/mpi/config.hpp index ff91b17..c83277f 100644 --- a/include/boost/mpi/config.hpp +++ b/include/boost/mpi/config.hpp @@ -20,15 +20,15 @@ #include #include -/** @brief Comment this macro is you are running in an heterogeneous environement. +/** @brief Comment this macro is you are running in an heterogeneous environment. * - * When this flags is enabled, we assume some simple, POD like, type can be - * transmited without paying the cost of portable serialization. + * When this flag is enabled, we assume some simple, POD-like, type can be + * transmitted without paying the cost of portable serialization. * * Comment this if your platform is not homogeneous and that portable * serialization/deserialization must be performed. * - * It you do so, check that you MPI implementation supports thats kind of environement. + * It you do so, check that your MPI implementation supports thats kind of environment. */ #define BOOST_MPI_HOMOGENEOUS @@ -57,7 +57,7 @@ # define BOOST_MPI_HAS_NOARG_INITIALIZATION #else // If this is an MPI-1.x implementation, no arg initialization for -// mpi environement could still be available, but not mandatory. +// mpi environment could still be available, but not mandatory. // Undef this if no arg init is available: //# define BOOST_MPI_HAS_NOARG_INITIALIZATION #endif From 85bb58d75be9a3129e3132df7386e002ffc86525 Mon Sep 17 00:00:00 2001 From: "K. Noel Belcourt" Date: Thu, 21 Jul 2016 21:48:02 -0600 Subject: [PATCH 17/99] Add send and recv overloads for std::vector. --- include/boost/mpi/communicator.hpp | 70 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/include/boost/mpi/communicator.hpp b/include/boost/mpi/communicator.hpp index 46f7375..af616a3 100644 --- a/include/boost/mpi/communicator.hpp +++ b/include/boost/mpi/communicator.hpp @@ -272,6 +272,15 @@ class BOOST_MPI_DECL communicator template void send(int dest, int tag, const T& value) const; + template + void send(int dest, int tag, const std::vector& value) const; + + template + void send_vector(int dest, int tag, const std::vector& value, mpl::true_) const; + + template + void send_vector(int dest, int tag, const std::vector& value, mpl::false_) const; + /** * @brief Send the skeleton of an object. * @@ -384,6 +393,17 @@ class BOOST_MPI_DECL communicator template status recv(int source, int tag, T& value) const; + template + status recv(int source, int tag, std::vector& value) const; + + template + status recv_vector(int source, int tag, std::vector& value + , mpl::true_) const; + + template + status recv_vector(int source, int tag, std::vector& value + , mpl::false_) const; + /** * @brief Receive a skeleton from a remote process. * @@ -1187,6 +1207,30 @@ communicator::array_send_impl(int dest, int tag, const T* values, int n, send(dest, tag, oa); } +template +void communicator::send_vector(int dest, int tag, const std::vector& value + , mpl::true_ true_type) const +{ + // send the vector size + typename std::vector::size_type size = value.size(); + send(dest, tag, size); + // send the data + this->array_send_impl(dest, tag, value.data(), size, true_type); +} + +template +void communicator::send_vector(int dest, int tag, const std::vector& value + , mpl::false_ false_type) const +{ + this->send_impl(dest, tag, value, false_type); +} + +template +void communicator::send(int dest, int tag, const std::vector& value) const +{ + send_vector(dest, tag, value, is_mpi_datatype()); +} + // Array send must send the elements directly template void communicator::send(int dest, int tag, const T* values, int n) const @@ -1269,6 +1313,32 @@ communicator::array_recv_impl(int source, int tag, T* values, int n, return stat; } +template +status communicator::recv_vector(int source, int tag, std::vector& value + , mpl::true_ true_type) const +{ + // receive the vector size + typename std::vector::size_type size = 0; + recv(source, tag, size); + // size the vector + value.resize(size); + // receive the data + return this->array_recv_impl(source, tag, value.data(), size, true_type); +} + +template +status communicator::recv_vector(int source, int tag, std::vector& value + , mpl::false_ false_type) const +{ + return this->recv_impl(source, tag, value, false_type); +} + +template +status communicator::recv(int source, int tag, std::vector& value) const +{ + return recv_vector(source, tag, value, is_mpi_datatype()); +} + // Array receive must receive the elements directly into a buffer. template status communicator::recv(int source, int tag, T* values, int n) const From 3eabcce9b60059be53917c9008a879e510916e23 Mon Sep 17 00:00:00 2001 From: "K. Noel Belcourt" Date: Fri, 22 Jul 2016 12:14:58 -0600 Subject: [PATCH 18/99] Add allocator support to vector send recv api. Added test case to send recv a vector of udt with an overload of get_mpi_datatype. Added test to Jamfile for nightly testing. --- include/boost/mpi/communicator.hpp | 68 ++++++++++++++++-------------- test/Jamfile.v2 | 1 + test/sendrecv_vector.cpp | 86 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 123 insertions(+), 32 deletions(-) create mode 100644 test/sendrecv_vector.cpp diff --git a/include/boost/mpi/communicator.hpp b/include/boost/mpi/communicator.hpp index af616a3..a491086 100644 --- a/include/boost/mpi/communicator.hpp +++ b/include/boost/mpi/communicator.hpp @@ -1,4 +1,5 @@ // Copyright (C) 2005, 2006 Douglas Gregor . +// Copyright (C) 2016 K. Noel Belcourt . // Use, modification and distribution is subject to the Boost Software // License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at @@ -23,6 +24,7 @@ #include #include #include // for std::range_error +#include // For (de-)serializing sends and receives #include @@ -272,14 +274,16 @@ class BOOST_MPI_DECL communicator template void send(int dest, int tag, const T& value) const; - template - void send(int dest, int tag, const std::vector& value) const; + template + void send(int dest, int tag, const std::vector& value) const; - template - void send_vector(int dest, int tag, const std::vector& value, mpl::true_) const; + template + void send_vector(int dest, int tag, const std::vector& value, + mpl::true_) const; - template - void send_vector(int dest, int tag, const std::vector& value, mpl::false_) const; + template + void send_vector(int dest, int tag, const std::vector& value, + mpl::false_) const; /** * @brief Send the skeleton of an object. @@ -393,16 +397,16 @@ class BOOST_MPI_DECL communicator template status recv(int source, int tag, T& value) const; - template - status recv(int source, int tag, std::vector& value) const; + template + status recv(int source, int tag, std::vector& value) const; - template - status recv_vector(int source, int tag, std::vector& value - , mpl::true_) const; + template + status recv_vector(int source, int tag, std::vector& value, + mpl::true_) const; - template - status recv_vector(int source, int tag, std::vector& value - , mpl::false_) const; + template + status recv_vector(int source, int tag, std::vector& value, + mpl::false_) const; /** * @brief Receive a skeleton from a remote process. @@ -1207,26 +1211,26 @@ communicator::array_send_impl(int dest, int tag, const T* values, int n, send(dest, tag, oa); } -template -void communicator::send_vector(int dest, int tag, const std::vector& value - , mpl::true_ true_type) const +template +void communicator::send_vector(int dest, int tag, + const std::vector& value, mpl::true_ true_type) const { // send the vector size - typename std::vector::size_type size = value.size(); + typename std::vector::size_type size = value.size(); send(dest, tag, size); // send the data this->array_send_impl(dest, tag, value.data(), size, true_type); } -template -void communicator::send_vector(int dest, int tag, const std::vector& value - , mpl::false_ false_type) const +template +void communicator::send_vector(int dest, int tag, + const std::vector& value, mpl::false_ false_type) const { this->send_impl(dest, tag, value, false_type); } -template -void communicator::send(int dest, int tag, const std::vector& value) const +template +void communicator::send(int dest, int tag, const std::vector& value) const { send_vector(dest, tag, value, is_mpi_datatype()); } @@ -1313,12 +1317,12 @@ communicator::array_recv_impl(int source, int tag, T* values, int n, return stat; } -template -status communicator::recv_vector(int source, int tag, std::vector& value - , mpl::true_ true_type) const +template +status communicator::recv_vector(int source, int tag, + std::vector& value, mpl::true_ true_type) const { // receive the vector size - typename std::vector::size_type size = 0; + typename std::vector::size_type size = 0; recv(source, tag, size); // size the vector value.resize(size); @@ -1326,15 +1330,15 @@ status communicator::recv_vector(int source, int tag, std::vector& value return this->array_recv_impl(source, tag, value.data(), size, true_type); } -template -status communicator::recv_vector(int source, int tag, std::vector& value - , mpl::false_ false_type) const +template +status communicator::recv_vector(int source, int tag, + std::vector& value, mpl::false_ false_type) const { return this->recv_impl(source, tag, value, false_type); } -template -status communicator::recv(int source, int tag, std::vector& value) const +template +status communicator::recv(int source, int tag, std::vector& value) const { return recv_vector(source, tag, value, is_mpi_datatype()); } diff --git a/test/Jamfile.v2 b/test/Jamfile.v2 index 6c31448..5973eb2 100644 --- a/test/Jamfile.v2 +++ b/test/Jamfile.v2 @@ -41,5 +41,6 @@ test-suite mpi [ mpi-test graph_topology_test : : : 2 7 13 ] [ mpi-test pointer_test : : : 2 ] [ mpi-test groups_test ] + [ mpi-test sendrecv_vector : : : 2 ] ; } diff --git a/test/sendrecv_vector.cpp b/test/sendrecv_vector.cpp new file mode 100644 index 0000000..0d032e9 --- /dev/null +++ b/test/sendrecv_vector.cpp @@ -0,0 +1,86 @@ +// Author: K. Noel Belcourt + +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +#include +#include +#include + +#include "boost/mpi/environment.hpp" +#include "boost/mpi/communicator.hpp" + +using std::array; +using std::vector; + +namespace mpi = boost::mpi; + +struct blob : array, array, array { +}; + +template <> +struct mpi::is_mpi_datatype : mpl::true_ { +}; + +template <> +MPI_Datatype +mpi::get_mpi_datatype(const blob& b) +{ + const array block_lengths{ + { 9, 3, 5 } + }; + + const array displacements{ + { 0, 40, 64 } + }; + + const array datatypes{ + { MPI_INT, MPI_DOUBLE, MPI_CHAR } + }; + + MPI_Datatype blob_type; + MPI_Type_create_struct(block_lengths.size() + , (const int *)block_lengths.data(), displacements.data(), + datatypes.data(), &blob_type); + + MPI_Type_commit(&blob_type); + return blob_type; + +} + +int main(int argc, char* argv[]) { + mpi::environment env(argc, argv); + mpi::communicator world; + + vector data; + + if (world.rank() == 0) { + int size = 10000000; + data.resize(size); + // initialize data at vector ends + blob& b1= data[0]; + array& i = b1; + i[0] = -1; +#if 0 + blob& b2= data[size-1]; + array& c = b2; + c[4] = 'a'; +#endif + world.send(1, 0, data); + } + else { + world.recv(0, 0, data); + // check data at vector ends + blob& b1 = data[0]; + array& i = b1; + assert(i[0] == -1); +#if 0 + blob& b2 = data[data.size()-1]; + array& c = b2; + assert(c[4] == 'a'); +#endif + } + + return 0; +} From cbefb9c312b0910735a9d5848c89910ff45745fa Mon Sep 17 00:00:00 2001 From: Noel Belcourt Date: Fri, 22 Jul 2016 13:59:07 -0600 Subject: [PATCH 19/99] Update test case to check values communicated correctly. Add remark in Jamfile that test case requires -std=c++11. --- test/Jamfile.v2 | 1 + test/sendrecv_vector.cpp | 66 +++++++++++++++++++++++++----------------------- 2 files changed, 35 insertions(+), 32 deletions(-) diff --git a/test/Jamfile.v2 b/test/Jamfile.v2 index 5973eb2..35a522d 100644 --- a/test/Jamfile.v2 +++ b/test/Jamfile.v2 @@ -41,6 +41,7 @@ test-suite mpi [ mpi-test graph_topology_test : : : 2 7 13 ] [ mpi-test pointer_test : : : 2 ] [ mpi-test groups_test ] + # tests that require -std=c++11 [ mpi-test sendrecv_vector : : : 2 ] ; } diff --git a/test/sendrecv_vector.cpp b/test/sendrecv_vector.cpp index 0d032e9..ffebdf8 100644 --- a/test/sendrecv_vector.cpp +++ b/test/sendrecv_vector.cpp @@ -16,38 +16,44 @@ using std::vector; namespace mpi = boost::mpi; -struct blob : array, array, array { +struct blob : array, array, array { }; -template <> -struct mpi::is_mpi_datatype : mpl::true_ { -}; +namespace boost { + namespace mpi { -template <> -MPI_Datatype -mpi::get_mpi_datatype(const blob& b) -{ - const array block_lengths{ - { 9, 3, 5 } - }; + template <> + struct is_mpi_datatype : mpl::true_ { + }; - const array displacements{ - { 0, 40, 64 } - }; + template <> + MPI_Datatype get_mpi_datatype(const blob& b) { + array block_lengths{ + { 9, 3, 8 } + }; - const array datatypes{ - { MPI_INT, MPI_DOUBLE, MPI_CHAR } - }; + array displacements{ + { 0, 40, 64 } + }; - MPI_Datatype blob_type; - MPI_Type_create_struct(block_lengths.size() - , (const int *)block_lengths.data(), displacements.data(), - datatypes.data(), &blob_type); + array datatypes{ + { MPI_INT, MPI_DOUBLE, MPI_CHAR } + }; - MPI_Type_commit(&blob_type); - return blob_type; + MPI_Datatype blob_type; + MPI_Type_create_struct( + block_lengths.size() + , reinterpret_cast(block_lengths.data()) + , displacements.data() + , datatypes.data() + , &blob_type); -} + MPI_Type_commit(&blob_type); + return blob_type; + } + + } // namespace mpi +} // namespace boost int main(int argc, char* argv[]) { mpi::environment env(argc, argv); @@ -62,11 +68,9 @@ int main(int argc, char* argv[]) { blob& b1= data[0]; array& i = b1; i[0] = -1; -#if 0 blob& b2= data[size-1]; - array& c = b2; - c[4] = 'a'; -#endif + array& d = b2; + d[2] = -17; world.send(1, 0, data); } else { @@ -75,11 +79,9 @@ int main(int argc, char* argv[]) { blob& b1 = data[0]; array& i = b1; assert(i[0] == -1); -#if 0 blob& b2 = data[data.size()-1]; - array& c = b2; - assert(c[4] == 'a'); -#endif + array& d = b2; + assert(d[2] == -17); } return 0; From 330bf8c1780a80b327d0f784c10ccc089ef6ab1f Mon Sep 17 00:00:00 2001 From: "K. Noel Belcourt" Date: Sun, 24 Jul 2016 13:27:30 -0600 Subject: [PATCH 20/99] Disable for stds before c++11. --- test/sendrecv_vector.cpp | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/test/sendrecv_vector.cpp b/test/sendrecv_vector.cpp index ffebdf8..305628e 100644 --- a/test/sendrecv_vector.cpp +++ b/test/sendrecv_vector.cpp @@ -4,6 +4,8 @@ // (See accompanying file LICENSE_1_0.txt or copy at // http://www.boost.org/LICENSE_1_0.txt) +#if defined(__cplusplus) && (201103L <= __cplusplus) + #include #include #include @@ -55,14 +57,19 @@ namespace boost { } // namespace mpi } // namespace boost +#endif // defined(__cplusplus) + + int main(int argc, char* argv[]) { +#if defined(__cplusplus) && (201103L <= __cplusplus) + mpi::environment env(argc, argv); mpi::communicator world; vector data; if (world.rank() == 0) { - int size = 10000000; + int size = 10; data.resize(size); // initialize data at vector ends blob& b1= data[0]; @@ -79,10 +86,11 @@ int main(int argc, char* argv[]) { blob& b1 = data[0]; array& i = b1; assert(i[0] == -1); - blob& b2 = data[data.size()-1]; - array& d = b2; - assert(d[2] == -17); + // blob& b2 = data[data.size()-1]; + // array& d = b2; + // assert(d[2] == -17); } +#endif // defined(__cplusplus) return 0; } From e33c16da78259bf81d913baeb4283a11859910be Mon Sep 17 00:00:00 2001 From: Noel Belcourt Date: Sun, 24 Jul 2016 13:33:02 -0600 Subject: [PATCH 21/99] Bump test size to 1e7 items in vector. --- test/sendrecv_vector.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/sendrecv_vector.cpp b/test/sendrecv_vector.cpp index 305628e..29b9751 100644 --- a/test/sendrecv_vector.cpp +++ b/test/sendrecv_vector.cpp @@ -69,7 +69,7 @@ int main(int argc, char* argv[]) { vector data; if (world.rank() == 0) { - int size = 10; + int size = 10000000; data.resize(size); // initialize data at vector ends blob& b1= data[0]; From 1c082e1ff8e5805f08a6077adc6973fa228f53cb Mon Sep 17 00:00:00 2001 From: Florian Weik Date: Tue, 20 Sep 2016 16:37:46 -0600 Subject: [PATCH 22/99] specialization of is_mpi_datatype for std::array --- include/boost/mpi/datatype.hpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/include/boost/mpi/datatype.hpp b/include/boost/mpi/datatype.hpp index c26dfdf..1f06997 100644 --- a/include/boost/mpi/datatype.hpp +++ b/include/boost/mpi/datatype.hpp @@ -30,6 +30,10 @@ #include #include // for std::pair +#if defined(__cplusplus) && (201103L <= __cplusplus) +#include +#endif + namespace boost { namespace mpi { /** @@ -265,6 +269,15 @@ struct is_mpi_datatype > { }; +/// specialization of is_mpi_datatype for arrays +#if defined(__cplusplus) && (201103L <= __cplusplus) +template +struct is_mpi_datatype > + : public is_mpi_datatype +{ +}; +#endif + // Define wchar_t specialization of is_mpi_datatype, if possible. #if !defined(BOOST_NO_INTRINSIC_WCHAR_T) && \ (defined(MPI_WCHAR) || (defined(MPI_VERSION) && MPI_VERSION >= 2)) From e6349ee63f487c9e15918c300af0b9b9d7893058 Mon Sep 17 00:00:00 2001 From: Rene Rivera Date: Fri, 7 Oct 2016 23:07:35 -0500 Subject: [PATCH 23/99] Add, and update, documentation build targets. --- doc/Jamfile.v2 | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/doc/Jamfile.v2 b/doc/Jamfile.v2 index 92216e6..a8226f3 100644 --- a/doc/Jamfile.v2 +++ b/doc/Jamfile.v2 @@ -49,4 +49,12 @@ boostbook standalone : mpi mpi_autodoc pdf:boost.url.prefix=http://www.boost.org/doc/libs/release/doc/html ; - +############################################################################### +alias boostdoc + : mpi + : + : mpi_autodoc + : ; +explicit boostdoc ; +alias boostrelease ; +explicit boostrelease ; From ebb22ff06ba1c3386c05fb9ed6e7ee11962e9d84 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Sat, 15 Oct 2016 22:45:18 +0200 Subject: [PATCH 24/99] typo in doc --- include/boost/mpi/cartesian_communicator.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/boost/mpi/cartesian_communicator.hpp b/include/boost/mpi/cartesian_communicator.hpp index 2b9d623..16db3de 100644 --- a/include/boost/mpi/cartesian_communicator.hpp +++ b/include/boost/mpi/cartesian_communicator.hpp @@ -225,7 +225,7 @@ class BOOST_MPI_DECL cartesian_communicator : public communicator */ int rank(const std::vector& coords) const; /** - * Return the rank of the source and targetdestination process through a shift. + * Return the rank of the source and target destination process through a shift. * @param dim the dimension in which the shift takes place. 0 <= dim <= ndim(). * @param disp the shift displacement, can be positive (upward) or negative (downward). */ From cd2db2b9d63a9f9e834dc95cc59f19f1375a1793 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Sat, 15 Oct 2016 22:46:06 +0200 Subject: [PATCH 25/99] use plain assert instead of boost's --- src/cartesian_communicator.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/cartesian_communicator.cpp b/src/cartesian_communicator.cpp index 147b41a..8fc7519 100644 --- a/src/cartesian_communicator.cpp +++ b/src/cartesian_communicator.cpp @@ -7,6 +7,7 @@ // Authors: Alain Miniussi #include +#include #include @@ -70,10 +71,10 @@ cartesian_communicator::cartesian_communicator(const cartesian_communicator& com { int const max_dims = comm.ndims(); int const nbkept = keep.size(); - BOOST_ASSERT(nbkept <= max_dims); + assert(nbkept <= max_dims); std::vector bitset(max_dims, int(false)); for(int i = 0; i < nbkept; ++i) { - BOOST_ASSERT(keep[i] < max_dims); + assert(keep[i] < max_dims); bitset[keep[i]] = true; } @@ -96,7 +97,7 @@ cartesian_communicator::ndims() const { int cartesian_communicator::rank(const std::vector& coords ) const { int r = -1; - BOOST_ASSERT(int(coords.size()) == ndims()); + assert(int(coords.size()) == ndims()); BOOST_MPI_CHECK_RESULT(MPI_Cart_rank, (MPI_Comm(*this), const_cast&>(coords).data(), &r)); @@ -106,8 +107,7 @@ cartesian_communicator::rank(const std::vector& coords ) const { std::pair cartesian_communicator::shifted_ranks(int dim, int disp) const { std::pair r(-1,-1); - assert((0 <= dim && dim < ndims()) || (std::abort(), false)); - BOOST_ASSERT(0 <= dim && dim < ndims()); + assert(0 <= dim && dim < ndims()); BOOST_MPI_CHECK_RESULT(MPI_Cart_shift, (MPI_Comm(*this), dim, disp, &(r.first), &(r.second))); return r; From 3117df1b364309f971d640e39e86bd0c6933b22c Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Sat, 15 Oct 2016 22:46:45 +0200 Subject: [PATCH 26/99] only test shift on periodic dmentions --- test/cartesian_topology_test.cpp | 75 +++++++++++++++++++++++++--------------- 1 file changed, 47 insertions(+), 28 deletions(-) diff --git a/test/cartesian_topology_test.cpp b/test/cartesian_topology_test.cpp index 893c9a1..bc96148 100644 --- a/test/cartesian_topology_test.cpp +++ b/test/cartesian_topology_test.cpp @@ -42,14 +42,19 @@ std::string topology_description( mpi::cartesian_topology const& topo ) { void test_coordinates_consistency( mpi::cartesian_communicator const& cc, std::vector const& coords ) { + cc.barrier(); // flush IOs for nice printing + bool master = cc.rank() == 0; + if (master) { + std::cout << "Test coordinates consistency.\n"; + } for(int p = 0; p < cc.size(); ++p) { std::vector min(cc.ndims()); std::vector local(cc.coords(p)); mpi::reduce(cc, local.data(), local.size(), min.data(), mpi::minimum(), p); + cc.barrier(); if (p == cc.rank()) { - BOOST_CHECK(std::equal(coords.begin(), coords.end(), - min.begin())); + BOOST_CHECK(std::equal(coords.begin(), coords.end(), min.begin())); std::ostringstream out; out << "proc " << p << " at ("; std::copy(min.begin(), min.end(), std::ostream_iterator(out, " ")); @@ -61,29 +66,34 @@ void test_coordinates_consistency( mpi::cartesian_communicator const& cc, void test_shifted_coords( mpi::cartesian_communicator const& cc, int pos, mpi::cartesian_dimension desc, int dim ) { - for (int i = -(desc.size); i < desc.size; ++i) { - std::pair rks = cc.shifted_ranks(pos, i); - int src = cc.coords(rks.first)[dim]; - int dst = cc.coords(rks.second)[dim]; - if (pos == (dim/2)) { - std::ostringstream out; - out << "Rank " << cc.rank() << ", dim. " << dim << ", pos " << pos << ", in " << desc << ' '; - out << "shifted pos: " << src << ", " << dst << '\n'; - std::cout << out.str(); + if (desc.periodic) { + for (int i = -(desc.size); i < desc.size; ++i) { + std::pair rks = cc.shifted_ranks(dim, i); + int src = cc.coords(rks.first)[dim]; + int dst = cc.coords(rks.second)[dim]; + if (pos == (dim/2)) { + std::ostringstream out; + out << "Rank " << cc.rank() << ", dim. " << dim << ", pos " << pos << ", in " << desc << ' '; + out << "shifted pos: " << src << ", " << dst << '\n'; + std::cout << out.str(); + } } } } void test_shifted_coords( mpi::cartesian_communicator const& cc) { - std::vector coords; mpi::cartesian_topology topo(cc.ndims()); + cc.barrier(); // flush IOs for nice printing + std::vector coords; + mpi::cartesian_topology topo(cc.ndims()); cc.topology(topo, coords); - if (cc.rank() == 0) { + bool master = cc.rank() == 0; + if (master) { std::cout << "Testing shifts with topology " << topo << '\n'; } for(int i = 0; i < cc.ndims(); ++i) { - if (cc.rank() == 0) { - std::cout << " for dimension " << i << ' ' << topo[i] << '\n'; + if (master) { + std::cout << " for dimension " << i << ": " << topo[i] << '\n'; } test_shifted_coords( cc, coords[i], topo[i], i ); } @@ -91,19 +101,22 @@ void test_shifted_coords( mpi::cartesian_communicator const& cc) void test_topology_consistency( mpi::cartesian_communicator const& cc) { + cc.barrier(); // flush IOs for nice printing mpi::cartesian_topology itopo(cc.ndims()); mpi::cartesian_topology otopo(cc.ndims()); std::vector coords(cc.ndims()); cc.topology(itopo, coords); - - // Check that everyone agrees on the dimensions + bool master = cc.rank() == 0; + if (master) { + std::cout << "Test topology consistency of" << itopo << "(on master)\n"; + std::cout << "Check that everyone agrees on the dimensions.\n"; + } mpi::all_reduce(cc, &(itopo[0]), itopo.size(), &(otopo[0]), topo_minimum()); - BOOST_CHECK(std::equal(itopo.begin(), itopo.end(), - otopo.begin())); - if (cc.rank() == 0) { - std::cout << topology_description(otopo) << '\n'; + BOOST_CHECK(std::equal(itopo.begin(), itopo.end(), otopo.begin())); + if (master) { + std::cout << "We agree on " << topology_description(otopo) << '\n'; } test_coordinates_consistency( cc, coords ); } @@ -117,18 +130,18 @@ void test_cartesian_topology( mpi::cartesian_communicator const& cc) std::vector coords = cc.coords(r); std::cout << "Process of cartesian rank " << cc.rank() << " has coordinates ("; - std::copy(coords.begin(), coords.end(), std::ostream_iterator(std::cout,",")); + std::copy(coords.begin(), coords.end(), std::ostream_iterator(std::cout," ")); std::cout << ")\n"; } } test_topology_consistency(cc); + test_shifted_coords(cc); std::vector even; for(int i = 0; i < cc.ndims(); i += 2) { even.push_back(i); } + cc.barrier(); mpi::cartesian_communicator cce(cc, even); - test_topology_consistency(cce); - test_shifted_coords(cce); } void test_cartesian_topology( mpi::communicator const& world, mpi::cartesian_topology const& topo) @@ -157,11 +170,17 @@ int test_main(int argc, char* argv[]) mpi::cartesian_topology topo(ndim); typedef mpi::cartesian_dimension cd; if (topo.size() == 3) { - topo[0] = cd(2,true); topo[1] = cd(3,false); topo[2] = cd(4, true); - } else if (world.size() >= 6) { - topo[0] = cd(0,true); topo[1] = cd(3, false); + topo[0] = cd(2,true); + topo[1] = cd(3,false); + topo[2] = cd(4, true); } else { - topo[0] = cd(0,true); topo[1] = cd(0, false); + if (world.size() >= 6) { + topo[0] = cd(2,true); + topo[1] = cd(3, false); + } else { + topo[0] = cd(1,true); + topo[1] = cd(1, false); + } } test_cartesian_topology( world, topo); From 7d73ef198fdf607c1db44c6c0eb00a028f42dbfc Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Wed, 19 Oct 2016 14:43:59 +0200 Subject: [PATCH 27/99] Added a few topology ctors. Documentation --- include/boost/mpi/cartesian_communicator.hpp | 119 ++++++++++++++++++++++++--- 1 file changed, 109 insertions(+), 10 deletions(-) diff --git a/include/boost/mpi/cartesian_communicator.hpp b/include/boost/mpi/cartesian_communicator.hpp index 16db3de..159c7c2 100644 --- a/include/boost/mpi/cartesian_communicator.hpp +++ b/include/boost/mpi/cartesian_communicator.hpp @@ -24,10 +24,12 @@ #include #include #include +#include // Headers required to implement cartesian topologies #include #include +#include namespace boost { namespace mpi { @@ -55,12 +57,18 @@ struct cartesian_dimension { template <> struct is_mpi_datatype : mpl::true_ { }; +/** + * @brief Test if the dimensions values are identical. + */ inline bool operator==(cartesian_dimension const& d1, cartesian_dimension const& d2) { - return d1.size == d2.size && d1.periodic == d2.periodic; + return &d1 == &d2 || (d1.size == d2.size && d1.periodic == d2.periodic); } +/** + * @brief Test if the dimension values are different. + */ inline bool operator!=(cartesian_dimension const& d1, cartesian_dimension const& d2) { @@ -83,34 +91,125 @@ class BOOST_MPI_DECL cartesian_topology friend class cartesian_communicator; typedef std::vector super; public: + /** + * Retrieve a specific dimension. + */ using super::operator[]; + /** + * @brief Topology dimentionality. + */ using super::size; using super::begin; using super::end; using super::swap; - cartesian_topology(int sz) - : super(sz) {} + /** + * @brief Create a N dimension space. + * Each dimension is initialized as non periodic of size 0. + */ + cartesian_topology(int ndim) + : super(ndim) {} + + /** + * @brief Use the provided dimensions specification as initial values. + */ + cartesian_topology(std::vector const& dims) + : super(dims) {} + /** + * @brief Use dimensions specification provided in the sequence container as initial values. + * #param dims must be a sequence container. + */ + template + explicit cartesian_topology(InitArr dims) + : super(0) { + BOOST_FOREACH(cartesian_dimension const& d, dims) { + push_back(d); + } + } +#if !defined(BOOST_NO_CXX11_HDR_INITIALIZER_LIST) + /** + * @brief Use dimensions specification provided in the initialization list as initial values. + * #param dims can be of the form { dim_1, false}, .... {dim_n, true} + */ + explicit cartesian_topology(std::initializer_list dims) + : super(dims) {} +#endif + /** + * @brief Use dimensions specification provided in the array. + * #param dims can be of the form { dim_1, false}, .... {dim_n, true} + */ template - explicit cartesian_topology(array const& dims) - : super(NDIM) { - std::copy(dims.begin(), dims.end(), begin()); + explicit cartesian_topology(cartesian_dimension (&dims)[NDIM]) + : super(dims, dims+NDIM) {} + + /** + * @brief Use dimensions specification provided in the input ranges + * The ranges do not need to be the same size. If the sizes are different, + * the missing values will be complete with zeros of the dim and assumed non periodic. + * @param dim_rg the dimensions, values must convert to integers. + * @param period_rg the periodicities, values must convert to booleans. + * #param dims can be of the form { dim_1, false}, .... {dim_n, true} + */ + template + cartesian_topology(DimRg const& dim_rg, PerRg const& period_rg) + : super(0) { + BOOST_FOREACH(int d, dim_rg) { + super::push_back(cartesian_dimension(d)); + } + super::iterator it = begin(); + BOOST_FOREACH(bool p, period_rg) { + if (it < end()) { + it->periodic = p; + } else { + push_back(cartesian_dimension(0,p)); + } + ++it; + } } + + /** + * @brief Iterator based initializer. + * Will use the first n iterated values. + * Both iterators can be single pass. + * @param dit dimension iterator, value must convert to integer type. + * @param pit periodicity iterator, value must convert to booleans.. + */ template - cartesian_topology(DimIter dim_iter, PerIter period_iter, int ndim) - : super(ndim) { - for(int i = 0; i < ndim; ++i) { - (*this)[i] = cartesian_dimension(*dim_iter++, *period_iter++); + cartesian_topology(DimIter dit, PerIter pit, int n) + : super(n) { + for(int i = 0; i < n; ++i) { + (*this)[i] = cartesian_dimension(*dit++, *pit++); } } + + /** + * Export as an stl sequence. + */ + std::vector& stl() { return *this; } + /** + * Export as an stl sequence. + */ + std::vector const& stl() const{ return *this; } /** * Split the topology in two sequences of sizes and periodicities. */ void split(std::vector& dims, std::vector& periodics) const; }; +inline +bool +operator==(cartesian_topology const& t1, cartesian_topology const& t2) { + return t1.stl() == t2.stl(); +} + +inline +bool +operator!=(cartesian_topology const& t1, cartesian_topology const& t2) { + return t1.stl() != t2.stl(); +} + /** * @brief Pretty printing of a cartesian topology */ From 77d902c35b735776d65a98db6980d21bde6757bd Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Wed, 19 Oct 2016 14:44:36 +0200 Subject: [PATCH 28/99] Added topology initialization tests. --- test/cartesian_topology_init_test.cpp | 84 +++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 test/cartesian_topology_init_test.cpp diff --git a/test/cartesian_topology_init_test.cpp b/test/cartesian_topology_init_test.cpp new file mode 100644 index 0000000..6bcd547 --- /dev/null +++ b/test/cartesian_topology_init_test.cpp @@ -0,0 +1,84 @@ + +// Copyright Alain Miniussi 2014. +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +// Authors: Alain Miniussi + +#include +#include +#include +#include +#include +#include +#include + +#define BOOST_TEST_MODULE BoostMPI +#include + +#include +#include +#include +#include + +namespace mpi = boost::mpi; + + +BOOST_AUTO_TEST_CASE(cartesian_dimension_init) +{ + { + // Check the basic ctor + mpi::cartesian_dimension def; + mpi::cartesian_topology t1(10); + BOOST_CHECK(t1.stl() == std::vector(10, def)); + } +#if !defined(BOOST_NO_CXX11_HDR_INITIALIZER_LIST) + { + // Intializer list ctor vs range based + int dims[] = {2,3,4}; + bool per[] = {true, false, true}; + mpi::cartesian_topology t1(dims, per); + mpi::cartesian_topology t2({{2,true},{3, false},{4, true}}); + BOOST_CHECK(t1.size() == 3); + BOOST_CHECK(t1 == t2); + } +#endif + // Container based ctor only available as a replacement for initializer list ctor + { + // seq ctor vs C array ctor + mpi::cartesian_dimension d[] = {{2,true},{3, false},{4, true}}; + std::list seq; + std::copy(d, d+3, std::back_inserter(seq)); + mpi::cartesian_topology t1(seq); + mpi::cartesian_topology t2(d); + BOOST_CHECK(t1 == t2); + } + { + // Check range based with array based ctor. + boost::array d = {{{2,true},{3, false},{4, true}}}; + int dims[] = {2,3,4}; + bool per[] = {true, false, true}; + mpi::cartesian_topology t1(dims, per); + mpi::cartesian_topology t2(d); + BOOST_CHECK(t1.size() == 3); + BOOST_CHECK(t1 == t2); + } + { + // Iterator based ctor vs C array based ctor + mpi::cartesian_dimension d[] = {{2,true},{3, false},{4, true}}; + std::vector vdims(d, d+3); + mpi::cartesian_topology t1(vdims); + mpi::cartesian_topology t2(d); + BOOST_CHECK(t1.size() == 3); + BOOST_CHECK(t1 == t2); + BOOST_CHECK(!(t1 != t2)); + t1[1].periodic = true; + BOOST_CHECK(t1 != t2); + t1[2].periodic = false; + t1[2].size = 0; + vdims.push_back(mpi::cartesian_dimension(3, false)); + mpi::cartesian_topology t3(vdims); + BOOST_CHECK(t1 != t3); + } +} From 480fcebd70baccb041d5ab2bdde9018d1e5925f4 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Wed, 19 Oct 2016 14:45:12 +0200 Subject: [PATCH 29/99] Activate topology ctor tests. --- test/Jamfile.v2 | 1 + 1 file changed, 1 insertion(+) diff --git a/test/Jamfile.v2 b/test/Jamfile.v2 index 801e61b..e6ba795 100644 --- a/test/Jamfile.v2 +++ b/test/Jamfile.v2 @@ -17,6 +17,7 @@ if [ mpi.configured ] { test-suite mpi : + [ mpi-test cartesian_topology_init_test : : : 1 ] [ mpi-test broadcast_stl_test : : : 2 ] [ mpi-test all_gather_test ] [ mpi-test all_reduce_test ] From c41d5e847669cfd5cd5c233776c82a699615b8e0 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Wed, 19 Oct 2016 15:02:57 +0200 Subject: [PATCH 30/99] Renamed coords -> coordinates Removed inplace coortdinate retreiver (move ctor makes them useless) --- include/boost/mpi/cartesian_communicator.hpp | 9 +-------- src/cartesian_communicator.cpp | 14 ++++---------- test/cartesian_topology_test.cpp | 8 ++++---- 3 files changed, 9 insertions(+), 22 deletions(-) diff --git a/include/boost/mpi/cartesian_communicator.hpp b/include/boost/mpi/cartesian_communicator.hpp index 159c7c2..d053bdc 100644 --- a/include/boost/mpi/cartesian_communicator.hpp +++ b/include/boost/mpi/cartesian_communicator.hpp @@ -330,18 +330,11 @@ class BOOST_MPI_DECL cartesian_communicator : public communicator */ std::pair shifted_ranks(int dim, int disp) const; /** - * Provides the coordinates of a process with the given rank. - * @param rk the rank in this communicator. - * @param cbuf a buffer were to store the coordinates. - * @returns a reference to cbuf. - */ - std::vector& coords(int rk, std::vector& cbuf) const; - /** * Provides the coordinates of the process with the given rank. * @param rk the ranks in this communicator. * @returns the coordinates. */ - std::vector coords(int rk) const; + std::vector coordinates(int rk) const; /** * Retrieve the topology and coordinates of this process in the grid. * diff --git a/src/cartesian_communicator.cpp b/src/cartesian_communicator.cpp index 8fc7519..ad3178c 100644 --- a/src/cartesian_communicator.cpp +++ b/src/cartesian_communicator.cpp @@ -113,20 +113,13 @@ cartesian_communicator::shifted_ranks(int dim, int disp) const { return r; } -std::vector& -cartesian_communicator::coords(int rk, std::vector& cbuf) const { - cbuf.resize(ndims()); +std::vector +cartesian_communicator::coordinates(int rk) const { + std::vector cbuf(ndims()); BOOST_MPI_CHECK_RESULT(MPI_Cart_coords, (MPI_Comm(*this), rk, cbuf.size(), cbuf.data() )); return cbuf; } - -std::vector -cartesian_communicator::coords(int rk) const { - std::vector coords; - this->coords(rk, coords); - return coords; -} void cartesian_communicator::topology( cartesian_topology& topo, @@ -149,6 +142,7 @@ cartesian_communicator::topology() const { topology(topo, coords); return topo; } + void cartesian_topology::split(std::vector& dims, std::vector& periodics) const { int ndims = size(); diff --git a/test/cartesian_topology_test.cpp b/test/cartesian_topology_test.cpp index bc96148..3875a1b 100644 --- a/test/cartesian_topology_test.cpp +++ b/test/cartesian_topology_test.cpp @@ -49,7 +49,7 @@ void test_coordinates_consistency( mpi::cartesian_communicator const& cc, } for(int p = 0; p < cc.size(); ++p) { std::vector min(cc.ndims()); - std::vector local(cc.coords(p)); + std::vector local(cc.coordinates(p)); mpi::reduce(cc, local.data(), local.size(), min.data(), mpi::minimum(), p); cc.barrier(); @@ -69,8 +69,8 @@ void test_shifted_coords( mpi::cartesian_communicator const& cc, int pos, mpi:: if (desc.periodic) { for (int i = -(desc.size); i < desc.size; ++i) { std::pair rks = cc.shifted_ranks(dim, i); - int src = cc.coords(rks.first)[dim]; - int dst = cc.coords(rks.second)[dim]; + int src = cc.coordinates(rks.first)[dim]; + int dst = cc.coordinates(rks.second)[dim]; if (pos == (dim/2)) { std::ostringstream out; out << "Rank " << cc.rank() << ", dim. " << dim << ", pos " << pos << ", in " << desc << ' '; @@ -127,7 +127,7 @@ void test_cartesian_topology( mpi::cartesian_communicator const& cc) for( int r = 0; r < cc.size(); ++r) { cc.barrier(); if (r == cc.rank()) { - std::vector coords = cc.coords(r); + std::vector coords = cc.coordinates(r); std::cout << "Process of cartesian rank " << cc.rank() << " has coordinates ("; std::copy(coords.begin(), coords.end(), std::ostream_iterator(std::cout," ")); From 876ac2e6f4623b248f1e61f7946d1b1287c10fae Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Wed, 19 Oct 2016 15:05:03 +0200 Subject: [PATCH 31/99] Not worth a function. --- include/boost/mpi/cartesian_communicator.hpp | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/include/boost/mpi/cartesian_communicator.hpp b/include/boost/mpi/cartesian_communicator.hpp index d053bdc..415e46e 100644 --- a/include/boost/mpi/cartesian_communicator.hpp +++ b/include/boost/mpi/cartesian_communicator.hpp @@ -356,19 +356,6 @@ class BOOST_MPI_DECL cartesian_communicator : public communicator */ std::vector& cartesian_dimensions(int nb_proc, std::vector& dims); -/** - * Given en communicator and a partially filled sequence - * of dimension, try to complete the dimension sequence to produce an acceptable - * cartesian topology. - * @param comm the prospective parent communicator. - * @param dims a sequence of positive or null dimensions. Non zero dimension - * will be left untouched. - */ -inline -std::vector& cartesian_dimensions(const communicator& comm, std::vector& dims) { - return cartesian_dimensions(comm.size(), dims); -} - } } // end namespace boost::mpi #endif // BOOST_MPI_CARTESIAN_COMMUNICATOR_HPP From b5140a7dda26e4364c630f3c5684d342794297a5 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Wed, 19 Oct 2016 15:47:51 +0200 Subject: [PATCH 32/99] Add basic ctor/dtor to topology class. --- include/boost/mpi/cartesian_communicator.hpp | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/include/boost/mpi/cartesian_communicator.hpp b/include/boost/mpi/cartesian_communicator.hpp index 415e46e..0468659 100644 --- a/include/boost/mpi/cartesian_communicator.hpp +++ b/include/boost/mpi/cartesian_communicator.hpp @@ -85,6 +85,8 @@ std::ostream& operator<<(std::ostream& out, cartesian_dimension const& d); * * Behave mostly like a sequence of @c cartesian_dimension with the notable * exception that its size is fixed. + * This is a lightweight object, so that any constructor that could be considered + * missing could be replaced with a function (move constructor provided when supported). */ class BOOST_MPI_DECL cartesian_topology : private std::vector { @@ -103,6 +105,22 @@ class BOOST_MPI_DECL cartesian_topology using super::end; using super::swap; +#if !defined(BOOST_NO_CXX11_DELETED_FUNCTIONS) + cartesian_topology() = delete; +#endif +#if !defined(BOOST_NO_CXX11_DEFAULTED_FUNCTIONS) + cartesian_topology(cartesian_topology const&) = default; + cartesian_topology& operator=(cartesian_topology const&) = default; + // There is apparently no macro for checking the support of move constructor. + // Assume that defaulted function is close enough. + cartesian_topology(cartesian_topology const&& other) : super(other) {} + cartesian_topology& operator=(cartesian_topology const&& other) { + (*this) = std::move(other.stl()); + return *this; + } + + ~cartesian_topology() = default; +#endif /** * @brief Create a N dimension space. * Each dimension is initialized as non periodic of size 0. From 9a90ae9048ea1a4113db3ebba109ea1a12afb0e7 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Wed, 19 Oct 2016 15:49:36 +0200 Subject: [PATCH 33/99] Doc --- include/boost/mpi/cartesian_communicator.hpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/include/boost/mpi/cartesian_communicator.hpp b/include/boost/mpi/cartesian_communicator.hpp index 0468659..a353a63 100644 --- a/include/boost/mpi/cartesian_communicator.hpp +++ b/include/boost/mpi/cartesian_communicator.hpp @@ -34,7 +34,9 @@ namespace boost { namespace mpi { /** - * Specify the size and periodicity of the grid in a single dimension. + * @brief Specify the size and periodicity of the grid in a single dimension. + * + * POD lightweight object. */ struct cartesian_dimension { /** The size of the grid n this dimension. */ From 916e51dc62e6b041aadda09a8224cad5ef9aaa26 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Thu, 20 Oct 2016 16:11:12 +0200 Subject: [PATCH 34/99] =?UTF-8?q?Use=20broadcast=20imlementation=20propose?= =?UTF-8?q?d=20by=20Lorenz=20H=C3=BCbschle-Schneider=20(https://gist.githu?= =?UTF-8?q?b.com/lorenzhs/79dab54552fd1f9381da):=20ofr=20non=20primitive/m?= =?UTF-8?q?pi=20type,=20broadcast=20the=20archive=20content=20as=20a=20pri?= =?UTF-8?q?mitive=20byte=20array.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- include/boost/mpi/collectives/broadcast.hpp | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/include/boost/mpi/collectives/broadcast.hpp b/include/boost/mpi/collectives/broadcast.hpp index d5160cf..c48db8d 100644 --- a/include/boost/mpi/collectives/broadcast.hpp +++ b/include/boost/mpi/collectives/broadcast.hpp @@ -106,16 +106,31 @@ namespace detail { template void broadcast_impl(const communicator& comm, T* values, int n, int root, - mpl::false_) + mpl::false_ non_mpi_datatype) { + // Implementation proposed by Lorenz Hübschle-Schneider if (comm.rank() == root) { packed_oarchive oa(comm); - for (int i = 0; i < n; ++i) + for (int i = 0; i < n; ++i) { oa << values[i]; - broadcast(comm, oa, root); + } + std::size_t asize = oa.size(); + broadcast(comm, asize, root); + void const* aptr = oa.address(); + BOOST_MPI_CHECK_RESULT(MPI_Bcast, + (const_cast(aptr), asize, + MPI_BYTE, + root, MPI_Comm(comm))); } else { packed_iarchive ia(comm); - broadcast(comm, ia, root); + std::size_t asize; + broadcast(comm, asize, root); + ia.resize(asize); + void* aptr = ia.address(); + BOOST_MPI_CHECK_RESULT(MPI_Bcast, + (aptr, asize, + MPI_BYTE, + root, MPI_Comm(comm))); for (int i = 0; i < n; ++i) ia >> values[i]; } From 5e4beaccda858670d31cd7c8d469d86584e6b34b Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Thu, 20 Oct 2016 19:16:27 +0200 Subject: [PATCH 35/99] Implement gather without any point to point communication. --- include/boost/mpi/collectives/gather.hpp | 144 +++++++++++++++++++------------ 1 file changed, 91 insertions(+), 53 deletions(-) diff --git a/include/boost/mpi/collectives/gather.hpp b/include/boost/mpi/collectives/gather.hpp index 70dfd65..2207f59 100644 --- a/include/boost/mpi/collectives/gather.hpp +++ b/include/boost/mpi/collectives/gather.hpp @@ -8,6 +8,9 @@ #ifndef BOOST_MPI_GATHER_HPP #define BOOST_MPI_GATHER_HPP +#include +#include +#include #include #include #include @@ -20,67 +23,102 @@ namespace boost { namespace mpi { +template +void gather(const communicator& comm, const T& in_value, std::vector& out_values, int root); + namespace detail { - // We're gathering at the root for a type that has an associated MPI - // datatype, so we'll use MPI_Gather to do all of the work. - template - void - gather_impl(const communicator& comm, const T* in_values, int n, - T* out_values, int root, mpl::true_) - { - MPI_Datatype type = get_mpi_datatype(*in_values); - BOOST_MPI_CHECK_RESULT(MPI_Gather, - (const_cast(in_values), n, type, - out_values, n, type, root, comm)); - } +// We're gathering at the root for a type that has an associated MPI +// datatype, so we'll use MPI_Gather to do all of the work. +template +void +gather_impl(const communicator& comm, const T* in_values, int n, + T* out_values, int root, mpl::true_) +{ + MPI_Datatype type = get_mpi_datatype(*in_values); + BOOST_MPI_CHECK_RESULT(MPI_Gather, + (const_cast(in_values), n, type, + out_values, n, type, root, comm)); +} - // We're gathering from a non-root for a type that has an associated MPI - // datatype, so we'll use MPI_Gather to do all of the work. - template - void - gather_impl(const communicator& comm, const T* in_values, int n, int root, - mpl::true_) - { - MPI_Datatype type = get_mpi_datatype(*in_values); - BOOST_MPI_CHECK_RESULT(MPI_Gather, - (const_cast(in_values), n, type, - 0, n, type, root, comm)); +// We're gathering from a non-root for a type that has an associated MPI +// datatype, so we'll use MPI_Gather to do all of the work. +template +void +gather_impl(const communicator& comm, const T* in_values, int n, int root, + mpl::true_) +{ + MPI_Datatype type = get_mpi_datatype(*in_values); + BOOST_MPI_CHECK_RESULT(MPI_Gather, + (const_cast(in_values), n, type, + 0, n, type, root, comm)); +} + +// Convert a sequence of sizes [S0..Sn] to a sequence displacement +// [O0..On] where O[0] = 0 and O[k+1] = O[k]+S[k] +template +void +sizes2offset(std::vector const& sizes, std::vector& offsets) +{ + assert(offsets.size() == sizes.size()); + offsets[0] = 0; + for(int i = 0; i < sizes.size()-1; ++i) { + offsets[i+1] = offsets[i] + sizes[i]; } +} - // We're gathering at the root for a type that does not have an - // associated MPI datatype, so we'll need to serialize - // it. Unfortunately, this means that we cannot use MPI_Gather, so - // we'll just have all of the non-root nodes send individual - // messages to the root. - template - void - gather_impl(const communicator& comm, const T* in_values, int n, - T* out_values, int root, mpl::false_) - { - int tag = environment::collectives_tag(); - int size = comm.size(); - - for (int src = 0; src < size; ++src) { - if (src == root) +// We're gathering at the root for a type that does not have an +// associated MPI datatype, so we'll need to serialize +// it. +template +void +gather_impl(const communicator& comm, const T* in_values, int n, + T* out_values, int root, mpl::false_) +{ + int tag = environment::collectives_tag(); + int nproc = comm.size(); + // first, gather all size, these size can be different for + // each process + packed_oarchive oa(comm); + for (int i = 0; i < n; ++i) { + oa << in_values[i]; + } + std::vector asizes; + gather(comm, int(oa.size()), asizes, root); + // Gather the archives, which can be of different sizes, so + // we need to use gatherv. + // Every thing is contiguous, so the offsets can be + // deduced from the collected sizes. + std::vector offsets(nproc); + if (comm.rank() == root) sizes2offset(asizes, offsets); + packed_iarchive::buffer_type recv_buffer(std::accumulate(asizes.begin(), asizes.end(), 0)); + BOOST_MPI_CHECK_RESULT(MPI_Gatherv, + (const_cast(oa.address()), int(oa.size()), MPI_BYTE, + recv_buffer.data(), asizes.data(), offsets.data(), MPI_BYTE, + root, MPI_Comm(comm))); + if (comm.rank() == root) { + for (int src = 0; src < nproc; ++src) { + if (src == root) { std::copy(in_values, in_values + n, out_values + n * src); - else - comm.recv(src, tag, out_values + n * src, n); + } else { + packed_iarchive ia(comm, recv_buffer, boost::archive::no_header, offsets[src]); + for (int i = 0; i < n; ++i) { + ia >> out_values[n*src + i]; + } + } } } +} - // We're gathering at a non-root for a type that does not have an - // associated MPI datatype, so we'll need to serialize - // it. Unfortunately, this means that we cannot use MPI_Gather, so - // we'll just have all of the non-root nodes send individual - // messages to the root. - template - void - gather_impl(const communicator& comm, const T* in_values, int n, int root, - mpl::false_) - { - int tag = environment::collectives_tag(); - comm.send(root, tag, in_values, n); - } +// We're gathering at a non-root for a type that does not have an +// associated MPI datatype, so we'll need to serialize +// it. +template +void +gather_impl(const communicator& comm, const T* in_values, int n, int root, + mpl::false_ is_mpi_type) +{ + gather_impl(comm, in_values, n, (T*)0, root, is_mpi_type); +} } // end namespace detail template From a5b5a2c4d3015907c369a109c40c20cbbb41cb63 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Thu, 20 Oct 2016 19:32:55 +0200 Subject: [PATCH 36/99] Fixed a pessimistic comment, all_to_all never uses individual send reciv. --- include/boost/mpi/collectives/all_to_all.hpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/include/boost/mpi/collectives/all_to_all.hpp b/include/boost/mpi/collectives/all_to_all.hpp index 8c33c2a..4f20be7 100644 --- a/include/boost/mpi/collectives/all_to_all.hpp +++ b/include/boost/mpi/collectives/all_to_all.hpp @@ -22,7 +22,7 @@ namespace boost { namespace mpi { namespace detail { - // We're performaing an all-to-all with a type that has an + // We're performing an all-to-all with a type that has an // associated MPI datatype, so we'll use MPI_Alltoall to do all of // the work. template @@ -38,9 +38,7 @@ namespace detail { // We're performing an all-to-all with a type that does not have an // associated MPI datatype, so we'll need to serialize - // it. Unfortunately, this means that we cannot use MPI_Alltoall, so - // we'll just have to send individual messages to the other - // processes. + // it. template void all_to_all_impl(const communicator& comm, const T* in_values, int n, From 2bdf8df315c44ad3145c1a56d4be0c58a08ed440 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Thu, 20 Oct 2016 19:34:14 +0200 Subject: [PATCH 37/99] Pessimistic comment, broadcast never uses point to point --- include/boost/mpi/collectives/broadcast.hpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/include/boost/mpi/collectives/broadcast.hpp b/include/boost/mpi/collectives/broadcast.hpp index c48db8d..f8b27f0 100644 --- a/include/boost/mpi/collectives/broadcast.hpp +++ b/include/boost/mpi/collectives/broadcast.hpp @@ -100,9 +100,7 @@ namespace detail { } // We're sending a type that does not have an associated MPI - // datatype, so we'll need to serialize it. Unfortunately, this - // means that we cannot use MPI_Bcast, so we'll just send from the - // root to everyone else. + // datatype, so we'll need to serialize it. template void broadcast_impl(const communicator& comm, T* values, int n, int root, From 0efb1415f6c57fba21997899c7fcdc5539f13276 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Thu, 20 Oct 2016 20:10:02 +0200 Subject: [PATCH 38/99] identation --- include/boost/mpi/collectives/gatherv.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/boost/mpi/collectives/gatherv.hpp b/include/boost/mpi/collectives/gatherv.hpp index eb5f9c1..2bfebe3 100644 --- a/include/boost/mpi/collectives/gatherv.hpp +++ b/include/boost/mpi/collectives/gatherv.hpp @@ -66,7 +66,7 @@ namespace detail { // Our own values will never be transmitted: just copy them. std::copy(in_values, in_values + in_size, out_values + displs[src]); else { -// comm.recv(src, tag, out_values + displs[src], sizes[src]); + // comm.recv(src, tag, out_values + displs[src], sizes[src]); // Receive archive packed_iarchive ia(comm); MPI_Status status; From 8158aebc0187c4178ace7e8609bbb8dce196ae6a Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Thu, 20 Oct 2016 20:25:27 +0200 Subject: [PATCH 39/99] Use C API in gather implementation --- include/boost/mpi/collectives/gather.hpp | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/include/boost/mpi/collectives/gather.hpp b/include/boost/mpi/collectives/gather.hpp index 2207f59..c513417 100644 --- a/include/boost/mpi/collectives/gather.hpp +++ b/include/boost/mpi/collectives/gather.hpp @@ -23,9 +23,6 @@ namespace boost { namespace mpi { -template -void gather(const communicator& comm, const T& in_value, std::vector& out_values, int root); - namespace detail { // We're gathering at the root for a type that has an associated MPI // datatype, so we'll use MPI_Gather to do all of the work. @@ -82,18 +79,22 @@ gather_impl(const communicator& comm, const T* in_values, int n, for (int i = 0; i < n; ++i) { oa << in_values[i]; } - std::vector asizes; - gather(comm, int(oa.size()), asizes, root); + std::vector oasizes(nproc); + int oasize = oa.size(); + BOOST_MPI_CHECK_RESULT(MPI_Gather, + (&oasize, 1, MPI_INTEGER, + oasizes.data(), 1, MPI_INTEGER, + root, MPI_Comm(comm))); // Gather the archives, which can be of different sizes, so // we need to use gatherv. // Every thing is contiguous, so the offsets can be // deduced from the collected sizes. std::vector offsets(nproc); - if (comm.rank() == root) sizes2offset(asizes, offsets); - packed_iarchive::buffer_type recv_buffer(std::accumulate(asizes.begin(), asizes.end(), 0)); + if (comm.rank() == root) sizes2offset(oasizes, offsets); + packed_iarchive::buffer_type recv_buffer(std::accumulate(oasizes.begin(), oasizes.end(), 0)); BOOST_MPI_CHECK_RESULT(MPI_Gatherv, (const_cast(oa.address()), int(oa.size()), MPI_BYTE, - recv_buffer.data(), asizes.data(), offsets.data(), MPI_BYTE, + recv_buffer.data(), oasizes.data(), offsets.data(), MPI_BYTE, root, MPI_Comm(comm))); if (comm.rank() == root) { for (int src = 0; src < nproc; ++src) { From f8340229af0d8b6a9657e23d26b6b33642b89f32 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Thu, 20 Oct 2016 21:20:42 +0200 Subject: [PATCH 40/99] identation --- include/boost/mpi/cartesian_communicator.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/include/boost/mpi/cartesian_communicator.hpp b/include/boost/mpi/cartesian_communicator.hpp index a353a63..f3cc2a0 100644 --- a/include/boost/mpi/cartesian_communicator.hpp +++ b/include/boost/mpi/cartesian_communicator.hpp @@ -1,4 +1,3 @@ - // Copyright Alain Miniussi 2014. // Distributed under the Boost Software License, Version 1.0. // (See accompanying file LICENSE_1_0.txt or copy at From 6e06e914c6995270c83370cdfe0cb59cb216f3c9 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Thu, 20 Oct 2016 21:21:11 +0200 Subject: [PATCH 41/99] convert a sequence of size to a sequence of offsets --- include/boost/mpi/detail/offsets.hpp | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 include/boost/mpi/detail/offsets.hpp diff --git a/include/boost/mpi/detail/offsets.hpp b/include/boost/mpi/detail/offsets.hpp new file mode 100644 index 0000000..c19779d --- /dev/null +++ b/include/boost/mpi/detail/offsets.hpp @@ -0,0 +1,29 @@ +// Copyright Alain Miniussi 2014. +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +// Authors: Alain Miniussi +#ifndef BOOST_MPI_OFFSETS_HPP +#define BOOST_MPI_OFFSETS_HPP + +namespace boost { namespace mpi { +namespace detail { + +// Convert a sequence of sizes [S0..Sn] to a sequence displacement +// [O0..On] where O[0] = 0 and O[k+1] = O[k]+S[k] +template +void +sizes2offset(std::vector const& sizes, std::vector& offsets) +{ + assert(offsets.size() == sizes.size()); + offsets[0] = 0; + for(int i = 0; i < sizes.size()-1; ++i) { + offsets[i+1] = offsets[i] + sizes[i]; + } +} + + +} +}} +#endif // BOOST_MPI_ALLGATHER_HPP From 23a64c3ef6709309c2ddde1b7d6993277087417b Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Thu, 20 Oct 2016 21:21:45 +0200 Subject: [PATCH 42/99] all_gather does not use point to point anymore. --- include/boost/mpi/collectives/all_gather.hpp | 114 ++++++++++++++++++--------- include/boost/mpi/collectives/gather.hpp | 14 +--- 2 files changed, 77 insertions(+), 51 deletions(-) diff --git a/include/boost/mpi/collectives/all_gather.hpp b/include/boost/mpi/collectives/all_gather.hpp index da73186..1a05f38 100644 --- a/include/boost/mpi/collectives/all_gather.hpp +++ b/include/boost/mpi/collectives/all_gather.hpp @@ -1,52 +1,92 @@ -// Copyright (C) 2005-2006 Douglas Gregor . +// Copyright (C) 2005, 2006 Douglas Gregor. // Use, modification and distribution is subject to the Boost Software // License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at // http://www.boost.org/LICENSE_1_0.txt) -// Message Passing Interface 1.1 -- Section 4.7. Gather-to-all -#ifndef BOOST_MPI_ALL_GATHER_HPP -#define BOOST_MPI_ALL_GATHER_HPP +// Message Passing Interface 1.1 -- Section 4.5. Gather +#ifndef BOOST_MPI_ALLGATHER_HPP +#define BOOST_MPI_ALLGATHER_HPP +#include +#include +#include #include #include #include -#include - -// all_gather falls back to gather+broadcast in some cases -#include -#include +#include +#include +#include +#include +#include +#include +#include namespace boost { namespace mpi { namespace detail { - // We're all-gathering for a type that has an associated MPI - // datatype, so we'll use MPI_Gather to do all of the work. - template - void - all_gather_impl(const communicator& comm, const T* in_values, int n, - T* out_values, mpl::true_) - { - MPI_Datatype type = boost::mpi::get_mpi_datatype(*in_values); - BOOST_MPI_CHECK_RESULT(MPI_Allgather, - (const_cast(in_values), n, type, - out_values, n, type, comm)); - } +// We're gathering at the root for a type that has an associated MPI +// datatype, so we'll use MPI_Gather to do all of the work. +template +void +all_gather_impl(const communicator& comm, const T* in_values, int n, + T* out_values, mpl::true_) +{ + MPI_Datatype type = get_mpi_datatype(*in_values); + BOOST_MPI_CHECK_RESULT(MPI_Allgather, + (const_cast(in_values), n, type, + out_values, n, type, comm)); +} - // We're all-gathering for a type that has no associated MPI - // type. So, we'll do a manual gather followed by a broadcast. - template - void - all_gather_impl(const communicator& comm, const T* in_values, int n, - T* out_values, mpl::false_) - { - gather(comm, in_values, n, out_values, 0); - broadcast(comm, out_values, comm.size() * n, 0); +// We're gathering for a type that does not have an +// associated MPI datatype, so we'll need to serialize +// it. +template +void +all_gather_impl(const communicator& comm, const T* in_values, int n, + T* out_values, mpl::false_) +{ + int tag = environment::collectives_tag(); + int nproc = comm.size(); + // first, gather all size, these size can be different for + // each process + packed_oarchive oa(comm); + for (int i = 0; i < n; ++i) { + oa << in_values[i]; } + std::vector oasizes(nproc); + int oasize = oa.size(); + BOOST_MPI_CHECK_RESULT(MPI_Allgather, + (&oasize, 1, MPI_INTEGER, + oasizes.data(), 1, MPI_INTEGER, + MPI_Comm(comm))); + // Gather the archives, which can be of different sizes, so + // we need to use allgatherv. + // Every thing is contiguous, so the offsets can be + // deduced from the collected sizes. + std::vector offsets(nproc); + sizes2offset(oasizes, offsets); + packed_iarchive::buffer_type recv_buffer(std::accumulate(oasizes.begin(), oasizes.end(), 0)); + BOOST_MPI_CHECK_RESULT(MPI_Allgatherv, + (const_cast(oa.address()), int(oa.size()), MPI_BYTE, + recv_buffer.data(), oasizes.data(), offsets.data(), MPI_BYTE, + MPI_Comm(comm))); + for (int src = 0; src < nproc; ++src) { + if (src == comm.rank()) { // this is our local data + std::copy(in_values, in_values + n, out_values + n * src); + } else { + packed_iarchive ia(comm, recv_buffer, boost::archive::no_header, offsets[src]); + for (int i = 0; i < n; ++i) { + ia >> out_values[n*src + i]; + } + } + } +} + } // end namespace detail template -inline void +void all_gather(const communicator& comm, const T& in_value, T* out_values) { detail::all_gather_impl(comm, &in_value, 1, out_values, is_mpi_datatype()); @@ -54,15 +94,14 @@ all_gather(const communicator& comm, const T& in_value, T* out_values) template void -all_gather(const communicator& comm, const T& in_value, - std::vector& out_values) +all_gather(const communicator& comm, const T& in_value, std::vector& out_values) { out_values.resize(comm.size()); - ::boost::mpi::all_gather(comm, &in_value, 1, &out_values[0]); + ::boost::mpi::all_gather(comm, in_value, &out_values[0]); } template -inline void +void all_gather(const communicator& comm, const T* in_values, int n, T* out_values) { detail::all_gather_impl(comm, in_values, n, out_values, is_mpi_datatype()); @@ -70,11 +109,10 @@ all_gather(const communicator& comm, const T* in_values, int n, T* out_values) template void -all_gather(const communicator& comm, const T* in_values, int n, - std::vector& out_values) +all_gather(const communicator& comm, const T* in_values, int n, std::vector& out_values) { out_values.resize(comm.size() * n); - ::boost::mpi::all_gather(comm, in_values, n, &out_values[0]); + ::boost::mpi::gather(comm, in_values, n, &out_values[0]); } } } // end namespace boost::mpi diff --git a/include/boost/mpi/collectives/gather.hpp b/include/boost/mpi/collectives/gather.hpp index c513417..83a94bd 100644 --- a/include/boost/mpi/collectives/gather.hpp +++ b/include/boost/mpi/collectives/gather.hpp @@ -19,6 +19,7 @@ #include #include #include +#include #include namespace boost { namespace mpi { @@ -50,19 +51,6 @@ gather_impl(const communicator& comm, const T* in_values, int n, int root, 0, n, type, root, comm)); } -// Convert a sequence of sizes [S0..Sn] to a sequence displacement -// [O0..On] where O[0] = 0 and O[k+1] = O[k]+S[k] -template -void -sizes2offset(std::vector const& sizes, std::vector& offsets) -{ - assert(offsets.size() == sizes.size()); - offsets[0] = 0; - for(int i = 0; i < sizes.size()-1; ++i) { - offsets[i+1] = offsets[i] + sizes[i]; - } -} - // We're gathering at the root for a type that does not have an // associated MPI datatype, so we'll need to serialize // it. From f254852f3d4a05be139e77bd6ec23ceb0bd8caaa Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Fri, 21 Oct 2016 10:48:24 +0200 Subject: [PATCH 43/99] Comment typo, a template was wrongly calling gather (a test should be added for that case) --- include/boost/mpi/collectives/all_gather.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/boost/mpi/collectives/all_gather.hpp b/include/boost/mpi/collectives/all_gather.hpp index 1a05f38..7d85e24 100644 --- a/include/boost/mpi/collectives/all_gather.hpp +++ b/include/boost/mpi/collectives/all_gather.hpp @@ -25,7 +25,7 @@ namespace boost { namespace mpi { namespace detail { -// We're gathering at the root for a type that has an associated MPI +// We're all-gathering for a type that has an associated MPI // datatype, so we'll use MPI_Gather to do all of the work. template void @@ -38,7 +38,7 @@ all_gather_impl(const communicator& comm, const T* in_values, int n, out_values, n, type, comm)); } -// We're gathering for a type that does not have an +// We're all-gathering for a type that does not have an // associated MPI datatype, so we'll need to serialize // it. template @@ -112,7 +112,7 @@ void all_gather(const communicator& comm, const T* in_values, int n, std::vector& out_values) { out_values.resize(comm.size() * n); - ::boost::mpi::gather(comm, in_values, n, &out_values[0]); + ::boost::mpi::all_gather(comm, in_values, n, &out_values[0]); } } } // end namespace boost::mpi From 590c8c69b0d7c9c2055b73c31f3cb4a3ef3029a8 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Fri, 21 Oct 2016 11:10:46 +0200 Subject: [PATCH 44/99] Set the output buffer size. This function is often called on the root node only, so it is more convenient to allocate memory on demand. --- include/boost/mpi/detail/offsets.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/boost/mpi/detail/offsets.hpp b/include/boost/mpi/detail/offsets.hpp index c19779d..9787257 100644 --- a/include/boost/mpi/detail/offsets.hpp +++ b/include/boost/mpi/detail/offsets.hpp @@ -16,7 +16,7 @@ template void sizes2offset(std::vector const& sizes, std::vector& offsets) { - assert(offsets.size() == sizes.size()); + offsets.resize(sizes.size()); offsets[0] = 0; for(int i = 0; i < sizes.size()-1; ++i) { offsets[i+1] = offsets[i] + sizes[i]; From a8b8049995d8a3f20334b2b8921865a5f083319c Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Fri, 21 Oct 2016 11:12:02 +0200 Subject: [PATCH 45/99] use data() instead of hacky &array[0] --- include/boost/mpi/collectives/all_gather.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/boost/mpi/collectives/all_gather.hpp b/include/boost/mpi/collectives/all_gather.hpp index 7d85e24..cba422e 100644 --- a/include/boost/mpi/collectives/all_gather.hpp +++ b/include/boost/mpi/collectives/all_gather.hpp @@ -97,7 +97,7 @@ void all_gather(const communicator& comm, const T& in_value, std::vector& out_values) { out_values.resize(comm.size()); - ::boost::mpi::all_gather(comm, in_value, &out_values[0]); + ::boost::mpi::all_gather(comm, in_value, out_values.data()); } template @@ -112,7 +112,7 @@ void all_gather(const communicator& comm, const T* in_values, int n, std::vector& out_values) { out_values.resize(comm.size() * n); - ::boost::mpi::all_gather(comm, in_values, n, &out_values[0]); + ::boost::mpi::all_gather(comm, in_values, n, out_values.data()); } } } // end namespace boost::mpi From 8aec85cfb01667afe507c2a847ef3b9e096c25d2 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Fri, 21 Oct 2016 11:12:37 +0200 Subject: [PATCH 46/99] allocae sizes only on root. --- include/boost/mpi/collectives/gather.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/boost/mpi/collectives/gather.hpp b/include/boost/mpi/collectives/gather.hpp index 83a94bd..4cedea3 100644 --- a/include/boost/mpi/collectives/gather.hpp +++ b/include/boost/mpi/collectives/gather.hpp @@ -77,7 +77,7 @@ gather_impl(const communicator& comm, const T* in_values, int n, // we need to use gatherv. // Every thing is contiguous, so the offsets can be // deduced from the collected sizes. - std::vector offsets(nproc); + std::vector offsets; if (comm.rank() == root) sizes2offset(oasizes, offsets); packed_iarchive::buffer_type recv_buffer(std::accumulate(oasizes.begin(), oasizes.end(), 0)); BOOST_MPI_CHECK_RESULT(MPI_Gatherv, From cdabc4398d342c1980f9e5f93dfccb260c1c80f2 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Fri, 21 Oct 2016 13:46:52 +0200 Subject: [PATCH 47/99] Comply to boost indentation rules --- include/boost/mpi/collectives/scatter.hpp | 136 +++++++++++++++--------------- 1 file changed, 68 insertions(+), 68 deletions(-) diff --git a/include/boost/mpi/collectives/scatter.hpp b/include/boost/mpi/collectives/scatter.hpp index 196682d..c25bb3b 100644 --- a/include/boost/mpi/collectives/scatter.hpp +++ b/include/boost/mpi/collectives/scatter.hpp @@ -21,78 +21,78 @@ namespace boost { namespace mpi { namespace detail { - // We're scattering from the root for a type that has an associated MPI - // datatype, so we'll use MPI_Scatter to do all of the work. - template - void - scatter_impl(const communicator& comm, const T* in_values, T* out_values, - int n, int root, mpl::true_) - { - MPI_Datatype type = get_mpi_datatype(*in_values); - BOOST_MPI_CHECK_RESULT(MPI_Scatter, - (const_cast(in_values), n, type, - out_values, n, type, root, comm)); - } +// We're scattering from the root for a type that has an associated MPI +// datatype, so we'll use MPI_Scatter to do all of the work. +template +void +scatter_impl(const communicator& comm, const T* in_values, T* out_values, + int n, int root, mpl::true_) +{ + MPI_Datatype type = get_mpi_datatype(*in_values); + BOOST_MPI_CHECK_RESULT(MPI_Scatter, + (const_cast(in_values), n, type, + out_values, n, type, root, comm)); +} - // We're scattering from a non-root for a type that has an associated MPI - // datatype, so we'll use MPI_Scatter to do all of the work. - template - void - scatter_impl(const communicator& comm, T* out_values, int n, int root, - mpl::true_) - { - MPI_Datatype type = get_mpi_datatype(*out_values); - BOOST_MPI_CHECK_RESULT(MPI_Scatter, - (0, n, type, - out_values, n, type, - root, comm)); - } +// We're scattering from a non-root for a type that has an associated MPI +// datatype, so we'll use MPI_Scatter to do all of the work. +template +void +scatter_impl(const communicator& comm, T* out_values, int n, int root, + mpl::true_) +{ + MPI_Datatype type = get_mpi_datatype(*out_values); + BOOST_MPI_CHECK_RESULT(MPI_Scatter, + (0, n, type, + out_values, n, type, + root, comm)); +} - // We're scattering from the root for a type that does not have an - // associated MPI datatype, so we'll need to serialize - // it. Unfortunately, this means that we cannot use MPI_Scatter, so - // we'll just have the root send individual messages to the other - // processes. - template - void - scatter_impl(const communicator& comm, const T* in_values, T* out_values, - int n, int root, mpl::false_) - { - int tag = environment::collectives_tag(); - int size = comm.size(); - - for (int dest = 0; dest < size; ++dest) { - if (dest == root) { - // Our own values will never be transmitted: just copy them. - std::copy(in_values + dest * n, in_values + (dest + 1) * n, out_values); - } else { - // Send archive - packed_oarchive oa(comm); - for (int i = 0; i < n; ++i) - oa << in_values[dest * n + i]; - detail::packed_archive_send(comm, dest, tag, oa); - } +// We're scattering from the root for a type that does not have an +// associated MPI datatype, so we'll need to serialize +// it. Unfortunately, this means that we cannot use MPI_Scatter, so +// we'll just have the root send individual messages to the other +// processes. +template +void +scatter_impl(const communicator& comm, const T* in_values, T* out_values, + int n, int root, mpl::false_) +{ + int tag = environment::collectives_tag(); + int size = comm.size(); + + for (int dest = 0; dest < size; ++dest) { + if (dest == root) { + // Our own values will never be transmitted: just copy them. + std::copy(in_values + dest * n, in_values + (dest + 1) * n, out_values); + } else { + // Send archive + packed_oarchive oa(comm); + for (int i = 0; i < n; ++i) + oa << in_values[dest * n + i]; + detail::packed_archive_send(comm, dest, tag, oa); } } +} - // We're scattering to a non-root for a type that does not have an - // associated MPI datatype, so we'll need to de-serialize - // it. Unfortunately, this means that we cannot use MPI_Scatter, so - // we'll just have all of the non-root nodes send individual - // messages to the root. - template - void - scatter_impl(const communicator& comm, T* out_values, int n, int root, - mpl::false_) - { - int tag = environment::collectives_tag(); - - packed_iarchive ia(comm); - MPI_Status status; - detail::packed_archive_recv(comm, root, tag, ia, status); - for (int i = 0; i < n; ++i) - ia >> out_values[i]; - } +// We're scattering to a non-root for a type that does not have an +// associated MPI datatype, so we'll need to de-serialize +// it. Unfortunately, this means that we cannot use MPI_Scatter, so +// we'll just have all of the non-root nodes send individual +// messages to the root. +template +void +scatter_impl(const communicator& comm, T* out_values, int n, int root, + mpl::false_) +{ + int tag = environment::collectives_tag(); + + packed_iarchive ia(comm); + MPI_Status status; + detail::packed_archive_recv(comm, root, tag, ia, status); + for (int i = 0; i < n; ++i) + ia >> out_values[i]; +} } // end namespace detail template @@ -146,7 +146,7 @@ scatter(const communicator& comm, const std::vector& in_values, ::boost::mpi::scatter(comm, &in_values[0], out_values, n, root); else ::boost::mpi::scatter(comm, static_cast(0), out_values, - n, root); + n, root); } template From e3f076023ea19f5083cd7e7ecee2a9f0ff13d61a Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Fri, 21 Oct 2016 16:20:34 +0200 Subject: [PATCH 48/99] There are as many sizes as offsets --- include/boost/mpi/collectives/gather.hpp | 2 +- include/boost/mpi/detail/offsets.hpp | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/boost/mpi/collectives/gather.hpp b/include/boost/mpi/collectives/gather.hpp index 4cedea3..c0363eb 100644 --- a/include/boost/mpi/collectives/gather.hpp +++ b/include/boost/mpi/collectives/gather.hpp @@ -78,7 +78,7 @@ gather_impl(const communicator& comm, const T* in_values, int n, // Every thing is contiguous, so the offsets can be // deduced from the collected sizes. std::vector offsets; - if (comm.rank() == root) sizes2offset(oasizes, offsets); + if (comm.rank() == root) sizes2offsets(oasizes, offsets); packed_iarchive::buffer_type recv_buffer(std::accumulate(oasizes.begin(), oasizes.end(), 0)); BOOST_MPI_CHECK_RESULT(MPI_Gatherv, (const_cast(oa.address()), int(oa.size()), MPI_BYTE, diff --git a/include/boost/mpi/detail/offsets.hpp b/include/boost/mpi/detail/offsets.hpp index 9787257..28ccea8 100644 --- a/include/boost/mpi/detail/offsets.hpp +++ b/include/boost/mpi/detail/offsets.hpp @@ -11,10 +11,10 @@ namespace boost { namespace mpi { namespace detail { // Convert a sequence of sizes [S0..Sn] to a sequence displacement -// [O0..On] where O[0] = 0 and O[k+1] = O[k]+S[k] +// [O0..On] where O[0] = 0 and O[k+1] = O[k]+S[k]. template void -sizes2offset(std::vector const& sizes, std::vector& offsets) +sizes2offsets(std::vector const& sizes, std::vector& offsets) { offsets.resize(sizes.size()); offsets[0] = 0; @@ -26,4 +26,4 @@ sizes2offset(std::vector const& sizes, std::vector& of } }} -#endif // BOOST_MPI_ALLGATHER_HPP +#endif // BOOST_MPI_OFFSETS_HPP From 96def4ce61134a7629983fbf420a7a1a0952f292 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Fri, 21 Oct 2016 16:22:57 +0200 Subject: [PATCH 49/99] was missing in previous commit, the name of sizes2offsetshas changed. --- include/boost/mpi/collectives/all_gather.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/boost/mpi/collectives/all_gather.hpp b/include/boost/mpi/collectives/all_gather.hpp index cba422e..d2c741a 100644 --- a/include/boost/mpi/collectives/all_gather.hpp +++ b/include/boost/mpi/collectives/all_gather.hpp @@ -65,7 +65,7 @@ all_gather_impl(const communicator& comm, const T* in_values, int n, // Every thing is contiguous, so the offsets can be // deduced from the collected sizes. std::vector offsets(nproc); - sizes2offset(oasizes, offsets); + sizes2offsets(oasizes, offsets); packed_iarchive::buffer_type recv_buffer(std::accumulate(oasizes.begin(), oasizes.end(), 0)); BOOST_MPI_CHECK_RESULT(MPI_Allgatherv, (const_cast(oa.address()), int(oa.size()), MPI_BYTE, From ef30d347f7cf1aa772c3d3607f4fd3851688ada3 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Fri, 21 Oct 2016 16:23:43 +0200 Subject: [PATCH 50/99] scatter does not use point to point anymore --- include/boost/mpi/collectives/scatter.hpp | 101 +++++++++++++++--------------- 1 file changed, 52 insertions(+), 49 deletions(-) diff --git a/include/boost/mpi/collectives/scatter.hpp b/include/boost/mpi/collectives/scatter.hpp index c25bb3b..2d4b2a7 100644 --- a/include/boost/mpi/collectives/scatter.hpp +++ b/include/boost/mpi/collectives/scatter.hpp @@ -16,6 +16,7 @@ #include #include #include +#include #include namespace boost { namespace mpi { @@ -50,48 +51,66 @@ scatter_impl(const communicator& comm, T* out_values, int n, int root, // We're scattering from the root for a type that does not have an // associated MPI datatype, so we'll need to serialize -// it. Unfortunately, this means that we cannot use MPI_Scatter, so -// we'll just have the root send individual messages to the other -// processes. +// it. template void scatter_impl(const communicator& comm, const T* in_values, T* out_values, int n, int root, mpl::false_) { int tag = environment::collectives_tag(); - int size = comm.size(); - - for (int dest = 0; dest < size; ++dest) { - if (dest == root) { - // Our own values will never be transmitted: just copy them. - std::copy(in_values + dest * n, in_values + (dest + 1) * n, out_values); - } else { - // Send archive - packed_oarchive oa(comm); - for (int i = 0; i < n; ++i) - oa << in_values[dest * n + i]; - detail::packed_archive_send(comm, dest, tag, oa); + int nproc = comm.size(); + packed_oarchive::buffer_type sendbuf; + std::vector slotsizes; + + if (root == comm.rank()) { + // fill the sendbuf while keeping trac of the slot sizes + slotsizes.resize(nproc); + for (int dest = 0; dest < nproc; ++dest) { + packed_oarchive slotarchive(comm); + for (int i = dest*n; i < (dest+1)*n; ++i) { + slotarchive << in_values[i]; + } + std::cout << '\n'; + int slotsize = slotarchive.size(); + sendbuf.resize(sendbuf.size() + slotsize); + slotsizes[dest] = slotsize; + char const* aptr = static_cast(slotarchive.address()); + std::copy(aptr, aptr+slotsize, sendbuf.end()-slotsize); + } + } + // Distribute the sizes + int myslotsize; + BOOST_MPI_CHECK_RESULT(MPI_Scatter, + (slotsizes.data(), 1, MPI_INTEGER, + &myslotsize, 1, MPI_INTEGER, root, comm)); + std::vector offsets; + if (root == comm.rank()) { + sizes2offsets(slotsizes, offsets); + } + packed_iarchive::buffer_type recvbuf; + recvbuf.resize(myslotsize); + BOOST_MPI_CHECK_RESULT(MPI_Scatterv, + (sendbuf.data(), slotsizes.data(), offsets.data(), MPI_BYTE, + recvbuf.data(), recvbuf.size(), MPI_BYTE, + root, MPI_Comm(comm))); + if (root == comm.rank()) { + // Our own local values are already here: just copy them. + std::copy(in_values + root * n, in_values + (root + 1) * n, out_values); + } else { + // Otherwise deserialize: + packed_iarchive iarchv(comm, recvbuf); + for (int i = 0; i < n; ++i) { + iarchv >> out_values[i]; } } } -// We're scattering to a non-root for a type that does not have an -// associated MPI datatype, so we'll need to de-serialize -// it. Unfortunately, this means that we cannot use MPI_Scatter, so -// we'll just have all of the non-root nodes send individual -// messages to the root. template void scatter_impl(const communicator& comm, T* out_values, int n, int root, - mpl::false_) -{ - int tag = environment::collectives_tag(); - - packed_iarchive ia(comm); - MPI_Status status; - detail::packed_archive_recv(comm, root, tag, ia, status); - for (int i = 0; i < n; ++i) - ia >> out_values[i]; + mpl::false_ is_mpi_type) +{ + scatter_impl(comm, (T const*)0, out_values, n, root, is_mpi_type); } } // end namespace detail @@ -99,11 +118,7 @@ template void scatter(const communicator& comm, const T* in_values, T& out_value, int root) { - if (comm.rank() == root) - detail::scatter_impl(comm, in_values, &out_value, 1, root, - is_mpi_datatype()); - else - detail::scatter_impl(comm, &out_value, 1, root, is_mpi_datatype()); + detail::scatter_impl(comm, in_values, &out_value, 1, root, is_mpi_datatype()); } template @@ -111,11 +126,7 @@ void scatter(const communicator& comm, const std::vector& in_values, T& out_value, int root) { - if (comm.rank() == root) - ::boost::mpi::scatter(comm, &in_values[0], out_value, root); - else - ::boost::mpi::scatter(comm, static_cast(0), out_value, - root); + ::boost::mpi::scatter(comm, &in_values[0], out_value, root); } template @@ -130,11 +141,7 @@ void scatter(const communicator& comm, const T* in_values, T* out_values, int n, int root) { - if (comm.rank() == root) - detail::scatter_impl(comm, in_values, out_values, n, root, - is_mpi_datatype()); - else - detail::scatter_impl(comm, out_values, n, root, is_mpi_datatype()); + detail::scatter_impl(comm, in_values, out_values, n, root, is_mpi_datatype()); } template @@ -142,11 +149,7 @@ void scatter(const communicator& comm, const std::vector& in_values, T* out_values, int n, int root) { - if (comm.rank() == root) - ::boost::mpi::scatter(comm, &in_values[0], out_values, n, root); - else - ::boost::mpi::scatter(comm, static_cast(0), out_values, - n, root); + ::boost::mpi::scatter(comm, &in_values[0], out_values, n, root); } template From 7b27c12ab4c5edeb96d590c04990f6231bae58c6 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Fri, 21 Oct 2016 16:48:55 +0200 Subject: [PATCH 51/99] started isolating the scatter send bufferfilling. Will be shared with scatterv --- include/boost/mpi/collectives/scatter.hpp | 39 +++++++++++++++++++------------ 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/include/boost/mpi/collectives/scatter.hpp b/include/boost/mpi/collectives/scatter.hpp index 2d4b2a7..8bd7631 100644 --- a/include/boost/mpi/collectives/scatter.hpp +++ b/include/boost/mpi/collectives/scatter.hpp @@ -49,6 +49,28 @@ scatter_impl(const communicator& comm, T* out_values, int n, int root, root, comm)); } +// Fill the sendbuf while keeping trac of the slot sizes +// Used in the first steps of both scatter and scatterv +template +void +fill_scatter_sendbuf(const communicator& comm, T const* values, int n, + packed_oarchive::buffer_type& sendbuf, std::vector& archsizes) { + int nproc = comm.size(); + archsizes.resize(nproc); + + for (int dest = 0; dest < nproc; ++dest) { + packed_oarchive procarchive(comm); + for (int i = dest*n; i < (dest+1)*n; ++i) { + procarchive << values[i]; + } + int archsize = procarchive.size(); + sendbuf.resize(sendbuf.size() + archsize); + archsizes[dest] = archsize; + char const* aptr = static_cast(procarchive.address()); + std::copy(aptr, aptr+archsize, sendbuf.end()-archsize); + } +} + // We're scattering from the root for a type that does not have an // associated MPI datatype, so we'll need to serialize // it. @@ -61,22 +83,9 @@ scatter_impl(const communicator& comm, const T* in_values, T* out_values, int nproc = comm.size(); packed_oarchive::buffer_type sendbuf; std::vector slotsizes; - + if (root == comm.rank()) { - // fill the sendbuf while keeping trac of the slot sizes - slotsizes.resize(nproc); - for (int dest = 0; dest < nproc; ++dest) { - packed_oarchive slotarchive(comm); - for (int i = dest*n; i < (dest+1)*n; ++i) { - slotarchive << in_values[i]; - } - std::cout << '\n'; - int slotsize = slotarchive.size(); - sendbuf.resize(sendbuf.size() + slotsize); - slotsizes[dest] = slotsize; - char const* aptr = static_cast(slotarchive.address()); - std::copy(aptr, aptr+slotsize, sendbuf.end()-slotsize); - } + fill_scatter_sendbuf(comm, in_values, n, sendbuf, slotsizes); } // Distribute the sizes int myslotsize; From 3a0109112c6b5811cb90e84aa43e0136b8d2c6fc Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Fri, 21 Oct 2016 16:58:23 +0200 Subject: [PATCH 52/99] prepare for passing each proc's number of element explicitly. --- include/boost/mpi/collectives/scatter.hpp | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/include/boost/mpi/collectives/scatter.hpp b/include/boost/mpi/collectives/scatter.hpp index 8bd7631..2d4497e 100644 --- a/include/boost/mpi/collectives/scatter.hpp +++ b/include/boost/mpi/collectives/scatter.hpp @@ -51,17 +51,19 @@ scatter_impl(const communicator& comm, T* out_values, int n, int root, // Fill the sendbuf while keeping trac of the slot sizes // Used in the first steps of both scatter and scatterv +// Nslots contains the number of slots being sent +// to each process (identical values for scatter). template void -fill_scatter_sendbuf(const communicator& comm, T const* values, int n, +fill_scatter_sendbuf(const communicator& comm, T const* values, std::vector const& nslots, packed_oarchive::buffer_type& sendbuf, std::vector& archsizes) { int nproc = comm.size(); archsizes.resize(nproc); for (int dest = 0; dest < nproc; ++dest) { packed_oarchive procarchive(comm); - for (int i = dest*n; i < (dest+1)*n; ++i) { - procarchive << values[i]; + for (int i = 0; i < nslots[dest]; ++i) { + procarchive << *values++; } int archsize = procarchive.size(); sendbuf.resize(sendbuf.size() + archsize); @@ -72,8 +74,7 @@ fill_scatter_sendbuf(const communicator& comm, T const* values, int n, } // We're scattering from the root for a type that does not have an -// associated MPI datatype, so we'll need to serialize -// it. +// associated MPI datatype, so we'll need to serialize it. template void scatter_impl(const communicator& comm, const T* in_values, T* out_values, @@ -85,7 +86,8 @@ scatter_impl(const communicator& comm, const T* in_values, T* out_values, std::vector slotsizes; if (root == comm.rank()) { - fill_scatter_sendbuf(comm, in_values, n, sendbuf, slotsizes); + std::vector nslots(nproc, n); + fill_scatter_sendbuf(comm, in_values, nslots, sendbuf, slotsizes); } // Distribute the sizes int myslotsize; From 60360763f4dfcf421a173eada4651aa3b304b30a Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Fri, 21 Oct 2016 17:21:00 +0200 Subject: [PATCH 53/99] scatter is now clearly divided in 2 step: 1) fill the buffer, 2) dispatch the buffer --- include/boost/mpi/collectives/scatter.hpp | 52 +++++++++++++++++++------------ 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/include/boost/mpi/collectives/scatter.hpp b/include/boost/mpi/collectives/scatter.hpp index 2d4497e..539a348 100644 --- a/include/boost/mpi/collectives/scatter.hpp +++ b/include/boost/mpi/collectives/scatter.hpp @@ -73,38 +73,33 @@ fill_scatter_sendbuf(const communicator& comm, T const* values, std::vector } } -// We're scattering from the root for a type that does not have an -// associated MPI datatype, so we'll need to serialize it. +// Dispatch the sendbuf among proc. +// Used in the second steps of both scatter and scatterv template void -scatter_impl(const communicator& comm, const T* in_values, T* out_values, - int n, int root, mpl::false_) -{ - int tag = environment::collectives_tag(); - int nproc = comm.size(); - packed_oarchive::buffer_type sendbuf; - std::vector slotsizes; - - if (root == comm.rank()) { - std::vector nslots(nproc, n); - fill_scatter_sendbuf(comm, in_values, nslots, sendbuf, slotsizes); - } +dispatch_scatter_sendbuf(const communicator& comm, + packed_oarchive::buffer_type const& sendbuf, std::vector const& archsizes, + T const* in_values, + T* out_values, int n, int root) { // Distribute the sizes - int myslotsize; + int myarchsize; BOOST_MPI_CHECK_RESULT(MPI_Scatter, - (slotsizes.data(), 1, MPI_INTEGER, - &myslotsize, 1, MPI_INTEGER, root, comm)); + (archsizes.data(), 1, MPI_INTEGER, + &myarchsize, 1, MPI_INTEGER, root, comm)); std::vector offsets; if (root == comm.rank()) { - sizes2offsets(slotsizes, offsets); + sizes2offsets(archsizes, offsets); } + // Get my proc archive packed_iarchive::buffer_type recvbuf; - recvbuf.resize(myslotsize); + recvbuf.resize(myarchsize); BOOST_MPI_CHECK_RESULT(MPI_Scatterv, - (sendbuf.data(), slotsizes.data(), offsets.data(), MPI_BYTE, + (sendbuf.data(), archsizes.data(), offsets.data(), MPI_BYTE, recvbuf.data(), recvbuf.size(), MPI_BYTE, root, MPI_Comm(comm))); + // Unserialize if (root == comm.rank()) { + assert(in_values); // Our own local values are already here: just copy them. std::copy(in_values + root * n, in_values + (root + 1) * n, out_values); } else { @@ -116,6 +111,23 @@ scatter_impl(const communicator& comm, const T* in_values, T* out_values, } } +// We're scattering from the root for a type that does not have an +// associated MPI datatype, so we'll need to serialize it. +template +void +scatter_impl(const communicator& comm, const T* in_values, T* out_values, + int n, int root, mpl::false_) +{ + packed_oarchive::buffer_type sendbuf; + std::vector slotsizes; + + if (root == comm.rank()) { + std::vector nslots(comm.size(), n); + fill_scatter_sendbuf(comm, in_values, nslots, sendbuf, slotsizes); + } + dispatch_scatter_sendbuf(comm, sendbuf, slotsizes, in_values, out_values, n, root); +} + template void scatter_impl(const communicator& comm, T* out_values, int n, int root, From f9d9ac941497154c9d450b4604ddc45d04703f22 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Mon, 24 Oct 2016 13:21:03 +0200 Subject: [PATCH 54/99] try to get a core dump in debug mode --- include/boost/mpi/exception.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/boost/mpi/exception.hpp b/include/boost/mpi/exception.hpp index 1252307..154a431 100644 --- a/include/boost/mpi/exception.hpp +++ b/include/boost/mpi/exception.hpp @@ -15,6 +15,7 @@ #include #include +#include #include #include #include @@ -94,6 +95,7 @@ class BOOST_MPI_DECL exception : public std::exception #define BOOST_MPI_CHECK_RESULT( MPIFunc, Args ) \ { \ int _check_result = MPIFunc Args; \ + assert(_check_result == MPI_SUCCESS); \ if (_check_result != MPI_SUCCESS) \ boost::throw_exception(boost::mpi::exception(#MPIFunc, \ _check_result)); \ From 2003a352861ef5506886222b9accb599f5cfda75 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Mon, 24 Oct 2016 13:21:41 +0200 Subject: [PATCH 55/99] get a vector free version. --- include/boost/mpi/detail/offsets.hpp | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/include/boost/mpi/detail/offsets.hpp b/include/boost/mpi/detail/offsets.hpp index 28ccea8..e9374d9 100644 --- a/include/boost/mpi/detail/offsets.hpp +++ b/include/boost/mpi/detail/offsets.hpp @@ -12,14 +12,35 @@ namespace detail { // Convert a sequence of sizes [S0..Sn] to a sequence displacement // [O0..On] where O[0] = 0 and O[k+1] = O[k]+S[k]. -template void -sizes2offsets(std::vector const& sizes, std::vector& offsets) +sizes2offsets(int const* sizes, int* offsets, int n) { - offsets.resize(sizes.size()); offsets[0] = 0; - for(int i = 0; i < sizes.size()-1; ++i) { - offsets[i+1] = offsets[i] + sizes[i]; + for(int i = 1; i < n; ++i) { + offsets[i] = offsets[i-1] + sizes[i-1]; + } +} + +// Convert a sequence of sizes [S0..Sn] to a sequence displacement +// [O0..On] where O[0] = 0 and O[k+1] = O[k]+S[k]. +void +sizes2offsets(std::vector const& sizes, std::vector& offsets) +{ + int sz = sizes.size(); + offsets.resize(sz); + sizes2offsets(sizes.data(), offsets.data(), sz); +} + +// Given a sequence of sizes (typically the number of records dispatched +// to each process in a scater) and a sequence of displacements (typically the +// slot index at with those record starts), convert the later to a number +// of skipped slots. +void +offsets2skipped(int const* sizes, int const* offsets, int* skipped, int n) +{ + skipped[0] = 0; + for(int i = 1; i < n; ++i) { + skipped[i] -= offsets[i-1] + sizes[i-1]; } } From abded0963a1ce9b33b445e0d6819f525814b102b Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Mon, 24 Oct 2016 13:22:09 +0200 Subject: [PATCH 56/99] scatterv whithout point to point communication --- include/boost/mpi/collectives/scatter.hpp | 21 ++- include/boost/mpi/collectives/scatterv.hpp | 203 +++++++++++++++-------------- test/gps_position.hpp | 8 ++ test/scatter_test.cpp | 95 ++++++++------ 4 files changed, 177 insertions(+), 150 deletions(-) diff --git a/include/boost/mpi/collectives/scatter.hpp b/include/boost/mpi/collectives/scatter.hpp index 539a348..46c6b49 100644 --- a/include/boost/mpi/collectives/scatter.hpp +++ b/include/boost/mpi/collectives/scatter.hpp @@ -49,18 +49,25 @@ scatter_impl(const communicator& comm, T* out_values, int n, int root, root, comm)); } -// Fill the sendbuf while keeping trac of the slot sizes +// Fill the sendbuf while keeping trac of the slot's footprints // Used in the first steps of both scatter and scatterv // Nslots contains the number of slots being sent // to each process (identical values for scatter). +// skiped_slots, if present, is deduced from the +// displacement array authorised be the MPI API, +// for some yet to be determined reason. template void -fill_scatter_sendbuf(const communicator& comm, T const* values, std::vector const& nslots, +fill_scatter_sendbuf(const communicator& comm, T const* values, + int const* nslots, int const* skipped_slots, packed_oarchive::buffer_type& sendbuf, std::vector& archsizes) { int nproc = comm.size(); archsizes.resize(nproc); for (int dest = 0; dest < nproc; ++dest) { + if (skipped_slots) { // wee need to keep this for backward compatibility + for(int k= 0; k < skipped_slots[dest]; ++k) ++values; + } packed_oarchive procarchive(comm); for (int i = 0; i < nslots[dest]; ++i) { procarchive << *values++; @@ -75,6 +82,7 @@ fill_scatter_sendbuf(const communicator& comm, T const* values, std::vector // Dispatch the sendbuf among proc. // Used in the second steps of both scatter and scatterv +// in_value is only provide in the non variadic case. template void dispatch_scatter_sendbuf(const communicator& comm, @@ -98,8 +106,7 @@ dispatch_scatter_sendbuf(const communicator& comm, recvbuf.data(), recvbuf.size(), MPI_BYTE, root, MPI_Comm(comm))); // Unserialize - if (root == comm.rank()) { - assert(in_values); + if ( in_values != 0 && root == comm.rank()) { // Our own local values are already here: just copy them. std::copy(in_values + root * n, in_values + (root + 1) * n, out_values); } else { @@ -119,13 +126,13 @@ scatter_impl(const communicator& comm, const T* in_values, T* out_values, int n, int root, mpl::false_) { packed_oarchive::buffer_type sendbuf; - std::vector slotsizes; + std::vector archsizes; if (root == comm.rank()) { std::vector nslots(comm.size(), n); - fill_scatter_sendbuf(comm, in_values, nslots, sendbuf, slotsizes); + fill_scatter_sendbuf(comm, in_values, nslots.data(), (int const*)0, sendbuf, archsizes); } - dispatch_scatter_sendbuf(comm, sendbuf, slotsizes, in_values, out_values, n, root); + dispatch_scatter_sendbuf(comm, sendbuf, archsizes, in_values, out_values, n, root); } template diff --git a/include/boost/mpi/collectives/scatterv.hpp b/include/boost/mpi/collectives/scatterv.hpp index 6e6f270..3be2df2 100644 --- a/include/boost/mpi/collectives/scatterv.hpp +++ b/include/boost/mpi/collectives/scatterv.hpp @@ -8,93 +8,95 @@ #ifndef BOOST_MPI_SCATTERV_HPP #define BOOST_MPI_SCATTERV_HPP -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include namespace boost { namespace mpi { namespace detail { - // We're scattering from the root for a type that has an associated MPI - // datatype, so we'll use MPI_Scatterv to do all of the work. - template - void - scatterv_impl(const communicator& comm, const T* in_values, const int* sizes, - const int* displs, T* out_values, int out_size, int root, mpl::true_) - { - MPI_Datatype type = get_mpi_datatype(*in_values); - BOOST_MPI_CHECK_RESULT(MPI_Scatterv, - (const_cast(in_values), const_cast(sizes), - const_cast(displs), type, - out_values, out_size, type, root, comm)); - } - // We're scattering from a non-root for a type that has an associated MPI - // datatype, so we'll use MPI_Scatterv to do all of the work. - template - void - scatterv_impl(const communicator& comm, T* out_values, int out_size, int root, - mpl::true_) - { - MPI_Datatype type = get_mpi_datatype(*out_values); - BOOST_MPI_CHECK_RESULT(MPI_Scatterv, - (0, 0, 0, type, - out_values, out_size, type, - root, comm)); - } +////////////////////////////////////////////// +/// Implementation for MPI primitive types /// +////////////////////////////////////////////// - // We're scattering from the root for a type that does not have an - // associated MPI datatype, so we'll need to serialize - // it. Unfortunately, this means that we cannot use MPI_Scatterv, so - // we'll just have the root send individual messages to the other - // processes. - template - void - scatterv_impl(const communicator& comm, const T* in_values, const int* sizes, - const int* displs, T* out_values, int out_size, int root, mpl::false_) - { - int tag = environment::collectives_tag(); - int nprocs = comm.size(); - - for (int dest = 0; dest < nprocs; ++dest) { - if (dest == root) { - // Our own values will never be transmitted: just copy them. - std::copy(in_values + displs[dest], - in_values + displs[dest] + out_size, out_values); - } else { - // Send archive - packed_oarchive oa(comm); - for (int i = 0; i < sizes[dest]; ++i) - oa << in_values[ displs[dest] + i ]; - detail::packed_archive_send(comm, dest, tag, oa); - } +// We're scattering from the root for a type that has an associated MPI +// datatype, so we'll use MPI_Scatterv to do all of the work. +template +void +scatterv_impl(const communicator& comm, const T* in_values, T* out_values, int out_size, + const int* sizes, const int* displs, int root, mpl::true_) +{ + std::vector offsets; + if (sizes) { + assert(out_size == sizes[comm.rank()]); + assert(in_values); + if (!displs) { + int nproc = comm.size(); + offsets.resize(nproc); + sizes2offsets(sizes, offsets.data(), nproc); + displs = offsets.data(); } + } else { + assert(comm.rank() != root); } + MPI_Datatype type = get_mpi_datatype(*in_values); + BOOST_MPI_CHECK_RESULT(MPI_Scatterv, + (const_cast(in_values), const_cast(sizes), + const_cast(displs), type, + out_values, out_size, type, root, comm)); +} - // We're scattering to a non-root for a type that does not have an - // associated MPI datatype, so we'll need to de-serialize - // it. Unfortunately, this means that we cannot use MPI_Scatterv, so - // we'll just have all of the non-root nodes send individual - // messages to the root. - template - void - scatterv_impl(const communicator& comm, T* out_values, int out_size, int root, - mpl::false_) - { - int tag = environment::collectives_tag(); - - packed_iarchive ia(comm); - MPI_Status status; - detail::packed_archive_recv(comm, root, tag, ia, status); - for (int i = 0; i < out_size; ++i) - ia >> out_values[i]; +// We're scattering from a non-root for a type that has an associated MPI +// datatype, so we'll use MPI_Scatterv to do all of the work. +template +void +scatterv_impl(const communicator& comm, T* out_values, int out_size, int root, + mpl::true_ is_mpi_type) +{ + scatterv_impl(comm, (T const*)0, out_values, out_size, + (const int*)0, (const int*)0, root, is_mpi_type); +} + +////////////////////////////////////////////////// +/// Implementation for non MPI primitive types /// +////////////////////////////////////////////////// + +// We're scattering from the root for a type that does not have an +// associated MPI datatype, so we'll need to serialize it. +template +void +scatterv_impl(const communicator& comm, const T* in_values, T* out_values, int out_size, + int const* sizes, int const* displs, int root, mpl::false_) +{ + packed_oarchive::buffer_type sendbuf; + bool is_root = comm.rank() == root; + int nproc = comm.size(); + std::vector archsizes; + if (is_root) { + assert(out_size == sizes[comm.rank()]); + archsizes.resize(nproc); + std::vector skipped; + if (displs) { + skipped.resize(nproc); + offsets2skipped(sizes, displs, skipped.data(), nproc); + displs = skipped.data(); + } + fill_scatter_sendbuf(comm, in_values, sizes, (int const*)0, sendbuf, archsizes); } + dispatch_scatter_sendbuf(comm, sendbuf, archsizes, (T const*)0, out_values, out_size, root); +} + +// We're scattering to a non-root for a type that does not have an +// associated MPI datatype. input data not needed. +// it. +template +void +scatterv_impl(const communicator& comm, T* out_values, int n, int root, + mpl::false_ isnt_mpi_type) +{ + assert(root != comm.rank()); + scatterv_impl(comm, (T const*)0, out_values, n, (int const*)0, (int const*)0, root, isnt_mpi_type); +} + } // end namespace detail template @@ -103,13 +105,8 @@ scatterv(const communicator& comm, const T* in_values, const std::vector& sizes, const std::vector& displs, T* out_values, int out_size, int root) { - int rank = comm.rank(); - if (rank == root) - detail::scatterv_impl(comm, in_values, &sizes[0], &displs[0], - out_values, out_size, root, is_mpi_datatype()); - else - detail::scatterv_impl(comm, out_values, out_size, root, - is_mpi_datatype()); + scatterv_impl(comm, in_values, out_values, out_sizes, sizes.data(), displs.data(), + root, is_mpi_datatype()); } template @@ -118,12 +115,8 @@ scatterv(const communicator& comm, const std::vector& in_values, const std::vector& sizes, const std::vector& displs, T* out_values, int out_size, int root) { - if (comm.rank() == root) - ::boost::mpi::scatterv(comm, &in_values[0], sizes, displs, - out_values, out_size, root); - else - ::boost::mpi::scatterv(comm, static_cast(0), sizes, displs, - out_values, out_size, root); + ::boost::mpi::scatterv(comm, in_values.data(), sizes, displs, + out_values, out_size, root); } template @@ -141,16 +134,9 @@ void scatterv(const communicator& comm, const T* in_values, const std::vector& sizes, T* out_values, int root) { - int nprocs = comm.size(); - int myrank = comm.rank(); - - std::vector displs(nprocs); - for (int rank = 0, aux = 0; rank < nprocs; ++rank) { - displs[rank] = aux; - aux += sizes[rank]; - } - ::boost::mpi::scatterv(comm, in_values, sizes, displs, out_values, - sizes[myrank], root); + detail::scatterv_impl(comm, in_values, out_values, sizes[comm.rank()], + sizes.data(), (int const*)0, + root, is_mpi_datatype()); } template @@ -161,6 +147,23 @@ scatterv(const communicator& comm, const std::vector& in_values, ::boost::mpi::scatterv(comm, &in_values[0], sizes, out_values, root); } +template +void +scatterv(const communicator& comm, const T* in_values, + T* out_values, int n, int root) +{ + detail::scatterv_impl(comm, in_values, out_values, out_sizes, sizes.data(), (it const*)0, + root, is_mpi_datatype()); +} + +template +void +scatterv(const communicator& comm, const std::vector& in_values, + T* out_values, int out_size, int root) +{ + ::boost::mpi::scatterv(comm, &in_values[0], out_values, out_size, root); +} + } } // end namespace boost::mpi #endif // BOOST_MPI_SCATTERV_HPP diff --git a/test/gps_position.hpp b/test/gps_position.hpp index 109a3a9..e6910c4 100644 --- a/test/gps_position.hpp +++ b/test/gps_position.hpp @@ -10,6 +10,7 @@ #include #include #include +#include class gps_position { @@ -32,6 +33,8 @@ class gps_position degrees(d), minutes(m), seconds(s) {} + friend std::ostream& operator<<(std::ostream& out, const gps_position& g); + friend bool operator==(const gps_position& x, const gps_position& y) { return (x.degrees == y.degrees @@ -45,6 +48,11 @@ class gps_position } }; +inline +std::ostream& operator<<(std::ostream& out, const gps_position& g) { + out << "gps{" << g.degrees << 'd' << g.minutes << 'm' << g.seconds << "s}"; + return out; +} namespace boost { namespace mpi { diff --git a/test/scatter_test.cpp b/test/scatter_test.cpp index 4e91d8d..011a494 100644 --- a/test/scatter_test.cpp +++ b/test/scatter_test.cpp @@ -5,6 +5,7 @@ // http://www.boost.org/LICENSE_1_0.txt) // A test of the scatter() and scatterv() collectives. +#include #include #include #include @@ -54,49 +55,6 @@ scatter_test(const communicator& comm, Generator generator, } -template -void -scatterv_test(const communicator& comm, Generator generator, - const char* kind, int root = -1) -{ - typedef typename Generator::result_type value_type; - - if (root == -1) { - for (root = 0; root < comm.size(); ++root) - scatterv_test(comm, generator, kind, root); - } else { - using boost::mpi::scatterv; - - int mysize = comm.rank() + 1; - std::vector myvalues(mysize); - - if (comm.rank() == root) { - std::vector values; - std::vector sizes(comm.size()); - - // process p will receive p+1 identical generator(p) elements - for (int p = 0; p < comm.size(); ++p) { - for (int i = 0; i < p+1; ++i) - values.push_back(generator(p)); - sizes[p] = p + 1; - } - - std::cout << "Scatteringv " << kind << " from root " - << root << "..." << std::endl; - - scatterv(comm, values, sizes, &myvalues[0], root); - } else { - scatterv(comm, &myvalues[0], mysize, root); - } - - for (int i = 0; i < mysize; ++i) - BOOST_CHECK(myvalues[i] == generator(comm.rank())); - } - - (comm.barrier)(); -} - - // // Generators to test with scatter/scatterv // @@ -145,6 +103,57 @@ struct string_list_generator } }; +std::ostream& +operator<<(std::ostream& out, std::list const& l) { + out << '['; + std::copy(l.begin(), l.end(), std::ostream_iterator(out, " ")); + out << ']'; + return out; +} + +template +void +scatterv_test(const communicator& comm, Generator generator, + const char* kind, int root = -1) +{ + typedef typename Generator::result_type value_type; + + if (root == -1) { + for (root = 0; root < comm.size(); ++root) + scatterv_test(comm, generator, kind, root); + } else { + using boost::mpi::scatterv; + + int mysize = comm.rank() + 1; + std::vector myvalues(mysize); + + if (comm.rank() == root) { + std::vector values; + std::vector sizes(comm.size()); + + // process p will receive p+1 identical generator(p) elements + for (int p = 0; p < comm.size(); ++p) { + for (int i = 0; i < p+1; ++i) + values.push_back(generator(p)); + sizes[p] = p + 1; + } + + std::cout << "Scatteringv " << kind << " from root " + << root << "..." << std::endl; + assert(mysize == sizes[comm.rank()]); + scatterv(comm, values, sizes, myvalues.data(), root); + } else { + scatterv(comm, myvalues.data(), mysize, root); + } + + for (int i = 0; i < mysize; ++i) + BOOST_CHECK(myvalues[i] == generator(comm.rank())); + } + + (comm.barrier)(); +} + + int test_main(int argc, char* argv[]) { boost::mpi::environment env(argc, argv); From 2c9d9e6df72411e7f59e6cdf978204ebde800a00 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Mon, 24 Oct 2016 15:35:33 +0200 Subject: [PATCH 57/99] Move non template code in non template file. --- build/Jamfile.v2 | 1 + include/boost/mpi/collectives/scatterv.hpp | 20 +++------ include/boost/mpi/detail/offsets.hpp | 42 ++++++------------ src/offsets.cpp | 71 ++++++++++++++++++++++++++++++ 4 files changed, 93 insertions(+), 41 deletions(-) create mode 100644 src/offsets.cpp diff --git a/build/Jamfile.v2 b/build/Jamfile.v2 index c6138b7..455896b 100644 --- a/build/Jamfile.v2 +++ b/build/Jamfile.v2 @@ -62,6 +62,7 @@ lib boost_mpi request.cpp text_skeleton_oarchive.cpp timer.cpp + offsets.cpp : # Requirements ../../serialization/build//boost_serialization /mpi//mpi [ mpi.extra-requirements ] diff --git a/include/boost/mpi/collectives/scatterv.hpp b/include/boost/mpi/collectives/scatterv.hpp index 3be2df2..51773f8 100644 --- a/include/boost/mpi/collectives/scatterv.hpp +++ b/include/boost/mpi/collectives/scatterv.hpp @@ -8,7 +8,9 @@ #ifndef BOOST_MPI_SCATTERV_HPP #define BOOST_MPI_SCATTERV_HPP +#include #include +#include namespace boost { namespace mpi { @@ -25,19 +27,11 @@ void scatterv_impl(const communicator& comm, const T* in_values, T* out_values, int out_size, const int* sizes, const int* displs, int root, mpl::true_) { - std::vector offsets; - if (sizes) { - assert(out_size == sizes[comm.rank()]); - assert(in_values); - if (!displs) { - int nproc = comm.size(); - offsets.resize(nproc); - sizes2offsets(sizes, offsets.data(), nproc); - displs = offsets.data(); - } - } else { - assert(comm.rank() != root); - } + assert(!sizes || out_size == sizes[comm.rank()]); + assert(bool(sizes) == bool(in_values)); + + scoped_array new_offsets_mem(make_scatter_offsets(comm, sizes, displs, root)); + if (new_offsets_mem) displs = new_offsets_mem.get(); MPI_Datatype type = get_mpi_datatype(*in_values); BOOST_MPI_CHECK_RESULT(MPI_Scatterv, (const_cast(in_values), const_cast(sizes), diff --git a/include/boost/mpi/detail/offsets.hpp b/include/boost/mpi/detail/offsets.hpp index e9374d9..a5f79ea 100644 --- a/include/boost/mpi/detail/offsets.hpp +++ b/include/boost/mpi/detail/offsets.hpp @@ -4,47 +4,33 @@ // http://www.boost.org/LICENSE_1_0.txt) // Authors: Alain Miniussi -#ifndef BOOST_MPI_OFFSETS_HPP -#define BOOST_MPI_OFFSETS_HPP + +#include +#include namespace boost { namespace mpi { namespace detail { // Convert a sequence of sizes [S0..Sn] to a sequence displacement // [O0..On] where O[0] = 0 and O[k+1] = O[k]+S[k]. -void -sizes2offsets(int const* sizes, int* offsets, int n) -{ - offsets[0] = 0; - for(int i = 1; i < n; ++i) { - offsets[i] = offsets[i-1] + sizes[i-1]; - } -} +void sizes2offsets(int const* sizes, int* offsets, int n); -// Convert a sequence of sizes [S0..Sn] to a sequence displacement -// [O0..On] where O[0] = 0 and O[k+1] = O[k]+S[k]. -void -sizes2offsets(std::vector const& sizes, std::vector& offsets) -{ - int sz = sizes.size(); - offsets.resize(sz); - sizes2offsets(sizes.data(), offsets.data(), sz); -} +// Same as size2convert(sizes.data(), offsets.data(), sizes.size()) +void sizes2offsets(std::vector const& sizes, std::vector& offsets); // Given a sequence of sizes (typically the number of records dispatched // to each process in a scater) and a sequence of displacements (typically the // slot index at with those record starts), convert the later to a number // of skipped slots. -void -offsets2skipped(int const* sizes, int const* offsets, int* skipped, int n) -{ - skipped[0] = 0; - for(int i = 1; i < n; ++i) { - skipped[i] -= offsets[i-1] + sizes[i-1]; - } -} +void offsets2skipped(int const* sizes, int const* offsets, int* skipped, int n); +// Reconstruct offsets from sizes assuming no padding. +// Only takes place if on the root process and if +// displs are not already provided. +// If memory was allocated, returns a pointer to it +// otherwise null. +int* make_scatter_offsets(communicator const& comm, int const* sizes, int const* displs, int root); } }} -#endif // BOOST_MPI_OFFSETS_HPP + diff --git a/src/offsets.cpp b/src/offsets.cpp new file mode 100644 index 0000000..3259693 --- /dev/null +++ b/src/offsets.cpp @@ -0,0 +1,71 @@ +// Copyright Alain Miniussi 2014. +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +// Authors: Alain Miniussi + +#include + +namespace boost { namespace mpi { +namespace detail { + +// Convert a sequence of sizes [S0..Sn] to a sequence displacement +// [O0..On] where O[0] = 0 and O[k+1] = O[k]+S[k]. +void +sizes2offsets(int const* sizes, int* offsets, int n) +{ + offsets[0] = 0; + for(int i = 1; i < n; ++i) { + offsets[i] = offsets[i-1] + sizes[i-1]; + } +} + +// Convert a sequence of sizes [S0..Sn] to a sequence displacement +// [O0..On] where O[0] = 0 and O[k+1] = O[k]+S[k]. +void +sizes2offsets(std::vector const& sizes, std::vector& offsets) +{ + int sz = sizes.size(); + offsets.resize(sz); + sizes2offsets(sizes.data(), offsets.data(), sz); +} + +// Given a sequence of sizes (typically the number of records dispatched +// to each process in a scater) and a sequence of displacements (typically the +// slot index at with those record starts), convert the later to a number +// of skipped slots. +void +offsets2skipped(int const* sizes, int const* offsets, int* skipped, int n) +{ + skipped[0] = 0; + for(int i = 1; i < n; ++i) { + skipped[i] -= offsets[i-1] + sizes[i-1]; + } +} + +// Reconstruct offsets from sizes assuming no padding. +// Only takes place if on the root process and if +// displs are not already provided. +// If memory was allocated, returns a pointer to it +// otherwise null. +int* +make_scatter_offsets(communicator const& comm, int const* sizes, int const* displs, int root) +{ + if (root == comm.rank()) { + assert(sizes); + if (!displs) { + int nproc = comm.size(); + int* offsets = new int[nproc]; + displs = offsets; + sizes2offsets(sizes, offsets, nproc); + return offsets; + } else { + return 0; + } + } else { + return 0; + } +} +} +}} From 0408414e56f12af33bf2280f9c423da778205738 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Mon, 24 Oct 2016 15:47:00 +0200 Subject: [PATCH 58/99] coords=>coordinates --- doc/mpi.qbk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/mpi.qbk b/doc/mpi.qbk index 4ca3924..cece9a7 100644 --- a/doc/mpi.qbk +++ b/doc/mpi.qbk @@ -1758,7 +1758,7 @@ algorithms. [[[@http://www.mpi-forum.org/docs/mpi-1.1/mpi-11-html/node136.html#Node136 `MPI_Cart_rank`]] [[memberref boost::mpi::cartesian_communicator::rank `cartesian_communicator::rank` ]]] [[[@http://www.mpi-forum.org/docs/mpi-1.1/mpi-11-html/node136.html#Node136 - `MPI_Cart_coords`]] [[memberref boost::mpi::cartesian_communicator::coords `cartesian_communicator::coords` ]]] + `MPI_Cart_coords`]] [[memberref boost::mpi::cartesian_communicator::coordinates `cartesian_communicator::coordinates` ]]] [[[@http://www.mpi-forum.org/docs/mpi-1.1/mpi-11-html/node136.html#Node136 `MPI_Graph_neighbors_count`]] [[funcref boost::mpi::out_degree `out_degree`]]] From 6844f31d3a1f87cf0671b9265086385847cc1618 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Mon, 24 Oct 2016 17:00:10 +0200 Subject: [PATCH 59/99] indentation --- test/cartesian_topology_test.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/test/cartesian_topology_test.cpp b/test/cartesian_topology_test.cpp index 3875a1b..8a21523 100644 --- a/test/cartesian_topology_test.cpp +++ b/test/cartesian_topology_test.cpp @@ -1,4 +1,3 @@ - // Copyright Alain Miniussi 2014. // Distributed under the Boost Software License, Version 1.0. // (See accompanying file LICENSE_1_0.txt or copy at From 6c2fbc1c76cf4e87de89ef11522458da48702d12 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Mon, 24 Oct 2016 17:00:27 +0200 Subject: [PATCH 60/99] Basic cartesian example --- example/cartesian_communicator.cpp | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 example/cartesian_communicator.cpp diff --git a/example/cartesian_communicator.cpp b/example/cartesian_communicator.cpp new file mode 100644 index 0000000..cff2428 --- /dev/null +++ b/example/cartesian_communicator.cpp @@ -0,0 +1,37 @@ +// Copyright Alain Miniussi 2014. +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +// Authors: Alain Miniussi + +#include +#include + +#include +#include +#include +#include + +#include + +namespace mpi = boost::mpi; +int test_main(int argc, char* argv[]) +{ + mpi::environment env; + mpi::communicator world; + + if (world.size() != 24) return -1; + mpi::cartesian_dimension dims[] = {{2, true}, {3,true}, {4,true}}; + mpi::cartesian_communicator cart(world, mpi::cartesian_topology(dims)); + for (int r = 0; r < cart.size(); ++r) { + cart.barrier(); + if (r == cart.rank()) { + std::vector c = cart.coordinates(r); + std::cout << "rk :" << r << " coords: " + << c[0] << ' ' << c[1] << ' ' << c[2] << '\n'; + } + } + return 0; +} + From 24fffa408d030871bb38559da6d125eddc2a9910 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Mon, 24 Oct 2016 17:00:58 +0200 Subject: [PATCH 61/99] Make sure examples do compile --- test/Jamfile.v2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/Jamfile.v2 b/test/Jamfile.v2 index e6ba795..f7d796e 100644 --- a/test/Jamfile.v2 +++ b/test/Jamfile.v2 @@ -17,6 +17,8 @@ if [ mpi.configured ] { test-suite mpi : + [ mpi-test random_gather : ../example/random_gather.cpp : : 2 ] + [ mpi-test cartesian_communicator : ../example/cartesian_communicator.cpp : : 24 ] [ mpi-test cartesian_topology_init_test : : : 1 ] [ mpi-test broadcast_stl_test : : : 2 ] [ mpi-test all_gather_test ] From e7a98f6bc73279e92d774951b3ba33f62dc75688 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Mon, 24 Oct 2016 17:01:20 +0200 Subject: [PATCH 62/99] add doc entry for cartesian communicator. --- doc/mpi.qbk | 36 ++++++++++++++++++++++++++++++++++++ example/random_gather.cpp | 3 +-- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/doc/mpi.qbk b/doc/mpi.qbk index cece9a7..d5f613e 100644 --- a/doc/mpi.qbk +++ b/doc/mpi.qbk @@ -1097,6 +1097,42 @@ difference (`-`) of two groups, generate arbitrary subgroups, etc. [endsect] +[section:cartesian_communicator Cartesian communicator] + +A communicator can be organised as a cartesian grid, here a basic example: + + #include + #include + + #include + #include + #include + #include + + #include + + namespace mpi = boost::mpi; + int test_main(int argc, char* argv[]) + { + mpi::environment env; + mpi::communicator world; + + if (world.size() != 24) return -1; + mpi::cartesian_dimension dims[] = {{2, true}, {3,true}, {4,true}}; + mpi::cartesian_communicator cart(world, mpi::cartesian_topology(dims)); + for (int r = 0; r < cart.size(); ++r) { + cart.barrier(); + if (r == cart.rank()) { + std::vector c = cart.coordinates(r); + std::cout << "rk :" << r << " coords: " + << c[0] << ' ' << c[1] << ' ' << c[2] << '\n'; + } + } + return 0; + } + + + [section:skeleton_and_content Separating structure from content] When communicating data types over MPI that are not fundamental to MPI diff --git a/example/random_gather.cpp b/example/random_gather.cpp index 5483ba7..17825f1 100644 --- a/example/random_gather.cpp +++ b/example/random_gather.cpp @@ -4,7 +4,7 @@ // License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at // http://www.boost.org/LICENSE_1_0.txt) -// An example using Boost.MPI's gather() +// An example using Boost.MPI's gather(): [main] #include #include @@ -27,6 +27,5 @@ int main(int argc, char* argv[]) } else { gather(world, my_number, 0); } - return 0; } From 806d161cac8514a73885cf1359072071871df011 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Mon, 24 Oct 2016 17:33:10 +0200 Subject: [PATCH 63/99] add scatter example code --- example/random_scatter.cpp | 37 +++++++++++++++++++++++++++++++ include/boost/mpi/collectives/scatter.hpp | 2 +- test/Jamfile.v2 | 1 + 3 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 example/random_scatter.cpp diff --git a/example/random_scatter.cpp b/example/random_scatter.cpp new file mode 100644 index 0000000..fc8879e --- /dev/null +++ b/example/random_scatter.cpp @@ -0,0 +1,37 @@ +// Copyright (C) 2006 Douglas Gregor + +// Use, modification and distribution is subject to the Boost Software +// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +// An example using Boost.MPI's gather(): [main] + +#include +#include +#include +#include +#include + +namespace mpi = boost::mpi; + +int main(int argc, char* argv[]) +{ + mpi::environment env(argc, argv); + mpi::communicator world; + + std::srand(time(0) + world.rank()); + std::vector all; + int mine = -1; + if (world.rank() == 0) { + all.resize(world.size()); + std::generate(all.begin(), all.end(), std::rand); + } + mpi::scatter(world, all, mine, 0); + for (int r = 0; r < world.size(); ++r) { + world.barrier(); + if (r == world.rank()) { + std::cout << "Rank " << r << " got " << mine << '\n'; + } + } + return 0; +} diff --git a/include/boost/mpi/collectives/scatter.hpp b/include/boost/mpi/collectives/scatter.hpp index 46c6b49..0c3171f 100644 --- a/include/boost/mpi/collectives/scatter.hpp +++ b/include/boost/mpi/collectives/scatter.hpp @@ -156,7 +156,7 @@ void scatter(const communicator& comm, const std::vector& in_values, T& out_value, int root) { - ::boost::mpi::scatter(comm, &in_values[0], out_value, root); + ::boost::mpi::scatter(comm, in_values.data(), out_value, root); } template diff --git a/test/Jamfile.v2 b/test/Jamfile.v2 index f7d796e..7f2062d 100644 --- a/test/Jamfile.v2 +++ b/test/Jamfile.v2 @@ -18,6 +18,7 @@ if [ mpi.configured ] test-suite mpi : [ mpi-test random_gather : ../example/random_gather.cpp : : 2 ] + [ mpi-test random_scatter : ../example/random_scatter.cpp : : 2 ] [ mpi-test cartesian_communicator : ../example/cartesian_communicator.cpp : : 24 ] [ mpi-test cartesian_topology_init_test : : : 1 ] [ mpi-test broadcast_stl_test : : : 2 ] From 4b5c3735a188f9deb961aed7fb9131825bae99d4 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Mon, 24 Oct 2016 18:21:38 +0200 Subject: [PATCH 64/99] add an example for scatter --- doc/mpi.qbk | 58 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/doc/mpi.qbk b/doc/mpi.qbk index d5f613e..01f4f57 100644 --- a/doc/mpi.qbk +++ b/doc/mpi.qbk @@ -765,7 +765,7 @@ The [funcref boost::mpi::gather `gather`] collective gathers the values produced by every process in a communicator into a vector of values on the "root" process (specified by an argument to `gather`). The /i/th element in the vector will correspond to the -value gathered fro mthe /i/th process. For instance, in the following +value gathered from the /i/th process. For instance, in the following program each process computes its own random number. All of these random numbers are gathered at process 0 (the "root" in this case), which prints out the values that correspond to each processor. @@ -821,6 +821,62 @@ which is semantically equivalent to calling `gather` followed by a [endsect] +[section:scatter Scatter] +The [funcref boost::mpi::scatter `scatter`] collective scatters +the values from a vector in the "root" process in a communicator into +values in all the processes of the communicator. + The /i/th element in the vector will correspond to the +value received by the /i/th process. For instance, in the following +program, the root process produces a vector of random nomber and send +one value to each process that will print it. (`random_scatter.cpp`) + + #include + #include + #include + #include + #include + + namespace mpi = boost::mpi; + + int main(int argc, char* argv[]) + { + mpi::environment env(argc, argv); + mpi::communicator world; + + std::srand(time(0) + world.rank()); + std::vector all; + int mine = -1; + if (world.rank() == 0) { + all.resize(world.size()); + std::generate(all.begin(), all.end(), std::rand); + } + mpi::scatter(world, all, mine, 0); + for (int r = 0; r < world.size(); ++r) { + world.barrier(); + if (r == world.rank()) { + std::cout << "Rank " << r << " got " << mine << '\n'; + } + } + return 0; + } + +Executing this program with seven processes will result in output such +as the following. Although the random values will change from one run +to the next, the order of the processes in the output will remain the +same because of the barrier. + +[pre +Rank 0 got 1409381269 +Rank 1 got 17045268 +Rank 2 got 440120016 +Rank 3 got 936998224 +Rank 4 got 1827129182 +Rank 5 got 1951746047 +Rank 6 got 2117359639 +] + +[endsect] + [section:reduce Reduce] The [funcref boost::mpi::reduce `reduce`] collective From f55fee5e3679d9818a5723a97cf0ddba56372820 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Mon, 24 Oct 2016 19:19:44 +0200 Subject: [PATCH 65/99] Iterate on output instead of using indexes. It will be easier to merge with the variadic version. --- include/boost/mpi/collectives/gather.hpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/include/boost/mpi/collectives/gather.hpp b/include/boost/mpi/collectives/gather.hpp index c0363eb..2a16e8b 100644 --- a/include/boost/mpi/collectives/gather.hpp +++ b/include/boost/mpi/collectives/gather.hpp @@ -56,7 +56,8 @@ gather_impl(const communicator& comm, const T* in_values, int n, int root, // it. template void -gather_impl(const communicator& comm, const T* in_values, int n, +gather_impl(const communicator& comm, + const T* in_values, int n, T* out_values, int root, mpl::false_) { int tag = environment::collectives_tag(); @@ -75,8 +76,8 @@ gather_impl(const communicator& comm, const T* in_values, int n, root, MPI_Comm(comm))); // Gather the archives, which can be of different sizes, so // we need to use gatherv. - // Every thing is contiguous, so the offsets can be - // deduced from the collected sizes. + // Everything is contiguous (in the transmitted archive), so + // the offsets can be deduced from the collected sizes. std::vector offsets; if (comm.rank() == root) sizes2offsets(oasizes, offsets); packed_iarchive::buffer_type recv_buffer(std::accumulate(oasizes.begin(), oasizes.end(), 0)); @@ -87,11 +88,13 @@ gather_impl(const communicator& comm, const T* in_values, int n, if (comm.rank() == root) { for (int src = 0; src < nproc; ++src) { if (src == root) { - std::copy(in_values, in_values + n, out_values + n * src); + for (int i = 0; i < n; ++i) { + *out_values++ = *in_values++; + } } else { packed_iarchive ia(comm, recv_buffer, boost::archive::no_header, offsets[src]); for (int i = 0; i < n; ++i) { - ia >> out_values[n*src + i]; + ia >> *out_values++; } } } @@ -114,11 +117,8 @@ template void gather(const communicator& comm, const T& in_value, T* out_values, int root) { - if (comm.rank() == root) - detail::gather_impl(comm, &in_value, 1, out_values, root, - is_mpi_datatype()); - else - detail::gather_impl(comm, &in_value, 1, root, is_mpi_datatype()); + BOOST_ASSERT(out_values || (comm.rank() != root)); + detail::gather_impl(comm, &in_value, 1, out_values, root, is_mpi_datatype()); } template From 0a6092d5ffa69e18816adfd57e7d87920ef460df Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Mon, 24 Oct 2016 19:45:43 +0200 Subject: [PATCH 66/99] Generalized the gather MPI implementation function for non MPI datatypes in order to accomodate the variadic implementation. --- include/boost/mpi/collectives/gather.hpp | 35 +++++++++++++++----------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/include/boost/mpi/collectives/gather.hpp b/include/boost/mpi/collectives/gather.hpp index 2a16e8b..dd6a811 100644 --- a/include/boost/mpi/collectives/gather.hpp +++ b/include/boost/mpi/collectives/gather.hpp @@ -56,9 +56,8 @@ gather_impl(const communicator& comm, const T* in_values, int n, int root, // it. template void -gather_impl(const communicator& comm, - const T* in_values, int n, - T* out_values, int root, mpl::false_) +gather_impl(const communicator& comm, const T* in_values, int n, T* out_values, + int const* nslot, int const* nskip, int root, mpl::false_) { int tag = environment::collectives_tag(); int nproc = comm.size(); @@ -87,13 +86,18 @@ gather_impl(const communicator& comm, root, MPI_Comm(comm))); if (comm.rank() == root) { for (int src = 0; src < nproc; ++src) { + // handle variadic case + int nb = nslot ? nslot[src] : n; + int skip = nskip ? nskip[src] : 0; + std::advance(out_values, skip); if (src == root) { - for (int i = 0; i < n; ++i) { + BOOST_ASSERT(nb == n); + for (int i = 0; i < nb; ++i) { *out_values++ = *in_values++; } } else { packed_iarchive ia(comm, recv_buffer, boost::archive::no_header, offsets[src]); - for (int i = 0; i < n; ++i) { + for (int i = 0; i < nb; ++i) { ia >> *out_values++; } } @@ -106,10 +110,10 @@ gather_impl(const communicator& comm, // it. template void -gather_impl(const communicator& comm, const T* in_values, int n, int root, +gather_impl(const communicator& comm, const T* in_values, int n, T* out_values,int root, mpl::false_ is_mpi_type) { - gather_impl(comm, in_values, n, (T*)0, root, is_mpi_type); + gather_impl(comm, in_values, n, out_values, (int const*)0, (int const*)0, root, is_mpi_type); } } // end namespace detail @@ -135,10 +139,8 @@ gather(const communicator& comm, const T& in_value, std::vector& out_values, { if (comm.rank() == root) { out_values.resize(comm.size()); - ::boost::mpi::gather(comm, in_value, &out_values[0], root); - } else { - ::boost::mpi::gather(comm, in_value, root); } + ::boost::mpi::gather(comm, in_value, out_values.data(), root); } template @@ -146,11 +148,8 @@ void gather(const communicator& comm, const T* in_values, int n, T* out_values, int root) { - if (comm.rank() == root) - detail::gather_impl(comm, in_values, n, out_values, root, - is_mpi_datatype()); - else - detail::gather_impl(comm, in_values, n, root, is_mpi_datatype()); + detail::gather_impl(comm, in_values, n, out_values, root, + is_mpi_datatype()); } template @@ -160,10 +159,8 @@ gather(const communicator& comm, const T* in_values, int n, { if (comm.rank() == root) { out_values.resize(comm.size() * n); - ::boost::mpi::gather(comm, in_values, n, &out_values[0], root); - } - else - ::boost::mpi::gather(comm, in_values, n, root); + } + ::boost::mpi::gather(comm, in_values, n, out_values.data(), root); } template From d81cbab4a135aa875b052851f84726983f5901be Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Mon, 24 Oct 2016 20:49:23 +0200 Subject: [PATCH 67/99] gatherv is now point 2 point free. --- include/boost/mpi/collectives/gatherv.hpp | 37 +++++++++---------------------- include/boost/mpi/detail/offsets.hpp | 8 +++++++ src/offsets.cpp | 24 ++++++++++++++++++++ 3 files changed, 42 insertions(+), 27 deletions(-) diff --git a/include/boost/mpi/collectives/gatherv.hpp b/include/boost/mpi/collectives/gatherv.hpp index 2bfebe3..10345b7 100644 --- a/include/boost/mpi/collectives/gatherv.hpp +++ b/include/boost/mpi/collectives/gatherv.hpp @@ -8,15 +8,18 @@ #ifndef BOOST_MPI_GATHERV_HPP #define BOOST_MPI_GATHERV_HPP +#include + #include #include -#include #include #include #include #include #include +#include #include +#include namespace boost { namespace mpi { @@ -58,41 +61,21 @@ namespace detail { gatherv_impl(const communicator& comm, const T* in_values, int in_size, T* out_values, const int* sizes, const int* displs, int root, mpl::false_) { - int tag = environment::collectives_tag(); - int nprocs = comm.size(); - - for (int src = 0; src < nprocs; ++src) { - if (src == root) - // Our own values will never be transmitted: just copy them. - std::copy(in_values, in_values + in_size, out_values + displs[src]); - else { - // comm.recv(src, tag, out_values + displs[src], sizes[src]); - // Receive archive - packed_iarchive ia(comm); - MPI_Status status; - detail::packed_archive_recv(comm, src, tag, ia, status); - for (int i = 0; i < sizes[src]; ++i) - ia >> out_values[ displs[src] + i ]; - } - } + // convert displacement to offsets to skip + scoped_array skipped(make_gather_skipped(comm, sizes, displs, root)); + gather_impl(comm, in_values, in_size, out_values, sizes, skipped.get(), root, mpl::false_()); } // We're gathering at a non-root for a type that does not have an // associated MPI datatype, so we'll need to serialize - // it. Unfortunately, this means that we cannot use MPI_Gatherv, so - // we'll just have all of the non-root nodes send individual - // messages to the root. + // it. template void gatherv_impl(const communicator& comm, const T* in_values, int in_size, int root, mpl::false_) { - int tag = environment::collectives_tag(); -// comm.send(root, tag, in_values, in_size); - packed_oarchive oa(comm); - for (int i = 0; i < in_size; ++i) - oa << in_values[i]; - detail::packed_archive_send(comm, root, tag, oa); + gather_impl(comm, in_values, in_size, (T*)0,(int const*)0,(int const*)0, root, + mpl::false_()); } } // end namespace detail diff --git a/include/boost/mpi/detail/offsets.hpp b/include/boost/mpi/detail/offsets.hpp index a5f79ea..58ce981 100644 --- a/include/boost/mpi/detail/offsets.hpp +++ b/include/boost/mpi/detail/offsets.hpp @@ -31,6 +31,14 @@ void offsets2skipped(int const* sizes, int const* offsets, int* skipped, int n); // otherwise null. int* make_scatter_offsets(communicator const& comm, int const* sizes, int const* displs, int root); +// Reconstruct skip slots from sizes and offsets. +// Only takes place if on the root process and if +// displs are provided. +// If memory was allocated, returns a pointer to it +// otherwise null. +int* +make_gather_skipped(communicator const& comm, int const* sizes, int const* displs, int root); + } }} diff --git a/src/offsets.cpp b/src/offsets.cpp index 3259693..5c88e46 100644 --- a/src/offsets.cpp +++ b/src/offsets.cpp @@ -67,5 +67,29 @@ make_scatter_offsets(communicator const& comm, int const* sizes, int const* disp return 0; } } + +// Reconstruct skip slots from sizes and offsets. +// Only takes place if on the root process and if +// displs are provided. +// If memory was allocated, returns a pointer to it +// otherwise null. +int* +make_gather_skipped(communicator const& comm, int const* sizes, int const* displs, int root) +{ + if (root == comm.rank()) { + assert(sizes); + if (displs) { + int nproc = comm.size(); + int* skipped = new int[nproc]; + std::copy(displs, displs+nproc, skipped); + offsets2skipped(sizes, displs, skipped, nproc); + return skipped; + } else { + return 0; + } + } else { + return 0; + } +} } }} From 61e06540987f16c72102aebbf4b80750cfb95b3c Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Tue, 25 Oct 2016 12:05:10 +0200 Subject: [PATCH 68/99] Iterate on ouput instead of using indexed access. Will make sharing implementation with variadic version easier. --- include/boost/mpi/collectives/all_gather.hpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/include/boost/mpi/collectives/all_gather.hpp b/include/boost/mpi/collectives/all_gather.hpp index d2c741a..a168fc6 100644 --- a/include/boost/mpi/collectives/all_gather.hpp +++ b/include/boost/mpi/collectives/all_gather.hpp @@ -73,11 +73,13 @@ all_gather_impl(const communicator& comm, const T* in_values, int n, MPI_Comm(comm))); for (int src = 0; src < nproc; ++src) { if (src == comm.rank()) { // this is our local data - std::copy(in_values, in_values + n, out_values + n * src); + for (int i = 0; i < n; ++i) { + *out_values++ = *in_values++; + } } else { packed_iarchive ia(comm, recv_buffer, boost::archive::no_header, offsets[src]); for (int i = 0; i < n; ++i) { - ia >> out_values[n*src + i]; + ia >> *out_values++; } } } From 2a31dde18341a4644dd21f7a69cd24b7e84c4338 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Tue, 25 Oct 2016 13:42:58 +0200 Subject: [PATCH 69/99] Inject sizes and displacement support. --- include/boost/mpi/collectives/all_gather.hpp | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/include/boost/mpi/collectives/all_gather.hpp b/include/boost/mpi/collectives/all_gather.hpp index a168fc6..fd4f00a 100644 --- a/include/boost/mpi/collectives/all_gather.hpp +++ b/include/boost/mpi/collectives/all_gather.hpp @@ -44,7 +44,7 @@ all_gather_impl(const communicator& comm, const T* in_values, int n, template void all_gather_impl(const communicator& comm, const T* in_values, int n, - T* out_values, mpl::false_) + T* out_values, int const* sizes, int const* skips, mpl::false_) { int tag = environment::collectives_tag(); int nproc = comm.size(); @@ -72,19 +72,32 @@ all_gather_impl(const communicator& comm, const T* in_values, int n, recv_buffer.data(), oasizes.data(), offsets.data(), MPI_BYTE, MPI_Comm(comm))); for (int src = 0; src < nproc; ++src) { + int nb = sizes ? sizes[src] : n; + int skip = skips ? skips[src] : 0; + std::advance(out_values, skip); if (src == comm.rank()) { // this is our local data - for (int i = 0; i < n; ++i) { + for (int i = 0; i < nb; ++i) { *out_values++ = *in_values++; } } else { packed_iarchive ia(comm, recv_buffer, boost::archive::no_header, offsets[src]); - for (int i = 0; i < n; ++i) { + for (int i = 0; i < nb; ++i) { ia >> *out_values++; } } } } +// We're all-gathering for a type that does not have an +// associated MPI datatype, so we'll need to serialize +// it. +template +void +all_gather_impl(const communicator& comm, const T* in_values, int n, + T* out_values, mpl::false_ isnt_mpi_type) +{ + all_gather_impl(comm, in_values, n, out_values, (int const*)0, (int const*)0, isnt_mpi_type); +} } // end namespace detail template From da49ba7d4c928684cf5d06430202e6ea0fdf171a Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Tue, 25 Oct 2016 15:34:22 +0200 Subject: [PATCH 70/99] special flag to mark non rooted opertaions --- include/boost/mpi/detail/offsets.hpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/include/boost/mpi/detail/offsets.hpp b/include/boost/mpi/detail/offsets.hpp index 58ce981..dcb0749 100644 --- a/include/boost/mpi/detail/offsets.hpp +++ b/include/boost/mpi/detail/offsets.hpp @@ -5,6 +5,9 @@ // Authors: Alain Miniussi +#ifndef BOOST_MPI_OFFSETS_HPP +#define BOOST_MPI_OFFSETS_HPP + #include #include @@ -15,7 +18,7 @@ namespace detail { // [O0..On] where O[0] = 0 and O[k+1] = O[k]+S[k]. void sizes2offsets(int const* sizes, int* offsets, int n); -// Same as size2convert(sizes.data(), offsets.data(), sizes.size()) +// Same as size2offset(sizes.data(), offsets.data(), sizes.size()) void sizes2offsets(std::vector const& sizes, std::vector& offsets); // Given a sequence of sizes (typically the number of records dispatched @@ -36,9 +39,9 @@ int* make_scatter_offsets(communicator const& comm, int const* sizes, int const* // displs are provided. // If memory was allocated, returns a pointer to it // otherwise null. -int* -make_gather_skipped(communicator const& comm, int const* sizes, int const* displs, int root); +int* make_gather_skipped(communicator const& comm, int const* sizes, int const* displs, int root = -1); } -}} +}}// end namespace boost::mpi +#endif // BOOST_MPI_OFFSETS_HPP From 8ff3bf01e011875f87cf1eea556c7de29e80f5c5 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Tue, 25 Oct 2016 15:36:38 +0200 Subject: [PATCH 71/99] all_gatherv implementation --- include/boost/mpi/collectives/all_gatherv.hpp | 133 ++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 include/boost/mpi/collectives/all_gatherv.hpp diff --git a/include/boost/mpi/collectives/all_gatherv.hpp b/include/boost/mpi/collectives/all_gatherv.hpp new file mode 100644 index 0000000..20764cd --- /dev/null +++ b/include/boost/mpi/collectives/all_gatherv.hpp @@ -0,0 +1,133 @@ +// Copyright (C) 2005, 2006 Douglas Gregor. + +// Use, modification and distribution is subject to the Boost Software +// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +// Message Passing Interface 1.1 -- Section 4.5. Gatherv +#ifndef BOOST_MPI_ALLGATHERV_HPP +#define BOOST_MPI_ALLGATHERV_HPP + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace boost { namespace mpi { + +namespace detail { +// We're all-gathering for a type that has an associated MPI +// datatype, so we'll use MPI_Gather to do all of the work. +template +void +all_gatherv_impl(const communicator& comm, const T* in_values, + T* out_values, int const* sizes, int const* displs, mpl::true_) +{ + // Make displacements if not provided + scoped_array new_offsets_mem(make_scatter_offsets(comm, sizes, displs, -1)); + if (new_offsets_mem) displs = new_offsets_mem.get(); + MPI_Datatype type = get_mpi_datatype(*in_values); + BOOST_MPI_CHECK_RESULT(MPI_Allgatherv, + (const_cast(in_values), sizes[comm.rank()], type, + out_values, + const_cast(sizes), + const_cast(displs), + type, + comm)); +} + +// We're all-gathering for a type that does not have an +// associated MPI datatype, so we'll need to serialize +// it. +template +void +all_gatherv_impl(const communicator& comm, const T* in_values, + T* out_values, int const* sizes, int const* displs, + mpl::false_ isnt_mpi_type) +{ + // convert displacement to offsets to skip + scoped_array skipped(make_gather_skipped(comm, sizes, displs)); + all_gather_impl(comm, in_values, sizes[comm.rank()], out_values, + sizes, skipped.get(), isnt_mpi_type); +} +} // end namespace detail + +template +void +all_gatherv(const communicator& comm, const T& in_value, T* out_values, + const std::vector& sizes) +{ + assert(sizes.size() == comm.size()); + assert(sizes[comm.rank()] == 1); + detail::all_gatherv_impl(comm, &in_value, out_values, sizes.data(), 0, is_mpi_datatype()); +} + +template +void +all_gatherv(const communicator& comm, const T* in_values, T* out_values, + const std::vector& sizes) +{ + assert(sizes.size() == comm.size()); + detail::all_gatherv_impl(comm, in_values, out_values, sizes.data(), 0, is_mpi_datatype()); +} + +template +void +all_gatherv(const communicator& comm, std::vector const& in_values, std::vector& out_values, + const std::vector& sizes) +{ + assert(sizes.size() == comm.size()); + assert(in_values.size() == sizes[comm.rank()]); + out_values.resize(std::accumulate(sizes.begin(), sizes.end(), 0)); + ::boost::mpi::all_gatherv(comm, in_values.data(), out_values.data(), sizes); +} + + +template +void +all_gatherv(const communicator& comm, const T& in_value, T* out_values, + const std::vector& sizes, const std::vector& displs) +{ + assert(sizes.size() == comm.size()); + assert(displs.size() == comm.size()); + detail::all_gatherv_impl(comm, &in_value, 1, out_values, + sizes.data(), displs.data(), is_mpi_datatype()); +} + +template +void +all_gatherv(const communicator& comm, const T* in_values, T* out_values, + const std::vector& sizes, const std::vector& displs) +{ + assert(sizes.size() == comm.size()); + assert(displs.size() == comm.size()); + detail::all_gatherv_impl(comm, in_values, out_values, + sizes.data(), displs.data(), is_mpi_datatype()); +} + +template +void +all_gatherv(const communicator& comm, std::vector const& in_values, std::vector& out_values, + const std::vector& sizes, const std::vector& displs) +{ + assert(sizes.size() == comm.size()); + assert(displs.size() == comm.size()); + assert(in_values.size() == sizes[comm.rank()]); + out_values.resize(std::accumulate(sizes.begin(), sizes.end(), 0)); + ::boost::mpi::all_gatherv(comm, in_values.data(), out_values.data(), sizes, displs) +} + +} } // end namespace boost::mpi + +#endif // BOOST_MPI_ALL_GATHERV_HPP From bd5b96ec0a42c419d915ca7487502d24d67fc662 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Tue, 25 Oct 2016 15:37:01 +0200 Subject: [PATCH 72/99] handle non rooted version of operation --- src/offsets.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/offsets.cpp b/src/offsets.cpp index 5c88e46..19fd210 100644 --- a/src/offsets.cpp +++ b/src/offsets.cpp @@ -52,7 +52,7 @@ offsets2skipped(int const* sizes, int const* offsets, int* skipped, int n) int* make_scatter_offsets(communicator const& comm, int const* sizes, int const* displs, int root) { - if (root == comm.rank()) { + if (root == -1 || root == comm.rank()) { assert(sizes); if (!displs) { int nproc = comm.size(); @@ -76,7 +76,7 @@ make_scatter_offsets(communicator const& comm, int const* sizes, int const* disp int* make_gather_skipped(communicator const& comm, int const* sizes, int const* displs, int root) { - if (root == comm.rank()) { + if (root == -1 || root == comm.rank()) { assert(sizes); if (displs) { int nproc = comm.size(); From 457feb7419b06b493bf1dd6851e3f3126ef2c18e Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Tue, 25 Oct 2016 15:37:32 +0200 Subject: [PATCH 73/99] add all_gatherv test --- test/all_gather_test.cpp | 61 +++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 50 insertions(+), 11 deletions(-) diff --git a/test/all_gather_test.cpp b/test/all_gather_test.cpp index a0543fd..1bd4da9 100644 --- a/test/all_gather_test.cpp +++ b/test/all_gather_test.cpp @@ -5,39 +5,38 @@ // http://www.boost.org/LICENSE_1_0.txt) // A test of the all_gather() collective. + +#include + #include +#include #include #include #include -#include -#include "gps_position.hpp" #include #include #include #include -using boost::mpi::communicator; +#include "gps_position.hpp" -using boost::mpi::packed_skeleton_iarchive; -using boost::mpi::packed_skeleton_oarchive; +namespace mpi = boost::mpi; template void -all_gather_test(const communicator& comm, Generator generator, - const char* kind) +all_gather_test(const mpi::communicator& comm, Generator generator, + std::string kind) { typedef typename Generator::result_type value_type; value_type value = generator(comm.rank()); - using boost::mpi::all_gather; - std::vector values; if (comm.rank() == 0) { std::cout << "Gathering " << kind << "..."; std::cout.flush(); } - all_gather(comm, value, values); + mpi::all_gather(comm, value, values); std::vector expected_values; for (int p = 0; p < comm.size(); ++p) @@ -49,6 +48,41 @@ all_gather_test(const communicator& comm, Generator generator, (comm.barrier)(); } +template +void +all_gatherv_test(const mpi::communicator& comm, Generator generator, + std::string kind) +{ + typedef typename Generator::result_type value_type; + using boost::mpi::all_gatherv; + + std::vector myvalues, expected, values; + std::vector sizes; + for(int r = 0; r < comm.size(); ++r) { + value_type value = generator(r); + sizes.push_back(r+1); + for (int k=0; k < r+1; ++k) { + expected.push_back(value); + if(comm.rank() == r) { + myvalues.push_back(value); + } + } + } + if (comm.rank() == 0) { + std::cout << "Gathering " << kind << "..."; + std::cout.flush(); + } + + mpi::all_gatherv(comm, myvalues, values, sizes); + + BOOST_CHECK(values == expected); + + if (comm.rank() == 0 && values == expected) + std::cout << "OK." << std::endl; + + (comm.barrier)(); +} + // Generates integers to test with gather() struct int_generator { @@ -99,10 +133,15 @@ struct string_list_generator int test_main(int argc, char* argv[]) { boost::mpi::environment env(argc, argv); - communicator comm; + mpi::communicator comm; all_gather_test(comm, int_generator(), "integers"); all_gather_test(comm, gps_generator(), "GPS positions"); all_gather_test(comm, string_generator(), "string"); all_gather_test(comm, string_list_generator(), "list of strings"); + + all_gatherv_test(comm, int_generator(), "integers"); + all_gatherv_test(comm, gps_generator(), "GPS positions"); + all_gatherv_test(comm, string_generator(), "string"); + all_gatherv_test(comm, string_list_generator(), "list of strings"); return 0; } From c332690024445db3df6dc9dfbd027d2b59f041ae Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Tue, 25 Oct 2016 15:37:46 +0200 Subject: [PATCH 74/99] shorter test --- test/Jamfile.v2 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/Jamfile.v2 b/test/Jamfile.v2 index 7f2062d..d678935 100644 --- a/test/Jamfile.v2 +++ b/test/Jamfile.v2 @@ -22,11 +22,11 @@ test-suite mpi [ mpi-test cartesian_communicator : ../example/cartesian_communicator.cpp : : 24 ] [ mpi-test cartesian_topology_init_test : : : 1 ] [ mpi-test broadcast_stl_test : : : 2 ] - [ mpi-test all_gather_test ] - [ mpi-test all_reduce_test ] - [ mpi-test all_to_all_test ] + [ mpi-test all_gather_test : : : 1 2 11 ] + [ mpi-test all_reduce_test : : : 1 2 11 ] + [ mpi-test all_to_all_test : : : 1 2 11 ] [ mpi-test broadcast_test : : : 2 17 ] - [ mpi-test gather_test ] + [ mpi-test gather_test : : : 1 2 11 ] [ mpi-test is_mpi_op_test : : : 1 ] [ mpi-test mt_level_test : : : 1 ] [ mpi-test mt_init_test-single : mt_init_test.cpp : "single" : 1 4 ] From 4f6d3a5f8cedbc3ef232e6d8ca5a7214f6300c36 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Tue, 25 Oct 2016 15:47:39 +0200 Subject: [PATCH 75/99] typo --- include/boost/mpi/collectives/gather.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/boost/mpi/collectives/gather.hpp b/include/boost/mpi/collectives/gather.hpp index dd6a811..7116023 100644 --- a/include/boost/mpi/collectives/gather.hpp +++ b/include/boost/mpi/collectives/gather.hpp @@ -129,7 +129,7 @@ template void gather(const communicator& comm, const T& in_value, int root) { BOOST_ASSERT(comm.rank() != root); - detail::gather_impl(comm, &in_value, 1, root, is_mpi_datatype()); + detail::gather_impl(comm, &in_value, 1, (T*)0, root, is_mpi_datatype()); } template From e169ada11093afe63dd84d47e1e7274785f65dea Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Tue, 25 Oct 2016 15:47:45 +0200 Subject: [PATCH 76/99] renaming --- include/boost/mpi/collectives/all_gatherv.hpp | 2 +- include/boost/mpi/collectives/gatherv.hpp | 2 +- include/boost/mpi/detail/offsets.hpp | 2 +- src/offsets.cpp | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/boost/mpi/collectives/all_gatherv.hpp b/include/boost/mpi/collectives/all_gatherv.hpp index 20764cd..90fc0e3 100644 --- a/include/boost/mpi/collectives/all_gatherv.hpp +++ b/include/boost/mpi/collectives/all_gatherv.hpp @@ -57,7 +57,7 @@ all_gatherv_impl(const communicator& comm, const T* in_values, mpl::false_ isnt_mpi_type) { // convert displacement to offsets to skip - scoped_array skipped(make_gather_skipped(comm, sizes, displs)); + scoped_array skipped(make_skipped_slots(comm, sizes, displs)); all_gather_impl(comm, in_values, sizes[comm.rank()], out_values, sizes, skipped.get(), isnt_mpi_type); } diff --git a/include/boost/mpi/collectives/gatherv.hpp b/include/boost/mpi/collectives/gatherv.hpp index 10345b7..6b8d706 100644 --- a/include/boost/mpi/collectives/gatherv.hpp +++ b/include/boost/mpi/collectives/gatherv.hpp @@ -62,7 +62,7 @@ namespace detail { T* out_values, const int* sizes, const int* displs, int root, mpl::false_) { // convert displacement to offsets to skip - scoped_array skipped(make_gather_skipped(comm, sizes, displs, root)); + scoped_array skipped(make_skipped_slots(comm, sizes, displs, root)); gather_impl(comm, in_values, in_size, out_values, sizes, skipped.get(), root, mpl::false_()); } diff --git a/include/boost/mpi/detail/offsets.hpp b/include/boost/mpi/detail/offsets.hpp index dcb0749..e037630 100644 --- a/include/boost/mpi/detail/offsets.hpp +++ b/include/boost/mpi/detail/offsets.hpp @@ -39,7 +39,7 @@ int* make_scatter_offsets(communicator const& comm, int const* sizes, int const* // displs are provided. // If memory was allocated, returns a pointer to it // otherwise null. -int* make_gather_skipped(communicator const& comm, int const* sizes, int const* displs, int root = -1); +int* make_skipped_slots(communicator const& comm, int const* sizes, int const* displs, int root = -1); } }}// end namespace boost::mpi diff --git a/src/offsets.cpp b/src/offsets.cpp index 19fd210..58fc872 100644 --- a/src/offsets.cpp +++ b/src/offsets.cpp @@ -74,7 +74,7 @@ make_scatter_offsets(communicator const& comm, int const* sizes, int const* disp // If memory was allocated, returns a pointer to it // otherwise null. int* -make_gather_skipped(communicator const& comm, int const* sizes, int const* displs, int root) +make_skipped_slots(communicator const& comm, int const* sizes, int const* displs, int root) { if (root == -1 || root == comm.rank()) { assert(sizes); From debeb7c8c43018e18af8b7cb48bd75021b437439 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Tue, 25 Oct 2016 15:53:14 +0200 Subject: [PATCH 77/99] Renaming --- include/boost/mpi/collectives/all_gatherv.hpp | 2 +- include/boost/mpi/collectives/scatterv.hpp | 2 +- include/boost/mpi/detail/offsets.hpp | 2 +- src/offsets.cpp | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/boost/mpi/collectives/all_gatherv.hpp b/include/boost/mpi/collectives/all_gatherv.hpp index 90fc0e3..c979d6d 100644 --- a/include/boost/mpi/collectives/all_gatherv.hpp +++ b/include/boost/mpi/collectives/all_gatherv.hpp @@ -35,7 +35,7 @@ all_gatherv_impl(const communicator& comm, const T* in_values, T* out_values, int const* sizes, int const* displs, mpl::true_) { // Make displacements if not provided - scoped_array new_offsets_mem(make_scatter_offsets(comm, sizes, displs, -1)); + scoped_array new_offsets_mem(make_offsets(comm, sizes, displs, -1)); if (new_offsets_mem) displs = new_offsets_mem.get(); MPI_Datatype type = get_mpi_datatype(*in_values); BOOST_MPI_CHECK_RESULT(MPI_Allgatherv, diff --git a/include/boost/mpi/collectives/scatterv.hpp b/include/boost/mpi/collectives/scatterv.hpp index 51773f8..c8e286c 100644 --- a/include/boost/mpi/collectives/scatterv.hpp +++ b/include/boost/mpi/collectives/scatterv.hpp @@ -30,7 +30,7 @@ scatterv_impl(const communicator& comm, const T* in_values, T* out_values, int o assert(!sizes || out_size == sizes[comm.rank()]); assert(bool(sizes) == bool(in_values)); - scoped_array new_offsets_mem(make_scatter_offsets(comm, sizes, displs, root)); + scoped_array new_offsets_mem(make_offsets(comm, sizes, displs, root)); if (new_offsets_mem) displs = new_offsets_mem.get(); MPI_Datatype type = get_mpi_datatype(*in_values); BOOST_MPI_CHECK_RESULT(MPI_Scatterv, diff --git a/include/boost/mpi/detail/offsets.hpp b/include/boost/mpi/detail/offsets.hpp index e037630..7e5ab7d 100644 --- a/include/boost/mpi/detail/offsets.hpp +++ b/include/boost/mpi/detail/offsets.hpp @@ -32,7 +32,7 @@ void offsets2skipped(int const* sizes, int const* offsets, int* skipped, int n); // displs are not already provided. // If memory was allocated, returns a pointer to it // otherwise null. -int* make_scatter_offsets(communicator const& comm, int const* sizes, int const* displs, int root); +int* make_offsets(communicator const& comm, int const* sizes, int const* displs, int root = -1); // Reconstruct skip slots from sizes and offsets. // Only takes place if on the root process and if diff --git a/src/offsets.cpp b/src/offsets.cpp index 58fc872..8080501 100644 --- a/src/offsets.cpp +++ b/src/offsets.cpp @@ -50,7 +50,7 @@ offsets2skipped(int const* sizes, int const* offsets, int* skipped, int n) // If memory was allocated, returns a pointer to it // otherwise null. int* -make_scatter_offsets(communicator const& comm, int const* sizes, int const* displs, int root) +make_offsets(communicator const& comm, int const* sizes, int const* displs, int root) { if (root == -1 || root == comm.rank()) { assert(sizes); From d3f671ea0a9cdb477bef23d47db30068ce186890 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Tue, 25 Oct 2016 16:34:46 +0200 Subject: [PATCH 78/99] factorize --- include/boost/mpi/collectives/gather.hpp | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/include/boost/mpi/collectives/gather.hpp b/include/boost/mpi/collectives/gather.hpp index 7116023..badc8c8 100644 --- a/include/boost/mpi/collectives/gather.hpp +++ b/include/boost/mpi/collectives/gather.hpp @@ -43,12 +43,10 @@ gather_impl(const communicator& comm, const T* in_values, int n, template void gather_impl(const communicator& comm, const T* in_values, int n, int root, - mpl::true_) + mpl::true_ is_mpi_type) { - MPI_Datatype type = get_mpi_datatype(*in_values); - BOOST_MPI_CHECK_RESULT(MPI_Gather, - (const_cast(in_values), n, type, - 0, n, type, root, comm)); + assert(comm.rank() != root); + gather_impl(comm, in_values, n, (T*)0, root, is_mpi_type); } // We're gathering at the root for a type that does not have an From 864ad6011608d58bf55bd7b8e85c79ca3f0586ed Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Wed, 26 Oct 2016 10:10:04 +0200 Subject: [PATCH 79/99] Only allocate receiv buffer on root node in gather. --- include/boost/mpi/collectives/gather.hpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/include/boost/mpi/collectives/gather.hpp b/include/boost/mpi/collectives/gather.hpp index badc8c8..d478397 100644 --- a/include/boost/mpi/collectives/gather.hpp +++ b/include/boost/mpi/collectives/gather.hpp @@ -65,7 +65,8 @@ gather_impl(const communicator& comm, const T* in_values, int n, T* out_values, for (int i = 0; i < n; ++i) { oa << in_values[i]; } - std::vector oasizes(nproc); + bool on_root = comm.rank() == root; + std::vector oasizes(on_root ? nproc : 0); int oasize = oa.size(); BOOST_MPI_CHECK_RESULT(MPI_Gather, (&oasize, 1, MPI_INTEGER, @@ -76,13 +77,13 @@ gather_impl(const communicator& comm, const T* in_values, int n, T* out_values, // Everything is contiguous (in the transmitted archive), so // the offsets can be deduced from the collected sizes. std::vector offsets; - if (comm.rank() == root) sizes2offsets(oasizes, offsets); - packed_iarchive::buffer_type recv_buffer(std::accumulate(oasizes.begin(), oasizes.end(), 0)); + if (on_root) sizes2offsets(oasizes, offsets); + packed_iarchive::buffer_type recv_buffer(on_root ? std::accumulate(oasizes.begin(), oasizes.end(), 0) : 0); BOOST_MPI_CHECK_RESULT(MPI_Gatherv, (const_cast(oa.address()), int(oa.size()), MPI_BYTE, recv_buffer.data(), oasizes.data(), offsets.data(), MPI_BYTE, root, MPI_Comm(comm))); - if (comm.rank() == root) { + if (on_root) { for (int src = 0; src < nproc; ++src) { // handle variadic case int nb = nslot ? nslot[src] : n; From 91a19d2e6a64f5fbec6b773ff38cc118818f0462 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Wed, 26 Oct 2016 10:15:29 +0200 Subject: [PATCH 80/99] uniform var name. --- include/boost/mpi/collectives/gather.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/include/boost/mpi/collectives/gather.hpp b/include/boost/mpi/collectives/gather.hpp index d478397..5943a22 100644 --- a/include/boost/mpi/collectives/gather.hpp +++ b/include/boost/mpi/collectives/gather.hpp @@ -65,8 +65,8 @@ gather_impl(const communicator& comm, const T* in_values, int n, T* out_values, for (int i = 0; i < n; ++i) { oa << in_values[i]; } - bool on_root = comm.rank() == root; - std::vector oasizes(on_root ? nproc : 0); + bool is_root = comm.rank() == root; + std::vector oasizes(is_root ? nproc : 0); int oasize = oa.size(); BOOST_MPI_CHECK_RESULT(MPI_Gather, (&oasize, 1, MPI_INTEGER, @@ -77,13 +77,13 @@ gather_impl(const communicator& comm, const T* in_values, int n, T* out_values, // Everything is contiguous (in the transmitted archive), so // the offsets can be deduced from the collected sizes. std::vector offsets; - if (on_root) sizes2offsets(oasizes, offsets); - packed_iarchive::buffer_type recv_buffer(on_root ? std::accumulate(oasizes.begin(), oasizes.end(), 0) : 0); + if (is_root) sizes2offsets(oasizes, offsets); + packed_iarchive::buffer_type recv_buffer(is_root ? std::accumulate(oasizes.begin(), oasizes.end(), 0) : 0); BOOST_MPI_CHECK_RESULT(MPI_Gatherv, (const_cast(oa.address()), int(oa.size()), MPI_BYTE, recv_buffer.data(), oasizes.data(), offsets.data(), MPI_BYTE, root, MPI_Comm(comm))); - if (on_root) { + if (is_root) { for (int src = 0; src < nproc; ++src) { // handle variadic case int nb = nslot ? nslot[src] : n; From 8e3e9cd6d09ca8a21a8596a4a54b82d6b90d5b90 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Wed, 26 Oct 2016 10:28:08 +0200 Subject: [PATCH 81/99] activate non blocking test. Avoid case with 1 proc due to a problem with Microsoft's MPI. (I do not have Microsoft MPI, just trusted the comment). --- test/Jamfile.v2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Jamfile.v2 b/test/Jamfile.v2 index d678935..9e6cd60 100644 --- a/test/Jamfile.v2 +++ b/test/Jamfile.v2 @@ -34,7 +34,7 @@ test-suite mpi [ mpi-test mt_init_test-serialized : mt_init_test.cpp : "serialized" : 1 4 ] [ mpi-test mt_init_test-multiple : mt_init_test.cpp : "multiple" : 1 4 ] # Note: Microsoft MPI fails nonblocking_test on 1 processor -# [ mpi-test nonblocking_test ] + [ mpi-test nonblocking_test : : : 2 11 24 ] [ mpi-test reduce_test ] [ mpi-test ring_test : : : 2 3 4 7 8 13 17 ] [ mpi-test sendrecv_test : : : 1 4 7 48 ] From e3b0d08c7e1cf1d4df2ce348f65dd2fa59d97cb2 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Thu, 27 Oct 2016 00:54:51 +0200 Subject: [PATCH 82/99] Explicitly avoid temporary in isend. --- .../boost/mpi/detail/binary_buffer_oprimitive.hpp | 7 +++++- include/boost/mpi/detail/packed_oprimitive.hpp | 5 ++++ src/point_to_point.cpp | 28 +++++++++++----------- 3 files changed, 25 insertions(+), 15 deletions(-) diff --git a/include/boost/mpi/detail/binary_buffer_oprimitive.hpp b/include/boost/mpi/detail/binary_buffer_oprimitive.hpp index 1de441d..313097b 100644 --- a/include/boost/mpi/detail/binary_buffer_oprimitive.hpp +++ b/include/boost/mpi/detail/binary_buffer_oprimitive.hpp @@ -47,7 +47,12 @@ class BOOST_MPI_DECL binary_buffer_oprimitive { return size_ = buffer_.size(); } - + + const std::size_t* size_ptr() const + { + return &size(); + } + void save_binary(void const *address, std::size_t count) { save_impl(address,count); diff --git a/include/boost/mpi/detail/packed_oprimitive.hpp b/include/boost/mpi/detail/packed_oprimitive.hpp index 5b6b3b2..fbcde9a 100644 --- a/include/boost/mpi/detail/packed_oprimitive.hpp +++ b/include/boost/mpi/detail/packed_oprimitive.hpp @@ -47,6 +47,11 @@ class BOOST_MPI_DECL packed_oprimitive return size_ = buffer_.size(); } + const std::size_t* size_ptr() const + { + return &size(); + } + void save_binary(void const *address, std::size_t count) { save_impl(address,MPI_BYTE,count); diff --git a/src/point_to_point.cpp b/src/point_to_point.cpp index 7822152..c14b934 100644 --- a/src/point_to_point.cpp +++ b/src/point_to_point.cpp @@ -28,13 +28,13 @@ void packed_archive_send(MPI_Comm comm, int dest, int tag, const packed_oarchive& ar) { - std::size_t size = ar.size(); + std::size_t const& size = ar.size(); BOOST_MPI_CHECK_RESULT(MPI_Send, - (static_cast(&size), 1, - get_mpi_datatype(ar.size()), + (&size, 1, + get_mpi_datatype(size), dest, tag, comm)); BOOST_MPI_CHECK_RESULT(MPI_Send, - (const_cast(ar.address()), ar.size(), + (const_cast(ar.address()), size, MPI_PACKED, dest, tag, comm)); } @@ -45,13 +45,13 @@ packed_archive_isend(MPI_Comm comm, int dest, int tag, MPI_Request* out_requests, int num_out_requests) { assert(num_out_requests >= 2); - const void* size = &ar.size(); + std::size_t const& size = ar.size(); BOOST_MPI_CHECK_RESULT(MPI_Isend, - (const_cast(size), 1, - get_mpi_datatype(ar.size()), + (&size, 1, + get_mpi_datatype(size), dest, tag, comm, out_requests)); BOOST_MPI_CHECK_RESULT(MPI_Isend, - (const_cast(ar.address()), ar.size(), + (const_cast(ar.address()), size, MPI_PACKED, dest, tag, comm, out_requests + 1)); @@ -65,13 +65,13 @@ packed_archive_isend(MPI_Comm comm, int dest, int tag, { assert(num_out_requests >= 2); - const void* size = &ar.size(); + std::size_t const& size = ar.size(); BOOST_MPI_CHECK_RESULT(MPI_Isend, - (const_cast(size), 1, - get_mpi_datatype(ar.size()), + (&size, 1, + get_mpi_datatype(size), dest, tag, comm, out_requests)); BOOST_MPI_CHECK_RESULT(MPI_Isend, - (const_cast(ar.address()), ar.size(), + (const_cast(ar.address()), size, MPI_PACKED, dest, tag, comm, out_requests + 1)); @@ -84,13 +84,13 @@ packed_archive_recv(MPI_Comm comm, int source, int tag, packed_iarchive& ar, { std::size_t count; BOOST_MPI_CHECK_RESULT(MPI_Recv, - (&count, 1, get_mpi_datatype(count), + (&count, 1, get_mpi_datatype(count), source, tag, comm, &status)); // Prepare input buffer and receive the message ar.resize(count); BOOST_MPI_CHECK_RESULT(MPI_Recv, - (ar.address(), ar.size(), MPI_PACKED, + (ar.address(), count, MPI_PACKED, status.MPI_SOURCE, status.MPI_TAG, comm, &status)); } From 9d4e00a3c80f9c6cec70e830edd79b882e982ee5 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Fri, 28 Oct 2016 00:48:20 +0200 Subject: [PATCH 83/99] using g++5.4 detected a lot of compilation problems on template code that wasn't instantiated by the tests (which is an issue to deal with). --- include/boost/mpi/collectives/all_gather.hpp | 1 - include/boost/mpi/collectives/all_gatherv.hpp | 8 ++++---- include/boost/mpi/collectives/gather.hpp | 1 - include/boost/mpi/collectives/scatterv.hpp | 4 ++-- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/include/boost/mpi/collectives/all_gather.hpp b/include/boost/mpi/collectives/all_gather.hpp index fd4f00a..cd467a9 100644 --- a/include/boost/mpi/collectives/all_gather.hpp +++ b/include/boost/mpi/collectives/all_gather.hpp @@ -46,7 +46,6 @@ void all_gather_impl(const communicator& comm, const T* in_values, int n, T* out_values, int const* sizes, int const* skips, mpl::false_) { - int tag = environment::collectives_tag(); int nproc = comm.size(); // first, gather all size, these size can be different for // each process diff --git a/include/boost/mpi/collectives/all_gatherv.hpp b/include/boost/mpi/collectives/all_gatherv.hpp index c979d6d..d62cc7b 100644 --- a/include/boost/mpi/collectives/all_gatherv.hpp +++ b/include/boost/mpi/collectives/all_gatherv.hpp @@ -78,7 +78,7 @@ void all_gatherv(const communicator& comm, const T* in_values, T* out_values, const std::vector& sizes) { - assert(sizes.size() == comm.size()); + assert(int(sizes.size()) == comm.size()); detail::all_gatherv_impl(comm, in_values, out_values, sizes.data(), 0, is_mpi_datatype()); } @@ -87,8 +87,8 @@ void all_gatherv(const communicator& comm, std::vector const& in_values, std::vector& out_values, const std::vector& sizes) { - assert(sizes.size() == comm.size()); - assert(in_values.size() == sizes[comm.rank()]); + assert(int(sizes.size()) == comm.size()); + assert(int(in_values.size()) == sizes[comm.rank()]); out_values.resize(std::accumulate(sizes.begin(), sizes.end(), 0)); ::boost::mpi::all_gatherv(comm, in_values.data(), out_values.data(), sizes); } @@ -125,7 +125,7 @@ all_gatherv(const communicator& comm, std::vector const& in_values, std::vect assert(displs.size() == comm.size()); assert(in_values.size() == sizes[comm.rank()]); out_values.resize(std::accumulate(sizes.begin(), sizes.end(), 0)); - ::boost::mpi::all_gatherv(comm, in_values.data(), out_values.data(), sizes, displs) + ::boost::mpi::all_gatherv(comm, in_values.data(), out_values.data(), sizes, displs); } } } // end namespace boost::mpi diff --git a/include/boost/mpi/collectives/gather.hpp b/include/boost/mpi/collectives/gather.hpp index 5943a22..6ebfda5 100644 --- a/include/boost/mpi/collectives/gather.hpp +++ b/include/boost/mpi/collectives/gather.hpp @@ -57,7 +57,6 @@ void gather_impl(const communicator& comm, const T* in_values, int n, T* out_values, int const* nslot, int const* nskip, int root, mpl::false_) { - int tag = environment::collectives_tag(); int nproc = comm.size(); // first, gather all size, these size can be different for // each process diff --git a/include/boost/mpi/collectives/scatterv.hpp b/include/boost/mpi/collectives/scatterv.hpp index c8e286c..eb9f41d 100644 --- a/include/boost/mpi/collectives/scatterv.hpp +++ b/include/boost/mpi/collectives/scatterv.hpp @@ -99,7 +99,7 @@ scatterv(const communicator& comm, const T* in_values, const std::vector& sizes, const std::vector& displs, T* out_values, int out_size, int root) { - scatterv_impl(comm, in_values, out_values, out_sizes, sizes.data(), displs.data(), + scatterv_impl(comm, in_values, out_values, out_size, sizes.data(), displs.data(), root, is_mpi_datatype()); } @@ -146,7 +146,7 @@ void scatterv(const communicator& comm, const T* in_values, T* out_values, int n, int root) { - detail::scatterv_impl(comm, in_values, out_values, out_sizes, sizes.data(), (it const*)0, + detail::scatterv_impl(comm, in_values, out_values, n, (int const*)0, (int const*)0, root, is_mpi_datatype()); } From 7f3a324481b848086c90c8e4b5d1991f52c04b3d Mon Sep 17 00:00:00 2001 From: Vladimir Prus Date: Fri, 28 Oct 2016 11:57:43 +0300 Subject: [PATCH 84/99] Use Boost-global Python tagging. --- build/Jamfile.v2 | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/build/Jamfile.v2 b/build/Jamfile.v2 index 455896b..5a1a54f 100644 --- a/build/Jamfile.v2 +++ b/build/Jamfile.v2 @@ -19,27 +19,8 @@ if [ mpi.configured ] project boost/mpi : source-location ../src - : requirements - -@$(BOOST_JAMROOT_MODULE)%$(BOOST_JAMROOT_MODULE).tag - @$(__name__).tag ; -rule tag ( name : type ? : property-set ) -{ - local result = $(name) ; - if $(type) in STATIC_LIB SHARED_LIB IMPORT_LIB - { - if $(name) = boost_mpi_python && $(PYTHON_ID) - { - result = $(result)-$(PYTHON_ID) ; - } - } - - # forward to the boost tagging rule - return [ indirect.call $(BOOST_JAMROOT_MODULE)%$(BOOST_JAMROOT_MODULE).tag - $(result) : $(type) : $(property-set) ] ; -} - lib boost_mpi : broadcast.cpp @@ -90,6 +71,8 @@ libraries += boost_mpi ; shared:BOOST_MPI_PYTHON_DYN_LINK=1 shared:BOOST_PYTHON_DYN_LINK=1 BOOST_MPI_PYTHON_SOURCE=1 + -@$(BOOST_JAMROOT_MODULE)%$(BOOST_JAMROOT_MODULE).tag + @$(BOOST_JAMROOT_MODULE)%$(BOOST_JAMROOT_MODULE).python-tag : # Default build shared : # Usage requirements From 817914da2bd23e4d98e8baf69b066906e54f8d82 Mon Sep 17 00:00:00 2001 From: "K. Noel Belcourt" Date: Mon, 31 Oct 2016 19:06:50 -0600 Subject: [PATCH 85/99] Add Alain to list of MPI maintainers. --- meta/libraries.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/meta/libraries.json b/meta/libraries.json index 9b3a4bb..ab1e98f 100644 --- a/meta/libraries.json +++ b/meta/libraries.json @@ -8,5 +8,9 @@ "description": "Message Passing Interface library, for use in distributed-memory parallel application programming.", "category": [ "Concurrent" + ], + "maintainers": [ + "K. Noel Belcourt ", + "Alain Miniussi " ] } From 415a41f6ea87ce8636d6397dfa5251446f6f0d93 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Wed, 2 Nov 2016 14:52:01 +0100 Subject: [PATCH 86/99] Test that static and dynamic MPI versions matches --- include/boost/mpi/environment.hpp | 6 ++++++ src/environment.cpp | 7 +++++++ test/Jamfile.v2 | 1 + test/version_test.cpp | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 46 insertions(+) create mode 100644 test/version_test.cpp diff --git a/include/boost/mpi/environment.hpp b/include/boost/mpi/environment.hpp index 92af129..cb90b6d 100644 --- a/include/boost/mpi/environment.hpp +++ b/include/boost/mpi/environment.hpp @@ -265,6 +265,12 @@ class BOOST_MPI_DECL environment : noncopyable { */ static bool is_main_thread(); + /** @brief MPI version. + * + * Returns a pair with the version and sub-version number. + */ + static std::pair version(); + private: /// Whether this environment object called MPI_Init bool i_initialized; diff --git a/src/environment.cpp b/src/environment.cpp index c1bf9e2..ffdfc90 100644 --- a/src/environment.cpp +++ b/src/environment.cpp @@ -247,4 +247,11 @@ bool environment::is_main_thread() return static_cast(isit); } +std::pair environment::version() +{ + int major, minor; + BOOST_MPI_CHECK_RESULT(MPI_Get_version, (&major, &minor)); + return std::make_pair(major, minor); +} + } } // end namespace boost::mpi diff --git a/test/Jamfile.v2 b/test/Jamfile.v2 index 9e6cd60..46b0e84 100644 --- a/test/Jamfile.v2 +++ b/test/Jamfile.v2 @@ -17,6 +17,7 @@ if [ mpi.configured ] { test-suite mpi : + [ mpi-test version_test : : : 1 ] [ mpi-test random_gather : ../example/random_gather.cpp : : 2 ] [ mpi-test random_scatter : ../example/random_scatter.cpp : : 2 ] [ mpi-test cartesian_communicator : ../example/cartesian_communicator.cpp : : 24 ] diff --git a/test/version_test.cpp b/test/version_test.cpp new file mode 100644 index 0000000..5ebd907 --- /dev/null +++ b/test/version_test.cpp @@ -0,0 +1,32 @@ +// Copyright (C) 2013 Alain Miniussi + +// Use, modification and distribution is subject to the Boost Software +// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +// test mpi version + +#include +#include +#include + +namespace mpi = boost::mpi; + +int +test_main(int argc, char* argv[]) { +#if defined(MPI_VERSION) + int mpi_version = MPI_VERSION; + int mpi_subversion = MPI_SUBVERSION; +#else + int mpi_version = 0; + int mpi_subversion = 0; +#endif + + mpi::environment env(argc,argv); + std::pair version = env.version(); + std::cout << "MPI Version: " << version.first << ',' << version.second << '\n'; + + BOOST_CHECK(version.first == mpi_version); + BOOST_CHECK(version.second == mpi_subversion); + return 0; +} From c5df30a54d021233db62a4987f9ab077e6b5f971 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Fri, 4 Nov 2016 00:11:14 +0100 Subject: [PATCH 87/99] Support recent g++ without c++11. --- include/boost/mpi/cartesian_communicator.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/boost/mpi/cartesian_communicator.hpp b/include/boost/mpi/cartesian_communicator.hpp index f3cc2a0..7a31bce 100644 --- a/include/boost/mpi/cartesian_communicator.hpp +++ b/include/boost/mpi/cartesian_communicator.hpp @@ -23,7 +23,9 @@ #include #include #include +#if !defined(BOOST_NO_CXX11_HDR_INITIALIZER_LIST) #include +#endif // BOOST_NO_CXX11_HDR_INITIALIZER_LIST // Headers required to implement cartesian topologies #include From 3440bbe24efad437805a0bfbae5bc712069fcc90 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Wed, 22 Feb 2017 11:14:21 +0100 Subject: [PATCH 88/99] Boost array was used, but corresponding include was missing. It's in a test, so using boost array vs std::array is a no brainer. --- test/cartesian_topology_init_test.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/test/cartesian_topology_init_test.cpp b/test/cartesian_topology_init_test.cpp index 6bcd547..acc7021 100644 --- a/test/cartesian_topology_init_test.cpp +++ b/test/cartesian_topology_init_test.cpp @@ -19,6 +19,7 @@ #include #include +#include #include #include From dc9122c00dd2cebd744fd11f83f93e2e4cb7a961 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Thu, 23 Feb 2017 03:16:46 +0100 Subject: [PATCH 89/99] workaround an Open MPI bug: (#37) According to: http://mpi-forum.org/docs/mpi-3.1/mpi31-report.pdf input data on Scatter should be passed as a pointer to const. Open MPI miss that const which invalidate the implicit conversion. --- include/boost/mpi/collectives/scatter.hpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/include/boost/mpi/collectives/scatter.hpp b/include/boost/mpi/collectives/scatter.hpp index 0c3171f..2645612 100644 --- a/include/boost/mpi/collectives/scatter.hpp +++ b/include/boost/mpi/collectives/scatter.hpp @@ -80,6 +80,13 @@ fill_scatter_sendbuf(const communicator& comm, T const* values, } } +template +T* +non_const_data(std::vector const& v) { + T const* cptr = v.data(); + return const_cast(cptr); +} + // Dispatch the sendbuf among proc. // Used in the second steps of both scatter and scatterv // in_value is only provide in the non variadic case. @@ -92,7 +99,7 @@ dispatch_scatter_sendbuf(const communicator& comm, // Distribute the sizes int myarchsize; BOOST_MPI_CHECK_RESULT(MPI_Scatter, - (archsizes.data(), 1, MPI_INTEGER, + (non_const_data(archsizes), 1, MPI_INTEGER, &myarchsize, 1, MPI_INTEGER, root, comm)); std::vector offsets; if (root == comm.rank()) { @@ -102,7 +109,7 @@ dispatch_scatter_sendbuf(const communicator& comm, packed_iarchive::buffer_type recvbuf; recvbuf.resize(myarchsize); BOOST_MPI_CHECK_RESULT(MPI_Scatterv, - (sendbuf.data(), archsizes.data(), offsets.data(), MPI_BYTE, + (non_const_data(sendbuf), non_const_data(archsizes), offsets.data(), MPI_BYTE, recvbuf.data(), recvbuf.size(), MPI_BYTE, root, MPI_Comm(comm))); // Unserialize From b21d95676028e45a6743be9898306f13fe715282 Mon Sep 17 00:00:00 2001 From: Andrey Semashev Date: Thu, 23 Feb 2017 05:17:32 +0300 Subject: [PATCH 90/99] Fix incorrect usage of auto_ptr to free an array (#38) The `auto_ptr` would use `delete p` to free an array allocated with `operator new[]`, which could result in heap corruption. Replaced it with `scoped_array`. --- include/boost/mpi/python/serialize.hpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/include/boost/mpi/python/serialize.hpp b/include/boost/mpi/python/serialize.hpp index 5f9136b..d5e042b 100644 --- a/include/boost/mpi/python/serialize.hpp +++ b/include/boost/mpi/python/serialize.hpp @@ -26,7 +26,6 @@ #include #include -#include #include #include @@ -37,6 +36,8 @@ #include #include +#include + #include #include @@ -441,7 +442,7 @@ load_impl(Archiver& ar, boost::python::object& obj, int len; ar >> len; - std::auto_ptr string(new char[len]); + boost::scoped_array string(new char[len]); ar >> boost::serialization::make_array(string.get(), len); boost::python::str py_string(string.get(), len); obj = boost::python::pickle::loads(py_string); From f5bdcc1ebfe954bb64835f2a0efd94471da42207 Mon Sep 17 00:00:00 2001 From: Jonathan Wakely Date: Thu, 23 Feb 2017 02:18:58 +0000 Subject: [PATCH 91/99] Replace boost::serialization::detail::get_data function. (#39) --- include/boost/mpi/detail/mpi_datatype_primitive.hpp | 19 ++++++++++++------- include/boost/mpi/detail/packed_iprimitive.hpp | 8 ++++++-- include/boost/mpi/detail/packed_oprimitive.hpp | 8 ++++++-- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/include/boost/mpi/detail/mpi_datatype_primitive.hpp b/include/boost/mpi/detail/mpi_datatype_primitive.hpp index c230055..b95fc38 100644 --- a/include/boost/mpi/detail/mpi_datatype_primitive.hpp +++ b/include/boost/mpi/detail/mpi_datatype_primitive.hpp @@ -25,7 +25,6 @@ namespace std{ #include #include #include -#include #include #include #include @@ -80,18 +79,18 @@ class mpi_datatype_primitive BOOST_MPI_CHECK_RESULT(MPI_Type_create_struct, ( addresses.size(), - boost::serialization::detail::get_data(lengths), - boost::serialization::detail::get_data(addresses), - boost::serialization::detail::get_data(types), + get_data(lengths), + get_data(addresses), + get_data(types), &datatype_ )); #else BOOST_MPI_CHECK_RESULT(MPI_Type_struct, ( addresses.size(), - boost::serialization::detail::get_data(lengths), - boost::serialization::detail::get_data(addresses), - boost::serialization::detail::get_data(types), + get_data(lengths), + get_data(addresses), + get_data(types), &datatype_ )); #endif @@ -129,6 +128,12 @@ class mpi_datatype_primitive lengths.push_back(l); } + template + static T* get_data(std::vector& v) + { + return v.empty() ? 0 : &(v[0]); + } + std::vector addresses; std::vector types; std::vector lengths; diff --git a/include/boost/mpi/detail/packed_iprimitive.hpp b/include/boost/mpi/detail/packed_iprimitive.hpp index 7080cbf..227dc8e 100644 --- a/include/boost/mpi/detail/packed_iprimitive.hpp +++ b/include/boost/mpi/detail/packed_iprimitive.hpp @@ -16,7 +16,6 @@ #include #include #include -#include #include #include @@ -104,7 +103,12 @@ class BOOST_MPI_DECL packed_iprimitive void load_impl(void * p, MPI_Datatype t, int l) { BOOST_MPI_CHECK_RESULT(MPI_Unpack, - (const_cast(boost::serialization::detail::get_data(buffer_)), buffer_.size(), &position, p, l, t, comm)); + (get_data(buffer_), buffer_.size(), &position, p, l, t, comm)); + } + + static buffer_type::value_type* get_data(buffer_type& b) + { + return b.empty() ? 0 : &(b[0]); } buffer_type & buffer_; diff --git a/include/boost/mpi/detail/packed_oprimitive.hpp b/include/boost/mpi/detail/packed_oprimitive.hpp index fbcde9a..3c81a70 100644 --- a/include/boost/mpi/detail/packed_oprimitive.hpp +++ b/include/boost/mpi/detail/packed_oprimitive.hpp @@ -15,7 +15,6 @@ #include #include -#include #include #include #include @@ -103,13 +102,18 @@ class BOOST_MPI_DECL packed_oprimitive // pack the data into the buffer BOOST_MPI_CHECK_RESULT(MPI_Pack, - (const_cast(p), l, t, boost::serialization::detail::get_data(buffer_), buffer_.size(), &position, comm)); + (const_cast(p), l, t, get_data(buffer_), buffer_.size(), &position, comm)); // reduce the buffer size if needed BOOST_ASSERT(std::size_t(position) <= buffer_.size()); if (std::size_t(position) < buffer_.size()) buffer_.resize(position); } + static buffer_type::value_type* get_data(buffer_type& b) + { + return b.empty() ? 0 : &(b[0]); + } + buffer_type& buffer_; mutable std::size_t size_; MPI_Comm comm; From 947d17157304ceb93926ab961487e9c86fc7c2d2 Mon Sep 17 00:00:00 2001 From: Jonathan Wakely Date: Thu, 23 Feb 2017 02:19:45 +0000 Subject: [PATCH 92/99] Add header for serialization::make_array (#40) --- include/boost/mpi/python/serialize.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/include/boost/mpi/python/serialize.hpp b/include/boost/mpi/python/serialize.hpp index d5e042b..08f8dcc 100644 --- a/include/boost/mpi/python/serialize.hpp +++ b/include/boost/mpi/python/serialize.hpp @@ -35,6 +35,7 @@ #include #include +#include #include From 98c406a77f8506550276a8e81ad6ed0c961fca8f Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Thu, 23 Feb 2017 11:26:49 +0100 Subject: [PATCH 93/99] documentation typo --- include/boost/mpi/communicator.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/boost/mpi/communicator.hpp b/include/boost/mpi/communicator.hpp index 1ecd892..c51494f 100644 --- a/include/boost/mpi/communicator.hpp +++ b/include/boost/mpi/communicator.hpp @@ -858,7 +858,7 @@ class BOOST_MPI_DECL communicator optional as_graph_communicator() const; /** - * Determines whether this communicator has a Cartesian topology. + * Determines whether this communicator has a Graph topology. */ bool has_graph_topology() const; From 5afae486125406566577b8427c531aded2b1c20b Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Thu, 23 Feb 2017 11:31:39 +0100 Subject: [PATCH 94/99] Delete now clearly obsolete #if 0ed code. --- include/boost/mpi/communicator.hpp | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/include/boost/mpi/communicator.hpp b/include/boost/mpi/communicator.hpp index c51494f..39e629b 100644 --- a/include/boost/mpi/communicator.hpp +++ b/include/boost/mpi/communicator.hpp @@ -880,26 +880,6 @@ class BOOST_MPI_DECL communicator */ bool has_cartesian_topology() const; -#if 0 - template - communicator - with_cartesian_topology(const Extents& extents, - bool periodic = false, - bool reorder = false) const; - - template - communicator - with_cartesian_topology(DimInputIterator first_dim, - DimInputIterator last_dim, - PeriodicInputIterator first_periodic, - bool reorder = false); - - template - communicator - with_cartesian_topology(const multi_array& periods, - bool reorder = false); -#endif - /** Abort all tasks in the group of this communicator. * * Makes a "best attempt" to abort all of the tasks in the group of From 90e84fe143caacee583fa5aeea1d678dd53aebac Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Thu, 23 Feb 2017 13:47:23 +0100 Subject: [PATCH 95/99] Change partial specialization position to agree with nvcc/icc --- include/boost/mpi/communicator.hpp | 218 ++++++++++++++++++------------------- 1 file changed, 109 insertions(+), 109 deletions(-) diff --git a/include/boost/mpi/communicator.hpp b/include/boost/mpi/communicator.hpp index c51494f..6467446 100644 --- a/include/boost/mpi/communicator.hpp +++ b/include/boost/mpi/communicator.hpp @@ -1148,6 +1148,115 @@ inline bool operator!=(const communicator& comm1, const communicator& comm2) /************************************************************************ * Implementation details * ************************************************************************/ + +/** + * INTERNAL ONLY (using the same 'end' name might be considerd unfortunate + */ +template<> +BOOST_MPI_DECL void +communicator::send(int dest, int tag, + const packed_oarchive& ar) const; + +/** + * INTERNAL ONLY + */ +template<> +BOOST_MPI_DECL void +communicator::send + (int dest, int tag, const packed_skeleton_oarchive& ar) const; + +/** + * INTERNAL ONLY + */ +template<> +BOOST_MPI_DECL void +communicator::send(int dest, int tag, const content& c) const; + +/** + * INTERNAL ONLY + */ +template<> +BOOST_MPI_DECL status +communicator::recv(int source, int tag, + packed_iarchive& ar) const; + +/** + * INTERNAL ONLY + */ +template<> +BOOST_MPI_DECL status +communicator::recv + (int source, int tag, packed_skeleton_iarchive& ar) const; + +/** + * INTERNAL ONLY + */ +template<> +BOOST_MPI_DECL status +communicator::recv(int source, int tag, + const content& c) const; + +/** + * INTERNAL ONLY + */ +template<> +inline status +communicator::recv(int source, int tag, + content& c) const +{ + return recv(source,tag,c); +} + +/** + * INTERNAL ONLY + */ +template<> +BOOST_MPI_DECL request +communicator::isend(int dest, int tag, + const packed_oarchive& ar) const; + +/** + * INTERNAL ONLY + */ +template<> +BOOST_MPI_DECL request +communicator::isend + (int dest, int tag, const packed_skeleton_oarchive& ar) const; + +/** + * INTERNAL ONLY + */ +template<> +BOOST_MPI_DECL request +communicator::isend(int dest, int tag, const content& c) const; + +/** + * INTERNAL ONLY + */ +template<> +BOOST_MPI_DECL request +communicator::irecv + (int source, int tag, packed_skeleton_iarchive& ar) const; + +/** + * INTERNAL ONLY + */ +template<> +BOOST_MPI_DECL request +communicator::irecv(int source, int tag, + const content& c) const; + +/** + * INTERNAL ONLY + */ +template<> +inline request +communicator::irecv(int source, int tag, + content& c) const +{ + return irecv(source, tag, c); +} + // Count elements in a message template inline optional status::count() const @@ -1768,115 +1877,6 @@ request communicator::irecv(int source, int tag, T* values, int n) const return this->array_irecv_impl(source, tag, values, n, is_mpi_datatype()); } -/** - * INTERNAL ONLY - */ -template<> -BOOST_MPI_DECL void -communicator::send(int dest, int tag, - const packed_oarchive& ar) const; - -/** - * INTERNAL ONLY - */ -template<> -BOOST_MPI_DECL void -communicator::send - (int dest, int tag, const packed_skeleton_oarchive& ar) const; - -/** - * INTERNAL ONLY - */ -template<> -BOOST_MPI_DECL void -communicator::send(int dest, int tag, const content& c) const; - -/** - * INTERNAL ONLY - */ -template<> -BOOST_MPI_DECL status -communicator::recv(int source, int tag, - packed_iarchive& ar) const; - -/** - * INTERNAL ONLY - */ -template<> -BOOST_MPI_DECL status -communicator::recv - (int source, int tag, packed_skeleton_iarchive& ar) const; - -/** - * INTERNAL ONLY - */ -template<> -BOOST_MPI_DECL status -communicator::recv(int source, int tag, - const content& c) const; - -/** - * INTERNAL ONLY - */ -template<> -inline status -communicator::recv(int source, int tag, - content& c) const -{ - return recv(source,tag,c); -} - -/** - * INTERNAL ONLY - */ -template<> -BOOST_MPI_DECL request -communicator::isend(int dest, int tag, - const packed_oarchive& ar) const; - -/** - * INTERNAL ONLY - */ -template<> -BOOST_MPI_DECL request -communicator::isend - (int dest, int tag, const packed_skeleton_oarchive& ar) const; - -/** - * INTERNAL ONLY - */ -template<> -BOOST_MPI_DECL request -communicator::isend(int dest, int tag, const content& c) const; - -/** - * INTERNAL ONLY - */ -template<> -BOOST_MPI_DECL request -communicator::irecv - (int source, int tag, packed_skeleton_iarchive& ar) const; - -/** - * INTERNAL ONLY - */ -template<> -BOOST_MPI_DECL request -communicator::irecv(int source, int tag, - const content& c) const; - -/** - * INTERNAL ONLY - */ -template<> -inline request -communicator::irecv(int source, int tag, - content& c) const -{ - return irecv(source, tag, c); -} - - } } // end namespace boost::mpi // If the user has already included skeleton_and_content.hpp, include From 26a990d90f281814934b7fcc7983826456965bdf Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Fri, 24 Feb 2017 13:36:51 +0100 Subject: [PATCH 96/99] Support for antique compilers with poour brace init support. --- example/cartesian_communicator.cpp | 9 ++++++++- test/cartesian_topology_init_test.cpp | 11 +++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/example/cartesian_communicator.cpp b/example/cartesian_communicator.cpp index cff2428..e06f150 100644 --- a/example/cartesian_communicator.cpp +++ b/example/cartesian_communicator.cpp @@ -16,13 +16,20 @@ #include namespace mpi = boost::mpi; +// Curly brace init make this useless, but +// - Need to support obsolete like g++ 4.3.x. for some reason +// - Can't conditionnaly compile with bjam (unless you find +// the doc, and read it, which would only make sense if you +// actually wan't to use bjam, which does not (make sense)) +typedef mpi::cartesian_dimension cd; + int test_main(int argc, char* argv[]) { mpi::environment env; mpi::communicator world; if (world.size() != 24) return -1; - mpi::cartesian_dimension dims[] = {{2, true}, {3,true}, {4,true}}; + mpi::cartesian_dimension dims[] = {cd(2, true), cd(3,true), cd(4,true)}; mpi::cartesian_communicator cart(world, mpi::cartesian_topology(dims)); for (int r = 0; r < cart.size(); ++r) { cart.barrier(); diff --git a/test/cartesian_topology_init_test.cpp b/test/cartesian_topology_init_test.cpp index acc7021..9da72ea 100644 --- a/test/cartesian_topology_init_test.cpp +++ b/test/cartesian_topology_init_test.cpp @@ -25,9 +25,12 @@ namespace mpi = boost::mpi; - BOOST_AUTO_TEST_CASE(cartesian_dimension_init) { + // Curly brace initialization syntax not supported on (very) old gnu + // This typedef keeps things shorter + typedef mpi::cartesian_dimension cd; + { // Check the basic ctor mpi::cartesian_dimension def; @@ -48,7 +51,7 @@ BOOST_AUTO_TEST_CASE(cartesian_dimension_init) // Container based ctor only available as a replacement for initializer list ctor { // seq ctor vs C array ctor - mpi::cartesian_dimension d[] = {{2,true},{3, false},{4, true}}; + mpi::cartesian_dimension d[] = {cd(2,true),cd(3, false),cd(4, true)}; std::list seq; std::copy(d, d+3, std::back_inserter(seq)); mpi::cartesian_topology t1(seq); @@ -57,7 +60,7 @@ BOOST_AUTO_TEST_CASE(cartesian_dimension_init) } { // Check range based with array based ctor. - boost::array d = {{{2,true},{3, false},{4, true}}}; + boost::array d = {{cd(2,true),cd(3, false),cd(4, true)}}; int dims[] = {2,3,4}; bool per[] = {true, false, true}; mpi::cartesian_topology t1(dims, per); @@ -67,7 +70,7 @@ BOOST_AUTO_TEST_CASE(cartesian_dimension_init) } { // Iterator based ctor vs C array based ctor - mpi::cartesian_dimension d[] = {{2,true},{3, false},{4, true}}; + mpi::cartesian_dimension d[] = {cd(2,true),cd(3, false),cd(4, true)}; std::vector vdims(d, d+3); mpi::cartesian_topology t1(vdims); mpi::cartesian_topology t2(d); From c5964b439781978a2eff4f4b65d7ec4d694690c0 Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Tue, 28 Feb 2017 13:44:44 +0100 Subject: [PATCH 97/99] void using vector::data in order to comply to old pre C++ compilers. --- include/boost/mpi/collectives/all_gather.hpp | 11 ++++++---- include/boost/mpi/collectives/all_gatherv.hpp | 19 ++++++++++------ include/boost/mpi/collectives/gather.hpp | 8 ++++--- include/boost/mpi/collectives/scatter.hpp | 14 ++++++------ include/boost/mpi/collectives/scatterv.hpp | 14 +++++++----- include/boost/mpi/detail/antiques.hpp | 31 +++++++++++++++++++++++++++ src/cartesian_communicator.cpp | 19 ++++++++++------ src/offsets.cpp | 3 ++- test/cartesian_topology_test.cpp | 4 ++-- test/scatter_test.cpp | 4 ++-- 10 files changed, 91 insertions(+), 36 deletions(-) create mode 100644 include/boost/mpi/detail/antiques.hpp diff --git a/include/boost/mpi/collectives/all_gather.hpp b/include/boost/mpi/collectives/all_gather.hpp index cd467a9..4adaeb9 100644 --- a/include/boost/mpi/collectives/all_gather.hpp +++ b/include/boost/mpi/collectives/all_gather.hpp @@ -20,6 +20,7 @@ #include #include #include +#include #include namespace boost { namespace mpi { @@ -57,7 +58,7 @@ all_gather_impl(const communicator& comm, const T* in_values, int n, int oasize = oa.size(); BOOST_MPI_CHECK_RESULT(MPI_Allgather, (&oasize, 1, MPI_INTEGER, - oasizes.data(), 1, MPI_INTEGER, + c_data(oasizes), 1, MPI_INTEGER, MPI_Comm(comm))); // Gather the archives, which can be of different sizes, so // we need to use allgatherv. @@ -68,7 +69,7 @@ all_gather_impl(const communicator& comm, const T* in_values, int n, packed_iarchive::buffer_type recv_buffer(std::accumulate(oasizes.begin(), oasizes.end(), 0)); BOOST_MPI_CHECK_RESULT(MPI_Allgatherv, (const_cast(oa.address()), int(oa.size()), MPI_BYTE, - recv_buffer.data(), oasizes.data(), offsets.data(), MPI_BYTE, + c_data(recv_buffer), c_data(oasizes), c_data(offsets), MPI_BYTE, MPI_Comm(comm))); for (int src = 0; src < nproc; ++src) { int nb = sizes ? sizes[src] : n; @@ -110,8 +111,9 @@ template void all_gather(const communicator& comm, const T& in_value, std::vector& out_values) { + using detail::c_data; out_values.resize(comm.size()); - ::boost::mpi::all_gather(comm, in_value, out_values.data()); + ::boost::mpi::all_gather(comm, in_value, c_data(out_values)); } template @@ -125,8 +127,9 @@ template void all_gather(const communicator& comm, const T* in_values, int n, std::vector& out_values) { + using detail::c_data; out_values.resize(comm.size() * n); - ::boost::mpi::all_gather(comm, in_values, n, out_values.data()); + ::boost::mpi::all_gather(comm, in_values, n, c_data(out_values)); } } } // end namespace boost::mpi diff --git a/include/boost/mpi/collectives/all_gatherv.hpp b/include/boost/mpi/collectives/all_gatherv.hpp index d62cc7b..064412f 100644 --- a/include/boost/mpi/collectives/all_gatherv.hpp +++ b/include/boost/mpi/collectives/all_gatherv.hpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -68,9 +69,10 @@ void all_gatherv(const communicator& comm, const T& in_value, T* out_values, const std::vector& sizes) { + using detail::c_data; assert(sizes.size() == comm.size()); assert(sizes[comm.rank()] == 1); - detail::all_gatherv_impl(comm, &in_value, out_values, sizes.data(), 0, is_mpi_datatype()); + detail::all_gatherv_impl(comm, &in_value, out_values, c_data(sizes), 0, is_mpi_datatype()); } template @@ -78,8 +80,9 @@ void all_gatherv(const communicator& comm, const T* in_values, T* out_values, const std::vector& sizes) { + using detail::c_data; assert(int(sizes.size()) == comm.size()); - detail::all_gatherv_impl(comm, in_values, out_values, sizes.data(), 0, is_mpi_datatype()); + detail::all_gatherv_impl(comm, in_values, out_values, c_data(sizes), 0, is_mpi_datatype()); } template @@ -87,10 +90,11 @@ void all_gatherv(const communicator& comm, std::vector const& in_values, std::vector& out_values, const std::vector& sizes) { + using detail::c_data; assert(int(sizes.size()) == comm.size()); assert(int(in_values.size()) == sizes[comm.rank()]); out_values.resize(std::accumulate(sizes.begin(), sizes.end(), 0)); - ::boost::mpi::all_gatherv(comm, in_values.data(), out_values.data(), sizes); + ::boost::mpi::all_gatherv(comm, c_data(in_values), c_data(out_values), sizes); } @@ -99,10 +103,11 @@ void all_gatherv(const communicator& comm, const T& in_value, T* out_values, const std::vector& sizes, const std::vector& displs) { + using detail::c_data; assert(sizes.size() == comm.size()); assert(displs.size() == comm.size()); detail::all_gatherv_impl(comm, &in_value, 1, out_values, - sizes.data(), displs.data(), is_mpi_datatype()); + c_data(sizes), c_data(displs), is_mpi_datatype()); } template @@ -110,10 +115,11 @@ void all_gatherv(const communicator& comm, const T* in_values, T* out_values, const std::vector& sizes, const std::vector& displs) { + using detail::c_data; assert(sizes.size() == comm.size()); assert(displs.size() == comm.size()); detail::all_gatherv_impl(comm, in_values, out_values, - sizes.data(), displs.data(), is_mpi_datatype()); + c_data(sizes), c_data(displs), is_mpi_datatype()); } template @@ -121,11 +127,12 @@ void all_gatherv(const communicator& comm, std::vector const& in_values, std::vector& out_values, const std::vector& sizes, const std::vector& displs) { + using detail::c_data; assert(sizes.size() == comm.size()); assert(displs.size() == comm.size()); assert(in_values.size() == sizes[comm.rank()]); out_values.resize(std::accumulate(sizes.begin(), sizes.end(), 0)); - ::boost::mpi::all_gatherv(comm, in_values.data(), out_values.data(), sizes, displs); + ::boost::mpi::all_gatherv(comm, c_data(in_values), c_data(out_values), sizes, displs); } } } // end namespace boost::mpi diff --git a/include/boost/mpi/collectives/gather.hpp b/include/boost/mpi/collectives/gather.hpp index 6ebfda5..386bfdd 100644 --- a/include/boost/mpi/collectives/gather.hpp +++ b/include/boost/mpi/collectives/gather.hpp @@ -20,6 +20,7 @@ #include #include #include +#include #include namespace boost { namespace mpi { @@ -69,7 +70,7 @@ gather_impl(const communicator& comm, const T* in_values, int n, T* out_values, int oasize = oa.size(); BOOST_MPI_CHECK_RESULT(MPI_Gather, (&oasize, 1, MPI_INTEGER, - oasizes.data(), 1, MPI_INTEGER, + c_data(oasizes), 1, MPI_INTEGER, root, MPI_Comm(comm))); // Gather the archives, which can be of different sizes, so // we need to use gatherv. @@ -80,7 +81,7 @@ gather_impl(const communicator& comm, const T* in_values, int n, T* out_values, packed_iarchive::buffer_type recv_buffer(is_root ? std::accumulate(oasizes.begin(), oasizes.end(), 0) : 0); BOOST_MPI_CHECK_RESULT(MPI_Gatherv, (const_cast(oa.address()), int(oa.size()), MPI_BYTE, - recv_buffer.data(), oasizes.data(), offsets.data(), MPI_BYTE, + c_data(recv_buffer), c_data(oasizes), c_data(offsets), MPI_BYTE, root, MPI_Comm(comm))); if (is_root) { for (int src = 0; src < nproc; ++src) { @@ -135,10 +136,11 @@ void gather(const communicator& comm, const T& in_value, std::vector& out_values, int root) { + using detail::c_data; if (comm.rank() == root) { out_values.resize(comm.size()); } - ::boost::mpi::gather(comm, in_value, out_values.data(), root); + ::boost::mpi::gather(comm, in_value, c_data(out_values), root); } template diff --git a/include/boost/mpi/collectives/scatter.hpp b/include/boost/mpi/collectives/scatter.hpp index 2645612..0c91b1e 100644 --- a/include/boost/mpi/collectives/scatter.hpp +++ b/include/boost/mpi/collectives/scatter.hpp @@ -17,6 +17,7 @@ #include #include #include +#include #include namespace boost { namespace mpi { @@ -83,8 +84,8 @@ fill_scatter_sendbuf(const communicator& comm, T const* values, template T* non_const_data(std::vector const& v) { - T const* cptr = v.data(); - return const_cast(cptr); + using detail::c_data; + return const_cast(c_data(v)); } // Dispatch the sendbuf among proc. @@ -109,8 +110,8 @@ dispatch_scatter_sendbuf(const communicator& comm, packed_iarchive::buffer_type recvbuf; recvbuf.resize(myarchsize); BOOST_MPI_CHECK_RESULT(MPI_Scatterv, - (non_const_data(sendbuf), non_const_data(archsizes), offsets.data(), MPI_BYTE, - recvbuf.data(), recvbuf.size(), MPI_BYTE, + (non_const_data(sendbuf), non_const_data(archsizes), c_data(offsets), MPI_BYTE, + c_data(recvbuf), recvbuf.size(), MPI_BYTE, root, MPI_Comm(comm))); // Unserialize if ( in_values != 0 && root == comm.rank()) { @@ -137,7 +138,7 @@ scatter_impl(const communicator& comm, const T* in_values, T* out_values, if (root == comm.rank()) { std::vector nslots(comm.size(), n); - fill_scatter_sendbuf(comm, in_values, nslots.data(), (int const*)0, sendbuf, archsizes); + fill_scatter_sendbuf(comm, in_values, c_data(nslots), (int const*)0, sendbuf, archsizes); } dispatch_scatter_sendbuf(comm, sendbuf, archsizes, in_values, out_values, n, root); } @@ -163,7 +164,8 @@ void scatter(const communicator& comm, const std::vector& in_values, T& out_value, int root) { - ::boost::mpi::scatter(comm, in_values.data(), out_value, root); + using detail::c_data; + ::boost::mpi::scatter(comm, c_data(in_values), out_value, root); } template diff --git a/include/boost/mpi/collectives/scatterv.hpp b/include/boost/mpi/collectives/scatterv.hpp index eb9f41d..57e073c 100644 --- a/include/boost/mpi/collectives/scatterv.hpp +++ b/include/boost/mpi/collectives/scatterv.hpp @@ -11,6 +11,7 @@ #include #include #include +#include namespace boost { namespace mpi { @@ -71,8 +72,8 @@ scatterv_impl(const communicator& comm, const T* in_values, T* out_values, int o std::vector skipped; if (displs) { skipped.resize(nproc); - offsets2skipped(sizes, displs, skipped.data(), nproc); - displs = skipped.data(); + offsets2skipped(sizes, displs, c_data(skipped), nproc); + displs = c_data(skipped); } fill_scatter_sendbuf(comm, in_values, sizes, (int const*)0, sendbuf, archsizes); } @@ -99,7 +100,8 @@ scatterv(const communicator& comm, const T* in_values, const std::vector& sizes, const std::vector& displs, T* out_values, int out_size, int root) { - scatterv_impl(comm, in_values, out_values, out_size, sizes.data(), displs.data(), + using detail::c_data; + scatterv_impl(comm, in_values, out_values, out_size, c_data(sizes), c_data(displs), root, is_mpi_datatype()); } @@ -109,7 +111,8 @@ scatterv(const communicator& comm, const std::vector& in_values, const std::vector& sizes, const std::vector& displs, T* out_values, int out_size, int root) { - ::boost::mpi::scatterv(comm, in_values.data(), sizes, displs, + using detail::c_data; + ::boost::mpi::scatterv(comm, c_data(in_values), sizes, displs, out_values, out_size, root); } @@ -128,8 +131,9 @@ void scatterv(const communicator& comm, const T* in_values, const std::vector& sizes, T* out_values, int root) { + using detail::c_data; detail::scatterv_impl(comm, in_values, out_values, sizes[comm.rank()], - sizes.data(), (int const*)0, + c_data(sizes), (int const*)0, root, is_mpi_datatype()); } diff --git a/include/boost/mpi/detail/antiques.hpp b/include/boost/mpi/detail/antiques.hpp new file mode 100644 index 0000000..1913188 --- /dev/null +++ b/include/boost/mpi/detail/antiques.hpp @@ -0,0 +1,31 @@ +// Copyright Alain Miniussi 2014. +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +// Authors: Alain Miniussi + +#ifndef BOOST_MPI_ANTIQUES_HPP +#define BOOST_MPI_ANTIQUES_HPP + +#include + +// Support for some obsolette compilers + +namespace boost { namespace mpi { +namespace detail { + // Some old gnu compiler have no support for vector<>::data + // Use this in the mean time, the cumbersome syntax should + // serve as an incentive to get rid of this when those compilers + // are dropped. + template + T* c_data(std::vector& v) { return &(v[0]); } + + template + T const* c_data(std::vector const& v) { return &(v[0]); } + +} } } + +#endif + + diff --git a/src/cartesian_communicator.cpp b/src/cartesian_communicator.cpp index ad3178c..a46f0bc 100644 --- a/src/cartesian_communicator.cpp +++ b/src/cartesian_communicator.cpp @@ -13,6 +13,11 @@ namespace boost { namespace mpi { +namespace { + template + T* c_data(std::vector& v) { return &(v[0]); } +} + std::ostream& operator<<(std::ostream& out, cartesian_dimension const& d) { out << '(' << d.size << ','; @@ -57,8 +62,8 @@ cartesian_communicator::cartesian_communicator(const communicator& comm, } MPI_Comm newcomm; BOOST_MPI_CHECK_RESULT(MPI_Cart_create, - ((MPI_Comm)comm, dims.size(), - dims.data(), periodic.data(), + ((MPI_Comm)comm, dims.size(), + c_data(dims), c_data(periodic), int(reorder), &newcomm)); if(newcomm != MPI_COMM_NULL) { comm_ptr.reset(new MPI_Comm(newcomm), comm_free()); @@ -80,7 +85,7 @@ cartesian_communicator::cartesian_communicator(const cartesian_communicator& com MPI_Comm newcomm; BOOST_MPI_CHECK_RESULT(MPI_Cart_sub, - ((MPI_Comm)comm, bitset.data(), &newcomm)); + ((MPI_Comm)comm, c_data(bitset), &newcomm)); if(newcomm != MPI_COMM_NULL) { comm_ptr.reset(new MPI_Comm(newcomm), comm_free()); } @@ -99,7 +104,7 @@ cartesian_communicator::rank(const std::vector& coords ) const { int r = -1; assert(int(coords.size()) == ndims()); BOOST_MPI_CHECK_RESULT(MPI_Cart_rank, - (MPI_Comm(*this), const_cast&>(coords).data(), + (MPI_Comm(*this), c_data(const_cast&>(coords)), &r)); return r; } @@ -117,7 +122,7 @@ std::vector cartesian_communicator::coordinates(int rk) const { std::vector cbuf(ndims()); BOOST_MPI_CHECK_RESULT(MPI_Cart_coords, - (MPI_Comm(*this), rk, cbuf.size(), cbuf.data() )); + (MPI_Comm(*this), rk, cbuf.size(), c_data(cbuf) )); return cbuf; } @@ -130,7 +135,7 @@ cartesian_communicator::topology( cartesian_topology& topo, std::vector cdims(ndims); std::vector cperiods(ndims); BOOST_MPI_CHECK_RESULT(MPI_Cart_get, - (MPI_Comm(*this), ndims, cdims.data(), cperiods.data(), coords.data())); + (MPI_Comm(*this), ndims, c_data(cdims), c_data(cperiods), c_data(coords))); cartesian_topology res(cdims.begin(), cperiods.begin(), ndims); topo.swap(res); } @@ -167,7 +172,7 @@ cartesian_dimensions(int sz, std::vector& dims) { int leftover = sz % min; BOOST_MPI_CHECK_RESULT(MPI_Dims_create, - (sz-leftover, dims.size(), dims.data())); + (sz-leftover, dims.size(), c_data(dims))); return dims; } diff --git a/src/offsets.cpp b/src/offsets.cpp index 8080501..2382d45 100644 --- a/src/offsets.cpp +++ b/src/offsets.cpp @@ -6,6 +6,7 @@ // Authors: Alain Miniussi #include +#include namespace boost { namespace mpi { namespace detail { @@ -28,7 +29,7 @@ sizes2offsets(std::vector const& sizes, std::vector& offsets) { int sz = sizes.size(); offsets.resize(sz); - sizes2offsets(sizes.data(), offsets.data(), sz); + sizes2offsets(c_data(sizes), c_data(offsets), sz); } // Given a sequence of sizes (typically the number of records dispatched diff --git a/test/cartesian_topology_test.cpp b/test/cartesian_topology_test.cpp index 8a21523..f19d72a 100644 --- a/test/cartesian_topology_test.cpp +++ b/test/cartesian_topology_test.cpp @@ -49,8 +49,8 @@ void test_coordinates_consistency( mpi::cartesian_communicator const& cc, for(int p = 0; p < cc.size(); ++p) { std::vector min(cc.ndims()); std::vector local(cc.coordinates(p)); - mpi::reduce(cc, local.data(), local.size(), - min.data(), mpi::minimum(), p); + mpi::reduce(cc, &local.front(), local.size(), + &(min[0]), mpi::minimum(), p); cc.barrier(); if (p == cc.rank()) { BOOST_CHECK(std::equal(coords.begin(), coords.end(), min.begin())); diff --git a/test/scatter_test.cpp b/test/scatter_test.cpp index 011a494..5e5ecd2 100644 --- a/test/scatter_test.cpp +++ b/test/scatter_test.cpp @@ -141,9 +141,9 @@ scatterv_test(const communicator& comm, Generator generator, std::cout << "Scatteringv " << kind << " from root " << root << "..." << std::endl; assert(mysize == sizes[comm.rank()]); - scatterv(comm, values, sizes, myvalues.data(), root); + scatterv(comm, values, sizes, &(myvalues[0]), root); } else { - scatterv(comm, myvalues.data(), mysize, root); + scatterv(comm, &(myvalues[0]), mysize, root); } for (int i = 0; i < mysize; ++i) From b414ada8439511587cf8fe795f8f60f5df7482db Mon Sep 17 00:00:00 2001 From: Andrey Semashev Date: Thu, 20 Apr 2017 23:52:13 +0300 Subject: [PATCH 98/99] Fix mpi-python component compilation. Made mpi-python compilation and naming consistent with Boost.Python: - Add property requirement to the components dependent on Python. This ensures that Boost.Build passes include and library paths for the selected Python version while building these components. This fixes the build error of pyconfig.h not found. - Add property handling, to follow Boost.Python practice. This should ensure that correct Python headers and library are used by Boost.Python depending on this property. - Add suffix 3 to the library name when compiled against Python 3.x. This follows Boost.Python practice. The MPI plugin for Python is still named mpi (without suffix) as it may be visible to python programs. Users are expected to build into separate directories when plugins for different Python versions are required. --- build/Jamfile.v2 | 127 ++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 79 insertions(+), 48 deletions(-) diff --git a/build/Jamfile.v2 b/build/Jamfile.v2 index 5a1a54f..827e1d2 100644 --- a/build/Jamfile.v2 +++ b/build/Jamfile.v2 @@ -21,8 +21,8 @@ project boost/mpi : source-location ../src ; -lib boost_mpi - : +lib boost_mpi + : broadcast.cpp cartesian_communicator.cpp communicator.cpp @@ -55,55 +55,86 @@ lib boost_mpi ../../serialization/build//boost_serialization /mpi//mpi [ mpi.extra-requirements ] ; - -libraries += boost_mpi ; + +libraries += boost_mpi ; if [ python.configured ] { - lib boost_mpi_python - : # Sources - python/serialize.cpp - : # Requirements - boost_mpi - /mpi//mpi [ mpi.extra-requirements ] - /boost/python//boost_python - shared:BOOST_MPI_DYN_LINK=1 - shared:BOOST_MPI_PYTHON_DYN_LINK=1 - shared:BOOST_PYTHON_DYN_LINK=1 - BOOST_MPI_PYTHON_SOURCE=1 - -@$(BOOST_JAMROOT_MODULE)%$(BOOST_JAMROOT_MODULE).tag - @$(BOOST_JAMROOT_MODULE)%$(BOOST_JAMROOT_MODULE).python-tag - : # Default build - shared - : # Usage requirements - /mpi//mpi [ mpi.extra-requirements ] - ; - libraries += boost_mpi_python ; - - python-extension mpi - : # Sources - python/collectives.cpp - python/py_communicator.cpp - python/datatypes.cpp - python/documentation.cpp - python/py_environment.cpp - python/py_nonblocking.cpp - python/py_exception.cpp - python/module.cpp - python/py_request.cpp - python/skeleton_and_content.cpp - python/status.cpp - python/py_timer.cpp - : # Requirements - /boost/python//boost_python - boost_mpi_python - boost_mpi - /mpi//mpi [ mpi.extra-requirements ] - shared:BOOST_MPI_DYN_LINK=1 - shared:BOOST_MPI_PYTHON_DYN_LINK=1 - shared:BOOST_PYTHON_DYN_LINK=1 - shared shared - ; + py2-version = [ py-version 2 ] ; + py3-version = [ py-version 3 ] ; + + # These library names are synchronized with those defined by Boost.Python, see libs/python/build/Jamfile. + lib_boost_python(2) = boost_python ; + lib_boost_python(3) = boost_python3 ; + + lib_boost_python($(py2-version)) = $(lib_boost_python(2)) ; + lib_boost_python($(py3-version)) = $(lib_boost_python(3)) ; + + lib_boost_mpi_python(2) = boost_mpi_python ; + lib_boost_mpi_python(3) = boost_mpi_python3 ; + + lib_boost_mpi_python($(py2-version)) = $(lib_boost_mpi_python(2)) ; + lib_boost_mpi_python($(py3-version)) = $(lib_boost_mpi_python(3)) ; + + for local N in 2 3 + { + if $(py$(N)-version) + { + lib $(lib_boost_mpi_python($(py$(N)-version))) + : # Sources + python/serialize.cpp + : # Requirements + boost_mpi + /mpi//mpi [ mpi.extra-requirements ] + /boost/python//$(lib_boost_python($(py$(N)-version))) + shared:BOOST_MPI_DYN_LINK=1 + shared:BOOST_MPI_PYTHON_DYN_LINK=1 + shared:BOOST_PYTHON_DYN_LINK=1 + BOOST_MPI_PYTHON_SOURCE=1 + -@$(BOOST_JAMROOT_MODULE)%$(BOOST_JAMROOT_MODULE).tag + @$(BOOST_JAMROOT_MODULE)%$(BOOST_JAMROOT_MODULE).python-tag + on:BOOST_DEBUG_PYTHON + $(py$(N)-version) + : # Default build + shared + : # Usage requirements + /mpi//mpi [ mpi.extra-requirements ] + ; + + python-extension mpi + : # Sources + python/collectives.cpp + python/py_communicator.cpp + python/datatypes.cpp + python/documentation.cpp + python/py_environment.cpp + python/py_nonblocking.cpp + python/py_exception.cpp + python/module.cpp + python/py_request.cpp + python/skeleton_and_content.cpp + python/status.cpp + python/py_timer.cpp + : # Requirements + /boost/python//$(lib_boost_python($(py$(N)-version))) + $(lib_boost_mpi_python($(py$(N)-version))) + boost_mpi + /mpi//mpi [ mpi.extra-requirements ] + shared:BOOST_MPI_DYN_LINK=1 + shared:BOOST_MPI_PYTHON_DYN_LINK=1 + shared:BOOST_PYTHON_DYN_LINK=1 + shared shared + on:BOOST_DEBUG_PYTHON + $(py$(N)-version) + ; + + libraries += $(lib_boost_mpi_python($(py$(N)-version))) ; + } + else + { + alias $(lib_boost_mpi_python($(N))) ; + } + } } } else if ! ( --without-mpi in [ modules.peek : ARGV ] ) From 2c5ec6ef15eb4fd859f561b2fd208d2f5aeb09da Mon Sep 17 00:00:00 2001 From: Alain Miniussi Date: Mon, 24 Jul 2017 12:39:50 +0200 Subject: [PATCH 99/99] Add const conversion to accomodate open MPI 1.6 --- include/boost/mpi/detail/antiques.hpp | 13 ++++++++++++- src/point_to_point.cpp | 13 +++++++------ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/include/boost/mpi/detail/antiques.hpp b/include/boost/mpi/detail/antiques.hpp index 409905f..0bd235b 100644 --- a/include/boost/mpi/detail/antiques.hpp +++ b/include/boost/mpi/detail/antiques.hpp @@ -23,7 +23,18 @@ namespace detail { template T const* c_data(std::vector const& v) { return &(v[0]); } - + + // Some old MPI implementation (OpenMPI 1.6 for example) have non + // conforming API w.r.t. constness. + // We choose to fix this trhough this converter in order to + // explain/remember why we're doing this and remove it easilly + // when support for those MPI is dropped. + // The fix is as specific (un templatized, for one) as possible + // in order to encourage it usage for the probleme at hand. + // Problematic API include MPI_Send + inline + void *unconst(void const* addr) { return const_cast(addr); } + } } } #endif diff --git a/src/point_to_point.cpp b/src/point_to_point.cpp index c14b934..7b353f7 100644 --- a/src/point_to_point.cpp +++ b/src/point_to_point.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include namespace boost { namespace mpi { namespace detail { @@ -30,11 +31,11 @@ packed_archive_send(MPI_Comm comm, int dest, int tag, { std::size_t const& size = ar.size(); BOOST_MPI_CHECK_RESULT(MPI_Send, - (&size, 1, + (detail::unconst(&size), 1, get_mpi_datatype(size), dest, tag, comm)); BOOST_MPI_CHECK_RESULT(MPI_Send, - (const_cast(ar.address()), size, + (detail::unconst(ar.address()), size, MPI_PACKED, dest, tag, comm)); } @@ -47,11 +48,11 @@ packed_archive_isend(MPI_Comm comm, int dest, int tag, assert(num_out_requests >= 2); std::size_t const& size = ar.size(); BOOST_MPI_CHECK_RESULT(MPI_Isend, - (&size, 1, + (detail::unconst(&size), 1, get_mpi_datatype(size), dest, tag, comm, out_requests)); BOOST_MPI_CHECK_RESULT(MPI_Isend, - (const_cast(ar.address()), size, + (detail::unconst(ar.address()), size, MPI_PACKED, dest, tag, comm, out_requests + 1)); @@ -67,11 +68,11 @@ packed_archive_isend(MPI_Comm comm, int dest, int tag, std::size_t const& size = ar.size(); BOOST_MPI_CHECK_RESULT(MPI_Isend, - (&size, 1, + (detail::unconst(&size), 1, get_mpi_datatype(size), dest, tag, comm, out_requests)); BOOST_MPI_CHECK_RESULT(MPI_Isend, - (const_cast(ar.address()), size, + (detail::unconst(ar.address()), size, MPI_PACKED, dest, tag, comm, out_requests + 1));