diff --git a/build/Jamfile.v2 b/build/Jamfile.v2 index 58e7fcf..b1cd2c4 100644 --- a/build/Jamfile.v2 +++ b/build/Jamfile.v2 @@ -21,9 +21,10 @@ project boost/mpi : source-location ../src ; -lib boost_mpi - : +lib boost_mpi + : broadcast.cpp + cartesian_communicator.cpp communicator.cpp computation_tree.cpp content_oarchive.cpp @@ -42,6 +43,7 @@ lib boost_mpi request.cpp text_skeleton_oarchive.cpp timer.cpp + offsets.cpp : # Requirements ../../serialization/build//boost_serialization /mpi//mpi [ mpi.extra-requirements ] @@ -53,55 +55,86 @@ lib boost_mpi ../../serialization/build//boost_serialization /mpi//mpi [ mpi.extra-requirements ] ; - -libraries += boost_mpi ; + +libraries += boost_mpi ; if [ python.configured ] { - lib boost_mpi_python - : # Sources - python/serialize.cpp - : # Requirements - boost_mpi - /mpi//mpi [ mpi.extra-requirements ] - /boost/python//boost_python - shared:BOOST_MPI_DYN_LINK=1 - shared:BOOST_MPI_PYTHON_DYN_LINK=1 - shared:BOOST_PYTHON_DYN_LINK=1 - BOOST_MPI_PYTHON_SOURCE=1 - -@$(BOOST_JAMROOT_MODULE)%$(BOOST_JAMROOT_MODULE).tag - @$(BOOST_JAMROOT_MODULE)%$(BOOST_JAMROOT_MODULE).python-tag - : # Default build - shared - : # Usage requirements - /mpi//mpi [ mpi.extra-requirements ] - ; - libraries += boost_mpi_python ; - - python-extension mpi - : # Sources - python/collectives.cpp - python/py_communicator.cpp - python/datatypes.cpp - python/documentation.cpp - python/py_environment.cpp - python/py_nonblocking.cpp - python/py_exception.cpp - python/module.cpp - python/py_request.cpp - python/skeleton_and_content.cpp - python/status.cpp - python/py_timer.cpp - : # Requirements - /boost/python//boost_python - boost_mpi_python - boost_mpi - /mpi//mpi [ mpi.extra-requirements ] - shared:BOOST_MPI_DYN_LINK=1 - shared:BOOST_MPI_PYTHON_DYN_LINK=1 - shared:BOOST_PYTHON_DYN_LINK=1 - shared shared - ; + py2-version = [ py-version 2 ] ; + py3-version = [ py-version 3 ] ; + + # These library names are synchronized with those defined by Boost.Python, see libs/python/build/Jamfile. + lib_boost_python(2) = boost_python ; + lib_boost_python(3) = boost_python3 ; + + lib_boost_python($(py2-version)) = $(lib_boost_python(2)) ; + lib_boost_python($(py3-version)) = $(lib_boost_python(3)) ; + + lib_boost_mpi_python(2) = boost_mpi_python ; + lib_boost_mpi_python(3) = boost_mpi_python3 ; + + lib_boost_mpi_python($(py2-version)) = $(lib_boost_mpi_python(2)) ; + lib_boost_mpi_python($(py3-version)) = $(lib_boost_mpi_python(3)) ; + + for local N in 2 3 + { + if $(py$(N)-version) + { + lib $(lib_boost_mpi_python($(py$(N)-version))) + : # Sources + python/serialize.cpp + : # Requirements + boost_mpi + /mpi//mpi [ mpi.extra-requirements ] + /boost/python//$(lib_boost_python($(py$(N)-version))) + shared:BOOST_MPI_DYN_LINK=1 + shared:BOOST_MPI_PYTHON_DYN_LINK=1 + shared:BOOST_PYTHON_DYN_LINK=1 + BOOST_MPI_PYTHON_SOURCE=1 + -@$(BOOST_JAMROOT_MODULE)%$(BOOST_JAMROOT_MODULE).tag + @$(BOOST_JAMROOT_MODULE)%$(BOOST_JAMROOT_MODULE).python-tag + on:BOOST_DEBUG_PYTHON + $(py$(N)-version) + : # Default build + shared + : # Usage requirements + /mpi//mpi [ mpi.extra-requirements ] + ; + + python-extension mpi + : # Sources + python/collectives.cpp + python/py_communicator.cpp + python/datatypes.cpp + python/documentation.cpp + python/py_environment.cpp + python/py_nonblocking.cpp + python/py_exception.cpp + python/module.cpp + python/py_request.cpp + python/skeleton_and_content.cpp + python/status.cpp + python/py_timer.cpp + : # Requirements + /boost/python//$(lib_boost_python($(py$(N)-version))) + $(lib_boost_mpi_python($(py$(N)-version))) + boost_mpi + /mpi//mpi [ mpi.extra-requirements ] + shared:BOOST_MPI_DYN_LINK=1 + shared:BOOST_MPI_PYTHON_DYN_LINK=1 + shared:BOOST_PYTHON_DYN_LINK=1 + shared shared + on:BOOST_DEBUG_PYTHON + $(py$(N)-version) + ; + + libraries += $(lib_boost_mpi_python($(py$(N)-version))) ; + } + else + { + alias $(lib_boost_mpi_python($(N))) ; + } + } } } else if ! ( --without-mpi in [ modules.peek : ARGV ] ) diff --git a/doc/Jamfile.v2 b/doc/Jamfile.v2 index a8226f3..462fbe3 100644 --- a/doc/Jamfile.v2 +++ b/doc/Jamfile.v2 @@ -12,6 +12,7 @@ doxygen mpi_autodoc : [ glob ../../../boost/mpi.hpp ../../../boost/mpi/allocator.hpp + ../../../boost/mpi/cartesian_communicator.hpp ../../../boost/mpi/collectives.hpp ../../../boost/mpi/collectives_fwd.hpp ../../../boost/mpi/communicator.hpp diff --git a/doc/mpi.qbk b/doc/mpi.qbk index 2e0833a..aa8e00a 100644 --- a/doc/mpi.qbk +++ b/doc/mpi.qbk @@ -765,7 +765,7 @@ The [funcref boost::mpi::gather `gather`] collective gathers the values produced by every process in a communicator into a vector of values on the "root" process (specified by an argument to `gather`). The /i/th element in the vector will correspond to the -value gathered fro mthe /i/th process. For instance, in the following +value gathered from the /i/th process. For instance, in the following program each process computes its own random number. All of these random numbers are gathered at process 0 (the "root" in this case), which prints out the values that correspond to each processor. @@ -821,6 +821,62 @@ which is semantically equivalent to calling `gather` followed by a [endsect] +[section:scatter Scatter] +The [funcref boost::mpi::scatter `scatter`] collective scatters +the values from a vector in the "root" process in a communicator into +values in all the processes of the communicator. + The /i/th element in the vector will correspond to the +value received by the /i/th process. For instance, in the following +program, the root process produces a vector of random nomber and send +one value to each process that will print it. (`random_scatter.cpp`) + + #include + #include + #include + #include + #include + + namespace mpi = boost::mpi; + + int main(int argc, char* argv[]) + { + mpi::environment env(argc, argv); + mpi::communicator world; + + std::srand(time(0) + world.rank()); + std::vector all; + int mine = -1; + if (world.rank() == 0) { + all.resize(world.size()); + std::generate(all.begin(), all.end(), std::rand); + } + mpi::scatter(world, all, mine, 0); + for (int r = 0; r < world.size(); ++r) { + world.barrier(); + if (r == world.rank()) { + std::cout << "Rank " << r << " got " << mine << '\n'; + } + } + return 0; + } + +Executing this program with seven processes will result in output such +as the following. Although the random values will change from one run +to the next, the order of the processes in the output will remain the +same because of the barrier. + +[pre +Rank 0 got 1409381269 +Rank 1 got 17045268 +Rank 2 got 440120016 +Rank 3 got 936998224 +Rank 4 got 1827129182 +Rank 5 got 1951746047 +Rank 6 got 2117359639 +] + +[endsect] + [section:reduce Reduce] The [funcref boost::mpi::reduce `reduce`] collective @@ -1097,6 +1153,42 @@ difference (`-`) of two groups, generate arbitrary subgroups, etc. [endsect] +[section:cartesian_communicator Cartesian communicator] + +A communicator can be organised as a cartesian grid, here a basic example: + + #include + #include + + #include + #include + #include + #include + + #include + + namespace mpi = boost::mpi; + int test_main(int argc, char* argv[]) + { + mpi::environment env; + mpi::communicator world; + + if (world.size() != 24) return -1; + mpi::cartesian_dimension dims[] = {{2, true}, {3,true}, {4,true}}; + mpi::cartesian_communicator cart(world, mpi::cartesian_topology(dims)); + for (int r = 0; r < cart.size(); ++r) { + cart.barrier(); + if (r == cart.rank()) { + std::vector c = cart.coordinates(r); + std::cout << "rk :" << r << " coords: " + << c[0] << ' ' << c[1] << ' ' << c[2] << '\n'; + } + } + return 0; + } + + + [section:skeleton_and_content Separating structure from content] When communicating data types over MPI that are not fundamental to MPI @@ -1733,9 +1825,10 @@ algorithms. [[`MPI_CART`] [unnecessary; use [memberref boost::mpi::communicator::has_cartesian_topology `communicator::has_cartesian_topology`]]] [[[@http://www.mpi-forum.org/docs/mpi-1.1/mpi-11-html/node133.html#Node133 - `MPI_Cart_create`]] [unsupported]] + `MPI_Cart_create`]] [[classref boost::mpi::cartesian_communicator `cartesian_communicator`] + constructor]] [[[@http://www.mpi-forum.org/docs/mpi-1.1/mpi-11-html/node134.html#Node134 - `MPI_Dims_create`]] [unsupported]] + `MPI_Dims_create`]] [[funcref boost::mpi::cartesian_dimensions `cartesian_dimensions`]]] [[[@http://www.mpi-forum.org/docs/mpi-1.1/mpi-11-html/node135.html#Node135 `MPI_Graph_create`]] [[classref boost::mpi::graph_communicator @@ -1753,13 +1846,13 @@ algorithms. `MPI_Graph_get`]] [[funcref boost::mpi::vertices `vertices`], [funcref boost::mpi::edges `edges`]]] [[[@http://www.mpi-forum.org/docs/mpi-1.1/mpi-11-html/node136.html#Node136 - `MPI_Cartdim_get`]] [unsupported]] + `MPI_Cartdim_get`]] [[memberref boost::mpi::cartesian_communicator::ndims `cartesian_communicator::ndims` ]]] [[[@http://www.mpi-forum.org/docs/mpi-1.1/mpi-11-html/node136.html#Node136 - `MPI_Cart_get`]] [unsupported]] + `MPI_Cart_get`]] [[memberref boost::mpi::cartesian_communicator::topology `cartesian_communicator::topology` ]]] [[[@http://www.mpi-forum.org/docs/mpi-1.1/mpi-11-html/node136.html#Node136 - `MPI_Cart_rank`]] [unsupported]] + `MPI_Cart_rank`]] [[memberref boost::mpi::cartesian_communicator::rank `cartesian_communicator::rank` ]]] [[[@http://www.mpi-forum.org/docs/mpi-1.1/mpi-11-html/node136.html#Node136 - `MPI_Cart_coords`]] [unsupported]] + `MPI_Cart_coords`]] [[memberref boost::mpi::cartesian_communicator::coordinates `cartesian_communicator::coordinates` ]]] [[[@http://www.mpi-forum.org/docs/mpi-1.1/mpi-11-html/node136.html#Node136 `MPI_Graph_neighbors_count`]] [[funcref boost::mpi::out_degree `out_degree`]]] @@ -1769,7 +1862,8 @@ algorithms. [[[@http://www.mpi-forum.org/docs/mpi-1.1/mpi-11-html/node137.html#Node137 `MPI_Cart_shift`]] [unsupported]] [[[@http://www.mpi-forum.org/docs/mpi-1.1/mpi-11-html/node138.html#Node138 - `MPI_Cart_sub`]] [unsupported]] + `MPI_Cart_sub`]] [[classref boost::mpi::cartesian_communicator `cartesian_communicator`] + constructor]] [[[@http://www.mpi-forum.org/docs/mpi-1.1/mpi-11-html/node139.html#Node139 `MPI_Cart_map`]] [unsupported]] [[[@http://www.mpi-forum.org/docs/mpi-1.1/mpi-11-html/node139.html#Node139 diff --git a/example/cartesian_communicator.cpp b/example/cartesian_communicator.cpp new file mode 100644 index 0000000..e06f150 --- /dev/null +++ b/example/cartesian_communicator.cpp @@ -0,0 +1,44 @@ +// Copyright Alain Miniussi 2014. +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +// Authors: Alain Miniussi + +#include +#include + +#include +#include +#include +#include + +#include + +namespace mpi = boost::mpi; +// Curly brace init make this useless, but +// - Need to support obsolete like g++ 4.3.x. for some reason +// - Can't conditionnaly compile with bjam (unless you find +// the doc, and read it, which would only make sense if you +// actually wan't to use bjam, which does not (make sense)) +typedef mpi::cartesian_dimension cd; + +int test_main(int argc, char* argv[]) +{ + mpi::environment env; + mpi::communicator world; + + if (world.size() != 24) return -1; + mpi::cartesian_dimension dims[] = {cd(2, true), cd(3,true), cd(4,true)}; + mpi::cartesian_communicator cart(world, mpi::cartesian_topology(dims)); + for (int r = 0; r < cart.size(); ++r) { + cart.barrier(); + if (r == cart.rank()) { + std::vector c = cart.coordinates(r); + std::cout << "rk :" << r << " coords: " + << c[0] << ' ' << c[1] << ' ' << c[2] << '\n'; + } + } + return 0; +} + diff --git a/example/random_gather.cpp b/example/random_gather.cpp index 5483ba7..17825f1 100644 --- a/example/random_gather.cpp +++ b/example/random_gather.cpp @@ -4,7 +4,7 @@ // License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at // http://www.boost.org/LICENSE_1_0.txt) -// An example using Boost.MPI's gather() +// An example using Boost.MPI's gather(): [main] #include #include @@ -27,6 +27,5 @@ int main(int argc, char* argv[]) } else { gather(world, my_number, 0); } - return 0; } diff --git a/example/random_scatter.cpp b/example/random_scatter.cpp new file mode 100644 index 0000000..fc8879e --- /dev/null +++ b/example/random_scatter.cpp @@ -0,0 +1,37 @@ +// Copyright (C) 2006 Douglas Gregor + +// Use, modification and distribution is subject to the Boost Software +// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +// An example using Boost.MPI's gather(): [main] + +#include +#include +#include +#include +#include + +namespace mpi = boost::mpi; + +int main(int argc, char* argv[]) +{ + mpi::environment env(argc, argv); + mpi::communicator world; + + std::srand(time(0) + world.rank()); + std::vector all; + int mine = -1; + if (world.rank() == 0) { + all.resize(world.size()); + std::generate(all.begin(), all.end(), std::rand); + } + mpi::scatter(world, all, mine, 0); + for (int r = 0; r < world.size(); ++r) { + world.barrier(); + if (r == world.rank()) { + std::cout << "Rank " << r << " got " << mine << '\n'; + } + } + return 0; +} diff --git a/include/boost/mpi/cartesian_communicator.hpp b/include/boost/mpi/cartesian_communicator.hpp new file mode 100644 index 0000000..7a31bce --- /dev/null +++ b/include/boost/mpi/cartesian_communicator.hpp @@ -0,0 +1,382 @@ +// Copyright Alain Miniussi 2014. +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +// Authors: Alain Miniussi + +/** @file cartesian_communicator.hpp + * + * This header defines facilities to support MPI communicators with + * cartesian topologies. + * If known at compiled time, the dimension of the implied grid + * can be statically enforced, through the templatized communicator + * class. Otherwise, a non template, dynamic, base class is provided. + * + */ +#ifndef BOOST_MPI_CARTESIAN_COMMUNICATOR_HPP +#define BOOST_MPI_CARTESIAN_COMMUNICATOR_HPP + +#include + +#include +#include +#include +#include +#if !defined(BOOST_NO_CXX11_HDR_INITIALIZER_LIST) +#include +#endif // BOOST_NO_CXX11_HDR_INITIALIZER_LIST + +// Headers required to implement cartesian topologies +#include +#include +#include + +namespace boost { namespace mpi { + +/** + * @brief Specify the size and periodicity of the grid in a single dimension. + * + * POD lightweight object. + */ +struct cartesian_dimension { + /** The size of the grid n this dimension. */ + int size; + /** Is the grid periodic in this dimension. */ + bool periodic; + + cartesian_dimension(int sz = 0, bool p = false) : size(sz), periodic(p) {} + +private: + friend class boost::serialization::access; + template + void serialize(Archive & ar, const unsigned int version) + { + ar & size & periodic; + } + +}; + +template <> +struct is_mpi_datatype : mpl::true_ { }; + +/** + * @brief Test if the dimensions values are identical. + */ +inline +bool +operator==(cartesian_dimension const& d1, cartesian_dimension const& d2) { + return &d1 == &d2 || (d1.size == d2.size && d1.periodic == d2.periodic); +} + +/** + * @brief Test if the dimension values are different. + */ +inline +bool +operator!=(cartesian_dimension const& d1, cartesian_dimension const& d2) { + return !(d1 == d2); +} + +/** + * @brief Pretty printing of a cartesian dimension (size, periodic) + */ +std::ostream& operator<<(std::ostream& out, cartesian_dimension const& d); + +/** + * @brief Describe the topology of a cartesian grid. + * + * Behave mostly like a sequence of @c cartesian_dimension with the notable + * exception that its size is fixed. + * This is a lightweight object, so that any constructor that could be considered + * missing could be replaced with a function (move constructor provided when supported). + */ +class BOOST_MPI_DECL cartesian_topology + : private std::vector { + friend class cartesian_communicator; + typedef std::vector super; + public: + /** + * Retrieve a specific dimension. + */ + using super::operator[]; + /** + * @brief Topology dimentionality. + */ + using super::size; + using super::begin; + using super::end; + using super::swap; + +#if !defined(BOOST_NO_CXX11_DELETED_FUNCTIONS) + cartesian_topology() = delete; +#endif +#if !defined(BOOST_NO_CXX11_DEFAULTED_FUNCTIONS) + cartesian_topology(cartesian_topology const&) = default; + cartesian_topology& operator=(cartesian_topology const&) = default; + // There is apparently no macro for checking the support of move constructor. + // Assume that defaulted function is close enough. + cartesian_topology(cartesian_topology const&& other) : super(other) {} + cartesian_topology& operator=(cartesian_topology const&& other) { + (*this) = std::move(other.stl()); + return *this; + } + + ~cartesian_topology() = default; +#endif + /** + * @brief Create a N dimension space. + * Each dimension is initialized as non periodic of size 0. + */ + cartesian_topology(int ndim) + : super(ndim) {} + + /** + * @brief Use the provided dimensions specification as initial values. + */ + cartesian_topology(std::vector const& dims) + : super(dims) {} + + /** + * @brief Use dimensions specification provided in the sequence container as initial values. + * #param dims must be a sequence container. + */ + template + explicit cartesian_topology(InitArr dims) + : super(0) { + BOOST_FOREACH(cartesian_dimension const& d, dims) { + push_back(d); + } + } +#if !defined(BOOST_NO_CXX11_HDR_INITIALIZER_LIST) + /** + * @brief Use dimensions specification provided in the initialization list as initial values. + * #param dims can be of the form { dim_1, false}, .... {dim_n, true} + */ + explicit cartesian_topology(std::initializer_list dims) + : super(dims) {} +#endif + /** + * @brief Use dimensions specification provided in the array. + * #param dims can be of the form { dim_1, false}, .... {dim_n, true} + */ + template + explicit cartesian_topology(cartesian_dimension (&dims)[NDIM]) + : super(dims, dims+NDIM) {} + + /** + * @brief Use dimensions specification provided in the input ranges + * The ranges do not need to be the same size. If the sizes are different, + * the missing values will be complete with zeros of the dim and assumed non periodic. + * @param dim_rg the dimensions, values must convert to integers. + * @param period_rg the periodicities, values must convert to booleans. + * #param dims can be of the form { dim_1, false}, .... {dim_n, true} + */ + template + cartesian_topology(DimRg const& dim_rg, PerRg const& period_rg) + : super(0) { + BOOST_FOREACH(int d, dim_rg) { + super::push_back(cartesian_dimension(d)); + } + super::iterator it = begin(); + BOOST_FOREACH(bool p, period_rg) { + if (it < end()) { + it->periodic = p; + } else { + push_back(cartesian_dimension(0,p)); + } + ++it; + } + } + + + /** + * @brief Iterator based initializer. + * Will use the first n iterated values. + * Both iterators can be single pass. + * @param dit dimension iterator, value must convert to integer type. + * @param pit periodicity iterator, value must convert to booleans.. + */ + template + cartesian_topology(DimIter dit, PerIter pit, int n) + : super(n) { + for(int i = 0; i < n; ++i) { + (*this)[i] = cartesian_dimension(*dit++, *pit++); + } + } + + /** + * Export as an stl sequence. + */ + std::vector& stl() { return *this; } + /** + * Export as an stl sequence. + */ + std::vector const& stl() const{ return *this; } + /** + * Split the topology in two sequences of sizes and periodicities. + */ + void split(std::vector& dims, std::vector& periodics) const; +}; + +inline +bool +operator==(cartesian_topology const& t1, cartesian_topology const& t2) { + return t1.stl() == t2.stl(); +} + +inline +bool +operator!=(cartesian_topology const& t1, cartesian_topology const& t2) { + return t1.stl() != t2.stl(); +} + +/** + * @brief Pretty printing of a cartesian topology + */ +std::ostream& operator<<(std::ostream& out, cartesian_topology const& t); + +/** + * @brief An MPI communicator with a cartesian topology. + * + * A @c cartesian_communicator is a communicator whose topology is + * expressed as a grid. Cartesian communicators have the same + * functionality as (intra)communicators, but also allow one to query + * the relationships among processes and the properties of the grid. + */ +class BOOST_MPI_DECL cartesian_communicator : public communicator +{ + friend class communicator; + + /** + * INTERNAL ONLY + * + * Construct a cartesian communicator given a shared pointer to the + * underlying MPI_Comm (which must have a cartesian topology). + * This operation is used for "casting" from a communicator to + * a cartesian communicator. + */ + explicit cartesian_communicator(const shared_ptr& comm_ptr) + : communicator() + { + this->comm_ptr = comm_ptr; + BOOST_ASSERT(has_cartesian_topology()); + } + +public: + /** + * Build a new Boost.MPI cartesian communicator based on the MPI + * communicator @p comm with cartesian topology. + * + * @p comm may be any valid MPI communicator. If @p comm is + * MPI_COMM_NULL, an empty communicator (that cannot be used for + * communication) is created and the @p kind parameter is + * ignored. Otherwise, the @p kind parameter determines how the + * Boost.MPI communicator will be related to @p comm: + * + * - If @p kind is @c comm_duplicate, duplicate @c comm to create + * a new communicator. This new communicator will be freed when + * the Boost.MPI communicator (and all copies of it) is + * destroyed. This option is only permitted if the underlying MPI + * implementation supports MPI 2.0; duplication of + * intercommunicators is not available in MPI 1.x. + * + * - If @p kind is @c comm_take_ownership, take ownership of @c + * comm. It will be freed automatically when all of the Boost.MPI + * communicators go out of scope. + * + * - If @p kind is @c comm_attach, this Boost.MPI communicator + * will reference the existing MPI communicator @p comm but will + * not free @p comm when the Boost.MPI communicator goes out of + * scope. This option should only be used when the communicator is + * managed by the user. + */ + cartesian_communicator(const MPI_Comm& comm, comm_create_kind kind) + : communicator(comm, kind) + { + BOOST_ASSERT(has_cartesian_topology()); + } + + /** + * Create a new communicator whose topology is described by the + * given cartesian. The indices of the vertices in the cartesian will be + * assumed to be the ranks of the processes within the + * communicator. There may be fewer vertices in the cartesian than + * there are processes in the communicator; in this case, the + * resulting communicator will be a NULL communicator. + * + * @param comm The communicator that the new, cartesian communicator + * will be based on. + * + * @param dims the cartesian dimension of the new communicator. The size indicate + * the number of dimension. Some dimensions be set to zero, in which case + * the corresponding dimension value is left to the system. + * + * @param reorder Whether MPI is permitted to re-order the process + * ranks within the returned communicator, to better optimize + * communication. If false, the ranks of each process in the + * returned process will match precisely the rank of that process + * within the original communicator. + */ + cartesian_communicator(const communicator& comm, + const cartesian_topology& dims, + bool reorder = false); + + /** + * Create a new cartesian communicator whose topology is a subset of + * an existing cartesian cimmunicator. + * @param comm the original communicator. + * @param keep and array containiing the dimension to keep from the existing + * communicator. + */ + cartesian_communicator(const cartesian_communicator& comm, + const std::vector& keep ); + + using communicator::rank; + + /** + * Retrive the number of dimension of the underlying toppology. + */ + int ndims() const; + + /** + * Return the rank of the process at the given coordinates. + * @param coords the coordinates. the size must match the communicator's topology. + */ + int rank(const std::vector& coords) const; + /** + * Return the rank of the source and target destination process through a shift. + * @param dim the dimension in which the shift takes place. 0 <= dim <= ndim(). + * @param disp the shift displacement, can be positive (upward) or negative (downward). + */ + std::pair shifted_ranks(int dim, int disp) const; + /** + * Provides the coordinates of the process with the given rank. + * @param rk the ranks in this communicator. + * @returns the coordinates. + */ + std::vector coordinates(int rk) const; + /** + * Retrieve the topology and coordinates of this process in the grid. + * + */ + void topology( cartesian_topology& dims, std::vector& coords ) const; + /** + * Retrieve the topology of the grid. + * + */ + cartesian_topology topology() const; +}; + +/** + * Given en number of processes, and a partially filled sequence + * of dimension, try to complete the dimension sequence. + * @param nb_proc the numer of mpi processes.fill a sequence of dimension. + * @param dims a sequence of positive or null dimensions. Non zero dimension + * will be left untouched. + */ +std::vector& cartesian_dimensions(int nb_proc, std::vector& dims); + +} } // end namespace boost::mpi + +#endif // BOOST_MPI_CARTESIAN_COMMUNICATOR_HPP diff --git a/include/boost/mpi/collectives/all_gather.hpp b/include/boost/mpi/collectives/all_gather.hpp index da73186..4adaeb9 100644 --- a/include/boost/mpi/collectives/all_gather.hpp +++ b/include/boost/mpi/collectives/all_gather.hpp @@ -1,52 +1,107 @@ -// Copyright (C) 2005-2006 Douglas Gregor . +// Copyright (C) 2005, 2006 Douglas Gregor. // Use, modification and distribution is subject to the Boost Software // License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at // http://www.boost.org/LICENSE_1_0.txt) -// Message Passing Interface 1.1 -- Section 4.7. Gather-to-all -#ifndef BOOST_MPI_ALL_GATHER_HPP -#define BOOST_MPI_ALL_GATHER_HPP +// Message Passing Interface 1.1 -- Section 4.5. Gather +#ifndef BOOST_MPI_ALLGATHER_HPP +#define BOOST_MPI_ALLGATHER_HPP +#include +#include +#include #include #include #include -#include - -// all_gather falls back to gather+broadcast in some cases -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include namespace boost { namespace mpi { namespace detail { - // We're all-gathering for a type that has an associated MPI - // datatype, so we'll use MPI_Gather to do all of the work. - template - void - all_gather_impl(const communicator& comm, const T* in_values, int n, - T* out_values, mpl::true_) - { - MPI_Datatype type = boost::mpi::get_mpi_datatype(*in_values); - BOOST_MPI_CHECK_RESULT(MPI_Allgather, - (const_cast(in_values), n, type, - out_values, n, type, comm)); - } +// We're all-gathering for a type that has an associated MPI +// datatype, so we'll use MPI_Gather to do all of the work. +template +void +all_gather_impl(const communicator& comm, const T* in_values, int n, + T* out_values, mpl::true_) +{ + MPI_Datatype type = get_mpi_datatype(*in_values); + BOOST_MPI_CHECK_RESULT(MPI_Allgather, + (const_cast(in_values), n, type, + out_values, n, type, comm)); +} - // We're all-gathering for a type that has no associated MPI - // type. So, we'll do a manual gather followed by a broadcast. - template - void - all_gather_impl(const communicator& comm, const T* in_values, int n, - T* out_values, mpl::false_) - { - gather(comm, in_values, n, out_values, 0); - broadcast(comm, out_values, comm.size() * n, 0); +// We're all-gathering for a type that does not have an +// associated MPI datatype, so we'll need to serialize +// it. +template +void +all_gather_impl(const communicator& comm, const T* in_values, int n, + T* out_values, int const* sizes, int const* skips, mpl::false_) +{ + int nproc = comm.size(); + // first, gather all size, these size can be different for + // each process + packed_oarchive oa(comm); + for (int i = 0; i < n; ++i) { + oa << in_values[i]; } + std::vector oasizes(nproc); + int oasize = oa.size(); + BOOST_MPI_CHECK_RESULT(MPI_Allgather, + (&oasize, 1, MPI_INTEGER, + c_data(oasizes), 1, MPI_INTEGER, + MPI_Comm(comm))); + // Gather the archives, which can be of different sizes, so + // we need to use allgatherv. + // Every thing is contiguous, so the offsets can be + // deduced from the collected sizes. + std::vector offsets(nproc); + sizes2offsets(oasizes, offsets); + packed_iarchive::buffer_type recv_buffer(std::accumulate(oasizes.begin(), oasizes.end(), 0)); + BOOST_MPI_CHECK_RESULT(MPI_Allgatherv, + (const_cast(oa.address()), int(oa.size()), MPI_BYTE, + c_data(recv_buffer), c_data(oasizes), c_data(offsets), MPI_BYTE, + MPI_Comm(comm))); + for (int src = 0; src < nproc; ++src) { + int nb = sizes ? sizes[src] : n; + int skip = skips ? skips[src] : 0; + std::advance(out_values, skip); + if (src == comm.rank()) { // this is our local data + for (int i = 0; i < nb; ++i) { + *out_values++ = *in_values++; + } + } else { + packed_iarchive ia(comm, recv_buffer, boost::archive::no_header, offsets[src]); + for (int i = 0; i < nb; ++i) { + ia >> *out_values++; + } + } + } +} + +// We're all-gathering for a type that does not have an +// associated MPI datatype, so we'll need to serialize +// it. +template +void +all_gather_impl(const communicator& comm, const T* in_values, int n, + T* out_values, mpl::false_ isnt_mpi_type) +{ + all_gather_impl(comm, in_values, n, out_values, (int const*)0, (int const*)0, isnt_mpi_type); +} } // end namespace detail template -inline void +void all_gather(const communicator& comm, const T& in_value, T* out_values) { detail::all_gather_impl(comm, &in_value, 1, out_values, is_mpi_datatype()); @@ -54,15 +109,15 @@ all_gather(const communicator& comm, const T& in_value, T* out_values) template void -all_gather(const communicator& comm, const T& in_value, - std::vector& out_values) +all_gather(const communicator& comm, const T& in_value, std::vector& out_values) { + using detail::c_data; out_values.resize(comm.size()); - ::boost::mpi::all_gather(comm, &in_value, 1, &out_values[0]); + ::boost::mpi::all_gather(comm, in_value, c_data(out_values)); } template -inline void +void all_gather(const communicator& comm, const T* in_values, int n, T* out_values) { detail::all_gather_impl(comm, in_values, n, out_values, is_mpi_datatype()); @@ -70,11 +125,11 @@ all_gather(const communicator& comm, const T* in_values, int n, T* out_values) template void -all_gather(const communicator& comm, const T* in_values, int n, - std::vector& out_values) +all_gather(const communicator& comm, const T* in_values, int n, std::vector& out_values) { + using detail::c_data; out_values.resize(comm.size() * n); - ::boost::mpi::all_gather(comm, in_values, n, &out_values[0]); + ::boost::mpi::all_gather(comm, in_values, n, c_data(out_values)); } } } // end namespace boost::mpi diff --git a/include/boost/mpi/collectives/all_gatherv.hpp b/include/boost/mpi/collectives/all_gatherv.hpp new file mode 100644 index 0000000..064412f --- /dev/null +++ b/include/boost/mpi/collectives/all_gatherv.hpp @@ -0,0 +1,140 @@ +// Copyright (C) 2005, 2006 Douglas Gregor. + +// Use, modification and distribution is subject to the Boost Software +// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +// Message Passing Interface 1.1 -- Section 4.5. Gatherv +#ifndef BOOST_MPI_ALLGATHERV_HPP +#define BOOST_MPI_ALLGATHERV_HPP + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace boost { namespace mpi { + +namespace detail { +// We're all-gathering for a type that has an associated MPI +// datatype, so we'll use MPI_Gather to do all of the work. +template +void +all_gatherv_impl(const communicator& comm, const T* in_values, + T* out_values, int const* sizes, int const* displs, mpl::true_) +{ + // Make displacements if not provided + scoped_array new_offsets_mem(make_offsets(comm, sizes, displs, -1)); + if (new_offsets_mem) displs = new_offsets_mem.get(); + MPI_Datatype type = get_mpi_datatype(*in_values); + BOOST_MPI_CHECK_RESULT(MPI_Allgatherv, + (const_cast(in_values), sizes[comm.rank()], type, + out_values, + const_cast(sizes), + const_cast(displs), + type, + comm)); +} + +// We're all-gathering for a type that does not have an +// associated MPI datatype, so we'll need to serialize +// it. +template +void +all_gatherv_impl(const communicator& comm, const T* in_values, + T* out_values, int const* sizes, int const* displs, + mpl::false_ isnt_mpi_type) +{ + // convert displacement to offsets to skip + scoped_array skipped(make_skipped_slots(comm, sizes, displs)); + all_gather_impl(comm, in_values, sizes[comm.rank()], out_values, + sizes, skipped.get(), isnt_mpi_type); +} +} // end namespace detail + +template +void +all_gatherv(const communicator& comm, const T& in_value, T* out_values, + const std::vector& sizes) +{ + using detail::c_data; + assert(sizes.size() == comm.size()); + assert(sizes[comm.rank()] == 1); + detail::all_gatherv_impl(comm, &in_value, out_values, c_data(sizes), 0, is_mpi_datatype()); +} + +template +void +all_gatherv(const communicator& comm, const T* in_values, T* out_values, + const std::vector& sizes) +{ + using detail::c_data; + assert(int(sizes.size()) == comm.size()); + detail::all_gatherv_impl(comm, in_values, out_values, c_data(sizes), 0, is_mpi_datatype()); +} + +template +void +all_gatherv(const communicator& comm, std::vector const& in_values, std::vector& out_values, + const std::vector& sizes) +{ + using detail::c_data; + assert(int(sizes.size()) == comm.size()); + assert(int(in_values.size()) == sizes[comm.rank()]); + out_values.resize(std::accumulate(sizes.begin(), sizes.end(), 0)); + ::boost::mpi::all_gatherv(comm, c_data(in_values), c_data(out_values), sizes); +} + + +template +void +all_gatherv(const communicator& comm, const T& in_value, T* out_values, + const std::vector& sizes, const std::vector& displs) +{ + using detail::c_data; + assert(sizes.size() == comm.size()); + assert(displs.size() == comm.size()); + detail::all_gatherv_impl(comm, &in_value, 1, out_values, + c_data(sizes), c_data(displs), is_mpi_datatype()); +} + +template +void +all_gatherv(const communicator& comm, const T* in_values, T* out_values, + const std::vector& sizes, const std::vector& displs) +{ + using detail::c_data; + assert(sizes.size() == comm.size()); + assert(displs.size() == comm.size()); + detail::all_gatherv_impl(comm, in_values, out_values, + c_data(sizes), c_data(displs), is_mpi_datatype()); +} + +template +void +all_gatherv(const communicator& comm, std::vector const& in_values, std::vector& out_values, + const std::vector& sizes, const std::vector& displs) +{ + using detail::c_data; + assert(sizes.size() == comm.size()); + assert(displs.size() == comm.size()); + assert(in_values.size() == sizes[comm.rank()]); + out_values.resize(std::accumulate(sizes.begin(), sizes.end(), 0)); + ::boost::mpi::all_gatherv(comm, c_data(in_values), c_data(out_values), sizes, displs); +} + +} } // end namespace boost::mpi + +#endif // BOOST_MPI_ALL_GATHERV_HPP diff --git a/include/boost/mpi/collectives/all_to_all.hpp b/include/boost/mpi/collectives/all_to_all.hpp index 8c33c2a..4f20be7 100644 --- a/include/boost/mpi/collectives/all_to_all.hpp +++ b/include/boost/mpi/collectives/all_to_all.hpp @@ -22,7 +22,7 @@ namespace boost { namespace mpi { namespace detail { - // We're performaing an all-to-all with a type that has an + // We're performing an all-to-all with a type that has an // associated MPI datatype, so we'll use MPI_Alltoall to do all of // the work. template @@ -38,9 +38,7 @@ namespace detail { // We're performing an all-to-all with a type that does not have an // associated MPI datatype, so we'll need to serialize - // it. Unfortunately, this means that we cannot use MPI_Alltoall, so - // we'll just have to send individual messages to the other - // processes. + // it. template void all_to_all_impl(const communicator& comm, const T* in_values, int n, diff --git a/include/boost/mpi/collectives/broadcast.hpp b/include/boost/mpi/collectives/broadcast.hpp index d5160cf..f8b27f0 100644 --- a/include/boost/mpi/collectives/broadcast.hpp +++ b/include/boost/mpi/collectives/broadcast.hpp @@ -100,22 +100,35 @@ namespace detail { } // We're sending a type that does not have an associated MPI - // datatype, so we'll need to serialize it. Unfortunately, this - // means that we cannot use MPI_Bcast, so we'll just send from the - // root to everyone else. + // datatype, so we'll need to serialize it. template void broadcast_impl(const communicator& comm, T* values, int n, int root, - mpl::false_) + mpl::false_ non_mpi_datatype) { + // Implementation proposed by Lorenz Hübschle-Schneider if (comm.rank() == root) { packed_oarchive oa(comm); - for (int i = 0; i < n; ++i) + for (int i = 0; i < n; ++i) { oa << values[i]; - broadcast(comm, oa, root); + } + std::size_t asize = oa.size(); + broadcast(comm, asize, root); + void const* aptr = oa.address(); + BOOST_MPI_CHECK_RESULT(MPI_Bcast, + (const_cast(aptr), asize, + MPI_BYTE, + root, MPI_Comm(comm))); } else { packed_iarchive ia(comm); - broadcast(comm, ia, root); + std::size_t asize; + broadcast(comm, asize, root); + ia.resize(asize); + void* aptr = ia.address(); + BOOST_MPI_CHECK_RESULT(MPI_Bcast, + (aptr, asize, + MPI_BYTE, + root, MPI_Comm(comm))); for (int i = 0; i < n; ++i) ia >> values[i]; } diff --git a/include/boost/mpi/collectives/gather.hpp b/include/boost/mpi/collectives/gather.hpp index 70dfd65..386bfdd 100644 --- a/include/boost/mpi/collectives/gather.hpp +++ b/include/boost/mpi/collectives/gather.hpp @@ -8,6 +8,9 @@ #ifndef BOOST_MPI_GATHER_HPP #define BOOST_MPI_GATHER_HPP +#include +#include +#include #include #include #include @@ -16,89 +19,116 @@ #include #include #include +#include +#include #include namespace boost { namespace mpi { namespace detail { - // We're gathering at the root for a type that has an associated MPI - // datatype, so we'll use MPI_Gather to do all of the work. - template - void - gather_impl(const communicator& comm, const T* in_values, int n, - T* out_values, int root, mpl::true_) - { - MPI_Datatype type = get_mpi_datatype(*in_values); - BOOST_MPI_CHECK_RESULT(MPI_Gather, - (const_cast(in_values), n, type, - out_values, n, type, root, comm)); - } +// We're gathering at the root for a type that has an associated MPI +// datatype, so we'll use MPI_Gather to do all of the work. +template +void +gather_impl(const communicator& comm, const T* in_values, int n, + T* out_values, int root, mpl::true_) +{ + MPI_Datatype type = get_mpi_datatype(*in_values); + BOOST_MPI_CHECK_RESULT(MPI_Gather, + (const_cast(in_values), n, type, + out_values, n, type, root, comm)); +} - // We're gathering from a non-root for a type that has an associated MPI - // datatype, so we'll use MPI_Gather to do all of the work. - template - void - gather_impl(const communicator& comm, const T* in_values, int n, int root, - mpl::true_) - { - MPI_Datatype type = get_mpi_datatype(*in_values); - BOOST_MPI_CHECK_RESULT(MPI_Gather, - (const_cast(in_values), n, type, - 0, n, type, root, comm)); - } +// We're gathering from a non-root for a type that has an associated MPI +// datatype, so we'll use MPI_Gather to do all of the work. +template +void +gather_impl(const communicator& comm, const T* in_values, int n, int root, + mpl::true_ is_mpi_type) +{ + assert(comm.rank() != root); + gather_impl(comm, in_values, n, (T*)0, root, is_mpi_type); +} - // We're gathering at the root for a type that does not have an - // associated MPI datatype, so we'll need to serialize - // it. Unfortunately, this means that we cannot use MPI_Gather, so - // we'll just have all of the non-root nodes send individual - // messages to the root. - template - void - gather_impl(const communicator& comm, const T* in_values, int n, - T* out_values, int root, mpl::false_) - { - int tag = environment::collectives_tag(); - int size = comm.size(); - - for (int src = 0; src < size; ++src) { - if (src == root) - std::copy(in_values, in_values + n, out_values + n * src); - else - comm.recv(src, tag, out_values + n * src, n); +// We're gathering at the root for a type that does not have an +// associated MPI datatype, so we'll need to serialize +// it. +template +void +gather_impl(const communicator& comm, const T* in_values, int n, T* out_values, + int const* nslot, int const* nskip, int root, mpl::false_) +{ + int nproc = comm.size(); + // first, gather all size, these size can be different for + // each process + packed_oarchive oa(comm); + for (int i = 0; i < n; ++i) { + oa << in_values[i]; + } + bool is_root = comm.rank() == root; + std::vector oasizes(is_root ? nproc : 0); + int oasize = oa.size(); + BOOST_MPI_CHECK_RESULT(MPI_Gather, + (&oasize, 1, MPI_INTEGER, + c_data(oasizes), 1, MPI_INTEGER, + root, MPI_Comm(comm))); + // Gather the archives, which can be of different sizes, so + // we need to use gatherv. + // Everything is contiguous (in the transmitted archive), so + // the offsets can be deduced from the collected sizes. + std::vector offsets; + if (is_root) sizes2offsets(oasizes, offsets); + packed_iarchive::buffer_type recv_buffer(is_root ? std::accumulate(oasizes.begin(), oasizes.end(), 0) : 0); + BOOST_MPI_CHECK_RESULT(MPI_Gatherv, + (const_cast(oa.address()), int(oa.size()), MPI_BYTE, + c_data(recv_buffer), c_data(oasizes), c_data(offsets), MPI_BYTE, + root, MPI_Comm(comm))); + if (is_root) { + for (int src = 0; src < nproc; ++src) { + // handle variadic case + int nb = nslot ? nslot[src] : n; + int skip = nskip ? nskip[src] : 0; + std::advance(out_values, skip); + if (src == root) { + BOOST_ASSERT(nb == n); + for (int i = 0; i < nb; ++i) { + *out_values++ = *in_values++; + } + } else { + packed_iarchive ia(comm, recv_buffer, boost::archive::no_header, offsets[src]); + for (int i = 0; i < nb; ++i) { + ia >> *out_values++; + } + } } } +} - // We're gathering at a non-root for a type that does not have an - // associated MPI datatype, so we'll need to serialize - // it. Unfortunately, this means that we cannot use MPI_Gather, so - // we'll just have all of the non-root nodes send individual - // messages to the root. - template - void - gather_impl(const communicator& comm, const T* in_values, int n, int root, - mpl::false_) - { - int tag = environment::collectives_tag(); - comm.send(root, tag, in_values, n); - } +// We're gathering at a non-root for a type that does not have an +// associated MPI datatype, so we'll need to serialize +// it. +template +void +gather_impl(const communicator& comm, const T* in_values, int n, T* out_values,int root, + mpl::false_ is_mpi_type) +{ + gather_impl(comm, in_values, n, out_values, (int const*)0, (int const*)0, root, is_mpi_type); +} } // end namespace detail template void gather(const communicator& comm, const T& in_value, T* out_values, int root) { - if (comm.rank() == root) - detail::gather_impl(comm, &in_value, 1, out_values, root, - is_mpi_datatype()); - else - detail::gather_impl(comm, &in_value, 1, root, is_mpi_datatype()); + BOOST_ASSERT(out_values || (comm.rank() != root)); + detail::gather_impl(comm, &in_value, 1, out_values, root, is_mpi_datatype()); } template void gather(const communicator& comm, const T& in_value, int root) { BOOST_ASSERT(comm.rank() != root); - detail::gather_impl(comm, &in_value, 1, root, is_mpi_datatype()); + detail::gather_impl(comm, &in_value, 1, (T*)0, root, is_mpi_datatype()); } template @@ -106,12 +136,11 @@ void gather(const communicator& comm, const T& in_value, std::vector& out_values, int root) { + using detail::c_data; if (comm.rank() == root) { out_values.resize(comm.size()); - ::boost::mpi::gather(comm, in_value, &out_values[0], root); - } else { - ::boost::mpi::gather(comm, in_value, root); } + ::boost::mpi::gather(comm, in_value, c_data(out_values), root); } template @@ -119,11 +148,8 @@ void gather(const communicator& comm, const T* in_values, int n, T* out_values, int root) { - if (comm.rank() == root) - detail::gather_impl(comm, in_values, n, out_values, root, - is_mpi_datatype()); - else - detail::gather_impl(comm, in_values, n, root, is_mpi_datatype()); + detail::gather_impl(comm, in_values, n, out_values, root, + is_mpi_datatype()); } template @@ -133,10 +159,8 @@ gather(const communicator& comm, const T* in_values, int n, { if (comm.rank() == root) { out_values.resize(comm.size() * n); - ::boost::mpi::gather(comm, in_values, n, &out_values[0], root); - } - else - ::boost::mpi::gather(comm, in_values, n, root); + } + ::boost::mpi::gather(comm, in_values, n, out_values.data(), root); } template diff --git a/include/boost/mpi/collectives/gatherv.hpp b/include/boost/mpi/collectives/gatherv.hpp index eb5f9c1..6b8d706 100644 --- a/include/boost/mpi/collectives/gatherv.hpp +++ b/include/boost/mpi/collectives/gatherv.hpp @@ -8,15 +8,18 @@ #ifndef BOOST_MPI_GATHERV_HPP #define BOOST_MPI_GATHERV_HPP +#include + #include #include -#include #include #include #include #include #include +#include #include +#include namespace boost { namespace mpi { @@ -58,41 +61,21 @@ namespace detail { gatherv_impl(const communicator& comm, const T* in_values, int in_size, T* out_values, const int* sizes, const int* displs, int root, mpl::false_) { - int tag = environment::collectives_tag(); - int nprocs = comm.size(); - - for (int src = 0; src < nprocs; ++src) { - if (src == root) - // Our own values will never be transmitted: just copy them. - std::copy(in_values, in_values + in_size, out_values + displs[src]); - else { -// comm.recv(src, tag, out_values + displs[src], sizes[src]); - // Receive archive - packed_iarchive ia(comm); - MPI_Status status; - detail::packed_archive_recv(comm, src, tag, ia, status); - for (int i = 0; i < sizes[src]; ++i) - ia >> out_values[ displs[src] + i ]; - } - } + // convert displacement to offsets to skip + scoped_array skipped(make_skipped_slots(comm, sizes, displs, root)); + gather_impl(comm, in_values, in_size, out_values, sizes, skipped.get(), root, mpl::false_()); } // We're gathering at a non-root for a type that does not have an // associated MPI datatype, so we'll need to serialize - // it. Unfortunately, this means that we cannot use MPI_Gatherv, so - // we'll just have all of the non-root nodes send individual - // messages to the root. + // it. template void gatherv_impl(const communicator& comm, const T* in_values, int in_size, int root, mpl::false_) { - int tag = environment::collectives_tag(); -// comm.send(root, tag, in_values, in_size); - packed_oarchive oa(comm); - for (int i = 0; i < in_size; ++i) - oa << in_values[i]; - detail::packed_archive_send(comm, root, tag, oa); + gather_impl(comm, in_values, in_size, (T*)0,(int const*)0,(int const*)0, root, + mpl::false_()); } } // end namespace detail diff --git a/include/boost/mpi/collectives/scatter.hpp b/include/boost/mpi/collectives/scatter.hpp index 196682d..0c91b1e 100644 --- a/include/boost/mpi/collectives/scatter.hpp +++ b/include/boost/mpi/collectives/scatter.hpp @@ -16,94 +16,147 @@ #include #include #include +#include +#include #include namespace boost { namespace mpi { namespace detail { - // We're scattering from the root for a type that has an associated MPI - // datatype, so we'll use MPI_Scatter to do all of the work. - template - void - scatter_impl(const communicator& comm, const T* in_values, T* out_values, - int n, int root, mpl::true_) - { - MPI_Datatype type = get_mpi_datatype(*in_values); - BOOST_MPI_CHECK_RESULT(MPI_Scatter, - (const_cast(in_values), n, type, - out_values, n, type, root, comm)); - } +// We're scattering from the root for a type that has an associated MPI +// datatype, so we'll use MPI_Scatter to do all of the work. +template +void +scatter_impl(const communicator& comm, const T* in_values, T* out_values, + int n, int root, mpl::true_) +{ + MPI_Datatype type = get_mpi_datatype(*in_values); + BOOST_MPI_CHECK_RESULT(MPI_Scatter, + (const_cast(in_values), n, type, + out_values, n, type, root, comm)); +} - // We're scattering from a non-root for a type that has an associated MPI - // datatype, so we'll use MPI_Scatter to do all of the work. - template - void - scatter_impl(const communicator& comm, T* out_values, int n, int root, - mpl::true_) - { - MPI_Datatype type = get_mpi_datatype(*out_values); - BOOST_MPI_CHECK_RESULT(MPI_Scatter, - (0, n, type, - out_values, n, type, - root, comm)); +// We're scattering from a non-root for a type that has an associated MPI +// datatype, so we'll use MPI_Scatter to do all of the work. +template +void +scatter_impl(const communicator& comm, T* out_values, int n, int root, + mpl::true_) +{ + MPI_Datatype type = get_mpi_datatype(*out_values); + BOOST_MPI_CHECK_RESULT(MPI_Scatter, + (0, n, type, + out_values, n, type, + root, comm)); +} + +// Fill the sendbuf while keeping trac of the slot's footprints +// Used in the first steps of both scatter and scatterv +// Nslots contains the number of slots being sent +// to each process (identical values for scatter). +// skiped_slots, if present, is deduced from the +// displacement array authorised be the MPI API, +// for some yet to be determined reason. +template +void +fill_scatter_sendbuf(const communicator& comm, T const* values, + int const* nslots, int const* skipped_slots, + packed_oarchive::buffer_type& sendbuf, std::vector& archsizes) { + int nproc = comm.size(); + archsizes.resize(nproc); + + for (int dest = 0; dest < nproc; ++dest) { + if (skipped_slots) { // wee need to keep this for backward compatibility + for(int k= 0; k < skipped_slots[dest]; ++k) ++values; + } + packed_oarchive procarchive(comm); + for (int i = 0; i < nslots[dest]; ++i) { + procarchive << *values++; + } + int archsize = procarchive.size(); + sendbuf.resize(sendbuf.size() + archsize); + archsizes[dest] = archsize; + char const* aptr = static_cast(procarchive.address()); + std::copy(aptr, aptr+archsize, sendbuf.end()-archsize); } +} - // We're scattering from the root for a type that does not have an - // associated MPI datatype, so we'll need to serialize - // it. Unfortunately, this means that we cannot use MPI_Scatter, so - // we'll just have the root send individual messages to the other - // processes. - template - void - scatter_impl(const communicator& comm, const T* in_values, T* out_values, - int n, int root, mpl::false_) - { - int tag = environment::collectives_tag(); - int size = comm.size(); - - for (int dest = 0; dest < size; ++dest) { - if (dest == root) { - // Our own values will never be transmitted: just copy them. - std::copy(in_values + dest * n, in_values + (dest + 1) * n, out_values); - } else { - // Send archive - packed_oarchive oa(comm); - for (int i = 0; i < n; ++i) - oa << in_values[dest * n + i]; - detail::packed_archive_send(comm, dest, tag, oa); - } +template +T* +non_const_data(std::vector const& v) { + using detail::c_data; + return const_cast(c_data(v)); +} + +// Dispatch the sendbuf among proc. +// Used in the second steps of both scatter and scatterv +// in_value is only provide in the non variadic case. +template +void +dispatch_scatter_sendbuf(const communicator& comm, + packed_oarchive::buffer_type const& sendbuf, std::vector const& archsizes, + T const* in_values, + T* out_values, int n, int root) { + // Distribute the sizes + int myarchsize; + BOOST_MPI_CHECK_RESULT(MPI_Scatter, + (non_const_data(archsizes), 1, MPI_INTEGER, + &myarchsize, 1, MPI_INTEGER, root, comm)); + std::vector offsets; + if (root == comm.rank()) { + sizes2offsets(archsizes, offsets); + } + // Get my proc archive + packed_iarchive::buffer_type recvbuf; + recvbuf.resize(myarchsize); + BOOST_MPI_CHECK_RESULT(MPI_Scatterv, + (non_const_data(sendbuf), non_const_data(archsizes), c_data(offsets), MPI_BYTE, + c_data(recvbuf), recvbuf.size(), MPI_BYTE, + root, MPI_Comm(comm))); + // Unserialize + if ( in_values != 0 && root == comm.rank()) { + // Our own local values are already here: just copy them. + std::copy(in_values + root * n, in_values + (root + 1) * n, out_values); + } else { + // Otherwise deserialize: + packed_iarchive iarchv(comm, recvbuf); + for (int i = 0; i < n; ++i) { + iarchv >> out_values[i]; } } +} - // We're scattering to a non-root for a type that does not have an - // associated MPI datatype, so we'll need to de-serialize - // it. Unfortunately, this means that we cannot use MPI_Scatter, so - // we'll just have all of the non-root nodes send individual - // messages to the root. - template - void - scatter_impl(const communicator& comm, T* out_values, int n, int root, - mpl::false_) - { - int tag = environment::collectives_tag(); - - packed_iarchive ia(comm); - MPI_Status status; - detail::packed_archive_recv(comm, root, tag, ia, status); - for (int i = 0; i < n; ++i) - ia >> out_values[i]; +// We're scattering from the root for a type that does not have an +// associated MPI datatype, so we'll need to serialize it. +template +void +scatter_impl(const communicator& comm, const T* in_values, T* out_values, + int n, int root, mpl::false_) +{ + packed_oarchive::buffer_type sendbuf; + std::vector archsizes; + + if (root == comm.rank()) { + std::vector nslots(comm.size(), n); + fill_scatter_sendbuf(comm, in_values, c_data(nslots), (int const*)0, sendbuf, archsizes); } + dispatch_scatter_sendbuf(comm, sendbuf, archsizes, in_values, out_values, n, root); +} + +template +void +scatter_impl(const communicator& comm, T* out_values, int n, int root, + mpl::false_ is_mpi_type) +{ + scatter_impl(comm, (T const*)0, out_values, n, root, is_mpi_type); +} } // end namespace detail template void scatter(const communicator& comm, const T* in_values, T& out_value, int root) { - if (comm.rank() == root) - detail::scatter_impl(comm, in_values, &out_value, 1, root, - is_mpi_datatype()); - else - detail::scatter_impl(comm, &out_value, 1, root, is_mpi_datatype()); + detail::scatter_impl(comm, in_values, &out_value, 1, root, is_mpi_datatype()); } template @@ -111,11 +164,8 @@ void scatter(const communicator& comm, const std::vector& in_values, T& out_value, int root) { - if (comm.rank() == root) - ::boost::mpi::scatter(comm, &in_values[0], out_value, root); - else - ::boost::mpi::scatter(comm, static_cast(0), out_value, - root); + using detail::c_data; + ::boost::mpi::scatter(comm, c_data(in_values), out_value, root); } template @@ -130,11 +180,7 @@ void scatter(const communicator& comm, const T* in_values, T* out_values, int n, int root) { - if (comm.rank() == root) - detail::scatter_impl(comm, in_values, out_values, n, root, - is_mpi_datatype()); - else - detail::scatter_impl(comm, out_values, n, root, is_mpi_datatype()); + detail::scatter_impl(comm, in_values, out_values, n, root, is_mpi_datatype()); } template @@ -142,11 +188,7 @@ void scatter(const communicator& comm, const std::vector& in_values, T* out_values, int n, int root) { - if (comm.rank() == root) - ::boost::mpi::scatter(comm, &in_values[0], out_values, n, root); - else - ::boost::mpi::scatter(comm, static_cast(0), out_values, - n, root); + ::boost::mpi::scatter(comm, &in_values[0], out_values, n, root); } template diff --git a/include/boost/mpi/collectives/scatterv.hpp b/include/boost/mpi/collectives/scatterv.hpp index 6e6f270..57e073c 100644 --- a/include/boost/mpi/collectives/scatterv.hpp +++ b/include/boost/mpi/collectives/scatterv.hpp @@ -8,93 +8,90 @@ #ifndef BOOST_MPI_SCATTERV_HPP #define BOOST_MPI_SCATTERV_HPP -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include namespace boost { namespace mpi { namespace detail { - // We're scattering from the root for a type that has an associated MPI - // datatype, so we'll use MPI_Scatterv to do all of the work. - template - void - scatterv_impl(const communicator& comm, const T* in_values, const int* sizes, - const int* displs, T* out_values, int out_size, int root, mpl::true_) - { - MPI_Datatype type = get_mpi_datatype(*in_values); - BOOST_MPI_CHECK_RESULT(MPI_Scatterv, - (const_cast(in_values), const_cast(sizes), - const_cast(displs), type, - out_values, out_size, type, root, comm)); - } - // We're scattering from a non-root for a type that has an associated MPI - // datatype, so we'll use MPI_Scatterv to do all of the work. - template - void - scatterv_impl(const communicator& comm, T* out_values, int out_size, int root, - mpl::true_) - { - MPI_Datatype type = get_mpi_datatype(*out_values); - BOOST_MPI_CHECK_RESULT(MPI_Scatterv, - (0, 0, 0, type, - out_values, out_size, type, - root, comm)); - } +////////////////////////////////////////////// +/// Implementation for MPI primitive types /// +////////////////////////////////////////////// + +// We're scattering from the root for a type that has an associated MPI +// datatype, so we'll use MPI_Scatterv to do all of the work. +template +void +scatterv_impl(const communicator& comm, const T* in_values, T* out_values, int out_size, + const int* sizes, const int* displs, int root, mpl::true_) +{ + assert(!sizes || out_size == sizes[comm.rank()]); + assert(bool(sizes) == bool(in_values)); + + scoped_array new_offsets_mem(make_offsets(comm, sizes, displs, root)); + if (new_offsets_mem) displs = new_offsets_mem.get(); + MPI_Datatype type = get_mpi_datatype(*in_values); + BOOST_MPI_CHECK_RESULT(MPI_Scatterv, + (const_cast(in_values), const_cast(sizes), + const_cast(displs), type, + out_values, out_size, type, root, comm)); +} - // We're scattering from the root for a type that does not have an - // associated MPI datatype, so we'll need to serialize - // it. Unfortunately, this means that we cannot use MPI_Scatterv, so - // we'll just have the root send individual messages to the other - // processes. - template - void - scatterv_impl(const communicator& comm, const T* in_values, const int* sizes, - const int* displs, T* out_values, int out_size, int root, mpl::false_) - { - int tag = environment::collectives_tag(); - int nprocs = comm.size(); - - for (int dest = 0; dest < nprocs; ++dest) { - if (dest == root) { - // Our own values will never be transmitted: just copy them. - std::copy(in_values + displs[dest], - in_values + displs[dest] + out_size, out_values); - } else { - // Send archive - packed_oarchive oa(comm); - for (int i = 0; i < sizes[dest]; ++i) - oa << in_values[ displs[dest] + i ]; - detail::packed_archive_send(comm, dest, tag, oa); - } +// We're scattering from a non-root for a type that has an associated MPI +// datatype, so we'll use MPI_Scatterv to do all of the work. +template +void +scatterv_impl(const communicator& comm, T* out_values, int out_size, int root, + mpl::true_ is_mpi_type) +{ + scatterv_impl(comm, (T const*)0, out_values, out_size, + (const int*)0, (const int*)0, root, is_mpi_type); +} + +////////////////////////////////////////////////// +/// Implementation for non MPI primitive types /// +////////////////////////////////////////////////// + +// We're scattering from the root for a type that does not have an +// associated MPI datatype, so we'll need to serialize it. +template +void +scatterv_impl(const communicator& comm, const T* in_values, T* out_values, int out_size, + int const* sizes, int const* displs, int root, mpl::false_) +{ + packed_oarchive::buffer_type sendbuf; + bool is_root = comm.rank() == root; + int nproc = comm.size(); + std::vector archsizes; + if (is_root) { + assert(out_size == sizes[comm.rank()]); + archsizes.resize(nproc); + std::vector skipped; + if (displs) { + skipped.resize(nproc); + offsets2skipped(sizes, displs, c_data(skipped), nproc); + displs = c_data(skipped); } + fill_scatter_sendbuf(comm, in_values, sizes, (int const*)0, sendbuf, archsizes); } + dispatch_scatter_sendbuf(comm, sendbuf, archsizes, (T const*)0, out_values, out_size, root); +} + +// We're scattering to a non-root for a type that does not have an +// associated MPI datatype. input data not needed. +// it. +template +void +scatterv_impl(const communicator& comm, T* out_values, int n, int root, + mpl::false_ isnt_mpi_type) +{ + assert(root != comm.rank()); + scatterv_impl(comm, (T const*)0, out_values, n, (int const*)0, (int const*)0, root, isnt_mpi_type); +} - // We're scattering to a non-root for a type that does not have an - // associated MPI datatype, so we'll need to de-serialize - // it. Unfortunately, this means that we cannot use MPI_Scatterv, so - // we'll just have all of the non-root nodes send individual - // messages to the root. - template - void - scatterv_impl(const communicator& comm, T* out_values, int out_size, int root, - mpl::false_) - { - int tag = environment::collectives_tag(); - - packed_iarchive ia(comm); - MPI_Status status; - detail::packed_archive_recv(comm, root, tag, ia, status); - for (int i = 0; i < out_size; ++i) - ia >> out_values[i]; - } } // end namespace detail template @@ -103,13 +100,9 @@ scatterv(const communicator& comm, const T* in_values, const std::vector& sizes, const std::vector& displs, T* out_values, int out_size, int root) { - int rank = comm.rank(); - if (rank == root) - detail::scatterv_impl(comm, in_values, &sizes[0], &displs[0], - out_values, out_size, root, is_mpi_datatype()); - else - detail::scatterv_impl(comm, out_values, out_size, root, - is_mpi_datatype()); + using detail::c_data; + scatterv_impl(comm, in_values, out_values, out_size, c_data(sizes), c_data(displs), + root, is_mpi_datatype()); } template @@ -118,12 +111,9 @@ scatterv(const communicator& comm, const std::vector& in_values, const std::vector& sizes, const std::vector& displs, T* out_values, int out_size, int root) { - if (comm.rank() == root) - ::boost::mpi::scatterv(comm, &in_values[0], sizes, displs, - out_values, out_size, root); - else - ::boost::mpi::scatterv(comm, static_cast(0), sizes, displs, - out_values, out_size, root); + using detail::c_data; + ::boost::mpi::scatterv(comm, c_data(in_values), sizes, displs, + out_values, out_size, root); } template @@ -141,16 +131,10 @@ void scatterv(const communicator& comm, const T* in_values, const std::vector& sizes, T* out_values, int root) { - int nprocs = comm.size(); - int myrank = comm.rank(); - - std::vector displs(nprocs); - for (int rank = 0, aux = 0; rank < nprocs; ++rank) { - displs[rank] = aux; - aux += sizes[rank]; - } - ::boost::mpi::scatterv(comm, in_values, sizes, displs, out_values, - sizes[myrank], root); + using detail::c_data; + detail::scatterv_impl(comm, in_values, out_values, sizes[comm.rank()], + c_data(sizes), (int const*)0, + root, is_mpi_datatype()); } template @@ -161,6 +145,23 @@ scatterv(const communicator& comm, const std::vector& in_values, ::boost::mpi::scatterv(comm, &in_values[0], sizes, out_values, root); } +template +void +scatterv(const communicator& comm, const T* in_values, + T* out_values, int n, int root) +{ + detail::scatterv_impl(comm, in_values, out_values, n, (int const*)0, (int const*)0, + root, is_mpi_datatype()); +} + +template +void +scatterv(const communicator& comm, const std::vector& in_values, + T* out_values, int out_size, int root) +{ + ::boost::mpi::scatterv(comm, &in_values[0], out_values, out_size, root); +} + } } // end namespace boost::mpi #endif // BOOST_MPI_SCATTERV_HPP diff --git a/include/boost/mpi/communicator.hpp b/include/boost/mpi/communicator.hpp index a491086..af29d1c 100644 --- a/include/boost/mpi/communicator.hpp +++ b/include/boost/mpi/communicator.hpp @@ -116,6 +116,14 @@ class intercommunicator; class graph_communicator; /** + * INTERNAL ONLY + * + * Forward declaration of @c cartesian_communicator needed for the "cast" + * from a communicator to a cartesian communicator. + */ +class cartesian_communicator; + +/** * @brief A communicator that permits communication and * synchronization among a set of processes. * @@ -850,30 +858,28 @@ class BOOST_MPI_DECL communicator optional as_graph_communicator() const; /** + * Determines whether this communicator has a Graph topology. + */ + bool has_graph_topology() const; + + /** + * Determine if the communicator has a cartesian topology and, if so, + * return that @c cartesian_communicator. Even though the communicators + * have different types, they refer to the same underlying + * communication space and can be used interchangeably for + * communication. + * + * @returns an @c optional containing the cartesian communicator, if this + * communicator does in fact have a cartesian topology. Otherwise, returns + * an empty @c optional. + */ + optional as_cartesian_communicator() const; + + /** * Determines whether this communicator has a Cartesian topology. */ bool has_cartesian_topology() const; -#if 0 - template - communicator - with_cartesian_topology(const Extents& extents, - bool periodic = false, - bool reorder = false) const; - - template - communicator - with_cartesian_topology(DimInputIterator first_dim, - DimInputIterator last_dim, - PeriodicInputIterator first_periodic, - bool reorder = false); - - template - communicator - with_cartesian_topology(const multi_array& periods, - bool reorder = false); -#endif - /** Abort all tasks in the group of this communicator. * * Makes a "best attempt" to abort all of the tasks in the group of @@ -1122,6 +1128,115 @@ inline bool operator!=(const communicator& comm1, const communicator& comm2) /************************************************************************ * Implementation details * ************************************************************************/ + +/** + * INTERNAL ONLY (using the same 'end' name might be considerd unfortunate + */ +template<> +BOOST_MPI_DECL void +communicator::send(int dest, int tag, + const packed_oarchive& ar) const; + +/** + * INTERNAL ONLY + */ +template<> +BOOST_MPI_DECL void +communicator::send + (int dest, int tag, const packed_skeleton_oarchive& ar) const; + +/** + * INTERNAL ONLY + */ +template<> +BOOST_MPI_DECL void +communicator::send(int dest, int tag, const content& c) const; + +/** + * INTERNAL ONLY + */ +template<> +BOOST_MPI_DECL status +communicator::recv(int source, int tag, + packed_iarchive& ar) const; + +/** + * INTERNAL ONLY + */ +template<> +BOOST_MPI_DECL status +communicator::recv + (int source, int tag, packed_skeleton_iarchive& ar) const; + +/** + * INTERNAL ONLY + */ +template<> +BOOST_MPI_DECL status +communicator::recv(int source, int tag, + const content& c) const; + +/** + * INTERNAL ONLY + */ +template<> +inline status +communicator::recv(int source, int tag, + content& c) const +{ + return recv(source,tag,c); +} + +/** + * INTERNAL ONLY + */ +template<> +BOOST_MPI_DECL request +communicator::isend(int dest, int tag, + const packed_oarchive& ar) const; + +/** + * INTERNAL ONLY + */ +template<> +BOOST_MPI_DECL request +communicator::isend + (int dest, int tag, const packed_skeleton_oarchive& ar) const; + +/** + * INTERNAL ONLY + */ +template<> +BOOST_MPI_DECL request +communicator::isend(int dest, int tag, const content& c) const; + +/** + * INTERNAL ONLY + */ +template<> +BOOST_MPI_DECL request +communicator::irecv + (int source, int tag, packed_skeleton_iarchive& ar) const; + +/** + * INTERNAL ONLY + */ +template<> +BOOST_MPI_DECL request +communicator::irecv(int source, int tag, + const content& c) const; + +/** + * INTERNAL ONLY + */ +template<> +inline request +communicator::irecv(int source, int tag, + content& c) const +{ + return irecv(source, tag, c); +} + // Count elements in a message template inline optional status::count() const @@ -1742,115 +1857,6 @@ request communicator::irecv(int source, int tag, T* values, int n) const return this->array_irecv_impl(source, tag, values, n, is_mpi_datatype()); } -/** - * INTERNAL ONLY - */ -template<> -BOOST_MPI_DECL void -communicator::send(int dest, int tag, - const packed_oarchive& ar) const; - -/** - * INTERNAL ONLY - */ -template<> -BOOST_MPI_DECL void -communicator::send - (int dest, int tag, const packed_skeleton_oarchive& ar) const; - -/** - * INTERNAL ONLY - */ -template<> -BOOST_MPI_DECL void -communicator::send(int dest, int tag, const content& c) const; - -/** - * INTERNAL ONLY - */ -template<> -BOOST_MPI_DECL status -communicator::recv(int source, int tag, - packed_iarchive& ar) const; - -/** - * INTERNAL ONLY - */ -template<> -BOOST_MPI_DECL status -communicator::recv - (int source, int tag, packed_skeleton_iarchive& ar) const; - -/** - * INTERNAL ONLY - */ -template<> -BOOST_MPI_DECL status -communicator::recv(int source, int tag, - const content& c) const; - -/** - * INTERNAL ONLY - */ -template<> -inline status -communicator::recv(int source, int tag, - content& c) const -{ - return recv(source,tag,c); -} - -/** - * INTERNAL ONLY - */ -template<> -BOOST_MPI_DECL request -communicator::isend(int dest, int tag, - const packed_oarchive& ar) const; - -/** - * INTERNAL ONLY - */ -template<> -BOOST_MPI_DECL request -communicator::isend - (int dest, int tag, const packed_skeleton_oarchive& ar) const; - -/** - * INTERNAL ONLY - */ -template<> -BOOST_MPI_DECL request -communicator::isend(int dest, int tag, const content& c) const; - -/** - * INTERNAL ONLY - */ -template<> -BOOST_MPI_DECL request -communicator::irecv - (int source, int tag, packed_skeleton_iarchive& ar) const; - -/** - * INTERNAL ONLY - */ -template<> -BOOST_MPI_DECL request -communicator::irecv(int source, int tag, - const content& c) const; - -/** - * INTERNAL ONLY - */ -template<> -inline request -communicator::irecv(int source, int tag, - content& c) const -{ - return irecv(source, tag, c); -} - - } } // end namespace boost::mpi // If the user has already included skeleton_and_content.hpp, include diff --git a/include/boost/mpi/detail/antiques.hpp b/include/boost/mpi/detail/antiques.hpp index 6ece779..0bd235b 100644 --- a/include/boost/mpi/detail/antiques.hpp +++ b/include/boost/mpi/detail/antiques.hpp @@ -13,17 +13,28 @@ // Support for some obsolette compilers namespace boost { namespace mpi { - namespace detail { - // Some old gnu compiler have no support for vector<>::data - // Use this in the mean time, the cumbersome syntax should - // serve as an incentive to get rid of this when those compilers - // are dropped. - template - T* c_data(std::vector& v) { return &(v[0]); } - - template - T const* c_data(std::vector const& v) { return &(v[0]); } - +namespace detail { + // Some old gnu compiler have no support for vector<>::data + // Use this in the mean time, the cumbersome syntax should + // serve as an incentive to get rid of this when those compilers + // are dropped. + template + T* c_data(std::vector& v) { return &(v[0]); } + + template + T const* c_data(std::vector const& v) { return &(v[0]); } + + // Some old MPI implementation (OpenMPI 1.6 for example) have non + // conforming API w.r.t. constness. + // We choose to fix this trhough this converter in order to + // explain/remember why we're doing this and remove it easilly + // when support for those MPI is dropped. + // The fix is as specific (un templatized, for one) as possible + // in order to encourage it usage for the probleme at hand. + // Problematic API include MPI_Send + inline + void *unconst(void const* addr) { return const_cast(addr); } + } } } #endif diff --git a/include/boost/mpi/detail/binary_buffer_oprimitive.hpp b/include/boost/mpi/detail/binary_buffer_oprimitive.hpp index 1de441d..313097b 100644 --- a/include/boost/mpi/detail/binary_buffer_oprimitive.hpp +++ b/include/boost/mpi/detail/binary_buffer_oprimitive.hpp @@ -47,7 +47,12 @@ class BOOST_MPI_DECL binary_buffer_oprimitive { return size_ = buffer_.size(); } - + + const std::size_t* size_ptr() const + { + return &size(); + } + void save_binary(void const *address, std::size_t count) { save_impl(address,count); diff --git a/include/boost/mpi/detail/mpi_datatype_primitive.hpp b/include/boost/mpi/detail/mpi_datatype_primitive.hpp index b2263fa..6a82624 100644 --- a/include/boost/mpi/detail/mpi_datatype_primitive.hpp +++ b/include/boost/mpi/detail/mpi_datatype_primitive.hpp @@ -21,6 +21,7 @@ namespace std{ #include #include +#include #include #include #include @@ -129,6 +130,12 @@ class mpi_datatype_primitive lengths.push_back(l); } + template + static T* get_data(std::vector& v) + { + return v.empty() ? 0 : &(v[0]); + } + std::vector addresses; std::vector types; std::vector lengths; diff --git a/include/boost/mpi/detail/offsets.hpp b/include/boost/mpi/detail/offsets.hpp new file mode 100644 index 0000000..7e5ab7d --- /dev/null +++ b/include/boost/mpi/detail/offsets.hpp @@ -0,0 +1,47 @@ +// Copyright Alain Miniussi 2014. +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +// Authors: Alain Miniussi + +#ifndef BOOST_MPI_OFFSETS_HPP +#define BOOST_MPI_OFFSETS_HPP + +#include +#include + +namespace boost { namespace mpi { +namespace detail { + +// Convert a sequence of sizes [S0..Sn] to a sequence displacement +// [O0..On] where O[0] = 0 and O[k+1] = O[k]+S[k]. +void sizes2offsets(int const* sizes, int* offsets, int n); + +// Same as size2offset(sizes.data(), offsets.data(), sizes.size()) +void sizes2offsets(std::vector const& sizes, std::vector& offsets); + +// Given a sequence of sizes (typically the number of records dispatched +// to each process in a scater) and a sequence of displacements (typically the +// slot index at with those record starts), convert the later to a number +// of skipped slots. +void offsets2skipped(int const* sizes, int const* offsets, int* skipped, int n); + +// Reconstruct offsets from sizes assuming no padding. +// Only takes place if on the root process and if +// displs are not already provided. +// If memory was allocated, returns a pointer to it +// otherwise null. +int* make_offsets(communicator const& comm, int const* sizes, int const* displs, int root = -1); + +// Reconstruct skip slots from sizes and offsets. +// Only takes place if on the root process and if +// displs are provided. +// If memory was allocated, returns a pointer to it +// otherwise null. +int* make_skipped_slots(communicator const& comm, int const* sizes, int const* displs, int root = -1); + +} +}}// end namespace boost::mpi + +#endif // BOOST_MPI_OFFSETS_HPP diff --git a/include/boost/mpi/detail/packed_iprimitive.hpp b/include/boost/mpi/detail/packed_iprimitive.hpp index 69d2c73..85a4010 100644 --- a/include/boost/mpi/detail/packed_iprimitive.hpp +++ b/include/boost/mpi/detail/packed_iprimitive.hpp @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/include/boost/mpi/detail/packed_oprimitive.hpp b/include/boost/mpi/detail/packed_oprimitive.hpp index 1cb4ba0..4ca8e07 100644 --- a/include/boost/mpi/detail/packed_oprimitive.hpp +++ b/include/boost/mpi/detail/packed_oprimitive.hpp @@ -12,7 +12,6 @@ #include #include // size_t #include - #include #include #include @@ -47,6 +46,11 @@ class BOOST_MPI_DECL packed_oprimitive return size_ = buffer_.size(); } + const std::size_t* size_ptr() const + { + return &size(); + } + void save_binary(void const *address, std::size_t count) { save_impl(address,MPI_BYTE,count); @@ -108,6 +112,11 @@ class BOOST_MPI_DECL packed_oprimitive buffer_.resize(position); } + static buffer_type::value_type* get_data(buffer_type& b) + { + return b.empty() ? 0 : &(b[0]); + } + buffer_type& buffer_; mutable std::size_t size_; MPI_Comm comm; diff --git a/include/boost/mpi/environment.hpp b/include/boost/mpi/environment.hpp index 92af129..cb90b6d 100644 --- a/include/boost/mpi/environment.hpp +++ b/include/boost/mpi/environment.hpp @@ -265,6 +265,12 @@ class BOOST_MPI_DECL environment : noncopyable { */ static bool is_main_thread(); + /** @brief MPI version. + * + * Returns a pair with the version and sub-version number. + */ + static std::pair version(); + private: /// Whether this environment object called MPI_Init bool i_initialized; diff --git a/include/boost/mpi/exception.hpp b/include/boost/mpi/exception.hpp index 1252307..154a431 100644 --- a/include/boost/mpi/exception.hpp +++ b/include/boost/mpi/exception.hpp @@ -15,6 +15,7 @@ #include #include +#include #include #include #include @@ -94,6 +95,7 @@ class BOOST_MPI_DECL exception : public std::exception #define BOOST_MPI_CHECK_RESULT( MPIFunc, Args ) \ { \ int _check_result = MPIFunc Args; \ + assert(_check_result == MPI_SUCCESS); \ if (_check_result != MPI_SUCCESS) \ boost::throw_exception(boost::mpi::exception(#MPIFunc, \ _check_result)); \ diff --git a/include/boost/mpi/python/serialize.hpp b/include/boost/mpi/python/serialize.hpp index 8933b34..feb703d 100644 --- a/include/boost/mpi/python/serialize.hpp +++ b/include/boost/mpi/python/serialize.hpp @@ -26,7 +26,6 @@ #include #include -#include #include #include @@ -37,6 +36,7 @@ #include #include #include +#include #include @@ -442,7 +442,7 @@ load_impl(Archiver& ar, boost::python::object& obj, int len; ar >> len; - std::auto_ptr string(new char[len]); + boost::scoped_array string(new char[len]); ar >> boost::serialization::make_array(string.get(), len); boost::python::str py_string(string.get(), len); obj = boost::python::pickle::loads(py_string); diff --git a/src/cartesian_communicator.cpp b/src/cartesian_communicator.cpp new file mode 100644 index 0000000..a46f0bc --- /dev/null +++ b/src/cartesian_communicator.cpp @@ -0,0 +1,179 @@ + +// Copyright Alain Miniussi 2014. +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +// Authors: Alain Miniussi + +#include +#include + +#include + +namespace boost { namespace mpi { + +namespace { + template + T* c_data(std::vector& v) { return &(v[0]); } +} + +std::ostream& +operator<<(std::ostream& out, cartesian_dimension const& d) { + out << '(' << d.size << ','; + if (d.periodic) { + out << "periodic"; + } else { + out << "bounded"; + } + out << ')'; + return out; +} + +std::ostream& +operator<<(std::ostream& out, cartesian_topology const& topo) { + out << '{'; + int const sz = topo.size(); + for (int i = 0; i < sz; ++i) { + out << topo[i]; + if ( i < (sz-1) ) { + out << ','; + } + } + out << '}'; + return out; +} + +cartesian_communicator::cartesian_communicator(const communicator& comm, + const cartesian_topology& topology, + bool reorder ) + : communicator(MPI_COMM_NULL, comm_attach) +{ + std::vector dims(topology.size()); + std::vector periodic(topology.size()); + int tsz = topology.size(); + for(int i = 0; i < tsz; ++i) { + dims[i] = topology[i].size; + periodic[i] = topology[i].periodic; + } + // Fill the gaps, if any + if (std::count(dims.begin(), dims.end(), 0) > 0) { + cartesian_dimensions(comm, dims); + } + MPI_Comm newcomm; + BOOST_MPI_CHECK_RESULT(MPI_Cart_create, + ((MPI_Comm)comm, dims.size(), + c_data(dims), c_data(periodic), + int(reorder), &newcomm)); + if(newcomm != MPI_COMM_NULL) { + comm_ptr.reset(new MPI_Comm(newcomm), comm_free()); + } +} + +cartesian_communicator::cartesian_communicator(const cartesian_communicator& comm, + const std::vector& keep ) + : communicator(MPI_COMM_NULL, comm_attach) +{ + int const max_dims = comm.ndims(); + int const nbkept = keep.size(); + assert(nbkept <= max_dims); + std::vector bitset(max_dims, int(false)); + for(int i = 0; i < nbkept; ++i) { + assert(keep[i] < max_dims); + bitset[keep[i]] = true; + } + + MPI_Comm newcomm; + BOOST_MPI_CHECK_RESULT(MPI_Cart_sub, + ((MPI_Comm)comm, c_data(bitset), &newcomm)); + if(newcomm != MPI_COMM_NULL) { + comm_ptr.reset(new MPI_Comm(newcomm), comm_free()); + } +} + +int +cartesian_communicator::ndims() const { + int n = -1; + BOOST_MPI_CHECK_RESULT(MPI_Cartdim_get, + (MPI_Comm(*this), &n)); + return n; +} + +int +cartesian_communicator::rank(const std::vector& coords ) const { + int r = -1; + assert(int(coords.size()) == ndims()); + BOOST_MPI_CHECK_RESULT(MPI_Cart_rank, + (MPI_Comm(*this), c_data(const_cast&>(coords)), + &r)); + return r; +} + +std::pair +cartesian_communicator::shifted_ranks(int dim, int disp) const { + std::pair r(-1,-1); + assert(0 <= dim && dim < ndims()); + BOOST_MPI_CHECK_RESULT(MPI_Cart_shift, + (MPI_Comm(*this), dim, disp, &(r.first), &(r.second))); + return r; +} + +std::vector +cartesian_communicator::coordinates(int rk) const { + std::vector cbuf(ndims()); + BOOST_MPI_CHECK_RESULT(MPI_Cart_coords, + (MPI_Comm(*this), rk, cbuf.size(), c_data(cbuf) )); + return cbuf; +} + +void +cartesian_communicator::topology( cartesian_topology& topo, + std::vector& coords ) const { + int ndims = this->ndims(); + topo.resize(ndims); + coords.resize(ndims); + std::vector cdims(ndims); + std::vector cperiods(ndims); + BOOST_MPI_CHECK_RESULT(MPI_Cart_get, + (MPI_Comm(*this), ndims, c_data(cdims), c_data(cperiods), c_data(coords))); + cartesian_topology res(cdims.begin(), cperiods.begin(), ndims); + topo.swap(res); +} + +cartesian_topology +cartesian_communicator::topology() const { + cartesian_topology topo(ndims()); + std::vector coords; + topology(topo, coords); + return topo; +} + +void +cartesian_topology::split(std::vector& dims, std::vector& periodics) const { + int ndims = size(); + dims.resize(ndims); + periodics.resize(ndims); + for(int i = 0; i < ndims; ++i) { + cartesian_dimension const& d = (*this)[i]; + dims[i] = d.size; + periodics[i] = d.periodic; + } +} + +std::vector& +cartesian_dimensions(int sz, std::vector& dims) { + int min = 1; + int const dimsz = dims.size(); + for(int i = 0; i < dimsz; ++i) { + if (dims[i] > 0) { + min *= dims[i]; + } + } + int leftover = sz % min; + + BOOST_MPI_CHECK_RESULT(MPI_Dims_create, + (sz-leftover, dims.size(), c_data(dims))); + return dims; +} + +} } // end namespace boost::mpi diff --git a/src/communicator.cpp b/src/communicator.cpp index 640feae..a172edd 100644 --- a/src/communicator.cpp +++ b/src/communicator.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -160,34 +161,52 @@ optional communicator::as_intercommunicator() const return optional(); } -optional communicator::as_graph_communicator() const +bool communicator::has_graph_topology() const { - optional graph; + bool is_graph = false; // topology test not allowed on MPI_NULL_COMM if (bool(*this)) { int status; BOOST_MPI_CHECK_RESULT(MPI_Topo_test, ((MPI_Comm)*this, &status)); - if (status == MPI_GRAPH) - graph = graph_communicator(comm_ptr); + is_graph = status == MPI_GRAPH; + } + return is_graph; +} + +optional communicator::as_graph_communicator() const +{ + if (has_graph_topology()) { + return graph_communicator(comm_ptr); + } else { + return optional(); } - return graph; } bool communicator::has_cartesian_topology() const { + bool is_cart = false; // topology test not allowed on MPI_NULL_COM - if (!bool(*this)) { - return false; - } else { + if (bool(*this)) { int status; BOOST_MPI_CHECK_RESULT(MPI_Topo_test, ((MPI_Comm)*this, &status)); - return status == MPI_CART; + is_cart = status == MPI_CART; + } + return is_cart; +} + +optional communicator::as_cartesian_communicator() const +{ + if (has_cartesian_topology()) { + return cartesian_communicator(comm_ptr); + } else { + return optional(); } } void communicator::abort(int errcode) const { BOOST_MPI_CHECK_RESULT(MPI_Abort, (MPI_Comm(*this), errcode)); + std::abort(); } /************************************************************* diff --git a/src/environment.cpp b/src/environment.cpp index c1bf9e2..ffdfc90 100644 --- a/src/environment.cpp +++ b/src/environment.cpp @@ -247,4 +247,11 @@ bool environment::is_main_thread() return static_cast(isit); } +std::pair environment::version() +{ + int major, minor; + BOOST_MPI_CHECK_RESULT(MPI_Get_version, (&major, &minor)); + return std::make_pair(major, minor); +} + } } // end namespace boost::mpi diff --git a/src/offsets.cpp b/src/offsets.cpp new file mode 100644 index 0000000..2382d45 --- /dev/null +++ b/src/offsets.cpp @@ -0,0 +1,96 @@ +// Copyright Alain Miniussi 2014. +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +// Authors: Alain Miniussi + +#include +#include + +namespace boost { namespace mpi { +namespace detail { + +// Convert a sequence of sizes [S0..Sn] to a sequence displacement +// [O0..On] where O[0] = 0 and O[k+1] = O[k]+S[k]. +void +sizes2offsets(int const* sizes, int* offsets, int n) +{ + offsets[0] = 0; + for(int i = 1; i < n; ++i) { + offsets[i] = offsets[i-1] + sizes[i-1]; + } +} + +// Convert a sequence of sizes [S0..Sn] to a sequence displacement +// [O0..On] where O[0] = 0 and O[k+1] = O[k]+S[k]. +void +sizes2offsets(std::vector const& sizes, std::vector& offsets) +{ + int sz = sizes.size(); + offsets.resize(sz); + sizes2offsets(c_data(sizes), c_data(offsets), sz); +} + +// Given a sequence of sizes (typically the number of records dispatched +// to each process in a scater) and a sequence of displacements (typically the +// slot index at with those record starts), convert the later to a number +// of skipped slots. +void +offsets2skipped(int const* sizes, int const* offsets, int* skipped, int n) +{ + skipped[0] = 0; + for(int i = 1; i < n; ++i) { + skipped[i] -= offsets[i-1] + sizes[i-1]; + } +} + +// Reconstruct offsets from sizes assuming no padding. +// Only takes place if on the root process and if +// displs are not already provided. +// If memory was allocated, returns a pointer to it +// otherwise null. +int* +make_offsets(communicator const& comm, int const* sizes, int const* displs, int root) +{ + if (root == -1 || root == comm.rank()) { + assert(sizes); + if (!displs) { + int nproc = comm.size(); + int* offsets = new int[nproc]; + displs = offsets; + sizes2offsets(sizes, offsets, nproc); + return offsets; + } else { + return 0; + } + } else { + return 0; + } +} + +// Reconstruct skip slots from sizes and offsets. +// Only takes place if on the root process and if +// displs are provided. +// If memory was allocated, returns a pointer to it +// otherwise null. +int* +make_skipped_slots(communicator const& comm, int const* sizes, int const* displs, int root) +{ + if (root == -1 || root == comm.rank()) { + assert(sizes); + if (displs) { + int nproc = comm.size(); + int* skipped = new int[nproc]; + std::copy(displs, displs+nproc, skipped); + offsets2skipped(sizes, displs, skipped, nproc); + return skipped; + } else { + return 0; + } + } else { + return 0; + } +} +} +}} diff --git a/src/point_to_point.cpp b/src/point_to_point.cpp index 7822152..7b353f7 100644 --- a/src/point_to_point.cpp +++ b/src/point_to_point.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include namespace boost { namespace mpi { namespace detail { @@ -28,13 +29,13 @@ void packed_archive_send(MPI_Comm comm, int dest, int tag, const packed_oarchive& ar) { - std::size_t size = ar.size(); + std::size_t const& size = ar.size(); BOOST_MPI_CHECK_RESULT(MPI_Send, - (static_cast(&size), 1, - get_mpi_datatype(ar.size()), + (detail::unconst(&size), 1, + get_mpi_datatype(size), dest, tag, comm)); BOOST_MPI_CHECK_RESULT(MPI_Send, - (const_cast(ar.address()), ar.size(), + (detail::unconst(ar.address()), size, MPI_PACKED, dest, tag, comm)); } @@ -45,13 +46,13 @@ packed_archive_isend(MPI_Comm comm, int dest, int tag, MPI_Request* out_requests, int num_out_requests) { assert(num_out_requests >= 2); - const void* size = &ar.size(); + std::size_t const& size = ar.size(); BOOST_MPI_CHECK_RESULT(MPI_Isend, - (const_cast(size), 1, - get_mpi_datatype(ar.size()), + (detail::unconst(&size), 1, + get_mpi_datatype(size), dest, tag, comm, out_requests)); BOOST_MPI_CHECK_RESULT(MPI_Isend, - (const_cast(ar.address()), ar.size(), + (detail::unconst(ar.address()), size, MPI_PACKED, dest, tag, comm, out_requests + 1)); @@ -65,13 +66,13 @@ packed_archive_isend(MPI_Comm comm, int dest, int tag, { assert(num_out_requests >= 2); - const void* size = &ar.size(); + std::size_t const& size = ar.size(); BOOST_MPI_CHECK_RESULT(MPI_Isend, - (const_cast(size), 1, - get_mpi_datatype(ar.size()), + (detail::unconst(&size), 1, + get_mpi_datatype(size), dest, tag, comm, out_requests)); BOOST_MPI_CHECK_RESULT(MPI_Isend, - (const_cast(ar.address()), ar.size(), + (detail::unconst(ar.address()), size, MPI_PACKED, dest, tag, comm, out_requests + 1)); @@ -84,13 +85,13 @@ packed_archive_recv(MPI_Comm comm, int source, int tag, packed_iarchive& ar, { std::size_t count; BOOST_MPI_CHECK_RESULT(MPI_Recv, - (&count, 1, get_mpi_datatype(count), + (&count, 1, get_mpi_datatype(count), source, tag, comm, &status)); // Prepare input buffer and receive the message ar.resize(count); BOOST_MPI_CHECK_RESULT(MPI_Recv, - (ar.address(), ar.size(), MPI_PACKED, + (ar.address(), count, MPI_PACKED, status.MPI_SOURCE, status.MPI_TAG, comm, &status)); } diff --git a/test/Jamfile.v2 b/test/Jamfile.v2 index 35a522d..46b0e84 100644 --- a/test/Jamfile.v2 +++ b/test/Jamfile.v2 @@ -17,12 +17,17 @@ if [ mpi.configured ] { test-suite mpi : + [ mpi-test version_test : : : 1 ] + [ mpi-test random_gather : ../example/random_gather.cpp : : 2 ] + [ mpi-test random_scatter : ../example/random_scatter.cpp : : 2 ] + [ mpi-test cartesian_communicator : ../example/cartesian_communicator.cpp : : 24 ] + [ mpi-test cartesian_topology_init_test : : : 1 ] [ mpi-test broadcast_stl_test : : : 2 ] - [ mpi-test all_gather_test ] - [ mpi-test all_reduce_test ] - [ mpi-test all_to_all_test ] + [ mpi-test all_gather_test : : : 1 2 11 ] + [ mpi-test all_reduce_test : : : 1 2 11 ] + [ mpi-test all_to_all_test : : : 1 2 11 ] [ mpi-test broadcast_test : : : 2 17 ] - [ mpi-test gather_test ] + [ mpi-test gather_test : : : 1 2 11 ] [ mpi-test is_mpi_op_test : : : 1 ] [ mpi-test mt_level_test : : : 1 ] [ mpi-test mt_init_test-single : mt_init_test.cpp : "single" : 1 4 ] @@ -30,7 +35,7 @@ test-suite mpi [ mpi-test mt_init_test-serialized : mt_init_test.cpp : "serialized" : 1 4 ] [ mpi-test mt_init_test-multiple : mt_init_test.cpp : "multiple" : 1 4 ] # Note: Microsoft MPI fails nonblocking_test on 1 processor - [ mpi-test nonblocking_test ] + [ mpi-test nonblocking_test : : : 2 11 24 ] [ mpi-test reduce_test ] [ mpi-test ring_test : : : 2 3 4 7 8 13 17 ] [ mpi-test sendrecv_test : : : 1 4 7 48 ] @@ -39,6 +44,7 @@ test-suite mpi # Note: Microsoft MPI fails all skeleton-content tests [ mpi-test skeleton_content_test : : : 2 3 4 7 8 13 17 ] [ mpi-test graph_topology_test : : : 2 7 13 ] + [ mpi-test cartesian_topology_test : : : 24 ] [ mpi-test pointer_test : : : 2 ] [ mpi-test groups_test ] # tests that require -std=c++11 diff --git a/test/all_gather_test.cpp b/test/all_gather_test.cpp index a0543fd..1bd4da9 100644 --- a/test/all_gather_test.cpp +++ b/test/all_gather_test.cpp @@ -5,39 +5,38 @@ // http://www.boost.org/LICENSE_1_0.txt) // A test of the all_gather() collective. + +#include + #include +#include #include #include #include -#include -#include "gps_position.hpp" #include #include #include #include -using boost::mpi::communicator; +#include "gps_position.hpp" -using boost::mpi::packed_skeleton_iarchive; -using boost::mpi::packed_skeleton_oarchive; +namespace mpi = boost::mpi; template void -all_gather_test(const communicator& comm, Generator generator, - const char* kind) +all_gather_test(const mpi::communicator& comm, Generator generator, + std::string kind) { typedef typename Generator::result_type value_type; value_type value = generator(comm.rank()); - using boost::mpi::all_gather; - std::vector values; if (comm.rank() == 0) { std::cout << "Gathering " << kind << "..."; std::cout.flush(); } - all_gather(comm, value, values); + mpi::all_gather(comm, value, values); std::vector expected_values; for (int p = 0; p < comm.size(); ++p) @@ -49,6 +48,41 @@ all_gather_test(const communicator& comm, Generator generator, (comm.barrier)(); } +template +void +all_gatherv_test(const mpi::communicator& comm, Generator generator, + std::string kind) +{ + typedef typename Generator::result_type value_type; + using boost::mpi::all_gatherv; + + std::vector myvalues, expected, values; + std::vector sizes; + for(int r = 0; r < comm.size(); ++r) { + value_type value = generator(r); + sizes.push_back(r+1); + for (int k=0; k < r+1; ++k) { + expected.push_back(value); + if(comm.rank() == r) { + myvalues.push_back(value); + } + } + } + if (comm.rank() == 0) { + std::cout << "Gathering " << kind << "..."; + std::cout.flush(); + } + + mpi::all_gatherv(comm, myvalues, values, sizes); + + BOOST_CHECK(values == expected); + + if (comm.rank() == 0 && values == expected) + std::cout << "OK." << std::endl; + + (comm.barrier)(); +} + // Generates integers to test with gather() struct int_generator { @@ -99,10 +133,15 @@ struct string_list_generator int test_main(int argc, char* argv[]) { boost::mpi::environment env(argc, argv); - communicator comm; + mpi::communicator comm; all_gather_test(comm, int_generator(), "integers"); all_gather_test(comm, gps_generator(), "GPS positions"); all_gather_test(comm, string_generator(), "string"); all_gather_test(comm, string_list_generator(), "list of strings"); + + all_gatherv_test(comm, int_generator(), "integers"); + all_gatherv_test(comm, gps_generator(), "GPS positions"); + all_gatherv_test(comm, string_generator(), "string"); + all_gatherv_test(comm, string_list_generator(), "list of strings"); return 0; } diff --git a/test/cartesian_topology_init_test.cpp b/test/cartesian_topology_init_test.cpp new file mode 100644 index 0000000..9da72ea --- /dev/null +++ b/test/cartesian_topology_init_test.cpp @@ -0,0 +1,88 @@ + +// Copyright Alain Miniussi 2014. +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +// Authors: Alain Miniussi + +#include +#include +#include +#include +#include +#include +#include + +#define BOOST_TEST_MODULE BoostMPI +#include + +#include +#include +#include +#include +#include + +namespace mpi = boost::mpi; + +BOOST_AUTO_TEST_CASE(cartesian_dimension_init) +{ + // Curly brace initialization syntax not supported on (very) old gnu + // This typedef keeps things shorter + typedef mpi::cartesian_dimension cd; + + { + // Check the basic ctor + mpi::cartesian_dimension def; + mpi::cartesian_topology t1(10); + BOOST_CHECK(t1.stl() == std::vector(10, def)); + } +#if !defined(BOOST_NO_CXX11_HDR_INITIALIZER_LIST) + { + // Intializer list ctor vs range based + int dims[] = {2,3,4}; + bool per[] = {true, false, true}; + mpi::cartesian_topology t1(dims, per); + mpi::cartesian_topology t2({{2,true},{3, false},{4, true}}); + BOOST_CHECK(t1.size() == 3); + BOOST_CHECK(t1 == t2); + } +#endif + // Container based ctor only available as a replacement for initializer list ctor + { + // seq ctor vs C array ctor + mpi::cartesian_dimension d[] = {cd(2,true),cd(3, false),cd(4, true)}; + std::list seq; + std::copy(d, d+3, std::back_inserter(seq)); + mpi::cartesian_topology t1(seq); + mpi::cartesian_topology t2(d); + BOOST_CHECK(t1 == t2); + } + { + // Check range based with array based ctor. + boost::array d = {{cd(2,true),cd(3, false),cd(4, true)}}; + int dims[] = {2,3,4}; + bool per[] = {true, false, true}; + mpi::cartesian_topology t1(dims, per); + mpi::cartesian_topology t2(d); + BOOST_CHECK(t1.size() == 3); + BOOST_CHECK(t1 == t2); + } + { + // Iterator based ctor vs C array based ctor + mpi::cartesian_dimension d[] = {cd(2,true),cd(3, false),cd(4, true)}; + std::vector vdims(d, d+3); + mpi::cartesian_topology t1(vdims); + mpi::cartesian_topology t2(d); + BOOST_CHECK(t1.size() == 3); + BOOST_CHECK(t1 == t2); + BOOST_CHECK(!(t1 != t2)); + t1[1].periodic = true; + BOOST_CHECK(t1 != t2); + t1[2].periodic = false; + t1[2].size = 0; + vdims.push_back(mpi::cartesian_dimension(3, false)); + mpi::cartesian_topology t3(vdims); + BOOST_CHECK(t1 != t3); + } +} diff --git a/test/cartesian_topology_test.cpp b/test/cartesian_topology_test.cpp new file mode 100644 index 0000000..f19d72a --- /dev/null +++ b/test/cartesian_topology_test.cpp @@ -0,0 +1,187 @@ +// Copyright Alain Miniussi 2014. +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +// Authors: Alain Miniussi + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include + +namespace mpi = boost::mpi; + +struct topo_minimum { + mpi::cartesian_dimension + operator()(mpi::cartesian_dimension const& d1, + mpi::cartesian_dimension const& d2 ) const { + return mpi::cartesian_dimension(std::min(d1.size, d2.size), + d1.periodic && d2.periodic); + } +}; + +std::string topology_description( mpi::cartesian_topology const& topo ) { + std::ostringstream out; + std::copy(topo.begin(), topo.end(), std::ostream_iterator(out, " ")); + out << std::flush; + return out.str(); +} + +// Check that everyone agrees on the coordinates +void test_coordinates_consistency( mpi::cartesian_communicator const& cc, + std::vector const& coords ) +{ + cc.barrier(); // flush IOs for nice printing + bool master = cc.rank() == 0; + if (master) { + std::cout << "Test coordinates consistency.\n"; + } + for(int p = 0; p < cc.size(); ++p) { + std::vector min(cc.ndims()); + std::vector local(cc.coordinates(p)); + mpi::reduce(cc, &local.front(), local.size(), + &(min[0]), mpi::minimum(), p); + cc.barrier(); + if (p == cc.rank()) { + BOOST_CHECK(std::equal(coords.begin(), coords.end(), min.begin())); + std::ostringstream out; + out << "proc " << p << " at ("; + std::copy(min.begin(), min.end(), std::ostream_iterator(out, " ")); + out << ")\n"; + std::cout << out.str(); + } + } +} + +void test_shifted_coords( mpi::cartesian_communicator const& cc, int pos, mpi::cartesian_dimension desc, int dim ) +{ + if (desc.periodic) { + for (int i = -(desc.size); i < desc.size; ++i) { + std::pair rks = cc.shifted_ranks(dim, i); + int src = cc.coordinates(rks.first)[dim]; + int dst = cc.coordinates(rks.second)[dim]; + if (pos == (dim/2)) { + std::ostringstream out; + out << "Rank " << cc.rank() << ", dim. " << dim << ", pos " << pos << ", in " << desc << ' '; + out << "shifted pos: " << src << ", " << dst << '\n'; + std::cout << out.str(); + } + } + } +} + +void test_shifted_coords( mpi::cartesian_communicator const& cc) +{ + cc.barrier(); // flush IOs for nice printing + std::vector coords; + mpi::cartesian_topology topo(cc.ndims()); + cc.topology(topo, coords); + bool master = cc.rank() == 0; + if (master) { + std::cout << "Testing shifts with topology " << topo << '\n'; + } + for(int i = 0; i < cc.ndims(); ++i) { + if (master) { + std::cout << " for dimension " << i << ": " << topo[i] << '\n'; + } + test_shifted_coords( cc, coords[i], topo[i], i ); + } +} + +void test_topology_consistency( mpi::cartesian_communicator const& cc) +{ + cc.barrier(); // flush IOs for nice printing + mpi::cartesian_topology itopo(cc.ndims()); + mpi::cartesian_topology otopo(cc.ndims()); + std::vector coords(cc.ndims()); + cc.topology(itopo, coords); + bool master = cc.rank() == 0; + if (master) { + std::cout << "Test topology consistency of" << itopo << "(on master)\n"; + std::cout << "Check that everyone agrees on the dimensions.\n"; + } + mpi::all_reduce(cc, + &(itopo[0]), itopo.size(), &(otopo[0]), + topo_minimum()); + BOOST_CHECK(std::equal(itopo.begin(), itopo.end(), otopo.begin())); + if (master) { + std::cout << "We agree on " << topology_description(otopo) << '\n'; + } + test_coordinates_consistency( cc, coords ); +} + +void test_cartesian_topology( mpi::cartesian_communicator const& cc) +{ + BOOST_CHECK(cc.has_cartesian_topology()); + for( int r = 0; r < cc.size(); ++r) { + cc.barrier(); + if (r == cc.rank()) { + std::vector coords = cc.coordinates(r); + std::cout << "Process of cartesian rank " << cc.rank() + << " has coordinates ("; + std::copy(coords.begin(), coords.end(), std::ostream_iterator(std::cout," ")); + std::cout << ")\n"; + } + } + test_topology_consistency(cc); + test_shifted_coords(cc); + std::vector even; + for(int i = 0; i < cc.ndims(); i += 2) { + even.push_back(i); + } + cc.barrier(); + mpi::cartesian_communicator cce(cc, even); +} + +void test_cartesian_topology( mpi::communicator const& world, mpi::cartesian_topology const& topo) +{ + mpi::cartesian_communicator cc(world, topo, true); + if (cc) { + BOOST_CHECK(cc.has_cartesian_topology()); + BOOST_CHECK(cc.ndims() == int(topo.size())); + if (cc.rank() == 0) { + std::cout << "Asked topology " << topo << ", got " << cc.topology() << '\n'; + } + test_cartesian_topology(cc); + } else { + std::ostringstream out; + out << world.rank() << " was left outside the cartesian grid\n"; + std::cout << out.str(); + } +} + +int test_main(int argc, char* argv[]) +{ + mpi::environment env(argc, argv); + + mpi::communicator world; + int const ndim = world.size() >= 24 ? 3 : 2; + mpi::cartesian_topology topo(ndim); + typedef mpi::cartesian_dimension cd; + if (topo.size() == 3) { + topo[0] = cd(2,true); + topo[1] = cd(3,false); + topo[2] = cd(4, true); + } else { + if (world.size() >= 6) { + topo[0] = cd(2,true); + topo[1] = cd(3, false); + } else { + topo[0] = cd(1,true); + topo[1] = cd(1, false); + } + } + test_cartesian_topology( world, topo); + + return 0; +} diff --git a/test/gps_position.hpp b/test/gps_position.hpp index 109a3a9..e6910c4 100644 --- a/test/gps_position.hpp +++ b/test/gps_position.hpp @@ -10,6 +10,7 @@ #include #include #include +#include class gps_position { @@ -32,6 +33,8 @@ class gps_position degrees(d), minutes(m), seconds(s) {} + friend std::ostream& operator<<(std::ostream& out, const gps_position& g); + friend bool operator==(const gps_position& x, const gps_position& y) { return (x.degrees == y.degrees @@ -45,6 +48,11 @@ class gps_position } }; +inline +std::ostream& operator<<(std::ostream& out, const gps_position& g) { + out << "gps{" << g.degrees << 'd' << g.minutes << 'm' << g.seconds << "s}"; + return out; +} namespace boost { namespace mpi { diff --git a/test/scatter_test.cpp b/test/scatter_test.cpp index 4e91d8d..5e5ecd2 100644 --- a/test/scatter_test.cpp +++ b/test/scatter_test.cpp @@ -5,6 +5,7 @@ // http://www.boost.org/LICENSE_1_0.txt) // A test of the scatter() and scatterv() collectives. +#include #include #include #include @@ -54,49 +55,6 @@ scatter_test(const communicator& comm, Generator generator, } -template -void -scatterv_test(const communicator& comm, Generator generator, - const char* kind, int root = -1) -{ - typedef typename Generator::result_type value_type; - - if (root == -1) { - for (root = 0; root < comm.size(); ++root) - scatterv_test(comm, generator, kind, root); - } else { - using boost::mpi::scatterv; - - int mysize = comm.rank() + 1; - std::vector myvalues(mysize); - - if (comm.rank() == root) { - std::vector values; - std::vector sizes(comm.size()); - - // process p will receive p+1 identical generator(p) elements - for (int p = 0; p < comm.size(); ++p) { - for (int i = 0; i < p+1; ++i) - values.push_back(generator(p)); - sizes[p] = p + 1; - } - - std::cout << "Scatteringv " << kind << " from root " - << root << "..." << std::endl; - - scatterv(comm, values, sizes, &myvalues[0], root); - } else { - scatterv(comm, &myvalues[0], mysize, root); - } - - for (int i = 0; i < mysize; ++i) - BOOST_CHECK(myvalues[i] == generator(comm.rank())); - } - - (comm.barrier)(); -} - - // // Generators to test with scatter/scatterv // @@ -145,6 +103,57 @@ struct string_list_generator } }; +std::ostream& +operator<<(std::ostream& out, std::list const& l) { + out << '['; + std::copy(l.begin(), l.end(), std::ostream_iterator(out, " ")); + out << ']'; + return out; +} + +template +void +scatterv_test(const communicator& comm, Generator generator, + const char* kind, int root = -1) +{ + typedef typename Generator::result_type value_type; + + if (root == -1) { + for (root = 0; root < comm.size(); ++root) + scatterv_test(comm, generator, kind, root); + } else { + using boost::mpi::scatterv; + + int mysize = comm.rank() + 1; + std::vector myvalues(mysize); + + if (comm.rank() == root) { + std::vector values; + std::vector sizes(comm.size()); + + // process p will receive p+1 identical generator(p) elements + for (int p = 0; p < comm.size(); ++p) { + for (int i = 0; i < p+1; ++i) + values.push_back(generator(p)); + sizes[p] = p + 1; + } + + std::cout << "Scatteringv " << kind << " from root " + << root << "..." << std::endl; + assert(mysize == sizes[comm.rank()]); + scatterv(comm, values, sizes, &(myvalues[0]), root); + } else { + scatterv(comm, &(myvalues[0]), mysize, root); + } + + for (int i = 0; i < mysize; ++i) + BOOST_CHECK(myvalues[i] == generator(comm.rank())); + } + + (comm.barrier)(); +} + + int test_main(int argc, char* argv[]) { boost::mpi::environment env(argc, argv); diff --git a/test/version_test.cpp b/test/version_test.cpp new file mode 100644 index 0000000..5ebd907 --- /dev/null +++ b/test/version_test.cpp @@ -0,0 +1,32 @@ +// Copyright (C) 2013 Alain Miniussi + +// Use, modification and distribution is subject to the Boost Software +// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +// test mpi version + +#include +#include +#include + +namespace mpi = boost::mpi; + +int +test_main(int argc, char* argv[]) { +#if defined(MPI_VERSION) + int mpi_version = MPI_VERSION; + int mpi_subversion = MPI_SUBVERSION; +#else + int mpi_version = 0; + int mpi_subversion = 0; +#endif + + mpi::environment env(argc,argv); + std::pair version = env.version(); + std::cout << "MPI Version: " << version.first << ',' << version.second << '\n'; + + BOOST_CHECK(version.first == mpi_version); + BOOST_CHECK(version.second == mpi_subversion); + return 0; +}