// Copyright (C) 2006 Douglas Gregor <doug.gregor -at- gmail.com>. | |
// Use, modification and distribution is subject to the Boost Software | |
// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at | |
// http://www.boost.org/LICENSE_1_0.txt) | |
/** @file nonblocking.hpp | |
* | |
* This header defines operations for completing non-blocking | |
* communication requests. | |
*/ | |
#ifndef BOOST_MPI_NONBLOCKING_HPP | |
#define BOOST_MPI_NONBLOCKING_HPP | |
#include <boost/mpi/config.hpp> | |
#include <vector> | |
#include <iterator> // for std::iterator_traits | |
#include <boost/optional.hpp> | |
#include <utility> // for std::pair | |
#include <algorithm> // for iter_swap, reverse | |
#include <boost/static_assert.hpp> | |
#include <boost/mpi/request.hpp> | |
#include <boost/mpi/status.hpp> | |
#include <boost/mpi/exception.hpp> | |
namespace boost { namespace mpi { | |
/** | |
* @brief Wait until any non-blocking request has completed. | |
* | |
* This routine takes in a set of requests stored in the iterator | |
* range @c [first,last) and waits until any of these requests has | |
* been completed. It provides functionality equivalent to | |
* @c MPI_Waitany. | |
* | |
* @param first The iterator that denotes the beginning of the | |
* sequence of request objects. | |
* | |
* @param last The iterator that denotes the end of the sequence of | |
* request objects. This may not be equal to @c first. | |
* | |
* @returns A pair containing the status object that corresponds to | |
* the completed operation and the iterator referencing the completed | |
* request. | |
*/ | |
template<typename ForwardIterator> | |
std::pair<status, ForwardIterator> | |
wait_any(ForwardIterator first, ForwardIterator last) | |
{ | |
using std::advance; | |
BOOST_ASSERT(first != last); | |
typedef typename std::iterator_traits<ForwardIterator>::difference_type | |
difference_type; | |
bool all_trivial_requests = true; | |
difference_type n = 0; | |
ForwardIterator current = first; | |
while (true) { | |
// Check if we have found a completed request. If so, return it. | |
if (optional<status> result = current->test()) | |
return std::make_pair(*result, current); | |
// Check if this request (and all others before it) are "trivial" | |
// requests, e.g., they can be represented with a single | |
// MPI_Request. | |
all_trivial_requests = | |
all_trivial_requests | |
&& !current->m_handler | |
&& current->m_requests[1] == MPI_REQUEST_NULL; | |
// Move to the next request. | |
++n; | |
if (++current == last) { | |
// We have reached the end of the list. If all requests thus far | |
// have been trivial, we can call MPI_Waitany directly, because | |
// it may be more efficient than our busy-wait semantics. | |
if (all_trivial_requests) { | |
std::vector<MPI_Request> requests; | |
requests.reserve(n); | |
for (current = first; current != last; ++current) | |
requests.push_back(current->m_requests[0]); | |
// Let MPI wait until one of these operations completes. | |
int index; | |
status stat; | |
BOOST_MPI_CHECK_RESULT(MPI_Waitany, | |
(n, &requests[0], &index, &stat.m_status)); | |
// We don't have a notion of empty requests or status objects, | |
// so this is an error. | |
if (index == MPI_UNDEFINED) | |
boost::throw_exception(exception("MPI_Waitany", MPI_ERR_REQUEST)); | |
// Find the iterator corresponding to the completed request. | |
current = first; | |
advance(current, index); | |
current->m_requests[0] = requests[index]; | |
return std::make_pair(stat, current); | |
} | |
// There are some nontrivial requests, so we must continue our | |
// busy waiting loop. | |
n = 0; | |
current = first; | |
all_trivial_requests = true; | |
} | |
} | |
// We cannot ever get here | |
BOOST_ASSERT(false); | |
} | |
/** | |
* @brief Test whether any non-blocking request has completed. | |
* | |
* This routine takes in a set of requests stored in the iterator | |
* range @c [first,last) and tests whether any of these requests has | |
* been completed. This routine is similar to @c wait_any, but will | |
* not block waiting for requests to completed. It provides | |
* functionality equivalent to @c MPI_Testany. | |
* | |
* @param first The iterator that denotes the beginning of the | |
* sequence of request objects. | |
* | |
* @param last The iterator that denotes the end of the sequence of | |
* request objects. | |
* | |
* @returns If any outstanding requests have completed, a pair | |
* containing the status object that corresponds to the completed | |
* operation and the iterator referencing the completed | |
* request. Otherwise, an empty @c optional<>. | |
*/ | |
template<typename ForwardIterator> | |
optional<std::pair<status, ForwardIterator> > | |
test_any(ForwardIterator first, ForwardIterator last) | |
{ | |
for (ForwardIterator current = first; first != last; ++first) { | |
// Check if we have found a completed request. If so, return it. | |
if (optional<status> result = current->test()) | |
return std::make_pair(*result, current); | |
} | |
// We found nothing | |
return optional<std::pair<status, ForwardIterator> >(); | |
} | |
/** | |
* @brief Wait until all non-blocking requests have completed. | |
* | |
* This routine takes in a set of requests stored in the iterator | |
* range @c [first,last) and waits until all of these requests have | |
* been completed. It provides functionality equivalent to | |
* @c MPI_Waitall. | |
* | |
* @param first The iterator that denotes the beginning of the | |
* sequence of request objects. | |
* | |
* @param last The iterator that denotes the end of the sequence of | |
* request objects. | |
* | |
* @param out If provided, an output iterator through which the | |
* status of each request will be emitted. The @c status objects are | |
* emitted in the same order as the requests are retrieved from | |
* @c [first,last). | |
* | |
* @returns If an @p out parameter was provided, the value @c out | |
* after all of the @c status objects have been emitted. | |
*/ | |
template<typename ForwardIterator, typename OutputIterator> | |
OutputIterator | |
wait_all(ForwardIterator first, ForwardIterator last, OutputIterator out) | |
{ | |
typedef typename std::iterator_traits<ForwardIterator>::difference_type | |
difference_type; | |
using std::distance; | |
difference_type num_outstanding_requests = distance(first, last); | |
std::vector<status> results(num_outstanding_requests); | |
std::vector<bool> completed(num_outstanding_requests); | |
while (num_outstanding_requests > 0) { | |
bool all_trivial_requests = true; | |
difference_type idx = 0; | |
for (ForwardIterator current = first; current != last; ++current, ++idx) { | |
if (!completed[idx]) { | |
if (optional<status> stat = current->test()) { | |
// This outstanding request has been completed. We're done. | |
results[idx] = *stat; | |
completed[idx] = true; | |
--num_outstanding_requests; | |
all_trivial_requests = false; | |
} else { | |
// Check if this request (and all others before it) are "trivial" | |
// requests, e.g., they can be represented with a single | |
// MPI_Request. | |
all_trivial_requests = | |
all_trivial_requests | |
&& !current->m_handler | |
&& current->m_requests[1] == MPI_REQUEST_NULL; | |
} | |
} | |
} | |
// If we have yet to fulfill any requests and all of the requests | |
// are trivial (i.e., require only a single MPI_Request to be | |
// fulfilled), call MPI_Waitall directly. | |
if (all_trivial_requests | |
&& num_outstanding_requests == (difference_type)results.size()) { | |
std::vector<MPI_Request> requests; | |
requests.reserve(num_outstanding_requests); | |
for (ForwardIterator current = first; current != last; ++current) | |
requests.push_back(current->m_requests[0]); | |
// Let MPI wait until all of these operations completes. | |
std::vector<MPI_Status> stats(num_outstanding_requests); | |
BOOST_MPI_CHECK_RESULT(MPI_Waitall, | |
(num_outstanding_requests, &requests[0], | |
&stats[0])); | |
for (std::vector<MPI_Status>::iterator i = stats.begin(); | |
i != stats.end(); ++i, ++out) { | |
status stat; | |
stat.m_status = *i; | |
*out = stat; | |
} | |
return out; | |
} | |
all_trivial_requests = false; | |
} | |
return std::copy(results.begin(), results.end(), out); | |
} | |
/** | |
* \overload | |
*/ | |
template<typename ForwardIterator> | |
void | |
wait_all(ForwardIterator first, ForwardIterator last) | |
{ | |
typedef typename std::iterator_traits<ForwardIterator>::difference_type | |
difference_type; | |
using std::distance; | |
difference_type num_outstanding_requests = distance(first, last); | |
std::vector<bool> completed(num_outstanding_requests); | |
while (num_outstanding_requests > 0) { | |
bool all_trivial_requests = true; | |
difference_type idx = 0; | |
for (ForwardIterator current = first; current != last; ++current, ++idx) { | |
if (!completed[idx]) { | |
if (optional<status> stat = current->test()) { | |
// This outstanding request has been completed. | |
completed[idx] = true; | |
--num_outstanding_requests; | |
all_trivial_requests = false; | |
} else { | |
// Check if this request (and all others before it) are "trivial" | |
// requests, e.g., they can be represented with a single | |
// MPI_Request. | |
all_trivial_requests = | |
all_trivial_requests | |
&& !current->m_handler | |
&& current->m_requests[1] == MPI_REQUEST_NULL; | |
} | |
} | |
} | |
// If we have yet to fulfill any requests and all of the requests | |
// are trivial (i.e., require only a single MPI_Request to be | |
// fulfilled), call MPI_Waitall directly. | |
if (all_trivial_requests | |
&& num_outstanding_requests == (difference_type)completed.size()) { | |
std::vector<MPI_Request> requests; | |
requests.reserve(num_outstanding_requests); | |
for (ForwardIterator current = first; current != last; ++current) | |
requests.push_back(current->m_requests[0]); | |
// Let MPI wait until all of these operations completes. | |
BOOST_MPI_CHECK_RESULT(MPI_Waitall, | |
(num_outstanding_requests, &requests[0], | |
MPI_STATUSES_IGNORE)); | |
// Signal completion | |
num_outstanding_requests = 0; | |
} | |
} | |
} | |
/** | |
* @brief Tests whether all non-blocking requests have completed. | |
* | |
* This routine takes in a set of requests stored in the iterator | |
* range @c [first,last) and determines whether all of these requests | |
* have been completed. However, due to limitations of the underlying | |
* MPI implementation, if any of the requests refers to a | |
* non-blocking send or receive of a serialized data type, @c | |
* test_all will always return the equivalent of @c false (i.e., the | |
* requests cannot all be finished at this time). This routine | |
* performs the same functionality as @c wait_all, except that this | |
* routine will not block. This routine provides functionality | |
* equivalent to @c MPI_Testall. | |
* | |
* @param first The iterator that denotes the beginning of the | |
* sequence of request objects. | |
* | |
* @param last The iterator that denotes the end of the sequence of | |
* request objects. | |
* | |
* @param out If provided and all requests hav been completed, an | |
* output iterator through which the status of each request will be | |
* emitted. The @c status objects are emitted in the same order as | |
* the requests are retrieved from @c [first,last). | |
* | |
* @returns If an @p out parameter was provided, the value @c out | |
* after all of the @c status objects have been emitted (if all | |
* requests were completed) or an empty @c optional<>. If no @p out | |
* parameter was provided, returns @c true if all requests have | |
* completed or @c false otherwise. | |
*/ | |
template<typename ForwardIterator, typename OutputIterator> | |
optional<OutputIterator> | |
test_all(ForwardIterator first, ForwardIterator last, OutputIterator out) | |
{ | |
std::vector<MPI_Request> requests; | |
for (; first != last; ++first) { | |
// If we have a non-trivial request, then no requests can be | |
// completed. | |
if (first->m_handler || first->m_requests[1] != MPI_REQUEST_NULL) | |
return optional<OutputIterator>(); | |
requests.push_back(first->m_requests[0]); | |
} | |
int flag = 0; | |
int n = requests.size(); | |
std::vector<MPI_Status> stats(n); | |
BOOST_MPI_CHECK_RESULT(MPI_Testall, (n, &requests[0], &flag, &stats[0])); | |
if (flag) { | |
for (int i = 0; i < n; ++i, ++out) { | |
status stat; | |
stat.m_status = stats[i]; | |
*out = stat; | |
} | |
return out; | |
} else { | |
return optional<OutputIterator>(); | |
} | |
} | |
/** | |
* \overload | |
*/ | |
template<typename ForwardIterator> | |
bool | |
test_all(ForwardIterator first, ForwardIterator last) | |
{ | |
std::vector<MPI_Request> requests; | |
for (; first != last; ++first) { | |
// If we have a non-trivial request, then no requests can be | |
// completed. | |
if (first->m_handler || first->m_requests[1] != MPI_REQUEST_NULL) | |
return false; | |
requests.push_back(first->m_requests[0]); | |
} | |
int flag = 0; | |
int n = requests.size(); | |
BOOST_MPI_CHECK_RESULT(MPI_Testall, | |
(n, &requests[0], &flag, MPI_STATUSES_IGNORE)); | |
return flag != 0; | |
} | |
/** | |
* @brief Wait until some non-blocking requests have completed. | |
* | |
* This routine takes in a set of requests stored in the iterator | |
* range @c [first,last) and waits until at least one of the requests | |
* has completed. It then completes all of the requests it can, | |
* partitioning the input sequence into pending requests followed by | |
* completed requests. If an output iterator is provided, @c status | |
* objects will be emitted for each of the completed requests. This | |
* routine provides functionality equivalent to @c MPI_Waitsome. | |
* | |
* @param first The iterator that denotes the beginning of the | |
* sequence of request objects. | |
* | |
* @param last The iterator that denotes the end of the sequence of | |
* request objects. This may not be equal to @c first. | |
* | |
* @param out If provided, the @c status objects corresponding to | |
* completed requests will be emitted through this output iterator. | |
* @returns If the @p out parameter was provided, a pair containing | |
* the output iterator @p out after all of the @c status objects have | |
* been written through it and an iterator referencing the first | |
* completed request. If no @p out parameter was provided, only the | |
* iterator referencing the first completed request will be emitted. | |
*/ | |
template<typename BidirectionalIterator, typename OutputIterator> | |
std::pair<OutputIterator, BidirectionalIterator> | |
wait_some(BidirectionalIterator first, BidirectionalIterator last, | |
OutputIterator out) | |
{ | |
using std::advance; | |
if (first == last) | |
return std::make_pair(out, first); | |
typedef typename std::iterator_traits<BidirectionalIterator>::difference_type | |
difference_type; | |
bool all_trivial_requests = true; | |
difference_type n = 0; | |
BidirectionalIterator current = first; | |
BidirectionalIterator start_of_completed = last; | |
while (true) { | |
// Check if we have found a completed request. | |
if (optional<status> result = current->test()) { | |
using std::iter_swap; | |
// Emit the resulting status object | |
*out++ = *result; | |
// We're expanding the set of completed requests | |
--start_of_completed; | |
if (current == start_of_completed) { | |
// If we have hit the end of the list of pending | |
// requests. Finish up by fixing the order of the completed | |
// set to match the order in which we emitted status objects, | |
// then return. | |
std::reverse(start_of_completed, last); | |
return std::make_pair(out, start_of_completed); | |
} | |
// Swap the request we just completed with the last request that | |
// has not yet been tested. | |
iter_swap(current, start_of_completed); | |
continue; | |
} | |
// Check if this request (and all others before it) are "trivial" | |
// requests, e.g., they can be represented with a single | |
// MPI_Request. | |
all_trivial_requests = | |
all_trivial_requests | |
&& !current->m_handler | |
&& current->m_requests[1] == MPI_REQUEST_NULL; | |
// Move to the next request. | |
++n; | |
if (++current == start_of_completed) { | |
if (start_of_completed != last) { | |
// We have satisfied some requests. Make the order of the | |
// completed requests match that of the status objects we've | |
// already emitted and we're done. | |
std::reverse(start_of_completed, last); | |
return std::make_pair(out, start_of_completed); | |
} | |
// We have reached the end of the list. If all requests thus far | |
// have been trivial, we can call MPI_Waitsome directly, because | |
// it may be more efficient than our busy-wait semantics. | |
if (all_trivial_requests) { | |
std::vector<MPI_Request> requests; | |
std::vector<int> indices(n); | |
std::vector<MPI_Status> stats(n); | |
requests.reserve(n); | |
for (current = first; current != last; ++current) | |
requests.push_back(current->m_requests[0]); | |
// Let MPI wait until some of these operations complete. | |
int num_completed; | |
BOOST_MPI_CHECK_RESULT(MPI_Waitsome, | |
(n, &requests[0], &num_completed, &indices[0], | |
&stats[0])); | |
// Translate the index-based result of MPI_Waitsome into a | |
// partitioning on the requests. | |
int current_offset = 0; | |
current = first; | |
for (int index = 0; index < num_completed; ++index, ++out) { | |
using std::iter_swap; | |
// Move "current" to the request object at this index | |
advance(current, indices[index] - current_offset); | |
current_offset = indices[index]; | |
// Emit the status object | |
status stat; | |
stat.m_status = stats[index]; | |
*out = stat; | |
// Finish up the request and swap it into the "completed | |
// requests" partition. | |
current->m_requests[0] = requests[indices[index]]; | |
--start_of_completed; | |
iter_swap(current, start_of_completed); | |
} | |
// We have satisfied some requests. Make the order of the | |
// completed requests match that of the status objects we've | |
// already emitted and we're done. | |
std::reverse(start_of_completed, last); | |
return std::make_pair(out, start_of_completed); | |
} | |
// There are some nontrivial requests, so we must continue our | |
// busy waiting loop. | |
n = 0; | |
current = first; | |
} | |
} | |
// We cannot ever get here | |
BOOST_ASSERT(false); | |
} | |
/** | |
* \overload | |
*/ | |
template<typename BidirectionalIterator> | |
BidirectionalIterator | |
wait_some(BidirectionalIterator first, BidirectionalIterator last) | |
{ | |
using std::advance; | |
if (first == last) | |
return first; | |
typedef typename std::iterator_traits<BidirectionalIterator>::difference_type | |
difference_type; | |
bool all_trivial_requests = true; | |
difference_type n = 0; | |
BidirectionalIterator current = first; | |
BidirectionalIterator start_of_completed = last; | |
while (true) { | |
// Check if we have found a completed request. | |
if (optional<status> result = current->test()) { | |
using std::iter_swap; | |
// We're expanding the set of completed requests | |
--start_of_completed; | |
// If we have hit the end of the list of pending requests, we're | |
// done. | |
if (current == start_of_completed) | |
return start_of_completed; | |
// Swap the request we just completed with the last request that | |
// has not yet been tested. | |
iter_swap(current, start_of_completed); | |
continue; | |
} | |
// Check if this request (and all others before it) are "trivial" | |
// requests, e.g., they can be represented with a single | |
// MPI_Request. | |
all_trivial_requests = | |
all_trivial_requests | |
&& !current->m_handler | |
&& current->m_requests[1] == MPI_REQUEST_NULL; | |
// Move to the next request. | |
++n; | |
if (++current == start_of_completed) { | |
// If we have satisfied some requests, we're done. | |
if (start_of_completed != last) | |
return start_of_completed; | |
// We have reached the end of the list. If all requests thus far | |
// have been trivial, we can call MPI_Waitsome directly, because | |
// it may be more efficient than our busy-wait semantics. | |
if (all_trivial_requests) { | |
std::vector<MPI_Request> requests; | |
std::vector<int> indices(n); | |
requests.reserve(n); | |
for (current = first; current != last; ++current) | |
requests.push_back(current->m_requests[0]); | |
// Let MPI wait until some of these operations complete. | |
int num_completed; | |
BOOST_MPI_CHECK_RESULT(MPI_Waitsome, | |
(n, &requests[0], &num_completed, &indices[0], | |
MPI_STATUSES_IGNORE)); | |
// Translate the index-based result of MPI_Waitsome into a | |
// partitioning on the requests. | |
int current_offset = 0; | |
current = first; | |
for (int index = 0; index < num_completed; ++index) { | |
using std::iter_swap; | |
// Move "current" to the request object at this index | |
advance(current, indices[index] - current_offset); | |
current_offset = indices[index]; | |
// Finish up the request and swap it into the "completed | |
// requests" partition. | |
current->m_requests[0] = requests[indices[index]]; | |
--start_of_completed; | |
iter_swap(current, start_of_completed); | |
} | |
// We have satisfied some requests, so we are done. | |
return start_of_completed; | |
} | |
// There are some nontrivial requests, so we must continue our | |
// busy waiting loop. | |
n = 0; | |
current = first; | |
} | |
} | |
// We cannot ever get here | |
BOOST_ASSERT(false); | |
} | |
/** | |
* @brief Test whether some non-blocking requests have completed. | |
* | |
* This routine takes in a set of requests stored in the iterator | |
* range @c [first,last) and tests to see if any of the requests has | |
* completed. It completes all of the requests it can, partitioning | |
* the input sequence into pending requests followed by completed | |
* requests. If an output iterator is provided, @c status objects | |
* will be emitted for each of the completed requests. This routine | |
* is similar to @c wait_some, but does not wait until any requests | |
* have completed. This routine provides functionality equivalent to | |
* @c MPI_Testsome. | |
* | |
* @param first The iterator that denotes the beginning of the | |
* sequence of request objects. | |
* | |
* @param last The iterator that denotes the end of the sequence of | |
* request objects. This may not be equal to @c first. | |
* | |
* @param out If provided, the @c status objects corresponding to | |
* completed requests will be emitted through this output iterator. | |
* @returns If the @p out parameter was provided, a pair containing | |
* the output iterator @p out after all of the @c status objects have | |
* been written through it and an iterator referencing the first | |
* completed request. If no @p out parameter was provided, only the | |
* iterator referencing the first completed request will be emitted. | |
*/ | |
template<typename BidirectionalIterator, typename OutputIterator> | |
std::pair<OutputIterator, BidirectionalIterator> | |
test_some(BidirectionalIterator first, BidirectionalIterator last, | |
OutputIterator out) | |
{ | |
BidirectionalIterator current = first; | |
BidirectionalIterator start_of_completed = last; | |
while (current != start_of_completed) { | |
// Check if we have found a completed request. | |
if (optional<status> result = current->test()) { | |
using std::iter_swap; | |
// Emit the resulting status object | |
*out++ = *result; | |
// We're expanding the set of completed requests | |
--start_of_completed; | |
// Swap the request we just completed with the last request that | |
// has not yet been tested. | |
iter_swap(current, start_of_completed); | |
continue; | |
} | |
// Move to the next request. | |
++current; | |
} | |
// Finish up by fixing the order of the completed set to match the | |
// order in which we emitted status objects, then return. | |
std::reverse(start_of_completed, last); | |
return std::make_pair(out, start_of_completed); | |
} | |
/** | |
* \overload | |
*/ | |
template<typename BidirectionalIterator> | |
BidirectionalIterator | |
test_some(BidirectionalIterator first, BidirectionalIterator last) | |
{ | |
BidirectionalIterator current = first; | |
BidirectionalIterator start_of_completed = last; | |
while (current != start_of_completed) { | |
// Check if we have found a completed request. | |
if (optional<status> result = current->test()) { | |
using std::iter_swap; | |
// We're expanding the set of completed requests | |
--start_of_completed; | |
// Swap the request we just completed with the last request that | |
// has not yet been tested. | |
iter_swap(current, start_of_completed); | |
continue; | |
} | |
// Move to the next request. | |
++current; | |
} | |
return start_of_completed; | |
} | |
} } // end namespace boost::mpi | |
#endif // BOOST_MPI_NONBLOCKING_HPP |