blob: 5fecef4f7cd0b76fe8c79c76f6f3aba7513f2bd7 [file] [log] [blame]
// -*- C++ -*-
//===-- algorithm_impl.h --------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#ifndef _PSTL_ALGORITHM_IMPL_H
#define _PSTL_ALGORITHM_IMPL_H
#include <iterator>
#include <type_traits>
#include <utility>
#include <functional>
#include <algorithm>
#include "execution_impl.h"
#include "memory_impl.h"
#include "parallel_backend_utils.h"
#include "parallel_backend.h"
#include "parallel_impl.h"
#include "unseq_backend_simd.h"
namespace __pstl
{
namespace __internal
{
//------------------------------------------------------------------------
// any_of
//------------------------------------------------------------------------
template <class _ForwardIterator, class _Pred>
bool
__brick_any_of(const _ForwardIterator __first, const _ForwardIterator __last, _Pred __pred,
/*__is_vector=*/std::false_type) noexcept
{
return std::any_of(__first, __last, __pred);
};
template <class _ForwardIterator, class _Pred>
bool
__brick_any_of(const _ForwardIterator __first, const _ForwardIterator __last, _Pred __pred,
/*__is_vector=*/std::true_type) noexcept
{
return __unseq_backend::__simd_or(__first, __last - __first, __pred);
};
template <class _ExecutionPolicy, class _ForwardIterator, class _Pred, class _IsVector>
bool
__pattern_any_of(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Pred __pred,
_IsVector __is_vector, /*parallel=*/std::false_type) noexcept
{
return __internal::__brick_any_of(__first, __last, __pred, __is_vector);
}
template <class _ExecutionPolicy, class _ForwardIterator, class _Pred, class _IsVector>
bool
__pattern_any_of(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Pred __pred,
_IsVector __is_vector, /*parallel=*/std::true_type)
{
return __internal::__except_handler([&]() {
return __internal::__parallel_or(std::forward<_ExecutionPolicy>(__exec), __first, __last,
[__pred, __is_vector](_ForwardIterator __i, _ForwardIterator __j) {
return __internal::__brick_any_of(__i, __j, __pred, __is_vector);
});
});
}
// [alg.foreach]
// for_each_n with no policy
template <class _ForwardIterator, class _Size, class _Function>
_ForwardIterator
__for_each_n_it_serial(_ForwardIterator __first, _Size __n, _Function __f)
{
for (; __n > 0; ++__first, --__n)
__f(__first);
return __first;
}
//------------------------------------------------------------------------
// walk1 (pseudo)
//
// walk1 evaluates f(x) for each dereferenced value x drawn from [first,last)
//------------------------------------------------------------------------
template <class _ForwardIterator, class _Function>
void
__brick_walk1(_ForwardIterator __first, _ForwardIterator __last, _Function __f, /*vector=*/std::false_type) noexcept
{
std::for_each(__first, __last, __f);
}
template <class _RandomAccessIterator, class _Function>
void
__brick_walk1(_RandomAccessIterator __first, _RandomAccessIterator __last, _Function __f,
/*vector=*/std::true_type) noexcept
{
__unseq_backend::__simd_walk_1(__first, __last - __first, __f);
}
template <class _ExecutionPolicy, class _ForwardIterator, class _Function, class _IsVector>
void
__pattern_walk1(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Function __f,
_IsVector __is_vector,
/*parallel=*/std::false_type) noexcept
{
__internal::__brick_walk1(__first, __last, __f, __is_vector);
}
template <class _ExecutionPolicy, class _ForwardIterator, class _Function, class _IsVector>
void
__pattern_walk1(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Function __f,
_IsVector __is_vector,
/*parallel=*/std::true_type)
{
__internal::__except_handler([&]() {
__par_backend::__parallel_for(std::forward<_ExecutionPolicy>(__exec), __first, __last,
[__f, __is_vector](_ForwardIterator __i, _ForwardIterator __j) {
__internal::__brick_walk1(__i, __j, __f, __is_vector);
});
});
}
template <class _ExecutionPolicy, class _ForwardIterator, class _Brick>
void
__pattern_walk_brick(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Brick __brick,
/*parallel=*/std::false_type) noexcept
{
__brick(__first, __last);
}
template <class _ExecutionPolicy, class _ForwardIterator, class _Brick>
void
__pattern_walk_brick(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Brick __brick,
/*parallel=*/std::true_type)
{
__internal::__except_handler([&]() {
__par_backend::__parallel_for(std::forward<_ExecutionPolicy>(__exec), __first, __last,
[__brick](_ForwardIterator __i, _ForwardIterator __j) { __brick(__i, __j); });
});
}
//------------------------------------------------------------------------
// walk1_n
//------------------------------------------------------------------------
template <class _ForwardIterator, class _Size, class _Function>
_ForwardIterator
__brick_walk1_n(_ForwardIterator __first, _Size __n, _Function __f, /*_IsVectorTag=*/std::false_type)
{
return __internal::__for_each_n_it_serial(__first, __n,
[&__f](_ForwardIterator __it) { __f(*__it); }); // calling serial version
}
template <class _RandomAccessIterator, class _DifferenceType, class _Function>
_RandomAccessIterator
__brick_walk1_n(_RandomAccessIterator __first, _DifferenceType __n, _Function __f,
/*vectorTag=*/std::true_type) noexcept
{
return __unseq_backend::__simd_walk_1(__first, __n, __f);
}
template <class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Function, class _IsVector>
_ForwardIterator
__pattern_walk1_n(_ExecutionPolicy&&, _ForwardIterator __first, _Size __n, _Function __f, _IsVector __is_vector,
/*is_parallel=*/std::false_type) noexcept
{
return __internal::__brick_walk1_n(__first, __n, __f, __is_vector);
}
template <class _ExecutionPolicy, class _RandomAccessIterator, class _Size, class _Function, class _IsVector>
_RandomAccessIterator
__pattern_walk1_n(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _Size __n, _Function __f,
_IsVector __is_vector,
/*is_parallel=*/std::true_type)
{
__internal::__pattern_walk1(std::forward<_ExecutionPolicy>(__exec), __first, __first + __n, __f, __is_vector,
std::true_type());
return __first + __n;
}
template <class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Brick>
_ForwardIterator
__pattern_walk_brick_n(_ExecutionPolicy&&, _ForwardIterator __first, _Size __n, _Brick __brick,
/*is_parallel=*/std::false_type) noexcept
{
return __brick(__first, __n);
}
template <class _ExecutionPolicy, class _RandomAccessIterator, class _Size, class _Brick>
_RandomAccessIterator
__pattern_walk_brick_n(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _Size __n, _Brick __brick,
/*is_parallel=*/std::true_type)
{
return __internal::__except_handler([&]() {
__par_backend::__parallel_for(
std::forward<_ExecutionPolicy>(__exec), __first, __first + __n,
[__brick](_RandomAccessIterator __i, _RandomAccessIterator __j) { __brick(__i, __j - __i); });
return __first + __n;
});
}
//------------------------------------------------------------------------
// walk2 (pseudo)
//
// walk2 evaluates f(x,y) for deferenced values (x,y) drawn from [first1,last1) and [first2,...)
//------------------------------------------------------------------------
template <class _ForwardIterator1, class _ForwardIterator2, class _Function>
_ForwardIterator2
__brick_walk2(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2, _Function __f,
/*vector=*/std::false_type) noexcept
{
for (; __first1 != __last1; ++__first1, ++__first2)
__f(*__first1, *__first2);
return __first2;
}
template <class _ForwardIterator1, class _ForwardIterator2, class _Function>
_ForwardIterator2
__brick_walk2(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2, _Function __f,
/*vector=*/std::true_type) noexcept
{
return __unseq_backend::__simd_walk_2(__first1, __last1 - __first1, __first2, __f);
}
template <class _ForwardIterator1, class _Size, class _ForwardIterator2, class _Function>
_ForwardIterator2
__brick_walk2_n(_ForwardIterator1 __first1, _Size __n, _ForwardIterator2 __first2, _Function __f,
/*vector=*/std::false_type) noexcept
{
for (; __n > 0; --__n, ++__first1, ++__first2)
__f(*__first1, *__first2);
return __first2;
}
template <class _ForwardIterator1, class _Size, class _ForwardIterator2, class _Function>
_ForwardIterator2
__brick_walk2_n(_ForwardIterator1 __first1, _Size __n, _ForwardIterator2 __first2, _Function __f,
/*vector=*/std::true_type) noexcept
{
return __unseq_backend::__simd_walk_2(__first1, __n, __first2, __f);
}
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Function, class _IsVector>
_ForwardIterator2
__pattern_walk2(_ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
_Function __f, _IsVector __is_vector, /*parallel=*/std::false_type) noexcept
{
return __internal::__brick_walk2(__first1, __last1, __first2, __f, __is_vector);
}
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Function, class _IsVector>
_ForwardIterator2
__pattern_walk2(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
_ForwardIterator2 __first2, _Function __f, _IsVector __is_vector, /*parallel=*/std::true_type)
{
return __internal::__except_handler([&]() {
__par_backend::__parallel_for(
std::forward<_ExecutionPolicy>(__exec), __first1, __last1,
[__f, __first1, __first2, __is_vector](_ForwardIterator1 __i, _ForwardIterator1 __j) {
__internal::__brick_walk2(__i, __j, __first2 + (__i - __first1), __f, __is_vector);
});
return __first2 + (__last1 - __first1);
});
}
template <class _ExecutionPolicy, class _ForwardIterator1, class _Size, class _ForwardIterator2, class _Function,
class _IsVector>
_ForwardIterator2
__pattern_walk2_n(_ExecutionPolicy&&, _ForwardIterator1 __first1, _Size n, _ForwardIterator2 __first2, _Function f,
_IsVector is_vector, /*parallel=*/std::false_type) noexcept
{
return __internal::__brick_walk2_n(__first1, n, __first2, f, is_vector);
}
template <class _ExecutionPolicy, class _RandomAccessIterator1, class _Size, class _RandomAccessIterator2,
class _Function, class _IsVector>
_RandomAccessIterator2
__pattern_walk2_n(_ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1, _Size n, _RandomAccessIterator2 __first2,
_Function f, _IsVector is_vector, /*parallel=*/std::true_type)
{
return __internal::__pattern_walk2(std::forward<_ExecutionPolicy>(__exec), __first1, __first1 + n, __first2, f,
is_vector, std::true_type());
}
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Brick>
_ForwardIterator2
__pattern_walk2_brick(_ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
_ForwardIterator2 __first2, _Brick __brick, /*parallel=*/std::false_type) noexcept
{
return __brick(__first1, __last1, __first2);
}
template <class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2, class _Brick>
_RandomAccessIterator2
__pattern_walk2_brick(_ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1,
_RandomAccessIterator2 __first2, _Brick __brick, /*parallel=*/std::true_type)
{
return __internal::__except_handler([&]() {
__par_backend::__parallel_for(
std::forward<_ExecutionPolicy>(__exec), __first1, __last1,
[__first1, __first2, __brick](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j) {
__brick(__i, __j, __first2 + (__i - __first1));
});
return __first2 + (__last1 - __first1);
});
}
template <class _ExecutionPolicy, class _RandomAccessIterator1, class _Size, class _RandomAccessIterator2, class _Brick>
_RandomAccessIterator2
__pattern_walk2_brick_n(_ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1, _Size __n,
_RandomAccessIterator2 __first2, _Brick __brick, /*parallel=*/std::true_type)
{
return __internal::__except_handler([&]() {
__par_backend::__parallel_for(
std::forward<_ExecutionPolicy>(__exec), __first1, __first1 + __n,
[__first1, __first2, __brick](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j) {
__brick(__i, __j - __i, __first2 + (__i - __first1));
});
return __first2 + __n;
});
}
template <class _ExecutionPolicy, class _ForwardIterator1, class _Size, class _ForwardIterator2, class _Brick>
_ForwardIterator2
__pattern_walk2_brick_n(_ExecutionPolicy&&, _ForwardIterator1 __first1, _Size __n, _ForwardIterator2 __first2,
_Brick __brick, /*parallel=*/std::false_type) noexcept
{
return __brick(__first1, __n, __first2);
}
//------------------------------------------------------------------------
// walk3 (pseudo)
//
// walk3 evaluates f(x,y,z) for (x,y,z) drawn from [first1,last1), [first2,...), [first3,...)
//------------------------------------------------------------------------
template <class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator3, class _Function>
_ForwardIterator3
__brick_walk3(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
_ForwardIterator3 __first3, _Function __f, /*vector=*/std::false_type) noexcept
{
for (; __first1 != __last1; ++__first1, ++__first2, ++__first3)
__f(*__first1, *__first2, *__first3);
return __first3;
}
template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _RandomAccessIterator3, class _Function>
_RandomAccessIterator3
__brick_walk3(_RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2,
_RandomAccessIterator3 __first3, _Function __f, /*vector=*/std::true_type) noexcept
{
return __unseq_backend::__simd_walk_3(__first1, __last1 - __first1, __first2, __first3, __f);
}
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator3,
class _Function, class _IsVector>
_ForwardIterator3
__pattern_walk3(_ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
_ForwardIterator3 __first3, _Function __f, _IsVector __is_vector, /*parallel=*/std::false_type) noexcept
{
return __internal::__brick_walk3(__first1, __last1, __first2, __first3, __f, __is_vector);
}
template <class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
class _RandomAccessIterator3, class _Function, class _IsVector>
_RandomAccessIterator3
__pattern_walk3(_ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1,
_RandomAccessIterator2 __first2, _RandomAccessIterator3 __first3, _Function __f, _IsVector __is_vector,
/*parallel=*/std::true_type)
{
return __internal::__except_handler([&]() {
__par_backend::__parallel_for(
std::forward<_ExecutionPolicy>(__exec), __first1, __last1,
[__f, __first1, __first2, __first3, __is_vector](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j) {
__internal::__brick_walk3(__i, __j, __first2 + (__i - __first1), __first3 + (__i - __first1), __f,
__is_vector);
});
return __first3 + (__last1 - __first1);
});
}
//------------------------------------------------------------------------
// equal
//------------------------------------------------------------------------
template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
bool
__brick_equal(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
_ForwardIterator2 __last2, _BinaryPredicate __p, /* IsVector = */ std::false_type) noexcept
{
return std::equal(__first1, __last1, __first2, __last2, __p);
}
template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate>
bool
__brick_equal(_RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2,
_RandomAccessIterator2 __last2, _BinaryPredicate __p, /* is_vector = */ std::true_type) noexcept
{
if (__last1 - __first1 != __last2 - __first2)
return false;
return __unseq_backend::__simd_first(__first1, __last1 - __first1, __first2,
__internal::__not_pred<_BinaryPredicate>(__p))
.first == __last1;
}
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate,
class _IsVector>
bool
__pattern_equal(_ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
_ForwardIterator2 __last2, _BinaryPredicate __p, _IsVector __is_vector, /* is_parallel = */
std::false_type) noexcept
{
return __internal::__brick_equal(__first1, __last1, __first2, __last2, __p, __is_vector);
}
template <class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate,
class _IsVector>
bool
__pattern_equal(_ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1,
_RandomAccessIterator2 __first2, _RandomAccessIterator2 __last2, _BinaryPredicate __p,
_IsVector __is_vector, /*is_parallel=*/std::true_type)
{
if (__last1 - __first1 != __last2 - __first2)
return false;
return __internal::__except_handler([&]() {
return !__internal::__parallel_or(
std::forward<_ExecutionPolicy>(__exec), __first1, __last1,
[__first1, __first2, __p, __is_vector](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j) {
return !__internal::__brick_equal(__i, __j, __first2 + (__i - __first1), __first2 + (__j - __first1),
__p, __is_vector);
});
});
}
//------------------------------------------------------------------------
// equal version for sequences with equal length
//------------------------------------------------------------------------
template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
bool
__brick_equal(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2, _BinaryPredicate __p,
/* IsVector = */ std::false_type) noexcept
{
return std::equal(__first1, __last1, __first2, __p);
}
template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate>
bool
__brick_equal(_RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2,
_BinaryPredicate __p, /* is_vector = */ std::true_type) noexcept
{
return __unseq_backend::__simd_first(__first1, __last1 - __first1, __first2, __not_pred<_BinaryPredicate>(__p))
.first == __last1;
}
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate,
class _IsVector>
bool
__pattern_equal(_ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
_BinaryPredicate __p, _IsVector __is_vector, /* is_parallel = */ std::false_type) noexcept
{
return __internal::__brick_equal(__first1, __last1, __first2, __p, __is_vector);
}
template <class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate,
class _IsVector>
bool
__pattern_equal(_ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1,
_RandomAccessIterator2 __first2, _BinaryPredicate __p, _IsVector __is_vector,
/*is_parallel=*/std::true_type)
{
return __internal::__except_handler([&]() {
return !__internal::__parallel_or(
std::forward<_ExecutionPolicy>(__exec), __first1, __last1,
[__first1, __first2, __p, __is_vector](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j) {
return !__internal::__brick_equal(__i, __j, __first2 + (__i - __first1), __p, __is_vector);
});
});
}
//------------------------------------------------------------------------
// find_if
//------------------------------------------------------------------------
template <class _ForwardIterator, class _Predicate>
_ForwardIterator
__brick_find_if(_ForwardIterator __first, _ForwardIterator __last, _Predicate __pred,
/*is_vector=*/std::false_type) noexcept
{
return std::find_if(__first, __last, __pred);
}
template <class _RandomAccessIterator, class _Predicate>
_RandomAccessIterator
__brick_find_if(_RandomAccessIterator __first, _RandomAccessIterator __last, _Predicate __pred,
/*is_vector=*/std::true_type) noexcept
{
typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type _SizeType;
return __unseq_backend::__simd_first(
__first, _SizeType(0), __last - __first,
[&__pred](_RandomAccessIterator __it, _SizeType __i) { return __pred(__it[__i]); });
}
template <class _ExecutionPolicy, class _ForwardIterator, class _Predicate, class _IsVector>
_ForwardIterator
__pattern_find_if(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred,
_IsVector __is_vector,
/*is_parallel=*/std::false_type) noexcept
{
return __internal::__brick_find_if(__first, __last, __pred, __is_vector);
}
template <class _ExecutionPolicy, class _ForwardIterator, class _Predicate, class _IsVector>
_ForwardIterator
__pattern_find_if(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred,
_IsVector __is_vector,
/*is_parallel=*/std::true_type)
{
return __internal::__except_handler([&]() {
return __internal::__parallel_find(
std::forward<_ExecutionPolicy>(__exec), __first, __last,
[__pred, __is_vector](_ForwardIterator __i, _ForwardIterator __j) {
return __internal::__brick_find_if(__i, __j, __pred, __is_vector);
},
std::less<typename std::iterator_traits<_ForwardIterator>::difference_type>(),
/*is_first=*/true);
});
}
//------------------------------------------------------------------------
// find_end
//------------------------------------------------------------------------
// find the first occurrence of the subsequence [s_first, s_last)
// or the last occurrence of the subsequence in the range [first, last)
// b_first determines what occurrence we want to find (first or last)
template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate, class _IsVector>
_RandomAccessIterator1
__find_subrange(_RandomAccessIterator1 __first, _RandomAccessIterator1 __last, _RandomAccessIterator1 __global_last,
_RandomAccessIterator2 __s_first, _RandomAccessIterator2 __s_last, _BinaryPredicate __pred,
bool __b_first, _IsVector __is_vector) noexcept
{
typedef typename std::iterator_traits<_RandomAccessIterator2>::value_type _ValueType;
auto __n2 = __s_last - __s_first;
if (__n2 < 1)
{
return __b_first ? __first : __last;
}
auto __n1 = __global_last - __first;
if (__n1 < __n2)
{
return __last;
}
auto __cur = __last;
while (__first != __last && (__global_last - __first >= __n2))
{
// find position of *s_first in [first, last) (it can be start of subsequence)
__first = __internal::__brick_find_if(
__first, __last, __equal_value_by_pred<_ValueType, _BinaryPredicate>(*__s_first, __pred), __is_vector);
// if position that was found previously is the start of subsequence
// then we can exit the loop (b_first == true) or keep the position
// (b_first == false)
if (__first != __last && (__global_last - __first >= __n2) &&
__internal::__brick_equal(__s_first + 1, __s_last, __first + 1, __pred, __is_vector))
{
if (__b_first)
{
return __first;
}
else
{
__cur = __first;
}
}
else if (__first == __last)
{
break;
}
else
{
}
// in case of b_first == false we try to find new start position
// for the next subsequence
++__first;
}
return __cur;
}
template <class _RandomAccessIterator, class _Size, class _Tp, class _BinaryPredicate, class _IsVector>
_RandomAccessIterator
__find_subrange(_RandomAccessIterator __first, _RandomAccessIterator __last, _RandomAccessIterator __global_last,
_Size __count, const _Tp& __value, _BinaryPredicate __pred, _IsVector __is_vector) noexcept
{
if (__global_last - __first < __count || __count < 1)
{
return __last; // According to the standard last shall be returned when count < 1
}
auto __n = __global_last - __first;
auto __unary_pred = __equal_value_by_pred<_Tp, _BinaryPredicate>(__value, __pred);
while (__first != __last && (__global_last - __first >= __count))
{
__first = __internal::__brick_find_if(__first, __last, __unary_pred, __is_vector);
// check that all of elements in [first+1, first+count) equal to value
if (__first != __last && (__global_last - __first >= __count) &&
!__internal::__brick_any_of(__first + 1, __first + __count,
__not_pred<decltype(__unary_pred)>(__unary_pred), __is_vector))
{
return __first;
}
else if (__first == __last)
{
break;
}
else
{
++__first;
}
}
return __last;
}
template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
_ForwardIterator1
__brick_find_end(_ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
_ForwardIterator2 __s_last, _BinaryPredicate __pred, /*__is_vector=*/std::false_type) noexcept
{
return std::find_end(__first, __last, __s_first, __s_last, __pred);
}
template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
_ForwardIterator1
__brick_find_end(_ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
_ForwardIterator2 __s_last, _BinaryPredicate __pred, /*__is_vector=*/std::true_type) noexcept
{
return __find_subrange(__first, __last, __last, __s_first, __s_last, __pred, false, std::true_type());
}
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate,
class _IsVector>
_ForwardIterator1
__pattern_find_end(_ExecutionPolicy&&, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
_ForwardIterator2 __s_last, _BinaryPredicate __pred, _IsVector __is_vector,
/*is_parallel=*/std::false_type) noexcept
{
return __internal::__brick_find_end(__first, __last, __s_first, __s_last, __pred, __is_vector);
}
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate,
class _IsVector>
_ForwardIterator1
__pattern_find_end(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
_ForwardIterator2 __s_first, _ForwardIterator2 __s_last, _BinaryPredicate __pred,
_IsVector __is_vector, /*is_parallel=*/std::true_type) noexcept
{
if (__last - __first == __s_last - __s_first)
{
const bool __res = __internal::__pattern_equal(std::forward<_ExecutionPolicy>(__exec), __first, __last,
__s_first, __pred, __is_vector, std::true_type());
return __res ? __first : __last;
}
else
{
return __internal::__except_handler([&]() {
return __internal::__parallel_find(
std::forward<_ExecutionPolicy>(__exec), __first, __last,
[__last, __s_first, __s_last, __pred, __is_vector](_ForwardIterator1 __i, _ForwardIterator1 __j) {
return __internal::__find_subrange(__i, __j, __last, __s_first, __s_last, __pred, false,
__is_vector);
},
std::greater<typename std::iterator_traits<_ForwardIterator1>::difference_type>(), /*is_first=*/false);
});
}
}
//------------------------------------------------------------------------
// find_first_of
//------------------------------------------------------------------------
template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
_ForwardIterator1
__brick_find_first_of(_ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
_ForwardIterator2 __s_last, _BinaryPredicate __pred, /*__is_vector=*/std::false_type) noexcept
{
return std::find_first_of(__first, __last, __s_first, __s_last, __pred);
}
template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
_ForwardIterator1
__brick_find_first_of(_ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
_ForwardIterator2 __s_last, _BinaryPredicate __pred, /*__is_vector=*/std::true_type) noexcept
{
return __unseq_backend::__simd_find_first_of(__first, __last, __s_first, __s_last, __pred);
}
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate,
class _IsVector>
_ForwardIterator1
__pattern_find_first_of(_ExecutionPolicy&&, _ForwardIterator1 __first, _ForwardIterator1 __last,
_ForwardIterator2 __s_first, _ForwardIterator2 __s_last, _BinaryPredicate __pred,
_IsVector __is_vector, /*is_parallel=*/std::false_type) noexcept
{
return __internal::__brick_find_first_of(__first, __last, __s_first, __s_last, __pred, __is_vector);
}
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate,
class _IsVector>
_ForwardIterator1
__pattern_find_first_of(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
_ForwardIterator2 __s_first, _ForwardIterator2 __s_last, _BinaryPredicate __pred,
_IsVector __is_vector, /*is_parallel=*/std::true_type) noexcept
{
return __internal::__except_handler([&]() {
return __internal::__parallel_find(
std::forward<_ExecutionPolicy>(__exec), __first, __last,
[__s_first, __s_last, __pred, __is_vector](_ForwardIterator1 __i, _ForwardIterator1 __j) {
return __internal::__brick_find_first_of(__i, __j, __s_first, __s_last, __pred, __is_vector);
},
std::less<typename std::iterator_traits<_ForwardIterator1>::difference_type>(), /*is_first=*/true);
});
}
//------------------------------------------------------------------------
// search
//------------------------------------------------------------------------
template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
_ForwardIterator1
__brick_search(_ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
_ForwardIterator2 __s_last, _BinaryPredicate __pred, /*vector=*/std::false_type) noexcept
{
return std::search(__first, __last, __s_first, __s_last, __pred);
}
template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
_ForwardIterator1
__brick_search(_ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
_ForwardIterator2 __s_last, _BinaryPredicate __pred, /*vector=*/std::true_type) noexcept
{
return __internal::__find_subrange(__first, __last, __last, __s_first, __s_last, __pred, true, std::true_type());
}
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate,
class _IsVector>
_ForwardIterator1
__pattern_search(_ExecutionPolicy&&, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
_ForwardIterator2 __s_last, _BinaryPredicate __pred, _IsVector __is_vector,
/*is_parallel=*/std::false_type) noexcept
{
return __internal::__brick_search(__first, __last, __s_first, __s_last, __pred, __is_vector);
}
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate,
class _IsVector>
_ForwardIterator1
__pattern_search(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
_ForwardIterator2 __s_first, _ForwardIterator2 __s_last, _BinaryPredicate __pred,
_IsVector __is_vector,
/*is_parallel=*/std::true_type) noexcept
{
if (__last - __first == __s_last - __s_first)
{
const bool __res = __internal::__pattern_equal(std::forward<_ExecutionPolicy>(__exec), __first, __last,
__s_first, __pred, __is_vector, std::true_type());
return __res ? __first : __last;
}
else
{
return __internal::__except_handler([&]() {
return __internal::__parallel_find(
std::forward<_ExecutionPolicy>(__exec), __first, __last,
[__last, __s_first, __s_last, __pred, __is_vector](_ForwardIterator1 __i, _ForwardIterator1 __j) {
return __internal::__find_subrange(__i, __j, __last, __s_first, __s_last, __pred, true,
__is_vector);
},
std::less<typename std::iterator_traits<_ForwardIterator1>::difference_type>(), /*is_first=*/true);
});
}
}
//------------------------------------------------------------------------
// search_n
//------------------------------------------------------------------------
template <class _ForwardIterator, class _Size, class _Tp, class _BinaryPredicate>
_ForwardIterator
__brick_search_n(_ForwardIterator __first, _ForwardIterator __last, _Size __count, const _Tp& __value,
_BinaryPredicate __pred, /*vector=*/std::false_type) noexcept
{
return std::search_n(__first, __last, __count, __value, __pred);
}
template <class _ForwardIterator, class _Size, class _Tp, class _BinaryPredicate>
_ForwardIterator
__brick_search_n(_ForwardIterator __first, _ForwardIterator __last, _Size __count, const _Tp& __value,
_BinaryPredicate __pred, /*vector=*/std::true_type) noexcept
{
return __internal::__find_subrange(__first, __last, __last, __count, __value, __pred, std::true_type());
}
template <class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Tp, class _BinaryPredicate,
class _IsVector>
_ForwardIterator
__pattern_search_n(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Size __count,
const _Tp& __value, _BinaryPredicate __pred, _IsVector __is_vector,
/*is_parallel=*/std::false_type) noexcept
{
return __internal::__brick_search_n(__first, __last, __count, __value, __pred, __is_vector);
}
template <class _ExecutionPolicy, class _RandomAccessIterator, class _Size, class _Tp, class _BinaryPredicate,
class _IsVector>
_RandomAccessIterator
__pattern_search_n(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last,
_Size __count, const _Tp& __value, _BinaryPredicate __pred, _IsVector __is_vector,
/*is_parallel=*/std::true_type) noexcept
{
if (__last - __first == __count)
{
const bool __result = !__internal::__pattern_any_of(
std::forward<_ExecutionPolicy>(__exec), __first, __last,
[&__value, &__pred](const _Tp& __val) { return !__pred(__val, __value); }, __is_vector,
/*is_parallel*/ std::true_type());
return __result ? __first : __last;
}
else
{
return __internal::__except_handler([&__exec, __first, __last, __count, &__value, __pred, __is_vector]() {
return __internal::__parallel_find(
std::forward<_ExecutionPolicy>(__exec), __first, __last,
[__last, __count, &__value, __pred, __is_vector](_RandomAccessIterator __i, _RandomAccessIterator __j) {
return __internal::__find_subrange(__i, __j, __last, __count, __value, __pred, __is_vector);
},
std::less<typename std::iterator_traits<_RandomAccessIterator>::difference_type>(), /*is_first=*/true);
});
}
}
//------------------------------------------------------------------------
// copy_n
//------------------------------------------------------------------------
template <class _ForwardIterator, class _Size, class _OutputIterator>
_OutputIterator
__brick_copy_n(_ForwardIterator __first, _Size __n, _OutputIterator __result, /*vector=*/std::false_type) noexcept
{
return std::copy_n(__first, __n, __result);
}
template <class _ForwardIterator, class _Size, class _OutputIterator>
_OutputIterator
__brick_copy_n(_ForwardIterator __first, _Size __n, _OutputIterator __result, /*vector=*/std::true_type) noexcept
{
return __unseq_backend::__simd_assign(
__first, __n, __result, [](_ForwardIterator __first, _OutputIterator __result) { *__result = *__first; });
}
//------------------------------------------------------------------------
// copy
//------------------------------------------------------------------------
template <class _ForwardIterator, class _OutputIterator>
_OutputIterator
__brick_copy(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result,
/*vector=*/std::false_type) noexcept
{
return std::copy(__first, __last, __result);
}
template <class _RandomAccessIterator, class _OutputIterator>
_OutputIterator
__brick_copy(_RandomAccessIterator __first, _RandomAccessIterator __last, _OutputIterator __result,
/*vector=*/std::true_type) noexcept
{
return __unseq_backend::__simd_assign(
__first, __last - __first, __result,
[](_RandomAccessIterator __first, _OutputIterator __result) { *__result = *__first; });
}
//------------------------------------------------------------------------
// move
//------------------------------------------------------------------------
template <class _ForwardIterator, class _OutputIterator>
_OutputIterator
__brick_move(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result,
/*vector=*/std::false_type) noexcept
{
return std::move(__first, __last, __result);
}
template <class _RandomAccessIterator, class _OutputIterator>
_OutputIterator
__brick_move(_RandomAccessIterator __first, _RandomAccessIterator __last, _OutputIterator __result,
/*vector=*/std::true_type) noexcept
{
return __unseq_backend::__simd_assign(
__first, __last - __first, __result,
[](_RandomAccessIterator __first, _OutputIterator __result) { *__result = std::move(*__first); });
}
//------------------------------------------------------------------------
// swap_ranges
//------------------------------------------------------------------------
template <class _ForwardIterator, class _OutputIterator>
_OutputIterator
__brick_swap_ranges(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result,
/*vector=*/std::false_type) noexcept
{
return std::swap_ranges(__first, __last, __result);
}
template <class _ForwardIterator, class _OutputIterator>
_OutputIterator
__brick_swap_ranges(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result,
/*vector=*/std::true_type) noexcept
{
using std::iter_swap;
return __unseq_backend::__simd_assign(__first, __last - __first, __result,
iter_swap<_ForwardIterator, _OutputIterator>);
}
//------------------------------------------------------------------------
// copy_if
//------------------------------------------------------------------------
template <class _ForwardIterator, class _OutputIterator, class _UnaryPredicate>
_OutputIterator
__brick_copy_if(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result, _UnaryPredicate __pred,
/*vector=*/std::false_type) noexcept
{
return std::copy_if(__first, __last, __result, __pred);
}
template <class _ForwardIterator, class _OutputIterator, class _UnaryPredicate>
_OutputIterator
__brick_copy_if(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result, _UnaryPredicate __pred,
/*vector=*/std::true_type) noexcept
{
#if (_PSTL_MONOTONIC_PRESENT)
return __unseq_backend::__simd_copy_if(__first, __last - __first, __result, __pred);
#else
return std::copy_if(__first, __last, __result, __pred);
#endif
}
// TODO: Try to use transform_reduce for combining __brick_copy_if_phase1 on IsVector.
template <class _DifferenceType, class _ForwardIterator, class _UnaryPredicate>
std::pair<_DifferenceType, _DifferenceType>
__brick_calc_mask_1(_ForwardIterator __first, _ForwardIterator __last, bool* __restrict __mask, _UnaryPredicate __pred,
/*vector=*/std::false_type) noexcept
{
auto __count_true = _DifferenceType(0);
auto __size = __last - __first;
static_assert(__is_random_access_iterator<_ForwardIterator>::value,
"Pattern-brick error. Should be a random access iterator.");
for (; __first != __last; ++__first, ++__mask)
{
*__mask = __pred(*__first);
if (*__mask)
{
++__count_true;
}
}
return std::make_pair(__count_true, __size - __count_true);
}
template <class _DifferenceType, class _RandomAccessIterator, class _UnaryPredicate>
std::pair<_DifferenceType, _DifferenceType>
__brick_calc_mask_1(_RandomAccessIterator __first, _RandomAccessIterator __last, bool* __mask, _UnaryPredicate __pred,
/*vector=*/std::true_type) noexcept
{
auto __result = __unseq_backend::__simd_calc_mask_1(__first, __last - __first, __mask, __pred);
return std::make_pair(__result, (__last - __first) - __result);
}
template <class _ForwardIterator, class _OutputIterator, class _Assigner>
void
__brick_copy_by_mask(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result, bool* __mask,
_Assigner __assigner, /*vector=*/std::false_type) noexcept
{
for (; __first != __last; ++__first, ++__mask)
{
if (*__mask)
{
__assigner(__first, __result);
++__result;
}
}
}
template <class _ForwardIterator, class _OutputIterator, class _Assigner>
void
__brick_copy_by_mask(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result,
bool* __restrict __mask, _Assigner __assigner, /*vector=*/std::true_type) noexcept
{
#if (_PSTL_MONOTONIC_PRESENT)
__unseq_backend::__simd_copy_by_mask(__first, __last - __first, __result, __mask, __assigner);
#else
__internal::__brick_copy_by_mask(__first, __last, __result, __mask, __assigner, std::false_type());
#endif
}
template <class _ForwardIterator, class _OutputIterator1, class _OutputIterator2>
void
__brick_partition_by_mask(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator1 __out_true,
_OutputIterator2 __out_false, bool* __mask, /*vector=*/std::false_type) noexcept
{
for (; __first != __last; ++__first, ++__mask)
{
if (*__mask)
{
*__out_true = *__first;
++__out_true;
}
else
{
*__out_false = *__first;
++__out_false;
}
}
}
template <class _RandomAccessIterator, class _OutputIterator1, class _OutputIterator2>
void
__brick_partition_by_mask(_RandomAccessIterator __first, _RandomAccessIterator __last, _OutputIterator1 __out_true,
_OutputIterator2 __out_false, bool* __mask, /*vector=*/std::true_type) noexcept
{
#if (_PSTL_MONOTONIC_PRESENT)
__unseq_backend::__simd_partition_by_mask(__first, __last - __first, __out_true, __out_false, __mask);
#else
__internal::__brick_partition_by_mask(__first, __last, __out_true, __out_false, __mask, std::false_type());
#endif
}
template <class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator, class _UnaryPredicate, class _IsVector>
_OutputIterator
__pattern_copy_if(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result,
_UnaryPredicate __pred, _IsVector __is_vector, /*parallel=*/std::false_type) noexcept
{
return __internal::__brick_copy_if(__first, __last, __result, __pred, __is_vector);
}
template <class _ExecutionPolicy, class _RandomAccessIterator, class _OutputIterator, class _UnaryPredicate,
class _IsVector>
_OutputIterator
__pattern_copy_if(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last,
_OutputIterator __result, _UnaryPredicate __pred, _IsVector __is_vector, /*parallel=*/std::true_type)
{
typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type _DifferenceType;
const _DifferenceType __n = __last - __first;
if (_DifferenceType(1) < __n)
{
__par_backend::__buffer<bool> __mask_buf(__n);
return __internal::__except_handler([&__exec, __n, __first, __result, __is_vector, __pred, &__mask_buf]() {
bool* __mask = __mask_buf.get();
_DifferenceType __m{};
__par_backend::__parallel_strict_scan(
std::forward<_ExecutionPolicy>(__exec), __n, _DifferenceType(0),
[=](_DifferenceType __i, _DifferenceType __len) { // Reduce
return __internal::__brick_calc_mask_1<_DifferenceType>(__first + __i, __first + (__i + __len),
__mask + __i, __pred, __is_vector)
.first;
},
std::plus<_DifferenceType>(), // Combine
[=](_DifferenceType __i, _DifferenceType __len, _DifferenceType __initial) { // Scan
__internal::__brick_copy_by_mask(
__first + __i, __first + (__i + __len), __result + __initial, __mask + __i,
[](_RandomAccessIterator __x, _OutputIterator __z) { *__z = *__x; }, __is_vector);
},
[&__m](_DifferenceType __total) { __m = __total; });
return __result + __m;
});
}
// trivial sequence - use serial algorithm
return __internal::__brick_copy_if(__first, __last, __result, __pred, __is_vector);
}
//------------------------------------------------------------------------
// count
//------------------------------------------------------------------------
template <class _ForwardIterator, class _Predicate>
typename std::iterator_traits<_ForwardIterator>::difference_type
__brick_count(_ForwardIterator __first, _ForwardIterator __last, _Predicate __pred,
/* is_vector = */ std::true_type) noexcept
{
return __unseq_backend::__simd_count(__first, __last - __first, __pred);
}
template <class _ForwardIterator, class _Predicate>
typename std::iterator_traits<_ForwardIterator>::difference_type
__brick_count(_ForwardIterator __first, _ForwardIterator __last, _Predicate __pred,
/* is_vector = */ std::false_type) noexcept
{
return std::count_if(__first, __last, __pred);
}
template <class _ExecutionPolicy, class _ForwardIterator, class _Predicate, class _IsVector>
typename std::iterator_traits<_ForwardIterator>::difference_type
__pattern_count(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred,
/* is_parallel */ std::false_type, _IsVector __is_vector) noexcept
{
return __internal::__brick_count(__first, __last, __pred, __is_vector);
}
template <class _ExecutionPolicy, class _ForwardIterator, class _Predicate, class _IsVector>
typename std::iterator_traits<_ForwardIterator>::difference_type
__pattern_count(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred,
/* is_parallel */ std::true_type, _IsVector __is_vector)
{
typedef typename std::iterator_traits<_ForwardIterator>::difference_type _SizeType;
return __internal::__except_handler([&]() {
return __par_backend::__parallel_reduce(
std::forward<_ExecutionPolicy>(__exec), __first, __last, _SizeType(0),
[__pred, __is_vector](_ForwardIterator __begin, _ForwardIterator __end, _SizeType __value) -> _SizeType {
return __value + __internal::__brick_count(__begin, __end, __pred, __is_vector);
},
std::plus<_SizeType>());
});
}
//------------------------------------------------------------------------
// unique
//------------------------------------------------------------------------
template <class _ForwardIterator, class _BinaryPredicate>
_ForwardIterator
__brick_unique(_ForwardIterator __first, _ForwardIterator __last, _BinaryPredicate __pred,
/*is_vector=*/std::false_type) noexcept
{
return std::unique(__first, __last, __pred);
}
template <class _ForwardIterator, class _BinaryPredicate>
_ForwardIterator
__brick_unique(_ForwardIterator __first, _ForwardIterator __last, _BinaryPredicate __pred,
/*is_vector=*/std::true_type) noexcept
{
_PSTL_PRAGMA_MESSAGE("Vectorized algorithm unimplemented, redirected to serial");
return std::unique(__first, __last, __pred);
}
template <class _ExecutionPolicy, class _ForwardIterator, class _BinaryPredicate, class _IsVector>
_ForwardIterator
__pattern_unique(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _BinaryPredicate __pred,
_IsVector __is_vector, /*is_parallel=*/std::false_type) noexcept
{
return __internal::__brick_unique(__first, __last, __pred, __is_vector);
}
// That function is shared between two algorithms - remove_if (__pattern_remove_if) and unique (pattern unique). But a mask calculation is different.
// So, a caller passes _CalcMask brick into remove_elements.
template <class _ExecutionPolicy, class _ForwardIterator, class _CalcMask, class _IsVector>
_ForwardIterator
__remove_elements(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _CalcMask __calc_mask,
_IsVector __is_vector)
{
typedef typename std::iterator_traits<_ForwardIterator>::difference_type _DifferenceType;
typedef typename std::iterator_traits<_ForwardIterator>::value_type _Tp;
_DifferenceType __n = __last - __first;
__par_backend::__buffer<bool> __mask_buf(__n);
// 1. find a first iterator that should be removed
return __internal::__except_handler([&]() {
bool* __mask = __mask_buf.get();
_DifferenceType __min = __par_backend::__parallel_reduce(
std::forward<_ExecutionPolicy>(__exec), _DifferenceType(0), __n, __n,
[__first, __mask, &__calc_mask, __is_vector](_DifferenceType __i, _DifferenceType __j,
_DifferenceType __local_min) -> _DifferenceType {
// Create mask
__calc_mask(__mask + __i, __mask + __j, __first + __i);
// if minimum was found in a previous range we shouldn't do anymore
if (__local_min < __i)
{
return __local_min;
}
// find first iterator that should be removed
bool* __result = __internal::__brick_find_if(__mask + __i, __mask + __j,
[](bool __val) { return !__val; }, __is_vector);
if (__result - __mask == __j)
{
return __local_min;
}
return std::min(__local_min, _DifferenceType(__result - __mask));
},
[](_DifferenceType __local_min1, _DifferenceType __local_min2) -> _DifferenceType {
return std::min(__local_min1, __local_min2);
});
// No elements to remove - exit
if (__min == __n)
{
return __last;
}
__n -= __min;
__first += __min;
__par_backend::__buffer<_Tp> __buf(__n);
_Tp* __result = __buf.get();
__mask += __min;
_DifferenceType __m{};
// 2. Elements that doesn't satisfy pred are moved to result
__par_backend::__parallel_strict_scan(
std::forward<_ExecutionPolicy>(__exec), __n, _DifferenceType(0),
[__mask, __is_vector](_DifferenceType __i, _DifferenceType __len) {
return __internal::__brick_count(__mask + __i, __mask + __i + __len, [](bool __val) { return __val; },
__is_vector);
},
std::plus<_DifferenceType>(),
[=](_DifferenceType __i, _DifferenceType __len, _DifferenceType __initial) {
__internal::__brick_copy_by_mask(
__first + __i, __first + __i + __len, __result + __initial, __mask + __i,
[](_ForwardIterator __x, _Tp* __z) {
__internal::__invoke_if_else(std::is_trivial<_Tp>(), [&]() { *__z = std::move(*__x); },
[&]() { ::new (std::addressof(*__z)) _Tp(std::move(*__x)); });
},
__is_vector);
},
[&__m](_DifferenceType __total) { __m = __total; });
// 3. Elements from result are moved to [first, last)
__par_backend::__parallel_for(std::forward<_ExecutionPolicy>(__exec), __result, __result + __m,
[__result, __first, __is_vector](_Tp* __i, _Tp* __j) {
__internal::__brick_move(__i, __j, __first + (__i - __result), __is_vector);
});
return __first + __m;
});
}
template <class _ExecutionPolicy, class _ForwardIterator, class _BinaryPredicate, class _IsVector>
_ForwardIterator
__pattern_unique(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _BinaryPredicate __pred,
_IsVector __is_vector, /*is_parallel=*/std::true_type) noexcept
{
typedef typename std::iterator_traits<_ForwardIterator>::reference _ReferenceType;
if (__first == __last)
{
return __last;
}
if (__first + 1 == __last || __first + 2 == __last)
{
// Trivial sequence - use serial algorithm
return __internal::__brick_unique(__first, __last, __pred, __is_vector);
}
return __internal::__remove_elements(
std::forward<_ExecutionPolicy>(__exec), ++__first, __last,
[&__pred, __is_vector](bool* __b, bool* __e, _ForwardIterator __it) {
__internal::__brick_walk3(
__b, __e, __it - 1, __it,
[&__pred](bool& __x, _ReferenceType __y, _ReferenceType __z) { __x = !__pred(__y, __z); }, __is_vector);
},
__is_vector);
}
//------------------------------------------------------------------------
// unique_copy
//------------------------------------------------------------------------
template <class _ForwardIterator, class OutputIterator, class _BinaryPredicate>
OutputIterator
__brick_unique_copy(_ForwardIterator __first, _ForwardIterator __last, OutputIterator __result, _BinaryPredicate __pred,
/*vector=*/std::false_type) noexcept
{
return std::unique_copy(__first, __last, __result, __pred);
}
template <class _RandomAccessIterator, class OutputIterator, class _BinaryPredicate>
OutputIterator
__brick_unique_copy(_RandomAccessIterator __first, _RandomAccessIterator __last, OutputIterator __result,
_BinaryPredicate __pred, /*vector=*/std::true_type) noexcept
{
#if (_PSTL_MONOTONIC_PRESENT)
return __unseq_backend::__simd_unique_copy(__first, __last - __first, __result, __pred);
#else
return std::unique_copy(__first, __last, __result, __pred);
#endif
}
template <class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator, class _BinaryPredicate,
class _IsVector>
_OutputIterator
__pattern_unique_copy(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result,
_BinaryPredicate __pred, _IsVector __is_vector, /*parallel=*/std::false_type) noexcept
{
return __internal::__brick_unique_copy(__first, __last, __result, __pred, __is_vector);
}
template <class _DifferenceType, class _RandomAccessIterator, class _BinaryPredicate>
_DifferenceType
__brick_calc_mask_2(_RandomAccessIterator __first, _RandomAccessIterator __last, bool* __restrict __mask,
_BinaryPredicate __pred, /*vector=*/std::false_type) noexcept
{
_DifferenceType __count = 0;
for (; __first != __last; ++__first, ++__mask)
{
*__mask = !__pred(*__first, *(__first - 1));
__count += *__mask;
}
return __count;
}
template <class _DifferenceType, class _RandomAccessIterator, class _BinaryPredicate>
_DifferenceType
__brick_calc_mask_2(_RandomAccessIterator __first, _RandomAccessIterator __last, bool* __restrict __mask,
_BinaryPredicate __pred, /*vector=*/std::true_type) noexcept
{
return __unseq_backend::__simd_calc_mask_2(__first, __last - __first, __mask, __pred);
}
template <class _ExecutionPolicy, class _RandomAccessIterator, class _OutputIterator, class _BinaryPredicate,
class _IsVector>
_OutputIterator
__pattern_unique_copy(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last,
_OutputIterator __result, _BinaryPredicate __pred, _IsVector __is_vector,
/*parallel=*/std::true_type)
{
typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type _DifferenceType;
const _DifferenceType __n = __last - __first;
if (_DifferenceType(2) < __n)
{
__par_backend::__buffer<bool> __mask_buf(__n);
if (_DifferenceType(2) < __n)
{
return __internal::__except_handler([&__exec, __n, __first, __result, __pred, __is_vector, &__mask_buf]() {
bool* __mask = __mask_buf.get();
_DifferenceType __m{};
__par_backend::__parallel_strict_scan(
std::forward<_ExecutionPolicy>(__exec), __n, _DifferenceType(0),
[=](_DifferenceType __i, _DifferenceType __len) -> _DifferenceType { // Reduce
_DifferenceType __extra = 0;
if (__i == 0)
{
// Special boundary case
__mask[__i] = true;
if (--__len == 0)
return 1;
++__i;
++__extra;
}
return __internal::__brick_calc_mask_2<_DifferenceType>(__first + __i, __first + (__i + __len),
__mask + __i, __pred, __is_vector) +
__extra;
},
std::plus<_DifferenceType>(), // Combine
[=](_DifferenceType __i, _DifferenceType __len, _DifferenceType __initial) { // Scan
// Phase 2 is same as for __pattern_copy_if
__internal::__brick_copy_by_mask(
__first + __i, __first + (__i + __len), __result + __initial, __mask + __i,
[](_RandomAccessIterator __x, _OutputIterator __z) { *__z = *__x; }, __is_vector);
},
[&__m](_DifferenceType __total) { __m = __total; });
return __result + __m;
});
}
}
// trivial sequence - use serial algorithm
return __internal::__brick_unique_copy(__first, __last, __result, __pred, __is_vector);
}
//------------------------------------------------------------------------
// reverse
//------------------------------------------------------------------------
template <class _BidirectionalIterator>
void
__brick_reverse(_BidirectionalIterator __first, _BidirectionalIterator __last, /*__is_vector=*/std::false_type) noexcept
{
std::reverse(__first, __last);
}
template <class _BidirectionalIterator>
void
__brick_reverse(_BidirectionalIterator __first, _BidirectionalIterator __last, /*__is_vector=*/std::true_type) noexcept
{
typedef typename std::iterator_traits<_BidirectionalIterator>::reference _ReferenceType;
const auto __n = (__last - __first) / 2;
__unseq_backend::__simd_walk_2(__first, __n, std::reverse_iterator<_BidirectionalIterator>(__last),
[](_ReferenceType __x, _ReferenceType __y) {
using std::swap;
swap(__x, __y);
});
}
// this brick is called in parallel version, so we can use iterator arithmetic
template <class _BidirectionalIterator>
void
__brick_reverse(_BidirectionalIterator __first, _BidirectionalIterator __last, _BidirectionalIterator __d_last,
/*is_vector=*/std::false_type) noexcept
{
for (--__d_last; __first != __last; ++__first, --__d_last)
{
using std::iter_swap;
iter_swap(__first, __d_last);
}
}
// this brick is called in parallel version, so we can use iterator arithmetic
template <class _BidirectionalIterator>
void
__brick_reverse(_BidirectionalIterator __first, _BidirectionalIterator __last, _BidirectionalIterator __d_last,
/*is_vector=*/std::true_type) noexcept
{
typedef typename std::iterator_traits<_BidirectionalIterator>::reference _ReferenceType;
__unseq_backend::__simd_walk_2(__first, __last - __first, std::reverse_iterator<_BidirectionalIterator>(__d_last),
[](_ReferenceType __x, _ReferenceType __y) {
using std::swap;
swap(__x, __y);
});
}
template <class _ExecutionPolicy, class _BidirectionalIterator, class _IsVector>
void
__pattern_reverse(_ExecutionPolicy&&, _BidirectionalIterator __first, _BidirectionalIterator __last,
_IsVector _is_vector,
/*is_parallel=*/std::false_type) noexcept
{
__internal::__brick_reverse(__first, __last, _is_vector);
}
template <class _ExecutionPolicy, class _BidirectionalIterator, class _IsVector>
void
__pattern_reverse(_ExecutionPolicy&& __exec, _BidirectionalIterator __first, _BidirectionalIterator __last,
_IsVector __is_vector, /*is_parallel=*/std::true_type)
{
__par_backend::__parallel_for(
std::forward<_ExecutionPolicy>(__exec), __first, __first + (__last - __first) / 2,
[__is_vector, __first, __last](_BidirectionalIterator __inner_first, _BidirectionalIterator __inner_last) {
__internal::__brick_reverse(__inner_first, __inner_last, __last - (__inner_first - __first), __is_vector);
});
}
//------------------------------------------------------------------------
// reverse_copy
//------------------------------------------------------------------------
template <class _BidirectionalIterator, class _OutputIterator>
_OutputIterator
__brick_reverse_copy(_BidirectionalIterator __first, _BidirectionalIterator __last, _OutputIterator __d_first,
/*is_vector=*/std::false_type) noexcept
{
return std::reverse_copy(__first, __last, __d_first);
}
template <class _BidirectionalIterator, class _OutputIterator>
_OutputIterator
__brick_reverse_copy(_BidirectionalIterator __first, _BidirectionalIterator __last, _OutputIterator __d_first,
/*is_vector=*/std::true_type) noexcept
{
typedef typename std::iterator_traits<_BidirectionalIterator>::reference _ReferenceType1;
typedef typename std::iterator_traits<_OutputIterator>::reference _ReferenceType2;
return __unseq_backend::__simd_walk_2(std::reverse_iterator<_BidirectionalIterator>(__last), __last - __first,
__d_first, [](_ReferenceType1 __x, _ReferenceType2 __y) { __y = __x; });
}
template <class _ExecutionPolicy, class _BidirectionalIterator, class _OutputIterator, class _IsVector>
_OutputIterator
__pattern_reverse_copy(_ExecutionPolicy&&, _BidirectionalIterator __first, _BidirectionalIterator __last,
_OutputIterator __d_first, _IsVector __is_vector, /*is_parallel=*/std::false_type) noexcept
{
return __internal::__brick_reverse_copy(__first, __last, __d_first, __is_vector);
}
template <class _ExecutionPolicy, class _BidirectionalIterator, class _OutputIterator, class _IsVector>
_OutputIterator
__pattern_reverse_copy(_ExecutionPolicy&& __exec, _BidirectionalIterator __first, _BidirectionalIterator __last,
_OutputIterator __d_first, _IsVector __is_vector, /*is_parallel=*/std::true_type)
{
auto __len = __last - __first;
__par_backend::__parallel_for(std::forward<_ExecutionPolicy>(__exec), __first, __last,
[__is_vector, __first, __len, __d_first](_BidirectionalIterator __inner_first,
_BidirectionalIterator __inner_last) {
__internal::__brick_reverse_copy(__inner_first, __inner_last,
__d_first + (__len - (__inner_last - __first)),
__is_vector);
});
return __d_first + __len;
}
//------------------------------------------------------------------------
// rotate
//------------------------------------------------------------------------
template <class _ForwardIterator>
_ForwardIterator
__brick_rotate(_ForwardIterator __first, _ForwardIterator __middle, _ForwardIterator __last,
/*is_vector=*/std::false_type) noexcept
{
#if _PSTL_CPP11_STD_ROTATE_BROKEN
std::rotate(__first, __middle, __last);
return std::next(__first, std::distance(__middle, __last));
#else
return std::rotate(__first, __middle, __last);
#endif
}
template <class _ForwardIterator>
_ForwardIterator
__brick_rotate(_ForwardIterator __first, _ForwardIterator __middle, _ForwardIterator __last,
/*is_vector=*/std::true_type) noexcept
{
auto __n = __last - __first;
auto __m = __middle - __first;
const _ForwardIterator __ret = __first + (__last - __middle);
bool __is_left = (__m <= __n / 2);
if (!__is_left)
__m = __n - __m;
while (__n > 1 && __m > 0)
{
using std::iter_swap;
const auto __m_2 = __m * 2;
if (__is_left)
{
for (; __last - __first >= __m_2; __first += __m)
{
__unseq_backend::__simd_assign(__first, __m, __first + __m,
iter_swap<_ForwardIterator, _ForwardIterator>);
}
}
else
{
for (; __last - __first >= __m_2; __last -= __m)
{
__unseq_backend::__simd_assign(__last - __m, __m, __last - __m_2,
iter_swap<_ForwardIterator, _ForwardIterator>);
}
}
__is_left = !__is_left;
__m = __n % __m;
__n = __last - __first;
}
return __ret;
}
template <class _ExecutionPolicy, class _ForwardIterator, class _IsVector>
_ForwardIterator
__pattern_rotate(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __middle, _ForwardIterator __last,
_IsVector __is_vector, /*is_parallel=*/std::false_type) noexcept
{
return __internal::__brick_rotate(__first, __middle, __last, __is_vector);
}
template <class _ExecutionPolicy, class _ForwardIterator, class _IsVector>
_ForwardIterator
__pattern_rotate(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __middle,
_ForwardIterator __last, _IsVector __is_vector, /*is_parallel=*/std::true_type)
{
typedef typename std::iterator_traits<_ForwardIterator>::value_type _Tp;
auto __n = __last - __first;
auto __m = __middle - __first;
if (__m <= __n / 2)
{
__par_backend::__buffer<_Tp> __buf(__n - __m);
return __internal::__except_handler([&__exec, __n, __m, __first, __middle, __last, __is_vector, &__buf]() {
_Tp* __result = __buf.get();
__par_backend::__parallel_for(
std::forward<_ExecutionPolicy>(__exec), __middle, __last,
[__middle, __result, __is_vector](_ForwardIterator __b, _ForwardIterator __e) {
__internal::__brick_uninitialized_move(__b, __e, __result + (__b - __middle), __is_vector);
});
__par_backend::__parallel_for(std::forward<_ExecutionPolicy>(__exec), __first, __middle,
[__last, __middle, __is_vector](_ForwardIterator __b, _ForwardIterator __e) {
__internal::__brick_move(__b, __e, __b + (__last - __middle),
__is_vector);
});
__par_backend::__parallel_for(std::forward<_ExecutionPolicy>(__exec), __result, __result + (__n - __m),
[__first, __result, __is_vector](_Tp* __b, _Tp* __e) {
__internal::__brick_move(__b, __e, __first + (__b - __result),
__is_vector);
});
return __first + (__last - __middle);
});
}
else
{
__par_backend::__buffer<_Tp> __buf(__m);
return __internal::__except_handler([&__exec, __n, __m, __first, __middle, __last, __is_vector, &__buf]() {
_Tp* __result = __buf.get();
__par_backend::__parallel_for(std::forward<_ExecutionPolicy>(__exec), __first, __middle,
[__first, __result, __is_vector](_ForwardIterator __b, _ForwardIterator __e) {
__internal::__brick_uninitialized_move(
__b, __e, __result + (__b - __first), __is_vector);
});
__par_backend::__parallel_for(std::forward<_ExecutionPolicy>(__exec), __middle, __last,
[__first, __middle, __is_vector](_ForwardIterator __b, _ForwardIterator __e) {
__internal::__brick_move(__b, __e, __first + (__b - __middle),
__is_vector);
});
__par_backend::__parallel_for(std::forward<_ExecutionPolicy>(__exec), __result, __result + __m,
[__n, __m, __first, __result, __is_vector](_Tp* __b, _Tp* __e) {
__internal::__brick_move(
__b, __e, __first + ((__n - __m) + (__b - __result)), __is_vector);
});
return __first + (__last - __middle);
});
}
}
//------------------------------------------------------------------------
// rotate_copy
//------------------------------------------------------------------------
template <class _ForwardIterator, class _OutputIterator>
_OutputIterator
__brick_rotate_copy(_ForwardIterator __first, _ForwardIterator __middle, _ForwardIterator __last,
_OutputIterator __result, /*__is_vector=*/std::false_type) noexcept
{
return std::rotate_copy(__first, __middle, __last, __result);
}
template <class _ForwardIterator, class _OutputIterator>
_OutputIterator
__brick_rotate_copy(_ForwardIterator __first, _ForwardIterator __middle, _ForwardIterator __last,
_OutputIterator __result, /*__is_vector=*/std::true_type) noexcept
{
_OutputIterator __res = __internal::__brick_copy(__middle, __last, __result, std::true_type());
return __internal::__brick_copy(__first, __middle, __res, std::true_type());
}
template <class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator, class _IsVector>
_OutputIterator
__pattern_rotate_copy(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __middle, _ForwardIterator __last,
_OutputIterator __result, _IsVector __is_vector, /*is_parallel=*/std::false_type) noexcept
{
return __internal::__brick_rotate_copy(__first, __middle, __last, __result, __is_vector);
}
template <class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator, class _IsVector>
_OutputIterator
__pattern_rotate_copy(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __middle,
_ForwardIterator __last, _OutputIterator __result, _IsVector __is_vector,
/*is_parallel=*/std::true_type)
{
__par_backend::__parallel_for(
std::forward<_ExecutionPolicy>(__exec), __first, __last,
[__first, __last, __middle, __result, __is_vector](_ForwardIterator __b, _ForwardIterator __e) {
if (__b > __middle)
{
__internal::__brick_copy(__b, __e, __result + (__b - __middle), __is_vector);
}
else
{
_OutputIterator __new_result = __result + ((__last - __middle) + (__b - __first));
if (__e < __middle)
{
__internal::__brick_copy(__b, __e, __new_result, __is_vector);
}
else
{
__internal::__brick_copy(__b, __middle, __new_result, __is_vector);
__internal::__brick_copy(__middle, __e, __result, __is_vector);
}
}
});
return __result + (__last - __first);
}
//------------------------------------------------------------------------
// is_partitioned
//------------------------------------------------------------------------
template <class _ForwardIterator, class _UnaryPredicate>
bool
__brick_is_partitioned(_ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred,
/*is_vector=*/std::false_type) noexcept
{
return std::is_partitioned(__first, __last, __pred);
}
template <class _ForwardIterator, class _UnaryPredicate>
bool
__brick_is_partitioned(_ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred,
/*is_vector=*/std::true_type) noexcept
{
typedef typename std::iterator_traits<_ForwardIterator>::difference_type _SizeType;
if (__first == __last)
{
return true;
}
else
{
_ForwardIterator __result = __unseq_backend::__simd_first(
__first, _SizeType(0), __last - __first,
[&__pred](_ForwardIterator __it, _SizeType __i) { return !__pred(__it[__i]); });
if (__result == __last)
{
return true;
}
else
{
++__result;
return !__unseq_backend::__simd_or(__result, __last - __result, __pred);
}
}
}
template <class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate, class _IsVector>
bool
__pattern_is_partitioned(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred,
_IsVector __is_vector, /*is_parallel=*/std::false_type) noexcept
{
return __internal::__brick_is_partitioned(__first, __last, __pred, __is_vector);
}
template <class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate, class _IsVector>
bool
__pattern_is_partitioned(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last,
_UnaryPredicate __pred, _IsVector __is_vector, /*is_parallel=*/std::true_type)
{
if (__first == __last)
{
return true;
}
else
{
return __internal::__except_handler([&]() {
// State of current range:
// broken - current range is not partitioned by pred
// all_true - all elements in current range satisfy pred
// all_false - all elements in current range don't satisfy pred
// true_false - elements satisfy pred are placed before elements that don't satisfy pred
enum _ReduceType
{
__not_init = -1,
__broken,
__all_true,
__all_false,
__true_false
};
_ReduceType __init = __not_init;
// Array with states that we'll have when state from the left branch is merged with state from the right branch.
// State is calculated by formula: new_state = table[left_state * 4 + right_state]
_ReduceType __table[] = {__broken, __broken, __broken, __broken, __broken, __all_true,
__true_false, __true_false, __broken, __broken, __all_false, __broken,
__broken, __broken, __true_false, __broken};
__init = __par_backend::__parallel_reduce(
std::forward<_ExecutionPolicy>(__exec), __first, __last, __init,
[&__pred, &__table, __is_vector](_ForwardIterator __i, _ForwardIterator __j,
_ReduceType __value) -> _ReduceType {
if (__value == __broken)
{
return __broken;
}
_ReduceType __res = __not_init;
// if first element satisfy pred
if (__pred(*__i))
{
// find first element that don't satisfy pred
_ForwardIterator __x =
__internal::__brick_find_if(__i + 1, __j, __not_pred<_UnaryPredicate>(__pred), __is_vector);
if (__x != __j)
{
// find first element after "x" that satisfy pred
_ForwardIterator __y = __internal::__brick_find_if(__x + 1, __j, __pred, __is_vector);
// if it was found then range isn't partitioned by pred
if (__y != __j)
{
return __broken;
}
else
{
__res = __true_false;
}
}
else
{
__res = __all_true;
}
}
else
{ // if first element doesn't satisfy pred
// then we should find the first element that satisfy pred.
// If we found it then range isn't partitioned by pred
if (__internal::__brick_find_if(__i + 1, __j, __pred, __is_vector) != __j)
{
return __broken;
}
else
{
__res = __all_false;
}
}
// if we have value from left range then we should calculate the result
return (__value == -1) ? __res : __table[__value * 4 + __res];
},
[&__table](_ReduceType __val1, _ReduceType __val2) -> _ReduceType {
if (__val1 == __broken || __val2 == __broken)
{
return __broken;
}
// calculate the result for new big range
return __table[__val1 * 4 + __val2];
});
return __init != __broken;
});
}
}
//------------------------------------------------------------------------
// partition
//------------------------------------------------------------------------
template <class _ForwardIterator, class _UnaryPredicate>
_ForwardIterator
__brick_partition(_ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred,
/*is_vector=*/std::false_type) noexcept
{
return std::partition(__first, __last, __pred);
}
template <class _ForwardIterator, class _UnaryPredicate>
_ForwardIterator
__brick_partition(_ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred,
/*is_vector=*/std::true_type) noexcept
{
_PSTL_PRAGMA_MESSAGE("Vectorized algorithm unimplemented, redirected to serial");
return std::partition(__first, __last, __pred);
}
template <class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate, class _IsVector>
_ForwardIterator
__pattern_partition(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred,
_IsVector __is_vector, /*is_parallel=*/std::false_type) noexcept
{
return __internal::__brick_partition(__first, __last, __pred, __is_vector);
}
template <class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate, class _IsVector>
_ForwardIterator
__pattern_partition(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last,
_UnaryPredicate __pred, _IsVector __is_vector, /*is_parallel=*/std::true_type)
{
// partitioned range: elements before pivot satisfy pred (true part),
// elements after pivot don't satisfy pred (false part)
struct _PartitionRange
{
_ForwardIterator __begin;
_ForwardIterator __pivot;
_ForwardIterator __end;
};
return __internal::__except_handler([&]() {
_PartitionRange __init{__last, __last, __last};
// lambda for merging two partitioned ranges to one partitioned range
auto __reductor = [&__exec, __is_vector](_PartitionRange __val1, _PartitionRange __val2) -> _PartitionRange {
auto __size1 = __val1.__end - __val1.__pivot;
auto __size2 = __val2.__pivot - __val2.__begin;
auto __new_begin = __val2.__begin - (__val1.__end - __val1.__begin);
// if all elements in left range satisfy pred then we can move new pivot to pivot of right range
if (__val1.__end == __val1.__pivot)
{
return {__new_begin, __val2.__pivot, __val2.__end};
}
// if true part of right range greater than false part of left range
// then we should swap the false part of left range and last part of true part of right range
else if (__size2 > __size1)
{
__par_backend::__parallel_for(
std::forward<_ExecutionPolicy>(__exec), __val1.__pivot, __val1.__pivot + __size1,
[__val1, __val2, __size1, __is_vector](_ForwardIterator __i, _ForwardIterator __j) {
__internal::__brick_swap_ranges(__i, __j, (__val2.__pivot - __size1) + (__i - __val1.__pivot),
__is_vector);
});
return {__new_begin, __val2.__pivot - __size1, __val2.__end};
}
// else we should swap the first part of false part of left range and true part of right range
else
{
__par_backend::__parallel_for(
std::forward<_ExecutionPolicy>(__exec), __val1.__pivot, __val1.__pivot + __size2,
[__val1, __val2, __is_vector](_ForwardIterator __i, _ForwardIterator __j) {
__internal::__brick_swap_ranges(__i, __j, __val2.__begin + (__i - __val1.__pivot), __is_vector);
});
return {__new_begin, __val1.__pivot + __size2, __val2.__end};
}
};
_PartitionRange __result = __par_backend::__parallel_reduce(
std::forward<_ExecutionPolicy>(__exec), __first, __last, __init,
[__pred, __is_vector, __reductor](_ForwardIterator __i, _ForwardIterator __j,
_PartitionRange __value) -> _PartitionRange {
//1. serial partition
_ForwardIterator __pivot = __internal::__brick_partition(__i, __j, __pred, __is_vector);
// 2. merging of two ranges (left and right respectively)
return __reductor(__value, {__i, __pivot, __j});
},
__reductor);
return __result.__pivot;
});
}
//------------------------------------------------------------------------
// stable_partition
//------------------------------------------------------------------------
template <class _BidirectionalIterator, class _UnaryPredicate>
_BidirectionalIterator
__brick_stable_partition(_BidirectionalIterator __first, _BidirectionalIterator __last, _UnaryPredicate __pred,
/*__is_vector=*/std::false_type) noexcept
{
return std::stable_partition(__first, __last, __pred);
}
template <class _BidirectionalIterator, class _UnaryPredicate>
_BidirectionalIterator
__brick_stable_partition(_BidirectionalIterator __first, _BidirectionalIterator __last, _UnaryPredicate __pred,
/*__is_vector=*/std::true_type) noexcept
{
_PSTL_PRAGMA_MESSAGE("Vectorized algorithm unimplemented, redirected to serial");
return std::stable_partition(__first, __last, __pred);
}
template <class _ExecutionPolicy, class _BidirectionalIterator, class _UnaryPredicate, class _IsVector>
_BidirectionalIterator
__pattern_stable_partition(_ExecutionPolicy&&, _BidirectionalIterator __first, _BidirectionalIterator __last,
_UnaryPredicate __pred, _IsVector __is_vector,
/*is_parallelization=*/std::false_type) noexcept
{
return __internal::__brick_stable_partition(__first, __last, __pred, __is_vector);
}
template <class _ExecutionPolicy, class _BidirectionalIterator, class _UnaryPredicate, class _IsVector>
_BidirectionalIterator
__pattern_stable_partition(_ExecutionPolicy&& __exec, _BidirectionalIterator __first, _BidirectionalIterator __last,
_UnaryPredicate __pred, _IsVector __is_vector,
/*is_parallelization=*/std::true_type) noexcept
{
// partitioned range: elements before pivot satisfy pred (true part),
// elements after pivot don't satisfy pred (false part)
struct _PartitionRange
{
_BidirectionalIterator __begin;
_BidirectionalIterator __pivot;
_BidirectionalIterator __end;
};
return __internal::__except_handler([&]() {
_PartitionRange __init{__last, __last, __last};
// lambda for merging two partitioned ranges to one partitioned range
auto __reductor = [__is_vector](_PartitionRange __val1, _PartitionRange __val2) -> _PartitionRange {
auto __size1 = __val1.__end - __val1.__pivot;
auto __new_begin = __val2.__begin - (__val1.__end - __val1.__begin);
// if all elements in left range satisfy pred then we can move new pivot to pivot of right range
if (__val1.__end == __val1.__pivot)
{
return {__new_begin, __val2.__pivot, __val2.__end};
}
// if true part of right range greater than false part of left range
// then we should swap the false part of left range and last part of true part of right range
else
{
__internal::__brick_rotate(__val1.__pivot, __val2.__begin, __val2.__pivot, __is_vector);
return {__new_begin, __val2.__pivot - __size1, __val2.__end};
}
};
_PartitionRange __result = __par_backend::__parallel_reduce(
std::forward<_ExecutionPolicy>(__exec), __first, __last, __init,
[&__pred, __is_vector, __reductor](_BidirectionalIterator __i, _BidirectionalIterator __j,
_PartitionRange __value) -> _PartitionRange {
//1. serial stable_partition
_BidirectionalIterator __pivot = __internal::__brick_stable_partition(__i, __j, __pred, __is_vector);
// 2. merging of two ranges (left and right respectively)
return __reductor(__value, {__i, __pivot, __j});
},
__reductor);
return __result.__pivot;
});
}
//------------------------------------------------------------------------
// partition_copy
//------------------------------------------------------------------------
template <class _ForwardIterator, class _OutputIterator1, class _OutputIterator2, class _UnaryPredicate>
std::pair<_OutputIterator1, _OutputIterator2>
__brick_partition_copy(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator1 __out_true,
_OutputIterator2 __out_false, _UnaryPredicate __pred, /*is_vector=*/std::false_type) noexcept
{
return std::partition_copy(__first, __last, __out_true, __out_false, __pred);
}
template <class _ForwardIterator, class _OutputIterator1, class _OutputIterator2, class _UnaryPredicate>
std::pair<_OutputIterator1, _OutputIterator2>
__brick_partition_copy(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator1 __out_true,
_OutputIterator2 __out_false, _UnaryPredicate __pred, /*is_vector=*/std::true_type) noexcept
{
#if (_PSTL_MONOTONIC_PRESENT)
return __unseq_backend::__simd_partition_copy(__first, __last - __first, __out_true, __out_false, __pred);
#else
return std::partition_copy(__first, __last, __out_true, __out_false, __pred);
#endif
}
template <class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator1, class _OutputIterator2,
class _UnaryPredicate, class _IsVector>
std::pair<_OutputIterator1, _OutputIterator2>
__pattern_partition_copy(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last,
_OutputIterator1 __out_true, _OutputIterator2 __out_false, _UnaryPredicate __pred,
_IsVector __is_vector, /*is_parallelization=*/std::false_type) noexcept
{
return __internal::__brick_partition_copy(__first, __last, __out_true, __out_false, __pred, __is_vector);
}
template <class _ExecutionPolicy, class _RandomAccessIterator, class _OutputIterator1, class _OutputIterator2,
class _UnaryPredicate, class _IsVector>
std::pair<_OutputIterator1, _OutputIterator2>
__pattern_partition_copy(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last,
_OutputIterator1 __out_true, _OutputIterator2 __out_false, _UnaryPredicate __pred,
_IsVector __is_vector, /*is_parallelization=*/std::true_type)
{
typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type _DifferenceType;
typedef std::pair<_DifferenceType, _DifferenceType> _ReturnType;
const _DifferenceType __n = __last - __first;
if (_DifferenceType(1) < __n)
{
__par_backend::__buffer<bool> __mask_buf(__n);
return __internal::__except_handler([&__exec, __n, __first, __out_true, __out_false, __is_vector, __pred,
&__mask_buf]() {
bool* __mask = __mask_buf.get();
_ReturnType __m{};
__par_backend::__parallel_strict_scan(
std::forward<_ExecutionPolicy>(__exec), __n, std::make_pair(_DifferenceType(0), _DifferenceType(0)),
[=](_DifferenceType __i, _DifferenceType __len) { // Reduce
return __internal::__brick_calc_mask_1<_DifferenceType>(__first + __i, __first + (__i + __len),
__mask + __i, __pred, __is_vector);
},
[](const _ReturnType& __x, const _ReturnType& __y) -> _ReturnType {
return std::make_pair(__x.first + __y.first, __x.second + __y.second);
}, // Combine
[=](_DifferenceType __i, _DifferenceType __len, _ReturnType __initial) { // Scan
__internal::__brick_partition_by_mask(__first + __i, __first + (__i + __len),
__out_true + __initial.first, __out_false + __initial.second,
__mask + __i, __is_vector);
},
[&__m](_ReturnType __total) { __m = __total; });
return std::make_pair(__out_true + __m.first, __out_false + __m.second);
});
}
// trivial sequence - use serial algorithm
return __internal::__brick_partition_copy(__first, __last, __out_true, __out_false, __pred, __is_vector);
}
//------------------------------------------------------------------------
// sort
//------------------------------------------------------------------------
template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare, class _IsVector,
class _IsMoveConstructible>
void
__pattern_sort(_ExecutionPolicy&&, _RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp,
_IsVector /*is_vector*/, /*is_parallel=*/std::false_type, _IsMoveConstructible) noexcept
{
std::sort(__first, __last, __comp);
}
template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare, class _IsVector>
void
__pattern_sort(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp,
_IsVector /*is_vector*/, /*is_parallel=*/std::true_type, /*is_move_constructible=*/std::true_type)
{
__internal::__except_handler([&]() {
__par_backend::__parallel_stable_sort(std::forward<_ExecutionPolicy>(__exec), __first, __last, __comp,
[](_RandomAccessIterator __first, _RandomAccessIterator __last,
_Compare __comp) { std::sort(__first, __last, __comp); },
__last - __first);
});
}
//------------------------------------------------------------------------
// stable_sort
//------------------------------------------------------------------------
template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare, class _IsVector>
void
__pattern_stable_sort(_ExecutionPolicy&&, _RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp,
_IsVector /*is_vector*/, /*is_parallel=*/std::false_type) noexcept
{
std::stable_sort(__first, __last, __comp);
}
template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare, class _IsVector>
void
__pattern_stable_sort(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last,
_Compare __comp, _IsVector /*is_vector*/, /*is_parallel=*/std::true_type)
{
__internal::__except_handler([&]() {
__par_backend::__parallel_stable_sort(std::forward<_ExecutionPolicy>(__exec), __first, __last, __comp,
[](_RandomAccessIterator __first, _RandomAccessIterator __last,
_Compare __comp) { std::stable_sort(__first, __last, __comp); });
});
}
//------------------------------------------------------------------------
// partial_sort
//------------------------------------------------------------------------
template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare, class _IsVector>
void
__pattern_partial_sort(_ExecutionPolicy&&, _RandomAccessIterator __first, _RandomAccessIterator __middle,
_RandomAccessIterator __last, _Compare __comp, _IsVector,
/*is_parallel=*/std::false_type) noexcept
{
std::partial_sort(__first, __middle, __last, __comp);
}
template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare, class _IsVector>
void
__pattern_partial_sort(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __middle,
_RandomAccessIterator __last, _Compare __comp, _IsVector, /*is_parallel=*/std::true_type)
{
const auto __n = __middle - __first;
__internal::__except_handler([&]() {
__par_backend::__parallel_stable_sort(
std::forward<_ExecutionPolicy>(__exec), __first, __last, __comp,
[__n](_RandomAccessIterator __begin, _RandomAccessIterator __end, _Compare __comp) {
if (__n < __end - __begin)
std::partial_sort(__begin, __begin + __n, __end, __comp);
else
std::sort(__begin, __end, __comp);
},
__n);
});
}
//------------------------------------------------------------------------
// partial_sort_copy
//------------------------------------------------------------------------
template <class _ExecutionPolicy, class _ForwardIterator, class _RandomAccessIterator, class _Compare, class _IsVector>
_RandomAccessIterator
__pattern_partial_sort_copy(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last,
_RandomAccessIterator __d_first, _RandomAccessIterator __d_last, _Compare __comp, _IsVector,
/*is_parallel=*/std::false_type) noexcept
{
return std::partial_sort_copy(__first, __last, __d_first, __d_last, __comp);
}
template <class _ExecutionPolicy, class _ForwardIterator, class _RandomAccessIterator, class _Compare, class _IsVector>
_RandomAccessIterator
__pattern_partial_sort_copy(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last,
_RandomAccessIterator __d_first, _RandomAccessIterator __d_last, _Compare __comp,
_IsVector __is_vector, /*is_parallel=*/std::true_type)
{
if (__last == __first || __d_last == __d_first)
{
return __d_first;
}
auto __n1 = __last - __first;
auto __n2 = __d_last - __d_first;
return __internal::__except_handler([&]() {
if (__n2 >= __n1)
{
__par_backend::__parallel_stable_sort(
std::forward<_ExecutionPolicy>(__exec), __d_first, __d_first + __n1, __comp,
[__first, __d_first, __is_vector](_RandomAccessIterator __i, _RandomAccessIterator __j,
_Compare __comp) {
_ForwardIterator __i1 = __first + (__i - __d_first);
_ForwardIterator __j1 = __first + (__j - __d_first);
// 1. Copy elements from input to output
# if !_PSTL_ICC_18_OMP_SIMD_BROKEN
__internal::__brick_copy(__i1, __j1, __i, __is_vector);
# else
std::copy(__i1, __j1, __i);
# endif
// 2. Sort elements in output sequence
std::sort(__i, __j, __comp);
},
__n1);
return __d_first + __n1;
}
else
{
typedef typename std::iterator_traits<_ForwardIterator>::value_type _T1;
typedef typename std::iterator_traits<_RandomAccessIterator>::value_type _T2;
__par_backend::__buffer<_T1> __buf(__n1);
_T1* __r = __buf.get();
__par_backend::__parallel_stable_sort(std::forward<_ExecutionPolicy>(__exec), __r, __r + __n1, __comp,
[__n2, __first, __r](_T1* __i, _T1* __j, _Compare __comp) {
_ForwardIterator __it = __first + (__i - __r);
// 1. Copy elements from input to raw memory
for (_T1* __k = __i; __k != __j; ++__k, ++__it)
{
::new (__k) _T2(*__it);
}
// 2. Sort elements in temporary __buffer
if (__n2 < __j - __i)
std::partial_sort(__i, __i + __n2, __j, __comp);
else
std::sort(__i, __j, __comp);
},
__n2);
// 3. Move elements from temporary __buffer to output
__par_backend::__parallel_for(std::forward<_ExecutionPolicy>(__exec), __r, __r + __n2,
[__r, __d_first, __is_vector](_T1* __i, _T1* __j) {
__internal::__brick_move(__i, __j, __d_first + (__i - __r), __is_vector);
});
return __d_first + __n2;
}
});
}
//------------------------------------------------------------------------
// adjacent_find
//------------------------------------------------------------------------
template <class _ForwardIterator, class _BinaryPredicate>
_ForwardIterator
__brick_adjacent_find(_ForwardIterator __first, _ForwardIterator __last, _BinaryPredicate __pred,
/* IsVector = */ std::true_type, bool __or_semantic) noexcept
{
return __unseq_backend::__simd_adjacent_find(__first, __last, __pred, __or_semantic);
}
template <class _ForwardIterator, class _BinaryPredicate>
_ForwardIterator
__brick_adjacent_find(_ForwardIterator __first, _ForwardIterator __last, _BinaryPredicate __pred,
/* IsVector = */ std::false_type, bool __or_semantic) noexcept
{
return std::adjacent_find(__first, __last, __pred);
}
template <class _ExecutionPolicy, class _ForwardIterator, class _BinaryPredicate, class _IsVector>
_ForwardIterator
__pattern_adjacent_find(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _BinaryPredicate __pred,
/* is_parallel */ std::false_type, _IsVector __is_vector, bool __or_semantic) noexcept
{
return __internal::__brick_adjacent_find(__first, __last, __pred, __is_vector, __or_semantic);
}
template <class _ExecutionPolicy, class _RandomAccessIterator, class _BinaryPredicate, class _IsVector>
_RandomAccessIterator
__pattern_adjacent_find(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last,
_BinaryPredicate __pred, /* is_parallel */ std::true_type, _IsVector __is_vector,
bool __or_semantic)
{
if (__last - __first < 2)
return __last;
return __internal::__except_handler([&]() {
return __par_backend::__parallel_reduce(
std::forward<_ExecutionPolicy>(__exec), __first, __last, __last,
[__last, __pred, __is_vector, __or_semantic](_RandomAccessIterator __begin, _RandomAccessIterator __end,
_RandomAccessIterator __value) -> _RandomAccessIterator {
// TODO: investigate performance benefits from the use of shared variable for the result,
// checking (compare_and_swap idiom) its __value at __first.
if (__or_semantic && __value < __last)
{ //found
__par_backend::__cancel_execution();
return __value;
}
if (__value > __begin)
{
// modify __end to check the predicate on the boundary __values;
// TODO: to use a custom range with boundaries overlapping
// TODO: investigate what if we remove "if" below and run algorithm on range [__first, __last-1)
// then check the pair [__last-1, __last)
if (__end != __last)
++__end;
//correct the global result iterator if the "brick" returns a local "__last"
const _RandomAccessIterator __res =
__internal::__brick_adjacent_find(__begin, __end, __pred, __is_vector, __or_semantic);
if (__res < __end)
__value = __res;
}
return __value;
},
[](_RandomAccessIterator __x, _RandomAccessIterator __y) -> _RandomAccessIterator {
return __x < __y ? __x : __y;
} //reduce a __value
);
});
}
//------------------------------------------------------------------------
// nth_element
//------------------------------------------------------------------------
template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare, class _IsVector>
void
__pattern_nth_element(_ExecutionPolicy&&, _RandomAccessIterator __first, _RandomAccessIterator __nth,
_RandomAccessIterator __last, _Compare __comp, _IsVector,
/*is_parallel=*/std::false_type) noexcept
{
std::nth_element(__first, __nth, __last, __comp);
}
template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare, class _IsVector>
void
__pattern_nth_element(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __nth,
_RandomAccessIterator __last, _Compare __comp, _IsVector __is_vector,
/*is_parallel=*/std::true_type) noexcept
{
if (__first == __last || __nth == __last)
{
return;
}
using std::iter_swap;
typedef typename std::iterator_traits<_RandomAccessIterator>::value_type _Tp;
_RandomAccessIterator __x;
do
{
__x = __internal::__pattern_partition(std