1 // -*- C++ -*-
2 //===----------------------------------------------------------------------===//
3 //
4 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5 // See https://llvm.org/LICENSE.txt for license information.
6 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //
8 //===----------------------------------------------------------------------===//
9
10 #ifndef _PSTL_ALGORITHM_IMPL_H
11 #define _PSTL_ALGORITHM_IMPL_H
12
13 #include <iterator>
14 #include <type_traits>
15 #include <utility>
16 #include <functional>
17 #include <algorithm>
18
19 #include "execution_impl.h"
20 #include "memory_impl.h"
21 #include "parallel_backend.h"
22 #include "parallel_backend_utils.h"
23 #include "parallel_impl.h"
24 #include "pstl_config.h"
25 #include "unseq_backend_simd.h"
26
27 _PSTL_HIDE_FROM_ABI_PUSH
28
29 namespace __pstl
30 {
31 namespace __internal
32 {
33
34 //------------------------------------------------------------------------
35 // any_of
36 //------------------------------------------------------------------------
37
38 template <class _ForwardIterator, class _Pred>
39 bool
__brick_any_of(const _ForwardIterator __first,const _ForwardIterator __last,_Pred __pred,std::false_type)40 __brick_any_of(const _ForwardIterator __first, const _ForwardIterator __last, _Pred __pred,
41 /*__is_vector=*/std::false_type) noexcept
42 {
43 return std::any_of(__first, __last, __pred);
44 };
45
46 template <class _ForwardIterator, class _Pred>
47 bool
__brick_any_of(const _ForwardIterator __first,const _ForwardIterator __last,_Pred __pred,std::true_type)48 __brick_any_of(const _ForwardIterator __first, const _ForwardIterator __last, _Pred __pred,
49 /*__is_vector=*/std::true_type) noexcept
50 {
51 return __unseq_backend::__simd_or(__first, __last - __first, __pred);
52 };
53
54 template <class _ExecutionPolicy, class _ForwardIterator, class _Pred, class _IsVector>
55 bool
__pattern_any_of(_ExecutionPolicy &&,_ForwardIterator __first,_ForwardIterator __last,_Pred __pred,_IsVector __is_vector,std::false_type)56 __pattern_any_of(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Pred __pred,
57 _IsVector __is_vector, /*parallel=*/std::false_type) noexcept
58 {
59 return __internal::__brick_any_of(__first, __last, __pred, __is_vector);
60 }
61
62 template <class _ExecutionPolicy, class _ForwardIterator, class _Pred, class _IsVector>
63 bool
__pattern_any_of(_ExecutionPolicy && __exec,_ForwardIterator __first,_ForwardIterator __last,_Pred __pred,_IsVector __is_vector,std::true_type)64 __pattern_any_of(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Pred __pred,
65 _IsVector __is_vector, /*parallel=*/std::true_type)
66 {
67 return __internal::__except_handler([&]() {
68 return __internal::__parallel_or(std::forward<_ExecutionPolicy>(__exec), __first, __last,
69 [__pred, __is_vector](_ForwardIterator __i, _ForwardIterator __j) {
70 return __internal::__brick_any_of(__i, __j, __pred, __is_vector);
71 });
72 });
73 }
74
75 // [alg.foreach]
76 // for_each_n with no policy
77
78 template <class _ForwardIterator, class _Size, class _Function>
79 _ForwardIterator
__for_each_n_it_serial(_ForwardIterator __first,_Size __n,_Function __f)80 __for_each_n_it_serial(_ForwardIterator __first, _Size __n, _Function __f)
81 {
82 for (; __n > 0; ++__first, --__n)
83 __f(__first);
84 return __first;
85 }
86
87 //------------------------------------------------------------------------
88 // walk1 (pseudo)
89 //
90 // walk1 evaluates f(x) for each dereferenced value x drawn from [first,last)
91 //------------------------------------------------------------------------
92 template <class _ForwardIterator, class _Function>
93 void
__brick_walk1(_ForwardIterator __first,_ForwardIterator __last,_Function __f,std::false_type)94 __brick_walk1(_ForwardIterator __first, _ForwardIterator __last, _Function __f, /*vector=*/std::false_type) noexcept
95 {
96 std::for_each(__first, __last, __f);
97 }
98
99 template <class _RandomAccessIterator, class _Function>
100 void
__brick_walk1(_RandomAccessIterator __first,_RandomAccessIterator __last,_Function __f,std::true_type)101 __brick_walk1(_RandomAccessIterator __first, _RandomAccessIterator __last, _Function __f,
102 /*vector=*/std::true_type) noexcept
103 {
104 __unseq_backend::__simd_walk_1(__first, __last - __first, __f);
105 }
106
107 template <class _ExecutionPolicy, class _ForwardIterator, class _Function, class _IsVector>
108 void
__pattern_walk1(_ExecutionPolicy &&,_ForwardIterator __first,_ForwardIterator __last,_Function __f,_IsVector __is_vector,std::false_type)109 __pattern_walk1(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Function __f,
110 _IsVector __is_vector,
111 /*parallel=*/std::false_type) noexcept
112 {
113 __internal::__brick_walk1(__first, __last, __f, __is_vector);
114 }
115
116 template <class _ExecutionPolicy, class _ForwardIterator, class _Function, class _IsVector>
117 void
__pattern_walk1(_ExecutionPolicy && __exec,_ForwardIterator __first,_ForwardIterator __last,_Function __f,_IsVector __is_vector,std::true_type)118 __pattern_walk1(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Function __f,
119 _IsVector __is_vector,
120 /*parallel=*/std::true_type)
121 {
122 __internal::__except_handler([&]() {
123 __par_backend::__parallel_for(std::forward<_ExecutionPolicy>(__exec), __first, __last,
124 [__f, __is_vector](_ForwardIterator __i, _ForwardIterator __j) {
125 __internal::__brick_walk1(__i, __j, __f, __is_vector);
126 });
127 });
128 }
129
130 template <class _ExecutionPolicy, class _ForwardIterator, class _Brick>
131 void
__pattern_walk_brick(_ExecutionPolicy &&,_ForwardIterator __first,_ForwardIterator __last,_Brick __brick,std::false_type)132 __pattern_walk_brick(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Brick __brick,
133 /*parallel=*/std::false_type) noexcept
134 {
135 __brick(__first, __last);
136 }
137
138 template <class _ExecutionPolicy, class _ForwardIterator, class _Brick>
139 void
__pattern_walk_brick(_ExecutionPolicy && __exec,_ForwardIterator __first,_ForwardIterator __last,_Brick __brick,std::true_type)140 __pattern_walk_brick(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Brick __brick,
141 /*parallel=*/std::true_type)
142 {
143 __internal::__except_handler([&]() {
144 __par_backend::__parallel_for(std::forward<_ExecutionPolicy>(__exec), __first, __last,
145 [__brick](_ForwardIterator __i, _ForwardIterator __j) { __brick(__i, __j); });
146 });
147 }
148
149 //------------------------------------------------------------------------
150 // walk1_n
151 //------------------------------------------------------------------------
152 template <class _ForwardIterator, class _Size, class _Function>
153 _ForwardIterator
__brick_walk1_n(_ForwardIterator __first,_Size __n,_Function __f,std::false_type)154 __brick_walk1_n(_ForwardIterator __first, _Size __n, _Function __f, /*_IsVectorTag=*/std::false_type)
155 {
156 return __internal::__for_each_n_it_serial(__first, __n,
157 [&__f](_ForwardIterator __it) { __f(*__it); }); // calling serial version
158 }
159
160 template <class _RandomAccessIterator, class _DifferenceType, class _Function>
161 _RandomAccessIterator
__brick_walk1_n(_RandomAccessIterator __first,_DifferenceType __n,_Function __f,std::true_type)162 __brick_walk1_n(_RandomAccessIterator __first, _DifferenceType __n, _Function __f,
163 /*vectorTag=*/std::true_type) noexcept
164 {
165 return __unseq_backend::__simd_walk_1(__first, __n, __f);
166 }
167
168 template <class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Function, class _IsVector>
169 _ForwardIterator
__pattern_walk1_n(_ExecutionPolicy &&,_ForwardIterator __first,_Size __n,_Function __f,_IsVector __is_vector,std::false_type)170 __pattern_walk1_n(_ExecutionPolicy&&, _ForwardIterator __first, _Size __n, _Function __f, _IsVector __is_vector,
171 /*is_parallel=*/std::false_type) noexcept
172 {
173 return __internal::__brick_walk1_n(__first, __n, __f, __is_vector);
174 }
175
176 template <class _ExecutionPolicy, class _RandomAccessIterator, class _Size, class _Function, class _IsVector>
177 _RandomAccessIterator
__pattern_walk1_n(_ExecutionPolicy && __exec,_RandomAccessIterator __first,_Size __n,_Function __f,_IsVector __is_vector,std::true_type)178 __pattern_walk1_n(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _Size __n, _Function __f,
179 _IsVector __is_vector,
180 /*is_parallel=*/std::true_type)
181 {
182 __internal::__pattern_walk1(std::forward<_ExecutionPolicy>(__exec), __first, __first + __n, __f, __is_vector,
183 std::true_type());
184 return __first + __n;
185 }
186
187 template <class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Brick>
188 _ForwardIterator
__pattern_walk_brick_n(_ExecutionPolicy &&,_ForwardIterator __first,_Size __n,_Brick __brick,std::false_type)189 __pattern_walk_brick_n(_ExecutionPolicy&&, _ForwardIterator __first, _Size __n, _Brick __brick,
190 /*is_parallel=*/std::false_type) noexcept
191 {
192 return __brick(__first, __n);
193 }
194
195 template <class _ExecutionPolicy, class _RandomAccessIterator, class _Size, class _Brick>
196 _RandomAccessIterator
__pattern_walk_brick_n(_ExecutionPolicy && __exec,_RandomAccessIterator __first,_Size __n,_Brick __brick,std::true_type)197 __pattern_walk_brick_n(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _Size __n, _Brick __brick,
198 /*is_parallel=*/std::true_type)
199 {
200 return __internal::__except_handler([&]() {
201 __par_backend::__parallel_for(
202 std::forward<_ExecutionPolicy>(__exec), __first, __first + __n,
203 [__brick](_RandomAccessIterator __i, _RandomAccessIterator __j) { __brick(__i, __j - __i); });
204 return __first + __n;
205 });
206 }
207
208 //------------------------------------------------------------------------
209 // walk2 (pseudo)
210 //
211 // walk2 evaluates f(x,y) for deferenced values (x,y) drawn from [first1,last1) and [first2,...)
212 //------------------------------------------------------------------------
213 template <class _ForwardIterator1, class _ForwardIterator2, class _Function>
214 _ForwardIterator2
__brick_walk2(_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_Function __f,std::false_type)215 __brick_walk2(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2, _Function __f,
216 /*vector=*/std::false_type) noexcept
217 {
218 for (; __first1 != __last1; ++__first1, ++__first2)
219 __f(*__first1, *__first2);
220 return __first2;
221 }
222
223 template <class _ForwardIterator1, class _ForwardIterator2, class _Function>
224 _ForwardIterator2
__brick_walk2(_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_Function __f,std::true_type)225 __brick_walk2(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2, _Function __f,
226 /*vector=*/std::true_type) noexcept
227 {
228 return __unseq_backend::__simd_walk_2(__first1, __last1 - __first1, __first2, __f);
229 }
230
231 template <class _ForwardIterator1, class _Size, class _ForwardIterator2, class _Function>
232 _ForwardIterator2
__brick_walk2_n(_ForwardIterator1 __first1,_Size __n,_ForwardIterator2 __first2,_Function __f,std::false_type)233 __brick_walk2_n(_ForwardIterator1 __first1, _Size __n, _ForwardIterator2 __first2, _Function __f,
234 /*vector=*/std::false_type) noexcept
235 {
236 for (; __n > 0; --__n, ++__first1, ++__first2)
237 __f(*__first1, *__first2);
238 return __first2;
239 }
240
241 template <class _ForwardIterator1, class _Size, class _ForwardIterator2, class _Function>
242 _ForwardIterator2
__brick_walk2_n(_ForwardIterator1 __first1,_Size __n,_ForwardIterator2 __first2,_Function __f,std::true_type)243 __brick_walk2_n(_ForwardIterator1 __first1, _Size __n, _ForwardIterator2 __first2, _Function __f,
244 /*vector=*/std::true_type) noexcept
245 {
246 return __unseq_backend::__simd_walk_2(__first1, __n, __first2, __f);
247 }
248
249 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Function, class _IsVector>
250 _ForwardIterator2
__pattern_walk2(_ExecutionPolicy &&,_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_Function __f,_IsVector __is_vector,std::false_type)251 __pattern_walk2(_ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
252 _Function __f, _IsVector __is_vector, /*parallel=*/std::false_type) noexcept
253 {
254 return __internal::__brick_walk2(__first1, __last1, __first2, __f, __is_vector);
255 }
256
257 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Function, class _IsVector>
258 _ForwardIterator2
__pattern_walk2(_ExecutionPolicy && __exec,_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_Function __f,_IsVector __is_vector,std::true_type)259 __pattern_walk2(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
260 _ForwardIterator2 __first2, _Function __f, _IsVector __is_vector, /*parallel=*/std::true_type)
261 {
262 return __internal::__except_handler([&]() {
263 __par_backend::__parallel_for(
264 std::forward<_ExecutionPolicy>(__exec), __first1, __last1,
265 [__f, __first1, __first2, __is_vector](_ForwardIterator1 __i, _ForwardIterator1 __j) {
266 __internal::__brick_walk2(__i, __j, __first2 + (__i - __first1), __f, __is_vector);
267 });
268 return __first2 + (__last1 - __first1);
269 });
270 }
271
272 template <class _ExecutionPolicy, class _ForwardIterator1, class _Size, class _ForwardIterator2, class _Function,
273 class _IsVector>
274 _ForwardIterator2
__pattern_walk2_n(_ExecutionPolicy &&,_ForwardIterator1 __first1,_Size __n,_ForwardIterator2 __first2,_Function __f,_IsVector is_vector,std::false_type)275 __pattern_walk2_n(_ExecutionPolicy&&, _ForwardIterator1 __first1, _Size __n, _ForwardIterator2 __first2, _Function __f,
276 _IsVector is_vector, /*parallel=*/std::false_type) noexcept
277 {
278 return __internal::__brick_walk2_n(__first1, __n, __first2, __f, is_vector);
279 }
280
281 template <class _ExecutionPolicy, class _RandomAccessIterator1, class _Size, class _RandomAccessIterator2,
282 class _Function, class _IsVector>
283 _RandomAccessIterator2
__pattern_walk2_n(_ExecutionPolicy && __exec,_RandomAccessIterator1 __first1,_Size __n,_RandomAccessIterator2 __first2,_Function __f,_IsVector is_vector,std::true_type)284 __pattern_walk2_n(_ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1, _Size __n, _RandomAccessIterator2 __first2,
285 _Function __f, _IsVector is_vector, /*parallel=*/std::true_type)
286 {
287 return __internal::__pattern_walk2(std::forward<_ExecutionPolicy>(__exec), __first1, __first1 + __n, __first2, __f,
288 is_vector, std::true_type());
289 }
290
291 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Brick>
292 _ForwardIterator2
__pattern_walk2_brick(_ExecutionPolicy &&,_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_Brick __brick,std::false_type)293 __pattern_walk2_brick(_ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
294 _ForwardIterator2 __first2, _Brick __brick, /*parallel=*/std::false_type) noexcept
295 {
296 return __brick(__first1, __last1, __first2);
297 }
298
299 template <class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2, class _Brick>
300 _RandomAccessIterator2
__pattern_walk2_brick(_ExecutionPolicy && __exec,_RandomAccessIterator1 __first1,_RandomAccessIterator1 __last1,_RandomAccessIterator2 __first2,_Brick __brick,std::true_type)301 __pattern_walk2_brick(_ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1,
302 _RandomAccessIterator2 __first2, _Brick __brick, /*parallel=*/std::true_type)
303 {
304 return __internal::__except_handler([&]() {
305 __par_backend::__parallel_for(
306 std::forward<_ExecutionPolicy>(__exec), __first1, __last1,
307 [__first1, __first2, __brick](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j) {
308 __brick(__i, __j, __first2 + (__i - __first1));
309 });
310 return __first2 + (__last1 - __first1);
311 });
312 }
313
314 template <class _ExecutionPolicy, class _RandomAccessIterator1, class _Size, class _RandomAccessIterator2, class _Brick>
315 _RandomAccessIterator2
__pattern_walk2_brick_n(_ExecutionPolicy && __exec,_RandomAccessIterator1 __first1,_Size __n,_RandomAccessIterator2 __first2,_Brick __brick,std::true_type)316 __pattern_walk2_brick_n(_ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1, _Size __n,
317 _RandomAccessIterator2 __first2, _Brick __brick, /*parallel=*/std::true_type)
318 {
319 return __internal::__except_handler([&]() {
320 __par_backend::__parallel_for(
321 std::forward<_ExecutionPolicy>(__exec), __first1, __first1 + __n,
322 [__first1, __first2, __brick](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j) {
323 __brick(__i, __j - __i, __first2 + (__i - __first1));
324 });
325 return __first2 + __n;
326 });
327 }
328
329 template <class _ExecutionPolicy, class _ForwardIterator1, class _Size, class _ForwardIterator2, class _Brick>
330 _ForwardIterator2
__pattern_walk2_brick_n(_ExecutionPolicy &&,_ForwardIterator1 __first1,_Size __n,_ForwardIterator2 __first2,_Brick __brick,std::false_type)331 __pattern_walk2_brick_n(_ExecutionPolicy&&, _ForwardIterator1 __first1, _Size __n, _ForwardIterator2 __first2,
332 _Brick __brick, /*parallel=*/std::false_type) noexcept
333 {
334 return __brick(__first1, __n, __first2);
335 }
336
337 //------------------------------------------------------------------------
338 // walk3 (pseudo)
339 //
340 // walk3 evaluates f(x,y,z) for (x,y,z) drawn from [first1,last1), [first2,...), [first3,...)
341 //------------------------------------------------------------------------
342 template <class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator3, class _Function>
343 _ForwardIterator3
__brick_walk3(_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator3 __first3,_Function __f,std::false_type)344 __brick_walk3(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
345 _ForwardIterator3 __first3, _Function __f, /*vector=*/std::false_type) noexcept
346 {
347 for (; __first1 != __last1; ++__first1, ++__first2, ++__first3)
348 __f(*__first1, *__first2, *__first3);
349 return __first3;
350 }
351
352 template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _RandomAccessIterator3, class _Function>
353 _RandomAccessIterator3
__brick_walk3(_RandomAccessIterator1 __first1,_RandomAccessIterator1 __last1,_RandomAccessIterator2 __first2,_RandomAccessIterator3 __first3,_Function __f,std::true_type)354 __brick_walk3(_RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2,
355 _RandomAccessIterator3 __first3, _Function __f, /*vector=*/std::true_type) noexcept
356 {
357 return __unseq_backend::__simd_walk_3(__first1, __last1 - __first1, __first2, __first3, __f);
358 }
359
360 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator3,
361 class _Function, class _IsVector>
362 _ForwardIterator3
__pattern_walk3(_ExecutionPolicy &&,_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator3 __first3,_Function __f,_IsVector __is_vector,std::false_type)363 __pattern_walk3(_ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
364 _ForwardIterator3 __first3, _Function __f, _IsVector __is_vector, /*parallel=*/std::false_type) noexcept
365 {
366 return __internal::__brick_walk3(__first1, __last1, __first2, __first3, __f, __is_vector);
367 }
368
369 template <class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
370 class _RandomAccessIterator3, class _Function, class _IsVector>
371 _RandomAccessIterator3
__pattern_walk3(_ExecutionPolicy && __exec,_RandomAccessIterator1 __first1,_RandomAccessIterator1 __last1,_RandomAccessIterator2 __first2,_RandomAccessIterator3 __first3,_Function __f,_IsVector __is_vector,std::true_type)372 __pattern_walk3(_ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1,
373 _RandomAccessIterator2 __first2, _RandomAccessIterator3 __first3, _Function __f, _IsVector __is_vector,
374 /*parallel=*/std::true_type)
375 {
376 return __internal::__except_handler([&]() {
377 __par_backend::__parallel_for(
378 std::forward<_ExecutionPolicy>(__exec), __first1, __last1,
379 [__f, __first1, __first2, __first3, __is_vector](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j) {
380 __internal::__brick_walk3(__i, __j, __first2 + (__i - __first1), __first3 + (__i - __first1), __f,
381 __is_vector);
382 });
383 return __first3 + (__last1 - __first1);
384 });
385 }
386
387 //------------------------------------------------------------------------
388 // equal
389 //------------------------------------------------------------------------
390
391 template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
392 bool
__brick_equal(_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_BinaryPredicate __p,std::false_type)393 __brick_equal(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
394 _ForwardIterator2 __last2, _BinaryPredicate __p, /* IsVector = */ std::false_type) noexcept
395 {
396 return std::equal(__first1, __last1, __first2, __last2, __p);
397 }
398
399 template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate>
400 bool
__brick_equal(_RandomAccessIterator1 __first1,_RandomAccessIterator1 __last1,_RandomAccessIterator2 __first2,_RandomAccessIterator2 __last2,_BinaryPredicate __p,std::true_type)401 __brick_equal(_RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2,
402 _RandomAccessIterator2 __last2, _BinaryPredicate __p, /* is_vector = */ std::true_type) noexcept
403 {
404 if (__last1 - __first1 != __last2 - __first2)
405 return false;
406
407 return __unseq_backend::__simd_first(__first1, __last1 - __first1, __first2, std::not_fn(__p)).first == __last1;
408 }
409
410 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate,
411 class _IsVector>
412 bool
__pattern_equal(_ExecutionPolicy &&,_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_BinaryPredicate __p,_IsVector __is_vector,std::false_type)413 __pattern_equal(_ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
414 _ForwardIterator2 __last2, _BinaryPredicate __p, _IsVector __is_vector, /* is_parallel = */
415 std::false_type) noexcept
416 {
417 return __internal::__brick_equal(__first1, __last1, __first2, __last2, __p, __is_vector);
418 }
419
420 template <class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate,
421 class _IsVector>
422 bool
__pattern_equal(_ExecutionPolicy && __exec,_RandomAccessIterator1 __first1,_RandomAccessIterator1 __last1,_RandomAccessIterator2 __first2,_RandomAccessIterator2 __last2,_BinaryPredicate __p,_IsVector __is_vector,std::true_type)423 __pattern_equal(_ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1,
424 _RandomAccessIterator2 __first2, _RandomAccessIterator2 __last2, _BinaryPredicate __p,
425 _IsVector __is_vector, /*is_parallel=*/std::true_type)
426 {
427 if (__last1 - __first1 != __last2 - __first2)
428 return false;
429
430 return __internal::__except_handler([&]() {
431 return !__internal::__parallel_or(
432 std::forward<_ExecutionPolicy>(__exec), __first1, __last1,
433 [__first1, __first2, __p, __is_vector](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j) {
434 return !__internal::__brick_equal(__i, __j, __first2 + (__i - __first1), __first2 + (__j - __first1),
435 __p, __is_vector);
436 });
437 });
438 }
439
440 //------------------------------------------------------------------------
441 // equal version for sequences with equal length
442 //------------------------------------------------------------------------
443
444 template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
445 bool
__brick_equal(_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_BinaryPredicate __p,std::false_type)446 __brick_equal(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2, _BinaryPredicate __p,
447 /* IsVector = */ std::false_type) noexcept
448 {
449 return std::equal(__first1, __last1, __first2, __p);
450 }
451
452 template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate>
453 bool
__brick_equal(_RandomAccessIterator1 __first1,_RandomAccessIterator1 __last1,_RandomAccessIterator2 __first2,_BinaryPredicate __p,std::true_type)454 __brick_equal(_RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2,
455 _BinaryPredicate __p, /* is_vector = */ std::true_type) noexcept
456 {
457 return __unseq_backend::__simd_first(__first1, __last1 - __first1, __first2, std::not_fn(__p)).first == __last1;
458 }
459
460 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate,
461 class _IsVector>
462 bool
__pattern_equal(_ExecutionPolicy &&,_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_BinaryPredicate __p,_IsVector __is_vector,std::false_type)463 __pattern_equal(_ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
464 _BinaryPredicate __p, _IsVector __is_vector, /* is_parallel = */ std::false_type) noexcept
465 {
466 return __internal::__brick_equal(__first1, __last1, __first2, __p, __is_vector);
467 }
468
469 template <class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate,
470 class _IsVector>
471 bool
__pattern_equal(_ExecutionPolicy && __exec,_RandomAccessIterator1 __first1,_RandomAccessIterator1 __last1,_RandomAccessIterator2 __first2,_BinaryPredicate __p,_IsVector __is_vector,std::true_type)472 __pattern_equal(_ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1,
473 _RandomAccessIterator2 __first2, _BinaryPredicate __p, _IsVector __is_vector,
474 /*is_parallel=*/std::true_type)
475 {
476 return __internal::__except_handler([&]() {
477 return !__internal::__parallel_or(
478 std::forward<_ExecutionPolicy>(__exec), __first1, __last1,
479 [__first1, __first2, __p, __is_vector](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j) {
480 return !__internal::__brick_equal(__i, __j, __first2 + (__i - __first1), __p, __is_vector);
481 });
482 });
483 }
484
485 //------------------------------------------------------------------------
486 // find_if
487 //------------------------------------------------------------------------
488 template <class _ForwardIterator, class _Predicate>
489 _ForwardIterator
__brick_find_if(_ForwardIterator __first,_ForwardIterator __last,_Predicate __pred,std::false_type)490 __brick_find_if(_ForwardIterator __first, _ForwardIterator __last, _Predicate __pred,
491 /*is_vector=*/std::false_type) noexcept
492 {
493 return std::find_if(__first, __last, __pred);
494 }
495
496 template <class _RandomAccessIterator, class _Predicate>
497 _RandomAccessIterator
__brick_find_if(_RandomAccessIterator __first,_RandomAccessIterator __last,_Predicate __pred,std::true_type)498 __brick_find_if(_RandomAccessIterator __first, _RandomAccessIterator __last, _Predicate __pred,
499 /*is_vector=*/std::true_type) noexcept
500 {
501 typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type _SizeType;
502 return __unseq_backend::__simd_first(
503 __first, _SizeType(0), __last - __first,
504 [&__pred](_RandomAccessIterator __it, _SizeType __i) { return __pred(__it[__i]); });
505 }
506
507 template <class _ExecutionPolicy, class _ForwardIterator, class _Predicate, class _IsVector>
508 _ForwardIterator
__pattern_find_if(_ExecutionPolicy &&,_ForwardIterator __first,_ForwardIterator __last,_Predicate __pred,_IsVector __is_vector,std::false_type)509 __pattern_find_if(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred,
510 _IsVector __is_vector,
511 /*is_parallel=*/std::false_type) noexcept
512 {
513 return __internal::__brick_find_if(__first, __last, __pred, __is_vector);
514 }
515
516 template <class _ExecutionPolicy, class _ForwardIterator, class _Predicate, class _IsVector>
517 _ForwardIterator
__pattern_find_if(_ExecutionPolicy && __exec,_ForwardIterator __first,_ForwardIterator __last,_Predicate __pred,_IsVector __is_vector,std::true_type)518 __pattern_find_if(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred,
519 _IsVector __is_vector,
520 /*is_parallel=*/std::true_type)
521 {
522 return __internal::__except_handler([&]() {
523 return __internal::__parallel_find(
524 std::forward<_ExecutionPolicy>(__exec), __first, __last,
525 [__pred, __is_vector](_ForwardIterator __i, _ForwardIterator __j) {
526 return __internal::__brick_find_if(__i, __j, __pred, __is_vector);
527 },
528 std::less<typename std::iterator_traits<_ForwardIterator>::difference_type>(),
529 /*is_first=*/true);
530 });
531 }
532
533 //------------------------------------------------------------------------
534 // find_end
535 //------------------------------------------------------------------------
536
537 // find the first occurrence of the subsequence [s_first, s_last)
538 // or the last occurrence of the subsequence in the range [first, last)
539 // b_first determines what occurrence we want to find (first or last)
540 template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate, class _IsVector>
541 _RandomAccessIterator1
__find_subrange(_RandomAccessIterator1 __first,_RandomAccessIterator1 __last,_RandomAccessIterator1 __global_last,_RandomAccessIterator2 __s_first,_RandomAccessIterator2 __s_last,_BinaryPredicate __pred,bool __b_first,_IsVector __is_vector)542 __find_subrange(_RandomAccessIterator1 __first, _RandomAccessIterator1 __last, _RandomAccessIterator1 __global_last,
543 _RandomAccessIterator2 __s_first, _RandomAccessIterator2 __s_last, _BinaryPredicate __pred,
544 bool __b_first, _IsVector __is_vector) noexcept
545 {
546 typedef typename std::iterator_traits<_RandomAccessIterator2>::value_type _ValueType;
547 auto __n2 = __s_last - __s_first;
548 if (__n2 < 1)
549 {
550 return __b_first ? __first : __last;
551 }
552
553 auto __n1 = __global_last - __first;
554 if (__n1 < __n2)
555 {
556 return __last;
557 }
558
559 auto __cur = __last;
560 while (__first != __last && (__global_last - __first >= __n2))
561 {
562 // find position of *s_first in [first, last) (it can be start of subsequence)
563 __first = __internal::__brick_find_if(
564 __first, __last, __equal_value_by_pred<_ValueType, _BinaryPredicate>(*__s_first, __pred), __is_vector);
565
566 // if position that was found previously is the start of subsequence
567 // then we can exit the loop (b_first == true) or keep the position
568 // (b_first == false)
569 if (__first != __last && (__global_last - __first >= __n2) &&
570 __internal::__brick_equal(__s_first + 1, __s_last, __first + 1, __pred, __is_vector))
571 {
572 if (__b_first)
573 {
574 return __first;
575 }
576 else
577 {
578 __cur = __first;
579 }
580 }
581 else if (__first == __last)
582 {
583 break;
584 }
585 else
586 {
587 }
588
589 // in case of b_first == false we try to find new start position
590 // for the next subsequence
591 ++__first;
592 }
593 return __cur;
594 }
595
596 template <class _RandomAccessIterator, class _Size, class _Tp, class _BinaryPredicate, class _IsVector>
597 _RandomAccessIterator
__find_subrange(_RandomAccessIterator __first,_RandomAccessIterator __last,_RandomAccessIterator __global_last,_Size __count,const _Tp & __value,_BinaryPredicate __pred,_IsVector __is_vector)598 __find_subrange(_RandomAccessIterator __first, _RandomAccessIterator __last, _RandomAccessIterator __global_last,
599 _Size __count, const _Tp& __value, _BinaryPredicate __pred, _IsVector __is_vector) noexcept
600 {
601 if (static_cast<_Size>(__global_last - __first) < __count || __count < 1)
602 {
603 return __last; // According to the standard last shall be returned when count < 1
604 }
605
606 auto __unary_pred = __equal_value_by_pred<_Tp, _BinaryPredicate>(__value, __pred);
607 while (__first != __last && (static_cast<_Size>(__global_last - __first) >= __count))
608 {
609 __first = __internal::__brick_find_if(__first, __last, __unary_pred, __is_vector);
610
611 // check that all of elements in [first+1, first+count) equal to value
612 if (__first != __last && (static_cast<_Size>(__global_last - __first) >= __count) &&
613 !__internal::__brick_any_of(__first + 1, __first + __count, std::not_fn(__unary_pred), __is_vector))
614 {
615 return __first;
616 }
617 else if (__first == __last)
618 {
619 break;
620 }
621 else
622 {
623 ++__first;
624 }
625 }
626 return __last;
627 }
628
629 template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
630 _ForwardIterator1
__brick_find_end(_ForwardIterator1 __first,_ForwardIterator1 __last,_ForwardIterator2 __s_first,_ForwardIterator2 __s_last,_BinaryPredicate __pred,std::false_type)631 __brick_find_end(_ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
632 _ForwardIterator2 __s_last, _BinaryPredicate __pred, /*__is_vector=*/std::false_type) noexcept
633 {
634 return std::find_end(__first, __last, __s_first, __s_last, __pred);
635 }
636
637 template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
638 _ForwardIterator1
__brick_find_end(_ForwardIterator1 __first,_ForwardIterator1 __last,_ForwardIterator2 __s_first,_ForwardIterator2 __s_last,_BinaryPredicate __pred,std::true_type)639 __brick_find_end(_ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
640 _ForwardIterator2 __s_last, _BinaryPredicate __pred, /*__is_vector=*/std::true_type) noexcept
641 {
642 return __find_subrange(__first, __last, __last, __s_first, __s_last, __pred, false, std::true_type());
643 }
644
645 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate,
646 class _IsVector>
647 _ForwardIterator1
__pattern_find_end(_ExecutionPolicy &&,_ForwardIterator1 __first,_ForwardIterator1 __last,_ForwardIterator2 __s_first,_ForwardIterator2 __s_last,_BinaryPredicate __pred,_IsVector __is_vector,std::false_type)648 __pattern_find_end(_ExecutionPolicy&&, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
649 _ForwardIterator2 __s_last, _BinaryPredicate __pred, _IsVector __is_vector,
650 /*is_parallel=*/std::false_type) noexcept
651 {
652 return __internal::__brick_find_end(__first, __last, __s_first, __s_last, __pred, __is_vector);
653 }
654
655 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate,
656 class _IsVector>
657 _ForwardIterator1
__pattern_find_end(_ExecutionPolicy && __exec,_ForwardIterator1 __first,_ForwardIterator1 __last,_ForwardIterator2 __s_first,_ForwardIterator2 __s_last,_BinaryPredicate __pred,_IsVector __is_vector,std::true_type)658 __pattern_find_end(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
659 _ForwardIterator2 __s_first, _ForwardIterator2 __s_last, _BinaryPredicate __pred,
660 _IsVector __is_vector, /*is_parallel=*/std::true_type) noexcept
661 {
662 if (__last - __first == __s_last - __s_first)
663 {
664 const bool __res = __internal::__pattern_equal(std::forward<_ExecutionPolicy>(__exec), __first, __last,
665 __s_first, __pred, __is_vector, std::true_type());
666 return __res ? __first : __last;
667 }
668 else
669 {
670 return __internal::__except_handler([&]() {
671 return __internal::__parallel_find(
672 std::forward<_ExecutionPolicy>(__exec), __first, __last,
673 [__last, __s_first, __s_last, __pred, __is_vector](_ForwardIterator1 __i, _ForwardIterator1 __j) {
674 return __internal::__find_subrange(__i, __j, __last, __s_first, __s_last, __pred, false,
675 __is_vector);
676 },
677 std::greater<typename std::iterator_traits<_ForwardIterator1>::difference_type>(), /*is_first=*/false);
678 });
679 }
680 }
681
682 //------------------------------------------------------------------------
683 // find_first_of
684 //------------------------------------------------------------------------
685 template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
686 _ForwardIterator1
__brick_find_first_of(_ForwardIterator1 __first,_ForwardIterator1 __last,_ForwardIterator2 __s_first,_ForwardIterator2 __s_last,_BinaryPredicate __pred,std::false_type)687 __brick_find_first_of(_ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
688 _ForwardIterator2 __s_last, _BinaryPredicate __pred, /*__is_vector=*/std::false_type) noexcept
689 {
690 return std::find_first_of(__first, __last, __s_first, __s_last, __pred);
691 }
692
693 template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
694 _ForwardIterator1
__brick_find_first_of(_ForwardIterator1 __first,_ForwardIterator1 __last,_ForwardIterator2 __s_first,_ForwardIterator2 __s_last,_BinaryPredicate __pred,std::true_type)695 __brick_find_first_of(_ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
696 _ForwardIterator2 __s_last, _BinaryPredicate __pred, /*__is_vector=*/std::true_type) noexcept
697 {
698 return __unseq_backend::__simd_find_first_of(__first, __last, __s_first, __s_last, __pred);
699 }
700
701 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate,
702 class _IsVector>
703 _ForwardIterator1
__pattern_find_first_of(_ExecutionPolicy &&,_ForwardIterator1 __first,_ForwardIterator1 __last,_ForwardIterator2 __s_first,_ForwardIterator2 __s_last,_BinaryPredicate __pred,_IsVector __is_vector,std::false_type)704 __pattern_find_first_of(_ExecutionPolicy&&, _ForwardIterator1 __first, _ForwardIterator1 __last,
705 _ForwardIterator2 __s_first, _ForwardIterator2 __s_last, _BinaryPredicate __pred,
706 _IsVector __is_vector, /*is_parallel=*/std::false_type) noexcept
707 {
708 return __internal::__brick_find_first_of(__first, __last, __s_first, __s_last, __pred, __is_vector);
709 }
710
711 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate,
712 class _IsVector>
713 _ForwardIterator1
__pattern_find_first_of(_ExecutionPolicy && __exec,_ForwardIterator1 __first,_ForwardIterator1 __last,_ForwardIterator2 __s_first,_ForwardIterator2 __s_last,_BinaryPredicate __pred,_IsVector __is_vector,std::true_type)714 __pattern_find_first_of(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
715 _ForwardIterator2 __s_first, _ForwardIterator2 __s_last, _BinaryPredicate __pred,
716 _IsVector __is_vector, /*is_parallel=*/std::true_type) noexcept
717 {
718 return __internal::__except_handler([&]() {
719 return __internal::__parallel_find(
720 std::forward<_ExecutionPolicy>(__exec), __first, __last,
721 [__s_first, __s_last, __pred, __is_vector](_ForwardIterator1 __i, _ForwardIterator1 __j) {
722 return __internal::__brick_find_first_of(__i, __j, __s_first, __s_last, __pred, __is_vector);
723 },
724 std::less<typename std::iterator_traits<_ForwardIterator1>::difference_type>(), /*is_first=*/true);
725 });
726 }
727
728 //------------------------------------------------------------------------
729 // search
730 //------------------------------------------------------------------------
731 template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
732 _ForwardIterator1
__brick_search(_ForwardIterator1 __first,_ForwardIterator1 __last,_ForwardIterator2 __s_first,_ForwardIterator2 __s_last,_BinaryPredicate __pred,std::false_type)733 __brick_search(_ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
734 _ForwardIterator2 __s_last, _BinaryPredicate __pred, /*vector=*/std::false_type) noexcept
735 {
736 return std::search(__first, __last, __s_first, __s_last, __pred);
737 }
738
739 template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
740 _ForwardIterator1
__brick_search(_ForwardIterator1 __first,_ForwardIterator1 __last,_ForwardIterator2 __s_first,_ForwardIterator2 __s_last,_BinaryPredicate __pred,std::true_type)741 __brick_search(_ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
742 _ForwardIterator2 __s_last, _BinaryPredicate __pred, /*vector=*/std::true_type) noexcept
743 {
744 return __internal::__find_subrange(__first, __last, __last, __s_first, __s_last, __pred, true, std::true_type());
745 }
746
747 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate,
748 class _IsVector>
749 _ForwardIterator1
__pattern_search(_ExecutionPolicy &&,_ForwardIterator1 __first,_ForwardIterator1 __last,_ForwardIterator2 __s_first,_ForwardIterator2 __s_last,_BinaryPredicate __pred,_IsVector __is_vector,std::false_type)750 __pattern_search(_ExecutionPolicy&&, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
751 _ForwardIterator2 __s_last, _BinaryPredicate __pred, _IsVector __is_vector,
752 /*is_parallel=*/std::false_type) noexcept
753 {
754 return __internal::__brick_search(__first, __last, __s_first, __s_last, __pred, __is_vector);
755 }
756
757 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate,
758 class _IsVector>
759 _ForwardIterator1
__pattern_search(_ExecutionPolicy && __exec,_ForwardIterator1 __first,_ForwardIterator1 __last,_ForwardIterator2 __s_first,_ForwardIterator2 __s_last,_BinaryPredicate __pred,_IsVector __is_vector,std::true_type)760 __pattern_search(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
761 _ForwardIterator2 __s_first, _ForwardIterator2 __s_last, _BinaryPredicate __pred,
762 _IsVector __is_vector,
763 /*is_parallel=*/std::true_type) noexcept
764 {
765 if (__last - __first == __s_last - __s_first)
766 {
767 const bool __res = __internal::__pattern_equal(std::forward<_ExecutionPolicy>(__exec), __first, __last,
768 __s_first, __pred, __is_vector, std::true_type());
769 return __res ? __first : __last;
770 }
771 else
772 {
773 return __internal::__except_handler([&]() {
774 return __internal::__parallel_find(
775 std::forward<_ExecutionPolicy>(__exec), __first, __last,
776 [__last, __s_first, __s_last, __pred, __is_vector](_ForwardIterator1 __i, _ForwardIterator1 __j) {
777 return __internal::__find_subrange(__i, __j, __last, __s_first, __s_last, __pred, true,
778 __is_vector);
779 },
780 std::less<typename std::iterator_traits<_ForwardIterator1>::difference_type>(), /*is_first=*/true);
781 });
782 }
783 }
784
785 //------------------------------------------------------------------------
786 // search_n
787 //------------------------------------------------------------------------
788 template <class _ForwardIterator, class _Size, class _Tp, class _BinaryPredicate>
789 _ForwardIterator
__brick_search_n(_ForwardIterator __first,_ForwardIterator __last,_Size __count,const _Tp & __value,_BinaryPredicate __pred,std::false_type)790 __brick_search_n(_ForwardIterator __first, _ForwardIterator __last, _Size __count, const _Tp& __value,
791 _BinaryPredicate __pred, /*vector=*/std::false_type) noexcept
792 {
793 return std::search_n(__first, __last, __count, __value, __pred);
794 }
795
796 template <class _ForwardIterator, class _Size, class _Tp, class _BinaryPredicate>
797 _ForwardIterator
__brick_search_n(_ForwardIterator __first,_ForwardIterator __last,_Size __count,const _Tp & __value,_BinaryPredicate __pred,std::true_type)798 __brick_search_n(_ForwardIterator __first, _ForwardIterator __last, _Size __count, const _Tp& __value,
799 _BinaryPredicate __pred, /*vector=*/std::true_type) noexcept
800 {
801 return __internal::__find_subrange(__first, __last, __last, __count, __value, __pred, std::true_type());
802 }
803
804 template <class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Tp, class _BinaryPredicate,
805 class _IsVector>
806 _ForwardIterator
__pattern_search_n(_ExecutionPolicy &&,_ForwardIterator __first,_ForwardIterator __last,_Size __count,const _Tp & __value,_BinaryPredicate __pred,_IsVector __is_vector,std::false_type)807 __pattern_search_n(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Size __count,
808 const _Tp& __value, _BinaryPredicate __pred, _IsVector __is_vector,
809 /*is_parallel=*/std::false_type) noexcept
810 {
811 return __internal::__brick_search_n(__first, __last, __count, __value, __pred, __is_vector);
812 }
813
814 template <class _ExecutionPolicy, class _RandomAccessIterator, class _Size, class _Tp, class _BinaryPredicate,
815 class _IsVector>
816 _RandomAccessIterator
__pattern_search_n(_ExecutionPolicy && __exec,_RandomAccessIterator __first,_RandomAccessIterator __last,_Size __count,const _Tp & __value,_BinaryPredicate __pred,_IsVector __is_vector,std::true_type)817 __pattern_search_n(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last,
818 _Size __count, const _Tp& __value, _BinaryPredicate __pred, _IsVector __is_vector,
819 /*is_parallel=*/std::true_type) noexcept
820 {
821 if (static_cast<_Size>(__last - __first) == __count)
822 {
823 const bool __result = !__internal::__pattern_any_of(
824 std::forward<_ExecutionPolicy>(__exec), __first, __last,
825 [&__value, &__pred](const _Tp& __val) { return !__pred(__val, __value); }, __is_vector,
826 /*is_parallel*/ std::true_type());
827 return __result ? __first : __last;
828 }
829 else
830 {
831 return __internal::__except_handler([&__exec, __first, __last, __count, &__value, __pred, __is_vector]() {
832 return __internal::__parallel_find(
833 std::forward<_ExecutionPolicy>(__exec), __first, __last,
834 [__last, __count, &__value, __pred, __is_vector](_RandomAccessIterator __i, _RandomAccessIterator __j) {
835 return __internal::__find_subrange(__i, __j, __last, __count, __value, __pred, __is_vector);
836 },
837 std::less<typename std::iterator_traits<_RandomAccessIterator>::difference_type>(), /*is_first=*/true);
838 });
839 }
840 }
841
842 //------------------------------------------------------------------------
843 // copy_n
844 //------------------------------------------------------------------------
845
846 template <class _ForwardIterator, class _Size, class _OutputIterator>
847 _OutputIterator
__brick_copy_n(_ForwardIterator __first,_Size __n,_OutputIterator __result,std::false_type)848 __brick_copy_n(_ForwardIterator __first, _Size __n, _OutputIterator __result, /*vector=*/std::false_type) noexcept
849 {
850 return std::copy_n(__first, __n, __result);
851 }
852
853 template <class _ForwardIterator, class _Size, class _OutputIterator>
854 _OutputIterator
__brick_copy_n(_ForwardIterator __first,_Size __n,_OutputIterator __result,std::true_type)855 __brick_copy_n(_ForwardIterator __first, _Size __n, _OutputIterator __result, /*vector=*/std::true_type) noexcept
856 {
857 return __unseq_backend::__simd_assign(
858 __first, __n, __result, [](_ForwardIterator __first, _OutputIterator __result) { *__result = *__first; });
859 }
860
861 //------------------------------------------------------------------------
862 // copy
863 //------------------------------------------------------------------------
864 template <class _ForwardIterator, class _OutputIterator>
865 _OutputIterator
__brick_copy(_ForwardIterator __first,_ForwardIterator __last,_OutputIterator __result,std::false_type)866 __brick_copy(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result,
867 /*vector=*/std::false_type) noexcept
868 {
869 return std::copy(__first, __last, __result);
870 }
871
872 template <class _RandomAccessIterator, class _OutputIterator>
873 _OutputIterator
__brick_copy(_RandomAccessIterator __first,_RandomAccessIterator __last,_OutputIterator __result,std::true_type)874 __brick_copy(_RandomAccessIterator __first, _RandomAccessIterator __last, _OutputIterator __result,
875 /*vector=*/std::true_type) noexcept
876 {
877 return __unseq_backend::__simd_assign(
878 __first, __last - __first, __result,
879 [](_RandomAccessIterator __first, _OutputIterator __result) { *__result = *__first; });
880 }
881
882 //------------------------------------------------------------------------
883 // move
884 //------------------------------------------------------------------------
885 template <class _ForwardIterator, class _OutputIterator>
886 _OutputIterator
__brick_move(_ForwardIterator __first,_ForwardIterator __last,_OutputIterator __result,std::false_type)887 __brick_move(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result,
888 /*vector=*/std::false_type) noexcept
889 {
890 return std::move(__first, __last, __result);
891 }
892
893 template <class _RandomAccessIterator, class _OutputIterator>
894 _OutputIterator
__brick_move(_RandomAccessIterator __first,_RandomAccessIterator __last,_OutputIterator __result,std::true_type)895 __brick_move(_RandomAccessIterator __first, _RandomAccessIterator __last, _OutputIterator __result,
896 /*vector=*/std::true_type) noexcept
897 {
898 return __unseq_backend::__simd_assign(
899 __first, __last - __first, __result,
900 [](_RandomAccessIterator __first, _OutputIterator __result) { *__result = std::move(*__first); });
901 }
902
903 struct __brick_move_destroy
904 {
905 template <typename _Iterator, typename _OutputIterator>
906 _OutputIterator
operator__brick_move_destroy907 operator()(_Iterator __first, _Iterator __last, _OutputIterator __result, /*vec*/ std::true_type) const
908 {
909 using _IteratorValueType = typename std::iterator_traits<_Iterator>::value_type;
910
911 return __unseq_backend::__simd_assign(__first, __last - __first, __result,
912 [](_Iterator __first, _OutputIterator __result) {
913 *__result = std::move(*__first);
914 (*__first).~_IteratorValueType();
915 });
916 }
917
918 template <typename _Iterator, typename _OutputIterator>
919 _OutputIterator
operator__brick_move_destroy920 operator()(_Iterator __first, _Iterator __last, _OutputIterator __result, /*vec*/ std::false_type) const
921 {
922 using _IteratorValueType = typename std::iterator_traits<_Iterator>::value_type;
923
924 for (; __first != __last; ++__first, ++__result)
925 {
926 *__result = std::move(*__first);
927 (*__first).~_IteratorValueType();
928 }
929 return __result;
930 }
931 };
932
933 //------------------------------------------------------------------------
934 // swap_ranges
935 //------------------------------------------------------------------------
936 template <class _ForwardIterator, class _OutputIterator>
937 _OutputIterator
__brick_swap_ranges(_ForwardIterator __first,_ForwardIterator __last,_OutputIterator __result,std::false_type)938 __brick_swap_ranges(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result,
939 /*vector=*/std::false_type) noexcept
940 {
941 return std::swap_ranges(__first, __last, __result);
942 }
943
944 template <class _ForwardIterator, class _OutputIterator>
945 _OutputIterator
__brick_swap_ranges(_ForwardIterator __first,_ForwardIterator __last,_OutputIterator __result,std::true_type)946 __brick_swap_ranges(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result,
947 /*vector=*/std::true_type) noexcept
948 {
949 using std::iter_swap;
950 return __unseq_backend::__simd_assign(__first, __last - __first, __result,
951 iter_swap<_ForwardIterator, _OutputIterator>);
952 }
953
954 //------------------------------------------------------------------------
955 // copy_if
956 //------------------------------------------------------------------------
957 template <class _ForwardIterator, class _OutputIterator, class _UnaryPredicate>
958 _OutputIterator
__brick_copy_if(_ForwardIterator __first,_ForwardIterator __last,_OutputIterator __result,_UnaryPredicate __pred,std::false_type)959 __brick_copy_if(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result, _UnaryPredicate __pred,
960 /*vector=*/std::false_type) noexcept
961 {
962 return std::copy_if(__first, __last, __result, __pred);
963 }
964
965 template <class _ForwardIterator, class _OutputIterator, class _UnaryPredicate>
966 _OutputIterator
__brick_copy_if(_ForwardIterator __first,_ForwardIterator __last,_OutputIterator __result,_UnaryPredicate __pred,std::true_type)967 __brick_copy_if(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result, _UnaryPredicate __pred,
968 /*vector=*/std::true_type) noexcept
969 {
970 #if (_PSTL_MONOTONIC_PRESENT)
971 return __unseq_backend::__simd_copy_if(__first, __last - __first, __result, __pred);
972 #else
973 return std::copy_if(__first, __last, __result, __pred);
974 #endif
975 }
976
977 // TODO: Try to use transform_reduce for combining __brick_copy_if_phase1 on IsVector.
978 template <class _DifferenceType, class _ForwardIterator, class _UnaryPredicate>
979 std::pair<_DifferenceType, _DifferenceType>
__brick_calc_mask_1(_ForwardIterator __first,_ForwardIterator __last,bool * __restrict __mask,_UnaryPredicate __pred,std::false_type)980 __brick_calc_mask_1(_ForwardIterator __first, _ForwardIterator __last, bool* __restrict __mask, _UnaryPredicate __pred,
981 /*vector=*/std::false_type) noexcept
982 {
983 auto __count_true = _DifferenceType(0);
984 auto __size = __last - __first;
985
986 static_assert(__is_random_access_iterator<_ForwardIterator>::value,
987 "Pattern-brick error. Should be a random access iterator.");
988
989 for (; __first != __last; ++__first, ++__mask)
990 {
991 *__mask = __pred(*__first);
992 if (*__mask)
993 {
994 ++__count_true;
995 }
996 }
997 return std::make_pair(__count_true, __size - __count_true);
998 }
999
1000 template <class _DifferenceType, class _RandomAccessIterator, class _UnaryPredicate>
1001 std::pair<_DifferenceType, _DifferenceType>
__brick_calc_mask_1(_RandomAccessIterator __first,_RandomAccessIterator __last,bool * __mask,_UnaryPredicate __pred,std::true_type)1002 __brick_calc_mask_1(_RandomAccessIterator __first, _RandomAccessIterator __last, bool* __mask, _UnaryPredicate __pred,
1003 /*vector=*/std::true_type) noexcept
1004 {
1005 auto __result = __unseq_backend::__simd_calc_mask_1(__first, __last - __first, __mask, __pred);
1006 return std::make_pair(__result, (__last - __first) - __result);
1007 }
1008
1009 template <class _ForwardIterator, class _OutputIterator, class _Assigner>
1010 void
__brick_copy_by_mask(_ForwardIterator __first,_ForwardIterator __last,_OutputIterator __result,bool * __mask,_Assigner __assigner,std::false_type)1011 __brick_copy_by_mask(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result, bool* __mask,
1012 _Assigner __assigner, /*vector=*/std::false_type) noexcept
1013 {
1014 for (; __first != __last; ++__first, ++__mask)
1015 {
1016 if (*__mask)
1017 {
1018 __assigner(__first, __result);
1019 ++__result;
1020 }
1021 }
1022 }
1023
1024 template <class _ForwardIterator, class _OutputIterator, class _Assigner>
1025 void
__brick_copy_by_mask(_ForwardIterator __first,_ForwardIterator __last,_OutputIterator __result,bool * __restrict __mask,_Assigner __assigner,std::true_type)1026 __brick_copy_by_mask(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result,
1027 bool* __restrict __mask, _Assigner __assigner, /*vector=*/std::true_type) noexcept
1028 {
1029 #if (_PSTL_MONOTONIC_PRESENT)
1030 __unseq_backend::__simd_copy_by_mask(__first, __last - __first, __result, __mask, __assigner);
1031 #else
1032 __internal::__brick_copy_by_mask(__first, __last, __result, __mask, __assigner, std::false_type());
1033 #endif
1034 }
1035
1036 template <class _ForwardIterator, class _OutputIterator1, class _OutputIterator2>
1037 void
__brick_partition_by_mask(_ForwardIterator __first,_ForwardIterator __last,_OutputIterator1 __out_true,_OutputIterator2 __out_false,bool * __mask,std::false_type)1038 __brick_partition_by_mask(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator1 __out_true,
1039 _OutputIterator2 __out_false, bool* __mask, /*vector=*/std::false_type) noexcept
1040 {
1041 for (; __first != __last; ++__first, ++__mask)
1042 {
1043 if (*__mask)
1044 {
1045 *__out_true = *__first;
1046 ++__out_true;
1047 }
1048 else
1049 {
1050 *__out_false = *__first;
1051 ++__out_false;
1052 }
1053 }
1054 }
1055
1056 template <class _RandomAccessIterator, class _OutputIterator1, class _OutputIterator2>
1057 void
__brick_partition_by_mask(_RandomAccessIterator __first,_RandomAccessIterator __last,_OutputIterator1 __out_true,_OutputIterator2 __out_false,bool * __mask,std::true_type)1058 __brick_partition_by_mask(_RandomAccessIterator __first, _RandomAccessIterator __last, _OutputIterator1 __out_true,
1059 _OutputIterator2 __out_false, bool* __mask, /*vector=*/std::true_type) noexcept
1060 {
1061 #if (_PSTL_MONOTONIC_PRESENT)
1062 __unseq_backend::__simd_partition_by_mask(__first, __last - __first, __out_true, __out_false, __mask);
1063 #else
1064 __internal::__brick_partition_by_mask(__first, __last, __out_true, __out_false, __mask, std::false_type());
1065 #endif
1066 }
1067
1068 template <class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator, class _UnaryPredicate, class _IsVector>
1069 _OutputIterator
__pattern_copy_if(_ExecutionPolicy &&,_ForwardIterator __first,_ForwardIterator __last,_OutputIterator __result,_UnaryPredicate __pred,_IsVector __is_vector,std::false_type)1070 __pattern_copy_if(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result,
1071 _UnaryPredicate __pred, _IsVector __is_vector, /*parallel=*/std::false_type) noexcept
1072 {
1073 return __internal::__brick_copy_if(__first, __last, __result, __pred, __is_vector);
1074 }
1075
1076 template <class _ExecutionPolicy, class _RandomAccessIterator, class _OutputIterator, class _UnaryPredicate,
1077 class _IsVector>
1078 _OutputIterator
__pattern_copy_if(_ExecutionPolicy && __exec,_RandomAccessIterator __first,_RandomAccessIterator __last,_OutputIterator __result,_UnaryPredicate __pred,_IsVector __is_vector,std::true_type)1079 __pattern_copy_if(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last,
1080 _OutputIterator __result, _UnaryPredicate __pred, _IsVector __is_vector, /*parallel=*/std::true_type)
1081 {
1082 typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type _DifferenceType;
1083 const _DifferenceType __n = __last - __first;
1084 if (_DifferenceType(1) < __n)
1085 {
1086 __par_backend::__buffer<bool> __mask_buf(__n);
1087 return __internal::__except_handler([&__exec, __n, __first, __result, __is_vector, __pred, &__mask_buf]() {
1088 bool* __mask = __mask_buf.get();
1089 _DifferenceType __m{};
1090 __par_backend::__parallel_strict_scan(
1091 std::forward<_ExecutionPolicy>(__exec), __n, _DifferenceType(0),
1092 [=](_DifferenceType __i, _DifferenceType __len) { // Reduce
1093 return __internal::__brick_calc_mask_1<_DifferenceType>(__first + __i, __first + (__i + __len),
1094 __mask + __i, __pred, __is_vector)
1095 .first;
1096 },
1097 std::plus<_DifferenceType>(), // Combine
1098 [=](_DifferenceType __i, _DifferenceType __len, _DifferenceType __initial) { // Scan
1099 __internal::__brick_copy_by_mask(
1100 __first + __i, __first + (__i + __len), __result + __initial, __mask + __i,
1101 [](_RandomAccessIterator __x, _OutputIterator __z) { *__z = *__x; }, __is_vector);
1102 },
1103 [&__m](_DifferenceType __total) { __m = __total; });
1104 return __result + __m;
1105 });
1106 }
1107 // trivial sequence - use serial algorithm
1108 return __internal::__brick_copy_if(__first, __last, __result, __pred, __is_vector);
1109 }
1110
1111 //------------------------------------------------------------------------
1112 // count
1113 //------------------------------------------------------------------------
1114 template <class _ForwardIterator, class _Predicate>
1115 typename std::iterator_traits<_ForwardIterator>::difference_type
__brick_count(_ForwardIterator __first,_ForwardIterator __last,_Predicate __pred,std::true_type)1116 __brick_count(_ForwardIterator __first, _ForwardIterator __last, _Predicate __pred,
1117 /* is_vector = */ std::true_type) noexcept
1118 {
1119 return __unseq_backend::__simd_count(__first, __last - __first, __pred);
1120 }
1121
1122 template <class _ForwardIterator, class _Predicate>
1123 typename std::iterator_traits<_ForwardIterator>::difference_type
__brick_count(_ForwardIterator __first,_ForwardIterator __last,_Predicate __pred,std::false_type)1124 __brick_count(_ForwardIterator __first, _ForwardIterator __last, _Predicate __pred,
1125 /* is_vector = */ std::false_type) noexcept
1126 {
1127 return std::count_if(__first, __last, __pred);
1128 }
1129
1130 template <class _ExecutionPolicy, class _ForwardIterator, class _Predicate, class _IsVector>
1131 typename std::iterator_traits<_ForwardIterator>::difference_type
__pattern_count(_ExecutionPolicy &&,_ForwardIterator __first,_ForwardIterator __last,_Predicate __pred,std::false_type,_IsVector __is_vector)1132 __pattern_count(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred,
1133 /* is_parallel */ std::false_type, _IsVector __is_vector) noexcept
1134 {
1135 return __internal::__brick_count(__first, __last, __pred, __is_vector);
1136 }
1137
1138 template <class _ExecutionPolicy, class _ForwardIterator, class _Predicate, class _IsVector>
1139 typename std::iterator_traits<_ForwardIterator>::difference_type
__pattern_count(_ExecutionPolicy && __exec,_ForwardIterator __first,_ForwardIterator __last,_Predicate __pred,std::true_type,_IsVector __is_vector)1140 __pattern_count(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred,
1141 /* is_parallel */ std::true_type, _IsVector __is_vector)
1142 {
1143 typedef typename std::iterator_traits<_ForwardIterator>::difference_type _SizeType;
1144 return __internal::__except_handler([&]() {
1145 return __par_backend::__parallel_reduce(
1146 std::forward<_ExecutionPolicy>(__exec), __first, __last, _SizeType(0),
1147 [__pred, __is_vector](_ForwardIterator __begin, _ForwardIterator __end, _SizeType __value) -> _SizeType {
1148 return __value + __internal::__brick_count(__begin, __end, __pred, __is_vector);
1149 },
1150 std::plus<_SizeType>());
1151 });
1152 }
1153
1154 //------------------------------------------------------------------------
1155 // unique
1156 //------------------------------------------------------------------------
1157
1158 template <class _ForwardIterator, class _BinaryPredicate>
1159 _ForwardIterator
__brick_unique(_ForwardIterator __first,_ForwardIterator __last,_BinaryPredicate __pred,std::false_type)1160 __brick_unique(_ForwardIterator __first, _ForwardIterator __last, _BinaryPredicate __pred,
1161 /*is_vector=*/std::false_type) noexcept
1162 {
1163 return std::unique(__first, __last, __pred);
1164 }
1165
1166 template <class _ForwardIterator, class _BinaryPredicate>
1167 _ForwardIterator
__brick_unique(_ForwardIterator __first,_ForwardIterator __last,_BinaryPredicate __pred,std::true_type)1168 __brick_unique(_ForwardIterator __first, _ForwardIterator __last, _BinaryPredicate __pred,
1169 /*is_vector=*/std::true_type) noexcept
1170 {
1171 _PSTL_PRAGMA_MESSAGE("Vectorized algorithm unimplemented, redirected to serial");
1172 return std::unique(__first, __last, __pred);
1173 }
1174
1175 template <class _ExecutionPolicy, class _ForwardIterator, class _BinaryPredicate, class _IsVector>
1176 _ForwardIterator
__pattern_unique(_ExecutionPolicy &&,_ForwardIterator __first,_ForwardIterator __last,_BinaryPredicate __pred,_IsVector __is_vector,std::false_type)1177 __pattern_unique(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _BinaryPredicate __pred,
1178 _IsVector __is_vector, /*is_parallel=*/std::false_type) noexcept
1179 {
1180 return __internal::__brick_unique(__first, __last, __pred, __is_vector);
1181 }
1182
1183 // That function is shared between two algorithms - remove_if (__pattern_remove_if) and unique (pattern unique). But a mask calculation is different.
1184 // So, a caller passes _CalcMask brick into remove_elements.
1185 template <class _ExecutionPolicy, class _ForwardIterator, class _CalcMask, class _IsVector>
1186 _ForwardIterator
__remove_elements(_ExecutionPolicy && __exec,_ForwardIterator __first,_ForwardIterator __last,_CalcMask __calc_mask,_IsVector __is_vector)1187 __remove_elements(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _CalcMask __calc_mask,
1188 _IsVector __is_vector)
1189 {
1190 typedef typename std::iterator_traits<_ForwardIterator>::difference_type _DifferenceType;
1191 typedef typename std::iterator_traits<_ForwardIterator>::value_type _Tp;
1192 _DifferenceType __n = __last - __first;
1193 __par_backend::__buffer<bool> __mask_buf(__n);
1194 // 1. find a first iterator that should be removed
1195 return __internal::__except_handler([&]() {
1196 bool* __mask = __mask_buf.get();
1197 _DifferenceType __min = __par_backend::__parallel_reduce(
1198 std::forward<_ExecutionPolicy>(__exec), _DifferenceType(0), __n, __n,
1199 [__first, __mask, &__calc_mask, __is_vector](_DifferenceType __i, _DifferenceType __j,
1200 _DifferenceType __local_min) -> _DifferenceType {
1201 // Create mask
1202 __calc_mask(__mask + __i, __mask + __j, __first + __i);
1203
1204 // if minimum was found in a previous range we shouldn't do anymore
1205 if (__local_min < __i)
1206 {
1207 return __local_min;
1208 }
1209 // find first iterator that should be removed
1210 bool* __result = __internal::__brick_find_if(__mask + __i, __mask + __j,
1211 [](bool __val) { return !__val; }, __is_vector);
1212 if (__result - __mask == __j)
1213 {
1214 return __local_min;
1215 }
1216 return std::min(__local_min, _DifferenceType(__result - __mask));
1217 },
1218 [](_DifferenceType __local_min1, _DifferenceType __local_min2) -> _DifferenceType {
1219 return std::min(__local_min1, __local_min2);
1220 });
1221
1222 // No elements to remove - exit
1223 if (__min == __n)
1224 {
1225 return __last;
1226 }
1227 __n -= __min;
1228 __first += __min;
1229
1230 __par_backend::__buffer<_Tp> __buf(__n);
1231 _Tp* __result = __buf.get();
1232 __mask += __min;
1233 _DifferenceType __m{};
1234 // 2. Elements that doesn't satisfy pred are moved to result
1235 __par_backend::__parallel_strict_scan(
1236 std::forward<_ExecutionPolicy>(__exec), __n, _DifferenceType(0),
1237 [__mask, __is_vector](_DifferenceType __i, _DifferenceType __len) {
1238 return __internal::__brick_count(__mask + __i, __mask + __i + __len, [](bool __val) { return __val; },
1239 __is_vector);
1240 },
1241 std::plus<_DifferenceType>(),
1242 [=](_DifferenceType __i, _DifferenceType __len, _DifferenceType __initial) {
1243 __internal::__brick_copy_by_mask(
1244 __first + __i, __first + __i + __len, __result + __initial, __mask + __i,
1245 [](_ForwardIterator __x, _Tp* __z) {
1246 __internal::__invoke_if_else(std::is_trivial<_Tp>(), [&]() { *__z = std::move(*__x); },
1247 [&]() { ::new (std::addressof(*__z)) _Tp(std::move(*__x)); });
1248 },
1249 __is_vector);
1250 },
1251 [&__m](_DifferenceType __total) { __m = __total; });
1252
1253 // 3. Elements from result are moved to [first, last)
1254 __par_backend::__parallel_for(
1255 std::forward<_ExecutionPolicy>(__exec), __result, __result + __m,
1256 [__result, __first, __is_vector](_Tp* __i, _Tp* __j) {
1257 __invoke_if_else(
1258 std::is_trivial<_Tp>(),
1259 [&]() { __brick_move(__i, __j, __first + (__i - __result), __is_vector); },
1260 [&]() {
1261 __brick_move_destroy()(__i, __j, __first + (__i - __result), __is_vector);
1262 });
1263 });
1264 return __first + __m;
1265 });
1266 }
1267
1268 template <class _ExecutionPolicy, class _ForwardIterator, class _BinaryPredicate, class _IsVector>
1269 _ForwardIterator
__pattern_unique(_ExecutionPolicy && __exec,_ForwardIterator __first,_ForwardIterator __last,_BinaryPredicate __pred,_IsVector __is_vector,std::true_type)1270 __pattern_unique(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _BinaryPredicate __pred,
1271 _IsVector __is_vector, /*is_parallel=*/std::true_type) noexcept
1272 {
1273 typedef typename std::iterator_traits<_ForwardIterator>::reference _ReferenceType;
1274
1275 if (__first == __last)
1276 {
1277 return __last;
1278 }
1279 if (__first + 1 == __last || __first + 2 == __last)
1280 {
1281 // Trivial sequence - use serial algorithm
1282 return __internal::__brick_unique(__first, __last, __pred, __is_vector);
1283 }
1284 return __internal::__remove_elements(
1285 std::forward<_ExecutionPolicy>(__exec), ++__first, __last,
1286 [&__pred, __is_vector](bool* __b, bool* __e, _ForwardIterator __it) {
1287 __internal::__brick_walk3(
1288 __b, __e, __it - 1, __it,
1289 [&__pred](bool& __x, _ReferenceType __y, _ReferenceType __z) { __x = !__pred(__y, __z); }, __is_vector);
1290 },
1291 __is_vector);
1292 }
1293
1294 //------------------------------------------------------------------------
1295 // unique_copy
1296 //------------------------------------------------------------------------
1297
1298 template <class _ForwardIterator, class OutputIterator, class _BinaryPredicate>
1299 OutputIterator
__brick_unique_copy(_ForwardIterator __first,_ForwardIterator __last,OutputIterator __result,_BinaryPredicate __pred,std::false_type)1300 __brick_unique_copy(_ForwardIterator __first, _ForwardIterator __last, OutputIterator __result, _BinaryPredicate __pred,
1301 /*vector=*/std::false_type) noexcept
1302 {
1303 return std::unique_copy(__first, __last, __result, __pred);
1304 }
1305
1306 template <class _RandomAccessIterator, class OutputIterator, class _BinaryPredicate>
1307 OutputIterator
__brick_unique_copy(_RandomAccessIterator __first,_RandomAccessIterator __last,OutputIterator __result,_BinaryPredicate __pred,std::true_type)1308 __brick_unique_copy(_RandomAccessIterator __first, _RandomAccessIterator __last, OutputIterator __result,
1309 _BinaryPredicate __pred, /*vector=*/std::true_type) noexcept
1310 {
1311 #if (_PSTL_MONOTONIC_PRESENT)
1312 return __unseq_backend::__simd_unique_copy(__first, __last - __first, __result, __pred);
1313 #else
1314 return std::unique_copy(__first, __last, __result, __pred);
1315 #endif
1316 }
1317
1318 template <class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator, class _BinaryPredicate,
1319 class _IsVector>
1320 _OutputIterator
__pattern_unique_copy(_ExecutionPolicy &&,_ForwardIterator __first,_ForwardIterator __last,_OutputIterator __result,_BinaryPredicate __pred,_IsVector __is_vector,std::false_type)1321 __pattern_unique_copy(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result,
1322 _BinaryPredicate __pred, _IsVector __is_vector, /*parallel=*/std::false_type) noexcept
1323 {
1324 return __internal::__brick_unique_copy(__first, __last, __result, __pred, __is_vector);
1325 }
1326
1327 template <class _DifferenceType, class _RandomAccessIterator, class _BinaryPredicate>
1328 _DifferenceType
__brick_calc_mask_2(_RandomAccessIterator __first,_RandomAccessIterator __last,bool * __restrict __mask,_BinaryPredicate __pred,std::false_type)1329 __brick_calc_mask_2(_RandomAccessIterator __first, _RandomAccessIterator __last, bool* __restrict __mask,
1330 _BinaryPredicate __pred, /*vector=*/std::false_type) noexcept
1331 {
1332 _DifferenceType __count = 0;
1333 for (; __first != __last; ++__first, ++__mask)
1334 {
1335 *__mask = !__pred(*__first, *(__first - 1));
1336 __count += *__mask;
1337 }
1338 return __count;
1339 }
1340
1341 template <class _DifferenceType, class _RandomAccessIterator, class _BinaryPredicate>
1342 _DifferenceType
__brick_calc_mask_2(_RandomAccessIterator __first,_RandomAccessIterator __last,bool * __restrict __mask,_BinaryPredicate __pred,std::true_type)1343 __brick_calc_mask_2(_RandomAccessIterator __first, _RandomAccessIterator __last, bool* __restrict __mask,
1344 _BinaryPredicate __pred, /*vector=*/std::true_type) noexcept
1345 {
1346 return __unseq_backend::__simd_calc_mask_2(__first, __last - __first, __mask, __pred);
1347 }
1348
1349 template <class _ExecutionPolicy, class _RandomAccessIterator, class _OutputIterator, class _BinaryPredicate,
1350 class _IsVector>
1351 _OutputIterator
__pattern_unique_copy(_ExecutionPolicy && __exec,_RandomAccessIterator __first,_RandomAccessIterator __last,_OutputIterator __result,_BinaryPredicate __pred,_IsVector __is_vector,std::true_type)1352 __pattern_unique_copy(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last,
1353 _OutputIterator __result, _BinaryPredicate __pred, _IsVector __is_vector,
1354 /*parallel=*/std::true_type)
1355 {
1356 typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type _DifferenceType;
1357 const _DifferenceType __n = __last - __first;
1358 if (_DifferenceType(2) < __n)
1359 {
1360 __par_backend::__buffer<bool> __mask_buf(__n);
1361 if (_DifferenceType(2) < __n)
1362 {
1363 return __internal::__except_handler([&__exec, __n, __first, __result, __pred, __is_vector, &__mask_buf]() {
1364 bool* __mask = __mask_buf.get();
1365 _DifferenceType __m{};
1366 __par_backend::__parallel_strict_scan(
1367 std::forward<_ExecutionPolicy>(__exec), __n, _DifferenceType(0),
1368 [=](_DifferenceType __i, _DifferenceType __len) -> _DifferenceType { // Reduce
1369 _DifferenceType __extra = 0;
1370 if (__i == 0)
1371 {
1372 // Special boundary case
1373 __mask[__i] = true;
1374 if (--__len == 0)
1375 return 1;
1376 ++__i;
1377 ++__extra;
1378 }
1379 return __internal::__brick_calc_mask_2<_DifferenceType>(__first + __i, __first + (__i + __len),
1380 __mask + __i, __pred, __is_vector) +
1381 __extra;
1382 },
1383 std::plus<_DifferenceType>(), // Combine
1384 [=](_DifferenceType __i, _DifferenceType __len, _DifferenceType __initial) { // Scan
1385 // Phase 2 is same as for __pattern_copy_if
1386 __internal::__brick_copy_by_mask(
1387 __first + __i, __first + (__i + __len), __result + __initial, __mask + __i,
1388 [](_RandomAccessIterator __x, _OutputIterator __z) { *__z = *__x; }, __is_vector);
1389 },
1390 [&__m](_DifferenceType __total) { __m = __total; });
1391 return __result + __m;
1392 });
1393 }
1394 }
1395 // trivial sequence - use serial algorithm
1396 return __internal::__brick_unique_copy(__first, __last, __result, __pred, __is_vector);
1397 }
1398
1399 //------------------------------------------------------------------------
1400 // reverse
1401 //------------------------------------------------------------------------
1402 template <class _BidirectionalIterator>
1403 void
__brick_reverse(_BidirectionalIterator __first,_BidirectionalIterator __last,std::false_type)1404 __brick_reverse(_BidirectionalIterator __first, _BidirectionalIterator __last, /*__is_vector=*/std::false_type) noexcept
1405 {
1406 std::reverse(__first, __last);
1407 }
1408
1409 template <class _BidirectionalIterator>
1410 void
__brick_reverse(_BidirectionalIterator __first,_BidirectionalIterator __last,std::true_type)1411 __brick_reverse(_BidirectionalIterator __first, _BidirectionalIterator __last, /*__is_vector=*/std::true_type) noexcept
1412 {
1413 typedef typename std::iterator_traits<_BidirectionalIterator>::reference _ReferenceType;
1414
1415 const auto __n = (__last - __first) / 2;
1416 __unseq_backend::__simd_walk_2(__first, __n, std::reverse_iterator<_BidirectionalIterator>(__last),
1417 [](_ReferenceType __x, _ReferenceType __y) {
1418 using std::swap;
1419 swap(__x, __y);
1420 });
1421 }
1422
1423 // this brick is called in parallel version, so we can use iterator arithmetic
1424 template <class _BidirectionalIterator>
1425 void
__brick_reverse(_BidirectionalIterator __first,_BidirectionalIterator __last,_BidirectionalIterator __d_last,std::false_type)1426 __brick_reverse(_BidirectionalIterator __first, _BidirectionalIterator __last, _BidirectionalIterator __d_last,
1427 /*is_vector=*/std::false_type) noexcept
1428 {
1429 for (--__d_last; __first != __last; ++__first, --__d_last)
1430 {
1431 using std::iter_swap;
1432 iter_swap(__first, __d_last);
1433 }
1434 }
1435
1436 // this brick is called in parallel version, so we can use iterator arithmetic
1437 template <class _BidirectionalIterator>
1438 void
__brick_reverse(_BidirectionalIterator __first,_BidirectionalIterator __last,_BidirectionalIterator __d_last,std::true_type)1439 __brick_reverse(_BidirectionalIterator __first, _BidirectionalIterator __last, _BidirectionalIterator __d_last,
1440 /*is_vector=*/std::true_type) noexcept
1441 {
1442 typedef typename std::iterator_traits<_BidirectionalIterator>::reference _ReferenceType;
1443
1444 __unseq_backend::__simd_walk_2(__first, __last - __first, std::reverse_iterator<_BidirectionalIterator>(__d_last),
1445 [](_ReferenceType __x, _ReferenceType __y) {
1446 using std::swap;
1447 swap(__x, __y);
1448 });
1449 }
1450
1451 template <class _ExecutionPolicy, class _BidirectionalIterator, class _IsVector>
1452 void
__pattern_reverse(_ExecutionPolicy &&,_BidirectionalIterator __first,_BidirectionalIterator __last,_IsVector _is_vector,std::false_type)1453 __pattern_reverse(_ExecutionPolicy&&, _BidirectionalIterator __first, _BidirectionalIterator __last,
1454 _IsVector _is_vector,
1455 /*is_parallel=*/std::false_type) noexcept
1456 {
1457 __internal::__brick_reverse(__first, __last, _is_vector);
1458 }
1459
1460 template <class _ExecutionPolicy, class _BidirectionalIterator, class _IsVector>
1461 void
__pattern_reverse(_ExecutionPolicy && __exec,_BidirectionalIterator __first,_BidirectionalIterator __last,_IsVector __is_vector,std::true_type)1462 __pattern_reverse(_ExecutionPolicy&& __exec, _BidirectionalIterator __first, _BidirectionalIterator __last,
1463 _IsVector __is_vector, /*is_parallel=*/std::true_type)
1464 {
1465 __par_backend::__parallel_for(
1466 std::forward<_ExecutionPolicy>(__exec), __first, __first + (__last - __first) / 2,
1467 [__is_vector, __first, __last](_BidirectionalIterator __inner_first, _BidirectionalIterator __inner_last) {
1468 __internal::__brick_reverse(__inner_first, __inner_last, __last - (__inner_first - __first), __is_vector);
1469 });
1470 }
1471
1472 //------------------------------------------------------------------------
1473 // reverse_copy
1474 //------------------------------------------------------------------------
1475
1476 template <class _BidirectionalIterator, class _OutputIterator>
1477 _OutputIterator
__brick_reverse_copy(_BidirectionalIterator __first,_BidirectionalIterator __last,_OutputIterator __d_first,std::false_type)1478 __brick_reverse_copy(_BidirectionalIterator __first, _BidirectionalIterator __last, _OutputIterator __d_first,
1479 /*is_vector=*/std::false_type) noexcept
1480 {
1481 return std::reverse_copy(__first, __last, __d_first);
1482 }
1483
1484 template <class _BidirectionalIterator, class _OutputIterator>
1485 _OutputIterator
__brick_reverse_copy(_BidirectionalIterator __first,_BidirectionalIterator __last,_OutputIterator __d_first,std::true_type)1486 __brick_reverse_copy(_BidirectionalIterator __first, _BidirectionalIterator __last, _OutputIterator __d_first,
1487 /*is_vector=*/std::true_type) noexcept
1488 {
1489 typedef typename std::iterator_traits<_BidirectionalIterator>::reference _ReferenceType1;
1490 typedef typename std::iterator_traits<_OutputIterator>::reference _ReferenceType2;
1491
1492 return __unseq_backend::__simd_walk_2(std::reverse_iterator<_BidirectionalIterator>(__last), __last - __first,
1493 __d_first, [](_ReferenceType1 __x, _ReferenceType2 __y) { __y = __x; });
1494 }
1495
1496 template <class _ExecutionPolicy, class _BidirectionalIterator, class _OutputIterator, class _IsVector>
1497 _OutputIterator
__pattern_reverse_copy(_ExecutionPolicy &&,_BidirectionalIterator __first,_BidirectionalIterator __last,_OutputIterator __d_first,_IsVector __is_vector,std::false_type)1498 __pattern_reverse_copy(_ExecutionPolicy&&, _BidirectionalIterator __first, _BidirectionalIterator __last,
1499 _OutputIterator __d_first, _IsVector __is_vector, /*is_parallel=*/std::false_type) noexcept
1500 {
1501 return __internal::__brick_reverse_copy(__first, __last, __d_first, __is_vector);
1502 }
1503
1504 template <class _ExecutionPolicy, class _BidirectionalIterator, class _OutputIterator, class _IsVector>
1505 _OutputIterator
__pattern_reverse_copy(_ExecutionPolicy && __exec,_BidirectionalIterator __first,_BidirectionalIterator __last,_OutputIterator __d_first,_IsVector __is_vector,std::true_type)1506 __pattern_reverse_copy(_ExecutionPolicy&& __exec, _BidirectionalIterator __first, _BidirectionalIterator __last,
1507 _OutputIterator __d_first, _IsVector __is_vector, /*is_parallel=*/std::true_type)
1508 {
1509 auto __len = __last - __first;
1510 __par_backend::__parallel_for(std::forward<_ExecutionPolicy>(__exec), __first, __last,
1511 [__is_vector, __first, __len, __d_first](_BidirectionalIterator __inner_first,
1512 _BidirectionalIterator __inner_last) {
1513 __internal::__brick_reverse_copy(__inner_first, __inner_last,
1514 __d_first + (__len - (__inner_last - __first)),
1515 __is_vector);
1516 });
1517 return __d_first + __len;
1518 }
1519
1520 //------------------------------------------------------------------------
1521 // rotate
1522 //------------------------------------------------------------------------
1523 template <class _ForwardIterator>
1524 _ForwardIterator
__brick_rotate(_ForwardIterator __first,_ForwardIterator __middle,_ForwardIterator __last,std::false_type)1525 __brick_rotate(_ForwardIterator __first, _ForwardIterator __middle, _ForwardIterator __last,
1526 /*is_vector=*/std::false_type) noexcept
1527 {
1528 #if _PSTL_CPP11_STD_ROTATE_BROKEN
1529 std::rotate(__first, __middle, __last);
1530 return std::next(__first, std::distance(__middle, __last));
1531 #else
1532 return std::rotate(__first, __middle, __last);
1533 #endif
1534 }
1535
1536 template <class _ForwardIterator>
1537 _ForwardIterator
__brick_rotate(_ForwardIterator __first,_ForwardIterator __middle,_ForwardIterator __last,std::true_type)1538 __brick_rotate(_ForwardIterator __first, _ForwardIterator __middle, _ForwardIterator __last,
1539 /*is_vector=*/std::true_type) noexcept
1540 {
1541 auto __n = __last - __first;
1542 auto __m = __middle - __first;
1543 const _ForwardIterator __ret = __first + (__last - __middle);
1544
1545 bool __is_left = (__m <= __n / 2);
1546 if (!__is_left)
1547 __m = __n - __m;
1548
1549 while (__n > 1 && __m > 0)
1550 {
1551 using std::iter_swap;
1552 const auto __m_2 = __m * 2;
1553 if (__is_left)
1554 {
1555 for (; __last - __first >= __m_2; __first += __m)
1556 {
1557 __unseq_backend::__simd_assign(__first, __m, __first + __m,
1558 iter_swap<_ForwardIterator, _ForwardIterator>);
1559 }
1560 }
1561 else
1562 {
1563 for (; __last - __first >= __m_2; __last -= __m)
1564 {
1565 __unseq_backend::__simd_assign(__last - __m, __m, __last - __m_2,
1566 iter_swap<_ForwardIterator, _ForwardIterator>);
1567 }
1568 }
1569 __is_left = !__is_left;
1570 __m = __n % __m;
1571 __n = __last - __first;
1572 }
1573
1574 return __ret;
1575 }
1576
1577 template <class _ExecutionPolicy, class _ForwardIterator, class _IsVector>
1578 _ForwardIterator
__pattern_rotate(_ExecutionPolicy &&,_ForwardIterator __first,_ForwardIterator __middle,_ForwardIterator __last,_IsVector __is_vector,std::false_type)1579 __pattern_rotate(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __middle, _ForwardIterator __last,
1580 _IsVector __is_vector, /*is_parallel=*/std::false_type) noexcept
1581 {
1582 return __internal::__brick_rotate(__first, __middle, __last, __is_vector);
1583 }
1584
1585 template <class _ExecutionPolicy, class _ForwardIterator, class _IsVector>
1586 _ForwardIterator
__pattern_rotate(_ExecutionPolicy && __exec,_ForwardIterator __first,_ForwardIterator __middle,_ForwardIterator __last,_IsVector __is_vector,std::true_type)1587 __pattern_rotate(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __middle,
1588 _ForwardIterator __last, _IsVector __is_vector, /*is_parallel=*/std::true_type)
1589 {
1590 typedef typename std::iterator_traits<_ForwardIterator>::value_type _Tp;
1591 auto __n = __last - __first;
1592 auto __m = __middle - __first;
1593 if (__m <= __n / 2)
1594 {
1595 __par_backend::__buffer<_Tp> __buf(__n - __m);
1596 return __internal::__except_handler([&__exec, __n, __m, __first, __middle, __last, __is_vector, &__buf]() {
1597 _Tp* __result = __buf.get();
1598 __par_backend::__parallel_for(
1599 std::forward<_ExecutionPolicy>(__exec), __middle, __last,
1600 [__middle, __result, __is_vector](_ForwardIterator __b, _ForwardIterator __e) {
1601 __internal::__brick_uninitialized_move(__b, __e, __result + (__b - __middle), __is_vector);
1602 });
1603
1604 __par_backend::__parallel_for(std::forward<_ExecutionPolicy>(__exec), __first, __middle,
1605 [__last, __middle, __is_vector](_ForwardIterator __b, _ForwardIterator __e) {
1606 __internal::__brick_move(__b, __e, __b + (__last - __middle),
1607 __is_vector);
1608 });
1609
1610 __par_backend::__parallel_for(std::forward<_ExecutionPolicy>(__exec), __result, __result + (__n - __m),
1611 [__first, __result, __is_vector](_Tp* __b, _Tp* __e) {
1612 __brick_move_destroy()(
1613 __b, __e, __first + (__b - __result), __is_vector);
1614 });
1615
1616 return __first + (__last - __middle);
1617 });
1618 }
1619 else
1620 {
1621 __par_backend::__buffer<_Tp> __buf(__m);
1622 return __internal::__except_handler([&__exec, __n, __m, __first, __middle, __last, __is_vector, &__buf]() {
1623 _Tp* __result = __buf.get();
1624 __par_backend::__parallel_for(std::forward<_ExecutionPolicy>(__exec), __first, __middle,
1625 [__first, __result, __is_vector](_ForwardIterator __b, _ForwardIterator __e) {
1626 __internal::__brick_uninitialized_move(
1627 __b, __e, __result + (__b - __first), __is_vector);
1628 });
1629
1630 __par_backend::__parallel_for(std::forward<_ExecutionPolicy>(__exec), __middle, __last,
1631 [__first, __middle, __is_vector](_ForwardIterator __b, _ForwardIterator __e) {
1632 __internal::__brick_move(__b, __e, __first + (__b - __middle),
1633 __is_vector);
1634 });
1635
1636 __par_backend::__parallel_for(std::forward<_ExecutionPolicy>(__exec), __result, __result + __m,
1637 [__n, __m, __first, __result, __is_vector](_Tp* __b, _Tp* __e) {
1638 __brick_move_destroy()(
1639 __b, __e, __first + ((__n - __m) + (__b - __result)), __is_vector);
1640 });
1641
1642 return __first + (__last - __middle);
1643 });
1644 }
1645 }
1646
1647 //------------------------------------------------------------------------
1648 // rotate_copy
1649 //------------------------------------------------------------------------
1650
1651 template <class _ForwardIterator, class _OutputIterator>
1652 _OutputIterator
__brick_rotate_copy(_ForwardIterator __first,_ForwardIterator __middle,_ForwardIterator __last,_OutputIterator __result,std::false_type)1653 __brick_rotate_copy(_ForwardIterator __first, _ForwardIterator __middle, _ForwardIterator __last,
1654 _OutputIterator __result, /*__is_vector=*/std::false_type) noexcept
1655 {
1656 return std::rotate_copy(__first, __middle, __last, __result);
1657 }
1658
1659 template <class _ForwardIterator, class _OutputIterator>
1660 _OutputIterator
__brick_rotate_copy(_ForwardIterator __first,_ForwardIterator __middle,_ForwardIterator __last,_OutputIterator __result,std::true_type)1661 __brick_rotate_copy(_ForwardIterator __first, _ForwardIterator __middle, _ForwardIterator __last,
1662 _OutputIterator __result, /*__is_vector=*/std::true_type) noexcept
1663 {
1664 _OutputIterator __res = __internal::__brick_copy(__middle, __last, __result, std::true_type());
1665 return __internal::__brick_copy(__first, __middle, __res, std::true_type());
1666 }
1667
1668 template <class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator, class _IsVector>
1669 _OutputIterator
__pattern_rotate_copy(_ExecutionPolicy &&,_ForwardIterator __first,_ForwardIterator __middle,_ForwardIterator __last,_OutputIterator __result,_IsVector __is_vector,std::false_type)1670 __pattern_rotate_copy(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __middle, _ForwardIterator __last,
1671 _OutputIterator __result, _IsVector __is_vector, /*is_parallel=*/std::false_type) noexcept
1672 {
1673 return __internal::__brick_rotate_copy(__first, __middle, __last, __result, __is_vector);
1674 }
1675
1676 template <class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator, class _IsVector>
1677 _OutputIterator
__pattern_rotate_copy(_ExecutionPolicy && __exec,_ForwardIterator __first,_ForwardIterator __middle,_ForwardIterator __last,_OutputIterator __result,_IsVector __is_vector,std::true_type)1678 __pattern_rotate_copy(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __middle,
1679 _ForwardIterator __last, _OutputIterator __result, _IsVector __is_vector,
1680 /*is_parallel=*/std::true_type)
1681 {
1682 __par_backend::__parallel_for(
1683 std::forward<_ExecutionPolicy>(__exec), __first, __last,
1684 [__first, __last, __middle, __result, __is_vector](_ForwardIterator __b, _ForwardIterator __e) {
1685 if (__b > __middle)
1686 {
1687 __internal::__brick_copy(__b, __e, __result + (__b - __middle), __is_vector);
1688 }
1689 else
1690 {
1691 _OutputIterator __new_result = __result + ((__last - __middle) + (__b - __first));
1692 if (__e < __middle)
1693 {
1694 __internal::__brick_copy(__b, __e, __new_result, __is_vector);
1695 }
1696 else
1697 {
1698 __internal::__brick_copy(__b, __middle, __new_result, __is_vector);
1699 __internal::__brick_copy(__middle, __e, __result, __is_vector);
1700 }
1701 }
1702 });
1703 return __result + (__last - __first);
1704 }
1705
1706 //------------------------------------------------------------------------
1707 // is_partitioned
1708 //------------------------------------------------------------------------
1709
1710 template <class _ForwardIterator, class _UnaryPredicate>
1711 bool
__brick_is_partitioned(_ForwardIterator __first,_ForwardIterator __last,_UnaryPredicate __pred,std::false_type)1712 __brick_is_partitioned(_ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred,
1713 /*is_vector=*/std::false_type) noexcept
1714 {
1715 return std::is_partitioned(__first, __last, __pred);
1716 }
1717
1718 template <class _ForwardIterator, class _UnaryPredicate>
1719 bool
__brick_is_partitioned(_ForwardIterator __first,_ForwardIterator __last,_UnaryPredicate __pred,std::true_type)1720 __brick_is_partitioned(_ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred,
1721 /*is_vector=*/std::true_type) noexcept
1722 {
1723 typedef typename std::iterator_traits<_ForwardIterator>::difference_type _SizeType;
1724 if (__first == __last)
1725 {
1726 return true;
1727 }
1728 else
1729 {
1730 _ForwardIterator __result = __unseq_backend::__simd_first(
1731 __first, _SizeType(0), __last - __first,
1732 [&__pred](_ForwardIterator __it, _SizeType __i) { return !__pred(__it[__i]); });
1733 if (__result == __last)
1734 {
1735 return true;
1736 }
1737 else
1738 {
1739 ++__result;
1740 return !__unseq_backend::__simd_or(__result, __last - __result, __pred);
1741 }
1742 }
1743 }
1744
1745 template <class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate, class _IsVector>
1746 bool
__pattern_is_partitioned(_ExecutionPolicy &&,_ForwardIterator __first,_ForwardIterator __last,_UnaryPredicate __pred,_IsVector __is_vector,std::false_type)1747 __pattern_is_partitioned(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred,
1748 _IsVector __is_vector, /*is_parallel=*/std::false_type) noexcept
1749 {
1750 return __internal::__brick_is_partitioned(__first, __last, __pred, __is_vector);
1751 }
1752
1753 template <class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate, class _IsVector>
1754 bool
__pattern_is_partitioned(_ExecutionPolicy && __exec,_ForwardIterator __first,_ForwardIterator __last,_UnaryPredicate __pred,_IsVector __is_vector,std::true_type)1755 __pattern_is_partitioned(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last,
1756 _UnaryPredicate __pred, _IsVector __is_vector, /*is_parallel=*/std::true_type)
1757 {
1758 if (__first == __last)
1759 {
1760 return true;
1761 }
1762 else
1763 {
1764 return __internal::__except_handler([&]() {
1765 // State of current range:
1766 // broken - current range is not partitioned by pred
1767 // all_true - all elements in current range satisfy pred
1768 // all_false - all elements in current range don't satisfy pred
1769 // true_false - elements satisfy pred are placed before elements that don't satisfy pred
1770 enum _ReduceType
1771 {
1772 __not_init = -1,
1773 __broken,
1774 __all_true,
1775 __all_false,
1776 __true_false
1777 };
1778 _ReduceType __init = __not_init;
1779
1780 // Array with states that we'll have when state from the left branch is merged with state from the right branch.
1781 // State is calculated by formula: new_state = table[left_state * 4 + right_state]
1782 _ReduceType __table[] = {__broken, __broken, __broken, __broken, __broken, __all_true,
1783 __true_false, __true_false, __broken, __broken, __all_false, __broken,
1784 __broken, __broken, __true_false, __broken};
1785
1786 __init = __par_backend::__parallel_reduce(
1787 std::forward<_ExecutionPolicy>(__exec), __first, __last, __init,
1788 [&__pred, &__table, __is_vector](_ForwardIterator __i, _ForwardIterator __j,
1789 _ReduceType __value) -> _ReduceType {
1790 if (__value == __broken)
1791 {
1792 return __broken;
1793 }
1794 _ReduceType __res = __not_init;
1795 // if first element satisfy pred
1796 if (__pred(*__i))
1797 {
1798 // find first element that don't satisfy pred
1799 _ForwardIterator __x =
1800 __internal::__brick_find_if(__i + 1, __j, std::not_fn(__pred), __is_vector);
1801 if (__x != __j)
1802 {
1803 // find first element after "x" that satisfy pred
1804 _ForwardIterator __y = __internal::__brick_find_if(__x + 1, __j, __pred, __is_vector);
1805 // if it was found then range isn't partitioned by pred
1806 if (__y != __j)
1807 {
1808 return __broken;
1809 }
1810 else
1811 {
1812 __res = __true_false;
1813 }
1814 }
1815 else
1816 {
1817 __res = __all_true;
1818 }
1819 }
1820 else
1821 { // if first element doesn't satisfy pred
1822 // then we should find the first element that satisfy pred.
1823 // If we found it then range isn't partitioned by pred
1824 if (__internal::__brick_find_if(__i + 1, __j, __pred, __is_vector) != __j)
1825 {
1826 return __broken;
1827 }
1828 else
1829 {
1830 __res = __all_false;
1831 }
1832 }
1833 // if we have value from left range then we should calculate the result
1834 return (__value == -1) ? __res : __table[__value * 4 + __res];
1835 },
1836
1837 [&__table](_ReduceType __val1, _ReduceType __val2) -> _ReduceType {
1838 if (__val1 == __broken || __val2 == __broken)
1839 {
1840 return __broken;
1841 }
1842 // calculate the result for new big range
1843 return __table[__val1 * 4 + __val2];
1844 });
1845 return __init != __broken;
1846 });
1847 }
1848 }
1849
1850 //------------------------------------------------------------------------
1851 // partition
1852 //------------------------------------------------------------------------
1853
1854 template <class _ForwardIterator, class _UnaryPredicate>
1855 _ForwardIterator
__brick_partition(_ForwardIterator __first,_ForwardIterator __last,_UnaryPredicate __pred,std::false_type)1856 __brick_partition(_ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred,
1857 /*is_vector=*/std::false_type) noexcept
1858 {
1859 return std::partition(__first, __last, __pred);
1860 }
1861
1862 template <class _ForwardIterator, class _UnaryPredicate>
1863 _ForwardIterator
__brick_partition(_ForwardIterator __first,_ForwardIterator __last,_UnaryPredicate __pred,std::true_type)1864 __brick_partition(_ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred,
1865 /*is_vector=*/std::true_type) noexcept
1866 {
1867 _PSTL_PRAGMA_MESSAGE("Vectorized algorithm unimplemented, redirected to serial");
1868 return std::partition(__first, __last, __pred);
1869 }
1870
1871 template <class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate, class _IsVector>
1872 _ForwardIterator
__pattern_partition(_ExecutionPolicy &&,_ForwardIterator __first,_ForwardIterator __last,_UnaryPredicate __pred,_IsVector __is_vector,std::false_type)1873 __pattern_partition(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred,
1874 _IsVector __is_vector, /*is_parallel=*/std::false_type) noexcept
1875 {
1876 return __internal::__brick_partition(__first, __last, __pred, __is_vector);
1877 }
1878
1879 template <class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate, class _IsVector>
1880 _ForwardIterator
__pattern_partition(_ExecutionPolicy && __exec,_ForwardIterator __first,_ForwardIterator __last,_UnaryPredicate __pred,_IsVector __is_vector,std::true_type)1881 __pattern_partition(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last,
1882 _UnaryPredicate __pred, _IsVector __is_vector, /*is_parallel=*/std::true_type)
1883 {
1884
1885 // partitioned range: elements before pivot satisfy pred (true part),
1886 // elements after pivot don't satisfy pred (false part)
1887 struct _PartitionRange
1888 {
1889 _ForwardIterator __begin;
1890 _ForwardIterator __pivot;
1891 _ForwardIterator __end;
1892 };
1893
1894 return __internal::__except_handler([&]() {
1895 _PartitionRange __init{__last, __last, __last};
1896
1897 // lambda for merging two partitioned ranges to one partitioned range
1898 auto __reductor = [&__exec, __is_vector](_PartitionRange __val1, _PartitionRange __val2) -> _PartitionRange {
1899 auto __size1 = __val1.__end - __val1.__pivot;
1900 auto __size2 = __val2.__pivot - __val2.__begin;
1901 auto __new_begin = __val2.__begin - (__val1.__end - __val1.__begin);
1902
1903 // if all elements in left range satisfy pred then we can move new pivot to pivot of right range
1904 if (__val1.__end == __val1.__pivot)
1905 {
1906 return {__new_begin, __val2.__pivot, __val2.__end};
1907 }
1908 // if true part of right range greater than false part of left range
1909 // then we should swap the false part of left range and last part of true part of right range
1910 else if (__size2 > __size1)
1911 {
1912 __par_backend::__parallel_for(
1913 std::forward<_ExecutionPolicy>(__exec), __val1.__pivot, __val1.__pivot + __size1,
1914 [__val1, __val2, __size1, __is_vector](_ForwardIterator __i, _ForwardIterator __j) {
1915 __internal::__brick_swap_ranges(__i, __j, (__val2.__pivot - __size1) + (__i - __val1.__pivot),
1916 __is_vector);
1917 });
1918 return {__new_begin, __val2.__pivot - __size1, __val2.__end};
1919 }
1920 // else we should swap the first part of false part of left range and true part of right range
1921 else
1922 {
1923 __par_backend::__parallel_for(
1924 std::forward<_ExecutionPolicy>(__exec), __val1.__pivot, __val1.__pivot + __size2,
1925 [__val1, __val2, __is_vector](_ForwardIterator __i, _ForwardIterator __j) {
1926 __internal::__brick_swap_ranges(__i, __j, __val2.__begin + (__i - __val1.__pivot), __is_vector);
1927 });
1928 return {__new_begin, __val1.__pivot + __size2, __val2.__end};
1929 }
1930 };
1931
1932 _PartitionRange __result = __par_backend::__parallel_reduce(
1933 std::forward<_ExecutionPolicy>(__exec), __first, __last, __init,
1934 [__pred, __is_vector, __reductor](_ForwardIterator __i, _ForwardIterator __j,
1935 _PartitionRange __value) -> _PartitionRange {
1936 //1. serial partition
1937 _ForwardIterator __pivot = __internal::__brick_partition(__i, __j, __pred, __is_vector);
1938
1939 // 2. merging of two ranges (left and right respectively)
1940 return __reductor(__value, {__i, __pivot, __j});
1941 },
1942 __reductor);
1943 return __result.__pivot;
1944 });
1945 }
1946
1947 //------------------------------------------------------------------------
1948 // stable_partition
1949 //------------------------------------------------------------------------
1950
1951 template <class _BidirectionalIterator, class _UnaryPredicate>
1952 _BidirectionalIterator
__brick_stable_partition(_BidirectionalIterator __first,_BidirectionalIterator __last,_UnaryPredicate __pred,std::false_type)1953 __brick_stable_partition(_BidirectionalIterator __first, _BidirectionalIterator __last, _UnaryPredicate __pred,
1954 /*__is_vector=*/std::false_type) noexcept
1955 {
1956 return std::stable_partition(__first, __last, __pred);
1957 }
1958
1959 template <class _BidirectionalIterator, class _UnaryPredicate>
1960 _BidirectionalIterator
__brick_stable_partition(_BidirectionalIterator __first,_BidirectionalIterator __last,_UnaryPredicate __pred,std::true_type)1961 __brick_stable_partition(_BidirectionalIterator __first, _BidirectionalIterator __last, _UnaryPredicate __pred,
1962 /*__is_vector=*/std::true_type) noexcept
1963 {
1964 _PSTL_PRAGMA_MESSAGE("Vectorized algorithm unimplemented, redirected to serial");
1965 return std::stable_partition(__first, __last, __pred);
1966 }
1967
1968 template <class _ExecutionPolicy, class _BidirectionalIterator, class _UnaryPredicate, class _IsVector>
1969 _BidirectionalIterator
__pattern_stable_partition(_ExecutionPolicy &&,_BidirectionalIterator __first,_BidirectionalIterator __last,_UnaryPredicate __pred,_IsVector __is_vector,std::false_type)1970 __pattern_stable_partition(_ExecutionPolicy&&, _BidirectionalIterator __first, _BidirectionalIterator __last,
1971 _UnaryPredicate __pred, _IsVector __is_vector,
1972 /*is_parallelization=*/std::false_type) noexcept
1973 {
1974 return __internal::__brick_stable_partition(__first, __last, __pred, __is_vector);
1975 }
1976
1977 template <class _ExecutionPolicy, class _BidirectionalIterator, class _UnaryPredicate, class _IsVector>
1978 _BidirectionalIterator
__pattern_stable_partition(_ExecutionPolicy && __exec,_BidirectionalIterator __first,_BidirectionalIterator __last,_UnaryPredicate __pred,_IsVector __is_vector,std::true_type)1979 __pattern_stable_partition(_ExecutionPolicy&& __exec, _BidirectionalIterator __first, _BidirectionalIterator __last,
1980 _UnaryPredicate __pred, _IsVector __is_vector,
1981 /*is_parallelization=*/std::true_type) noexcept
1982 {
1983 // partitioned range: elements before pivot satisfy pred (true part),
1984 // elements after pivot don't satisfy pred (false part)
1985 struct _PartitionRange
1986 {
1987 _BidirectionalIterator __begin;
1988 _BidirectionalIterator __pivot;
1989 _BidirectionalIterator __end;
1990 };
1991
1992 return __internal::__except_handler([&]() {
1993 _PartitionRange __init{__last, __last, __last};
1994
1995 // lambda for merging two partitioned ranges to one partitioned range
1996 auto __reductor = [__is_vector](_PartitionRange __val1, _PartitionRange __val2) -> _PartitionRange {
1997 auto __size1 = __val1.__end - __val1.__pivot;
1998 auto __new_begin = __val2.__begin - (__val1.__end - __val1.__begin);
1999
2000 // if all elements in left range satisfy pred then we can move new pivot to pivot of right range
2001 if (__val1.__end == __val1.__pivot)
2002 {
2003 return {__new_begin, __val2.__pivot, __val2.__end};
2004 }
2005 // if true part of right range greater than false part of left range
2006 // then we should swap the false part of left range and last part of true part of right range
2007 else
2008 {
2009 __internal::__brick_rotate(__val1.__pivot, __val2.__begin, __val2.__pivot, __is_vector);
2010 return {__new_begin, __val2.__pivot - __size1, __val2.__end};
2011 }
2012 };
2013
2014 _PartitionRange __result = __par_backend::__parallel_reduce(
2015 std::forward<_ExecutionPolicy>(__exec), __first, __last, __init,
2016 [&__pred, __is_vector, __reductor](_BidirectionalIterator __i, _BidirectionalIterator __j,
2017 _PartitionRange __value) -> _PartitionRange {
2018 //1. serial stable_partition
2019 _BidirectionalIterator __pivot = __internal::__brick_stable_partition(__i, __j, __pred, __is_vector);
2020
2021 // 2. merging of two ranges (left and right respectively)
2022 return __reductor(__value, {__i, __pivot, __j});
2023 },
2024 __reductor);
2025 return __result.__pivot;
2026 });
2027 }
2028
2029 //------------------------------------------------------------------------
2030 // partition_copy
2031 //------------------------------------------------------------------------
2032
2033 template <class _ForwardIterator, class _OutputIterator1, class _OutputIterator2, class _UnaryPredicate>
2034 std::pair<_OutputIterator1, _OutputIterator2>
__brick_partition_copy(_ForwardIterator __first,_ForwardIterator __last,_OutputIterator1 __out_true,_OutputIterator2 __out_false,_UnaryPredicate __pred,std::false_type)2035 __brick_partition_copy(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator1 __out_true,
2036 _OutputIterator2 __out_false, _UnaryPredicate __pred, /*is_vector=*/std::false_type) noexcept
2037 {
2038 return std::partition_copy(__first, __last, __out_true, __out_false, __pred);
2039 }
2040
2041 template <class _ForwardIterator, class _OutputIterator1, class _OutputIterator2, class _UnaryPredicate>
2042 std::pair<_OutputIterator1, _OutputIterator2>
__brick_partition_copy(_ForwardIterator __first,_ForwardIterator __last,_OutputIterator1 __out_true,_OutputIterator2 __out_false,_UnaryPredicate __pred,std::true_type)2043 __brick_partition_copy(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator1 __out_true,
2044 _OutputIterator2 __out_false, _UnaryPredicate __pred, /*is_vector=*/std::true_type) noexcept
2045 {
2046 #if (_PSTL_MONOTONIC_PRESENT)
2047 return __unseq_backend::__simd_partition_copy(__first, __last - __first, __out_true, __out_false, __pred);
2048 #else
2049 return std::partition_copy(__first, __last, __out_true, __out_false, __pred);
2050 #endif
2051 }
2052
2053 template <class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator1, class _OutputIterator2,
2054 class _UnaryPredicate, class _IsVector>
2055 std::pair<_OutputIterator1, _OutputIterator2>
__pattern_partition_copy(_ExecutionPolicy &&,_ForwardIterator __first,_ForwardIterator __last,_OutputIterator1 __out_true,_OutputIterator2 __out_false,_UnaryPredicate __pred,_IsVector __is_vector,std::false_type)2056 __pattern_partition_copy(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last,
2057 _OutputIterator1 __out_true, _OutputIterator2 __out_false, _UnaryPredicate __pred,
2058 _IsVector __is_vector, /*is_parallelization=*/std::false_type) noexcept
2059 {
2060 return __internal::__brick_partition_copy(__first, __last, __out_true, __out_false, __pred, __is_vector);
2061 }
2062
2063 template <class _ExecutionPolicy, class _RandomAccessIterator, class _OutputIterator1, class _OutputIterator2,
2064 class _UnaryPredicate, class _IsVector>
2065 std::pair<_OutputIterator1, _OutputIterator2>
__pattern_partition_copy(_ExecutionPolicy && __exec,_RandomAccessIterator __first,_RandomAccessIterator __last,_OutputIterator1 __out_true,_OutputIterator2 __out_false,_UnaryPredicate __pred,_IsVector __is_vector,std::true_type)2066 __pattern_partition_copy(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last,
2067 _OutputIterator1 __out_true, _OutputIterator2 __out_false, _UnaryPredicate __pred,
2068 _IsVector __is_vector, /*is_parallelization=*/std::true_type)
2069 {
2070 typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type _DifferenceType;
2071 typedef std::pair<_DifferenceType, _DifferenceType> _ReturnType;
2072 const _DifferenceType __n = __last - __first;
2073 if (_DifferenceType(1) < __n)
2074 {
2075 __par_backend::__buffer<bool> __mask_buf(__n);
2076 return __internal::__except_handler([&__exec, __n, __first, __out_true, __out_false, __is_vector, __pred,
2077 &__mask_buf]() {
2078 bool* __mask = __mask_buf.get();
2079 _ReturnType __m{};
2080 __par_backend::__parallel_strict_scan(
2081 std::forward<_ExecutionPolicy>(__exec), __n, std::make_pair(_DifferenceType(0), _DifferenceType(0)),
2082 [=](_DifferenceType __i, _DifferenceType __len) { // Reduce
2083 return __internal::__brick_calc_mask_1<_DifferenceType>(__first + __i, __first + (__i + __len),
2084 __mask + __i, __pred, __is_vector);
2085 },
2086 [](const _ReturnType& __x, const _ReturnType& __y) -> _ReturnType {
2087 return std::make_pair(__x.first + __y.first, __x.second + __y.second);
2088 }, // Combine
2089 [=](_DifferenceType __i, _DifferenceType __len, _ReturnType __initial) { // Scan
2090 __internal::__brick_partition_by_mask(__first + __i, __first + (__i + __len),
2091 __out_true + __initial.first, __out_false + __initial.second,
2092 __mask + __i, __is_vector);
2093 },
2094 [&__m](_ReturnType __total) { __m = __total; });
2095 return std::make_pair(__out_true + __m.first, __out_false + __m.second);
2096 });
2097 }
2098 // trivial sequence - use serial algorithm
2099 return __internal::__brick_partition_copy(__first, __last, __out_true, __out_false, __pred, __is_vector);
2100 }
2101
2102 //------------------------------------------------------------------------
2103 // sort
2104 //------------------------------------------------------------------------
2105
2106 template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare, class _IsVector,
2107 class _IsMoveConstructible>
2108 void
__pattern_sort(_ExecutionPolicy &&,_RandomAccessIterator __first,_RandomAccessIterator __last,_Compare __comp,_IsVector,std::false_type,_IsMoveConstructible)2109 __pattern_sort(_ExecutionPolicy&&, _RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp,
2110 _IsVector /*is_vector*/, /*is_parallel=*/std::false_type, _IsMoveConstructible) noexcept
2111 {
2112 std::sort(__first, __last, __comp);
2113 }
2114
2115 template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare, class _IsVector>
2116 void
__pattern_sort(_ExecutionPolicy && __exec,_RandomAccessIterator __first,_RandomAccessIterator __last,_Compare __comp,_IsVector,std::true_type,std::true_type)2117 __pattern_sort(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp,
2118 _IsVector /*is_vector*/, /*is_parallel=*/std::true_type, /*is_move_constructible=*/std::true_type)
2119 {
2120 __internal::__except_handler([&]() {
2121 __par_backend::__parallel_stable_sort(std::forward<_ExecutionPolicy>(__exec), __first, __last, __comp,
2122 [](_RandomAccessIterator __first, _RandomAccessIterator __last,
2123 _Compare __comp) { std::sort(__first, __last, __comp); });
2124 });
2125 }
2126
2127 //------------------------------------------------------------------------
2128 // stable_sort
2129 //------------------------------------------------------------------------
2130
2131 template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare, class _IsVector>
2132 void
__pattern_stable_sort(_ExecutionPolicy &&,_RandomAccessIterator __first,_RandomAccessIterator __last,_Compare __comp,_IsVector,std::false_type)2133 __pattern_stable_sort(_ExecutionPolicy&&, _RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp,
2134 _IsVector /*is_vector*/, /*is_parallel=*/std::false_type) noexcept
2135 {
2136 std::stable_sort(__first, __last, __comp);
2137 }
2138
2139 template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare, class _IsVector>
2140 void
__pattern_stable_sort(_ExecutionPolicy && __exec,_RandomAccessIterator __first,_RandomAccessIterator __last,_Compare __comp,_IsVector,std::true_type)2141 __pattern_stable_sort(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last,
2142 _Compare __comp, _IsVector /*is_vector*/, /*is_parallel=*/std::true_type)
2143 {
2144 __internal::__except_handler([&]() {
2145 __par_backend::__parallel_stable_sort(std::forward<_ExecutionPolicy>(__exec), __first, __last, __comp,
2146 [](_RandomAccessIterator __first, _RandomAccessIterator __last,
2147 _Compare __comp) { std::stable_sort(__first, __last, __comp); });
2148 });
2149 }
2150
2151 //------------------------------------------------------------------------
2152 // partial_sort
2153 //------------------------------------------------------------------------
2154
2155 template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare, class _IsVector>
2156 void
__pattern_partial_sort(_ExecutionPolicy &&,_RandomAccessIterator __first,_RandomAccessIterator __middle,_RandomAccessIterator __last,_Compare __comp,_IsVector,std::false_type)2157 __pattern_partial_sort(_ExecutionPolicy&&, _RandomAccessIterator __first, _RandomAccessIterator __middle,
2158 _RandomAccessIterator __last, _Compare __comp, _IsVector,
2159 /*is_parallel=*/std::false_type) noexcept
2160 {
2161 std::partial_sort(__first, __middle, __last, __comp);
2162 }
2163
2164 template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare, class _IsVector>
2165 void
__pattern_partial_sort(_ExecutionPolicy && __exec,_RandomAccessIterator __first,_RandomAccessIterator __middle,_RandomAccessIterator __last,_Compare __comp,_IsVector,std::true_type)2166 __pattern_partial_sort(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __middle,
2167 _RandomAccessIterator __last, _Compare __comp, _IsVector, /*is_parallel=*/std::true_type)
2168 {
2169 const auto __n = __middle - __first;
2170 if (__n == 0)
2171 return;
2172
2173 __internal::__except_handler([&]() {
2174 __par_backend::__parallel_stable_sort(
2175 std::forward<_ExecutionPolicy>(__exec), __first, __last, __comp,
2176 [__n](_RandomAccessIterator __begin, _RandomAccessIterator __end, _Compare __comp) {
2177 if (__n < __end - __begin)
2178 std::partial_sort(__begin, __begin + __n, __end, __comp);
2179 else
2180 std::sort(__begin, __end, __comp);
2181 },
2182 __n);
2183 });
2184 }
2185
2186 //------------------------------------------------------------------------
2187 // partial_sort_copy
2188 //------------------------------------------------------------------------
2189
2190 template <class _ExecutionPolicy, class _ForwardIterator, class _RandomAccessIterator, class _Compare, class _IsVector>
2191 _RandomAccessIterator
__pattern_partial_sort_copy(_ExecutionPolicy &&,_ForwardIterator __first,_ForwardIterator __last,_RandomAccessIterator __d_first,_RandomAccessIterator __d_last,_Compare __comp,_IsVector,std::false_type)2192 __pattern_partial_sort_copy(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last,
2193 _RandomAccessIterator __d_first, _RandomAccessIterator __d_last, _Compare __comp, _IsVector,
2194 /*is_parallel=*/std::false_type) noexcept
2195 {
2196 return std::partial_sort_copy(__first, __last, __d_first, __d_last, __comp);
2197 }
2198
2199 template <class _ExecutionPolicy, class _ForwardIterator, class _RandomAccessIterator, class _Compare, class _IsVector>
2200 _RandomAccessIterator
__pattern_partial_sort_copy(_ExecutionPolicy && __exec,_ForwardIterator __first,_ForwardIterator __last,_RandomAccessIterator __d_first,_RandomAccessIterator __d_last,_Compare __comp,_IsVector __is_vector,std::true_type)2201 __pattern_partial_sort_copy(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last,
2202 _RandomAccessIterator __d_first, _RandomAccessIterator __d_last, _Compare __comp,
2203 _IsVector __is_vector, /*is_parallel=*/std::true_type)
2204 {
2205 if (__last == __first || __d_last == __d_first)
2206 {
2207 return __d_first;
2208 }
2209 auto __n1 = __last - __first;
2210 auto __n2 = __d_last - __d_first;
2211 return __internal::__except_handler([&]() {
2212 if (__n2 >= __n1)
2213 {
2214 __par_backend::__parallel_stable_sort(
2215 std::forward<_ExecutionPolicy>(__exec), __d_first, __d_first + __n1, __comp,
2216 [__first, __d_first, __is_vector](_RandomAccessIterator __i, _RandomAccessIterator __j,
2217 _Compare __comp) {
2218 _ForwardIterator __i1 = __first + (__i - __d_first);
2219 _ForwardIterator __j1 = __first + (__j - __d_first);
2220
2221 // 1. Copy elements from input to output
2222 #if !_PSTL_ICC_18_OMP_SIMD_BROKEN
2223 __internal::__brick_copy(__i1, __j1, __i, __is_vector);
2224 #else
2225 std::copy(__i1, __j1, __i);
2226 #endif
2227 // 2. Sort elements in output sequence
2228 std::sort(__i, __j, __comp);
2229 },
2230 __n1);
2231 return __d_first + __n1;
2232 }
2233 else
2234 {
2235 typedef typename std::iterator_traits<_ForwardIterator>::value_type _T1;
2236 typedef typename std::iterator_traits<_RandomAccessIterator>::value_type _T2;
2237 __par_backend::__buffer<_T1> __buf(__n1);
2238 _T1* __r = __buf.get();
2239
2240 __par_backend::__parallel_stable_sort(std::forward<_ExecutionPolicy>(__exec), __r, __r + __n1, __comp,
2241 [__n2, __first, __r](_T1* __i, _T1* __j, _Compare __comp) {
2242 _ForwardIterator __it = __first + (__i - __r);
2243
2244 // 1. Copy elements from input to raw memory
2245 for (_T1* __k = __i; __k != __j; ++__k, ++__it)
2246 {
2247 ::new (__k) _T2(*__it);
2248 }
2249
2250 // 2. Sort elements in temporary __buffer
2251 if (__n2 < __j - __i)
2252 std::partial_sort(__i, __i + __n2, __j, __comp);
2253 else
2254 std::sort(__i, __j, __comp);
2255 },
2256 __n2);
2257
2258 // 3. Move elements from temporary __buffer to output
2259 __par_backend::__parallel_for(std::forward<_ExecutionPolicy>(__exec), __r, __r + __n2,
2260 [__r, __d_first, __is_vector](_T1* __i, _T1* __j) {
2261 __brick_move_destroy()(
2262 __i, __j, __d_first + (__i - __r), __is_vector);
2263 });
2264 __par_backend::__parallel_for(
2265 std::forward<_ExecutionPolicy>(__exec), __r + __n2, __r + __n1,
2266 [__is_vector](_T1* __i, _T1* __j) { __brick_destroy(__i, __j, __is_vector); });
2267
2268 return __d_first + __n2;
2269 }
2270 });
2271 }
2272
2273 //------------------------------------------------------------------------
2274 // adjacent_find
2275 //------------------------------------------------------------------------
2276 template <class _ForwardIterator, class _BinaryPredicate>
2277 _ForwardIterator
__brick_adjacent_find(_ForwardIterator __first,_ForwardIterator __last,_BinaryPredicate __pred,std::true_type,bool __or_semantic)2278 __brick_adjacent_find(_ForwardIterator __first, _ForwardIterator __last, _BinaryPredicate __pred,
2279 /* IsVector = */ std::true_type, bool __or_semantic) noexcept
2280 {
2281 return __unseq_backend::__simd_adjacent_find(__first, __last, __pred, __or_semantic);
2282 }
2283
2284 template <class _ForwardIterator, class _BinaryPredicate>
2285 _ForwardIterator
__brick_adjacent_find(_ForwardIterator __first,_ForwardIterator __last,_BinaryPredicate __pred,std::false_type,bool)2286 __brick_adjacent_find(_ForwardIterator __first, _ForwardIterator __last, _BinaryPredicate __pred,
2287 /* IsVector = */ std::false_type, bool) noexcept
2288 {
2289 return std::adjacent_find(__first, __last, __pred);
2290 }
2291
2292 template <class _ExecutionPolicy, class _ForwardIterator, class _BinaryPredicate, class _IsVector>
2293 _ForwardIterator
__pattern_adjacent_find(_ExecutionPolicy &&,_ForwardIterator __first,_ForwardIterator __last,_BinaryPredicate __pred,std::false_type,_IsVector __is_vector,bool __or_semantic)2294 __pattern_adjacent_find(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _BinaryPredicate __pred,
2295 /* is_parallel */ std::false_type, _IsVector __is_vector, bool __or_semantic) noexcept
2296 {
2297 return __internal::__brick_adjacent_find(__first, __last, __pred, __is_vector, __or_semantic);
2298 }
2299
2300 template <class _ExecutionPolicy, class _RandomAccessIterator, class _BinaryPredicate, class _IsVector>
2301 _RandomAccessIterator
__pattern_adjacent_find(_ExecutionPolicy && __exec,_RandomAccessIterator __first,_RandomAccessIterator __last,_BinaryPredicate __pred,std::true_type,_IsVector __is_vector,bool __or_semantic)2302 __pattern_adjacent_find(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last,
2303 _BinaryPredicate __pred, /* is_parallel */ std::true_type, _IsVector __is_vector,
2304 bool __or_semantic)
2305 {
2306 if (__last - __first < 2)
2307 return __last;
2308
2309 return __internal::__except_handler([&]() {
2310 return __par_backend::__parallel_reduce(
2311 std::forward<_ExecutionPolicy>(__exec), __first, __last, __last,
2312 [__last, __pred, __is_vector, __or_semantic](_RandomAccessIterator __begin, _RandomAccessIterator __end,
2313 _RandomAccessIterator __value) -> _RandomAccessIterator {
2314 // TODO: investigate performance benefits from the use of shared variable for the result,
2315 // checking (compare_and_swap idiom) its __value at __first.
2316 if (__or_semantic && __value < __last)
2317 { //found
2318 __par_backend::__cancel_execution();
2319 return __value;
2320 }
2321
2322 if (__value > __begin)
2323 {
2324 // modify __end to check the predicate on the boundary __values;
2325 // TODO: to use a custom range with boundaries overlapping
2326 // TODO: investigate what if we remove "if" below and run algorithm on range [__first, __last-1)
2327 // then check the pair [__last-1, __last)
2328 if (__end != __last)
2329 ++__end;
2330
2331 //correct the global result iterator if the "brick" returns a local "__last"
2332 const _RandomAccessIterator __res =
2333 __internal::__brick_adjacent_find(__begin, __end, __pred, __is_vector, __or_semantic);
2334 if (__res < __end)
2335 __value = __res;
2336 }
2337 return __value;
2338 },
2339 [](_RandomAccessIterator __x, _RandomAccessIterator __y) -> _RandomAccessIterator {
2340 return __x < __y ? __x : __y;
2341 } //reduce a __value
2342 );
2343 });
2344 }
2345
2346 //------------------------------------------------------------------------
2347 // nth_element
2348 //------------------------------------------------------------------------
2349
2350 template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare, class _IsVector>
2351 void
__pattern_nth_element(_ExecutionPolicy &&,_RandomAccessIterator __first,_RandomAccessIterator __nth,_RandomAccessIterator __last,_Compare __comp,_IsVector,std::false_type)2352 __pattern_nth_element(_ExecutionPolicy&&, _RandomAccessIterator __first, _RandomAccessIterator __nth,
2353 _RandomAccessIterator __last, _Compare __comp, _IsVector,
2354 /*is_parallel=*/std::false_type) noexcept
2355 {
2356 std::nth_element(__first, __nth, __last, __comp);
2357 }
2358
2359 template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare, class _IsVector>
2360 void
__pattern_nth_element(_ExecutionPolicy && __exec,_RandomAccessIterator __first,_RandomAccessIterator __nth,_RandomAccessIterator __last,_Compare __comp,_IsVector __is_vector,std::true_type)2361 __pattern_nth_element(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __nth,
2362 _RandomAccessIterator __last, _Compare __comp, _IsVector __is_vector,
2363 /*is_parallel=*/std::true_type) noexcept
2364 {
2365 if (__first == __last || __nth == __last)
2366 {
2367 return;
2368 }
2369
2370 using std::iter_swap;
2371 typedef typename std::iterator_traits<_RandomAccessIterator>::value_type _Tp;
2372 _RandomAccessIterator __x;
2373 do
2374 {
2375 __x = __internal::__pattern_partition(std::forward<_ExecutionPolicy>(__exec), __first + 1, __last,
2376 [&__comp, __first](const _Tp& __x) { return __comp(__x, *__first); },
2377 __is_vector,
2378 /*is_parallel=*/std::true_type());
2379 --__x;
2380 if (__x != __first)
2381 {
2382 iter_swap(__first, __x);
2383 }
2384 // if x > nth then our new range for partition is [first, x)
2385 if (__x - __nth > 0)
2386 {
2387 __last = __x;
2388 }
2389 // if x < nth then our new range for partition is [x, last)
2390 else if (__x - __nth < 0)
2391 {
2392 // if *x == *nth then we can start new partition with x+1
2393 if (!__comp(*__nth, *__x) && !__comp(*__x, *__nth))
2394 {
2395 ++__x;
2396 }
2397 else
2398 {
2399 iter_swap(__nth, __x);
2400 }
2401 __first = __x;
2402 }
2403 } while (__x != __nth);
2404 }
2405
2406 //------------------------------------------------------------------------
2407 // fill, fill_n
2408 //------------------------------------------------------------------------
2409 template <class _ForwardIterator, class _Tp>
2410 void
__brick_fill(_ForwardIterator __first,_ForwardIterator __last,const _Tp & __value,std::true_type)2411 __brick_fill(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __value,
2412 /* __is_vector = */ std::true_type) noexcept
2413 {
2414 __unseq_backend::__simd_fill_n(__first, __last - __first, __value);
2415 }
2416
2417 template <class _ForwardIterator, class _Tp>
2418 void
__brick_fill(_ForwardIterator __first,_ForwardIterator __last,const _Tp & __value,std::false_type)2419 __brick_fill(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __value,
2420 /* __is_vector = */ std::false_type) noexcept
2421 {
2422 std::fill(__first, __last, __value);
2423 }
2424
2425 template <class _ExecutionPolicy, class _ForwardIterator, class _Tp, class _IsVector>
2426 void
__pattern_fill(_ExecutionPolicy &&,_ForwardIterator __first,_ForwardIterator __last,const _Tp & __value,std::false_type,_IsVector __is_vector)2427 __pattern_fill(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value,
2428 /*is_parallel=*/std::false_type, _IsVector __is_vector) noexcept
2429 {
2430 __internal::__brick_fill(__first, __last, __value, __is_vector);
2431 }
2432
2433 template <class _ExecutionPolicy, class _ForwardIterator, class _Tp, class _IsVector>
2434 _ForwardIterator
__pattern_fill(_ExecutionPolicy && __exec,_ForwardIterator __first,_ForwardIterator __last,const _Tp & __value,std::true_type,_IsVector __is_vector)2435 __pattern_fill(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value,
2436 /*is_parallel=*/std::true_type, _IsVector __is_vector)
2437 {
2438 return __internal::__except_handler([&__exec, __first, __last, &__value, __is_vector]() {
2439 __par_backend::__parallel_for(std::forward<_ExecutionPolicy>(__exec), __first, __last,
2440 [&__value, __is_vector](_ForwardIterator __begin, _ForwardIterator __end) {
2441 __internal::__brick_fill(__begin, __end, __value, __is_vector);
2442 });
2443 return __last;
2444 });
2445 }
2446
2447 template <class _OutputIterator, class _Size, class _Tp>
2448 _OutputIterator
__brick_fill_n(_OutputIterator __first,_Size __count,const _Tp & __value,std::true_type)2449 __brick_fill_n(_OutputIterator __first, _Size __count, const _Tp& __value, /* __is_vector = */ std::true_type) noexcept
2450 {
2451 return __unseq_backend::__simd_fill_n(__first, __count, __value);
2452 }
2453
2454 template <class _OutputIterator, class _Size, class _Tp>
2455 _OutputIterator
__brick_fill_n(_OutputIterator __first,_Size __count,const _Tp & __value,std::false_type)2456 __brick_fill_n(_OutputIterator __first, _Size __count, const _Tp& __value, /* __is_vector = */ std::false_type) noexcept
2457 {
2458 return std::fill_n(__first, __count, __value);
2459 }
2460
2461 template <class _ExecutionPolicy, class _OutputIterator, class _Size, class _Tp, class _IsVector>
2462 _OutputIterator
__pattern_fill_n(_ExecutionPolicy &&,_OutputIterator __first,_Size __count,const _Tp & __value,std::false_type,_IsVector __is_vector)2463 __pattern_fill_n(_ExecutionPolicy&&, _OutputIterator __first, _Size __count, const _Tp& __value,
2464 /*is_parallel=*/std::false_type, _IsVector __is_vector) noexcept
2465 {
2466 return __internal::__brick_fill_n(__first, __count, __value, __is_vector);
2467 }
2468
2469 template <class _ExecutionPolicy, class _OutputIterator, class _Size, class _Tp, class _IsVector>
2470 _OutputIterator
__pattern_fill_n(_ExecutionPolicy && __exec,_OutputIterator __first,_Size __count,const _Tp & __value,std::true_type,_IsVector __is_vector)2471 __pattern_fill_n(_ExecutionPolicy&& __exec, _OutputIterator __first, _Size __count, const _Tp& __value,
2472 /*is_parallel=*/std::true_type, _IsVector __is_vector)
2473 {
2474 return __internal::__pattern_fill(std::forward<_ExecutionPolicy>(__exec), __first, __first + __count, __value,
2475 std::true_type(), __is_vector);
2476 }
2477
2478 //------------------------------------------------------------------------
2479 // generate, generate_n
2480 //------------------------------------------------------------------------
2481 template <class _RandomAccessIterator, class _Generator>
2482 void
__brick_generate(_RandomAccessIterator __first,_RandomAccessIterator __last,_Generator __g,std::true_type)2483 __brick_generate(_RandomAccessIterator __first, _RandomAccessIterator __last, _Generator __g,
2484 /* is_vector = */ std::true_type) noexcept
2485 {
2486 __unseq_backend::__simd_generate_n(__first, __last - __first, __g);
2487 }
2488
2489 template <class _ForwardIterator, class _Generator>
2490 void
__brick_generate(_ForwardIterator __first,_ForwardIterator __last,_Generator __g,std::false_type)2491 __brick_generate(_ForwardIterator __first, _ForwardIterator __last, _Generator __g,
2492 /* is_vector = */ std::false_type) noexcept
2493 {
2494 std::generate(__first, __last, __g);
2495 }
2496
2497 template <class _ExecutionPolicy, class _ForwardIterator, class _Generator, class _IsVector>
2498 void
__pattern_generate(_ExecutionPolicy &&,_ForwardIterator __first,_ForwardIterator __last,_Generator __g,std::false_type,_IsVector __is_vector)2499 __pattern_generate(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Generator __g,
2500 /*is_parallel=*/std::false_type, _IsVector __is_vector) noexcept
2501 {
2502 __internal::__brick_generate(__first, __last, __g, __is_vector);
2503 }
2504
2505 template <class _ExecutionPolicy, class _ForwardIterator, class _Generator, class _IsVector>
2506 _ForwardIterator
__pattern_generate(_ExecutionPolicy && __exec,_ForwardIterator __first,_ForwardIterator __last,_Generator __g,std::true_type,_IsVector __is_vector)2507 __pattern_generate(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Generator __g,
2508 /*is_parallel=*/std::true_type, _IsVector __is_vector)
2509 {
2510 return __internal::__except_handler([&]() {
2511 __par_backend::__parallel_for(std::forward<_ExecutionPolicy>(__exec), __first, __last,
2512 [__g, __is_vector](_ForwardIterator __begin, _ForwardIterator __end) {
2513 __internal::__brick_generate(__begin, __end, __g, __is_vector);
2514 });
2515 return __last;
2516 });
2517 }
2518
2519 template <class OutputIterator, class Size, class _Generator>
2520 OutputIterator
__brick_generate_n(OutputIterator __first,Size __count,_Generator __g,std::true_type)2521 __brick_generate_n(OutputIterator __first, Size __count, _Generator __g, /* is_vector = */ std::true_type) noexcept
2522 {
2523 return __unseq_backend::__simd_generate_n(__first, __count, __g);
2524 }
2525
2526 template <class OutputIterator, class Size, class _Generator>
2527 OutputIterator
__brick_generate_n(OutputIterator __first,Size __count,_Generator __g,std::false_type)2528 __brick_generate_n(OutputIterator __first, Size __count, _Generator __g, /* is_vector = */ std::false_type) noexcept
2529 {
2530 return std::generate_n(__first, __count, __g);
2531 }
2532
2533 template <class _ExecutionPolicy, class _OutputIterator, class _Size, class _Generator, class _IsVector>
2534 _OutputIterator
__pattern_generate_n(_ExecutionPolicy &&,_OutputIterator __first,_Size __count,_Generator __g,std::false_type,_IsVector __is_vector)2535 __pattern_generate_n(_ExecutionPolicy&&, _OutputIterator __first, _Size __count, _Generator __g,
2536 /*is_parallel=*/std::false_type, _IsVector __is_vector) noexcept
2537 {
2538 return __internal::__brick_generate_n(__first, __count, __g, __is_vector);
2539 }
2540
2541 template <class _ExecutionPolicy, class _OutputIterator, class _Size, class _Generator, class _IsVector>
2542 _OutputIterator
__pattern_generate_n(_ExecutionPolicy && __exec,_OutputIterator __first,_Size __count,_Generator __g,std::true_type,_IsVector __is_vector)2543 __pattern_generate_n(_ExecutionPolicy&& __exec, _OutputIterator __first, _Size __count, _Generator __g,
2544 /*is_parallel=*/std::true_type, _IsVector __is_vector)
2545 {
2546 static_assert(__is_random_access_iterator<_OutputIterator>::value,
2547 "Pattern-brick error. Should be a random access iterator.");
2548 return __internal::__pattern_generate(std::forward<_ExecutionPolicy>(__exec), __first, __first + __count, __g,
2549 std::true_type(), __is_vector);
2550 }
2551
2552 //------------------------------------------------------------------------
2553 // remove
2554 //------------------------------------------------------------------------
2555
2556 template <class _ForwardIterator, class _UnaryPredicate>
2557 _ForwardIterator
__brick_remove_if(_ForwardIterator __first,_ForwardIterator __last,_UnaryPredicate __pred,std::false_type)2558 __brick_remove_if(_ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred,
2559 /* __is_vector = */ std::false_type) noexcept
2560 {
2561 return std::remove_if(__first, __last, __pred);
2562 }
2563
2564 template <class _RandomAccessIterator, class _UnaryPredicate>
2565 _RandomAccessIterator
__brick_remove_if(_RandomAccessIterator __first,_RandomAccessIterator __last,_UnaryPredicate __pred,std::true_type)2566 __brick_remove_if(_RandomAccessIterator __first, _RandomAccessIterator __last, _UnaryPredicate __pred,
2567 /* __is_vector = */ std::true_type) noexcept
2568 {
2569 #if _PSTL_MONOTONIC_PRESENT
2570 return __unseq_backend::__simd_remove_if(__first, __last - __first, __pred);
2571 #else
2572 return std::remove_if(__first, __last, __pred);
2573 #endif
2574 }
2575
2576 template <class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate, class _IsVector>
2577 _ForwardIterator
__pattern_remove_if(_ExecutionPolicy &&,_ForwardIterator __first,_ForwardIterator __last,_UnaryPredicate __pred,_IsVector __is_vector,std::false_type)2578 __pattern_remove_if(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred,
2579 _IsVector __is_vector, /*is_parallel*/ std::false_type) noexcept
2580 {
2581 return __internal::__brick_remove_if(__first, __last, __pred, __is_vector);
2582 }
2583
2584 template <class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate, class _IsVector>
2585 _ForwardIterator
__pattern_remove_if(_ExecutionPolicy && __exec,_ForwardIterator __first,_ForwardIterator __last,_UnaryPredicate __pred,_IsVector __is_vector,std::true_type)2586 __pattern_remove_if(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last,
2587 _UnaryPredicate __pred, _IsVector __is_vector, /*is_parallel*/ std::true_type) noexcept
2588 {
2589 typedef typename std::iterator_traits<_ForwardIterator>::reference _ReferenceType;
2590
2591 if (__first == __last || __first + 1 == __last)
2592 {
2593 // Trivial sequence - use serial algorithm
2594 return __internal::__brick_remove_if(__first, __last, __pred, __is_vector);
2595 }
2596
2597 return __internal::__remove_elements(
2598 std::forward<_ExecutionPolicy>(__exec), __first, __last,
2599 [&__pred, __is_vector](bool* __b, bool* __e, _ForwardIterator __it) {
2600 __internal::__brick_walk2(__b, __e, __it, [&__pred](bool& __x, _ReferenceType __y) { __x = !__pred(__y); },
2601 __is_vector);
2602 },
2603 __is_vector);
2604 }
2605
2606 //------------------------------------------------------------------------
2607 // merge
2608 //------------------------------------------------------------------------
2609
2610 template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
2611 _OutputIterator
__brick_merge(_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_OutputIterator __d_first,_Compare __comp,std::false_type)2612 __brick_merge(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
2613 _ForwardIterator2 __last2, _OutputIterator __d_first, _Compare __comp,
2614 /* __is_vector = */ std::false_type) noexcept
2615 {
2616 return std::merge(__first1, __last1, __first2, __last2, __d_first, __comp);
2617 }
2618
2619 template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
2620 _OutputIterator
__brick_merge(_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_OutputIterator __d_first,_Compare __comp,std::true_type)2621 __brick_merge(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
2622 _ForwardIterator2 __last2, _OutputIterator __d_first, _Compare __comp,
2623 /* __is_vector = */ std::true_type) noexcept
2624 {
2625 _PSTL_PRAGMA_MESSAGE("Vectorized algorithm unimplemented, redirected to serial");
2626 return std::merge(__first1, __last1, __first2, __last2, __d_first, __comp);
2627 }
2628
2629 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator,
2630 class _Compare, class _IsVector>
2631 _OutputIterator
__pattern_merge(_ExecutionPolicy &&,_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_OutputIterator __d_first,_Compare __comp,_IsVector __is_vector,std::false_type)2632 __pattern_merge(_ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
2633 _ForwardIterator2 __last2, _OutputIterator __d_first, _Compare __comp, _IsVector __is_vector,
2634 /* is_parallel = */ std::false_type) noexcept
2635 {
2636 return __internal::__brick_merge(__first1, __last1, __first2, __last2, __d_first, __comp, __is_vector);
2637 }
2638
2639 template <class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2, class _OutputIterator,
2640 class _Compare, class _IsVector>
2641 _OutputIterator
__pattern_merge(_ExecutionPolicy && __exec,_RandomAccessIterator1 __first1,_RandomAccessIterator1 __last1,_RandomAccessIterator2 __first2,_RandomAccessIterator2 __last2,_OutputIterator __d_first,_Compare __comp,_IsVector __is_vector,std::true_type)2642 __pattern_merge(_ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1,
2643 _RandomAccessIterator2 __first2, _RandomAccessIterator2 __last2, _OutputIterator __d_first,
2644 _Compare __comp, _IsVector __is_vector, /* is_parallel = */ std::true_type)
2645 {
2646 __par_backend::__parallel_merge(
2647 std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __d_first, __comp,
2648 [__is_vector](_RandomAccessIterator1 __f1, _RandomAccessIterator1 __l1, _RandomAccessIterator2 __f2,
2649 _RandomAccessIterator2 __l2, _OutputIterator __f3, _Compare __comp) {
2650 return __internal::__brick_merge(__f1, __l1, __f2, __l2, __f3, __comp, __is_vector);
2651 });
2652 return __d_first + (__last1 - __first1) + (__last2 - __first2);
2653 }
2654
2655 //------------------------------------------------------------------------
2656 // inplace_merge
2657 //------------------------------------------------------------------------
2658 template <class _BidirectionalIterator, class _Compare>
2659 void
__brick_inplace_merge(_BidirectionalIterator __first,_BidirectionalIterator __middle,_BidirectionalIterator __last,_Compare __comp,std::false_type)2660 __brick_inplace_merge(_BidirectionalIterator __first, _BidirectionalIterator __middle, _BidirectionalIterator __last,
2661 _Compare __comp, /* __is_vector = */ std::false_type) noexcept
2662 {
2663 std::inplace_merge(__first, __middle, __last, __comp);
2664 }
2665
2666 template <class _BidirectionalIterator, class _Compare>
2667 void
__brick_inplace_merge(_BidirectionalIterator __first,_BidirectionalIterator __middle,_BidirectionalIterator __last,_Compare __comp,std::true_type)2668 __brick_inplace_merge(_BidirectionalIterator __first, _BidirectionalIterator __middle, _BidirectionalIterator __last,
2669 _Compare __comp, /* __is_vector = */ std::true_type) noexcept
2670 {
2671 _PSTL_PRAGMA_MESSAGE("Vectorized algorithm unimplemented, redirected to serial")
2672 std::inplace_merge(__first, __middle, __last, __comp);
2673 }
2674
2675 template <class _ExecutionPolicy, class _BidirectionalIterator, class _Compare, class _IsVector>
2676 void
__pattern_inplace_merge(_ExecutionPolicy &&,_BidirectionalIterator __first,_BidirectionalIterator __middle,_BidirectionalIterator __last,_Compare __comp,_IsVector __is_vector,std::false_type)2677 __pattern_inplace_merge(_ExecutionPolicy&&, _BidirectionalIterator __first, _BidirectionalIterator __middle,
2678 _BidirectionalIterator __last, _Compare __comp, _IsVector __is_vector,
2679 /* is_parallel = */ std::false_type) noexcept
2680 {
2681 __internal::__brick_inplace_merge(__first, __middle, __last, __comp, __is_vector);
2682 }
2683
2684 template <class _ExecutionPolicy, class _BidirectionalIterator, class _Compare, class _IsVector>
2685 void
__pattern_inplace_merge(_ExecutionPolicy && __exec,_BidirectionalIterator __first,_BidirectionalIterator __middle,_BidirectionalIterator __last,_Compare __comp,_IsVector __is_vector,std::true_type)2686 __pattern_inplace_merge(_ExecutionPolicy&& __exec, _BidirectionalIterator __first, _BidirectionalIterator __middle,
2687 _BidirectionalIterator __last, _Compare __comp, _IsVector __is_vector,
2688 /*is_parallel=*/std::true_type)
2689 {
2690 if (__first == __last || __first == __middle || __middle == __last)
2691 {
2692 return;
2693 }
2694 typedef typename std::iterator_traits<_BidirectionalIterator>::value_type _Tp;
2695 auto __n = __last - __first;
2696 __par_backend::__buffer<_Tp> __buf(__n);
2697 _Tp* __r = __buf.get();
2698 __internal::__except_handler([&]() {
2699 auto __move_values = [](_BidirectionalIterator __x, _Tp* __z) {
2700 __internal::__invoke_if_else(std::is_trivial<_Tp>(), [&]() { *__z = std::move(*__x); },
2701 [&]() { ::new (std::addressof(*__z)) _Tp(std::move(*__x)); });
2702 };
2703
2704 auto __move_sequences = [](_BidirectionalIterator __first1, _BidirectionalIterator __last1, _Tp* __first2) {
2705 return __internal::__brick_uninitialized_move(__first1, __last1, __first2, _IsVector());
2706 };
2707
2708 __par_backend::__parallel_merge(
2709 std::forward<_ExecutionPolicy>(__exec), __first, __middle, __middle, __last, __r, __comp,
2710 [__n, __move_values, __move_sequences](_BidirectionalIterator __f1, _BidirectionalIterator __l1,
2711 _BidirectionalIterator __f2, _BidirectionalIterator __l2, _Tp* __f3,
2712 _Compare __comp) {
2713 (__utils::__serial_move_merge(__n))(__f1, __l1, __f2, __l2, __f3, __comp, __move_values, __move_values,
2714 __move_sequences, __move_sequences);
2715 return __f3 + (__l1 - __f1) + (__l2 - __f2);
2716 });
2717 __par_backend::__parallel_for(
2718 std::forward<_ExecutionPolicy>(__exec), __r, __r + __n, [__r, __first, __is_vector](_Tp* __i, _Tp* __j) {
2719 __brick_move_destroy()(__i, __j, __first + (__i - __r), __is_vector);
2720 });
2721 });
2722 }
2723
2724 //------------------------------------------------------------------------
2725 // includes
2726 //------------------------------------------------------------------------
2727
2728 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Compare, class _IsVector>
2729 bool
__pattern_includes(_ExecutionPolicy &&,_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_Compare __comp,_IsVector,std::false_type)2730 __pattern_includes(_ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
2731 _ForwardIterator2 __first2, _ForwardIterator2 __last2, _Compare __comp, _IsVector,
2732 /*is_parallel=*/std::false_type) noexcept
2733 {
2734 return std::includes(__first1, __last1, __first2, __last2, __comp);
2735 }
2736
2737 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Compare, class _IsVector>
2738 bool
__pattern_includes(_ExecutionPolicy && __exec,_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_Compare __comp,_IsVector,std::true_type)2739 __pattern_includes(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
2740 _ForwardIterator2 __first2, _ForwardIterator2 __last2, _Compare __comp, _IsVector,
2741 /*is_parallel=*/std::true_type)
2742 {
2743 if (__first2 >= __last2)
2744 return true;
2745
2746 if (__first1 >= __last1 || __comp(*__first2, *__first1) || __comp(*(__last1 - 1), *(__last2 - 1)))
2747 return false;
2748
2749 __first1 = std::lower_bound(__first1, __last1, *__first2, __comp);
2750 if (__first1 == __last1)
2751 return false;
2752
2753 if (__last2 - __first2 == 1)
2754 return !__comp(*__first1, *__first2) && !__comp(*__first2, *__first1);
2755
2756 return __internal::__except_handler([&]() {
2757 return !__internal::__parallel_or(
2758 std::forward<_ExecutionPolicy>(__exec), __first2, __last2,
2759 [__first1, __last1, __first2, __last2, &__comp](_ForwardIterator2 __i, _ForwardIterator2 __j) {
2760 _PSTL_ASSERT(__j > __i);
2761 //_PSTL_ASSERT(__j - __i > 1);
2762
2763 //1. moving boundaries to "consume" subsequence of equal elements
2764 auto __is_equal = [&__comp](_ForwardIterator2 __a, _ForwardIterator2 __b) -> bool {
2765 return !__comp(*__a, *__b) && !__comp(*__b, *__a);
2766 };
2767
2768 //1.1 left bound, case "aaa[aaaxyz...]" - searching "x"
2769 if (__i > __first2 && __is_equal(__i, __i - 1))
2770 {
2771 //whole subrange continues to content equal elements - return "no op"
2772 if (__is_equal(__i, __j - 1))
2773 return false;
2774
2775 __i = std::upper_bound(__i, __last2, *__i, __comp);
2776 }
2777
2778 //1.2 right bound, case "[...aaa]aaaxyz" - searching "x"
2779 if (__j < __last2 && __is_equal(__j - 1, __j))
2780 __j = std::upper_bound(__j, __last2, *__j, __comp);
2781
2782 //2. testing is __a subsequence of the second range included into the first range
2783 auto __b = std::lower_bound(__first1, __last1, *__i, __comp);
2784
2785 _PSTL_ASSERT(!__comp(*(__last1 - 1), *__b));
2786 _PSTL_ASSERT(!__comp(*(__j - 1), *__i));
2787 return !std::includes(__b, __last1, __i, __j, __comp);
2788 });
2789 });
2790 }
2791
2792 constexpr auto __set_algo_cut_off = 1000;
2793
2794 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator,
2795 class _Compare, class _IsVector, class _SizeFunction, class _SetOP>
2796 _OutputIterator
__parallel_set_op(_ExecutionPolicy && __exec,_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_OutputIterator __result,_Compare __comp,_SizeFunction __size_func,_SetOP __set_op,_IsVector __is_vector)2797 __parallel_set_op(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
2798 _ForwardIterator2 __first2, _ForwardIterator2 __last2, _OutputIterator __result, _Compare __comp,
2799 _SizeFunction __size_func, _SetOP __set_op, _IsVector __is_vector)
2800 {
2801 typedef typename std::iterator_traits<_ForwardIterator1>::difference_type _DifferenceType;
2802 typedef typename std::iterator_traits<_OutputIterator>::value_type _Tp;
2803
2804 struct _SetRange
2805 {
2806 _DifferenceType __pos, __len, __buf_pos;
2807 bool
2808 empty() const
2809 {
2810 return __len == 0;
2811 }
2812 };
2813
2814 const _DifferenceType __n1 = __last1 - __first1;
2815 const _DifferenceType __n2 = __last2 - __first2;
2816
2817 __par_backend::__buffer<_Tp> __buf(__size_func(__n1, __n2));
2818
2819 return __internal::__except_handler([&__exec, __n1, __first1, __last1, __first2, __last2, __result, __is_vector,
2820 __comp, __size_func, __set_op, &__buf]() {
2821 auto __buffer = __buf.get();
2822 _DifferenceType __m{};
2823 auto __scan = [=](_DifferenceType, _DifferenceType, const _SetRange& __s) { // Scan
2824 if (!__s.empty())
2825 __brick_move_destroy()(__buffer + __s.__buf_pos,
2826 __buffer + (__s.__buf_pos + __s.__len), __result + __s.__pos,
2827 __is_vector);
2828 };
2829 __par_backend::__parallel_strict_scan(
2830 std::forward<_ExecutionPolicy>(__exec), __n1, _SetRange{0, 0, 0}, //-1, 0},
2831 [=](_DifferenceType __i, _DifferenceType __len) { // Reduce
2832 //[__b; __e) - a subrange of the first sequence, to reduce
2833 _ForwardIterator1 __b = __first1 + __i, __e = __first1 + (__i + __len);
2834
2835 //try searching for the first element which not equal to *__b
2836 if (__b != __first1)
2837 __b = std::upper_bound(__b, __last1, *__b, __comp);
2838
2839 //try searching for the first element which not equal to *__e
2840 if (__e != __last1)
2841 __e = std::upper_bound(__e, __last1, *__e, __comp);
2842
2843 //check is [__b; __e) empty
2844 if (__e - __b < 1)
2845 {
2846 _ForwardIterator2 __bb = __last2;
2847 if (__b != __last1)
2848 __bb = std::lower_bound(__first2, __last2, *__b, __comp);
2849
2850 const _DifferenceType __buf_pos = __size_func((__b - __first1), (__bb - __first2));
2851 return _SetRange{0, 0, __buf_pos};
2852 }
2853
2854 //try searching for "corresponding" subrange [__bb; __ee) in the second sequence
2855 _ForwardIterator2 __bb = __first2;
2856 if (__b != __first1)
2857 __bb = std::lower_bound(__first2, __last2, *__b, __comp);
2858
2859 _ForwardIterator2 __ee = __last2;
2860 if (__e != __last1)
2861 __ee = std::lower_bound(__bb, __last2, *__e, __comp);
2862
2863 const _DifferenceType __buf_pos = __size_func((__b - __first1), (__bb - __first2));
2864 auto __buffer_b = __buffer + __buf_pos;
2865 auto __res = __set_op(__b, __e, __bb, __ee, __buffer_b, __comp);
2866
2867 return _SetRange{0, __res - __buffer_b, __buf_pos};
2868 },
2869 [](const _SetRange& __a, const _SetRange& __b) { // Combine
2870 if (__b.__buf_pos > __a.__buf_pos || ((__b.__buf_pos == __a.__buf_pos) && !__b.empty()))
2871 return _SetRange{__a.__pos + __a.__len + __b.__pos, __b.__len, __b.__buf_pos};
2872 return _SetRange{__b.__pos + __b.__len + __a.__pos, __a.__len, __a.__buf_pos};
2873 },
2874 __scan, // Scan
2875 [&__m, &__scan](const _SetRange& __total) { // Apex
2876 //final scan
2877 __scan(0, 0, __total);
2878 __m = __total.__pos + __total.__len;
2879 });
2880 return __result + __m;
2881 });
2882 }
2883
2884 //a shared parallel pattern for '__pattern_set_union' and '__pattern_set_symmetric_difference'
2885 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator,
2886 class _Compare, class _SetUnionOp, class _IsVector>
2887 _OutputIterator
__parallel_set_union_op(_ExecutionPolicy && __exec,_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_OutputIterator __result,_Compare __comp,_SetUnionOp __set_union_op,_IsVector __is_vector)2888 __parallel_set_union_op(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
2889 _ForwardIterator2 __first2, _ForwardIterator2 __last2, _OutputIterator __result,
2890 _Compare __comp, _SetUnionOp __set_union_op, _IsVector __is_vector)
2891 {
2892 typedef typename std::iterator_traits<_ForwardIterator1>::difference_type _DifferenceType;
2893
2894 const auto __n1 = __last1 - __first1;
2895 const auto __n2 = __last2 - __first2;
2896
2897 auto copy_range1 = [__is_vector](_ForwardIterator1 __begin, _ForwardIterator1 __end, _OutputIterator __res) {
2898 return __internal::__brick_copy(__begin, __end, __res, __is_vector);
2899 };
2900 auto copy_range2 = [__is_vector](_ForwardIterator2 __begin, _ForwardIterator2 __end, _OutputIterator __res) {
2901 return __internal::__brick_copy(__begin, __end, __res, __is_vector);
2902 };
2903
2904 // {1} {}: parallel copying just first sequence
2905 if (__n2 == 0)
2906 return __internal::__pattern_walk2_brick(std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __result,
2907 copy_range1, std::true_type());
2908
2909 // {} {2}: parallel copying justmake second sequence
2910 if (__n1 == 0)
2911 return __internal::__pattern_walk2_brick(std::forward<_ExecutionPolicy>(__exec), __first2, __last2, __result,
2912 copy_range2, std::true_type());
2913
2914 // testing whether the sequences are intersected
2915 _ForwardIterator1 __left_bound_seq_1 = std::lower_bound(__first1, __last1, *__first2, __comp);
2916
2917 if (__left_bound_seq_1 == __last1)
2918 {
2919 //{1} < {2}: seq2 is wholly greater than seq1, so, do parallel copying seq1 and seq2
2920 __par_backend::__parallel_invoke(
2921 std::forward<_ExecutionPolicy>(__exec),
2922 [=] {
2923 __internal::__pattern_walk2_brick(std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __result,
2924 copy_range1, std::true_type());
2925 },
2926 [=] {
2927 __internal::__pattern_walk2_brick(std::forward<_ExecutionPolicy>(__exec), __first2, __last2,
2928 __result + __n1, copy_range2, std::true_type());
2929 });
2930 return __result + __n1 + __n2;
2931 }
2932
2933 // testing whether the sequences are intersected
2934 _ForwardIterator2 __left_bound_seq_2 = std::lower_bound(__first2, __last2, *__first1, __comp);
2935
2936 if (__left_bound_seq_2 == __last2)
2937 {
2938 //{2} < {1}: seq2 is wholly greater than seq1, so, do parallel copying seq1 and seq2
2939 __par_backend::__parallel_invoke(
2940 std::forward<_ExecutionPolicy>(__exec),
2941 [=] {
2942 __internal::__pattern_walk2_brick(std::forward<_ExecutionPolicy>(__exec), __first2, __last2, __result,
2943 copy_range2, std::true_type());
2944 },
2945 [=] {
2946 __internal::__pattern_walk2_brick(std::forward<_ExecutionPolicy>(__exec), __first1, __last1,
2947 __result + __n2, copy_range1, std::true_type());
2948 });
2949 return __result + __n1 + __n2;
2950 }
2951
2952 const auto __m1 = __left_bound_seq_1 - __first1;
2953 if (__m1 > __set_algo_cut_off)
2954 {
2955 auto __res_or = __result;
2956 __result += __m1; //we know proper offset due to [first1; left_bound_seq_1) < [first2; last2)
2957 __par_backend::__parallel_invoke(
2958 std::forward<_ExecutionPolicy>(__exec),
2959 //do parallel copying of [first1; left_bound_seq_1)
2960 [=] {
2961 __internal::__pattern_walk2_brick(std::forward<_ExecutionPolicy>(__exec), __first1, __left_bound_seq_1,
2962 __res_or, copy_range1, std::true_type());
2963 },
2964 [=, &__result] {
2965 __result = __internal::__parallel_set_op(
2966 std::forward<_ExecutionPolicy>(__exec), __left_bound_seq_1, __last1, __first2, __last2, __result,
2967 __comp, [](_DifferenceType __n, _DifferenceType __m) { return __n + __m; }, __set_union_op,
2968 __is_vector);
2969 });
2970 return __result;
2971 }
2972
2973 const auto __m2 = __left_bound_seq_2 - __first2;
2974 _PSTL_ASSERT(__m1 == 0 || __m2 == 0);
2975 if (__m2 > __set_algo_cut_off)
2976 {
2977 auto __res_or = __result;
2978 __result += __m2; //we know proper offset due to [first2; left_bound_seq_2) < [first1; last1)
2979 __par_backend::__parallel_invoke(
2980 std::forward<_ExecutionPolicy>(__exec),
2981 //do parallel copying of [first2; left_bound_seq_2)
2982 [=] {
2983 __internal::__pattern_walk2_brick(std::forward<_ExecutionPolicy>(__exec), __first2, __left_bound_seq_2,
2984 __res_or, copy_range2, std::true_type());
2985 },
2986 [=, &__result] {
2987 __result = __internal::__parallel_set_op(
2988 std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __left_bound_seq_2, __last2, __result,
2989 __comp, [](_DifferenceType __n, _DifferenceType __m) { return __n + __m; }, __set_union_op,
2990 __is_vector);
2991 });
2992 return __result;
2993 }
2994
2995 return __internal::__parallel_set_op(
2996 std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __result, __comp,
2997 [](_DifferenceType __n, _DifferenceType __m) { return __n + __m; }, __set_union_op, __is_vector);
2998 }
2999
3000 //------------------------------------------------------------------------
3001 // set_union
3002 //------------------------------------------------------------------------
3003
3004 template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
3005 _OutputIterator
__brick_set_union(_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_OutputIterator __result,_Compare __comp,std::false_type)3006 __brick_set_union(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
3007 _ForwardIterator2 __last2, _OutputIterator __result, _Compare __comp,
3008 /*__is_vector=*/std::false_type) noexcept
3009 {
3010 return std::set_union(__first1, __last1, __first2, __last2, __result, __comp);
3011 }
3012
3013 template <typename _IsVector>
3014 struct __BrickCopyConstruct
3015 {
3016 template <typename _ForwardIterator, typename _OutputIterator>
3017 _OutputIterator
operator__BrickCopyConstruct3018 operator()(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result)
3019 {
3020 return __brick_uninitialized_copy(__first, __last, __result, _IsVector());
3021 }
3022 };
3023
3024 template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
3025 _OutputIterator
__brick_set_union(_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_OutputIterator __result,_Compare __comp,std::true_type)3026 __brick_set_union(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
3027 _ForwardIterator2 __last2, _OutputIterator __result, _Compare __comp,
3028 /*__is_vector=*/std::true_type) noexcept
3029 {
3030 _PSTL_PRAGMA_MESSAGE("Vectorized algorithm unimplemented, redirected to serial");
3031 return std::set_union(__first1, __last1, __first2, __last2, __result, __comp);
3032 }
3033
3034 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator,
3035 class _Compare, class _IsVector>
3036 _OutputIterator
__pattern_set_union(_ExecutionPolicy &&,_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_OutputIterator __result,_Compare __comp,_IsVector __is_vector,std::false_type)3037 __pattern_set_union(_ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
3038 _ForwardIterator2 __first2, _ForwardIterator2 __last2, _OutputIterator __result, _Compare __comp,
3039 _IsVector __is_vector,
3040 /*is_parallel=*/std::false_type) noexcept
3041 {
3042 return __internal::__brick_set_union(__first1, __last1, __first2, __last2, __result, __comp, __is_vector);
3043 }
3044
3045 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator,
3046 class _Compare, class _IsVector>
3047 _OutputIterator
__pattern_set_union(_ExecutionPolicy && __exec,_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_OutputIterator __result,_Compare __comp,_IsVector __is_vector,std::true_type)3048 __pattern_set_union(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
3049 _ForwardIterator2 __first2, _ForwardIterator2 __last2, _OutputIterator __result, _Compare __comp,
3050 _IsVector __is_vector, /*__is_parallel=*/std::true_type)
3051 {
3052
3053 const auto __n1 = __last1 - __first1;
3054 const auto __n2 = __last2 - __first2;
3055
3056 // use serial algorithm
3057 if (__n1 + __n2 <= __set_algo_cut_off)
3058 return std::set_union(__first1, __last1, __first2, __last2, __result, __comp);
3059
3060 typedef typename std::iterator_traits<_OutputIterator>::value_type _Tp;
3061 return __parallel_set_union_op(
3062 std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __result, __comp,
3063 [](_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2, _ForwardIterator2 __last2,
3064 _Tp* __result, _Compare __comp) {
3065 return __pstl::__utils::__set_union_construct(__first1, __last1, __first2, __last2, __result, __comp,
3066 __BrickCopyConstruct<_IsVector>());
3067 },
3068 __is_vector);
3069 }
3070
3071 //------------------------------------------------------------------------
3072 // set_intersection
3073 //------------------------------------------------------------------------
3074
3075 template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
3076 _OutputIterator
__brick_set_intersection(_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_OutputIterator __result,_Compare __comp,std::false_type)3077 __brick_set_intersection(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
3078 _ForwardIterator2 __last2, _OutputIterator __result, _Compare __comp,
3079 /*__is_vector=*/std::false_type) noexcept
3080 {
3081 return std::set_intersection(__first1, __last1, __first2, __last2, __result, __comp);
3082 }
3083
3084 template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
3085 _OutputIterator
__brick_set_intersection(_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_OutputIterator __result,_Compare __comp,std::true_type)3086 __brick_set_intersection(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
3087 _ForwardIterator2 __last2, _OutputIterator __result, _Compare __comp,
3088 /*__is_vector=*/std::true_type) noexcept
3089 {
3090 _PSTL_PRAGMA_MESSAGE("Vectorized algorithm unimplemented, redirected to serial");
3091 return std::set_intersection(__first1, __last1, __first2, __last2, __result, __comp);
3092 }
3093
3094 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator,
3095 class _Compare, class _IsVector>
3096 _OutputIterator
__pattern_set_intersection(_ExecutionPolicy &&,_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_OutputIterator __result,_Compare __comp,_IsVector __is_vector,std::false_type)3097 __pattern_set_intersection(_ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
3098 _ForwardIterator2 __first2, _ForwardIterator2 __last2, _OutputIterator __result,
3099 _Compare __comp, _IsVector __is_vector, /*is_parallel=*/std::false_type) noexcept
3100 {
3101 return __internal::__brick_set_intersection(__first1, __last1, __first2, __last2, __result, __comp, __is_vector);
3102 }
3103
3104 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator,
3105 class _Compare, class _IsVector>
3106 _OutputIterator
__pattern_set_intersection(_ExecutionPolicy && __exec,_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_OutputIterator __result,_Compare __comp,_IsVector __is_vector,std::true_type)3107 __pattern_set_intersection(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
3108 _ForwardIterator2 __first2, _ForwardIterator2 __last2, _OutputIterator __result,
3109 _Compare __comp, _IsVector __is_vector, /*is_parallel=*/std::true_type)
3110 {
3111 typedef typename std::iterator_traits<_OutputIterator>::value_type _Tp;
3112 typedef typename std::iterator_traits<_ForwardIterator1>::difference_type _DifferenceType;
3113
3114 const auto __n1 = __last1 - __first1;
3115 const auto __n2 = __last2 - __first2;
3116
3117 // intersection is empty
3118 if (__n1 == 0 || __n2 == 0)
3119 return __result;
3120
3121 // testing whether the sequences are intersected
3122 _ForwardIterator1 __left_bound_seq_1 = std::lower_bound(__first1, __last1, *__first2, __comp);
3123 //{1} < {2}: seq 2 is wholly greater than seq 1, so, the intersection is empty
3124 if (__left_bound_seq_1 == __last1)
3125 return __result;
3126
3127 // testing whether the sequences are intersected
3128 _ForwardIterator2 __left_bound_seq_2 = std::lower_bound(__first2, __last2, *__first1, __comp);
3129 //{2} < {1}: seq 1 is wholly greater than seq 2, so, the intersection is empty
3130 if (__left_bound_seq_2 == __last2)
3131 return __result;
3132
3133 const auto __m1 = __last1 - __left_bound_seq_1 + __n2;
3134 if (__m1 > __set_algo_cut_off)
3135 {
3136 //we know proper offset due to [first1; left_bound_seq_1) < [first2; last2)
3137 return __internal::__parallel_set_op(
3138 std::forward<_ExecutionPolicy>(__exec), __left_bound_seq_1, __last1, __first2, __last2, __result, __comp,
3139 [](_DifferenceType __n, _DifferenceType __m) { return std::min(__n, __m); },
3140 [](_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
3141 _ForwardIterator2 __last2, _Tp* __result, _Compare __comp) {
3142 return __pstl::__utils::__set_intersection_construct(__first1, __last1, __first2, __last2, __result,
3143 __comp);
3144 },
3145 __is_vector);
3146 }
3147
3148 const auto __m2 = __last2 - __left_bound_seq_2 + __n1;
3149 if (__m2 > __set_algo_cut_off)
3150 {
3151 //we know proper offset due to [first2; left_bound_seq_2) < [first1; last1)
3152 __result = __internal::__parallel_set_op(
3153 std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __left_bound_seq_2, __last2, __result, __comp,
3154 [](_DifferenceType __n, _DifferenceType __m) { return std::min(__n, __m); },
3155 [](_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
3156 _ForwardIterator2 __last2, _Tp* __result, _Compare __comp) {
3157 return __pstl::__utils::__set_intersection_construct(__first2, __last2, __first1, __last1, __result,
3158 __comp);
3159 },
3160 __is_vector);
3161 return __result;
3162 }
3163
3164 // [left_bound_seq_1; last1) and [left_bound_seq_2; last2) - use serial algorithm
3165 return std::set_intersection(__left_bound_seq_1, __last1, __left_bound_seq_2, __last2, __result, __comp);
3166 }
3167
3168 //------------------------------------------------------------------------
3169 // set_difference
3170 //------------------------------------------------------------------------
3171
3172 template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
3173 _OutputIterator
__brick_set_difference(_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_OutputIterator __result,_Compare __comp,std::false_type)3174 __brick_set_difference(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
3175 _ForwardIterator2 __last2, _OutputIterator __result, _Compare __comp,
3176 /*__is_vector=*/std::false_type) noexcept
3177 {
3178 return std::set_difference(__first1, __last1, __first2, __last2, __result, __comp);
3179 }
3180
3181 template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
3182 _OutputIterator
__brick_set_difference(_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_OutputIterator __result,_Compare __comp,std::true_type)3183 __brick_set_difference(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
3184 _ForwardIterator2 __last2, _OutputIterator __result, _Compare __comp,
3185 /*__is_vector=*/std::true_type) noexcept
3186 {
3187 _PSTL_PRAGMA_MESSAGE("Vectorized algorithm unimplemented, redirected to serial");
3188 return std::set_difference(__first1, __last1, __first2, __last2, __result, __comp);
3189 }
3190
3191 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator,
3192 class _Compare, class _IsVector>
3193 _OutputIterator
__pattern_set_difference(_ExecutionPolicy &&,_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_OutputIterator __result,_Compare __comp,_IsVector __is_vector,std::false_type)3194 __pattern_set_difference(_ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
3195 _ForwardIterator2 __first2, _ForwardIterator2 __last2, _OutputIterator __result,
3196 _Compare __comp, _IsVector __is_vector, /*is_parallel=*/std::false_type) noexcept
3197 {
3198 return __internal::__brick_set_difference(__first1, __last1, __first2, __last2, __result, __comp, __is_vector);
3199 }
3200
3201 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator,
3202 class _Compare, class _IsVector>
3203 _OutputIterator
__pattern_set_difference(_ExecutionPolicy && __exec,_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_OutputIterator __result,_Compare __comp,_IsVector __is_vector,std::true_type)3204 __pattern_set_difference(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
3205 _ForwardIterator2 __first2, _ForwardIterator2 __last2, _OutputIterator __result,
3206 _Compare __comp, _IsVector __is_vector, /*is_parallel=*/std::true_type)
3207 {
3208 typedef typename std::iterator_traits<_OutputIterator>::value_type _Tp;
3209 typedef typename std::iterator_traits<_ForwardIterator1>::difference_type _DifferenceType;
3210
3211 const auto __n1 = __last1 - __first1;
3212 const auto __n2 = __last2 - __first2;
3213
3214 // {} \ {2}: the difference is empty
3215 if (__n1 == 0)
3216 return __result;
3217
3218 // {1} \ {}: parallel copying just first sequence
3219 if (__n2 == 0)
3220 return __internal::__pattern_walk2_brick(
3221 std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __result,
3222 [__is_vector](_ForwardIterator1 __begin, _ForwardIterator1 __end, _OutputIterator __res) {
3223 return __internal::__brick_copy(__begin, __end, __res, __is_vector);
3224 },
3225 std::true_type());
3226
3227 // testing whether the sequences are intersected
3228 _ForwardIterator1 __left_bound_seq_1 = std::lower_bound(__first1, __last1, *__first2, __comp);
3229 //{1} < {2}: seq 2 is wholly greater than seq 1, so, parallel copying just first sequence
3230 if (__left_bound_seq_1 == __last1)
3231 return __internal::__pattern_walk2_brick(
3232 std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __result,
3233 [__is_vector](_ForwardIterator1 __begin, _ForwardIterator1 __end, _OutputIterator __res) {
3234 return __internal::__brick_copy(__begin, __end, __res, __is_vector);
3235 },
3236 std::true_type());
3237
3238 // testing whether the sequences are intersected
3239 _ForwardIterator2 __left_bound_seq_2 = std::lower_bound(__first2, __last2, *__first1, __comp);
3240 //{2} < {1}: seq 1 is wholly greater than seq 2, so, parallel copying just first sequence
3241 if (__left_bound_seq_2 == __last2)
3242 return __internal::__pattern_walk2_brick(
3243 std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __result,
3244 [__is_vector](_ForwardIterator1 __begin, _ForwardIterator1 __end, _OutputIterator __res) {
3245 return __internal::__brick_copy(__begin, __end, __res, __is_vector);
3246 },
3247 std::true_type());
3248
3249 if (__n1 + __n2 > __set_algo_cut_off)
3250 return __parallel_set_op(std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __result,
3251 __comp, [](_DifferenceType __n, _DifferenceType) { return __n; },
3252 [](_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
3253 _ForwardIterator2 __last2, _Tp* __result, _Compare __comp) {
3254 return __pstl::__utils::__set_difference_construct(
3255 __first1, __last1, __first2, __last2, __result, __comp,
3256 __BrickCopyConstruct<_IsVector>());
3257 },
3258 __is_vector);
3259
3260 // use serial algorithm
3261 return std::set_difference(__first1, __last1, __first2, __last2, __result, __comp);
3262 }
3263
3264 //------------------------------------------------------------------------
3265 // set_symmetric_difference
3266 //------------------------------------------------------------------------
3267
3268 template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
3269 _OutputIterator
__brick_set_symmetric_difference(_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_OutputIterator __result,_Compare __comp,std::false_type)3270 __brick_set_symmetric_difference(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
3271 _ForwardIterator2 __last2, _OutputIterator __result, _Compare __comp,
3272 /*__is_vector=*/std::false_type) noexcept
3273 {
3274 return std::set_symmetric_difference(__first1, __last1, __first2, __last2, __result, __comp);
3275 }
3276
3277 template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
3278 _OutputIterator
__brick_set_symmetric_difference(_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_OutputIterator __result,_Compare __comp,std::true_type)3279 __brick_set_symmetric_difference(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
3280 _ForwardIterator2 __last2, _OutputIterator __result, _Compare __comp,
3281 /*__is_vector=*/std::true_type) noexcept
3282 {
3283 _PSTL_PRAGMA_MESSAGE("Vectorized algorithm unimplemented, redirected to serial");
3284 return std::set_symmetric_difference(__first1, __last1, __first2, __last2, __result, __comp);
3285 }
3286
3287 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator,
3288 class _Compare, class _IsVector>
3289 _OutputIterator
__pattern_set_symmetric_difference(_ExecutionPolicy &&,_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_OutputIterator __result,_Compare __comp,_IsVector __is_vector,std::false_type)3290 __pattern_set_symmetric_difference(_ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
3291 _ForwardIterator2 __first2, _ForwardIterator2 __last2, _OutputIterator __result,
3292 _Compare __comp, _IsVector __is_vector, /*is_parallel=*/std::false_type) noexcept
3293 {
3294 return __internal::__brick_set_symmetric_difference(__first1, __last1, __first2, __last2, __result, __comp,
3295 __is_vector);
3296 }
3297
3298 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator,
3299 class _Compare, class _IsVector>
3300 _OutputIterator
__pattern_set_symmetric_difference(_ExecutionPolicy && __exec,_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_OutputIterator __result,_Compare __comp,_IsVector __is_vector,std::true_type)3301 __pattern_set_symmetric_difference(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
3302 _ForwardIterator2 __first2, _ForwardIterator2 __last2, _OutputIterator __result,
3303 _Compare __comp, _IsVector __is_vector, /*is_parallel=*/std::true_type)
3304 {
3305
3306 const auto __n1 = __last1 - __first1;
3307 const auto __n2 = __last2 - __first2;
3308
3309 // use serial algorithm
3310 if (__n1 + __n2 <= __set_algo_cut_off)
3311 return std::set_symmetric_difference(__first1, __last1, __first2, __last2, __result, __comp);
3312
3313 typedef typename std::iterator_traits<_OutputIterator>::value_type _Tp;
3314 return __internal::__parallel_set_union_op(
3315 std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __result, __comp,
3316 [](_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2, _ForwardIterator2 __last2,
3317 _Tp* __result, _Compare __comp) {
3318 return __pstl::__utils::__set_symmetric_difference_construct(__first1, __last1, __first2, __last2, __result,
3319 __comp, __BrickCopyConstruct<_IsVector>());
3320 },
3321 __is_vector);
3322 }
3323
3324 //------------------------------------------------------------------------
3325 // is_heap_until
3326 //------------------------------------------------------------------------
3327
3328 template <class _RandomAccessIterator, class _Compare>
3329 _RandomAccessIterator
__brick_is_heap_until(_RandomAccessIterator __first,_RandomAccessIterator __last,_Compare __comp,std::false_type)3330 __brick_is_heap_until(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp,
3331 /* __is_vector = */ std::false_type) noexcept
3332 {
3333 return std::is_heap_until(__first, __last, __comp);
3334 }
3335
3336 template <class _RandomAccessIterator, class _Compare>
3337 _RandomAccessIterator
__brick_is_heap_until(_RandomAccessIterator __first,_RandomAccessIterator __last,_Compare __comp,std::true_type)3338 __brick_is_heap_until(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp,
3339 /* __is_vector = */ std::true_type) noexcept
3340 {
3341 if (__last - __first < 2)
3342 return __last;
3343 typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type _SizeType;
3344 return __unseq_backend::__simd_first(
3345 __first, _SizeType(0), __last - __first,
3346 [&__comp](_RandomAccessIterator __it, _SizeType __i) { return __comp(__it[(__i - 1) / 2], __it[__i]); });
3347 }
3348
3349 template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare, class _IsVector>
3350 _RandomAccessIterator
__pattern_is_heap_until(_ExecutionPolicy &&,_RandomAccessIterator __first,_RandomAccessIterator __last,_Compare __comp,_IsVector __is_vector,std::false_type)3351 __pattern_is_heap_until(_ExecutionPolicy&&, _RandomAccessIterator __first, _RandomAccessIterator __last,
3352 _Compare __comp, _IsVector __is_vector, /* is_parallel = */ std::false_type) noexcept
3353 {
3354 return __internal::__brick_is_heap_until(__first, __last, __comp, __is_vector);
3355 }
3356
3357 template <class _RandomAccessIterator, class _DifferenceType, class _Compare>
3358 _RandomAccessIterator
__is_heap_until_local(_RandomAccessIterator __first,_DifferenceType __begin,_DifferenceType __end,_Compare __comp,std::false_type)3359 __is_heap_until_local(_RandomAccessIterator __first, _DifferenceType __begin, _DifferenceType __end, _Compare __comp,
3360 /* __is_vector = */ std::false_type) noexcept
3361 {
3362 _DifferenceType __i = __begin;
3363 for (; __i < __end; ++__i)
3364 {
3365 if (__comp(__first[(__i - 1) / 2], __first[__i]))
3366 {
3367 break;
3368 }
3369 }
3370 return __first + __i;
3371 }
3372
3373 template <class _RandomAccessIterator, class _DifferenceType, class _Compare>
3374 _RandomAccessIterator
__is_heap_until_local(_RandomAccessIterator __first,_DifferenceType __begin,_DifferenceType __end,_Compare __comp,std::true_type)3375 __is_heap_until_local(_RandomAccessIterator __first, _DifferenceType __begin, _DifferenceType __end, _Compare __comp,
3376 /* __is_vector = */ std::true_type) noexcept
3377 {
3378 return __unseq_backend::__simd_first(
3379 __first, __begin, __end,
3380 [&__comp](_RandomAccessIterator __it, _DifferenceType __i) { return __comp(__it[(__i - 1) / 2], __it[__i]); });
3381 }
3382
3383 template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare, class _IsVector>
3384 _RandomAccessIterator
__pattern_is_heap_until(_ExecutionPolicy && __exec,_RandomAccessIterator __first,_RandomAccessIterator __last,_Compare __comp,_IsVector __is_vector,std::true_type)3385 __pattern_is_heap_until(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last,
3386 _Compare __comp, _IsVector __is_vector, /* is_parallel = */ std::true_type) noexcept
3387 {
3388 if (__last - __first < 2)
3389 return __last;
3390
3391 return __internal::__except_handler([&]() {
3392 return __parallel_find(
3393 std::forward<_ExecutionPolicy>(__exec), __first, __last,
3394 [__first, __comp, __is_vector](_RandomAccessIterator __i, _RandomAccessIterator __j) {
3395 return __internal::__is_heap_until_local(__first, __i - __first, __j - __first, __comp, __is_vector);
3396 },
3397 std::less<typename std::iterator_traits<_RandomAccessIterator>::difference_type>(), /*is_first=*/true);
3398 });
3399 }
3400
3401 //------------------------------------------------------------------------
3402 // min_element
3403 //------------------------------------------------------------------------
3404
3405 template <typename _ForwardIterator, typename _Compare>
3406 _ForwardIterator
__brick_min_element(_ForwardIterator __first,_ForwardIterator __last,_Compare __comp,std::false_type)3407 __brick_min_element(_ForwardIterator __first, _ForwardIterator __last, _Compare __comp,
3408 /* __is_vector = */ std::false_type) noexcept
3409 {
3410 return std::min_element(__first, __last, __comp);
3411 }
3412
3413 template <typename _ForwardIterator, typename _Compare>
3414 _ForwardIterator
__brick_min_element(_ForwardIterator __first,_ForwardIterator __last,_Compare __comp,std::true_type)3415 __brick_min_element(_ForwardIterator __first, _ForwardIterator __last, _Compare __comp,
3416 /* __is_vector = */ std::true_type) noexcept
3417 {
3418 #if _PSTL_UDR_PRESENT
3419 return __unseq_backend::__simd_min_element(__first, __last - __first, __comp);
3420 #else
3421 return std::min_element(__first, __last, __comp);
3422 #endif
3423 }
3424
3425 template <typename _ExecutionPolicy, typename _ForwardIterator, typename _Compare, typename _IsVector>
3426 _ForwardIterator
__pattern_min_element(_ExecutionPolicy &&,_ForwardIterator __first,_ForwardIterator __last,_Compare __comp,_IsVector __is_vector,std::false_type)3427 __pattern_min_element(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Compare __comp,
3428 _IsVector __is_vector, /* is_parallel = */ std::false_type) noexcept
3429 {
3430 return __internal::__brick_min_element(__first, __last, __comp, __is_vector);
3431 }
3432
3433 template <typename _ExecutionPolicy, typename _RandomAccessIterator, typename _Compare, typename _IsVector>
3434 _RandomAccessIterator
__pattern_min_element(_ExecutionPolicy && __exec,_RandomAccessIterator __first,_RandomAccessIterator __last,_Compare __comp,_IsVector __is_vector,std::true_type)3435 __pattern_min_element(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last,
3436 _Compare __comp, _IsVector __is_vector, /* is_parallel = */ std::true_type)
3437 {
3438 if (__first == __last)
3439 return __last;
3440
3441 return __internal::__except_handler([&]() {
3442 return __par_backend::__parallel_reduce(
3443 std::forward<_ExecutionPolicy>(__exec), __first + 1, __last, __first,
3444 [=](_RandomAccessIterator __begin, _RandomAccessIterator __end,
3445 _RandomAccessIterator __init) -> _RandomAccessIterator {
3446 const _RandomAccessIterator subresult =
3447 __internal::__brick_min_element(__begin, __end, __comp, __is_vector);
3448 return __internal::__cmp_iterators_by_values(__init, subresult, __comp);
3449 },
3450 [=](_RandomAccessIterator __it1, _RandomAccessIterator __it2) -> _RandomAccessIterator {
3451 return __internal::__cmp_iterators_by_values(__it1, __it2, __comp);
3452 });
3453 });
3454 }
3455
3456 //------------------------------------------------------------------------
3457 // minmax_element
3458 //------------------------------------------------------------------------
3459
3460 template <typename _ForwardIterator, typename _Compare>
3461 std::pair<_ForwardIterator, _ForwardIterator>
__brick_minmax_element(_ForwardIterator __first,_ForwardIterator __last,_Compare __comp,std::false_type)3462 __brick_minmax_element(_ForwardIterator __first, _ForwardIterator __last, _Compare __comp,
3463 /* __is_vector = */ std::false_type) noexcept
3464 {
3465 return std::minmax_element(__first, __last, __comp);
3466 }
3467
3468 template <typename _ForwardIterator, typename _Compare>
3469 std::pair<_ForwardIterator, _ForwardIterator>
__brick_minmax_element(_ForwardIterator __first,_ForwardIterator __last,_Compare __comp,std::true_type)3470 __brick_minmax_element(_ForwardIterator __first, _ForwardIterator __last, _Compare __comp,
3471 /* __is_vector = */ std::true_type) noexcept
3472 {
3473 #if _PSTL_UDR_PRESENT
3474 return __unseq_backend::__simd_minmax_element(__first, __last - __first, __comp);
3475 #else
3476 return std::minmax_element(__first, __last, __comp);
3477 #endif
3478 }
3479
3480 template <typename _ExecutionPolicy, typename _ForwardIterator, typename _Compare, typename _IsVector>
3481 std::pair<_ForwardIterator, _ForwardIterator>
__pattern_minmax_element(_ExecutionPolicy &&,_ForwardIterator __first,_ForwardIterator __last,_Compare __comp,_IsVector __is_vector,std::false_type)3482 __pattern_minmax_element(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Compare __comp,
3483 _IsVector __is_vector, /* is_parallel = */ std::false_type) noexcept
3484 {
3485 return __internal::__brick_minmax_element(__first, __last, __comp, __is_vector);
3486 }
3487
3488 template <typename _ExecutionPolicy, typename _ForwardIterator, typename _Compare, typename _IsVector>
3489 std::pair<_ForwardIterator, _ForwardIterator>
__pattern_minmax_element(_ExecutionPolicy && __exec,_ForwardIterator __first,_ForwardIterator __last,_Compare __comp,_IsVector __is_vector,std::true_type)3490 __pattern_minmax_element(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Compare __comp,
3491 _IsVector __is_vector, /* is_parallel = */ std::true_type)
3492 {
3493 if (__first == __last)
3494 return std::make_pair(__first, __first);
3495
3496 return __internal::__except_handler([&]() {
3497 typedef std::pair<_ForwardIterator, _ForwardIterator> _Result;
3498
3499 return __par_backend::__parallel_reduce(
3500 std::forward<_ExecutionPolicy>(__exec), __first + 1, __last, std::make_pair(__first, __first),
3501 [=](_ForwardIterator __begin, _ForwardIterator __end, _Result __init) -> _Result {
3502 const _Result __subresult = __internal::__brick_minmax_element(__begin, __end, __comp, __is_vector);
3503 return std::make_pair(
3504 __internal::__cmp_iterators_by_values(__subresult.first, __init.first, __comp),
3505 __internal::__cmp_iterators_by_values(__init.second, __subresult.second, std::not_fn(__comp)));
3506 },
3507 [=](_Result __p1, _Result __p2) -> _Result {
3508 return std::make_pair(
3509 __internal::__cmp_iterators_by_values(__p1.first, __p2.first, __comp),
3510 __internal::__cmp_iterators_by_values(__p2.second, __p1.second, std::not_fn(__comp)));
3511 });
3512 });
3513 }
3514
3515 //------------------------------------------------------------------------
3516 // mismatch
3517 //------------------------------------------------------------------------
3518 template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
3519 std::pair<_ForwardIterator1, _ForwardIterator2>
__mismatch_serial(_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_BinaryPredicate __pred)3520 __mismatch_serial(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
3521 _ForwardIterator2 __last2, _BinaryPredicate __pred)
3522 {
3523 #if _PSTL_CPP14_2RANGE_MISMATCH_EQUAL_PRESENT
3524 return std::mismatch(__first1, __last1, __first2, __last2, __pred);
3525 #else
3526 for (; __first1 != __last1 && __first2 != __last2 && __pred(*__first1, *__first2); ++__first1, ++__first2)
3527 {
3528 }
3529 return std::make_pair(__first1, __first2);
3530 #endif
3531 }
3532
3533 template <class _ForwardIterator1, class _ForwardIterator2, class _Predicate>
3534 std::pair<_ForwardIterator1, _ForwardIterator2>
__brick_mismatch(_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_Predicate __pred,std::false_type)3535 __brick_mismatch(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
3536 _ForwardIterator2 __last2, _Predicate __pred, /* __is_vector = */ std::false_type) noexcept
3537 {
3538 return __mismatch_serial(__first1, __last1, __first2, __last2, __pred);
3539 }
3540
3541 template <class _ForwardIterator1, class _ForwardIterator2, class _Predicate>
3542 std::pair<_ForwardIterator1, _ForwardIterator2>
__brick_mismatch(_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_Predicate __pred,std::true_type)3543 __brick_mismatch(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
3544 _ForwardIterator2 __last2, _Predicate __pred, /* __is_vector = */ std::true_type) noexcept
3545 {
3546 auto __n = std::min(__last1 - __first1, __last2 - __first2);
3547 return __unseq_backend::__simd_first(__first1, __n, __first2, std::not_fn(__pred));
3548 }
3549
3550 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Predicate, class _IsVector>
3551 std::pair<_ForwardIterator1, _ForwardIterator2>
__pattern_mismatch(_ExecutionPolicy &&,_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_Predicate __pred,_IsVector __is_vector,std::false_type)3552 __pattern_mismatch(_ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
3553 _ForwardIterator2 __first2, _ForwardIterator2 __last2, _Predicate __pred, _IsVector __is_vector,
3554 /* is_parallel = */ std::false_type) noexcept
3555 {
3556 return __internal::__brick_mismatch(__first1, __last1, __first2, __last2, __pred, __is_vector);
3557 }
3558
3559 template <class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2, class _Predicate,
3560 class _IsVector>
3561 std::pair<_RandomAccessIterator1, _RandomAccessIterator2>
__pattern_mismatch(_ExecutionPolicy && __exec,_RandomAccessIterator1 __first1,_RandomAccessIterator1 __last1,_RandomAccessIterator2 __first2,_RandomAccessIterator2 __last2,_Predicate __pred,_IsVector __is_vector,std::true_type)3562 __pattern_mismatch(_ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1,
3563 _RandomAccessIterator2 __first2, _RandomAccessIterator2 __last2, _Predicate __pred,
3564 _IsVector __is_vector, /* is_parallel = */ std::true_type) noexcept
3565 {
3566 return __internal::__except_handler([&]() {
3567 auto __n = std::min(__last1 - __first1, __last2 - __first2);
3568 auto __result = __internal::__parallel_find(
3569 std::forward<_ExecutionPolicy>(__exec), __first1, __first1 + __n,
3570 [__first1, __first2, __pred, __is_vector](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j) {
3571 return __internal::__brick_mismatch(__i, __j, __first2 + (__i - __first1), __first2 + (__j - __first1),
3572 __pred, __is_vector)
3573 .first;
3574 },
3575 std::less<typename std::iterator_traits<_RandomAccessIterator1>::difference_type>(), /*is_first=*/true);
3576 return std::make_pair(__result, __first2 + (__result - __first1));
3577 });
3578 }
3579
3580 //------------------------------------------------------------------------
3581 // lexicographical_compare
3582 //------------------------------------------------------------------------
3583
3584 template <class _ForwardIterator1, class _ForwardIterator2, class _Compare>
3585 bool
__brick_lexicographical_compare(_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_Compare __comp,std::false_type)3586 __brick_lexicographical_compare(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
3587 _ForwardIterator2 __last2, _Compare __comp,
3588 /* __is_vector = */ std::false_type) noexcept
3589 {
3590 return std::lexicographical_compare(__first1, __last1, __first2, __last2, __comp);
3591 }
3592
3593 template <class _ForwardIterator1, class _ForwardIterator2, class _Compare>
3594 bool
__brick_lexicographical_compare(_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_Compare __comp,std::true_type)3595 __brick_lexicographical_compare(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
3596 _ForwardIterator2 __last2, _Compare __comp, /* __is_vector = */ std::true_type) noexcept
3597 {
3598 if (__first2 == __last2)
3599 { // if second sequence is empty
3600 return false;
3601 }
3602 else if (__first1 == __last1)
3603 { // if first sequence is empty
3604 return true;
3605 }
3606 else
3607 {
3608 typedef typename std::iterator_traits<_ForwardIterator1>::reference ref_type1;
3609 typedef typename std::iterator_traits<_ForwardIterator2>::reference ref_type2;
3610 --__last1;
3611 --__last2;
3612 auto __n = std::min(__last1 - __first1, __last2 - __first2);
3613 std::pair<_ForwardIterator1, _ForwardIterator2> __result = __unseq_backend::__simd_first(
3614 __first1, __n, __first2, [__comp](const ref_type1 __x, const ref_type2 __y) mutable {
3615 return __comp(__x, __y) || __comp(__y, __x);
3616 });
3617
3618 if (__result.first == __last1 && __result.second != __last2)
3619 { // if first sequence shorter than second
3620 return !__comp(*__result.second, *__result.first);
3621 }
3622 else
3623 { // if second sequence shorter than first or both have the same number of elements
3624 return __comp(*__result.first, *__result.second);
3625 }
3626 }
3627 }
3628
3629 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Compare, class _IsVector>
3630 bool
__pattern_lexicographical_compare(_ExecutionPolicy &&,_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_Compare __comp,_IsVector __is_vector,std::false_type)3631 __pattern_lexicographical_compare(_ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
3632 _ForwardIterator2 __first2, _ForwardIterator2 __last2, _Compare __comp,
3633 _IsVector __is_vector, /* is_parallel = */ std::false_type) noexcept
3634 {
3635 return __internal::__brick_lexicographical_compare(__first1, __last1, __first2, __last2, __comp, __is_vector);
3636 }
3637
3638 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Compare, class _IsVector>
3639 bool
__pattern_lexicographical_compare(_ExecutionPolicy && __exec,_ForwardIterator1 __first1,_ForwardIterator1 __last1,_ForwardIterator2 __first2,_ForwardIterator2 __last2,_Compare __comp,_IsVector __is_vector,std::true_type)3640 __pattern_lexicographical_compare(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
3641 _ForwardIterator2 __first2, _ForwardIterator2 __last2, _Compare __comp,
3642 _IsVector __is_vector, /* is_parallel = */ std::true_type) noexcept
3643 {
3644 if (__first2 == __last2)
3645 { // if second sequence is empty
3646 return false;
3647 }
3648 else if (__first1 == __last1)
3649 { // if first sequence is empty
3650 return true;
3651 }
3652 else
3653 {
3654 typedef typename std::iterator_traits<_ForwardIterator1>::reference _RefType1;
3655 typedef typename std::iterator_traits<_ForwardIterator2>::reference _RefType2;
3656 --__last1;
3657 --__last2;
3658 auto __n = std::min(__last1 - __first1, __last2 - __first2);
3659 auto __result = __internal::__parallel_find(
3660 std::forward<_ExecutionPolicy>(__exec), __first1, __first1 + __n,
3661 [__first1, __first2, &__comp, __is_vector](_ForwardIterator1 __i, _ForwardIterator1 __j) {
3662 return __internal::__brick_mismatch(__i, __j, __first2 + (__i - __first1), __first2 + (__j - __first1),
3663 [&__comp](const _RefType1 __x, const _RefType2 __y) {
3664 return !__comp(__x, __y) && !__comp(__y, __x);
3665 },
3666 __is_vector)
3667 .first;
3668 },
3669 std::less<typename std::iterator_traits<_ForwardIterator1>::difference_type>(), /*is_first=*/true);
3670
3671 if (__result == __last1 && __first2 + (__result - __first1) != __last2)
3672 { // if first sequence shorter than second
3673 return !__comp(*(__first2 + (__result - __first1)), *__result);
3674 }
3675 else
3676 { // if second sequence shorter than first or both have the same number of elements
3677 return __comp(*__result, *(__first2 + (__result - __first1)));
3678 }
3679 }
3680 }
3681
3682 } // namespace __internal
3683 } // namespace __pstl
3684
3685 _PSTL_HIDE_FROM_ABI_POP
3686
3687 #endif /* _PSTL_ALGORITHM_IMPL_H */
3688