1/****************************************************************************
2 * Copyright (C) 2017 Intel Corporation.   All Rights Reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 ****************************************************************************/
23#if !defined(__SIMD_LIB_AVX2_HPP__)
24#error Do not include this file directly, use "simdlib.hpp" instead.
25#endif
26
27//============================================================================
28// SIMD256 AVX (2) implementation
29//
30// Since this implementation inherits from the AVX (1) implementation,
31// the only operations below ones that replace AVX (1) operations.
32// Mostly these are integer operations that are no longer emulated with SSE
33//============================================================================
34
35#define SIMD_IWRAPPER_1(op) \
36    static SIMDINLINE Integer SIMDCALL op(Integer const& a) { return _mm256_##op(a); }
37
38#define SIMD_IWRAPPER_1L(op)                                \
39    static SIMDINLINE Integer SIMDCALL op(Integer const& a) \
40    {                                                       \
41        return _mm256_##op(_mm256_castsi256_si128(a));      \
42    }
43
44#define SIMD_IWRAPPER_1I(op)                                \
45    template <int ImmT>                                     \
46    static SIMDINLINE Integer SIMDCALL op(Integer const& a) \
47    {                                                       \
48        return _mm256_##op(a, ImmT);                        \
49    }
50
51#define SIMD_IWRAPPER_1I_(op, intrin)                       \
52    template <int ImmT>                                     \
53    static SIMDINLINE Integer SIMDCALL op(Integer const& a) \
54    {                                                       \
55        return _mm256_##intrin(a, ImmT);                    \
56    }
57
58#define SIMD_IWRAPPER_2_(op, intrin)                                          \
59    static SIMDINLINE Integer SIMDCALL op(Integer const& a, Integer const& b) \
60    {                                                                         \
61        return _mm256_##intrin(a, b);                                         \
62    }
63
64#define SIMD_IWRAPPER_2(op)                                                   \
65    static SIMDINLINE Integer SIMDCALL op(Integer const& a, Integer const& b) \
66    {                                                                         \
67        return _mm256_##op(a, b);                                             \
68    }
69
70#define SIMD_IWRAPPER_2I(op)                                                  \
71    template <int ImmT>                                                       \
72    static SIMDINLINE Integer SIMDCALL op(Integer const& a, Integer const& b) \
73    {                                                                         \
74        return _mm256_##op(a, b, ImmT);                                       \
75    }
76
77#define SIMD_IWRAPPER_2I(op)                                                  \
78    template <int ImmT>                                                       \
79    static SIMDINLINE Integer SIMDCALL op(Integer const& a, Integer const& b) \
80    {                                                                         \
81        return _mm256_##op(a, b, ImmT);                                       \
82    }
83
84
85//-----------------------------------------------------------------------
86// Floating point arithmetic operations
87//-----------------------------------------------------------------------
88static SIMDINLINE Float SIMDCALL fmadd_ps(Float const& a,
89                                          Float const& b,
90                                          Float const& c) // return (a * b) + c
91{
92    return _mm256_fmadd_ps(a, b, c);
93}
94
95//-----------------------------------------------------------------------
96// Integer (various width) arithmetic operations
97//-----------------------------------------------------------------------
98SIMD_IWRAPPER_1(abs_epi32); // return absolute_value(a) (int32)
99SIMD_IWRAPPER_2(add_epi32); // return a + b (int32)
100SIMD_IWRAPPER_2(add_epi8);  // return a + b (int8)
101SIMD_IWRAPPER_2(adds_epu8); // return ((a + b) > 0xff) ? 0xff : (a + b) (uint8)
102SIMD_IWRAPPER_2(max_epi32); // return (a > b) ? a : b (int32)
103SIMD_IWRAPPER_2(max_epu32); // return (a > b) ? a : b (uint32)
104SIMD_IWRAPPER_2(min_epi32); // return (a < b) ? a : b (int32)
105SIMD_IWRAPPER_2(min_epu32); // return (a < b) ? a : b (uint32)
106SIMD_IWRAPPER_2(mul_epi32); // return a * b (int32)
107
108// return (a * b) & 0xFFFFFFFF
109//
110// Multiply the packed 32-bit integers in a and b, producing intermediate 64-bit integers,
111// and store the low 32 bits of the intermediate integers in dst.
112SIMD_IWRAPPER_2(mullo_epi32);
113SIMD_IWRAPPER_2(sub_epi32); // return a - b (int32)
114SIMD_IWRAPPER_2(sub_epi64); // return a - b (int64)
115SIMD_IWRAPPER_2(subs_epu8); // return (b > a) ? 0 : (a - b) (uint8)
116
117//-----------------------------------------------------------------------
118// Logical operations
119//-----------------------------------------------------------------------
120#if _MSC_VER >= 1920 // && _MSC_FULL_VER < [some_fixed_version]
121// Some versions of MSVC 2019 don't handle constant folding of and_si() correctly.
122// Using and_ps instead inhibits the compiler's constant folding and actually issues
123// the and intrinsic even though both inputs are constant values.
124#else
125// Use native integer and intrinsic
126SIMD_IWRAPPER_2_(and_si, and_si256); // return a & b       (int)
127#endif
128SIMD_IWRAPPER_2_(andnot_si, andnot_si256); // return (~a) & b    (int)
129SIMD_IWRAPPER_2_(or_si, or_si256);         // return a | b       (int)
130SIMD_IWRAPPER_2_(xor_si, xor_si256);       // return a ^ b       (int)
131
132//-----------------------------------------------------------------------
133// Shift operations
134//-----------------------------------------------------------------------
135SIMD_IWRAPPER_1I(slli_epi32);           // return a << ImmT
136SIMD_IWRAPPER_2(sllv_epi32);            // return a << b      (uint32)
137SIMD_IWRAPPER_1I(srai_epi32);           // return a >> ImmT   (int32)
138SIMD_IWRAPPER_1I(srli_epi32);           // return a >> ImmT   (uint32)
139SIMD_IWRAPPER_2(srlv_epi32);            // return a >> b      (uint32)
140SIMD_IWRAPPER_1I_(srli_si, srli_si256); // return a >> (ImmT*8) (uint)
141
142template <int ImmT> // same as srli_si, but with Float cast to int
143static SIMDINLINE Float SIMDCALL srlisi_ps(Float const& a)
144{
145    return castsi_ps(srli_si<ImmT>(castps_si(a)));
146}
147
148//-----------------------------------------------------------------------
149// Conversion operations
150//-----------------------------------------------------------------------
151SIMD_IWRAPPER_1L(cvtepu8_epi16);  // return (int16)a    (uint8 --> int16)
152SIMD_IWRAPPER_1L(cvtepu8_epi32);  // return (int32)a    (uint8 --> int32)
153SIMD_IWRAPPER_1L(cvtepu16_epi32); // return (int32)a    (uint16 --> int32)
154SIMD_IWRAPPER_1L(cvtepu16_epi64); // return (int64)a    (uint16 --> int64)
155SIMD_IWRAPPER_1L(cvtepu32_epi64); // return (int64)a    (uint32 --> int64)
156
157//-----------------------------------------------------------------------
158// Comparison operations
159//-----------------------------------------------------------------------
160SIMD_IWRAPPER_2(cmpeq_epi8);  // return a == b (int8)
161SIMD_IWRAPPER_2(cmpeq_epi16); // return a == b (int16)
162SIMD_IWRAPPER_2(cmpeq_epi32); // return a == b (int32)
163SIMD_IWRAPPER_2(cmpeq_epi64); // return a == b (int64)
164SIMD_IWRAPPER_2(cmpgt_epi8);  // return a > b (int8)
165SIMD_IWRAPPER_2(cmpgt_epi16); // return a > b (int16)
166SIMD_IWRAPPER_2(cmpgt_epi32); // return a > b (int32)
167SIMD_IWRAPPER_2(cmpgt_epi64); // return a > b (int64)
168
169static SIMDINLINE Integer SIMDCALL cmplt_epi32(Integer const& a,
170                                               Integer const& b) // return a < b (int32)
171{
172    return cmpgt_epi32(b, a);
173}
174
175//-----------------------------------------------------------------------
176// Blend / shuffle / permute operations
177//-----------------------------------------------------------------------
178SIMD_IWRAPPER_2I(blend_epi32); // return ImmT ? b : a  (int32)
179SIMD_IWRAPPER_2(packs_epi16);  // See documentation for _mm256_packs_epi16 and _mm512_packs_epi16
180SIMD_IWRAPPER_2(packs_epi32);  // See documentation for _mm256_packs_epi32 and _mm512_packs_epi32
181SIMD_IWRAPPER_2(packus_epi16); // See documentation for _mm256_packus_epi16 and _mm512_packus_epi16
182SIMD_IWRAPPER_2(packus_epi32); // See documentation for _mm256_packus_epi32 and _mm512_packus_epi32
183
184template <int ImmT>
185static SIMDINLINE Float SIMDCALL permute_ps(Float const& a)
186{
187    return _mm256_permute_ps(a, ImmT);
188}
189
190SIMD_IWRAPPER_2_(permute_epi32, permutevar8x32_epi32);
191
192static SIMDINLINE Float SIMDCALL
193                        permute_ps(Float const& a, Integer const& swiz) // return a[swiz[i]] for each 32-bit lane i (float)
194{
195    return _mm256_permutevar8x32_ps(a, swiz);
196}
197
198SIMD_IWRAPPER_1I(shuffle_epi32);
199template <int ImmT>
200static SIMDINLINE Integer SIMDCALL shuffle_epi64(Integer const& a, Integer const& b)
201{
202    return castpd_si(shuffle_pd<ImmT>(castsi_pd(a), castsi_pd(b)));
203}
204SIMD_IWRAPPER_2(shuffle_epi8);
205SIMD_IWRAPPER_2(unpackhi_epi16);
206SIMD_IWRAPPER_2(unpackhi_epi32);
207SIMD_IWRAPPER_2(unpackhi_epi64);
208SIMD_IWRAPPER_2(unpackhi_epi8);
209SIMD_IWRAPPER_2(unpacklo_epi16);
210SIMD_IWRAPPER_2(unpacklo_epi32);
211SIMD_IWRAPPER_2(unpacklo_epi64);
212SIMD_IWRAPPER_2(unpacklo_epi8);
213
214//-----------------------------------------------------------------------
215// Load / store operations
216//-----------------------------------------------------------------------
217template <ScaleFactor ScaleT = ScaleFactor::SF_1>
218static SIMDINLINE Float SIMDCALL
219                        i32gather_ps(float const* p, Integer const& idx) // return *(float*)(((int8*)p) + (idx * ScaleT))
220{
221    return _mm256_i32gather_ps(p, idx, static_cast<int>(ScaleT));
222}
223
224#if _MSC_VER == 1920 // && _MSC_FULL_VER < [some_fixed_version]
225// Don't use _mm256_mask_i32gather_ps(), the compiler doesn't preserve the mask register
226// correctly in early versions of MSVC 2019
227#else
228// for each element: (mask & (1 << 31)) ? (i32gather_ps<ScaleT>(p, idx), mask = 0) : old
229template <ScaleFactor ScaleT = ScaleFactor::SF_1>
230static SIMDINLINE Float SIMDCALL
231                        mask_i32gather_ps(Float const& old, float const* p, Integer const& idx, Float const& mask)
232{
233    // g++ in debug mode needs the explicit .v suffix instead of relying on operator __m256()
234    // Only for this intrinsic - not sure why. :(
235    return _mm256_mask_i32gather_ps(old.v, p, idx.v, mask.v, static_cast<int>(ScaleT));
236}
237#endif
238
239static SIMDINLINE uint32_t SIMDCALL movemask_epi8(Integer const& a)
240{
241    return static_cast<uint32_t>(_mm256_movemask_epi8(a));
242}
243
244//=======================================================================
245// Legacy interface (available only in SIMD256 width)
246//=======================================================================
247
248#undef SIMD_IWRAPPER_1
249#undef SIMD_IWRAPPER_1L
250#undef SIMD_IWRAPPER_1I
251#undef SIMD_IWRAPPER_1I_
252#undef SIMD_IWRAPPER_2_
253#undef SIMD_IWRAPPER_2
254#undef SIMD_IWRAPPER_2I
255#undef SIMD_IWRAPPER_2I
256