1/**************************************************************************** 2 * Copyright (C) 2017 Intel Corporation. All Rights Reserved. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 ****************************************************************************/ 23#if !defined(__SIMD_LIB_AVX512_HPP__) 24#error Do not include this file directly, use "simdlib.hpp" instead. 25#endif 26 27//============================================================================ 28// SIMD16 AVX512 (F) implementation for Core processors 29// 30//============================================================================ 31 32#define SIMD_WRAPPER_1_(op, intrin) \ 33 static SIMDINLINE Float SIMDCALL op(Float a) { return intrin(a); } 34 35#define SIMD_WRAPPER_1(op) SIMD_WRAPPER_1_(op, _mm512_##op) 36 37#define SIMD_WRAPPER_2_(op, intrin) \ 38 static SIMDINLINE Float SIMDCALL op(Float a, Float b) { return _mm512_##intrin(a, b); } 39#define SIMD_WRAPPER_2(op) SIMD_WRAPPER_2_(op, op) 40 41#define SIMD_WRAPPERI_2_(op, intrin) \ 42 static SIMDINLINE Float SIMDCALL op(Float a, Float b) \ 43 { \ 44 return _mm512_castsi512_ps( \ 45 _mm512_##intrin(_mm512_castps_si512(a), _mm512_castps_si512(b))); \ 46 } 47 48#define SIMD_DWRAPPER_2(op) \ 49 static SIMDINLINE Double SIMDCALL op(Double a, Double b) { return _mm512_##op(a, b); } 50 51#define SIMD_WRAPPER_2I_(op, intrin) \ 52 template <int ImmT> \ 53 static SIMDINLINE Float SIMDCALL op(Float a, Float b) \ 54 { \ 55 return _mm512_##intrin(a, b, ImmT); \ 56 } 57#define SIMD_WRAPPER_2I(op) SIMD_WRAPPER_2I_(op, op) 58 59#define SIMD_DWRAPPER_2I_(op, intrin) \ 60 template <int ImmT> \ 61 static SIMDINLINE Double SIMDCALL op(Double a, Double b) \ 62 { \ 63 return _mm512_##intrin(a, b, ImmT); \ 64 } 65#define SIMD_DWRAPPER_2I(op) SIMD_DWRAPPER_2I_(op, op) 66 67#define SIMD_WRAPPER_3(op) \ 68 static SIMDINLINE Float SIMDCALL op(Float a, Float b, Float c) { return _mm512_##op(a, b, c); } 69 70#define SIMD_IWRAPPER_1(op) \ 71 static SIMDINLINE Integer SIMDCALL op(Integer a) { return _mm512_##op(a); } 72#define SIMD_IWRAPPER_1_8(op) \ 73 static SIMDINLINE Integer SIMDCALL op(SIMD256Impl::Integer a) { return _mm512_##op(a); } 74 75#define SIMD_IWRAPPER_1_4(op) \ 76 static SIMDINLINE Integer SIMDCALL op(SIMD128Impl::Integer a) { return _mm512_##op(a); } 77 78#define SIMD_IWRAPPER_1I_(op, intrin) \ 79 template <int ImmT> \ 80 static SIMDINLINE Integer SIMDCALL op(Integer a) \ 81 { \ 82 return intrin(a, ImmT); \ 83 } 84#define SIMD_IWRAPPER_1I(op) SIMD_IWRAPPER_1I_(op, _mm512_##op) 85 86#define SIMD_IWRAPPER_2_(op, intrin) \ 87 static SIMDINLINE Integer SIMDCALL op(Integer a, Integer b) { return _mm512_##intrin(a, b); } 88#define SIMD_IWRAPPER_2(op) SIMD_IWRAPPER_2_(op, op) 89 90#define SIMD_IWRAPPER_2_CMP(op, cmp) \ 91 static SIMDINLINE Integer SIMDCALL op(Integer a, Integer b) { return cmp(a, b); } 92 93#define SIMD_IFWRAPPER_2(op, intrin) \ 94 static SIMDINLINE Integer SIMDCALL op(Integer a, Integer b) \ 95 { \ 96 return castps_si(_mm512_##intrin(castsi_ps(a), castsi_ps(b))); \ 97 } 98 99#define SIMD_IWRAPPER_2I_(op, intrin) \ 100 template <int ImmT> \ 101 static SIMDINLINE Integer SIMDCALL op(Integer a, Integer b) \ 102 { \ 103 return _mm512_##intrin(a, b, ImmT); \ 104 } 105#define SIMD_IWRAPPER_2I(op) SIMD_IWRAPPER_2I_(op, op) 106 107private: 108static SIMDINLINE Integer vmask(__mmask32 m) 109{ 110 return _mm512_maskz_set1_epi16(m, -1); 111} 112static SIMDINLINE Integer vmask(__mmask64 m) 113{ 114 return _mm512_maskz_set1_epi8(m, -1); 115} 116 117public: 118SIMD_IWRAPPER_2(add_epi8); // return a + b (int8) 119SIMD_IWRAPPER_2(adds_epu8); // return ((a + b) > 0xff) ? 0xff : (a + b) (uint8) 120SIMD_IWRAPPER_2(subs_epu8); // return (b > a) ? 0 : (a - b) (uint8) 121 122SIMD_WRAPPER_2(and_ps); // return a & b (float treated as int) 123SIMD_WRAPPER_2(andnot_ps); // return (~a) & b (float treated as int) 124SIMD_WRAPPER_2(or_ps); // return a | b (float treated as int) 125SIMD_WRAPPER_2(xor_ps); // return a ^ b (float treated as int) 126 127SIMD_IWRAPPER_1_8(cvtepu8_epi16); // return (int16)a (uint8 --> int16) 128 129template <CompareTypeInt CmpTypeT> 130static SIMDINLINE Integer SIMDCALL cmp_epi8(Integer a, Integer b) 131{ 132 // Legacy vector mask generator 133 __mmask64 result = _mm512_cmp_epi8_mask(a, b, static_cast<const int>(CmpTypeT)); 134 return vmask(result); 135} 136template <CompareTypeInt CmpTypeT> 137static SIMDINLINE Integer SIMDCALL cmp_epi16(Integer a, Integer b) 138{ 139 // Legacy vector mask generator 140 __mmask32 result = _mm512_cmp_epi16_mask(a, b, static_cast<const int>(CmpTypeT)); 141 return vmask(result); 142} 143 144SIMD_IWRAPPER_2_CMP(cmpeq_epi8, cmp_epi8<CompareTypeInt::EQ>); // return a == b (int8) 145SIMD_IWRAPPER_2_CMP(cmpeq_epi16, cmp_epi16<CompareTypeInt::EQ>); // return a == b (int16) 146SIMD_IWRAPPER_2_CMP(cmpgt_epi8, cmp_epi8<CompareTypeInt::GT>); // return a > b (int8) 147SIMD_IWRAPPER_2_CMP(cmpgt_epi16, cmp_epi16<CompareTypeInt::GT>); // return a > b (int16) 148 149SIMD_IWRAPPER_2(packs_epi16); // See documentation for _mm512_packs_epi16 150SIMD_IWRAPPER_2(packs_epi32); // See documentation for _mm512_packs_epi32 151SIMD_IWRAPPER_2(packus_epi16); // See documentation for _mm512_packus_epi16 152SIMD_IWRAPPER_2(packus_epi32); // See documentation for _mm512_packus_epi32 153 154SIMD_IWRAPPER_2(unpackhi_epi8); // See documentation for _mm512_unpackhi_epi8 155SIMD_IWRAPPER_2(unpacklo_epi16); // See documentation for _mm512_unpacklo_epi16 156SIMD_IWRAPPER_2(unpacklo_epi8); // See documentation for _mm512_unpacklo_epi8 157 158SIMD_IWRAPPER_2(shuffle_epi8); 159 160static SIMDINLINE uint64_t SIMDCALL movemask_epi8(Integer a) 161{ 162 __mmask64 m = _mm512_cmplt_epi8_mask(a, setzero_si()); 163 return static_cast<uint64_t>(m); 164} 165 166#undef SIMD_WRAPPER_1_ 167#undef SIMD_WRAPPER_1 168#undef SIMD_WRAPPER_2 169#undef SIMD_WRAPPER_2_ 170#undef SIMD_WRAPPERI_2_ 171#undef SIMD_DWRAPPER_2 172#undef SIMD_DWRAPPER_2I 173#undef SIMD_WRAPPER_2I_ 174#undef SIMD_WRAPPER_3_ 175#undef SIMD_WRAPPER_2I 176#undef SIMD_WRAPPER_3 177#undef SIMD_IWRAPPER_1 178#undef SIMD_IWRAPPER_2 179#undef SIMD_IFWRAPPER_2 180#undef SIMD_IWRAPPER_2I 181#undef SIMD_IWRAPPER_1 182#undef SIMD_IWRAPPER_1I 183#undef SIMD_IWRAPPER_1I_ 184#undef SIMD_IWRAPPER_2 185#undef SIMD_IWRAPPER_2_ 186#undef SIMD_IWRAPPER_2I 187