1 // This file is part of Eigen, a lightweight C++ template library 2 // for linear algebra. 3 // 4 // Copyright (C) 2016 Benoit Steiner <benoit.steiner.goog@gmail.com> 5 // 6 // This Source Code Form is subject to the terms of the Mozilla 7 // Public License v. 2.0. If a copy of the MPL was not distributed 8 // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 10 #ifndef EIGEN_PACKET_MATH_HALF_CUDA_H 11 #define EIGEN_PACKET_MATH_HALF_CUDA_H 12 13 14 namespace Eigen { 15 namespace internal { 16 17 // Most of the following operations require arch >= 3.0 18 #if defined(EIGEN_HAS_CUDA_FP16) && defined(__CUDACC__) && defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 300 19 20 template<> struct is_arithmetic<half2> { enum { value = true }; }; 21 22 template<> struct packet_traits<Eigen::half> : default_packet_traits 23 { 24 typedef half2 type; 25 typedef half2 half; 26 enum { 27 Vectorizable = 1, 28 AlignedOnScalar = 1, 29 size=2, 30 HasHalfPacket = 0, 31 HasAdd = 1, 32 HasMul = 1, 33 HasDiv = 1, 34 HasSqrt = 1, 35 HasRsqrt = 1, 36 HasExp = 1, 37 HasLog = 1, 38 HasLog1p = 1 39 }; 40 }; 41 42 template<> struct unpacket_traits<half2> { typedef Eigen::half type; enum {size=2, alignment=Aligned16}; typedef half2 half; }; 43 44 template<> __device__ EIGEN_STRONG_INLINE half2 pset1<half2>(const Eigen::half& from) { 45 return __half2half2(from); 46 } 47 48 template<> __device__ EIGEN_STRONG_INLINE half2 pload<half2>(const Eigen::half* from) { 49 return *reinterpret_cast<const half2*>(from); 50 } 51 52 template<> __device__ EIGEN_STRONG_INLINE half2 ploadu<half2>(const Eigen::half* from) { 53 return __halves2half2(from[0], from[1]); 54 } 55 56 template<> EIGEN_STRONG_INLINE half2 ploaddup<half2>(const Eigen::half* from) { 57 return __halves2half2(from[0], from[0]); 58 } 59 60 template<> __device__ EIGEN_STRONG_INLINE void pstore<Eigen::half>(Eigen::half* to, const half2& from) { 61 *reinterpret_cast<half2*>(to) = from; 62 } 63 64 template<> __device__ EIGEN_STRONG_INLINE void pstoreu<Eigen::half>(Eigen::half* to, const half2& from) { 65 to[0] = __low2half(from); 66 to[1] = __high2half(from); 67 } 68 69 template<> 70 __device__ EIGEN_ALWAYS_INLINE half2 ploadt_ro<half2, Aligned>(const Eigen::half* from) { 71 #if __CUDA_ARCH__ >= 350 72 return __ldg((const half2*)from); 73 #else 74 return __halves2half2(*(from+0), *(from+1)); 75 #endif 76 } 77 78 template<> 79 __device__ EIGEN_ALWAYS_INLINE half2 ploadt_ro<half2, Unaligned>(const Eigen::half* from) { 80 #if __CUDA_ARCH__ >= 350 81 return __halves2half2(__ldg(from+0), __ldg(from+1)); 82 #else 83 return __halves2half2(*(from+0), *(from+1)); 84 #endif 85 } 86 87 template<> __device__ EIGEN_STRONG_INLINE half2 pgather<Eigen::half, half2>(const Eigen::half* from, Index stride) { 88 return __halves2half2(from[0*stride], from[1*stride]); 89 } 90 91 template<> __device__ EIGEN_STRONG_INLINE void pscatter<Eigen::half, half2>(Eigen::half* to, const half2& from, Index stride) { 92 to[stride*0] = __low2half(from); 93 to[stride*1] = __high2half(from); 94 } 95 96 template<> __device__ EIGEN_STRONG_INLINE Eigen::half pfirst<half2>(const half2& a) { 97 return __low2half(a); 98 } 99 100 template<> __device__ EIGEN_STRONG_INLINE half2 pabs<half2>(const half2& a) { 101 half2 result; 102 result.x = a.x & 0x7FFF7FFF; 103 return result; 104 } 105 106 107 __device__ EIGEN_STRONG_INLINE void 108 ptranspose(PacketBlock<half2,2>& kernel) { 109 __half a1 = __low2half(kernel.packet[0]); 110 __half a2 = __high2half(kernel.packet[0]); 111 __half b1 = __low2half(kernel.packet[1]); 112 __half b2 = __high2half(kernel.packet[1]); 113 kernel.packet[0] = __halves2half2(a1, b1); 114 kernel.packet[1] = __halves2half2(a2, b2); 115 } 116 117 template<> __device__ EIGEN_STRONG_INLINE half2 plset<half2>(const Eigen::half& a) { 118 #if __CUDA_ARCH__ >= 530 119 return __halves2half2(a, __hadd(a, __float2half(1.0f))); 120 #else 121 float f = __half2float(a) + 1.0f; 122 return __halves2half2(a, __float2half(f)); 123 #endif 124 } 125 126 template<> __device__ EIGEN_STRONG_INLINE half2 padd<half2>(const half2& a, const half2& b) { 127 #if __CUDA_ARCH__ >= 530 128 return __hadd2(a, b); 129 #else 130 float a1 = __low2float(a); 131 float a2 = __high2float(a); 132 float b1 = __low2float(b); 133 float b2 = __high2float(b); 134 float r1 = a1 + b1; 135 float r2 = a2 + b2; 136 return __floats2half2_rn(r1, r2); 137 #endif 138 } 139 140 template<> __device__ EIGEN_STRONG_INLINE half2 psub<half2>(const half2& a, const half2& b) { 141 #if __CUDA_ARCH__ >= 530 142 return __hsub2(a, b); 143 #else 144 float a1 = __low2float(a); 145 float a2 = __high2float(a); 146 float b1 = __low2float(b); 147 float b2 = __high2float(b); 148 float r1 = a1 - b1; 149 float r2 = a2 - b2; 150 return __floats2half2_rn(r1, r2); 151 #endif 152 } 153 154 template<> __device__ EIGEN_STRONG_INLINE half2 pnegate(const half2& a) { 155 #if __CUDA_ARCH__ >= 530 156 return __hneg2(a); 157 #else 158 float a1 = __low2float(a); 159 float a2 = __high2float(a); 160 return __floats2half2_rn(-a1, -a2); 161 #endif 162 } 163 164 template<> __device__ EIGEN_STRONG_INLINE half2 pconj(const half2& a) { return a; } 165 166 template<> __device__ EIGEN_STRONG_INLINE half2 pmul<half2>(const half2& a, const half2& b) { 167 #if __CUDA_ARCH__ >= 530 168 return __hmul2(a, b); 169 #else 170 float a1 = __low2float(a); 171 float a2 = __high2float(a); 172 float b1 = __low2float(b); 173 float b2 = __high2float(b); 174 float r1 = a1 * b1; 175 float r2 = a2 * b2; 176 return __floats2half2_rn(r1, r2); 177 #endif 178 } 179 180 template<> __device__ EIGEN_STRONG_INLINE half2 pmadd<half2>(const half2& a, const half2& b, const half2& c) { 181 #if __CUDA_ARCH__ >= 530 182 return __hfma2(a, b, c); 183 #else 184 float a1 = __low2float(a); 185 float a2 = __high2float(a); 186 float b1 = __low2float(b); 187 float b2 = __high2float(b); 188 float c1 = __low2float(c); 189 float c2 = __high2float(c); 190 float r1 = a1 * b1 + c1; 191 float r2 = a2 * b2 + c2; 192 return __floats2half2_rn(r1, r2); 193 #endif 194 } 195 196 template<> __device__ EIGEN_STRONG_INLINE half2 pdiv<half2>(const half2& a, const half2& b) { 197 float a1 = __low2float(a); 198 float a2 = __high2float(a); 199 float b1 = __low2float(b); 200 float b2 = __high2float(b); 201 float r1 = a1 / b1; 202 float r2 = a2 / b2; 203 return __floats2half2_rn(r1, r2); 204 } 205 206 template<> __device__ EIGEN_STRONG_INLINE half2 pmin<half2>(const half2& a, const half2& b) { 207 float a1 = __low2float(a); 208 float a2 = __high2float(a); 209 float b1 = __low2float(b); 210 float b2 = __high2float(b); 211 __half r1 = a1 < b1 ? __low2half(a) : __low2half(b); 212 __half r2 = a2 < b2 ? __high2half(a) : __high2half(b); 213 return __halves2half2(r1, r2); 214 } 215 216 template<> __device__ EIGEN_STRONG_INLINE half2 pmax<half2>(const half2& a, const half2& b) { 217 float a1 = __low2float(a); 218 float a2 = __high2float(a); 219 float b1 = __low2float(b); 220 float b2 = __high2float(b); 221 __half r1 = a1 > b1 ? __low2half(a) : __low2half(b); 222 __half r2 = a2 > b2 ? __high2half(a) : __high2half(b); 223 return __halves2half2(r1, r2); 224 } 225 226 template<> __device__ EIGEN_STRONG_INLINE Eigen::half predux<half2>(const half2& a) { 227 #if __CUDA_ARCH__ >= 530 228 return __hadd(__low2half(a), __high2half(a)); 229 #else 230 float a1 = __low2float(a); 231 float a2 = __high2float(a); 232 return Eigen::half(half_impl::raw_uint16_to_half(__float2half_rn(a1 + a2))); 233 #endif 234 } 235 236 template<> __device__ EIGEN_STRONG_INLINE Eigen::half predux_max<half2>(const half2& a) { 237 #if __CUDA_ARCH__ >= 530 238 __half first = __low2half(a); 239 __half second = __high2half(a); 240 return __hgt(first, second) ? first : second; 241 #else 242 float a1 = __low2float(a); 243 float a2 = __high2float(a); 244 return a1 > a2 ? __low2half(a) : __high2half(a); 245 #endif 246 } 247 248 template<> __device__ EIGEN_STRONG_INLINE Eigen::half predux_min<half2>(const half2& a) { 249 #if __CUDA_ARCH__ >= 530 250 __half first = __low2half(a); 251 __half second = __high2half(a); 252 return __hlt(first, second) ? first : second; 253 #else 254 float a1 = __low2float(a); 255 float a2 = __high2float(a); 256 return a1 < a2 ? __low2half(a) : __high2half(a); 257 #endif 258 } 259 260 template<> __device__ EIGEN_STRONG_INLINE Eigen::half predux_mul<half2>(const half2& a) { 261 #if __CUDA_ARCH__ >= 530 262 return __hmul(__low2half(a), __high2half(a)); 263 #else 264 float a1 = __low2float(a); 265 float a2 = __high2float(a); 266 return Eigen::half(half_impl::raw_uint16_to_half(__float2half_rn(a1 * a2))); 267 #endif 268 } 269 270 template<> __device__ EIGEN_STRONG_INLINE half2 plog1p<half2>(const half2& a) { 271 float a1 = __low2float(a); 272 float a2 = __high2float(a); 273 float r1 = log1pf(a1); 274 float r2 = log1pf(a2); 275 return __floats2half2_rn(r1, r2); 276 } 277 278 #if defined __CUDACC_VER__ && __CUDACC_VER__ >= 80000 && defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 530 279 280 template<> __device__ EIGEN_STRONG_INLINE 281 half2 plog<half2>(const half2& a) { 282 return h2log(a); 283 } 284 285 template<> __device__ EIGEN_STRONG_INLINE 286 half2 pexp<half2>(const half2& a) { 287 return h2exp(a); 288 } 289 290 template<> __device__ EIGEN_STRONG_INLINE 291 half2 psqrt<half2>(const half2& a) { 292 return h2sqrt(a); 293 } 294 295 template<> __device__ EIGEN_STRONG_INLINE 296 half2 prsqrt<half2>(const half2& a) { 297 return h2rsqrt(a); 298 } 299 300 #else 301 302 template<> __device__ EIGEN_STRONG_INLINE half2 plog<half2>(const half2& a) { 303 float a1 = __low2float(a); 304 float a2 = __high2float(a); 305 float r1 = logf(a1); 306 float r2 = logf(a2); 307 return __floats2half2_rn(r1, r2); 308 } 309 310 template<> __device__ EIGEN_STRONG_INLINE half2 pexp<half2>(const half2& a) { 311 float a1 = __low2float(a); 312 float a2 = __high2float(a); 313 float r1 = expf(a1); 314 float r2 = expf(a2); 315 return __floats2half2_rn(r1, r2); 316 } 317 318 template<> __device__ EIGEN_STRONG_INLINE half2 psqrt<half2>(const half2& a) { 319 float a1 = __low2float(a); 320 float a2 = __high2float(a); 321 float r1 = sqrtf(a1); 322 float r2 = sqrtf(a2); 323 return __floats2half2_rn(r1, r2); 324 } 325 326 template<> __device__ EIGEN_STRONG_INLINE half2 prsqrt<half2>(const half2& a) { 327 float a1 = __low2float(a); 328 float a2 = __high2float(a); 329 float r1 = rsqrtf(a1); 330 float r2 = rsqrtf(a2); 331 return __floats2half2_rn(r1, r2); 332 } 333 334 #endif 335 336 #elif defined EIGEN_VECTORIZE_AVX512 337 338 typedef struct { 339 __m256i x; 340 } Packet16h; 341 342 343 template<> struct is_arithmetic<Packet16h> { enum { value = true }; }; 344 345 template <> 346 struct packet_traits<half> : default_packet_traits { 347 typedef Packet16h type; 348 // There is no half-size packet for Packet16h. 349 typedef Packet16h half; 350 enum { 351 Vectorizable = 1, 352 AlignedOnScalar = 1, 353 size = 16, 354 HasHalfPacket = 0, 355 HasAdd = 0, 356 HasSub = 0, 357 HasMul = 0, 358 HasNegate = 0, 359 HasAbs = 0, 360 HasAbs2 = 0, 361 HasMin = 0, 362 HasMax = 0, 363 HasConj = 0, 364 HasSetLinear = 0, 365 HasDiv = 0, 366 HasSqrt = 0, 367 HasRsqrt = 0, 368 HasExp = 0, 369 HasLog = 0, 370 HasBlend = 0 371 }; 372 }; 373 374 375 template<> struct unpacket_traits<Packet16h> { typedef Eigen::half type; enum {size=16, alignment=Aligned32}; typedef Packet16h half; }; 376 377 template<> EIGEN_STRONG_INLINE Packet16h pset1<Packet16h>(const Eigen::half& from) { 378 Packet16h result; 379 result.x = _mm256_set1_epi16(from.x); 380 return result; 381 } 382 383 template<> EIGEN_STRONG_INLINE Eigen::half pfirst<Packet16h>(const Packet16h& from) { 384 return half_impl::raw_uint16_to_half(static_cast<unsigned short>(_mm256_extract_epi16(from.x, 0))); 385 } 386 387 template<> EIGEN_STRONG_INLINE Packet16h pload<Packet16h>(const Eigen::half* from) { 388 Packet16h result; 389 result.x = _mm256_load_si256(reinterpret_cast<const __m256i*>(from)); 390 return result; 391 } 392 393 template<> EIGEN_STRONG_INLINE Packet16h ploadu<Packet16h>(const Eigen::half* from) { 394 Packet16h result; 395 result.x = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(from)); 396 return result; 397 } 398 399 template<> EIGEN_STRONG_INLINE void pstore<half>(Eigen::half* to, const Packet16h& from) { 400 _mm256_store_si256((__m256i*)to, from.x); 401 } 402 403 template<> EIGEN_STRONG_INLINE void pstoreu<half>(Eigen::half* to, const Packet16h& from) { 404 _mm256_storeu_si256((__m256i*)to, from.x); 405 } 406 407 template<> EIGEN_STRONG_INLINE Packet16h 408 ploadquad(const Eigen::half* from) { 409 Packet16h result; 410 unsigned short a = from[0].x; 411 unsigned short b = from[1].x; 412 unsigned short c = from[2].x; 413 unsigned short d = from[3].x; 414 result.x = _mm256_set_epi16(d, d, d, d, c, c, c, c, b, b, b, b, a, a, a, a); 415 return result; 416 } 417 418 EIGEN_STRONG_INLINE Packet16f half2float(const Packet16h& a) { 419 #ifdef EIGEN_HAS_FP16_C 420 return _mm512_cvtph_ps(a.x); 421 #else 422 EIGEN_ALIGN64 half aux[16]; 423 pstore(aux, a); 424 float f0(aux[0]); 425 float f1(aux[1]); 426 float f2(aux[2]); 427 float f3(aux[3]); 428 float f4(aux[4]); 429 float f5(aux[5]); 430 float f6(aux[6]); 431 float f7(aux[7]); 432 float f8(aux[8]); 433 float f9(aux[9]); 434 float fa(aux[10]); 435 float fb(aux[11]); 436 float fc(aux[12]); 437 float fd(aux[13]); 438 float fe(aux[14]); 439 float ff(aux[15]); 440 441 return _mm512_set_ps( 442 ff, fe, fd, fc, fb, fa, f9, f8, f7, f6, f5, f4, f3, f2, f1, f0); 443 #endif 444 } 445 446 EIGEN_STRONG_INLINE Packet16h float2half(const Packet16f& a) { 447 #ifdef EIGEN_HAS_FP16_C 448 Packet16h result; 449 result.x = _mm512_cvtps_ph(a, _MM_FROUND_TO_NEAREST_INT|_MM_FROUND_NO_EXC); 450 return result; 451 #else 452 EIGEN_ALIGN64 float aux[16]; 453 pstore(aux, a); 454 half h0(aux[0]); 455 half h1(aux[1]); 456 half h2(aux[2]); 457 half h3(aux[3]); 458 half h4(aux[4]); 459 half h5(aux[5]); 460 half h6(aux[6]); 461 half h7(aux[7]); 462 half h8(aux[8]); 463 half h9(aux[9]); 464 half ha(aux[10]); 465 half hb(aux[11]); 466 half hc(aux[12]); 467 half hd(aux[13]); 468 half he(aux[14]); 469 half hf(aux[15]); 470 471 Packet16h result; 472 result.x = _mm256_set_epi16( 473 hf.x, he.x, hd.x, hc.x, hb.x, ha.x, h9.x, h8.x, 474 h7.x, h6.x, h5.x, h4.x, h3.x, h2.x, h1.x, h0.x); 475 return result; 476 #endif 477 } 478 479 template<> EIGEN_STRONG_INLINE Packet16h padd<Packet16h>(const Packet16h& a, const Packet16h& b) { 480 Packet16f af = half2float(a); 481 Packet16f bf = half2float(b); 482 Packet16f rf = padd(af, bf); 483 return float2half(rf); 484 } 485 486 template<> EIGEN_STRONG_INLINE Packet16h pmul<Packet16h>(const Packet16h& a, const Packet16h& b) { 487 Packet16f af = half2float(a); 488 Packet16f bf = half2float(b); 489 Packet16f rf = pmul(af, bf); 490 return float2half(rf); 491 } 492 493 template<> EIGEN_STRONG_INLINE half predux<Packet16h>(const Packet16h& from) { 494 Packet16f from_float = half2float(from); 495 return half(predux(from_float)); 496 } 497 498 template<> EIGEN_STRONG_INLINE Packet16h pgather<Eigen::half, Packet16h>(const Eigen::half* from, Index stride) 499 { 500 Packet16h result; 501 result.x = _mm256_set_epi16( 502 from[15*stride].x, from[14*stride].x, from[13*stride].x, from[12*stride].x, 503 from[11*stride].x, from[10*stride].x, from[9*stride].x, from[8*stride].x, 504 from[7*stride].x, from[6*stride].x, from[5*stride].x, from[4*stride].x, 505 from[3*stride].x, from[2*stride].x, from[1*stride].x, from[0*stride].x); 506 return result; 507 } 508 509 template<> EIGEN_STRONG_INLINE void pscatter<half, Packet16h>(half* to, const Packet16h& from, Index stride) 510 { 511 EIGEN_ALIGN64 half aux[16]; 512 pstore(aux, from); 513 to[stride*0].x = aux[0].x; 514 to[stride*1].x = aux[1].x; 515 to[stride*2].x = aux[2].x; 516 to[stride*3].x = aux[3].x; 517 to[stride*4].x = aux[4].x; 518 to[stride*5].x = aux[5].x; 519 to[stride*6].x = aux[6].x; 520 to[stride*7].x = aux[7].x; 521 to[stride*8].x = aux[8].x; 522 to[stride*9].x = aux[9].x; 523 to[stride*10].x = aux[10].x; 524 to[stride*11].x = aux[11].x; 525 to[stride*12].x = aux[12].x; 526 to[stride*13].x = aux[13].x; 527 to[stride*14].x = aux[14].x; 528 to[stride*15].x = aux[15].x; 529 } 530 531 EIGEN_STRONG_INLINE void 532 ptranspose(PacketBlock<Packet16h,16>& kernel) { 533 __m256i a = kernel.packet[0].x; 534 __m256i b = kernel.packet[1].x; 535 __m256i c = kernel.packet[2].x; 536 __m256i d = kernel.packet[3].x; 537 __m256i e = kernel.packet[4].x; 538 __m256i f = kernel.packet[5].x; 539 __m256i g = kernel.packet[6].x; 540 __m256i h = kernel.packet[7].x; 541 __m256i i = kernel.packet[8].x; 542 __m256i j = kernel.packet[9].x; 543 __m256i k = kernel.packet[10].x; 544 __m256i l = kernel.packet[11].x; 545 __m256i m = kernel.packet[12].x; 546 __m256i n = kernel.packet[13].x; 547 __m256i o = kernel.packet[14].x; 548 __m256i p = kernel.packet[15].x; 549 550 __m256i ab_07 = _mm256_unpacklo_epi16(a, b); 551 __m256i cd_07 = _mm256_unpacklo_epi16(c, d); 552 __m256i ef_07 = _mm256_unpacklo_epi16(e, f); 553 __m256i gh_07 = _mm256_unpacklo_epi16(g, h); 554 __m256i ij_07 = _mm256_unpacklo_epi16(i, j); 555 __m256i kl_07 = _mm256_unpacklo_epi16(k, l); 556 __m256i mn_07 = _mm256_unpacklo_epi16(m, n); 557 __m256i op_07 = _mm256_unpacklo_epi16(o, p); 558 559 __m256i ab_8f = _mm256_unpackhi_epi16(a, b); 560 __m256i cd_8f = _mm256_unpackhi_epi16(c, d); 561 __m256i ef_8f = _mm256_unpackhi_epi16(e, f); 562 __m256i gh_8f = _mm256_unpackhi_epi16(g, h); 563 __m256i ij_8f = _mm256_unpackhi_epi16(i, j); 564 __m256i kl_8f = _mm256_unpackhi_epi16(k, l); 565 __m256i mn_8f = _mm256_unpackhi_epi16(m, n); 566 __m256i op_8f = _mm256_unpackhi_epi16(o, p); 567 568 __m256i abcd_03 = _mm256_unpacklo_epi32(ab_07, cd_07); 569 __m256i abcd_47 = _mm256_unpackhi_epi32(ab_07, cd_07); 570 __m256i efgh_03 = _mm256_unpacklo_epi32(ef_07, gh_07); 571 __m256i efgh_47 = _mm256_unpackhi_epi32(ef_07, gh_07); 572 __m256i ijkl_03 = _mm256_unpacklo_epi32(ij_07, kl_07); 573 __m256i ijkl_47 = _mm256_unpackhi_epi32(ij_07, kl_07); 574 __m256i mnop_03 = _mm256_unpacklo_epi32(mn_07, op_07); 575 __m256i mnop_47 = _mm256_unpackhi_epi32(mn_07, op_07); 576 577 __m256i abcd_8b = _mm256_unpacklo_epi32(ab_8f, cd_8f); 578 __m256i abcd_cf = _mm256_unpackhi_epi32(ab_8f, cd_8f); 579 __m256i efgh_8b = _mm256_unpacklo_epi32(ef_8f, gh_8f); 580 __m256i efgh_cf = _mm256_unpackhi_epi32(ef_8f, gh_8f); 581 __m256i ijkl_8b = _mm256_unpacklo_epi32(ij_8f, kl_8f); 582 __m256i ijkl_cf = _mm256_unpackhi_epi32(ij_8f, kl_8f); 583 __m256i mnop_8b = _mm256_unpacklo_epi32(mn_8f, op_8f); 584 __m256i mnop_cf = _mm256_unpackhi_epi32(mn_8f, op_8f); 585 586 __m256i abcdefgh_01 = _mm256_unpacklo_epi64(abcd_03, efgh_03); 587 __m256i abcdefgh_23 = _mm256_unpackhi_epi64(abcd_03, efgh_03); 588 __m256i ijklmnop_01 = _mm256_unpacklo_epi64(ijkl_03, mnop_03); 589 __m256i ijklmnop_23 = _mm256_unpackhi_epi64(ijkl_03, mnop_03); 590 __m256i abcdefgh_45 = _mm256_unpacklo_epi64(abcd_47, efgh_47); 591 __m256i abcdefgh_67 = _mm256_unpackhi_epi64(abcd_47, efgh_47); 592 __m256i ijklmnop_45 = _mm256_unpacklo_epi64(ijkl_47, mnop_47); 593 __m256i ijklmnop_67 = _mm256_unpackhi_epi64(ijkl_47, mnop_47); 594 __m256i abcdefgh_89 = _mm256_unpacklo_epi64(abcd_8b, efgh_8b); 595 __m256i abcdefgh_ab = _mm256_unpackhi_epi64(abcd_8b, efgh_8b); 596 __m256i ijklmnop_89 = _mm256_unpacklo_epi64(ijkl_8b, mnop_8b); 597 __m256i ijklmnop_ab = _mm256_unpackhi_epi64(ijkl_8b, mnop_8b); 598 __m256i abcdefgh_cd = _mm256_unpacklo_epi64(abcd_cf, efgh_cf); 599 __m256i abcdefgh_ef = _mm256_unpackhi_epi64(abcd_cf, efgh_cf); 600 __m256i ijklmnop_cd = _mm256_unpacklo_epi64(ijkl_cf, mnop_cf); 601 __m256i ijklmnop_ef = _mm256_unpackhi_epi64(ijkl_cf, mnop_cf); 602 603 // NOTE: no unpacklo/hi instr in this case, so using permute instr. 604 __m256i a_p_0 = _mm256_permute2x128_si256(abcdefgh_01, ijklmnop_01, 0x20); 605 __m256i a_p_1 = _mm256_permute2x128_si256(abcdefgh_01, ijklmnop_01, 0x31); 606 __m256i a_p_2 = _mm256_permute2x128_si256(abcdefgh_23, ijklmnop_23, 0x20); 607 __m256i a_p_3 = _mm256_permute2x128_si256(abcdefgh_23, ijklmnop_23, 0x31); 608 __m256i a_p_4 = _mm256_permute2x128_si256(abcdefgh_45, ijklmnop_45, 0x20); 609 __m256i a_p_5 = _mm256_permute2x128_si256(abcdefgh_45, ijklmnop_45, 0x31); 610 __m256i a_p_6 = _mm256_permute2x128_si256(abcdefgh_67, ijklmnop_67, 0x20); 611 __m256i a_p_7 = _mm256_permute2x128_si256(abcdefgh_67, ijklmnop_67, 0x31); 612 __m256i a_p_8 = _mm256_permute2x128_si256(abcdefgh_89, ijklmnop_89, 0x20); 613 __m256i a_p_9 = _mm256_permute2x128_si256(abcdefgh_89, ijklmnop_89, 0x31); 614 __m256i a_p_a = _mm256_permute2x128_si256(abcdefgh_ab, ijklmnop_ab, 0x20); 615 __m256i a_p_b = _mm256_permute2x128_si256(abcdefgh_ab, ijklmnop_ab, 0x31); 616 __m256i a_p_c = _mm256_permute2x128_si256(abcdefgh_cd, ijklmnop_cd, 0x20); 617 __m256i a_p_d = _mm256_permute2x128_si256(abcdefgh_cd, ijklmnop_cd, 0x31); 618 __m256i a_p_e = _mm256_permute2x128_si256(abcdefgh_ef, ijklmnop_ef, 0x20); 619 __m256i a_p_f = _mm256_permute2x128_si256(abcdefgh_ef, ijklmnop_ef, 0x31); 620 621 kernel.packet[0].x = a_p_0; 622 kernel.packet[1].x = a_p_1; 623 kernel.packet[2].x = a_p_2; 624 kernel.packet[3].x = a_p_3; 625 kernel.packet[4].x = a_p_4; 626 kernel.packet[5].x = a_p_5; 627 kernel.packet[6].x = a_p_6; 628 kernel.packet[7].x = a_p_7; 629 kernel.packet[8].x = a_p_8; 630 kernel.packet[9].x = a_p_9; 631 kernel.packet[10].x = a_p_a; 632 kernel.packet[11].x = a_p_b; 633 kernel.packet[12].x = a_p_c; 634 kernel.packet[13].x = a_p_d; 635 kernel.packet[14].x = a_p_e; 636 kernel.packet[15].x = a_p_f; 637 } 638 639 EIGEN_STRONG_INLINE void 640 ptranspose(PacketBlock<Packet16h,8>& kernel) { 641 EIGEN_ALIGN64 half in[8][16]; 642 pstore<half>(in[0], kernel.packet[0]); 643 pstore<half>(in[1], kernel.packet[1]); 644 pstore<half>(in[2], kernel.packet[2]); 645 pstore<half>(in[3], kernel.packet[3]); 646 pstore<half>(in[4], kernel.packet[4]); 647 pstore<half>(in[5], kernel.packet[5]); 648 pstore<half>(in[6], kernel.packet[6]); 649 pstore<half>(in[7], kernel.packet[7]); 650 651 EIGEN_ALIGN64 half out[8][16]; 652 653 for (int i = 0; i < 8; ++i) { 654 for (int j = 0; j < 8; ++j) { 655 out[i][j] = in[j][2*i]; 656 } 657 for (int j = 0; j < 8; ++j) { 658 out[i][j+8] = in[j][2*i+1]; 659 } 660 } 661 662 kernel.packet[0] = pload<Packet16h>(out[0]); 663 kernel.packet[1] = pload<Packet16h>(out[1]); 664 kernel.packet[2] = pload<Packet16h>(out[2]); 665 kernel.packet[3] = pload<Packet16h>(out[3]); 666 kernel.packet[4] = pload<Packet16h>(out[4]); 667 kernel.packet[5] = pload<Packet16h>(out[5]); 668 kernel.packet[6] = pload<Packet16h>(out[6]); 669 kernel.packet[7] = pload<Packet16h>(out[7]); 670 } 671 672 EIGEN_STRONG_INLINE void 673 ptranspose(PacketBlock<Packet16h,4>& kernel) { 674 EIGEN_ALIGN64 half in[4][16]; 675 pstore<half>(in[0], kernel.packet[0]); 676 pstore<half>(in[1], kernel.packet[1]); 677 pstore<half>(in[2], kernel.packet[2]); 678 pstore<half>(in[3], kernel.packet[3]); 679 680 EIGEN_ALIGN64 half out[4][16]; 681 682 for (int i = 0; i < 4; ++i) { 683 for (int j = 0; j < 4; ++j) { 684 out[i][j] = in[j][4*i]; 685 } 686 for (int j = 0; j < 4; ++j) { 687 out[i][j+4] = in[j][4*i+1]; 688 } 689 for (int j = 0; j < 4; ++j) { 690 out[i][j+8] = in[j][4*i+2]; 691 } 692 for (int j = 0; j < 4; ++j) { 693 out[i][j+12] = in[j][4*i+3]; 694 } 695 } 696 697 kernel.packet[0] = pload<Packet16h>(out[0]); 698 kernel.packet[1] = pload<Packet16h>(out[1]); 699 kernel.packet[2] = pload<Packet16h>(out[2]); 700 kernel.packet[3] = pload<Packet16h>(out[3]); 701 } 702 703 704 #elif defined EIGEN_VECTORIZE_AVX 705 706 typedef struct { 707 __m128i x; 708 } Packet8h; 709 710 711 template<> struct is_arithmetic<Packet8h> { enum { value = true }; }; 712 713 template <> 714 struct packet_traits<Eigen::half> : default_packet_traits { 715 typedef Packet8h type; 716 // There is no half-size packet for Packet8h. 717 typedef Packet8h half; 718 enum { 719 Vectorizable = 1, 720 AlignedOnScalar = 1, 721 size = 8, 722 HasHalfPacket = 0, 723 HasAdd = 0, 724 HasSub = 0, 725 HasMul = 0, 726 HasNegate = 0, 727 HasAbs = 0, 728 HasAbs2 = 0, 729 HasMin = 0, 730 HasMax = 0, 731 HasConj = 0, 732 HasSetLinear = 0, 733 HasDiv = 0, 734 HasSqrt = 0, 735 HasRsqrt = 0, 736 HasExp = 0, 737 HasLog = 0, 738 HasBlend = 0 739 }; 740 }; 741 742 743 template<> struct unpacket_traits<Packet8h> { typedef Eigen::half type; enum {size=8, alignment=Aligned16}; typedef Packet8h half; }; 744 745 template<> EIGEN_STRONG_INLINE Packet8h pset1<Packet8h>(const Eigen::half& from) { 746 Packet8h result; 747 result.x = _mm_set1_epi16(from.x); 748 return result; 749 } 750 751 template<> EIGEN_STRONG_INLINE Eigen::half pfirst<Packet8h>(const Packet8h& from) { 752 return half_impl::raw_uint16_to_half(static_cast<unsigned short>(_mm_extract_epi16(from.x, 0))); 753 } 754 755 template<> EIGEN_STRONG_INLINE Packet8h pload<Packet8h>(const Eigen::half* from) { 756 Packet8h result; 757 result.x = _mm_load_si128(reinterpret_cast<const __m128i*>(from)); 758 return result; 759 } 760 761 template<> EIGEN_STRONG_INLINE Packet8h ploadu<Packet8h>(const Eigen::half* from) { 762 Packet8h result; 763 result.x = _mm_loadu_si128(reinterpret_cast<const __m128i*>(from)); 764 return result; 765 } 766 767 template<> EIGEN_STRONG_INLINE void pstore<Eigen::half>(Eigen::half* to, const Packet8h& from) { 768 _mm_store_si128(reinterpret_cast<__m128i*>(to), from.x); 769 } 770 771 template<> EIGEN_STRONG_INLINE void pstoreu<Eigen::half>(Eigen::half* to, const Packet8h& from) { 772 _mm_storeu_si128(reinterpret_cast<__m128i*>(to), from.x); 773 } 774 775 template<> EIGEN_STRONG_INLINE Packet8h 776 ploadquad<Packet8h>(const Eigen::half* from) { 777 Packet8h result; 778 unsigned short a = from[0].x; 779 unsigned short b = from[1].x; 780 result.x = _mm_set_epi16(b, b, b, b, a, a, a, a); 781 return result; 782 } 783 784 EIGEN_STRONG_INLINE Packet8f half2float(const Packet8h& a) { 785 #ifdef EIGEN_HAS_FP16_C 786 return _mm256_cvtph_ps(a.x); 787 #else 788 EIGEN_ALIGN32 Eigen::half aux[8]; 789 pstore(aux, a); 790 float f0(aux[0]); 791 float f1(aux[1]); 792 float f2(aux[2]); 793 float f3(aux[3]); 794 float f4(aux[4]); 795 float f5(aux[5]); 796 float f6(aux[6]); 797 float f7(aux[7]); 798 799 return _mm256_set_ps(f7, f6, f5, f4, f3, f2, f1, f0); 800 #endif 801 } 802 803 EIGEN_STRONG_INLINE Packet8h float2half(const Packet8f& a) { 804 #ifdef EIGEN_HAS_FP16_C 805 Packet8h result; 806 result.x = _mm256_cvtps_ph(a, _MM_FROUND_TO_NEAREST_INT|_MM_FROUND_NO_EXC); 807 return result; 808 #else 809 EIGEN_ALIGN32 float aux[8]; 810 pstore(aux, a); 811 Eigen::half h0(aux[0]); 812 Eigen::half h1(aux[1]); 813 Eigen::half h2(aux[2]); 814 Eigen::half h3(aux[3]); 815 Eigen::half h4(aux[4]); 816 Eigen::half h5(aux[5]); 817 Eigen::half h6(aux[6]); 818 Eigen::half h7(aux[7]); 819 820 Packet8h result; 821 result.x = _mm_set_epi16(h7.x, h6.x, h5.x, h4.x, h3.x, h2.x, h1.x, h0.x); 822 return result; 823 #endif 824 } 825 826 template<> EIGEN_STRONG_INLINE Packet8h pconj(const Packet8h& a) { return a; } 827 828 template<> EIGEN_STRONG_INLINE Packet8h padd<Packet8h>(const Packet8h& a, const Packet8h& b) { 829 Packet8f af = half2float(a); 830 Packet8f bf = half2float(b); 831 Packet8f rf = padd(af, bf); 832 return float2half(rf); 833 } 834 835 template<> EIGEN_STRONG_INLINE Packet8h pmul<Packet8h>(const Packet8h& a, const Packet8h& b) { 836 Packet8f af = half2float(a); 837 Packet8f bf = half2float(b); 838 Packet8f rf = pmul(af, bf); 839 return float2half(rf); 840 } 841 842 template<> EIGEN_STRONG_INLINE Packet8h pgather<Eigen::half, Packet8h>(const Eigen::half* from, Index stride) 843 { 844 Packet8h result; 845 result.x = _mm_set_epi16(from[7*stride].x, from[6*stride].x, from[5*stride].x, from[4*stride].x, from[3*stride].x, from[2*stride].x, from[1*stride].x, from[0*stride].x); 846 return result; 847 } 848 849 template<> EIGEN_STRONG_INLINE void pscatter<Eigen::half, Packet8h>(Eigen::half* to, const Packet8h& from, Index stride) 850 { 851 EIGEN_ALIGN32 Eigen::half aux[8]; 852 pstore(aux, from); 853 to[stride*0].x = aux[0].x; 854 to[stride*1].x = aux[1].x; 855 to[stride*2].x = aux[2].x; 856 to[stride*3].x = aux[3].x; 857 to[stride*4].x = aux[4].x; 858 to[stride*5].x = aux[5].x; 859 to[stride*6].x = aux[6].x; 860 to[stride*7].x = aux[7].x; 861 } 862 863 template<> EIGEN_STRONG_INLINE Eigen::half predux<Packet8h>(const Packet8h& a) { 864 Packet8f af = half2float(a); 865 float reduced = predux<Packet8f>(af); 866 return Eigen::half(reduced); 867 } 868 869 template<> EIGEN_STRONG_INLINE Eigen::half predux_max<Packet8h>(const Packet8h& a) { 870 Packet8f af = half2float(a); 871 float reduced = predux_max<Packet8f>(af); 872 return Eigen::half(reduced); 873 } 874 875 template<> EIGEN_STRONG_INLINE Eigen::half predux_min<Packet8h>(const Packet8h& a) { 876 Packet8f af = half2float(a); 877 float reduced = predux_min<Packet8f>(af); 878 return Eigen::half(reduced); 879 } 880 881 template<> EIGEN_STRONG_INLINE Eigen::half predux_mul<Packet8h>(const Packet8h& a) { 882 Packet8f af = half2float(a); 883 float reduced = predux_mul<Packet8f>(af); 884 return Eigen::half(reduced); 885 } 886 887 EIGEN_STRONG_INLINE void 888 ptranspose(PacketBlock<Packet8h,8>& kernel) { 889 __m128i a = kernel.packet[0].x; 890 __m128i b = kernel.packet[1].x; 891 __m128i c = kernel.packet[2].x; 892 __m128i d = kernel.packet[3].x; 893 __m128i e = kernel.packet[4].x; 894 __m128i f = kernel.packet[5].x; 895 __m128i g = kernel.packet[6].x; 896 __m128i h = kernel.packet[7].x; 897 898 __m128i a03b03 = _mm_unpacklo_epi16(a, b); 899 __m128i c03d03 = _mm_unpacklo_epi16(c, d); 900 __m128i e03f03 = _mm_unpacklo_epi16(e, f); 901 __m128i g03h03 = _mm_unpacklo_epi16(g, h); 902 __m128i a47b47 = _mm_unpackhi_epi16(a, b); 903 __m128i c47d47 = _mm_unpackhi_epi16(c, d); 904 __m128i e47f47 = _mm_unpackhi_epi16(e, f); 905 __m128i g47h47 = _mm_unpackhi_epi16(g, h); 906 907 __m128i a01b01c01d01 = _mm_unpacklo_epi32(a03b03, c03d03); 908 __m128i a23b23c23d23 = _mm_unpackhi_epi32(a03b03, c03d03); 909 __m128i e01f01g01h01 = _mm_unpacklo_epi32(e03f03, g03h03); 910 __m128i e23f23g23h23 = _mm_unpackhi_epi32(e03f03, g03h03); 911 __m128i a45b45c45d45 = _mm_unpacklo_epi32(a47b47, c47d47); 912 __m128i a67b67c67d67 = _mm_unpackhi_epi32(a47b47, c47d47); 913 __m128i e45f45g45h45 = _mm_unpacklo_epi32(e47f47, g47h47); 914 __m128i e67f67g67h67 = _mm_unpackhi_epi32(e47f47, g47h47); 915 916 __m128i a0b0c0d0e0f0g0h0 = _mm_unpacklo_epi64(a01b01c01d01, e01f01g01h01); 917 __m128i a1b1c1d1e1f1g1h1 = _mm_unpackhi_epi64(a01b01c01d01, e01f01g01h01); 918 __m128i a2b2c2d2e2f2g2h2 = _mm_unpacklo_epi64(a23b23c23d23, e23f23g23h23); 919 __m128i a3b3c3d3e3f3g3h3 = _mm_unpackhi_epi64(a23b23c23d23, e23f23g23h23); 920 __m128i a4b4c4d4e4f4g4h4 = _mm_unpacklo_epi64(a45b45c45d45, e45f45g45h45); 921 __m128i a5b5c5d5e5f5g5h5 = _mm_unpackhi_epi64(a45b45c45d45, e45f45g45h45); 922 __m128i a6b6c6d6e6f6g6h6 = _mm_unpacklo_epi64(a67b67c67d67, e67f67g67h67); 923 __m128i a7b7c7d7e7f7g7h7 = _mm_unpackhi_epi64(a67b67c67d67, e67f67g67h67); 924 925 kernel.packet[0].x = a0b0c0d0e0f0g0h0; 926 kernel.packet[1].x = a1b1c1d1e1f1g1h1; 927 kernel.packet[2].x = a2b2c2d2e2f2g2h2; 928 kernel.packet[3].x = a3b3c3d3e3f3g3h3; 929 kernel.packet[4].x = a4b4c4d4e4f4g4h4; 930 kernel.packet[5].x = a5b5c5d5e5f5g5h5; 931 kernel.packet[6].x = a6b6c6d6e6f6g6h6; 932 kernel.packet[7].x = a7b7c7d7e7f7g7h7; 933 } 934 935 EIGEN_STRONG_INLINE void 936 ptranspose(PacketBlock<Packet8h,4>& kernel) { 937 EIGEN_ALIGN32 Eigen::half in[4][8]; 938 pstore<Eigen::half>(in[0], kernel.packet[0]); 939 pstore<Eigen::half>(in[1], kernel.packet[1]); 940 pstore<Eigen::half>(in[2], kernel.packet[2]); 941 pstore<Eigen::half>(in[3], kernel.packet[3]); 942 943 EIGEN_ALIGN32 Eigen::half out[4][8]; 944 945 for (int i = 0; i < 4; ++i) { 946 for (int j = 0; j < 4; ++j) { 947 out[i][j] = in[j][2*i]; 948 } 949 for (int j = 0; j < 4; ++j) { 950 out[i][j+4] = in[j][2*i+1]; 951 } 952 } 953 954 kernel.packet[0] = pload<Packet8h>(out[0]); 955 kernel.packet[1] = pload<Packet8h>(out[1]); 956 kernel.packet[2] = pload<Packet8h>(out[2]); 957 kernel.packet[3] = pload<Packet8h>(out[3]); 958 } 959 960 961 // Disable the following code since it's broken on too many platforms / compilers. 962 //#elif defined(EIGEN_VECTORIZE_SSE) && (!EIGEN_ARCH_x86_64) && (!EIGEN_COMP_MSVC) 963 #elif 0 964 965 typedef struct { 966 __m64 x; 967 } Packet4h; 968 969 970 template<> struct is_arithmetic<Packet4h> { enum { value = true }; }; 971 972 template <> 973 struct packet_traits<Eigen::half> : default_packet_traits { 974 typedef Packet4h type; 975 // There is no half-size packet for Packet4h. 976 typedef Packet4h half; 977 enum { 978 Vectorizable = 1, 979 AlignedOnScalar = 1, 980 size = 4, 981 HasHalfPacket = 0, 982 HasAdd = 0, 983 HasSub = 0, 984 HasMul = 0, 985 HasNegate = 0, 986 HasAbs = 0, 987 HasAbs2 = 0, 988 HasMin = 0, 989 HasMax = 0, 990 HasConj = 0, 991 HasSetLinear = 0, 992 HasDiv = 0, 993 HasSqrt = 0, 994 HasRsqrt = 0, 995 HasExp = 0, 996 HasLog = 0, 997 HasBlend = 0 998 }; 999 }; 1000 1001 1002 template<> struct unpacket_traits<Packet4h> { typedef Eigen::half type; enum {size=4, alignment=Aligned16}; typedef Packet4h half; }; 1003 1004 template<> EIGEN_STRONG_INLINE Packet4h pset1<Packet4h>(const Eigen::half& from) { 1005 Packet4h result; 1006 result.x = _mm_set1_pi16(from.x); 1007 return result; 1008 } 1009 1010 template<> EIGEN_STRONG_INLINE Eigen::half pfirst<Packet4h>(const Packet4h& from) { 1011 return half_impl::raw_uint16_to_half(static_cast<unsigned short>(_mm_cvtsi64_si32(from.x))); 1012 } 1013 1014 template<> EIGEN_STRONG_INLINE Packet4h pconj(const Packet4h& a) { return a; } 1015 1016 template<> EIGEN_STRONG_INLINE Packet4h padd<Packet4h>(const Packet4h& a, const Packet4h& b) { 1017 __int64_t a64 = _mm_cvtm64_si64(a.x); 1018 __int64_t b64 = _mm_cvtm64_si64(b.x); 1019 1020 Eigen::half h[4]; 1021 1022 Eigen::half ha = half_impl::raw_uint16_to_half(static_cast<unsigned short>(a64)); 1023 Eigen::half hb = half_impl::raw_uint16_to_half(static_cast<unsigned short>(b64)); 1024 h[0] = ha + hb; 1025 ha = half_impl::raw_uint16_to_half(static_cast<unsigned short>(a64 >> 16)); 1026 hb = half_impl::raw_uint16_to_half(static_cast<unsigned short>(b64 >> 16)); 1027 h[1] = ha + hb; 1028 ha = half_impl::raw_uint16_to_half(static_cast<unsigned short>(a64 >> 32)); 1029 hb = half_impl::raw_uint16_to_half(static_cast<unsigned short>(b64 >> 32)); 1030 h[2] = ha + hb; 1031 ha = half_impl::raw_uint16_to_half(static_cast<unsigned short>(a64 >> 48)); 1032 hb = half_impl::raw_uint16_to_half(static_cast<unsigned short>(b64 >> 48)); 1033 h[3] = ha + hb; 1034 Packet4h result; 1035 result.x = _mm_set_pi16(h[3].x, h[2].x, h[1].x, h[0].x); 1036 return result; 1037 } 1038 1039 template<> EIGEN_STRONG_INLINE Packet4h pmul<Packet4h>(const Packet4h& a, const Packet4h& b) { 1040 __int64_t a64 = _mm_cvtm64_si64(a.x); 1041 __int64_t b64 = _mm_cvtm64_si64(b.x); 1042 1043 Eigen::half h[4]; 1044 1045 Eigen::half ha = half_impl::raw_uint16_to_half(static_cast<unsigned short>(a64)); 1046 Eigen::half hb = half_impl::raw_uint16_to_half(static_cast<unsigned short>(b64)); 1047 h[0] = ha * hb; 1048 ha = half_impl::raw_uint16_to_half(static_cast<unsigned short>(a64 >> 16)); 1049 hb = half_impl::raw_uint16_to_half(static_cast<unsigned short>(b64 >> 16)); 1050 h[1] = ha * hb; 1051 ha = half_impl::raw_uint16_to_half(static_cast<unsigned short>(a64 >> 32)); 1052 hb = half_impl::raw_uint16_to_half(static_cast<unsigned short>(b64 >> 32)); 1053 h[2] = ha * hb; 1054 ha = half_impl::raw_uint16_to_half(static_cast<unsigned short>(a64 >> 48)); 1055 hb = half_impl::raw_uint16_to_half(static_cast<unsigned short>(b64 >> 48)); 1056 h[3] = ha * hb; 1057 Packet4h result; 1058 result.x = _mm_set_pi16(h[3].x, h[2].x, h[1].x, h[0].x); 1059 return result; 1060 } 1061 1062 template<> EIGEN_STRONG_INLINE Packet4h pload<Packet4h>(const Eigen::half* from) { 1063 Packet4h result; 1064 result.x = _mm_cvtsi64_m64(*reinterpret_cast<const __int64_t*>(from)); 1065 return result; 1066 } 1067 1068 template<> EIGEN_STRONG_INLINE Packet4h ploadu<Packet4h>(const Eigen::half* from) { 1069 Packet4h result; 1070 result.x = _mm_cvtsi64_m64(*reinterpret_cast<const __int64_t*>(from)); 1071 return result; 1072 } 1073 1074 template<> EIGEN_STRONG_INLINE void pstore<Eigen::half>(Eigen::half* to, const Packet4h& from) { 1075 __int64_t r = _mm_cvtm64_si64(from.x); 1076 *(reinterpret_cast<__int64_t*>(to)) = r; 1077 } 1078 1079 template<> EIGEN_STRONG_INLINE void pstoreu<Eigen::half>(Eigen::half* to, const Packet4h& from) { 1080 __int64_t r = _mm_cvtm64_si64(from.x); 1081 *(reinterpret_cast<__int64_t*>(to)) = r; 1082 } 1083 1084 template<> EIGEN_STRONG_INLINE Packet4h 1085 ploadquad<Packet4h>(const Eigen::half* from) { 1086 return pset1<Packet4h>(*from); 1087 } 1088 1089 template<> EIGEN_STRONG_INLINE Packet4h pgather<Eigen::half, Packet4h>(const Eigen::half* from, Index stride) 1090 { 1091 Packet4h result; 1092 result.x = _mm_set_pi16(from[3*stride].x, from[2*stride].x, from[1*stride].x, from[0*stride].x); 1093 return result; 1094 } 1095 1096 template<> EIGEN_STRONG_INLINE void pscatter<Eigen::half, Packet4h>(Eigen::half* to, const Packet4h& from, Index stride) 1097 { 1098 __int64_t a = _mm_cvtm64_si64(from.x); 1099 to[stride*0].x = static_cast<unsigned short>(a); 1100 to[stride*1].x = static_cast<unsigned short>(a >> 16); 1101 to[stride*2].x = static_cast<unsigned short>(a >> 32); 1102 to[stride*3].x = static_cast<unsigned short>(a >> 48); 1103 } 1104 1105 EIGEN_STRONG_INLINE void 1106 ptranspose(PacketBlock<Packet4h,4>& kernel) { 1107 __m64 T0 = _mm_unpacklo_pi16(kernel.packet[0].x, kernel.packet[1].x); 1108 __m64 T1 = _mm_unpacklo_pi16(kernel.packet[2].x, kernel.packet[3].x); 1109 __m64 T2 = _mm_unpackhi_pi16(kernel.packet[0].x, kernel.packet[1].x); 1110 __m64 T3 = _mm_unpackhi_pi16(kernel.packet[2].x, kernel.packet[3].x); 1111 1112 kernel.packet[0].x = _mm_unpacklo_pi32(T0, T1); 1113 kernel.packet[1].x = _mm_unpackhi_pi32(T0, T1); 1114 kernel.packet[2].x = _mm_unpacklo_pi32(T2, T3); 1115 kernel.packet[3].x = _mm_unpackhi_pi32(T2, T3); 1116 } 1117 1118 #endif 1119 1120 } 1121 } 1122 1123 #endif // EIGEN_PACKET_MATH_HALF_CUDA_H 1124