• Home
  • History
  • Annotate
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // This file is part of Eigen, a lightweight C++ template library
2 // for linear algebra.
3 //
4 // Copyright (C) 2008-2009 Gael Guennebaud <gael.guennebaud@inria.fr>
5 // Copyright (C) 2010 Konstantinos Margaritis <markos@codex.gr>
6 // Heavily based on Gael's SSE version.
7 //
8 // This Source Code Form is subject to the terms of the Mozilla
9 // Public License v. 2.0. If a copy of the MPL was not distributed
10 // with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
11 
12 #ifndef EIGEN_PACKET_MATH_NEON_H
13 #define EIGEN_PACKET_MATH_NEON_H
14 
15 namespace Eigen {
16 
17 namespace internal {
18 
19 #ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD
20 #define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8
21 #endif
22 
23 // FIXME NEON has 16 quad registers, but since the current register allocator
24 // is so bad, it is much better to reduce it to 8
25 #ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS
26 #define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 8
27 #endif
28 
29 typedef float32x4_t Packet4f;
30 typedef int32x4_t   Packet4i;
31 typedef uint32x4_t  Packet4ui;
32 
33 #define _EIGEN_DECLARE_CONST_Packet4f(NAME,X) \
34   const Packet4f p4f_##NAME = pset1<Packet4f>(X)
35 
36 #define _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(NAME,X) \
37   const Packet4f p4f_##NAME = vreinterpretq_f32_u32(pset1<int>(X))
38 
39 #define _EIGEN_DECLARE_CONST_Packet4i(NAME,X) \
40   const Packet4i p4i_##NAME = pset1<Packet4i>(X)
41 
42 #if defined(__llvm__) && !defined(__clang__)
43   //Special treatment for Apple's llvm-gcc, its NEON packet types are unions
44   #define EIGEN_INIT_NEON_PACKET2(X, Y)       {{X, Y}}
45   #define EIGEN_INIT_NEON_PACKET4(X, Y, Z, W) {{X, Y, Z, W}}
46 #else
47   //Default initializer for packets
48   #define EIGEN_INIT_NEON_PACKET2(X, Y)       {X, Y}
49   #define EIGEN_INIT_NEON_PACKET4(X, Y, Z, W) {X, Y, Z, W}
50 #endif
51 
52 // arm64 does have the pld instruction. If available, let's trust the __builtin_prefetch built-in function
53 // which available on LLVM and GCC (at least)
54 #if EIGEN_HAS_BUILTIN(__builtin_prefetch) || defined(__GNUC__)
55   #define EIGEN_ARM_PREFETCH(ADDR) __builtin_prefetch(ADDR);
56 #elif defined __pld
57   #define EIGEN_ARM_PREFETCH(ADDR) __pld(ADDR)
58 #elif !defined(__aarch64__)
59   #define EIGEN_ARM_PREFETCH(ADDR) __asm__ __volatile__ ( "   pld [%[addr]]\n" :: [addr] "r" (ADDR) : "cc" );
60 #else
61   // by default no explicit prefetching
62   #define EIGEN_ARM_PREFETCH(ADDR)
63 #endif
64 
65 template<> struct packet_traits<float>  : default_packet_traits
66 {
67   typedef Packet4f type;
68   enum {
69     Vectorizable = 1,
70     AlignedOnScalar = 1,
71     size = 4,
72 
73     HasDiv  = 1,
74     // FIXME check the Has*
75     HasSin  = 0,
76     HasCos  = 0,
77     HasLog  = 0,
78     HasExp  = 0,
79     HasSqrt = 0
80   };
81 };
82 template<> struct packet_traits<int>    : default_packet_traits
83 {
84   typedef Packet4i type;
85   enum {
86     Vectorizable = 1,
87     AlignedOnScalar = 1,
88     size=4
89     // FIXME check the Has*
90   };
91 };
92 
93 #if EIGEN_GNUC_AT_MOST(4,4) && !defined(__llvm__)
94 // workaround gcc 4.2, 4.3 and 4.4 compilatin issue
95 EIGEN_STRONG_INLINE float32x4_t vld1q_f32(const float* x) { return ::vld1q_f32((const float32_t*)x); }
96 EIGEN_STRONG_INLINE float32x2_t vld1_f32 (const float* x) { return ::vld1_f32 ((const float32_t*)x); }
97 EIGEN_STRONG_INLINE void        vst1q_f32(float* to, float32x4_t from) { ::vst1q_f32((float32_t*)to,from); }
98 EIGEN_STRONG_INLINE void        vst1_f32 (float* to, float32x2_t from) { ::vst1_f32 ((float32_t*)to,from); }
99 #endif
100 
101 template<> struct unpacket_traits<Packet4f> { typedef float  type; enum {size=4}; };
102 template<> struct unpacket_traits<Packet4i> { typedef int    type; enum {size=4}; };
103 
104 template<> EIGEN_STRONG_INLINE Packet4f pset1<Packet4f>(const float&  from) { return vdupq_n_f32(from); }
105 template<> EIGEN_STRONG_INLINE Packet4i pset1<Packet4i>(const int&    from)   { return vdupq_n_s32(from); }
106 
107 template<> EIGEN_STRONG_INLINE Packet4f plset<float>(const float& a)
108 {
109   Packet4f countdown = EIGEN_INIT_NEON_PACKET4(0, 1, 2, 3);
110   return vaddq_f32(pset1<Packet4f>(a), countdown);
111 }
112 template<> EIGEN_STRONG_INLINE Packet4i plset<int>(const int& a)
113 {
114   Packet4i countdown = EIGEN_INIT_NEON_PACKET4(0, 1, 2, 3);
115   return vaddq_s32(pset1<Packet4i>(a), countdown);
116 }
117 
118 template<> EIGEN_STRONG_INLINE Packet4f padd<Packet4f>(const Packet4f& a, const Packet4f& b) { return vaddq_f32(a,b); }
119 template<> EIGEN_STRONG_INLINE Packet4i padd<Packet4i>(const Packet4i& a, const Packet4i& b) { return vaddq_s32(a,b); }
120 
121 template<> EIGEN_STRONG_INLINE Packet4f psub<Packet4f>(const Packet4f& a, const Packet4f& b) { return vsubq_f32(a,b); }
122 template<> EIGEN_STRONG_INLINE Packet4i psub<Packet4i>(const Packet4i& a, const Packet4i& b) { return vsubq_s32(a,b); }
123 
124 template<> EIGEN_STRONG_INLINE Packet4f pnegate(const Packet4f& a) { return vnegq_f32(a); }
125 template<> EIGEN_STRONG_INLINE Packet4i pnegate(const Packet4i& a) { return vnegq_s32(a); }
126 
127 template<> EIGEN_STRONG_INLINE Packet4f pconj(const Packet4f& a) { return a; }
128 template<> EIGEN_STRONG_INLINE Packet4i pconj(const Packet4i& a) { return a; }
129 
130 template<> EIGEN_STRONG_INLINE Packet4f pmul<Packet4f>(const Packet4f& a, const Packet4f& b) { return vmulq_f32(a,b); }
131 template<> EIGEN_STRONG_INLINE Packet4i pmul<Packet4i>(const Packet4i& a, const Packet4i& b) { return vmulq_s32(a,b); }
132 
133 template<> EIGEN_STRONG_INLINE Packet4f pdiv<Packet4f>(const Packet4f& a, const Packet4f& b)
134 {
135   Packet4f inv, restep, div;
136 
137   // NEON does not offer a divide instruction, we have to do a reciprocal approximation
138   // However NEON in contrast to other SIMD engines (AltiVec/SSE), offers
139   // a reciprocal estimate AND a reciprocal step -which saves a few instructions
140   // vrecpeq_f32() returns an estimate to 1/b, which we will finetune with
141   // Newton-Raphson and vrecpsq_f32()
142   inv = vrecpeq_f32(b);
143 
144   // This returns a differential, by which we will have to multiply inv to get a better
145   // approximation of 1/b.
146   restep = vrecpsq_f32(b, inv);
147   inv = vmulq_f32(restep, inv);
148 
149   // Finally, multiply a by 1/b and get the wanted result of the division.
150   div = vmulq_f32(a, inv);
151 
152   return div;
153 }
154 template<> EIGEN_STRONG_INLINE Packet4i pdiv<Packet4i>(const Packet4i& /*a*/, const Packet4i& /*b*/)
155 { eigen_assert(false && "packet integer division are not supported by NEON");
156   return pset1<Packet4i>(0);
157 }
158 
159 // for some weird raisons, it has to be overloaded for packet of integers
160 template<> EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c) { return vmlaq_f32(c,a,b); }
161 template<> EIGEN_STRONG_INLINE Packet4i pmadd(const Packet4i& a, const Packet4i& b, const Packet4i& c) { return vmlaq_s32(c,a,b); }
162 
163 template<> EIGEN_STRONG_INLINE Packet4f pmin<Packet4f>(const Packet4f& a, const Packet4f& b) { return vminq_f32(a,b); }
164 template<> EIGEN_STRONG_INLINE Packet4i pmin<Packet4i>(const Packet4i& a, const Packet4i& b) { return vminq_s32(a,b); }
165 
166 template<> EIGEN_STRONG_INLINE Packet4f pmax<Packet4f>(const Packet4f& a, const Packet4f& b) { return vmaxq_f32(a,b); }
167 template<> EIGEN_STRONG_INLINE Packet4i pmax<Packet4i>(const Packet4i& a, const Packet4i& b) { return vmaxq_s32(a,b); }
168 
169 // Logical Operations are not supported for float, so we have to reinterpret casts using NEON intrinsics
170 template<> EIGEN_STRONG_INLINE Packet4f pand<Packet4f>(const Packet4f& a, const Packet4f& b)
171 {
172   return vreinterpretq_f32_u32(vandq_u32(vreinterpretq_u32_f32(a),vreinterpretq_u32_f32(b)));
173 }
174 template<> EIGEN_STRONG_INLINE Packet4i pand<Packet4i>(const Packet4i& a, const Packet4i& b) { return vandq_s32(a,b); }
175 
176 template<> EIGEN_STRONG_INLINE Packet4f por<Packet4f>(const Packet4f& a, const Packet4f& b)
177 {
178   return vreinterpretq_f32_u32(vorrq_u32(vreinterpretq_u32_f32(a),vreinterpretq_u32_f32(b)));
179 }
180 template<> EIGEN_STRONG_INLINE Packet4i por<Packet4i>(const Packet4i& a, const Packet4i& b) { return vorrq_s32(a,b); }
181 
182 template<> EIGEN_STRONG_INLINE Packet4f pxor<Packet4f>(const Packet4f& a, const Packet4f& b)
183 {
184   return vreinterpretq_f32_u32(veorq_u32(vreinterpretq_u32_f32(a),vreinterpretq_u32_f32(b)));
185 }
186 template<> EIGEN_STRONG_INLINE Packet4i pxor<Packet4i>(const Packet4i& a, const Packet4i& b) { return veorq_s32(a,b); }
187 
188 template<> EIGEN_STRONG_INLINE Packet4f pandnot<Packet4f>(const Packet4f& a, const Packet4f& b)
189 {
190   return vreinterpretq_f32_u32(vbicq_u32(vreinterpretq_u32_f32(a),vreinterpretq_u32_f32(b)));
191 }
192 template<> EIGEN_STRONG_INLINE Packet4i pandnot<Packet4i>(const Packet4i& a, const Packet4i& b) { return vbicq_s32(a,b); }
193 
194 template<> EIGEN_STRONG_INLINE Packet4f pload<Packet4f>(const float* from) { EIGEN_DEBUG_ALIGNED_LOAD return vld1q_f32(from); }
195 template<> EIGEN_STRONG_INLINE Packet4i pload<Packet4i>(const int*   from) { EIGEN_DEBUG_ALIGNED_LOAD return vld1q_s32(from); }
196 
197 template<> EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f>(const float* from) { EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_f32(from); }
198 template<> EIGEN_STRONG_INLINE Packet4i ploadu<Packet4i>(const int* from)   { EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_s32(from); }
199 
200 template<> EIGEN_STRONG_INLINE Packet4f ploaddup<Packet4f>(const float*   from)
201 {
202   float32x2_t lo, hi;
203   lo = vld1_dup_f32(from);
204   hi = vld1_dup_f32(from+1);
205   return vcombine_f32(lo, hi);
206 }
207 template<> EIGEN_STRONG_INLINE Packet4i ploaddup<Packet4i>(const int*     from)
208 {
209   int32x2_t lo, hi;
210   lo = vld1_dup_s32(from);
211   hi = vld1_dup_s32(from+1);
212   return vcombine_s32(lo, hi);
213 }
214 
215 template<> EIGEN_STRONG_INLINE void pstore<float>(float*   to, const Packet4f& from) { EIGEN_DEBUG_ALIGNED_STORE vst1q_f32(to, from); }
216 template<> EIGEN_STRONG_INLINE void pstore<int>(int*       to, const Packet4i& from) { EIGEN_DEBUG_ALIGNED_STORE vst1q_s32(to, from); }
217 
218 template<> EIGEN_STRONG_INLINE void pstoreu<float>(float*  to, const Packet4f& from) { EIGEN_DEBUG_UNALIGNED_STORE vst1q_f32(to, from); }
219 template<> EIGEN_STRONG_INLINE void pstoreu<int>(int*      to, const Packet4i& from) { EIGEN_DEBUG_UNALIGNED_STORE vst1q_s32(to, from); }
220 
221 template<> EIGEN_STRONG_INLINE void prefetch<float>(const float* addr) { EIGEN_ARM_PREFETCH(addr); }
222 template<> EIGEN_STRONG_INLINE void prefetch<int>(const int*     addr) { EIGEN_ARM_PREFETCH(addr); }
223 
224 // FIXME only store the 2 first elements ?
225 template<> EIGEN_STRONG_INLINE float  pfirst<Packet4f>(const Packet4f& a) { float EIGEN_ALIGN16 x[4]; vst1q_f32(x, a); return x[0]; }
226 template<> EIGEN_STRONG_INLINE int    pfirst<Packet4i>(const Packet4i& a) { int   EIGEN_ALIGN16 x[4]; vst1q_s32(x, a); return x[0]; }
227 
228 template<> EIGEN_STRONG_INLINE Packet4f preverse(const Packet4f& a) {
229   float32x2_t a_lo, a_hi;
230   Packet4f a_r64;
231 
232   a_r64 = vrev64q_f32(a);
233   a_lo = vget_low_f32(a_r64);
234   a_hi = vget_high_f32(a_r64);
235   return vcombine_f32(a_hi, a_lo);
236 }
237 template<> EIGEN_STRONG_INLINE Packet4i preverse(const Packet4i& a) {
238   int32x2_t a_lo, a_hi;
239   Packet4i a_r64;
240 
241   a_r64 = vrev64q_s32(a);
242   a_lo = vget_low_s32(a_r64);
243   a_hi = vget_high_s32(a_r64);
244   return vcombine_s32(a_hi, a_lo);
245 }
246 template<> EIGEN_STRONG_INLINE Packet4f pabs(const Packet4f& a) { return vabsq_f32(a); }
247 template<> EIGEN_STRONG_INLINE Packet4i pabs(const Packet4i& a) { return vabsq_s32(a); }
248 
249 template<> EIGEN_STRONG_INLINE float predux<Packet4f>(const Packet4f& a)
250 {
251   float32x2_t a_lo, a_hi, sum;
252 
253   a_lo = vget_low_f32(a);
254   a_hi = vget_high_f32(a);
255   sum = vpadd_f32(a_lo, a_hi);
256   sum = vpadd_f32(sum, sum);
257   return vget_lane_f32(sum, 0);
258 }
259 
260 template<> EIGEN_STRONG_INLINE Packet4f preduxp<Packet4f>(const Packet4f* vecs)
261 {
262   float32x4x2_t vtrn1, vtrn2, res1, res2;
263   Packet4f sum1, sum2, sum;
264 
265   // NEON zip performs interleaving of the supplied vectors.
266   // We perform two interleaves in a row to acquire the transposed vector
267   vtrn1 = vzipq_f32(vecs[0], vecs[2]);
268   vtrn2 = vzipq_f32(vecs[1], vecs[3]);
269   res1 = vzipq_f32(vtrn1.val[0], vtrn2.val[0]);
270   res2 = vzipq_f32(vtrn1.val[1], vtrn2.val[1]);
271 
272   // Do the addition of the resulting vectors
273   sum1 = vaddq_f32(res1.val[0], res1.val[1]);
274   sum2 = vaddq_f32(res2.val[0], res2.val[1]);
275   sum = vaddq_f32(sum1, sum2);
276 
277   return sum;
278 }
279 
280 template<> EIGEN_STRONG_INLINE int predux<Packet4i>(const Packet4i& a)
281 {
282   int32x2_t a_lo, a_hi, sum;
283 
284   a_lo = vget_low_s32(a);
285   a_hi = vget_high_s32(a);
286   sum = vpadd_s32(a_lo, a_hi);
287   sum = vpadd_s32(sum, sum);
288   return vget_lane_s32(sum, 0);
289 }
290 
291 template<> EIGEN_STRONG_INLINE Packet4i preduxp<Packet4i>(const Packet4i* vecs)
292 {
293   int32x4x2_t vtrn1, vtrn2, res1, res2;
294   Packet4i sum1, sum2, sum;
295 
296   // NEON zip performs interleaving of the supplied vectors.
297   // We perform two interleaves in a row to acquire the transposed vector
298   vtrn1 = vzipq_s32(vecs[0], vecs[2]);
299   vtrn2 = vzipq_s32(vecs[1], vecs[3]);
300   res1 = vzipq_s32(vtrn1.val[0], vtrn2.val[0]);
301   res2 = vzipq_s32(vtrn1.val[1], vtrn2.val[1]);
302 
303   // Do the addition of the resulting vectors
304   sum1 = vaddq_s32(res1.val[0], res1.val[1]);
305   sum2 = vaddq_s32(res2.val[0], res2.val[1]);
306   sum = vaddq_s32(sum1, sum2);
307 
308   return sum;
309 }
310 
311 // Other reduction functions:
312 // mul
313 template<> EIGEN_STRONG_INLINE float predux_mul<Packet4f>(const Packet4f& a)
314 {
315   float32x2_t a_lo, a_hi, prod;
316 
317   // Get a_lo = |a1|a2| and a_hi = |a3|a4|
318   a_lo = vget_low_f32(a);
319   a_hi = vget_high_f32(a);
320   // Get the product of a_lo * a_hi -> |a1*a3|a2*a4|
321   prod = vmul_f32(a_lo, a_hi);
322   // Multiply prod with its swapped value |a2*a4|a1*a3|
323   prod = vmul_f32(prod, vrev64_f32(prod));
324 
325   return vget_lane_f32(prod, 0);
326 }
327 template<> EIGEN_STRONG_INLINE int predux_mul<Packet4i>(const Packet4i& a)
328 {
329   int32x2_t a_lo, a_hi, prod;
330 
331   // Get a_lo = |a1|a2| and a_hi = |a3|a4|
332   a_lo = vget_low_s32(a);
333   a_hi = vget_high_s32(a);
334   // Get the product of a_lo * a_hi -> |a1*a3|a2*a4|
335   prod = vmul_s32(a_lo, a_hi);
336   // Multiply prod with its swapped value |a2*a4|a1*a3|
337   prod = vmul_s32(prod, vrev64_s32(prod));
338 
339   return vget_lane_s32(prod, 0);
340 }
341 
342 // min
343 template<> EIGEN_STRONG_INLINE float predux_min<Packet4f>(const Packet4f& a)
344 {
345   float32x2_t a_lo, a_hi, min;
346 
347   a_lo = vget_low_f32(a);
348   a_hi = vget_high_f32(a);
349   min = vpmin_f32(a_lo, a_hi);
350   min = vpmin_f32(min, min);
351 
352   return vget_lane_f32(min, 0);
353 }
354 
355 template<> EIGEN_STRONG_INLINE int predux_min<Packet4i>(const Packet4i& a)
356 {
357   int32x2_t a_lo, a_hi, min;
358 
359   a_lo = vget_low_s32(a);
360   a_hi = vget_high_s32(a);
361   min = vpmin_s32(a_lo, a_hi);
362   min = vpmin_s32(min, min);
363 
364   return vget_lane_s32(min, 0);
365 }
366 
367 // max
368 template<> EIGEN_STRONG_INLINE float predux_max<Packet4f>(const Packet4f& a)
369 {
370   float32x2_t a_lo, a_hi, max;
371 
372   a_lo = vget_low_f32(a);
373   a_hi = vget_high_f32(a);
374   max = vpmax_f32(a_lo, a_hi);
375   max = vpmax_f32(max, max);
376 
377   return vget_lane_f32(max, 0);
378 }
379 
380 template<> EIGEN_STRONG_INLINE int predux_max<Packet4i>(const Packet4i& a)
381 {
382   int32x2_t a_lo, a_hi, max;
383 
384   a_lo = vget_low_s32(a);
385   a_hi = vget_high_s32(a);
386   max = vpmax_s32(a_lo, a_hi);
387 
388   return vget_lane_s32(max, 0);
389 }
390 
391 // this PALIGN_NEON business is to work around a bug in LLVM Clang 3.0 causing incorrect compilation errors,
392 // see bug 347 and this LLVM bug: http://llvm.org/bugs/show_bug.cgi?id=11074
393 #define PALIGN_NEON(Offset,Type,Command) \
394 template<>\
395 struct palign_impl<Offset,Type>\
396 {\
397     EIGEN_STRONG_INLINE static void run(Type& first, const Type& second)\
398     {\
399         if (Offset!=0)\
400             first = Command(first, second, Offset);\
401     }\
402 };\
403 
404 PALIGN_NEON(0,Packet4f,vextq_f32)
405 PALIGN_NEON(1,Packet4f,vextq_f32)
406 PALIGN_NEON(2,Packet4f,vextq_f32)
407 PALIGN_NEON(3,Packet4f,vextq_f32)
408 PALIGN_NEON(0,Packet4i,vextq_s32)
409 PALIGN_NEON(1,Packet4i,vextq_s32)
410 PALIGN_NEON(2,Packet4i,vextq_s32)
411 PALIGN_NEON(3,Packet4i,vextq_s32)
412 
413 #undef PALIGN_NEON
414 
415 } // end namespace internal
416 
417 } // end namespace Eigen
418 
419 #endif // EIGEN_PACKET_MATH_NEON_H
420