1// This file is part of Eigen, a lightweight C++ template library 2// for linear algebra. 3// 4// Copyright (C) 2008 Gael Guennebaud <gael.guennebaud@inria.fr> 5// Copyright (C) 2007-2011 Benoit Jacob <jacob.benoit.1@gmail.com> 6// 7// This Source Code Form is subject to the terms of the Mozilla 8// Public License v. 2.0. If a copy of the MPL was not distributed 9// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 10 11#ifndef EIGEN_CORE_H 12#define EIGEN_CORE_H 13 14// first thing Eigen does: stop the compiler from committing suicide 15#include "src/Core/util/DisableStupidWarnings.h" 16 17// Handle NVCC/CUDA/SYCL 18#if defined(__CUDACC__) || defined(__SYCL_DEVICE_ONLY__) 19 // Do not try asserts on CUDA and SYCL! 20 #ifndef EIGEN_NO_DEBUG 21 #define EIGEN_NO_DEBUG 22 #endif 23 24 #ifdef EIGEN_INTERNAL_DEBUGGING 25 #undef EIGEN_INTERNAL_DEBUGGING 26 #endif 27 28 #ifdef EIGEN_EXCEPTIONS 29 #undef EIGEN_EXCEPTIONS 30 #endif 31 32 // All functions callable from CUDA code must be qualified with __device__ 33 #ifdef __CUDACC__ 34 // Do not try to vectorize on CUDA and SYCL! 35 #ifndef EIGEN_DONT_VECTORIZE 36 #define EIGEN_DONT_VECTORIZE 37 #endif 38 39 #define EIGEN_DEVICE_FUNC __host__ __device__ 40 // We need math_functions.hpp to ensure that that EIGEN_USING_STD_MATH macro 41 // works properly on the device side 42 #include <math_functions.hpp> 43 #else 44 #define EIGEN_DEVICE_FUNC 45 #endif 46 47#else 48 #define EIGEN_DEVICE_FUNC 49 50#endif 51 52// When compiling CUDA device code with NVCC, pull in math functions from the 53// global namespace. In host mode, and when device doee with clang, use the 54// std versions. 55#if defined(__CUDA_ARCH__) && defined(__NVCC__) 56 #define EIGEN_USING_STD_MATH(FUNC) using ::FUNC; 57#else 58 #define EIGEN_USING_STD_MATH(FUNC) using std::FUNC; 59#endif 60 61#if (defined(_CPPUNWIND) || defined(__EXCEPTIONS)) && !defined(__CUDA_ARCH__) && !defined(EIGEN_EXCEPTIONS) && !defined(EIGEN_USE_SYCL) 62 #define EIGEN_EXCEPTIONS 63#endif 64 65#ifdef EIGEN_EXCEPTIONS 66 #include <new> 67#endif 68 69// then include this file where all our macros are defined. It's really important to do it first because 70// it's where we do all the alignment settings (platform detection and honoring the user's will if he 71// defined e.g. EIGEN_DONT_ALIGN) so it needs to be done before we do anything with vectorization. 72#include "src/Core/util/Macros.h" 73 74// Disable the ipa-cp-clone optimization flag with MinGW 6.x or newer (enabled by default with -O3) 75// See http://eigen.tuxfamily.org/bz/show_bug.cgi?id=556 for details. 76#if EIGEN_COMP_MINGW && EIGEN_GNUC_AT_LEAST(4,6) 77 #pragma GCC optimize ("-fno-ipa-cp-clone") 78#endif 79 80#include <complex> 81 82// this include file manages BLAS and MKL related macros 83// and inclusion of their respective header files 84#include "src/Core/util/MKL_support.h" 85 86// if alignment is disabled, then disable vectorization. Note: EIGEN_MAX_ALIGN_BYTES is the proper check, it takes into 87// account both the user's will (EIGEN_MAX_ALIGN_BYTES,EIGEN_DONT_ALIGN) and our own platform checks 88#if EIGEN_MAX_ALIGN_BYTES==0 89 #ifndef EIGEN_DONT_VECTORIZE 90 #define EIGEN_DONT_VECTORIZE 91 #endif 92#endif 93 94#if EIGEN_COMP_MSVC 95 #include <malloc.h> // for _aligned_malloc -- need it regardless of whether vectorization is enabled 96 #if (EIGEN_COMP_MSVC >= 1500) // 2008 or later 97 // Remember that usage of defined() in a #define is undefined by the standard. 98 // a user reported that in 64-bit mode, MSVC doesn't care to define _M_IX86_FP. 99 #if (defined(_M_IX86_FP) && (_M_IX86_FP >= 2)) || EIGEN_ARCH_x86_64 100 #define EIGEN_SSE2_ON_MSVC_2008_OR_LATER 101 #endif 102 #endif 103#else 104 // Remember that usage of defined() in a #define is undefined by the standard 105 #if (defined __SSE2__) && ( (!EIGEN_COMP_GNUC) || EIGEN_COMP_ICC || EIGEN_GNUC_AT_LEAST(4,2) ) 106 #define EIGEN_SSE2_ON_NON_MSVC_BUT_NOT_OLD_GCC 107 #endif 108#endif 109 110#ifndef EIGEN_DONT_VECTORIZE 111 112 #if defined (EIGEN_SSE2_ON_NON_MSVC_BUT_NOT_OLD_GCC) || defined(EIGEN_SSE2_ON_MSVC_2008_OR_LATER) 113 114 // Defines symbols for compile-time detection of which instructions are 115 // used. 116 // EIGEN_VECTORIZE_YY is defined if and only if the instruction set YY is used 117 #define EIGEN_VECTORIZE 118 #define EIGEN_VECTORIZE_SSE 119 #define EIGEN_VECTORIZE_SSE2 120 121 // Detect sse3/ssse3/sse4: 122 // gcc and icc defines __SSE3__, ... 123 // there is no way to know about this on msvc. You can define EIGEN_VECTORIZE_SSE* if you 124 // want to force the use of those instructions with msvc. 125 #ifdef __SSE3__ 126 #define EIGEN_VECTORIZE_SSE3 127 #endif 128 #ifdef __SSSE3__ 129 #define EIGEN_VECTORIZE_SSSE3 130 #endif 131 #ifdef __SSE4_1__ 132 #define EIGEN_VECTORIZE_SSE4_1 133 #endif 134 #ifdef __SSE4_2__ 135 #define EIGEN_VECTORIZE_SSE4_2 136 #endif 137 #ifdef __AVX__ 138 #define EIGEN_VECTORIZE_AVX 139 #define EIGEN_VECTORIZE_SSE3 140 #define EIGEN_VECTORIZE_SSSE3 141 #define EIGEN_VECTORIZE_SSE4_1 142 #define EIGEN_VECTORIZE_SSE4_2 143 #endif 144 #ifdef __AVX2__ 145 #define EIGEN_VECTORIZE_AVX2 146 #endif 147 #ifdef __FMA__ 148 #define EIGEN_VECTORIZE_FMA 149 #endif 150 #if defined(__AVX512F__) && defined(EIGEN_ENABLE_AVX512) 151 #define EIGEN_VECTORIZE_AVX512 152 #define EIGEN_VECTORIZE_AVX2 153 #define EIGEN_VECTORIZE_AVX 154 #define EIGEN_VECTORIZE_FMA 155 #ifdef __AVX512DQ__ 156 #define EIGEN_VECTORIZE_AVX512DQ 157 #endif 158 #endif 159 160 // include files 161 162 // This extern "C" works around a MINGW-w64 compilation issue 163 // https://sourceforge.net/tracker/index.php?func=detail&aid=3018394&group_id=202880&atid=983354 164 // In essence, intrin.h is included by windows.h and also declares intrinsics (just as emmintrin.h etc. below do). 165 // However, intrin.h uses an extern "C" declaration, and g++ thus complains of duplicate declarations 166 // with conflicting linkage. The linkage for intrinsics doesn't matter, but at that stage the compiler doesn't know; 167 // so, to avoid compile errors when windows.h is included after Eigen/Core, ensure intrinsics are extern "C" here too. 168 // notice that since these are C headers, the extern "C" is theoretically needed anyways. 169 extern "C" { 170 // In theory we should only include immintrin.h and not the other *mmintrin.h header files directly. 171 // Doing so triggers some issues with ICC. However old gcc versions seems to not have this file, thus: 172 #if EIGEN_COMP_ICC >= 1110 173 #include <immintrin.h> 174 #else 175 #include <mmintrin.h> 176 #include <emmintrin.h> 177 #include <xmmintrin.h> 178 #ifdef EIGEN_VECTORIZE_SSE3 179 #include <pmmintrin.h> 180 #endif 181 #ifdef EIGEN_VECTORIZE_SSSE3 182 #include <tmmintrin.h> 183 #endif 184 #ifdef EIGEN_VECTORIZE_SSE4_1 185 #include <smmintrin.h> 186 #endif 187 #ifdef EIGEN_VECTORIZE_SSE4_2 188 #include <nmmintrin.h> 189 #endif 190 #if defined(EIGEN_VECTORIZE_AVX) || defined(EIGEN_VECTORIZE_AVX512) 191 #include <immintrin.h> 192 #endif 193 #endif 194 } // end extern "C" 195 #elif defined __VSX__ 196 #define EIGEN_VECTORIZE 197 #define EIGEN_VECTORIZE_VSX 198 #include <altivec.h> 199 // We need to #undef all these ugly tokens defined in <altivec.h> 200 // => use __vector instead of vector 201 #undef bool 202 #undef vector 203 #undef pixel 204 #elif defined __ALTIVEC__ 205 #define EIGEN_VECTORIZE 206 #define EIGEN_VECTORIZE_ALTIVEC 207 #include <altivec.h> 208 // We need to #undef all these ugly tokens defined in <altivec.h> 209 // => use __vector instead of vector 210 #undef bool 211 #undef vector 212 #undef pixel 213 #elif (defined __ARM_NEON) || (defined __ARM_NEON__) 214 #define EIGEN_VECTORIZE 215 #define EIGEN_VECTORIZE_NEON 216 #include <arm_neon.h> 217 #elif (defined __s390x__ && defined __VEC__) 218 #define EIGEN_VECTORIZE 219 #define EIGEN_VECTORIZE_ZVECTOR 220 #include <vecintrin.h> 221 #endif 222#endif 223 224#if defined(__F16C__) && !defined(EIGEN_COMP_CLANG) 225 // We can use the optimized fp16 to float and float to fp16 conversion routines 226 #define EIGEN_HAS_FP16_C 227#endif 228 229#if defined __CUDACC__ 230 #define EIGEN_VECTORIZE_CUDA 231 #include <vector_types.h> 232 #if defined __CUDACC_VER__ && __CUDACC_VER__ >= 70500 233 #define EIGEN_HAS_CUDA_FP16 234 #endif 235#endif 236 237#if defined EIGEN_HAS_CUDA_FP16 238 #include <host_defines.h> 239 #include <cuda_fp16.h> 240#endif 241 242#if (defined _OPENMP) && (!defined EIGEN_DONT_PARALLELIZE) 243 #define EIGEN_HAS_OPENMP 244#endif 245 246#ifdef EIGEN_HAS_OPENMP 247#include <omp.h> 248#endif 249 250// MSVC for windows mobile does not have the errno.h file 251#if !(EIGEN_COMP_MSVC && EIGEN_OS_WINCE) && !EIGEN_COMP_ARM 252#define EIGEN_HAS_ERRNO 253#endif 254 255#ifdef EIGEN_HAS_ERRNO 256#include <cerrno> 257#endif 258#include <cstddef> 259#include <cstdlib> 260#include <cmath> 261#include <cassert> 262#include <functional> 263#include <sstream> 264#include <iosfwd> 265#include <cstring> 266#include <string> 267#include <limits> 268#include <climits> // for CHAR_BIT 269// for min/max: 270#include <algorithm> 271 272// for std::is_nothrow_move_assignable 273#ifdef EIGEN_INCLUDE_TYPE_TRAITS 274#include <type_traits> 275#endif 276 277// for outputting debug info 278#ifdef EIGEN_DEBUG_ASSIGN 279#include <iostream> 280#endif 281 282// required for __cpuid, needs to be included after cmath 283#if EIGEN_COMP_MSVC && EIGEN_ARCH_i386_OR_x86_64 && !EIGEN_OS_WINCE 284 #include <intrin.h> 285#endif 286 287/** \brief Namespace containing all symbols from the %Eigen library. */ 288namespace Eigen { 289 290inline static const char *SimdInstructionSetsInUse(void) { 291#if defined(EIGEN_VECTORIZE_AVX512) 292 return "AVX512, FMA, AVX2, AVX, SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2"; 293#elif defined(EIGEN_VECTORIZE_AVX) 294 return "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2"; 295#elif defined(EIGEN_VECTORIZE_SSE4_2) 296 return "SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2"; 297#elif defined(EIGEN_VECTORIZE_SSE4_1) 298 return "SSE, SSE2, SSE3, SSSE3, SSE4.1"; 299#elif defined(EIGEN_VECTORIZE_SSSE3) 300 return "SSE, SSE2, SSE3, SSSE3"; 301#elif defined(EIGEN_VECTORIZE_SSE3) 302 return "SSE, SSE2, SSE3"; 303#elif defined(EIGEN_VECTORIZE_SSE2) 304 return "SSE, SSE2"; 305#elif defined(EIGEN_VECTORIZE_ALTIVEC) 306 return "AltiVec"; 307#elif defined(EIGEN_VECTORIZE_VSX) 308 return "VSX"; 309#elif defined(EIGEN_VECTORIZE_NEON) 310 return "ARM NEON"; 311#elif defined(EIGEN_VECTORIZE_ZVECTOR) 312 return "S390X ZVECTOR"; 313#else 314 return "None"; 315#endif 316} 317 318} // end namespace Eigen 319 320#if defined EIGEN2_SUPPORT_STAGE40_FULL_EIGEN3_STRICTNESS || defined EIGEN2_SUPPORT_STAGE30_FULL_EIGEN3_API || defined EIGEN2_SUPPORT_STAGE20_RESOLVE_API_CONFLICTS || defined EIGEN2_SUPPORT_STAGE10_FULL_EIGEN2_API || defined EIGEN2_SUPPORT 321// This will generate an error message: 322#error Eigen2-support is only available up to version 3.2. Please go to "http://eigen.tuxfamily.org/index.php?title=Eigen2" for further information 323#endif 324 325namespace Eigen { 326 327// we use size_t frequently and we'll never remember to prepend it with std:: everytime just to 328// ensure QNX/QCC support 329using std::size_t; 330// gcc 4.6.0 wants std:: for ptrdiff_t 331using std::ptrdiff_t; 332 333} 334 335/** \defgroup Core_Module Core module 336 * This is the main module of Eigen providing dense matrix and vector support 337 * (both fixed and dynamic size) with all the features corresponding to a BLAS library 338 * and much more... 339 * 340 * \code 341 * #include <Eigen/Core> 342 * \endcode 343 */ 344 345#include "src/Core/util/Constants.h" 346#include "src/Core/util/Meta.h" 347#include "src/Core/util/ForwardDeclarations.h" 348#include "src/Core/util/StaticAssert.h" 349#include "src/Core/util/XprHelper.h" 350#include "src/Core/util/Memory.h" 351 352#include "src/Core/NumTraits.h" 353#include "src/Core/MathFunctions.h" 354#include "src/Core/GenericPacketMath.h" 355#include "src/Core/MathFunctionsImpl.h" 356 357#if defined EIGEN_VECTORIZE_AVX512 358 #include "src/Core/arch/SSE/PacketMath.h" 359 #include "src/Core/arch/AVX/PacketMath.h" 360 #include "src/Core/arch/AVX512/PacketMath.h" 361 #include "src/Core/arch/AVX512/MathFunctions.h" 362#elif defined EIGEN_VECTORIZE_AVX 363 // Use AVX for floats and doubles, SSE for integers 364 #include "src/Core/arch/SSE/PacketMath.h" 365 #include "src/Core/arch/SSE/Complex.h" 366 #include "src/Core/arch/SSE/MathFunctions.h" 367 #include "src/Core/arch/AVX/PacketMath.h" 368 #include "src/Core/arch/AVX/MathFunctions.h" 369 #include "src/Core/arch/AVX/Complex.h" 370 #include "src/Core/arch/AVX/TypeCasting.h" 371#elif defined EIGEN_VECTORIZE_SSE 372 #include "src/Core/arch/SSE/PacketMath.h" 373 #include "src/Core/arch/SSE/MathFunctions.h" 374 #include "src/Core/arch/SSE/Complex.h" 375 #include "src/Core/arch/SSE/TypeCasting.h" 376#elif defined(EIGEN_VECTORIZE_ALTIVEC) || defined(EIGEN_VECTORIZE_VSX) 377 #include "src/Core/arch/AltiVec/PacketMath.h" 378 #include "src/Core/arch/AltiVec/MathFunctions.h" 379 #include "src/Core/arch/AltiVec/Complex.h" 380#elif defined EIGEN_VECTORIZE_NEON 381 #include "src/Core/arch/NEON/PacketMath.h" 382 #include "src/Core/arch/NEON/MathFunctions.h" 383 #include "src/Core/arch/NEON/Complex.h" 384#elif defined EIGEN_VECTORIZE_ZVECTOR 385 #include "src/Core/arch/ZVector/PacketMath.h" 386 #include "src/Core/arch/ZVector/MathFunctions.h" 387 #include "src/Core/arch/ZVector/Complex.h" 388#endif 389 390// Half float support 391#include "src/Core/arch/CUDA/Half.h" 392#include "src/Core/arch/CUDA/PacketMathHalf.h" 393#include "src/Core/arch/CUDA/TypeCasting.h" 394 395#if defined EIGEN_VECTORIZE_CUDA 396 #include "src/Core/arch/CUDA/PacketMath.h" 397 #include "src/Core/arch/CUDA/MathFunctions.h" 398#endif 399 400#include "src/Core/arch/Default/Settings.h" 401 402#include "src/Core/functors/TernaryFunctors.h" 403#include "src/Core/functors/BinaryFunctors.h" 404#include "src/Core/functors/UnaryFunctors.h" 405#include "src/Core/functors/NullaryFunctors.h" 406#include "src/Core/functors/StlFunctors.h" 407#include "src/Core/functors/AssignmentFunctors.h" 408 409// Specialized functors to enable the processing of complex numbers 410// on CUDA devices 411#include "src/Core/arch/CUDA/Complex.h" 412 413#include "src/Core/IO.h" 414#include "src/Core/DenseCoeffsBase.h" 415#include "src/Core/DenseBase.h" 416#include "src/Core/MatrixBase.h" 417#include "src/Core/EigenBase.h" 418 419#include "src/Core/Product.h" 420#include "src/Core/CoreEvaluators.h" 421#include "src/Core/AssignEvaluator.h" 422 423#ifndef EIGEN_PARSED_BY_DOXYGEN // work around Doxygen bug triggered by Assign.h r814874 424 // at least confirmed with Doxygen 1.5.5 and 1.5.6 425 #include "src/Core/Assign.h" 426#endif 427 428#include "src/Core/ArrayBase.h" 429#include "src/Core/util/BlasUtil.h" 430#include "src/Core/DenseStorage.h" 431#include "src/Core/NestByValue.h" 432 433// #include "src/Core/ForceAlignedAccess.h" 434 435#include "src/Core/ReturnByValue.h" 436#include "src/Core/NoAlias.h" 437#include "src/Core/PlainObjectBase.h" 438#include "src/Core/Matrix.h" 439#include "src/Core/Array.h" 440#include "src/Core/CwiseTernaryOp.h" 441#include "src/Core/CwiseBinaryOp.h" 442#include "src/Core/CwiseUnaryOp.h" 443#include "src/Core/CwiseNullaryOp.h" 444#include "src/Core/CwiseUnaryView.h" 445#include "src/Core/SelfCwiseBinaryOp.h" 446#include "src/Core/Dot.h" 447#include "src/Core/StableNorm.h" 448#include "src/Core/Stride.h" 449#include "src/Core/MapBase.h" 450#include "src/Core/Map.h" 451#include "src/Core/Ref.h" 452#include "src/Core/Block.h" 453#include "src/Core/VectorBlock.h" 454#include "src/Core/Transpose.h" 455#include "src/Core/DiagonalMatrix.h" 456#include "src/Core/Diagonal.h" 457#include "src/Core/DiagonalProduct.h" 458#include "src/Core/Redux.h" 459#include "src/Core/Visitor.h" 460#include "src/Core/Fuzzy.h" 461#include "src/Core/Swap.h" 462#include "src/Core/CommaInitializer.h" 463#include "src/Core/GeneralProduct.h" 464#include "src/Core/Solve.h" 465#include "src/Core/Inverse.h" 466#include "src/Core/SolverBase.h" 467#include "src/Core/PermutationMatrix.h" 468#include "src/Core/Transpositions.h" 469#include "src/Core/TriangularMatrix.h" 470#include "src/Core/SelfAdjointView.h" 471#include "src/Core/products/GeneralBlockPanelKernel.h" 472#include "src/Core/products/Parallelizer.h" 473#include "src/Core/ProductEvaluators.h" 474#include "src/Core/products/GeneralMatrixVector.h" 475#include "src/Core/products/GeneralMatrixMatrix.h" 476#include "src/Core/SolveTriangular.h" 477#include "src/Core/products/GeneralMatrixMatrixTriangular.h" 478#include "src/Core/products/SelfadjointMatrixVector.h" 479#include "src/Core/products/SelfadjointMatrixMatrix.h" 480#include "src/Core/products/SelfadjointProduct.h" 481#include "src/Core/products/SelfadjointRank2Update.h" 482#include "src/Core/products/TriangularMatrixVector.h" 483#include "src/Core/products/TriangularMatrixMatrix.h" 484#include "src/Core/products/TriangularSolverMatrix.h" 485#include "src/Core/products/TriangularSolverVector.h" 486#include "src/Core/BandMatrix.h" 487#include "src/Core/CoreIterators.h" 488#include "src/Core/ConditionEstimator.h" 489 490#include "src/Core/BooleanRedux.h" 491#include "src/Core/Select.h" 492#include "src/Core/VectorwiseOp.h" 493#include "src/Core/Random.h" 494#include "src/Core/Replicate.h" 495#include "src/Core/Reverse.h" 496#include "src/Core/ArrayWrapper.h" 497 498#ifdef EIGEN_USE_BLAS 499#include "src/Core/products/GeneralMatrixMatrix_BLAS.h" 500#include "src/Core/products/GeneralMatrixVector_BLAS.h" 501#include "src/Core/products/GeneralMatrixMatrixTriangular_BLAS.h" 502#include "src/Core/products/SelfadjointMatrixMatrix_BLAS.h" 503#include "src/Core/products/SelfadjointMatrixVector_BLAS.h" 504#include "src/Core/products/TriangularMatrixMatrix_BLAS.h" 505#include "src/Core/products/TriangularMatrixVector_BLAS.h" 506#include "src/Core/products/TriangularSolverMatrix_BLAS.h" 507#endif // EIGEN_USE_BLAS 508 509#ifdef EIGEN_USE_MKL_VML 510#include "src/Core/Assign_MKL.h" 511#endif 512 513#include "src/Core/GlobalFunctions.h" 514 515#include "src/Core/util/ReenableStupidWarnings.h" 516 517#endif // EIGEN_CORE_H 518