Home
last modified time | relevance | path

Searched refs:Inexact (Results 1 – 25 of 152) sorted by relevance

1234567

/external/python/cpython2/Lib/test/decimaltestdata/
Dpowersqrt.decTest26 -- x ** 0.5 (x>0) has no preferred exponent, and is Inexact
40 pwsx001 power 1 0.5 -> 1.00000000 Inexact Rounded
42 pwsx003 power 1.00 0.5 -> 1.00000000 Inexact Rounded
53 pwsx013 power 2 0.5 -> 1.41421356 Inexact Rounded
55 pwsx015 power 2.00 0.5 -> 1.41421356 Inexact Rounded
67 pwsx027 power 4.0 0.5 -> 2.00000000 Inexact Rounded
68 pwsx028 power 4.00 0.5 -> 2.00000000 Inexact Rounded
70 pwsx030 power +0.1 0.5 -> 0.316227766 Inexact Rounded
72 pwsx032 power +0.01 0.5 -> 0.100000000 Inexact Rounded
74 pwsx034 power +0.001 0.5 -> 0.0316227766 Inexact Rounded
[all …]
Dsquareroot.decTest42 sqtx013 squareroot 2 -> 1.41421356 Inexact Rounded
44 sqtx015 squareroot 2.00 -> 1.41421356 Inexact Rounded
59 sqtx030 squareroot +0.1 -> 0.316227766 Inexact Rounded
63 sqtx034 squareroot +0.001 -> 0.0316227766 Inexact Rounded
70 sqtx041 squareroot 1.1 -> 1.04880885 Inexact Rounded
71 sqtx042 squareroot 1.10 -> 1.04880885 Inexact Rounded
72 sqtx043 squareroot 1.100 -> 1.04880885 Inexact Rounded
73 sqtx044 squareroot 1.110 -> 1.05356538 Inexact Rounded
78 sqtx049 squareroot 9.9 -> 3.14642654 Inexact Rounded
79 sqtx050 squareroot 9.90 -> 3.14642654 Inexact Rounded
[all …]
Drounding.decTest43 radx100 add 12345 -0.1 -> 12344 Inexact Rounded
44 radx101 add 12345 -0.01 -> 12344 Inexact Rounded
45 radx102 add 12345 -0.001 -> 12344 Inexact Rounded
46 radx103 add 12345 -0.00001 -> 12344 Inexact Rounded
47 radx104 add 12345 -0.000001 -> 12344 Inexact Rounded
48 radx105 add 12345 -0.0000001 -> 12344 Inexact Rounded
50 radx107 add 12345 0.0000001 -> 12345 Inexact Rounded
51 radx108 add 12345 0.000001 -> 12345 Inexact Rounded
52 radx109 add 12345 0.00001 -> 12345 Inexact Rounded
53 radx110 add 12345 0.0001 -> 12345 Inexact Rounded
[all …]
Dln.decTest32 lnxs003 ln 2.71828183 -> 1.00000000 Inexact Rounded
33 lnxs004 ln 10 -> 2.30258509 Inexact Rounded
40 lnx0002 ln 1E-9 -> -20.72326583694641 Inexact Rounded
41 lnx0003 ln 0.0007 -> -7.264430222920869 Inexact Rounded
42 lnx0004 ln 0.1 -> -2.302585092994046 Inexact Rounded
43 lnx0005 ln 0.7 -> -0.3566749439387324 Inexact Rounded
46 lnx0008 ln 1.5 -> 0.4054651081081644 Inexact Rounded
47 lnx0009 ln 2 -> 0.6931471805599453 Inexact Rounded
48 lnx0010 ln 2.718281828459045 -> 0.9999999999999999 Inexact Rounded
49 lnx0011 ln 2.718281828459046 -> 1.000000000000000 Inexact Rounded
[all …]
Dexp.decTest33 expx002 exp -10 -> 0.0000453999298 Inexact Rounded
34 expx003 exp -1 -> 0.367879441 Inexact Rounded
37 expx006 exp 1 -> 2.71828183 Inexact Rounded
38 expx007 exp 0.693147181 -> 2.00000000 Inexact Rounded
39 expx008 exp 10 -> 22026.4658 Inexact Rounded
44 expx011 exp 0.1 -> 1.105171 Inexact Rounded
45 expx012 exp 0.01 -> 1.010050 Inexact Rounded
46 expx013 exp 0.001 -> 1.001001 Inexact Rounded
47 expx014 exp 0.0001 -> 1.000100 Inexact Rounded
48 expx015 exp 0.00001 -> 1.000010 Inexact Rounded
[all …]
Drandoms.decTest30 xadd001 add 905.67402 -202896611.E-780472620 -> 905.674020 Inexact Rounded
32 xdiv001 divide 905.67402 -202896611.E-780472620 -> -4.46372177E+780472614 Inexact Rounded
34 xmul001 multiply 905.67402 -202896611.E-780472620 -> -1.83758189E-780472609 Inexact Rounded
35 xpow001 power 905.67402 -2 -> 0.00000121914730 Inexact Rounded
37 xsub001 subtract 905.67402 -202896611.E-780472620 -> 905.674020 Inexact Rounded
38 xadd002 add 3915134.7 -597164907. -> -593249772 Inexact Rounded
40 xdiv002 divide 3915134.7 -597164907. -> -0.00655620358 Inexact Rounded
42 xmul002 multiply 3915134.7 -597164907. -> -2.33798105E+15 Inexact Rounded
43 xpow002 power 3915134.7 -597164907 -> 0E-1000000007 Underflow Subnormal Inexact Rounded Clamped
45 xsub002 subtract 3915134.7 -597164907. -> 601080042 Inexact Rounded
[all …]
Dlog10.decTest36 logxs003 log10 2 -> 0.301029996 Inexact Rounded
38 logxs005 log10 70 -> 1.84509804 Inexact Rounded
45 logx0001 log10 7E-1000 -> -999.1549019599857 Inexact Rounded
46 logx0002 log10 1.1E-9 -> -8.958607314841775 Inexact Rounded
47 logx0003 log10 0.0007 -> -3.154901959985743 Inexact Rounded
48 logx0004 log10 0.11 -> -0.9586073148417750 Inexact Rounded
49 logx0005 log10 0.7 -> -0.1549019599857432 Inexact Rounded
51 logx0007 log10 1.5 -> 0.1760912590556812 Inexact Rounded
52 logx0008 log10 2 -> 0.3010299956639812 Inexact Rounded
53 logx0009 log10 2.718281828459045 -> 0.4342944819032518 Inexact Rounded
[all …]
DrandomBound32.decTest32 …2579 0218.932010396534371704930714860E+797 -> 2.189320103965343717049307148600E+799 Inexact Rounded
34 …2579 0218.932010396534371704930714860E+797 -> 2.262681764507965005284080800438E-787 Inexact Rounded
36 …2579 0218.932010396534371704930714860E+797 -> 1.084531091568672041923151632066E+812 Inexact Rounded
37 powx3001 power 4953734675913.065314738743322579 2 -> 24539487239343522246155890.99495 Inexact Round…
39 …579 0218.932010396534371704930714860E+797 -> -2.189320103965343717049307148600E+799 Inexact Rounded
40 …44 -78864532047.12287484430980636798E+934 -> -7.886453204712287484430980636798E+944 Inexact Rounded
42 …4 -78864532047.12287484430980636798E+934 -> -1.222562801441069667849402782716E-1785 Inexact Rounded
44 …44 -78864532047.12287484430980636798E+934 -> -7.603869223099928141659831589905E+104 Inexact Rounded
45 … 9641.684323386955881595490347910E-844 -8 -> 1.338988152067180337738955757587E+6720 Inexact Rounded
47 …844 -78864532047.12287484430980636798E+934 -> 7.886453204712287484430980636798E+944 Inexact Rounded
[all …]
Dpower.decTest60 powx027 power '2' '31' -> '2.14748365E+9' Inexact Rounded
62 powx028 power '2' '32' -> '4.29496730E+9' Inexact Rounded
128 powx093 power 101 5 -> 1.05101005E+10 Inexact Rounded
129 powx094 power 101 6 -> 1.06152015E+12 Inexact Rounded
130 powx095 power 101 7 -> 1.07213535E+14 Inexact Rounded
134 powx100 power '3' '-1' -> 0.333333333 Inexact Rounded
139 powx105 power '2' '-16' -> 0.0000152587891 Inexact Rounded
140 powx106 power '2' '-32' -> 2.32830644E-10 Inexact Rounded
141 powx108 power '2' '-64' -> 5.42101086E-20 Inexact Rounded
190 powx219 power '123456789E+10' '-1.23000e+2' -> '5.54E-2226' Inexact Rounded
[all …]
DddAdd.decTest44 ddadd011 add '0.4444444444444446' '0.5555555555555555' -> '1.000000000000000' Inexact Rounded
47 ddadd014 add '4444444444444444' '0.49' -> '4444444444444444' Inexact Rounded
48 ddadd015 add '4444444444444444' '0.499' -> '4444444444444444' Inexact Rounded
49 ddadd016 add '4444444444444444' '0.4999' -> '4444444444444444' Inexact Rounded
50 ddadd017 add '4444444444444444' '0.5000' -> '4444444444444444' Inexact Rounded
51 ddadd018 add '4444444444444444' '0.5001' -> '4444444444444445' Inexact Rounded
52 ddadd019 add '4444444444444444' '0.501' -> '4444444444444445' Inexact Rounded
53 ddadd020 add '4444444444444444' '0.51' -> '4444444444444445' Inexact Rounded
72 ddadd035 add '70' '10000e+16' -> '1.000000000000000E+20' Inexact Rounded
73 ddadd036 add '700' '10000e+16' -> '1.000000000000000E+20' Inexact Rounded
[all …]
Dinexact.decTest33 inx006 add 123456789 0.1 -> 123456789 Inexact Rounded
34 inx007 add 123456789 0.01 -> 123456789 Inexact Rounded
35 inx008 add 123456789 0.001 -> 123456789 Inexact Rounded
36 inx009 add 123456789 0.000001 -> 123456789 Inexact Rounded
37 inx010 add 123456789 0.000000001 -> 123456789 Inexact Rounded
38 inx011 add 123456789 0.000000000001 -> 123456789 Inexact Rounded
40 inx012 add 123456789 0.9 -> 123456790 Inexact Rounded
41 inx013 add 123456789 0.09 -> 123456789 Inexact Rounded
42 inx014 add 123456789 0.009 -> 123456789 Inexact Rounded
43 inx015 add 123456789 0.000009 -> 123456789 Inexact Rounded
[all …]
DddQuantize.decTest36 ddqua003 quantize 0.1 1e+2 -> 0E+2 Inexact Rounded
37 ddqua005 quantize 0.1 1e+1 -> 0E+1 Inexact Rounded
38 ddqua006 quantize 0.1 1e0 -> 0 Inexact Rounded
42 ddqua010 quantize 0.9 1e+2 -> 0E+2 Inexact Rounded
43 ddqua011 quantize 0.9 1e+1 -> 0E+1 Inexact Rounded
44 ddqua012 quantize 0.9 1e+0 -> 1 Inexact Rounded
51 ddqua023 quantize -0.1 1e+2 -> -0E+2 Inexact Rounded
52 ddqua025 quantize -0.1 1e+1 -> -0E+1 Inexact Rounded
53 ddqua026 quantize -0.1 1e0 -> -0 Inexact Rounded
57 ddqua030 quantize -0.9 1e+2 -> -0E+2 Inexact Rounded
[all …]
DdqQuantize.decTest38 dqqua003 quantize 0.1 1e+2 -> 0E+2 Inexact Rounded
39 dqqua005 quantize 0.1 1e+1 -> 0E+1 Inexact Rounded
40 dqqua006 quantize 0.1 1e0 -> 0 Inexact Rounded
44 dqqua010 quantize 0.9 1e+2 -> 0E+2 Inexact Rounded
45 dqqua011 quantize 0.9 1e+1 -> 0E+1 Inexact Rounded
46 dqqua012 quantize 0.9 1e+0 -> 1 Inexact Rounded
53 dqqua023 quantize -0.1 1e+2 -> -0E+2 Inexact Rounded
54 dqqua025 quantize -0.1 1e+1 -> -0E+1 Inexact Rounded
55 dqqua026 quantize -0.1 1e0 -> -0 Inexact Rounded
59 dqqua030 quantize -0.9 1e+2 -> -0E+2 Inexact Rounded
[all …]
/external/python/cpython3/Lib/test/decimaltestdata/
Dpowersqrt.decTest26 -- x ** 0.5 (x>0) has no preferred exponent, and is Inexact
40 pwsx001 power 1 0.5 -> 1.00000000 Inexact Rounded
42 pwsx003 power 1.00 0.5 -> 1.00000000 Inexact Rounded
53 pwsx013 power 2 0.5 -> 1.41421356 Inexact Rounded
55 pwsx015 power 2.00 0.5 -> 1.41421356 Inexact Rounded
67 pwsx027 power 4.0 0.5 -> 2.00000000 Inexact Rounded
68 pwsx028 power 4.00 0.5 -> 2.00000000 Inexact Rounded
70 pwsx030 power +0.1 0.5 -> 0.316227766 Inexact Rounded
72 pwsx032 power +0.01 0.5 -> 0.100000000 Inexact Rounded
74 pwsx034 power +0.001 0.5 -> 0.0316227766 Inexact Rounded
[all …]
Dsquareroot.decTest42 sqtx013 squareroot 2 -> 1.41421356 Inexact Rounded
44 sqtx015 squareroot 2.00 -> 1.41421356 Inexact Rounded
59 sqtx030 squareroot +0.1 -> 0.316227766 Inexact Rounded
63 sqtx034 squareroot +0.001 -> 0.0316227766 Inexact Rounded
70 sqtx041 squareroot 1.1 -> 1.04880885 Inexact Rounded
71 sqtx042 squareroot 1.10 -> 1.04880885 Inexact Rounded
72 sqtx043 squareroot 1.100 -> 1.04880885 Inexact Rounded
73 sqtx044 squareroot 1.110 -> 1.05356538 Inexact Rounded
78 sqtx049 squareroot 9.9 -> 3.14642654 Inexact Rounded
79 sqtx050 squareroot 9.90 -> 3.14642654 Inexact Rounded
[all …]
Drounding.decTest43 radx100 add 12345 -0.1 -> 12344 Inexact Rounded
44 radx101 add 12345 -0.01 -> 12344 Inexact Rounded
45 radx102 add 12345 -0.001 -> 12344 Inexact Rounded
46 radx103 add 12345 -0.00001 -> 12344 Inexact Rounded
47 radx104 add 12345 -0.000001 -> 12344 Inexact Rounded
48 radx105 add 12345 -0.0000001 -> 12344 Inexact Rounded
50 radx107 add 12345 0.0000001 -> 12345 Inexact Rounded
51 radx108 add 12345 0.000001 -> 12345 Inexact Rounded
52 radx109 add 12345 0.00001 -> 12345 Inexact Rounded
53 radx110 add 12345 0.0001 -> 12345 Inexact Rounded
[all …]
Dln.decTest32 lnxs003 ln 2.71828183 -> 1.00000000 Inexact Rounded
33 lnxs004 ln 10 -> 2.30258509 Inexact Rounded
40 lnx0002 ln 1E-9 -> -20.72326583694641 Inexact Rounded
41 lnx0003 ln 0.0007 -> -7.264430222920869 Inexact Rounded
42 lnx0004 ln 0.1 -> -2.302585092994046 Inexact Rounded
43 lnx0005 ln 0.7 -> -0.3566749439387324 Inexact Rounded
46 lnx0008 ln 1.5 -> 0.4054651081081644 Inexact Rounded
47 lnx0009 ln 2 -> 0.6931471805599453 Inexact Rounded
48 lnx0010 ln 2.718281828459045 -> 0.9999999999999999 Inexact Rounded
49 lnx0011 ln 2.718281828459046 -> 1.000000000000000 Inexact Rounded
[all …]
Dexp.decTest33 expx002 exp -10 -> 0.0000453999298 Inexact Rounded
34 expx003 exp -1 -> 0.367879441 Inexact Rounded
37 expx006 exp 1 -> 2.71828183 Inexact Rounded
38 expx007 exp 0.693147181 -> 2.00000000 Inexact Rounded
39 expx008 exp 10 -> 22026.4658 Inexact Rounded
44 expx011 exp 0.1 -> 1.105171 Inexact Rounded
45 expx012 exp 0.01 -> 1.010050 Inexact Rounded
46 expx013 exp 0.001 -> 1.001001 Inexact Rounded
47 expx014 exp 0.0001 -> 1.000100 Inexact Rounded
48 expx015 exp 0.00001 -> 1.000010 Inexact Rounded
[all …]
Drandoms.decTest30 xadd001 add 905.67402 -202896611.E-780472620 -> 905.674020 Inexact Rounded
32 xdiv001 divide 905.67402 -202896611.E-780472620 -> -4.46372177E+780472614 Inexact Rounded
34 xmul001 multiply 905.67402 -202896611.E-780472620 -> -1.83758189E-780472609 Inexact Rounded
35 xpow001 power 905.67402 -2 -> 0.00000121914730 Inexact Rounded
37 xsub001 subtract 905.67402 -202896611.E-780472620 -> 905.674020 Inexact Rounded
38 xadd002 add 3915134.7 -597164907. -> -593249772 Inexact Rounded
40 xdiv002 divide 3915134.7 -597164907. -> -0.00655620358 Inexact Rounded
42 xmul002 multiply 3915134.7 -597164907. -> -2.33798105E+15 Inexact Rounded
43 xpow002 power 3915134.7 -597164907 -> 0E-1000000007 Underflow Subnormal Inexact Rounded Clamped
45 xsub002 subtract 3915134.7 -597164907. -> 601080042 Inexact Rounded
[all …]
Dlog10.decTest36 logxs003 log10 2 -> 0.301029996 Inexact Rounded
38 logxs005 log10 70 -> 1.84509804 Inexact Rounded
45 logx0001 log10 7E-1000 -> -999.1549019599857 Inexact Rounded
46 logx0002 log10 1.1E-9 -> -8.958607314841775 Inexact Rounded
47 logx0003 log10 0.0007 -> -3.154901959985743 Inexact Rounded
48 logx0004 log10 0.11 -> -0.9586073148417750 Inexact Rounded
49 logx0005 log10 0.7 -> -0.1549019599857432 Inexact Rounded
51 logx0007 log10 1.5 -> 0.1760912590556812 Inexact Rounded
52 logx0008 log10 2 -> 0.3010299956639812 Inexact Rounded
53 logx0009 log10 2.718281828459045 -> 0.4342944819032518 Inexact Rounded
[all …]
DrandomBound32.decTest32 …2579 0218.932010396534371704930714860E+797 -> 2.189320103965343717049307148600E+799 Inexact Rounded
34 …2579 0218.932010396534371704930714860E+797 -> 2.262681764507965005284080800438E-787 Inexact Rounded
36 …2579 0218.932010396534371704930714860E+797 -> 1.084531091568672041923151632066E+812 Inexact Rounded
37 powx3001 power 4953734675913.065314738743322579 2 -> 24539487239343522246155890.99495 Inexact Round…
39 …579 0218.932010396534371704930714860E+797 -> -2.189320103965343717049307148600E+799 Inexact Rounded
40 …44 -78864532047.12287484430980636798E+934 -> -7.886453204712287484430980636798E+944 Inexact Rounded
42 …4 -78864532047.12287484430980636798E+934 -> -1.222562801441069667849402782716E-1785 Inexact Rounded
44 …44 -78864532047.12287484430980636798E+934 -> -7.603869223099928141659831589905E+104 Inexact Rounded
45 … 9641.684323386955881595490347910E-844 -8 -> 1.338988152067180337738955757587E+6720 Inexact Rounded
47 …844 -78864532047.12287484430980636798E+934 -> 7.886453204712287484430980636798E+944 Inexact Rounded
[all …]
Dpower.decTest60 powx027 power '2' '31' -> '2.14748365E+9' Inexact Rounded
62 powx028 power '2' '32' -> '4.29496730E+9' Inexact Rounded
128 powx093 power 101 5 -> 1.05101005E+10 Inexact Rounded
129 powx094 power 101 6 -> 1.06152015E+12 Inexact Rounded
130 powx095 power 101 7 -> 1.07213535E+14 Inexact Rounded
134 powx100 power '3' '-1' -> 0.333333333 Inexact Rounded
139 powx105 power '2' '-16' -> 0.0000152587891 Inexact Rounded
140 powx106 power '2' '-32' -> 2.32830644E-10 Inexact Rounded
141 powx108 power '2' '-64' -> 5.42101086E-20 Inexact Rounded
190 powx219 power '123456789E+10' '-1.23000e+2' -> '5.54E-2226' Inexact Rounded
[all …]
DddAdd.decTest44 ddadd011 add '0.4444444444444446' '0.5555555555555555' -> '1.000000000000000' Inexact Rounded
47 ddadd014 add '4444444444444444' '0.49' -> '4444444444444444' Inexact Rounded
48 ddadd015 add '4444444444444444' '0.499' -> '4444444444444444' Inexact Rounded
49 ddadd016 add '4444444444444444' '0.4999' -> '4444444444444444' Inexact Rounded
50 ddadd017 add '4444444444444444' '0.5000' -> '4444444444444444' Inexact Rounded
51 ddadd018 add '4444444444444444' '0.5001' -> '4444444444444445' Inexact Rounded
52 ddadd019 add '4444444444444444' '0.501' -> '4444444444444445' Inexact Rounded
53 ddadd020 add '4444444444444444' '0.51' -> '4444444444444445' Inexact Rounded
72 ddadd035 add '70' '10000e+16' -> '1.000000000000000E+20' Inexact Rounded
73 ddadd036 add '700' '10000e+16' -> '1.000000000000000E+20' Inexact Rounded
[all …]
Dinexact.decTest33 inx006 add 123456789 0.1 -> 123456789 Inexact Rounded
34 inx007 add 123456789 0.01 -> 123456789 Inexact Rounded
35 inx008 add 123456789 0.001 -> 123456789 Inexact Rounded
36 inx009 add 123456789 0.000001 -> 123456789 Inexact Rounded
37 inx010 add 123456789 0.000000001 -> 123456789 Inexact Rounded
38 inx011 add 123456789 0.000000000001 -> 123456789 Inexact Rounded
40 inx012 add 123456789 0.9 -> 123456790 Inexact Rounded
41 inx013 add 123456789 0.09 -> 123456789 Inexact Rounded
42 inx014 add 123456789 0.009 -> 123456789 Inexact Rounded
43 inx015 add 123456789 0.000009 -> 123456789 Inexact Rounded
[all …]
DddQuantize.decTest36 ddqua003 quantize 0.1 1e+2 -> 0E+2 Inexact Rounded
37 ddqua005 quantize 0.1 1e+1 -> 0E+1 Inexact Rounded
38 ddqua006 quantize 0.1 1e0 -> 0 Inexact Rounded
42 ddqua010 quantize 0.9 1e+2 -> 0E+2 Inexact Rounded
43 ddqua011 quantize 0.9 1e+1 -> 0E+1 Inexact Rounded
44 ddqua012 quantize 0.9 1e+0 -> 1 Inexact Rounded
51 ddqua023 quantize -0.1 1e+2 -> -0E+2 Inexact Rounded
52 ddqua025 quantize -0.1 1e+1 -> -0E+1 Inexact Rounded
53 ddqua026 quantize -0.1 1e0 -> -0 Inexact Rounded
57 ddqua030 quantize -0.9 1e+2 -> -0E+2 Inexact Rounded
[all …]

1234567