| /device/linaro/bootloader/edk2/AppPkg/Applications/Python/Python-2.7.2/Lib/test/decimaltestdata/ |
| D | ln.decTest | 23 precision: 16
29 precision: 9
38 precision: 16
87 -- lower precision basics
143 -- group low-precision ln(1)s:
144 precision: 1
146 precision: 2
148 precision: 3
150 precision: 4
152 precision: 5
[all …]
|
| D | log10.decTest | 26 precision: 16
32 precision: 9
43 precision: 16
126 precision: 1
135 precision: 2
157 precision: 3
167 precision: 50
171 precision: 49
173 precision: 48
175 precision: 47
[all …]
|
| D | exp.decTest | 26 precision: 9
43 precision: 7
72 precision: 7
86 precision: 16
90 precision: 17
92 precision: 18
94 precision: 19
96 precision: 20
100 precision: 50
102 precision: 31
[all …]
|
| D | multiply.decTest | 23 precision: 9
45 precision: 6
53 precision: 15
55 precision: 30
57 precision: 9
103 precision: 9
109 precision: 8
112 precision: 7
116 precision: 9
122 precision: 8
[all …]
|
| D | divide.decTest | 23 precision: 9
95 precision: 6
106 precision: 9
151 precision: 9
228 precision: 7
269 precision: 9
298 precision: 15
348 precision: 7
365 precision: 7
372 precision: 9
[all …]
|
| D | abs.decTest | 26 precision: 9
54 precision: 7
57 precision: 6
60 precision: 3
76 precision: 9
83 precision: 9
93 precision: 3
97 precision: 3
129 precision: 9
137 precision: 15
[all …]
|
| D | power.decTest | 27 precision: 16
59 precision: 9
63 precision: 10
66 precision: 9
180 -- The Vienna case. Checks both setup and 1/acc working precision
187 precision: 3
195 precision: 7
230 precision: 9
248 precision: 15
261 precision: 5
[all …]
|
| D | plus.decTest | 26 precision: 9
53 precision: 7
56 precision: 6
59 precision: 3
64 precision: 9
124 precision: 3
129 precision: 3
159 precision: 16
175 precision: 9
184 precision: 15
[all …]
|
| D | inexact.decTest | 23 precision: 9
112 precision: 8
124 precision: 4
146 precision: 5
156 precision: 3
166 precision: 2
176 precision: 1
188 precision: 4
198 precision: 4
208 precision: 9
|
| D | subtract.decTest | 23 precision: 9
92 -- change precision
94 precision: 6
96 precision: 9
151 precision: 3
154 precision: 6
157 precision: 9
160 precision: 12
163 precision: 15
166 precision: 9
[all …]
|
| D | remainderNear.decTest | 23 precision: 9
99 precision: 6
110 precision: 9
119 precision: 9
347 precision: 6
349 precision: 3
352 precision: 5
372 precision: 9
387 precision: 9
389 precision: 8
[all …]
|
| D | fma.decTest | 23 precision: 9
144 precision: 6
152 precision: 15
154 precision: 30
156 precision: 9
202 precision: 9
208 precision: 8
211 precision: 7
215 precision: 9
221 precision: 8
[all …]
|
| D | minus.decTest | 26 precision: 9
72 precision: 7
75 precision: 6
78 precision: 3
83 precision: 9
113 precision: 3
118 precision: 3
151 precision: 9
159 precision: 15
|
| /device/linaro/bootloader/edk2/AppPkg/Applications/Python/Python-2.7.2/Python/ |
| D | pystrtod.c | 528 ensure_decimal_point(char* buffer, size_t buf_size, int precision) in ensure_decimal_point() argument
553 assert(precision == -1); in ensure_decimal_point()
561 if (digit_count == precision) { in ensure_decimal_point()
573 assert(precision == -1 || digit_count < precision); in ensure_decimal_point()
644 int precision) in _PyOS_ascii_formatd() argument
714 buffer = ensure_decimal_point(buffer, buf_size, precision); in _PyOS_ascii_formatd()
739 int precision, in PyOS_double_to_string() argument
769 if (precision != 0) { in PyOS_double_to_string()
779 precision = 17; in PyOS_double_to_string()
829 bufsize = 25 + precision; in PyOS_double_to_string()
[all …]
|
| /device/linaro/bootloader/edk2/AppPkg/Applications/Python/Python-2.7.10/Python/ |
| D | pystrtod.c | 528 ensure_decimal_point(char* buffer, size_t buf_size, int precision) in ensure_decimal_point() argument
553 assert(precision == -1); in ensure_decimal_point()
561 if (digit_count == precision) { in ensure_decimal_point()
573 assert(precision == -1 || digit_count < precision); in ensure_decimal_point()
644 int precision) in _PyOS_ascii_formatd() argument
714 buffer = ensure_decimal_point(buffer, buf_size, precision); in _PyOS_ascii_formatd()
739 int precision, in PyOS_double_to_string() argument
769 if (precision != 0) { in PyOS_double_to_string()
779 precision = 17; in PyOS_double_to_string()
829 bufsize = 25 + precision; in PyOS_double_to_string()
[all …]
|
| /device/linaro/bootloader/edk2/AppPkg/Applications/Python/Python-2.7.2/Lib/test/ |
| D | formatfloat_testcases.txt | 1 -- 'f' code formatting, with explicit precision (>= 0). Output always
12 -- precision 0; result should never include a .
28 -- precision 1
44 -- precision 2
61 -- makes a difference when the precision is 0.
71 -- if precision is omitted it defaults to 6
92 -- 'e' code formatting with explicit precision (>= 0). Output should
103 -- precision 0. no decimal point in the output
137 -- precision 1
149 -- precision 2
[all …]
|
| /device/linaro/bootloader/edk2/StdLib/LibC/Softfloat/templates/ |
| D | softfloat-specialize | 67 The pattern for a default generated single-precision NaN.
74 Returns 1 if the single-precision floating-point value `a' is a NaN;
87 Returns 1 if the single-precision floating-point value `a' is a signaling
100 Returns the result of converting the single-precision floating-point NaN
120 precision floating-point format.
132 Takes two single-precision floating-point values `a' and `b', one of which
159 The pattern for a default generated double-precision NaN.
166 Returns 1 if the double-precision floating-point value `a' is a NaN;
179 Returns 1 if the double-precision floating-point value `a' is a signaling
194 Returns the result of converting the double-precision floating-point NaN
[all …]
|
| /device/linaro/bootloader/edk2/StdLib/LibC/Softfloat/ |
| D | timesoftfloat.txt | 14 mode, tininess mode, and/or rounding precision.
69 precision operations affected by rounding precision control, `timesoftfloat'
70 also times the function for all three rounding precision modes, one after
72 mode, a single tininess mode, and/or a single rounding precision with
88 operations, particularly for extended double precision (`floatx80') and
89 quadruple precision (`float128'). This is inherent to the remainder
108 For extended double-precision functions affected by rounding precision
110 in which rounding precision is equivalent to single precision. The other
111 rounding precision options are not timed. Likewise, the `-precision64'
112 and `-precision80' options fix the rounding precision equivalent to double
[all …]
|
| D | softfloat-specialize | 117 The pattern for a default generated single-precision NaN.
124 Returns 1 if the single-precision floating-point value `a' is a NaN;
140 Returns 1 if the single-precision floating-point value `a' is a signaling
157 Returns the result of converting the single-precision floating-point NaN
177 precision floating-point format.
189 Takes two single-precision floating-point values `a' and `b', one of which
216 The pattern for a default generated double-precision NaN.
223 Returns 1 if the double-precision floating-point value `a' is a NaN;
240 Returns 1 if the double-precision floating-point value `a' is a signaling
259 Returns the result of converting the double-precision floating-point NaN
[all …]
|
| D | softfloat.txt | 14 formats are supported: single precision, double precision, extended double
15 precision, and quadruple precision. All operations required by the standard
33 Support for the extended double-precision and quadruple-precision formats
37 references in this document to the extended double precision, quadruple
38 precision, and 64-bit integers should be ignored.
86 file defines four types: `float32' (single precision), `float64' (double
87 precision), `floatx80' (extended double precision), and `float128'
88 (quadruple precision). The `float32' and `float64' types are defined in
146 For extended double precision (`floatx80') only, the rounding precision
153 operations are rounded (as usual) to the full precision of the extended
[all …]
|
| /device/linaro/bootloader/edk2/ArmPkg/Library/ArmSoftFloatLib/ |
| D | softfloat-specialize | 113 The pattern for a default generated single-precision NaN.
120 Returns 1 if the single-precision floating-point value `a' is a NaN;
136 Returns 1 if the single-precision floating-point value `a' is a signaling
153 Returns the result of converting the single-precision floating-point NaN
173 precision floating-point format.
185 Takes two single-precision floating-point values `a' and `b', one of which
212 The pattern for a default generated double-precision NaN.
219 Returns 1 if the double-precision floating-point value `a' is a NaN;
236 Returns 1 if the double-precision floating-point value `a' is a signaling
255 Returns the result of converting the double-precision floating-point NaN
[all …]
|
| /device/linaro/bootloader/edk2/AppPkg/Applications/Python/Python-2.7.2/Lib/ |
| D | timeit.py | 265 precision = 3
281 precision += 1
303 print "%d loops -> %.*g secs" % (number, precision, x)
313 print "raw times:", " ".join(["%.*g" % (precision, x) for x in r])
317 print "best of %d: %.*g usec per loop" % (repeat, precision, usec)
321 print "best of %d: %.*g msec per loop" % (repeat, precision, msec)
324 print "best of %d: %.*g sec per loop" % (repeat, precision, sec)
|
| /device/linaro/bootloader/edk2/AppPkg/Applications/Python/Python-2.7.10/Objects/stringlib/ |
| D | formatter.h | 138 Py_ssize_t precision; member
155 printf("internal format spec: precision %zd\n", format->precision);
191 format->precision = -1; in parse_internal_render_format_spec()
253 consumed = get_integer(&ptr, end, &format->precision); in parse_internal_render_format_spec()
707 if (format->precision >= 0 && len >= format->precision) { in format_string_internal()
708 len = format->precision; in format_string_internal()
762 if (format->precision != -1) { in format_int_or_long_internal()
931 Py_ssize_t precision; in format_float_internal() local
950 if (format->precision > INT_MAX) { in format_float_internal()
954 precision = (int)format->precision; in format_float_internal()
[all …]
|
| /device/google/contexthub/firmware/os/core/ |
| D | printf.c | 32 uint32_t precision; member
51 if (data->precision > sizeof(buf) - 1) in StrPrvPrintfEx_number()
52 data->precision = sizeof(buf) - 1; in StrPrvPrintfEx_number()
92 while (data->precision > numPrinted) { in StrPrvPrintfEx_number()
219 data.precision = 0; in cvprintf()
247 if (data.precision) in cvprintf()
248 len = StrVPrintf_StrLen_withMax(str,data.precision); in cvprintf()
301 data.precision = (data.precision * 10) + c - '0'; in cvprintf()
|
| /device/linaro/bootloader/edk2/AppPkg/Applications/Python/Python-2.7.2/Objects/stringlib/ |
| D | formatter.h | 140 Py_ssize_t precision; member
157 printf("internal format spec: precision %zd\n", format->precision);
192 format->precision = -1; in parse_internal_render_format_spec()
253 consumed = get_integer(&ptr, end, &format->precision); in parse_internal_render_format_spec()
707 if (format->precision >= 0 && len >= format->precision) { in format_string_internal()
708 len = format->precision; in format_string_internal()
763 if (format->precision != -1) { in format_int_or_long_internal()
933 Py_ssize_t precision = format->precision; in format_float_internal() local
983 if (precision < 0) in format_float_internal()
984 precision = default_precision; in format_float_internal()
[all …]
|