1/*
2 * Copyright (c) 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a copy
5 * of this software and associated documentation files (the "Software"), to deal
6 * in the Software without restriction, including without limitation the rights
7 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 * copies of the Software, and to permit persons to whom the Software is
9 * furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20 * THE SOFTWARE.
21 */
22
23#include <clc/clc.h>
24
25#include "config.h"
26#include "math.h"
27#include "../clcmacro.h"
28
29struct fp {
30	ulong mantissa;
31	int exponent;
32	uint sign;
33};
34
35_CLC_DEF _CLC_OVERLOAD float __clc_sw_fma(float a, float b, float c)
36{
37	/* special cases */
38	if (isnan(a) || isnan(b) || isnan(c) || isinf(a) || isinf(b))
39		return mad(a, b, c);
40
41	/* If only c is inf, and both a,b are regular numbers, the result is c*/
42	if (isinf(c))
43		return c;
44
45	a = __clc_flush_denormal_if_not_supported(a);
46	b = __clc_flush_denormal_if_not_supported(b);
47	c = __clc_flush_denormal_if_not_supported(c);
48
49	if (c == 0)
50		return a * b;
51
52	struct fp st_a, st_b, st_c;
53
54	st_a.exponent = a == .0f ? 0 : ((as_uint(a) & 0x7f800000) >> 23) - 127;
55	st_b.exponent = b == .0f ? 0 : ((as_uint(b) & 0x7f800000) >> 23) - 127;
56	st_c.exponent = c == .0f ? 0 : ((as_uint(c) & 0x7f800000) >> 23) - 127;
57
58	st_a.mantissa = a == .0f ? 0 : (as_uint(a) & 0x7fffff) | 0x800000;
59	st_b.mantissa = b == .0f ? 0 : (as_uint(b) & 0x7fffff) | 0x800000;
60	st_c.mantissa = c == .0f ? 0 : (as_uint(c) & 0x7fffff) | 0x800000;
61
62	st_a.sign = as_uint(a) & 0x80000000;
63	st_b.sign = as_uint(b) & 0x80000000;
64	st_c.sign = as_uint(c) & 0x80000000;
65
66	// Multiplication.
67	// Move the product to the highest bits to maximize precision
68	// mantissa is 24 bits => product is 48 bits, 2bits non-fraction.
69	// Add one bit for future addition overflow,
70	// add another bit to detect subtraction underflow
71	struct fp st_mul;
72	st_mul.sign = st_a.sign ^ st_b.sign;
73	st_mul.mantissa = (st_a.mantissa * st_b.mantissa) << 14ul;
74	st_mul.exponent = st_mul.mantissa ? st_a.exponent + st_b.exponent : 0;
75
76	// FIXME: Detecting a == 0 || b == 0 above crashed GCN isel
77	if (st_mul.exponent == 0 && st_mul.mantissa == 0)
78		return c;
79
80// Mantissa is 23 fractional bits, shift it the same way as product mantissa
81#define C_ADJUST 37ul
82
83	// both exponents are bias adjusted
84	int exp_diff = st_mul.exponent - st_c.exponent;
85
86	st_c.mantissa <<= C_ADJUST;
87	ulong cutoff_bits = 0;
88	ulong cutoff_mask = (1ul << abs(exp_diff)) - 1ul;
89	if (exp_diff > 0) {
90		cutoff_bits = exp_diff >= 64 ? st_c.mantissa : (st_c.mantissa & cutoff_mask);
91		st_c.mantissa = exp_diff >= 64 ? 0 : (st_c.mantissa >> exp_diff);
92	} else {
93		cutoff_bits = -exp_diff >= 64 ? st_mul.mantissa : (st_mul.mantissa & cutoff_mask);
94		st_mul.mantissa = -exp_diff >= 64 ? 0 : (st_mul.mantissa >> -exp_diff);
95	}
96
97	struct fp st_fma;
98	st_fma.sign = st_mul.sign;
99	st_fma.exponent = max(st_mul.exponent, st_c.exponent);
100	if (st_c.sign == st_mul.sign) {
101		st_fma.mantissa = st_mul.mantissa + st_c.mantissa;
102	} else {
103		// cutoff bits borrow one
104		st_fma.mantissa = st_mul.mantissa - st_c.mantissa - (cutoff_bits && (st_mul.exponent > st_c.exponent) ? 1 : 0);
105	}
106
107	// underflow: st_c.sign != st_mul.sign, and magnitude switches the sign
108	if (st_fma.mantissa > LONG_MAX) {
109		st_fma.mantissa = 0 - st_fma.mantissa;
110		st_fma.sign = st_mul.sign ^ 0x80000000;
111	}
112
113	// detect overflow/underflow
114	int overflow_bits = 3 - clz(st_fma.mantissa);
115
116	// adjust exponent
117	st_fma.exponent += overflow_bits;
118
119	// handle underflow
120	if (overflow_bits < 0) {
121		st_fma.mantissa <<= -overflow_bits;
122		overflow_bits = 0;
123	}
124
125	// rounding
126	ulong trunc_mask = (1ul << (C_ADJUST + overflow_bits)) - 1;
127	ulong trunc_bits = (st_fma.mantissa & trunc_mask) | (cutoff_bits != 0);
128	ulong last_bit = st_fma.mantissa & (1ul << (C_ADJUST + overflow_bits));
129	ulong grs_bits = (0x4ul << (C_ADJUST - 3 + overflow_bits));
130
131	// round to nearest even
132	if ((trunc_bits > grs_bits) ||
133	    (trunc_bits == grs_bits && last_bit != 0))
134		st_fma.mantissa += (1ul << (C_ADJUST + overflow_bits));
135
136	// Shift mantissa back to bit 23
137	st_fma.mantissa = (st_fma.mantissa >> (C_ADJUST + overflow_bits));
138
139	// Detect rounding overflow
140	if (st_fma.mantissa > 0xffffff) {
141		++st_fma.exponent;
142		st_fma.mantissa >>= 1;
143	}
144
145	if (st_fma.mantissa == 0)
146		return .0f;
147
148	// Flating point range limit
149	if (st_fma.exponent > 127)
150		return as_float(as_uint(INFINITY) | st_fma.sign);
151
152	// Flush denormals
153	if (st_fma.exponent <= -127)
154		return as_float(st_fma.sign);
155
156	return as_float(st_fma.sign | ((st_fma.exponent + 127) << 23) | ((uint)st_fma.mantissa & 0x7fffff));
157}
158_CLC_TERNARY_VECTORIZE(_CLC_DEF _CLC_OVERLOAD, float, __clc_sw_fma, float, float, float)
159