1 #include "ir3_nir.h"
2 
3 #include "nir.h"
4 #include "nir_builder.h"
5 #include "nir_search.h"
6 #include "nir_search_helpers.h"
7 
8 /* What follows is NIR algebraic transform code for the following 2
9  * transforms:
10  *    ('imul', 'a@32', 'b@32') => ('imadsh_mix16', 'b', 'a', ('imadsh_mix16', 'a', 'b', ('umul_low', 'a', 'b')))
11  *    ('iadd', ('imul24', 'a', 'b'), 'c') => ('imad24_ir3', 'a', 'b', 'c')
12  */
13 
14 
15    static const nir_search_variable search0_0 = {
16    { nir_search_value_variable, 32 },
17    0, /* a */
18    false,
19    nir_type_invalid,
20    NULL,
21    {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
22 };
23 
24 static const nir_search_variable search0_1 = {
25    { nir_search_value_variable, 32 },
26    1, /* b */
27    false,
28    nir_type_invalid,
29    NULL,
30    {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
31 };
32 static const nir_search_expression search0 = {
33    { nir_search_value_expression, 32 },
34    false, false,
35    0, 1,
36    nir_op_imul,
37    { &search0_0.value, &search0_1.value },
38    NULL,
39 };
40 
41    /* replace0_0 -> search0_1 in the cache */
42 
43 /* replace0_1 -> search0_0 in the cache */
44 
45 /* replace0_2_0 -> search0_0 in the cache */
46 
47 /* replace0_2_1 -> search0_1 in the cache */
48 
49 /* replace0_2_2_0 -> search0_0 in the cache */
50 
51 /* replace0_2_2_1 -> search0_1 in the cache */
52 static const nir_search_expression replace0_2_2 = {
53    { nir_search_value_expression, 32 },
54    false, false,
55    0, 1,
56    nir_op_umul_low,
57    { &search0_0.value, &search0_1.value },
58    NULL,
59 };
60 static const nir_search_expression replace0_2 = {
61    { nir_search_value_expression, 32 },
62    false, false,
63    -1, 1,
64    nir_op_imadsh_mix16,
65    { &search0_0.value, &search0_1.value, &replace0_2_2.value },
66    NULL,
67 };
68 static const nir_search_expression replace0 = {
69    { nir_search_value_expression, 32 },
70    false, false,
71    -1, 1,
72    nir_op_imadsh_mix16,
73    { &search0_1.value, &search0_0.value, &replace0_2.value },
74    NULL,
75 };
76 
77    /* search1_0_0 -> search0_0 in the cache */
78 
79 /* search1_0_1 -> search0_1 in the cache */
80 static const nir_search_expression search1_0 = {
81    { nir_search_value_expression, 32 },
82    false, false,
83    1, 1,
84    nir_op_imul24,
85    { &search0_0.value, &search0_1.value },
86    NULL,
87 };
88 
89 static const nir_search_variable search1_1 = {
90    { nir_search_value_variable, 32 },
91    2, /* c */
92    false,
93    nir_type_invalid,
94    NULL,
95    {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
96 };
97 static const nir_search_expression search1 = {
98    { nir_search_value_expression, 32 },
99    false, false,
100    0, 2,
101    nir_op_iadd,
102    { &search1_0.value, &search1_1.value },
103    NULL,
104 };
105 
106    /* replace1_0 -> search0_0 in the cache */
107 
108 /* replace1_1 -> search0_1 in the cache */
109 
110 /* replace1_2 -> search1_1 in the cache */
111 static const nir_search_expression replace1 = {
112    { nir_search_value_expression, 32 },
113    false, false,
114    0, 1,
115    nir_op_imad24_ir3,
116    { &search0_0.value, &search0_1.value, &search1_1.value },
117    NULL,
118 };
119 
120 
121 static const struct transform ir3_nir_lower_imul_state2_xforms[] = {
122   { &search0, &replace0.value, 0 },
123 };
124 static const struct transform ir3_nir_lower_imul_state4_xforms[] = {
125   { &search1, &replace1.value, 0 },
126 };
127 
128 static const struct per_op_table ir3_nir_lower_imul_table[nir_num_search_ops] = {
129    [nir_op_imul] = {
130       .filter = (uint16_t []) {
131          0,
132          0,
133          0,
134          0,
135          0,
136       },
137 
138       .num_filtered_states = 1,
139       .table = (uint16_t []) {
140 
141          2,
142       },
143    },
144    [nir_op_iadd] = {
145       .filter = (uint16_t []) {
146          0,
147          0,
148          0,
149          1,
150          0,
151       },
152 
153       .num_filtered_states = 2,
154       .table = (uint16_t []) {
155 
156          0,
157          4,
158          4,
159          4,
160       },
161    },
162    [nir_op_imul24] = {
163       .filter = (uint16_t []) {
164          0,
165          0,
166          0,
167          0,
168          0,
169       },
170 
171       .num_filtered_states = 1,
172       .table = (uint16_t []) {
173 
174          3,
175       },
176    },
177 };
178 
179 const struct transform *ir3_nir_lower_imul_transforms[] = {
180    NULL,
181    NULL,
182    ir3_nir_lower_imul_state2_xforms,
183    NULL,
184    ir3_nir_lower_imul_state4_xforms,
185 };
186 
187 const uint16_t ir3_nir_lower_imul_transform_counts[] = {
188    0,
189    0,
190    (uint16_t)ARRAY_SIZE(ir3_nir_lower_imul_state2_xforms),
191    0,
192    (uint16_t)ARRAY_SIZE(ir3_nir_lower_imul_state4_xforms),
193 };
194 
195 bool
ir3_nir_lower_imul(nir_shader * shader)196 ir3_nir_lower_imul(nir_shader *shader)
197 {
198    bool progress = false;
199    bool condition_flags[1];
200    const nir_shader_compiler_options *options = shader->options;
201    const shader_info *info = &shader->info;
202    (void) options;
203    (void) info;
204 
205    condition_flags[0] = true;
206 
207    nir_foreach_function(function, shader) {
208       if (function->impl) {
209          progress |= nir_algebraic_impl(function->impl, condition_flags,
210                                         ir3_nir_lower_imul_transforms,
211                                         ir3_nir_lower_imul_transform_counts,
212                                         ir3_nir_lower_imul_table);
213       }
214    }
215 
216    return progress;
217 }
218 
219