1 /*
2 * Copyright © 2010 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 */
23
24 /**
25 * \file lower_mat_op_to_vec.cpp
26 *
27 * Breaks matrix operation expressions down to a series of vector operations.
28 *
29 * Generally this is how we have to codegen matrix operations for a
30 * GPU, so this gives us the chance to constant fold operations on a
31 * column or row.
32 */
33
34 #include "ir.h"
35 #include "ir_expression_flattening.h"
36 #include "compiler/glsl_types.h"
37
38 namespace {
39
40 class ir_mat_op_to_vec_visitor : public ir_hierarchical_visitor {
41 public:
ir_mat_op_to_vec_visitor()42 ir_mat_op_to_vec_visitor()
43 {
44 this->made_progress = false;
45 this->mem_ctx = NULL;
46 }
47
48 ir_visitor_status visit_leave(ir_assignment *);
49
50 ir_dereference *get_column(ir_dereference *val, int col);
51 ir_rvalue *get_element(ir_dereference *val, int col, int row);
52
53 void do_mul_mat_mat(ir_dereference *result,
54 ir_dereference *a, ir_dereference *b);
55 void do_mul_mat_vec(ir_dereference *result,
56 ir_dereference *a, ir_dereference *b);
57 void do_mul_vec_mat(ir_dereference *result,
58 ir_dereference *a, ir_dereference *b);
59 void do_mul_mat_scalar(ir_dereference *result,
60 ir_dereference *a, ir_dereference *b);
61 void do_equal_mat_mat(ir_dereference *result, ir_dereference *a,
62 ir_dereference *b, bool test_equal);
63
64 void *mem_ctx;
65 bool made_progress;
66 };
67
68 } /* anonymous namespace */
69
70 static bool
mat_op_to_vec_predicate(ir_instruction * ir)71 mat_op_to_vec_predicate(ir_instruction *ir)
72 {
73 ir_expression *expr = ir->as_expression();
74 unsigned int i;
75
76 if (!expr)
77 return false;
78
79 for (i = 0; i < expr->num_operands; i++) {
80 if (expr->operands[i]->type->is_matrix())
81 return true;
82 }
83
84 return false;
85 }
86
87 bool
do_mat_op_to_vec(exec_list * instructions)88 do_mat_op_to_vec(exec_list *instructions)
89 {
90 ir_mat_op_to_vec_visitor v;
91
92 /* Pull out any matrix expression to a separate assignment to a
93 * temp. This will make our handling of the breakdown to
94 * operations on the matrix's vector components much easier.
95 */
96 do_expression_flattening(instructions, mat_op_to_vec_predicate);
97
98 visit_list_elements(&v, instructions);
99
100 return v.made_progress;
101 }
102
103 ir_rvalue *
get_element(ir_dereference * val,int col,int row)104 ir_mat_op_to_vec_visitor::get_element(ir_dereference *val, int col, int row)
105 {
106 val = get_column(val, col);
107
108 return new(mem_ctx) ir_swizzle(val, row, 0, 0, 0, 1);
109 }
110
111 ir_dereference *
get_column(ir_dereference * val,int row)112 ir_mat_op_to_vec_visitor::get_column(ir_dereference *val, int row)
113 {
114 val = val->clone(mem_ctx, NULL);
115
116 if (val->type->is_matrix()) {
117 val = new(mem_ctx) ir_dereference_array(val,
118 new(mem_ctx) ir_constant(row));
119 }
120
121 return val;
122 }
123
124 void
do_mul_mat_mat(ir_dereference * result,ir_dereference * a,ir_dereference * b)125 ir_mat_op_to_vec_visitor::do_mul_mat_mat(ir_dereference *result,
126 ir_dereference *a,
127 ir_dereference *b)
128 {
129 unsigned b_col, i;
130 ir_assignment *assign;
131 ir_expression *expr;
132
133 for (b_col = 0; b_col < b->type->matrix_columns; b_col++) {
134 /* first column */
135 expr = new(mem_ctx) ir_expression(ir_binop_mul,
136 get_column(a, 0),
137 get_element(b, b_col, 0));
138
139 /* following columns */
140 for (i = 1; i < a->type->matrix_columns; i++) {
141 ir_expression *mul_expr;
142
143 mul_expr = new(mem_ctx) ir_expression(ir_binop_mul,
144 get_column(a, i),
145 get_element(b, b_col, i));
146 expr = new(mem_ctx) ir_expression(ir_binop_add,
147 expr,
148 mul_expr);
149 }
150
151 assign = new(mem_ctx) ir_assignment(get_column(result, b_col), expr);
152 base_ir->insert_before(assign);
153 }
154 }
155
156 void
do_mul_mat_vec(ir_dereference * result,ir_dereference * a,ir_dereference * b)157 ir_mat_op_to_vec_visitor::do_mul_mat_vec(ir_dereference *result,
158 ir_dereference *a,
159 ir_dereference *b)
160 {
161 unsigned i;
162 ir_assignment *assign;
163 ir_expression *expr;
164
165 /* first column */
166 expr = new(mem_ctx) ir_expression(ir_binop_mul,
167 get_column(a, 0),
168 get_element(b, 0, 0));
169
170 /* following columns */
171 for (i = 1; i < a->type->matrix_columns; i++) {
172 ir_expression *mul_expr;
173
174 mul_expr = new(mem_ctx) ir_expression(ir_binop_mul,
175 get_column(a, i),
176 get_element(b, 0, i));
177 expr = new(mem_ctx) ir_expression(ir_binop_add, expr, mul_expr);
178 }
179
180 result = result->clone(mem_ctx, NULL);
181 assign = new(mem_ctx) ir_assignment(result, expr);
182 base_ir->insert_before(assign);
183 }
184
185 void
do_mul_vec_mat(ir_dereference * result,ir_dereference * a,ir_dereference * b)186 ir_mat_op_to_vec_visitor::do_mul_vec_mat(ir_dereference *result,
187 ir_dereference *a,
188 ir_dereference *b)
189 {
190 unsigned i;
191
192 for (i = 0; i < b->type->matrix_columns; i++) {
193 ir_rvalue *column_result;
194 ir_expression *column_expr;
195 ir_assignment *column_assign;
196
197 column_result = result->clone(mem_ctx, NULL);
198 column_result = new(mem_ctx) ir_swizzle(column_result, i, 0, 0, 0, 1);
199
200 column_expr = new(mem_ctx) ir_expression(ir_binop_dot,
201 a->clone(mem_ctx, NULL),
202 get_column(b, i));
203
204 column_assign = new(mem_ctx) ir_assignment(column_result,
205 column_expr);
206 base_ir->insert_before(column_assign);
207 }
208 }
209
210 void
do_mul_mat_scalar(ir_dereference * result,ir_dereference * a,ir_dereference * b)211 ir_mat_op_to_vec_visitor::do_mul_mat_scalar(ir_dereference *result,
212 ir_dereference *a,
213 ir_dereference *b)
214 {
215 unsigned i;
216
217 for (i = 0; i < a->type->matrix_columns; i++) {
218 ir_expression *column_expr;
219 ir_assignment *column_assign;
220
221 column_expr = new(mem_ctx) ir_expression(ir_binop_mul,
222 get_column(a, i),
223 b->clone(mem_ctx, NULL));
224
225 column_assign = new(mem_ctx) ir_assignment(get_column(result, i),
226 column_expr);
227 base_ir->insert_before(column_assign);
228 }
229 }
230
231 void
do_equal_mat_mat(ir_dereference * result,ir_dereference * a,ir_dereference * b,bool test_equal)232 ir_mat_op_to_vec_visitor::do_equal_mat_mat(ir_dereference *result,
233 ir_dereference *a,
234 ir_dereference *b,
235 bool test_equal)
236 {
237 /* This essentially implements the following GLSL:
238 *
239 * bool equal(mat4 a, mat4 b)
240 * {
241 * return !any(bvec4(a[0] != b[0],
242 * a[1] != b[1],
243 * a[2] != b[2],
244 * a[3] != b[3]);
245 * }
246 *
247 * bool nequal(mat4 a, mat4 b)
248 * {
249 * return any(bvec4(a[0] != b[0],
250 * a[1] != b[1],
251 * a[2] != b[2],
252 * a[3] != b[3]);
253 * }
254 */
255 const unsigned columns = a->type->matrix_columns;
256 const glsl_type *const bvec_type =
257 glsl_type::get_instance(GLSL_TYPE_BOOL, columns, 1);
258
259 ir_variable *const tmp_bvec =
260 new(this->mem_ctx) ir_variable(bvec_type, "mat_cmp_bvec",
261 ir_var_temporary);
262 this->base_ir->insert_before(tmp_bvec);
263
264 for (unsigned i = 0; i < columns; i++) {
265 ir_expression *const cmp =
266 new(this->mem_ctx) ir_expression(ir_binop_any_nequal,
267 get_column(a, i),
268 get_column(b, i));
269
270 ir_dereference *const lhs =
271 new(this->mem_ctx) ir_dereference_variable(tmp_bvec);
272
273 ir_assignment *const assign =
274 new(this->mem_ctx) ir_assignment(lhs, cmp, NULL, (1U << i));
275
276 this->base_ir->insert_before(assign);
277 }
278
279 ir_rvalue *const val = new(this->mem_ctx) ir_dereference_variable(tmp_bvec);
280 uint8_t vec_elems = val->type->vector_elements;
281 ir_expression *any =
282 new(this->mem_ctx) ir_expression(ir_binop_any_nequal, val,
283 new(this->mem_ctx) ir_constant(false,
284 vec_elems));
285
286 if (test_equal)
287 any = new(this->mem_ctx) ir_expression(ir_unop_logic_not, any);
288
289 ir_assignment *const assign =
290 new(mem_ctx) ir_assignment(result->clone(mem_ctx, NULL), any);
291 base_ir->insert_before(assign);
292 }
293
294 static bool
has_matrix_operand(const ir_expression * expr,unsigned & columns)295 has_matrix_operand(const ir_expression *expr, unsigned &columns)
296 {
297 for (unsigned i = 0; i < expr->num_operands; i++) {
298 if (expr->operands[i]->type->is_matrix()) {
299 columns = expr->operands[i]->type->matrix_columns;
300 return true;
301 }
302 }
303
304 return false;
305 }
306
307
308 ir_visitor_status
visit_leave(ir_assignment * orig_assign)309 ir_mat_op_to_vec_visitor::visit_leave(ir_assignment *orig_assign)
310 {
311 ir_expression *orig_expr = orig_assign->rhs->as_expression();
312 unsigned int i, matrix_columns = 1;
313 ir_dereference *op[2];
314
315 if (!orig_expr)
316 return visit_continue;
317
318 if (!has_matrix_operand(orig_expr, matrix_columns))
319 return visit_continue;
320
321 assert(orig_expr->num_operands <= 2);
322
323 mem_ctx = ralloc_parent(orig_assign);
324
325 ir_dereference_variable *result =
326 orig_assign->lhs->as_dereference_variable();
327 assert(result);
328
329 /* Store the expression operands in temps so we can use them
330 * multiple times.
331 */
332 for (i = 0; i < orig_expr->num_operands; i++) {
333 ir_assignment *assign;
334 ir_dereference *deref = orig_expr->operands[i]->as_dereference();
335
336 /* Avoid making a temporary if we don't need to to avoid aliasing. */
337 if (deref &&
338 deref->variable_referenced() != result->variable_referenced()) {
339 op[i] = deref;
340 continue;
341 }
342
343 /* Otherwise, store the operand in a temporary generally if it's
344 * not a dereference.
345 */
346 ir_variable *var = new(mem_ctx) ir_variable(orig_expr->operands[i]->type,
347 "mat_op_to_vec",
348 ir_var_temporary);
349 base_ir->insert_before(var);
350
351 /* Note that we use this dereference for the assignment. That means
352 * that others that want to use op[i] have to clone the deref.
353 */
354 op[i] = new(mem_ctx) ir_dereference_variable(var);
355 assign = new(mem_ctx) ir_assignment(op[i], orig_expr->operands[i]);
356 base_ir->insert_before(assign);
357 }
358
359 /* OK, time to break down this matrix operation. */
360 switch (orig_expr->operation) {
361 case ir_unop_d2f:
362 case ir_unop_f2d:
363 case ir_unop_f2f16:
364 case ir_unop_f2fmp:
365 case ir_unop_f162f:
366 case ir_unop_neg: {
367 /* Apply the operation to each column.*/
368 for (i = 0; i < matrix_columns; i++) {
369 ir_expression *column_expr;
370 ir_assignment *column_assign;
371
372 column_expr = new(mem_ctx) ir_expression(orig_expr->operation,
373 get_column(op[0], i));
374
375 column_assign = new(mem_ctx) ir_assignment(get_column(result, i),
376 column_expr);
377 assert(column_assign->write_mask != 0);
378 base_ir->insert_before(column_assign);
379 }
380 break;
381 }
382 case ir_binop_add:
383 case ir_binop_sub:
384 case ir_binop_div:
385 case ir_binop_mod: {
386 /* For most operations, the matrix version is just going
387 * column-wise through and applying the operation to each column
388 * if available.
389 */
390 for (i = 0; i < matrix_columns; i++) {
391 ir_expression *column_expr;
392 ir_assignment *column_assign;
393
394 column_expr = new(mem_ctx) ir_expression(orig_expr->operation,
395 get_column(op[0], i),
396 get_column(op[1], i));
397
398 column_assign = new(mem_ctx) ir_assignment(get_column(result, i),
399 column_expr);
400 assert(column_assign->write_mask != 0);
401 base_ir->insert_before(column_assign);
402 }
403 break;
404 }
405 case ir_binop_mul:
406 if (op[0]->type->is_matrix()) {
407 if (op[1]->type->is_matrix()) {
408 do_mul_mat_mat(result, op[0], op[1]);
409 } else if (op[1]->type->is_vector()) {
410 do_mul_mat_vec(result, op[0], op[1]);
411 } else {
412 assert(op[1]->type->is_scalar());
413 do_mul_mat_scalar(result, op[0], op[1]);
414 }
415 } else {
416 assert(op[1]->type->is_matrix());
417 if (op[0]->type->is_vector()) {
418 do_mul_vec_mat(result, op[0], op[1]);
419 } else {
420 assert(op[0]->type->is_scalar());
421 do_mul_mat_scalar(result, op[1], op[0]);
422 }
423 }
424 break;
425
426 case ir_binop_all_equal:
427 case ir_binop_any_nequal:
428 do_equal_mat_mat(result, op[1], op[0],
429 (orig_expr->operation == ir_binop_all_equal));
430 break;
431
432 default:
433 printf("FINISHME: Handle matrix operation for %s\n",
434 ir_expression_operation_strings[orig_expr->operation]);
435 abort();
436 }
437 orig_assign->remove();
438 this->made_progress = true;
439
440 return visit_continue;
441 }
442