1 /*
2  * Copyright © 2012 Intel Corporation
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21  * IN THE SOFTWARE.
22  */
23 
24 #include "blorp_nir_builder.h"
25 #include "compiler/nir/nir_format_convert.h"
26 
27 #include "blorp_priv.h"
28 
29 #include "util/format_rgb9e5.h"
30 /* header-only include needed for _mesa_unorm_to_float and friends. */
31 #include "mesa/main/format_utils.h"
32 #include "util/u_math.h"
33 
34 #define FILE_DEBUG_FLAG DEBUG_BLORP
35 
36 static const bool split_blorp_blit_debug = false;
37 
38 struct brw_blorp_blit_vars {
39    /* Input values from brw_blorp_wm_inputs */
40    nir_variable *v_discard_rect;
41    nir_variable *v_rect_grid;
42    nir_variable *v_coord_transform;
43    nir_variable *v_src_z;
44    nir_variable *v_src_offset;
45    nir_variable *v_dst_offset;
46    nir_variable *v_src_inv_size;
47 };
48 
49 static void
brw_blorp_blit_vars_init(nir_builder * b,struct brw_blorp_blit_vars * v,const struct brw_blorp_blit_prog_key * key)50 brw_blorp_blit_vars_init(nir_builder *b, struct brw_blorp_blit_vars *v,
51                          const struct brw_blorp_blit_prog_key *key)
52 {
53 #define LOAD_INPUT(name, type)\
54    v->v_##name = BLORP_CREATE_NIR_INPUT(b->shader, name, type);
55 
56    LOAD_INPUT(discard_rect, glsl_vec4_type())
57    LOAD_INPUT(rect_grid, glsl_vec4_type())
58    LOAD_INPUT(coord_transform, glsl_vec4_type())
59    LOAD_INPUT(src_z, glsl_float_type())
60    LOAD_INPUT(src_offset, glsl_vector_type(GLSL_TYPE_UINT, 2))
61    LOAD_INPUT(dst_offset, glsl_vector_type(GLSL_TYPE_UINT, 2))
62    LOAD_INPUT(src_inv_size, glsl_vector_type(GLSL_TYPE_FLOAT, 2))
63 
64 #undef LOAD_INPUT
65 }
66 
67 static nir_ssa_def *
blorp_blit_get_frag_coords(nir_builder * b,const struct brw_blorp_blit_prog_key * key,struct brw_blorp_blit_vars * v)68 blorp_blit_get_frag_coords(nir_builder *b,
69                            const struct brw_blorp_blit_prog_key *key,
70                            struct brw_blorp_blit_vars *v)
71 {
72    nir_ssa_def *coord = nir_f2i32(b, nir_load_frag_coord(b));
73 
74    /* Account for destination surface intratile offset
75     *
76     * Transformation parameters giving translation from destination to source
77     * coordinates don't take into account possible intra-tile destination
78     * offset.  Therefore it has to be first subtracted from the incoming
79     * coordinates.  Vertices are set up based on coordinates containing the
80     * intra-tile offset.
81     */
82    if (key->need_dst_offset)
83       coord = nir_isub(b, coord, nir_load_var(b, v->v_dst_offset));
84 
85    if (key->persample_msaa_dispatch) {
86       return nir_vec3(b, nir_channel(b, coord, 0), nir_channel(b, coord, 1),
87                       nir_load_sample_id(b));
88    } else {
89       return nir_vec2(b, nir_channel(b, coord, 0), nir_channel(b, coord, 1));
90    }
91 }
92 
93 /**
94  * Emit code to translate from destination (X, Y) coordinates to source (X, Y)
95  * coordinates.
96  */
97 static nir_ssa_def *
blorp_blit_apply_transform(nir_builder * b,nir_ssa_def * src_pos,struct brw_blorp_blit_vars * v)98 blorp_blit_apply_transform(nir_builder *b, nir_ssa_def *src_pos,
99                            struct brw_blorp_blit_vars *v)
100 {
101    nir_ssa_def *coord_transform = nir_load_var(b, v->v_coord_transform);
102 
103    nir_ssa_def *offset = nir_vec2(b, nir_channel(b, coord_transform, 1),
104                                      nir_channel(b, coord_transform, 3));
105    nir_ssa_def *mul = nir_vec2(b, nir_channel(b, coord_transform, 0),
106                                   nir_channel(b, coord_transform, 2));
107 
108    return nir_fadd(b, nir_fmul(b, src_pos, mul), offset);
109 }
110 
111 static inline void
blorp_nir_discard_if_outside_rect(nir_builder * b,nir_ssa_def * pos,struct brw_blorp_blit_vars * v)112 blorp_nir_discard_if_outside_rect(nir_builder *b, nir_ssa_def *pos,
113                                   struct brw_blorp_blit_vars *v)
114 {
115    nir_ssa_def *c0, *c1, *c2, *c3;
116    nir_ssa_def *discard_rect = nir_load_var(b, v->v_discard_rect);
117    nir_ssa_def *dst_x0 = nir_channel(b, discard_rect, 0);
118    nir_ssa_def *dst_x1 = nir_channel(b, discard_rect, 1);
119    nir_ssa_def *dst_y0 = nir_channel(b, discard_rect, 2);
120    nir_ssa_def *dst_y1 = nir_channel(b, discard_rect, 3);
121 
122    c0 = nir_ult(b, nir_channel(b, pos, 0), dst_x0);
123    c1 = nir_uge(b, nir_channel(b, pos, 0), dst_x1);
124    c2 = nir_ult(b, nir_channel(b, pos, 1), dst_y0);
125    c3 = nir_uge(b, nir_channel(b, pos, 1), dst_y1);
126 
127    nir_ssa_def *oob = nir_ior(b, nir_ior(b, c0, c1), nir_ior(b, c2, c3));
128 
129    nir_intrinsic_instr *discard =
130       nir_intrinsic_instr_create(b->shader, nir_intrinsic_discard_if);
131    discard->src[0] = nir_src_for_ssa(oob);
132    nir_builder_instr_insert(b, &discard->instr);
133 }
134 
135 static nir_tex_instr *
blorp_create_nir_tex_instr(nir_builder * b,struct brw_blorp_blit_vars * v,nir_texop op,nir_ssa_def * pos,unsigned num_srcs,nir_alu_type dst_type)136 blorp_create_nir_tex_instr(nir_builder *b, struct brw_blorp_blit_vars *v,
137                            nir_texop op, nir_ssa_def *pos, unsigned num_srcs,
138                            nir_alu_type dst_type)
139 {
140    nir_tex_instr *tex = nir_tex_instr_create(b->shader, num_srcs);
141 
142    tex->op = op;
143 
144    tex->dest_type = dst_type;
145    tex->is_array = false;
146    tex->is_shadow = false;
147 
148    /* Blorp only has one texture and it's bound at unit 0 */
149    tex->texture_index = 0;
150    tex->sampler_index = 0;
151 
152    /* To properly handle 3-D and 2-D array textures, we pull the Z component
153     * from an input.  TODO: This is a bit magic; we should probably make this
154     * more explicit in the future.
155     */
156    assert(pos->num_components >= 2);
157    if (op == nir_texop_txf || op == nir_texop_txf_ms || op == nir_texop_txf_ms_mcs) {
158       pos = nir_vec3(b, nir_channel(b, pos, 0), nir_channel(b, pos, 1),
159                         nir_f2i32(b, nir_load_var(b, v->v_src_z)));
160    } else {
161       pos = nir_vec3(b, nir_channel(b, pos, 0), nir_channel(b, pos, 1),
162                         nir_load_var(b, v->v_src_z));
163    }
164 
165    tex->src[0].src_type = nir_tex_src_coord;
166    tex->src[0].src = nir_src_for_ssa(pos);
167    tex->coord_components = 3;
168 
169    nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, NULL);
170 
171    return tex;
172 }
173 
174 static nir_ssa_def *
blorp_nir_tex(nir_builder * b,struct brw_blorp_blit_vars * v,const struct brw_blorp_blit_prog_key * key,nir_ssa_def * pos)175 blorp_nir_tex(nir_builder *b, struct brw_blorp_blit_vars *v,
176               const struct brw_blorp_blit_prog_key *key, nir_ssa_def *pos)
177 {
178    if (key->need_src_offset)
179       pos = nir_fadd(b, pos, nir_i2f32(b, nir_load_var(b, v->v_src_offset)));
180 
181    /* If the sampler requires normalized coordinates, we need to compensate. */
182    if (key->src_coords_normalized)
183       pos = nir_fmul(b, pos, nir_load_var(b, v->v_src_inv_size));
184 
185    nir_tex_instr *tex =
186       blorp_create_nir_tex_instr(b, v, nir_texop_tex, pos, 2,
187                                  key->texture_data_type);
188 
189    assert(pos->num_components == 2);
190    tex->sampler_dim = GLSL_SAMPLER_DIM_2D;
191    tex->src[1].src_type = nir_tex_src_lod;
192    tex->src[1].src = nir_src_for_ssa(nir_imm_int(b, 0));
193 
194    nir_builder_instr_insert(b, &tex->instr);
195 
196    return &tex->dest.ssa;
197 }
198 
199 static nir_ssa_def *
blorp_nir_txf(nir_builder * b,struct brw_blorp_blit_vars * v,nir_ssa_def * pos,nir_alu_type dst_type)200 blorp_nir_txf(nir_builder *b, struct brw_blorp_blit_vars *v,
201               nir_ssa_def *pos, nir_alu_type dst_type)
202 {
203    nir_tex_instr *tex =
204       blorp_create_nir_tex_instr(b, v, nir_texop_txf, pos, 2, dst_type);
205 
206    tex->sampler_dim = GLSL_SAMPLER_DIM_3D;
207    tex->src[1].src_type = nir_tex_src_lod;
208    tex->src[1].src = nir_src_for_ssa(nir_imm_int(b, 0));
209 
210    nir_builder_instr_insert(b, &tex->instr);
211 
212    return &tex->dest.ssa;
213 }
214 
215 static nir_ssa_def *
blorp_nir_txf_ms(nir_builder * b,struct brw_blorp_blit_vars * v,nir_ssa_def * pos,nir_ssa_def * mcs,nir_alu_type dst_type)216 blorp_nir_txf_ms(nir_builder *b, struct brw_blorp_blit_vars *v,
217                  nir_ssa_def *pos, nir_ssa_def *mcs, nir_alu_type dst_type)
218 {
219    nir_tex_instr *tex =
220       blorp_create_nir_tex_instr(b, v, nir_texop_txf_ms, pos,
221                                  mcs != NULL ? 3 : 2, dst_type);
222 
223    tex->sampler_dim = GLSL_SAMPLER_DIM_MS;
224 
225    tex->src[1].src_type = nir_tex_src_ms_index;
226    if (pos->num_components == 2) {
227       tex->src[1].src = nir_src_for_ssa(nir_imm_int(b, 0));
228    } else {
229       assert(pos->num_components == 3);
230       tex->src[1].src = nir_src_for_ssa(nir_channel(b, pos, 2));
231    }
232 
233    if (mcs) {
234       tex->src[2].src_type = nir_tex_src_ms_mcs;
235       tex->src[2].src = nir_src_for_ssa(mcs);
236    }
237 
238    nir_builder_instr_insert(b, &tex->instr);
239 
240    return &tex->dest.ssa;
241 }
242 
243 static nir_ssa_def *
blorp_blit_txf_ms_mcs(nir_builder * b,struct brw_blorp_blit_vars * v,nir_ssa_def * pos)244 blorp_blit_txf_ms_mcs(nir_builder *b, struct brw_blorp_blit_vars *v,
245                       nir_ssa_def *pos)
246 {
247    nir_tex_instr *tex =
248       blorp_create_nir_tex_instr(b, v, nir_texop_txf_ms_mcs,
249                                  pos, 1, nir_type_int);
250 
251    tex->sampler_dim = GLSL_SAMPLER_DIM_MS;
252 
253    nir_builder_instr_insert(b, &tex->instr);
254 
255    return &tex->dest.ssa;
256 }
257 
258 /**
259  * Emit code to compensate for the difference between Y and W tiling.
260  *
261  * This code modifies the X and Y coordinates according to the formula:
262  *
263  *   (X', Y', S') = detile(W-MAJOR, tile(Y-MAJOR, X, Y, S))
264  *
265  * (See brw_blorp_build_nir_shader).
266  */
267 static inline nir_ssa_def *
blorp_nir_retile_y_to_w(nir_builder * b,nir_ssa_def * pos)268 blorp_nir_retile_y_to_w(nir_builder *b, nir_ssa_def *pos)
269 {
270    assert(pos->num_components == 2);
271    nir_ssa_def *x_Y = nir_channel(b, pos, 0);
272    nir_ssa_def *y_Y = nir_channel(b, pos, 1);
273 
274    /* Given X and Y coordinates that describe an address using Y tiling,
275     * translate to the X and Y coordinates that describe the same address
276     * using W tiling.
277     *
278     * If we break down the low order bits of X and Y, using a
279     * single letter to represent each low-order bit:
280     *
281     *   X = A << 7 | 0bBCDEFGH
282     *   Y = J << 5 | 0bKLMNP                                       (1)
283     *
284     * Then we can apply the Y tiling formula to see the memory offset being
285     * addressed:
286     *
287     *   offset = (J * tile_pitch + A) << 12 | 0bBCDKLMNPEFGH       (2)
288     *
289     * If we apply the W detiling formula to this memory location, that the
290     * corresponding X' and Y' coordinates are:
291     *
292     *   X' = A << 6 | 0bBCDPFH                                     (3)
293     *   Y' = J << 6 | 0bKLMNEG
294     *
295     * Combining (1) and (3), we see that to transform (X, Y) to (X', Y'),
296     * we need to make the following computation:
297     *
298     *   X' = (X & ~0b1011) >> 1 | (Y & 0b1) << 2 | X & 0b1         (4)
299     *   Y' = (Y & ~0b1) << 1 | (X & 0b1000) >> 2 | (X & 0b10) >> 1
300     */
301    nir_ssa_def *x_W = nir_imm_int(b, 0);
302    x_W = nir_mask_shift_or(b, x_W, x_Y, 0xfffffff4, -1);
303    x_W = nir_mask_shift_or(b, x_W, y_Y, 0x1, 2);
304    x_W = nir_mask_shift_or(b, x_W, x_Y, 0x1, 0);
305 
306    nir_ssa_def *y_W = nir_imm_int(b, 0);
307    y_W = nir_mask_shift_or(b, y_W, y_Y, 0xfffffffe, 1);
308    y_W = nir_mask_shift_or(b, y_W, x_Y, 0x8, -2);
309    y_W = nir_mask_shift_or(b, y_W, x_Y, 0x2, -1);
310 
311    return nir_vec2(b, x_W, y_W);
312 }
313 
314 /**
315  * Emit code to compensate for the difference between Y and W tiling.
316  *
317  * This code modifies the X and Y coordinates according to the formula:
318  *
319  *   (X', Y', S') = detile(Y-MAJOR, tile(W-MAJOR, X, Y, S))
320  *
321  * (See brw_blorp_build_nir_shader).
322  */
323 static inline nir_ssa_def *
blorp_nir_retile_w_to_y(nir_builder * b,nir_ssa_def * pos)324 blorp_nir_retile_w_to_y(nir_builder *b, nir_ssa_def *pos)
325 {
326    assert(pos->num_components == 2);
327    nir_ssa_def *x_W = nir_channel(b, pos, 0);
328    nir_ssa_def *y_W = nir_channel(b, pos, 1);
329 
330    /* Applying the same logic as above, but in reverse, we obtain the
331     * formulas:
332     *
333     * X' = (X & ~0b101) << 1 | (Y & 0b10) << 2 | (Y & 0b1) << 1 | X & 0b1
334     * Y' = (Y & ~0b11) >> 1 | (X & 0b100) >> 2
335     */
336    nir_ssa_def *x_Y = nir_imm_int(b, 0);
337    x_Y = nir_mask_shift_or(b, x_Y, x_W, 0xfffffffa, 1);
338    x_Y = nir_mask_shift_or(b, x_Y, y_W, 0x2, 2);
339    x_Y = nir_mask_shift_or(b, x_Y, y_W, 0x1, 1);
340    x_Y = nir_mask_shift_or(b, x_Y, x_W, 0x1, 0);
341 
342    nir_ssa_def *y_Y = nir_imm_int(b, 0);
343    y_Y = nir_mask_shift_or(b, y_Y, y_W, 0xfffffffc, -1);
344    y_Y = nir_mask_shift_or(b, y_Y, x_W, 0x4, -2);
345 
346    return nir_vec2(b, x_Y, y_Y);
347 }
348 
349 /**
350  * Emit code to compensate for the difference between MSAA and non-MSAA
351  * surfaces.
352  *
353  * This code modifies the X and Y coordinates according to the formula:
354  *
355  *   (X', Y', S') = encode_msaa(num_samples, IMS, X, Y, S)
356  *
357  * (See brw_blorp_blit_program).
358  */
359 static inline nir_ssa_def *
blorp_nir_encode_msaa(nir_builder * b,nir_ssa_def * pos,unsigned num_samples,enum isl_msaa_layout layout)360 blorp_nir_encode_msaa(nir_builder *b, nir_ssa_def *pos,
361                       unsigned num_samples, enum isl_msaa_layout layout)
362 {
363    assert(pos->num_components == 2 || pos->num_components == 3);
364 
365    switch (layout) {
366    case ISL_MSAA_LAYOUT_NONE:
367       assert(pos->num_components == 2);
368       return pos;
369    case ISL_MSAA_LAYOUT_ARRAY:
370       /* No translation needed */
371       return pos;
372    case ISL_MSAA_LAYOUT_INTERLEAVED: {
373       nir_ssa_def *x_in = nir_channel(b, pos, 0);
374       nir_ssa_def *y_in = nir_channel(b, pos, 1);
375       nir_ssa_def *s_in = pos->num_components == 2 ? nir_imm_int(b, 0) :
376                                                      nir_channel(b, pos, 2);
377 
378       nir_ssa_def *x_out = nir_imm_int(b, 0);
379       nir_ssa_def *y_out = nir_imm_int(b, 0);
380       switch (num_samples) {
381       case 2:
382       case 4:
383          /* encode_msaa(2, IMS, X, Y, S) = (X', Y', 0)
384           *   where X' = (X & ~0b1) << 1 | (S & 0b1) << 1 | (X & 0b1)
385           *         Y' = Y
386           *
387           * encode_msaa(4, IMS, X, Y, S) = (X', Y', 0)
388           *   where X' = (X & ~0b1) << 1 | (S & 0b1) << 1 | (X & 0b1)
389           *         Y' = (Y & ~0b1) << 1 | (S & 0b10) | (Y & 0b1)
390           */
391          x_out = nir_mask_shift_or(b, x_out, x_in, 0xfffffffe, 1);
392          x_out = nir_mask_shift_or(b, x_out, s_in, 0x1, 1);
393          x_out = nir_mask_shift_or(b, x_out, x_in, 0x1, 0);
394          if (num_samples == 2) {
395             y_out = y_in;
396          } else {
397             y_out = nir_mask_shift_or(b, y_out, y_in, 0xfffffffe, 1);
398             y_out = nir_mask_shift_or(b, y_out, s_in, 0x2, 0);
399             y_out = nir_mask_shift_or(b, y_out, y_in, 0x1, 0);
400          }
401          break;
402 
403       case 8:
404          /* encode_msaa(8, IMS, X, Y, S) = (X', Y', 0)
405           *   where X' = (X & ~0b1) << 2 | (S & 0b100) | (S & 0b1) << 1
406           *              | (X & 0b1)
407           *         Y' = (Y & ~0b1) << 1 | (S & 0b10) | (Y & 0b1)
408           */
409          x_out = nir_mask_shift_or(b, x_out, x_in, 0xfffffffe, 2);
410          x_out = nir_mask_shift_or(b, x_out, s_in, 0x4, 0);
411          x_out = nir_mask_shift_or(b, x_out, s_in, 0x1, 1);
412          x_out = nir_mask_shift_or(b, x_out, x_in, 0x1, 0);
413          y_out = nir_mask_shift_or(b, y_out, y_in, 0xfffffffe, 1);
414          y_out = nir_mask_shift_or(b, y_out, s_in, 0x2, 0);
415          y_out = nir_mask_shift_or(b, y_out, y_in, 0x1, 0);
416          break;
417 
418       case 16:
419          /* encode_msaa(16, IMS, X, Y, S) = (X', Y', 0)
420           *   where X' = (X & ~0b1) << 2 | (S & 0b100) | (S & 0b1) << 1
421           *              | (X & 0b1)
422           *         Y' = (Y & ~0b1) << 2 | (S & 0b1000) >> 1 (S & 0b10)
423           *              | (Y & 0b1)
424           */
425          x_out = nir_mask_shift_or(b, x_out, x_in, 0xfffffffe, 2);
426          x_out = nir_mask_shift_or(b, x_out, s_in, 0x4, 0);
427          x_out = nir_mask_shift_or(b, x_out, s_in, 0x1, 1);
428          x_out = nir_mask_shift_or(b, x_out, x_in, 0x1, 0);
429          y_out = nir_mask_shift_or(b, y_out, y_in, 0xfffffffe, 2);
430          y_out = nir_mask_shift_or(b, y_out, s_in, 0x8, -1);
431          y_out = nir_mask_shift_or(b, y_out, s_in, 0x2, 0);
432          y_out = nir_mask_shift_or(b, y_out, y_in, 0x1, 0);
433          break;
434 
435       default:
436          unreachable("Invalid number of samples for IMS layout");
437       }
438 
439       return nir_vec2(b, x_out, y_out);
440    }
441 
442    default:
443       unreachable("Invalid MSAA layout");
444    }
445 }
446 
447 /**
448  * Emit code to compensate for the difference between MSAA and non-MSAA
449  * surfaces.
450  *
451  * This code modifies the X and Y coordinates according to the formula:
452  *
453  *   (X', Y', S) = decode_msaa(num_samples, IMS, X, Y, S)
454  *
455  * (See brw_blorp_blit_program).
456  */
457 static inline nir_ssa_def *
blorp_nir_decode_msaa(nir_builder * b,nir_ssa_def * pos,unsigned num_samples,enum isl_msaa_layout layout)458 blorp_nir_decode_msaa(nir_builder *b, nir_ssa_def *pos,
459                       unsigned num_samples, enum isl_msaa_layout layout)
460 {
461    assert(pos->num_components == 2 || pos->num_components == 3);
462 
463    switch (layout) {
464    case ISL_MSAA_LAYOUT_NONE:
465       /* No translation necessary, and S should already be zero. */
466       assert(pos->num_components == 2);
467       return pos;
468    case ISL_MSAA_LAYOUT_ARRAY:
469       /* No translation necessary. */
470       return pos;
471    case ISL_MSAA_LAYOUT_INTERLEAVED: {
472       assert(pos->num_components == 2);
473 
474       nir_ssa_def *x_in = nir_channel(b, pos, 0);
475       nir_ssa_def *y_in = nir_channel(b, pos, 1);
476 
477       nir_ssa_def *x_out = nir_imm_int(b, 0);
478       nir_ssa_def *y_out = nir_imm_int(b, 0);
479       nir_ssa_def *s_out = nir_imm_int(b, 0);
480       switch (num_samples) {
481       case 2:
482       case 4:
483          /* decode_msaa(2, IMS, X, Y, 0) = (X', Y', S)
484           *   where X' = (X & ~0b11) >> 1 | (X & 0b1)
485           *         S = (X & 0b10) >> 1
486           *
487           * decode_msaa(4, IMS, X, Y, 0) = (X', Y', S)
488           *   where X' = (X & ~0b11) >> 1 | (X & 0b1)
489           *         Y' = (Y & ~0b11) >> 1 | (Y & 0b1)
490           *         S = (Y & 0b10) | (X & 0b10) >> 1
491           */
492          x_out = nir_mask_shift_or(b, x_out, x_in, 0xfffffffc, -1);
493          x_out = nir_mask_shift_or(b, x_out, x_in, 0x1, 0);
494          if (num_samples == 2) {
495             y_out = y_in;
496             s_out = nir_mask_shift_or(b, s_out, x_in, 0x2, -1);
497          } else {
498             y_out = nir_mask_shift_or(b, y_out, y_in, 0xfffffffc, -1);
499             y_out = nir_mask_shift_or(b, y_out, y_in, 0x1, 0);
500             s_out = nir_mask_shift_or(b, s_out, x_in, 0x2, -1);
501             s_out = nir_mask_shift_or(b, s_out, y_in, 0x2, 0);
502          }
503          break;
504 
505       case 8:
506          /* decode_msaa(8, IMS, X, Y, 0) = (X', Y', S)
507           *   where X' = (X & ~0b111) >> 2 | (X & 0b1)
508           *         Y' = (Y & ~0b11) >> 1 | (Y & 0b1)
509           *         S = (X & 0b100) | (Y & 0b10) | (X & 0b10) >> 1
510           */
511          x_out = nir_mask_shift_or(b, x_out, x_in, 0xfffffff8, -2);
512          x_out = nir_mask_shift_or(b, x_out, x_in, 0x1, 0);
513          y_out = nir_mask_shift_or(b, y_out, y_in, 0xfffffffc, -1);
514          y_out = nir_mask_shift_or(b, y_out, y_in, 0x1, 0);
515          s_out = nir_mask_shift_or(b, s_out, x_in, 0x4, 0);
516          s_out = nir_mask_shift_or(b, s_out, y_in, 0x2, 0);
517          s_out = nir_mask_shift_or(b, s_out, x_in, 0x2, -1);
518          break;
519 
520       case 16:
521          /* decode_msaa(16, IMS, X, Y, 0) = (X', Y', S)
522           *   where X' = (X & ~0b111) >> 2 | (X & 0b1)
523           *         Y' = (Y & ~0b111) >> 2 | (Y & 0b1)
524           *         S = (Y & 0b100) << 1 | (X & 0b100) |
525           *             (Y & 0b10) | (X & 0b10) >> 1
526           */
527          x_out = nir_mask_shift_or(b, x_out, x_in, 0xfffffff8, -2);
528          x_out = nir_mask_shift_or(b, x_out, x_in, 0x1, 0);
529          y_out = nir_mask_shift_or(b, y_out, y_in, 0xfffffff8, -2);
530          y_out = nir_mask_shift_or(b, y_out, y_in, 0x1, 0);
531          s_out = nir_mask_shift_or(b, s_out, y_in, 0x4, 1);
532          s_out = nir_mask_shift_or(b, s_out, x_in, 0x4, 0);
533          s_out = nir_mask_shift_or(b, s_out, y_in, 0x2, 0);
534          s_out = nir_mask_shift_or(b, s_out, x_in, 0x2, -1);
535          break;
536 
537       default:
538          unreachable("Invalid number of samples for IMS layout");
539       }
540 
541       return nir_vec3(b, x_out, y_out, s_out);
542    }
543 
544    default:
545       unreachable("Invalid MSAA layout");
546    }
547 }
548 
549 /**
550  * Count the number of trailing 1 bits in the given value.  For example:
551  *
552  * count_trailing_one_bits(0) == 0
553  * count_trailing_one_bits(7) == 3
554  * count_trailing_one_bits(11) == 2
555  */
count_trailing_one_bits(unsigned value)556 static inline int count_trailing_one_bits(unsigned value)
557 {
558 #ifdef HAVE___BUILTIN_CTZ
559    return __builtin_ctz(~value);
560 #else
561    return util_bitcount(value & ~(value + 1));
562 #endif
563 }
564 
565 static nir_ssa_def *
blorp_nir_combine_samples(nir_builder * b,struct brw_blorp_blit_vars * v,nir_ssa_def * pos,unsigned tex_samples,enum isl_aux_usage tex_aux_usage,nir_alu_type dst_type,enum blorp_filter filter)566 blorp_nir_combine_samples(nir_builder *b, struct brw_blorp_blit_vars *v,
567                           nir_ssa_def *pos, unsigned tex_samples,
568                           enum isl_aux_usage tex_aux_usage,
569                           nir_alu_type dst_type,
570                           enum blorp_filter filter)
571 {
572    nir_variable *color =
573       nir_local_variable_create(b->impl, glsl_vec4_type(), "color");
574 
575    nir_ssa_def *mcs = NULL;
576    if (isl_aux_usage_has_mcs(tex_aux_usage))
577       mcs = blorp_blit_txf_ms_mcs(b, v, pos);
578 
579    nir_op combine_op;
580    switch (filter) {
581    case BLORP_FILTER_AVERAGE:
582       assert(dst_type == nir_type_float);
583       combine_op = nir_op_fadd;
584       break;
585 
586    case BLORP_FILTER_MIN_SAMPLE:
587       switch (dst_type) {
588       case nir_type_int:   combine_op = nir_op_imin;  break;
589       case nir_type_uint:  combine_op = nir_op_umin;  break;
590       case nir_type_float: combine_op = nir_op_fmin;  break;
591       default: unreachable("Invalid dst_type");
592       }
593       break;
594 
595    case BLORP_FILTER_MAX_SAMPLE:
596       switch (dst_type) {
597       case nir_type_int:   combine_op = nir_op_imax;  break;
598       case nir_type_uint:  combine_op = nir_op_umax;  break;
599       case nir_type_float: combine_op = nir_op_fmax;  break;
600       default: unreachable("Invalid dst_type");
601       }
602       break;
603 
604    default:
605       unreachable("Invalid filter");
606    }
607 
608    /* If true, we inserted an if statement that we need to pop at at the end.
609     */
610    bool inserted_if = false;
611 
612    /* We add together samples using a binary tree structure, e.g. for 4x MSAA:
613     *
614     *   result = ((sample[0] + sample[1]) + (sample[2] + sample[3])) / 4
615     *
616     * This ensures that when all samples have the same value, no numerical
617     * precision is lost, since each addition operation always adds two equal
618     * values, and summing two equal floating point values does not lose
619     * precision.
620     *
621     * We perform this computation by treating the texture_data array as a
622     * stack and performing the following operations:
623     *
624     * - push sample 0 onto stack
625     * - push sample 1 onto stack
626     * - add top two stack entries
627     * - push sample 2 onto stack
628     * - push sample 3 onto stack
629     * - add top two stack entries
630     * - add top two stack entries
631     * - divide top stack entry by 4
632     *
633     * Note that after pushing sample i onto the stack, the number of add
634     * operations we do is equal to the number of trailing 1 bits in i.  This
635     * works provided the total number of samples is a power of two, which it
636     * always is for i965.
637     *
638     * For integer formats, we replace the add operations with average
639     * operations and skip the final division.
640     */
641    nir_ssa_def *texture_data[5];
642    unsigned stack_depth = 0;
643    for (unsigned i = 0; i < tex_samples; ++i) {
644       assert(stack_depth == util_bitcount(i)); /* Loop invariant */
645 
646       /* Push sample i onto the stack */
647       assert(stack_depth < ARRAY_SIZE(texture_data));
648 
649       nir_ssa_def *ms_pos = nir_vec3(b, nir_channel(b, pos, 0),
650                                         nir_channel(b, pos, 1),
651                                         nir_imm_int(b, i));
652       texture_data[stack_depth++] = blorp_nir_txf_ms(b, v, ms_pos, mcs, dst_type);
653 
654       if (i == 0 && isl_aux_usage_has_mcs(tex_aux_usage)) {
655          /* The Ivy Bridge PRM, Vol4 Part1 p27 (Multisample Control Surface)
656           * suggests an optimization:
657           *
658           *     "A simple optimization with probable large return in
659           *     performance is to compare the MCS value to zero (indicating
660           *     all samples are on sample slice 0), and sample only from
661           *     sample slice 0 using ld2dss if MCS is zero."
662           *
663           * Note that in the case where the MCS value is zero, sampling from
664           * sample slice 0 using ld2dss and sampling from sample 0 using
665           * ld2dms are equivalent (since all samples are on sample slice 0).
666           * Since we have already sampled from sample 0, all we need to do is
667           * skip the remaining fetches and averaging if MCS is zero.
668           *
669           * It's also trivial to detect when the MCS has the magic clear color
670           * value.  In this case, the txf we did on sample 0 will return the
671           * clear color and we can skip the remaining fetches just like we do
672           * when MCS == 0.
673           */
674          nir_ssa_def *mcs_zero = nir_ieq_imm(b, nir_channel(b, mcs, 0), 0);
675          if (tex_samples == 16) {
676             mcs_zero = nir_iand(b, mcs_zero,
677                nir_ieq_imm(b, nir_channel(b, mcs, 1), 0));
678          }
679          nir_ssa_def *mcs_clear =
680             blorp_nir_mcs_is_clear_color(b, mcs, tex_samples);
681 
682          nir_push_if(b, nir_ior(b, mcs_zero, mcs_clear));
683          nir_store_var(b, color, texture_data[0], 0xf);
684 
685          nir_push_else(b, NULL);
686          inserted_if = true;
687       }
688 
689       for (int j = 0; j < count_trailing_one_bits(i); j++) {
690          assert(stack_depth >= 2);
691          --stack_depth;
692 
693          texture_data[stack_depth - 1] =
694             nir_build_alu(b, combine_op,
695                              texture_data[stack_depth - 1],
696                              texture_data[stack_depth],
697                              NULL, NULL);
698       }
699    }
700 
701    /* We should have just 1 sample on the stack now. */
702    assert(stack_depth == 1);
703 
704    if (filter == BLORP_FILTER_AVERAGE) {
705       assert(dst_type == nir_type_float);
706       texture_data[0] = nir_fmul(b, texture_data[0],
707                                  nir_imm_float(b, 1.0 / tex_samples));
708    }
709 
710    nir_store_var(b, color, texture_data[0], 0xf);
711 
712    if (inserted_if)
713       nir_pop_if(b, NULL);
714 
715    return nir_load_var(b, color);
716 }
717 
718 static nir_ssa_def *
blorp_nir_manual_blend_bilinear(nir_builder * b,nir_ssa_def * pos,unsigned tex_samples,const struct brw_blorp_blit_prog_key * key,struct brw_blorp_blit_vars * v)719 blorp_nir_manual_blend_bilinear(nir_builder *b, nir_ssa_def *pos,
720                                 unsigned tex_samples,
721                                 const struct brw_blorp_blit_prog_key *key,
722                                 struct brw_blorp_blit_vars *v)
723 {
724    nir_ssa_def *pos_xy = nir_channels(b, pos, 0x3);
725    nir_ssa_def *rect_grid = nir_load_var(b, v->v_rect_grid);
726    nir_ssa_def *scale = nir_imm_vec2(b, key->x_scale, key->y_scale);
727 
728    /* Translate coordinates to lay out the samples in a rectangular  grid
729     * roughly corresponding to sample locations.
730     */
731    pos_xy = nir_fmul(b, pos_xy, scale);
732    /* Adjust coordinates so that integers represent pixel centers rather
733     * than pixel edges.
734     */
735    pos_xy = nir_fadd(b, pos_xy, nir_imm_float(b, -0.5));
736    /* Clamp the X, Y texture coordinates to properly handle the sampling of
737     * texels on texture edges.
738     */
739    pos_xy = nir_fmin(b, nir_fmax(b, pos_xy, nir_imm_float(b, 0.0)),
740                         nir_vec2(b, nir_channel(b, rect_grid, 0),
741                                     nir_channel(b, rect_grid, 1)));
742 
743    /* Store the fractional parts to be used as bilinear interpolation
744     * coefficients.
745     */
746    nir_ssa_def *frac_xy = nir_ffract(b, pos_xy);
747    /* Round the float coordinates down to nearest integer */
748    pos_xy = nir_fdiv(b, nir_ftrunc(b, pos_xy), scale);
749 
750    nir_ssa_def *tex_data[4];
751    for (unsigned i = 0; i < 4; ++i) {
752       float sample_off_x = (float)(i & 0x1) / key->x_scale;
753       float sample_off_y = (float)((i >> 1) & 0x1) / key->y_scale;
754       nir_ssa_def *sample_off = nir_imm_vec2(b, sample_off_x, sample_off_y);
755 
756       nir_ssa_def *sample_coords = nir_fadd(b, pos_xy, sample_off);
757       nir_ssa_def *sample_coords_int = nir_f2i32(b, sample_coords);
758 
759       /* The MCS value we fetch has to match up with the pixel that we're
760        * sampling from. Since we sample from different pixels in each
761        * iteration of this "for" loop, the call to mcs_fetch() should be
762        * here inside the loop after computing the pixel coordinates.
763        */
764       nir_ssa_def *mcs = NULL;
765       if (isl_aux_usage_has_mcs(key->tex_aux_usage))
766          mcs = blorp_blit_txf_ms_mcs(b, v, sample_coords_int);
767 
768       /* Compute sample index and map the sample index to a sample number.
769        * Sample index layout shows the numbering of slots in a rectangular
770        * grid of samples with in a pixel. Sample number layout shows the
771        * rectangular grid of samples roughly corresponding to the real sample
772        * locations with in a pixel.
773        *
774        * In the case of 2x MSAA, the layout of sample indices is reversed from
775        * the layout of sample numbers:
776        *
777        * sample index layout :  ---------    sample number layout :  ---------
778        *                        | 0 | 1 |                            | 1 | 0 |
779        *                        ---------                            ---------
780        *
781        * In case of 4x MSAA, layout of sample indices matches the layout of
782        * sample numbers:
783        *           ---------
784        *           | 0 | 1 |
785        *           ---------
786        *           | 2 | 3 |
787        *           ---------
788        *
789        * In case of 8x MSAA the two layouts don't match.
790        * sample index layout :  ---------    sample number layout :  ---------
791        *                        | 0 | 1 |                            | 3 | 7 |
792        *                        ---------                            ---------
793        *                        | 2 | 3 |                            | 5 | 0 |
794        *                        ---------                            ---------
795        *                        | 4 | 5 |                            | 1 | 2 |
796        *                        ---------                            ---------
797        *                        | 6 | 7 |                            | 4 | 6 |
798        *                        ---------                            ---------
799        *
800        * Fortunately, this can be done fairly easily as:
801        * S' = (0x17306425 >> (S * 4)) & 0xf
802        *
803        * In the case of 16x MSAA the two layouts don't match.
804        * Sample index layout:                Sample number layout:
805        * ---------------------               ---------------------
806        * |  0 |  1 |  2 |  3 |               | 15 | 10 |  9 |  7 |
807        * ---------------------               ---------------------
808        * |  4 |  5 |  6 |  7 |               |  4 |  1 |  3 | 13 |
809        * ---------------------               ---------------------
810        * |  8 |  9 | 10 | 11 |               | 12 |  2 |  0 |  6 |
811        * ---------------------               ---------------------
812        * | 12 | 13 | 14 | 15 |               | 11 |  8 |  5 | 14 |
813        * ---------------------               ---------------------
814        *
815        * This is equivalent to
816        * S' = (0xe58b602cd31479af >> (S * 4)) & 0xf
817        */
818       nir_ssa_def *frac = nir_ffract(b, sample_coords);
819       nir_ssa_def *sample =
820          nir_fdot2(b, frac, nir_imm_vec2(b, key->x_scale,
821                                             key->x_scale * key->y_scale));
822       sample = nir_f2i32(b, sample);
823 
824       if (tex_samples == 2) {
825          sample = nir_isub(b, nir_imm_int(b, 1), sample);
826       } else if (tex_samples == 8) {
827          sample = nir_iand(b, nir_ishr(b, nir_imm_int(b, 0x64210573),
828                                        nir_ishl(b, sample, nir_imm_int(b, 2))),
829                            nir_imm_int(b, 0xf));
830       } else if (tex_samples == 16) {
831          nir_ssa_def *sample_low =
832             nir_iand(b, nir_ishr(b, nir_imm_int(b, 0xd31479af),
833                                  nir_ishl(b, sample, nir_imm_int(b, 2))),
834                      nir_imm_int(b, 0xf));
835          nir_ssa_def *sample_high =
836             nir_iand(b, nir_ishr(b, nir_imm_int(b, 0xe58b602c),
837                                  nir_ishl(b, nir_iadd(b, sample,
838                                                       nir_imm_int(b, -8)),
839                                           nir_imm_int(b, 2))),
840                      nir_imm_int(b, 0xf));
841 
842          sample = nir_bcsel(b, nir_ilt(b, sample, nir_imm_int(b, 8)),
843                             sample_low, sample_high);
844       }
845       nir_ssa_def *pos_ms = nir_vec3(b, nir_channel(b, sample_coords_int, 0),
846                                         nir_channel(b, sample_coords_int, 1),
847                                         sample);
848       tex_data[i] = blorp_nir_txf_ms(b, v, pos_ms, mcs, key->texture_data_type);
849    }
850 
851    nir_ssa_def *frac_x = nir_channel(b, frac_xy, 0);
852    nir_ssa_def *frac_y = nir_channel(b, frac_xy, 1);
853    return nir_flrp(b, nir_flrp(b, tex_data[0], tex_data[1], frac_x),
854                       nir_flrp(b, tex_data[2], tex_data[3], frac_x),
855                       frac_y);
856 }
857 
858 /** Perform a color bit-cast operation
859  *
860  * For copy operations involving CCS, we may need to use different formats for
861  * the source and destination surfaces.  The two formats must both be UINT
862  * formats and must have the same size but may have different bit layouts.
863  * For instance, we may be copying from R8G8B8A8_UINT to R32_UINT or R32_UINT
864  * to R16G16_UINT.  This function generates code to shuffle bits around to get
865  * us from one to the other.
866  */
867 static nir_ssa_def *
bit_cast_color(struct nir_builder * b,nir_ssa_def * color,const struct brw_blorp_blit_prog_key * key)868 bit_cast_color(struct nir_builder *b, nir_ssa_def *color,
869                const struct brw_blorp_blit_prog_key *key)
870 {
871    if (key->src_format == key->dst_format)
872       return color;
873 
874    const struct isl_format_layout *src_fmtl =
875       isl_format_get_layout(key->src_format);
876    const struct isl_format_layout *dst_fmtl =
877       isl_format_get_layout(key->dst_format);
878 
879    /* They must be formats with the same bit size */
880    assert(src_fmtl->bpb == dst_fmtl->bpb);
881 
882    if (src_fmtl->bpb <= 32) {
883       assert(src_fmtl->channels.r.type == ISL_UINT ||
884              src_fmtl->channels.r.type == ISL_UNORM);
885       assert(dst_fmtl->channels.r.type == ISL_UINT ||
886              dst_fmtl->channels.r.type == ISL_UNORM);
887 
888       nir_ssa_def *packed = nir_imm_int(b, 0);
889       for (unsigned c = 0; c < 4; c++) {
890          if (src_fmtl->channels_array[c].bits == 0)
891             continue;
892 
893          const unsigned chan_start_bit = src_fmtl->channels_array[c].start_bit;
894          const unsigned chan_bits = src_fmtl->channels_array[c].bits;
895 
896          nir_ssa_def *chan =  nir_channel(b, color, c);
897          if (src_fmtl->channels_array[c].type == ISL_UNORM)
898             chan = nir_format_float_to_unorm(b, chan, &chan_bits);
899 
900          packed = nir_ior(b, packed, nir_shift(b, chan, chan_start_bit));
901       }
902 
903       nir_ssa_def *chans[4] = { };
904       for (unsigned c = 0; c < 4; c++) {
905          if (dst_fmtl->channels_array[c].bits == 0) {
906             chans[c] = nir_imm_int(b, 0);
907             continue;
908          }
909 
910          const unsigned chan_start_bit = dst_fmtl->channels_array[c].start_bit;
911          const unsigned chan_bits = dst_fmtl->channels_array[c].bits;
912          chans[c] = nir_iand(b, nir_shift(b, packed, -(int)chan_start_bit),
913                                 nir_imm_int(b, BITFIELD_MASK(chan_bits)));
914 
915          if (dst_fmtl->channels_array[c].type == ISL_UNORM)
916             chans[c] = nir_format_unorm_to_float(b, chans[c], &chan_bits);
917       }
918       color = nir_vec(b, chans, 4);
919    } else {
920       /* This path only supports UINT formats */
921       assert(src_fmtl->channels.r.type == ISL_UINT);
922       assert(dst_fmtl->channels.r.type == ISL_UINT);
923 
924       const unsigned src_bpc = src_fmtl->channels.r.bits;
925       const unsigned dst_bpc = dst_fmtl->channels.r.bits;
926 
927       assert(src_fmtl->channels.g.bits == 0 ||
928              src_fmtl->channels.g.bits == src_fmtl->channels.r.bits);
929       assert(src_fmtl->channels.b.bits == 0 ||
930              src_fmtl->channels.b.bits == src_fmtl->channels.r.bits);
931       assert(src_fmtl->channels.a.bits == 0 ||
932              src_fmtl->channels.a.bits == src_fmtl->channels.r.bits);
933       assert(dst_fmtl->channels.g.bits == 0 ||
934              dst_fmtl->channels.g.bits == dst_fmtl->channels.r.bits);
935       assert(dst_fmtl->channels.b.bits == 0 ||
936              dst_fmtl->channels.b.bits == dst_fmtl->channels.r.bits);
937       assert(dst_fmtl->channels.a.bits == 0 ||
938              dst_fmtl->channels.a.bits == dst_fmtl->channels.r.bits);
939 
940       /* Restrict to only the channels we actually have */
941       const unsigned src_channels =
942          isl_format_get_num_channels(key->src_format);
943       color = nir_channels(b, color, (1 << src_channels) - 1);
944 
945       color = nir_format_bitcast_uvec_unmasked(b, color, src_bpc, dst_bpc);
946    }
947 
948    /* Blorp likes to assume that colors are vec4s */
949    nir_ssa_def *u = nir_ssa_undef(b, 1, 32);
950    nir_ssa_def *chans[4] = { u, u, u, u };
951    for (unsigned i = 0; i < color->num_components; i++)
952       chans[i] = nir_channel(b, color, i);
953    return nir_vec4(b, chans[0], chans[1], chans[2], chans[3]);
954 }
955 
956 static nir_ssa_def *
select_color_channel(struct nir_builder * b,nir_ssa_def * color,nir_alu_type data_type,enum isl_channel_select chan)957 select_color_channel(struct nir_builder *b, nir_ssa_def *color,
958                      nir_alu_type data_type,
959                      enum isl_channel_select chan)
960 {
961    if (chan == ISL_CHANNEL_SELECT_ZERO) {
962       return nir_imm_int(b, 0);
963    } else if (chan == ISL_CHANNEL_SELECT_ONE) {
964       switch (data_type) {
965       case nir_type_int:
966       case nir_type_uint:
967          return nir_imm_int(b, 1);
968       case nir_type_float:
969          return nir_imm_float(b, 1);
970       default:
971          unreachable("Invalid data type");
972       }
973    } else {
974       assert((unsigned)(chan - ISL_CHANNEL_SELECT_RED) < 4);
975       return nir_channel(b, color, chan - ISL_CHANNEL_SELECT_RED);
976    }
977 }
978 
979 static nir_ssa_def *
swizzle_color(struct nir_builder * b,nir_ssa_def * color,struct isl_swizzle swizzle,nir_alu_type data_type)980 swizzle_color(struct nir_builder *b, nir_ssa_def *color,
981               struct isl_swizzle swizzle, nir_alu_type data_type)
982 {
983    return nir_vec4(b,
984                    select_color_channel(b, color, data_type, swizzle.r),
985                    select_color_channel(b, color, data_type, swizzle.g),
986                    select_color_channel(b, color, data_type, swizzle.b),
987                    select_color_channel(b, color, data_type, swizzle.a));
988 }
989 
990 static nir_ssa_def *
convert_color(struct nir_builder * b,nir_ssa_def * color,const struct brw_blorp_blit_prog_key * key)991 convert_color(struct nir_builder *b, nir_ssa_def *color,
992               const struct brw_blorp_blit_prog_key *key)
993 {
994    /* All of our color conversions end up generating a single-channel color
995     * value that we need to write out.
996     */
997    nir_ssa_def *value;
998 
999    if (key->dst_format == ISL_FORMAT_R24_UNORM_X8_TYPELESS) {
1000       /* The destination image is bound as R32_UINT but the data needs to be
1001        * in R24_UNORM_X8_TYPELESS.  The bottom 24 are the actual data and the
1002        * top 8 need to be zero.  We can accomplish this by simply multiplying
1003        * by a factor to scale things down.
1004        */
1005       unsigned factor = (1 << 24) - 1;
1006       value = nir_fsat(b, nir_channel(b, color, 0));
1007       value = nir_f2i32(b, nir_fmul(b, value, nir_imm_float(b, factor)));
1008    } else if (key->dst_format == ISL_FORMAT_L8_UNORM_SRGB) {
1009       value = nir_format_linear_to_srgb(b, nir_channel(b, color, 0));
1010    } else if (key->dst_format == ISL_FORMAT_R8G8B8_UNORM_SRGB) {
1011       value = nir_format_linear_to_srgb(b, color);
1012    } else if (key->dst_format == ISL_FORMAT_R9G9B9E5_SHAREDEXP) {
1013       value = nir_format_pack_r9g9b9e5(b, color);
1014    } else {
1015       unreachable("Unsupported format conversion");
1016    }
1017 
1018    nir_ssa_def *out_comps[4];
1019    for (unsigned i = 0; i < 4; i++) {
1020       if (i < value->num_components)
1021          out_comps[i] = nir_channel(b, value, i);
1022       else
1023          out_comps[i] = nir_ssa_undef(b, 1, 32);
1024    }
1025    return nir_vec(b, out_comps, 4);
1026 }
1027 
1028 /**
1029  * Generator for WM programs used in BLORP blits.
1030  *
1031  * The bulk of the work done by the WM program is to wrap and unwrap the
1032  * coordinate transformations used by the hardware to store surfaces in
1033  * memory.  The hardware transforms a pixel location (X, Y, S) (where S is the
1034  * sample index for a multisampled surface) to a memory offset by the
1035  * following formulas:
1036  *
1037  *   offset = tile(tiling_format, encode_msaa(num_samples, layout, X, Y, S))
1038  *   (X, Y, S) = decode_msaa(num_samples, layout, detile(tiling_format, offset))
1039  *
1040  * For a single-sampled surface, or for a multisampled surface using
1041  * INTEL_MSAA_LAYOUT_UMS, encode_msaa() and decode_msaa are the identity
1042  * function:
1043  *
1044  *   encode_msaa(1, NONE, X, Y, 0) = (X, Y, 0)
1045  *   decode_msaa(1, NONE, X, Y, 0) = (X, Y, 0)
1046  *   encode_msaa(n, UMS, X, Y, S) = (X, Y, S)
1047  *   decode_msaa(n, UMS, X, Y, S) = (X, Y, S)
1048  *
1049  * For a 4x multisampled surface using INTEL_MSAA_LAYOUT_IMS, encode_msaa()
1050  * embeds the sample number into bit 1 of the X and Y coordinates:
1051  *
1052  *   encode_msaa(4, IMS, X, Y, S) = (X', Y', 0)
1053  *     where X' = (X & ~0b1) << 1 | (S & 0b1) << 1 | (X & 0b1)
1054  *           Y' = (Y & ~0b1 ) << 1 | (S & 0b10) | (Y & 0b1)
1055  *   decode_msaa(4, IMS, X, Y, 0) = (X', Y', S)
1056  *     where X' = (X & ~0b11) >> 1 | (X & 0b1)
1057  *           Y' = (Y & ~0b11) >> 1 | (Y & 0b1)
1058  *           S = (Y & 0b10) | (X & 0b10) >> 1
1059  *
1060  * For an 8x multisampled surface using INTEL_MSAA_LAYOUT_IMS, encode_msaa()
1061  * embeds the sample number into bits 1 and 2 of the X coordinate and bit 1 of
1062  * the Y coordinate:
1063  *
1064  *   encode_msaa(8, IMS, X, Y, S) = (X', Y', 0)
1065  *     where X' = (X & ~0b1) << 2 | (S & 0b100) | (S & 0b1) << 1 | (X & 0b1)
1066  *           Y' = (Y & ~0b1) << 1 | (S & 0b10) | (Y & 0b1)
1067  *   decode_msaa(8, IMS, X, Y, 0) = (X', Y', S)
1068  *     where X' = (X & ~0b111) >> 2 | (X & 0b1)
1069  *           Y' = (Y & ~0b11) >> 1 | (Y & 0b1)
1070  *           S = (X & 0b100) | (Y & 0b10) | (X & 0b10) >> 1
1071  *
1072  * For X tiling, tile() combines together the low-order bits of the X and Y
1073  * coordinates in the pattern 0byyyxxxxxxxxx, creating 4k tiles that are 512
1074  * bytes wide and 8 rows high:
1075  *
1076  *   tile(x_tiled, X, Y, S) = A
1077  *     where A = tile_num << 12 | offset
1078  *           tile_num = (Y' >> 3) * tile_pitch + (X' >> 9)
1079  *           offset = (Y' & 0b111) << 9
1080  *                    | (X & 0b111111111)
1081  *           X' = X * cpp
1082  *           Y' = Y + S * qpitch
1083  *   detile(x_tiled, A) = (X, Y, S)
1084  *     where X = X' / cpp
1085  *           Y = Y' % qpitch
1086  *           S = Y' / qpitch
1087  *           Y' = (tile_num / tile_pitch) << 3
1088  *                | (A & 0b111000000000) >> 9
1089  *           X' = (tile_num % tile_pitch) << 9
1090  *                | (A & 0b111111111)
1091  *
1092  * (In all tiling formulas, cpp is the number of bytes occupied by a single
1093  * sample ("chars per pixel"), tile_pitch is the number of 4k tiles required
1094  * to fill the width of the surface, and qpitch is the spacing (in rows)
1095  * between array slices).
1096  *
1097  * For Y tiling, tile() combines together the low-order bits of the X and Y
1098  * coordinates in the pattern 0bxxxyyyyyxxxx, creating 4k tiles that are 128
1099  * bytes wide and 32 rows high:
1100  *
1101  *   tile(y_tiled, X, Y, S) = A
1102  *     where A = tile_num << 12 | offset
1103  *           tile_num = (Y' >> 5) * tile_pitch + (X' >> 7)
1104  *           offset = (X' & 0b1110000) << 5
1105  *                    | (Y' & 0b11111) << 4
1106  *                    | (X' & 0b1111)
1107  *           X' = X * cpp
1108  *           Y' = Y + S * qpitch
1109  *   detile(y_tiled, A) = (X, Y, S)
1110  *     where X = X' / cpp
1111  *           Y = Y' % qpitch
1112  *           S = Y' / qpitch
1113  *           Y' = (tile_num / tile_pitch) << 5
1114  *                | (A & 0b111110000) >> 4
1115  *           X' = (tile_num % tile_pitch) << 7
1116  *                | (A & 0b111000000000) >> 5
1117  *                | (A & 0b1111)
1118  *
1119  * For W tiling, tile() combines together the low-order bits of the X and Y
1120  * coordinates in the pattern 0bxxxyyyyxyxyx, creating 4k tiles that are 64
1121  * bytes wide and 64 rows high (note that W tiling is only used for stencil
1122  * buffers, which always have cpp = 1 and S=0):
1123  *
1124  *   tile(w_tiled, X, Y, S) = A
1125  *     where A = tile_num << 12 | offset
1126  *           tile_num = (Y' >> 6) * tile_pitch + (X' >> 6)
1127  *           offset = (X' & 0b111000) << 6
1128  *                    | (Y' & 0b111100) << 3
1129  *                    | (X' & 0b100) << 2
1130  *                    | (Y' & 0b10) << 2
1131  *                    | (X' & 0b10) << 1
1132  *                    | (Y' & 0b1) << 1
1133  *                    | (X' & 0b1)
1134  *           X' = X * cpp = X
1135  *           Y' = Y + S * qpitch
1136  *   detile(w_tiled, A) = (X, Y, S)
1137  *     where X = X' / cpp = X'
1138  *           Y = Y' % qpitch = Y'
1139  *           S = Y / qpitch = 0
1140  *           Y' = (tile_num / tile_pitch) << 6
1141  *                | (A & 0b111100000) >> 3
1142  *                | (A & 0b1000) >> 2
1143  *                | (A & 0b10) >> 1
1144  *           X' = (tile_num % tile_pitch) << 6
1145  *                | (A & 0b111000000000) >> 6
1146  *                | (A & 0b10000) >> 2
1147  *                | (A & 0b100) >> 1
1148  *                | (A & 0b1)
1149  *
1150  * Finally, for a non-tiled surface, tile() simply combines together the X and
1151  * Y coordinates in the natural way:
1152  *
1153  *   tile(untiled, X, Y, S) = A
1154  *     where A = Y * pitch + X'
1155  *           X' = X * cpp
1156  *           Y' = Y + S * qpitch
1157  *   detile(untiled, A) = (X, Y, S)
1158  *     where X = X' / cpp
1159  *           Y = Y' % qpitch
1160  *           S = Y' / qpitch
1161  *           X' = A % pitch
1162  *           Y' = A / pitch
1163  *
1164  * (In these formulas, pitch is the number of bytes occupied by a single row
1165  * of samples).
1166  */
1167 static nir_shader *
brw_blorp_build_nir_shader(struct blorp_context * blorp,void * mem_ctx,const struct brw_blorp_blit_prog_key * key)1168 brw_blorp_build_nir_shader(struct blorp_context *blorp, void *mem_ctx,
1169                            const struct brw_blorp_blit_prog_key *key)
1170 {
1171    const struct gen_device_info *devinfo = blorp->isl_dev->info;
1172    nir_ssa_def *src_pos, *dst_pos, *color;
1173 
1174    /* Sanity checks */
1175    if (key->dst_tiled_w && key->rt_samples > 1) {
1176       /* If the destination image is W tiled and multisampled, then the thread
1177        * must be dispatched once per sample, not once per pixel.  This is
1178        * necessary because after conversion between W and Y tiling, there's no
1179        * guarantee that all samples corresponding to a single pixel will still
1180        * be together.
1181        */
1182       assert(key->persample_msaa_dispatch);
1183    }
1184 
1185    if (key->persample_msaa_dispatch) {
1186       /* It only makes sense to do persample dispatch if the render target is
1187        * configured as multisampled.
1188        */
1189       assert(key->rt_samples > 0);
1190    }
1191 
1192    /* Make sure layout is consistent with sample count */
1193    assert((key->tex_layout == ISL_MSAA_LAYOUT_NONE) ==
1194           (key->tex_samples <= 1));
1195    assert((key->rt_layout == ISL_MSAA_LAYOUT_NONE) ==
1196           (key->rt_samples <= 1));
1197    assert((key->src_layout == ISL_MSAA_LAYOUT_NONE) ==
1198           (key->src_samples <= 1));
1199    assert((key->dst_layout == ISL_MSAA_LAYOUT_NONE) ==
1200           (key->dst_samples <= 1));
1201 
1202    nir_builder b;
1203    blorp_nir_init_shader(&b, mem_ctx, MESA_SHADER_FRAGMENT, NULL);
1204 
1205    struct brw_blorp_blit_vars v;
1206    brw_blorp_blit_vars_init(&b, &v, key);
1207 
1208    dst_pos = blorp_blit_get_frag_coords(&b, key, &v);
1209 
1210    /* Render target and texture hardware don't support W tiling until Gen8. */
1211    const bool rt_tiled_w = false;
1212    const bool tex_tiled_w = devinfo->gen >= 8 && key->src_tiled_w;
1213 
1214    /* The address that data will be written to is determined by the
1215     * coordinates supplied to the WM thread and the tiling and sample count of
1216     * the render target, according to the formula:
1217     *
1218     * (X, Y, S) = decode_msaa(rt_samples, detile(rt_tiling, offset))
1219     *
1220     * If the actual tiling and sample count of the destination surface are not
1221     * the same as the configuration of the render target, then these
1222     * coordinates are wrong and we have to adjust them to compensate for the
1223     * difference.
1224     */
1225    if (rt_tiled_w != key->dst_tiled_w ||
1226        key->rt_samples != key->dst_samples ||
1227        key->rt_layout != key->dst_layout) {
1228       dst_pos = blorp_nir_encode_msaa(&b, dst_pos, key->rt_samples,
1229                                       key->rt_layout);
1230       /* Now (X, Y, S) = detile(rt_tiling, offset) */
1231       if (rt_tiled_w != key->dst_tiled_w)
1232          dst_pos = blorp_nir_retile_y_to_w(&b, dst_pos);
1233       /* Now (X, Y, S) = detile(rt_tiling, offset) */
1234       dst_pos = blorp_nir_decode_msaa(&b, dst_pos, key->dst_samples,
1235                                       key->dst_layout);
1236    }
1237 
1238    nir_ssa_def *comp = NULL;
1239    if (key->dst_rgb) {
1240       /* The destination image is bound as a red texture three times as wide
1241        * as the actual image.  Our shader is effectively running one color
1242        * component at a time.  We need to save off the component and adjust
1243        * the destination position.
1244        */
1245       assert(dst_pos->num_components == 2);
1246       nir_ssa_def *dst_x = nir_channel(&b, dst_pos, 0);
1247       comp = nir_umod(&b, dst_x, nir_imm_int(&b, 3));
1248       dst_pos = nir_vec2(&b, nir_idiv(&b, dst_x, nir_imm_int(&b, 3)),
1249                              nir_channel(&b, dst_pos, 1));
1250    }
1251 
1252    /* Now (X, Y, S) = decode_msaa(dst_samples, detile(dst_tiling, offset)).
1253     *
1254     * That is: X, Y and S now contain the true coordinates and sample index of
1255     * the data that the WM thread should output.
1256     *
1257     * If we need to kill pixels that are outside the destination rectangle,
1258     * now is the time to do it.
1259     */
1260    if (key->use_kill)
1261       blorp_nir_discard_if_outside_rect(&b, dst_pos, &v);
1262 
1263    src_pos = blorp_blit_apply_transform(&b, nir_i2f32(&b, dst_pos), &v);
1264    if (dst_pos->num_components == 3) {
1265       /* The sample coordinate is an integer that we want left alone but
1266        * blorp_blit_apply_transform() blindly applies the transform to all
1267        * three coordinates.  Grab the original sample index.
1268        */
1269       src_pos = nir_vec3(&b, nir_channel(&b, src_pos, 0),
1270                              nir_channel(&b, src_pos, 1),
1271                              nir_channel(&b, dst_pos, 2));
1272    }
1273 
1274    /* If the source image is not multisampled, then we want to fetch sample
1275     * number 0, because that's the only sample there is.
1276     */
1277    if (key->src_samples == 1)
1278       src_pos = nir_channels(&b, src_pos, 0x3);
1279 
1280    /* X, Y, and S are now the coordinates of the pixel in the source image
1281     * that we want to texture from.  Exception: if we are blending, then S is
1282     * irrelevant, because we are going to fetch all samples.
1283     */
1284    switch (key->filter) {
1285    case BLORP_FILTER_NONE:
1286    case BLORP_FILTER_NEAREST:
1287    case BLORP_FILTER_SAMPLE_0:
1288       /* We're going to use texelFetch, so we need integers */
1289       if (src_pos->num_components == 2) {
1290          src_pos = nir_f2i32(&b, src_pos);
1291       } else {
1292          assert(src_pos->num_components == 3);
1293          src_pos = nir_vec3(&b, nir_channel(&b, nir_f2i32(&b, src_pos), 0),
1294                                 nir_channel(&b, nir_f2i32(&b, src_pos), 1),
1295                                 nir_channel(&b, src_pos, 2));
1296       }
1297 
1298       /* We aren't blending, which means we just want to fetch a single
1299        * sample from the source surface.  The address that we want to fetch
1300        * from is related to the X, Y and S values according to the formula:
1301        *
1302        * (X, Y, S) = decode_msaa(src_samples, detile(src_tiling, offset)).
1303        *
1304        * If the actual tiling and sample count of the source surface are
1305        * not the same as the configuration of the texture, then we need to
1306        * adjust the coordinates to compensate for the difference.
1307        */
1308       if (tex_tiled_w != key->src_tiled_w ||
1309           key->tex_samples != key->src_samples ||
1310           key->tex_layout != key->src_layout) {
1311          src_pos = blorp_nir_encode_msaa(&b, src_pos, key->src_samples,
1312                                          key->src_layout);
1313          /* Now (X, Y, S) = detile(src_tiling, offset) */
1314          if (tex_tiled_w != key->src_tiled_w)
1315             src_pos = blorp_nir_retile_w_to_y(&b, src_pos);
1316          /* Now (X, Y, S) = detile(tex_tiling, offset) */
1317          src_pos = blorp_nir_decode_msaa(&b, src_pos, key->tex_samples,
1318                                          key->tex_layout);
1319       }
1320 
1321       if (key->need_src_offset)
1322          src_pos = nir_iadd(&b, src_pos, nir_load_var(&b, v.v_src_offset));
1323 
1324       /* Now (X, Y, S) = decode_msaa(tex_samples, detile(tex_tiling, offset)).
1325        *
1326        * In other words: X, Y, and S now contain values which, when passed to
1327        * the texturing unit, will cause data to be read from the correct
1328        * memory location.  So we can fetch the texel now.
1329        */
1330       if (key->src_samples == 1) {
1331          color = blorp_nir_txf(&b, &v, src_pos, key->texture_data_type);
1332       } else {
1333          nir_ssa_def *mcs = NULL;
1334          if (isl_aux_usage_has_mcs(key->tex_aux_usage))
1335             mcs = blorp_blit_txf_ms_mcs(&b, &v, src_pos);
1336 
1337          color = blorp_nir_txf_ms(&b, &v, src_pos, mcs, key->texture_data_type);
1338       }
1339       break;
1340 
1341    case BLORP_FILTER_BILINEAR:
1342       assert(!key->src_tiled_w);
1343       assert(key->tex_samples == key->src_samples);
1344       assert(key->tex_layout == key->src_layout);
1345 
1346       if (key->src_samples == 1) {
1347          color = blorp_nir_tex(&b, &v, key, src_pos);
1348       } else {
1349          assert(!key->use_kill);
1350          color = blorp_nir_manual_blend_bilinear(&b, src_pos, key->src_samples,
1351                                                  key, &v);
1352       }
1353       break;
1354 
1355    case BLORP_FILTER_AVERAGE:
1356    case BLORP_FILTER_MIN_SAMPLE:
1357    case BLORP_FILTER_MAX_SAMPLE:
1358       assert(!key->src_tiled_w);
1359       assert(key->tex_samples == key->src_samples);
1360       assert(key->tex_layout == key->src_layout);
1361 
1362       /* Resolves (effecively) use texelFetch, so we need integers and we
1363        * don't care about the sample index if we got one.
1364        */
1365       src_pos = nir_f2i32(&b, nir_channels(&b, src_pos, 0x3));
1366 
1367       if (devinfo->gen == 6) {
1368          /* Because gen6 only supports 4x interleved MSAA, we can do all the
1369           * blending we need with a single linear-interpolated texture lookup
1370           * at the center of the sample. The texture coordinates to be odd
1371           * integers so that they correspond to the center of a 2x2 block
1372           * representing the four samples that maxe up a pixel.  So we need
1373           * to multiply our X and Y coordinates each by 2 and then add 1.
1374           */
1375          assert(key->src_coords_normalized);
1376          assert(key->filter == BLORP_FILTER_AVERAGE);
1377          src_pos = nir_fadd(&b,
1378                             nir_i2f32(&b, src_pos),
1379                             nir_imm_float(&b, 0.5f));
1380          color = blorp_nir_tex(&b, &v, key, src_pos);
1381       } else {
1382          /* Gen7+ hardware doesn't automaticaly blend. */
1383          color = blorp_nir_combine_samples(&b, &v, src_pos, key->src_samples,
1384                                            key->tex_aux_usage,
1385                                            key->texture_data_type,
1386                                            key->filter);
1387       }
1388       break;
1389 
1390    default:
1391       unreachable("Invalid blorp filter");
1392    }
1393 
1394    if (!isl_swizzle_is_identity(key->src_swizzle)) {
1395       color = swizzle_color(&b, color, key->src_swizzle,
1396                             key->texture_data_type);
1397    }
1398 
1399    if (!isl_swizzle_is_identity(key->dst_swizzle)) {
1400       color = swizzle_color(&b, color, isl_swizzle_invert(key->dst_swizzle),
1401                             nir_type_int);
1402    }
1403 
1404    if (key->format_bit_cast) {
1405       assert(isl_swizzle_is_identity(key->src_swizzle));
1406       assert(isl_swizzle_is_identity(key->dst_swizzle));
1407       color = bit_cast_color(&b, color, key);
1408    } else if (key->dst_format) {
1409       color = convert_color(&b, color, key);
1410    } else if (key->uint32_to_sint) {
1411       /* Normally the hardware will take care of converting values from/to
1412        * the source and destination formats.  But a few cases need help.
1413        *
1414        * The Skylake PRM, volume 07, page 658 has a programming note:
1415        *
1416        *    "When using SINT or UINT rendertarget surface formats, Blending
1417        *     must be DISABLED. The Pre-Blend Color Clamp Enable and Color
1418        *     Clamp Range fields are ignored, and an implied clamp to the
1419        *     rendertarget surface format is performed."
1420        *
1421        * For UINT to SINT blits, our sample operation gives us a uint32_t,
1422        * but our render target write expects a signed int32_t number.  If we
1423        * simply passed the value along, the hardware would interpret a value
1424        * with bit 31 set as a negative value, clamping it to the largest
1425        * negative number the destination format could represent.  But the
1426        * actual source value is a positive number, so we want to clamp it
1427        * to INT_MAX.  To fix this, we explicitly take min(color, INT_MAX).
1428        */
1429       color = nir_umin(&b, color, nir_imm_int(&b, INT32_MAX));
1430    } else if (key->sint32_to_uint) {
1431       /* Similar to above, but clamping negative numbers to zero. */
1432       color = nir_imax(&b, color, nir_imm_int(&b, 0));
1433    }
1434 
1435    if (key->dst_rgb) {
1436       /* The destination image is bound as a red texture three times as wide
1437        * as the actual image.  Our shader is effectively running one color
1438        * component at a time.  We need to pick off the appropriate component
1439        * from the source color and write that to destination red.
1440        */
1441       assert(dst_pos->num_components == 2);
1442 
1443       nir_ssa_def *color_component =
1444          nir_bcsel(&b, nir_ieq_imm(&b, comp, 0),
1445                        nir_channel(&b, color, 0),
1446                        nir_bcsel(&b, nir_ieq_imm(&b, comp, 1),
1447                                      nir_channel(&b, color, 1),
1448                                      nir_channel(&b, color, 2)));
1449 
1450       nir_ssa_def *u = nir_ssa_undef(&b, 1, 32);
1451       color = nir_vec4(&b, color_component, u, u, u);
1452    }
1453 
1454    if (key->dst_usage == ISL_SURF_USAGE_RENDER_TARGET_BIT) {
1455       nir_variable *color_out =
1456          nir_variable_create(b.shader, nir_var_shader_out,
1457                              glsl_vec4_type(), "gl_FragColor");
1458       color_out->data.location = FRAG_RESULT_COLOR;
1459       nir_store_var(&b, color_out, color, 0xf);
1460    } else if (key->dst_usage == ISL_SURF_USAGE_DEPTH_BIT) {
1461       nir_variable *depth_out =
1462          nir_variable_create(b.shader, nir_var_shader_out,
1463                              glsl_float_type(), "gl_FragDepth");
1464       depth_out->data.location = FRAG_RESULT_DEPTH;
1465       nir_store_var(&b, depth_out, nir_channel(&b, color, 0), 0x1);
1466    } else if (key->dst_usage == ISL_SURF_USAGE_STENCIL_BIT) {
1467       nir_variable *stencil_out =
1468          nir_variable_create(b.shader, nir_var_shader_out,
1469                              glsl_int_type(), "gl_FragStencilRef");
1470       stencil_out->data.location = FRAG_RESULT_STENCIL;
1471       nir_store_var(&b, stencil_out, nir_channel(&b, color, 0), 0x1);
1472    } else {
1473       unreachable("Invalid destination usage");
1474    }
1475 
1476    return b.shader;
1477 }
1478 
1479 static bool
brw_blorp_get_blit_kernel(struct blorp_batch * batch,struct blorp_params * params,const struct brw_blorp_blit_prog_key * prog_key)1480 brw_blorp_get_blit_kernel(struct blorp_batch *batch,
1481                           struct blorp_params *params,
1482                           const struct brw_blorp_blit_prog_key *prog_key)
1483 {
1484    struct blorp_context *blorp = batch->blorp;
1485 
1486    if (blorp->lookup_shader(batch, prog_key, sizeof(*prog_key),
1487                             &params->wm_prog_kernel, &params->wm_prog_data))
1488       return true;
1489 
1490    void *mem_ctx = ralloc_context(NULL);
1491 
1492    const unsigned *program;
1493    struct brw_wm_prog_data prog_data;
1494 
1495    nir_shader *nir = brw_blorp_build_nir_shader(blorp, mem_ctx, prog_key);
1496    nir->info.name = ralloc_strdup(nir, blorp_shader_type_to_name(prog_key->shader_type));
1497 
1498    struct brw_wm_prog_key wm_key;
1499    brw_blorp_init_wm_prog_key(&wm_key);
1500    wm_key.base.tex.compressed_multisample_layout_mask =
1501       isl_aux_usage_has_mcs(prog_key->tex_aux_usage);
1502    wm_key.base.tex.msaa_16 = prog_key->tex_samples == 16;
1503    wm_key.multisample_fbo = prog_key->rt_samples > 1;
1504 
1505    program = blorp_compile_fs(blorp, mem_ctx, nir, &wm_key, false,
1506                               &prog_data);
1507 
1508    bool result =
1509       blorp->upload_shader(batch, MESA_SHADER_FRAGMENT,
1510                            prog_key, sizeof(*prog_key),
1511                            program, prog_data.base.program_size,
1512                            &prog_data.base, sizeof(prog_data),
1513                            &params->wm_prog_kernel, &params->wm_prog_data);
1514 
1515    ralloc_free(mem_ctx);
1516    return result;
1517 }
1518 
1519 static void
brw_blorp_setup_coord_transform(struct brw_blorp_coord_transform * xform,GLfloat src0,GLfloat src1,GLfloat dst0,GLfloat dst1,bool mirror)1520 brw_blorp_setup_coord_transform(struct brw_blorp_coord_transform *xform,
1521                                 GLfloat src0, GLfloat src1,
1522                                 GLfloat dst0, GLfloat dst1,
1523                                 bool mirror)
1524 {
1525    double scale = (double)(src1 - src0) / (double)(dst1 - dst0);
1526    if (!mirror) {
1527       /* When not mirroring a coordinate (say, X), we need:
1528        *   src_x - src_x0 = (dst_x - dst_x0 + 0.5) * scale
1529        * Therefore:
1530        *   src_x = src_x0 + (dst_x - dst_x0 + 0.5) * scale
1531        *
1532        * blorp program uses "round toward zero" to convert the
1533        * transformed floating point coordinates to integer coordinates,
1534        * whereas the behaviour we actually want is "round to nearest",
1535        * so 0.5 provides the necessary correction.
1536        */
1537       xform->multiplier = scale;
1538       xform->offset = src0 + (-(double)dst0 + 0.5) * scale;
1539    } else {
1540       /* When mirroring X we need:
1541        *   src_x - src_x0 = dst_x1 - dst_x - 0.5
1542        * Therefore:
1543        *   src_x = src_x0 + (dst_x1 -dst_x - 0.5) * scale
1544        */
1545       xform->multiplier = -scale;
1546       xform->offset = src0 + ((double)dst1 - 0.5) * scale;
1547    }
1548 }
1549 
1550 static inline void
surf_get_intratile_offset_px(struct brw_blorp_surface_info * info,uint32_t * tile_x_px,uint32_t * tile_y_px)1551 surf_get_intratile_offset_px(struct brw_blorp_surface_info *info,
1552                              uint32_t *tile_x_px, uint32_t *tile_y_px)
1553 {
1554    if (info->surf.msaa_layout == ISL_MSAA_LAYOUT_INTERLEAVED) {
1555       struct isl_extent2d px_size_sa =
1556          isl_get_interleaved_msaa_px_size_sa(info->surf.samples);
1557       assert(info->tile_x_sa % px_size_sa.width == 0);
1558       assert(info->tile_y_sa % px_size_sa.height == 0);
1559       *tile_x_px = info->tile_x_sa / px_size_sa.width;
1560       *tile_y_px = info->tile_y_sa / px_size_sa.height;
1561    } else {
1562       *tile_x_px = info->tile_x_sa;
1563       *tile_y_px = info->tile_y_sa;
1564    }
1565 }
1566 
1567 void
blorp_surf_convert_to_single_slice(const struct isl_device * isl_dev,struct brw_blorp_surface_info * info)1568 blorp_surf_convert_to_single_slice(const struct isl_device *isl_dev,
1569                                    struct brw_blorp_surface_info *info)
1570 {
1571    bool ok UNUSED;
1572 
1573    /* It would be insane to try and do this on a compressed surface */
1574    assert(info->aux_usage == ISL_AUX_USAGE_NONE);
1575 
1576    /* Just bail if we have nothing to do. */
1577    if (info->surf.dim == ISL_SURF_DIM_2D &&
1578        info->view.base_level == 0 && info->view.base_array_layer == 0 &&
1579        info->surf.levels == 1 && info->surf.logical_level0_px.array_len == 1)
1580       return;
1581 
1582    /* If this gets triggered then we've gotten here twice which.  This
1583     * shouldn't happen thanks to the above early return.
1584     */
1585    assert(info->tile_x_sa == 0 && info->tile_y_sa == 0);
1586 
1587    uint32_t layer = 0, z = 0;
1588    if (info->surf.dim == ISL_SURF_DIM_3D)
1589       z = info->view.base_array_layer + info->z_offset;
1590    else
1591       layer = info->view.base_array_layer;
1592 
1593    uint32_t byte_offset;
1594    isl_surf_get_image_surf(isl_dev, &info->surf,
1595                            info->view.base_level, layer, z,
1596                            &info->surf,
1597                            &byte_offset, &info->tile_x_sa, &info->tile_y_sa);
1598    info->addr.offset += byte_offset;
1599 
1600    uint32_t tile_x_px, tile_y_px;
1601    surf_get_intratile_offset_px(info, &tile_x_px, &tile_y_px);
1602 
1603    /* Instead of using the X/Y Offset fields in RENDER_SURFACE_STATE, we place
1604     * the image at the tile boundary and offset our sampling or rendering.
1605     * For this reason, we need to grow the image by the offset to ensure that
1606     * the hardware doesn't think we've gone past the edge.
1607     */
1608    info->surf.logical_level0_px.w += tile_x_px;
1609    info->surf.logical_level0_px.h += tile_y_px;
1610    info->surf.phys_level0_sa.w += info->tile_x_sa;
1611    info->surf.phys_level0_sa.h += info->tile_y_sa;
1612 
1613    /* The view is also different now. */
1614    info->view.base_level = 0;
1615    info->view.levels = 1;
1616    info->view.base_array_layer = 0;
1617    info->view.array_len = 1;
1618    info->z_offset = 0;
1619 }
1620 
1621 void
blorp_surf_fake_interleaved_msaa(const struct isl_device * isl_dev,struct brw_blorp_surface_info * info)1622 blorp_surf_fake_interleaved_msaa(const struct isl_device *isl_dev,
1623                                  struct brw_blorp_surface_info *info)
1624 {
1625    assert(info->surf.msaa_layout == ISL_MSAA_LAYOUT_INTERLEAVED);
1626 
1627    /* First, we need to convert it to a simple 1-level 1-layer 2-D surface */
1628    blorp_surf_convert_to_single_slice(isl_dev, info);
1629 
1630    info->surf.logical_level0_px = info->surf.phys_level0_sa;
1631    info->surf.samples = 1;
1632    info->surf.msaa_layout = ISL_MSAA_LAYOUT_NONE;
1633 }
1634 
1635 void
blorp_surf_retile_w_to_y(const struct isl_device * isl_dev,struct brw_blorp_surface_info * info)1636 blorp_surf_retile_w_to_y(const struct isl_device *isl_dev,
1637                          struct brw_blorp_surface_info *info)
1638 {
1639    assert(info->surf.tiling == ISL_TILING_W);
1640 
1641    /* First, we need to convert it to a simple 1-level 1-layer 2-D surface */
1642    blorp_surf_convert_to_single_slice(isl_dev, info);
1643 
1644    /* On gen7+, we don't have interleaved multisampling for color render
1645     * targets so we have to fake it.
1646     *
1647     * TODO: Are we sure we don't also need to fake it on gen6?
1648     */
1649    if (isl_dev->info->gen > 6 &&
1650        info->surf.msaa_layout == ISL_MSAA_LAYOUT_INTERLEAVED) {
1651       blorp_surf_fake_interleaved_msaa(isl_dev, info);
1652    }
1653 
1654    if (isl_dev->info->gen == 6) {
1655       /* Gen6 stencil buffers have a very large alignment coming in from the
1656        * miptree.  It's out-of-bounds for what the surface state can handle.
1657        * Since we have a single layer and level, it doesn't really matter as
1658        * long as we don't pass a bogus value into isl_surf_fill_state().
1659        */
1660       info->surf.image_alignment_el = isl_extent3d(4, 2, 1);
1661    }
1662 
1663    /* Now that we've converted everything to a simple 2-D surface with only
1664     * one miplevel, we can go about retiling it.
1665     */
1666    const unsigned x_align = 8, y_align = info->surf.samples != 0 ? 8 : 4;
1667    info->surf.tiling = ISL_TILING_Y0;
1668    info->surf.logical_level0_px.width =
1669       ALIGN(info->surf.logical_level0_px.width, x_align) * 2;
1670    info->surf.logical_level0_px.height =
1671       ALIGN(info->surf.logical_level0_px.height, y_align) / 2;
1672    info->tile_x_sa *= 2;
1673    info->tile_y_sa /= 2;
1674 }
1675 
1676 static bool
can_shrink_surface(const struct brw_blorp_surface_info * surf)1677 can_shrink_surface(const struct brw_blorp_surface_info *surf)
1678 {
1679    /* The current code doesn't support offsets into the aux buffers. This
1680     * should be possible, but we need to make sure the offset is page
1681     * aligned for both the surface and the aux buffer surface. Generally
1682     * this mean using the page aligned offset for the aux buffer.
1683     *
1684     * Currently the cases where we must split the blit are limited to cases
1685     * where we don't have a aux buffer.
1686     */
1687    if (surf->aux_addr.buffer != NULL)
1688       return false;
1689 
1690    /* We can't support splitting the blit for gen <= 7, because the qpitch
1691     * size is calculated by the hardware based on the surface height for
1692     * gen <= 7. In gen >= 8, the qpitch is controlled by the driver.
1693     */
1694    if (surf->surf.msaa_layout == ISL_MSAA_LAYOUT_ARRAY)
1695       return false;
1696 
1697    return true;
1698 }
1699 
1700 static unsigned
get_max_surface_size(const struct gen_device_info * devinfo,const struct brw_blorp_surface_info * surf)1701 get_max_surface_size(const struct gen_device_info *devinfo,
1702                      const struct brw_blorp_surface_info *surf)
1703 {
1704    const unsigned max = devinfo->gen >= 7 ? 16384 : 8192;
1705    if (split_blorp_blit_debug && can_shrink_surface(surf))
1706       return max >> 4; /* A smaller restriction when debug is enabled */
1707    else
1708       return max;
1709 }
1710 
1711 struct blt_axis {
1712    double src0, src1, dst0, dst1;
1713    bool mirror;
1714 };
1715 
1716 struct blt_coords {
1717    struct blt_axis x, y;
1718 };
1719 
1720 static enum isl_format
get_red_format_for_rgb_format(enum isl_format format)1721 get_red_format_for_rgb_format(enum isl_format format)
1722 {
1723    const struct isl_format_layout *fmtl = isl_format_get_layout(format);
1724 
1725    switch (fmtl->channels.r.bits) {
1726    case 8:
1727       switch (fmtl->channels.r.type) {
1728       case ISL_UNORM:
1729          return ISL_FORMAT_R8_UNORM;
1730       case ISL_SNORM:
1731          return ISL_FORMAT_R8_SNORM;
1732       case ISL_UINT:
1733          return ISL_FORMAT_R8_UINT;
1734       case ISL_SINT:
1735          return ISL_FORMAT_R8_SINT;
1736       default:
1737          unreachable("Invalid 8-bit RGB channel type");
1738       }
1739    case 16:
1740       switch (fmtl->channels.r.type) {
1741       case ISL_UNORM:
1742          return ISL_FORMAT_R16_UNORM;
1743       case ISL_SNORM:
1744          return ISL_FORMAT_R16_SNORM;
1745       case ISL_SFLOAT:
1746          return ISL_FORMAT_R16_FLOAT;
1747       case ISL_UINT:
1748          return ISL_FORMAT_R16_UINT;
1749       case ISL_SINT:
1750          return ISL_FORMAT_R16_SINT;
1751       default:
1752          unreachable("Invalid 8-bit RGB channel type");
1753       }
1754    case 32:
1755       switch (fmtl->channels.r.type) {
1756       case ISL_SFLOAT:
1757          return ISL_FORMAT_R32_FLOAT;
1758       case ISL_UINT:
1759          return ISL_FORMAT_R32_UINT;
1760       case ISL_SINT:
1761          return ISL_FORMAT_R32_SINT;
1762       default:
1763          unreachable("Invalid 8-bit RGB channel type");
1764       }
1765    default:
1766       unreachable("Invalid number of red channel bits");
1767    }
1768 }
1769 
1770 void
surf_fake_rgb_with_red(const struct isl_device * isl_dev,struct brw_blorp_surface_info * info)1771 surf_fake_rgb_with_red(const struct isl_device *isl_dev,
1772                        struct brw_blorp_surface_info *info)
1773 {
1774    blorp_surf_convert_to_single_slice(isl_dev, info);
1775 
1776    info->surf.logical_level0_px.width *= 3;
1777    info->surf.phys_level0_sa.width *= 3;
1778    info->tile_x_sa *= 3;
1779 
1780    enum isl_format red_format =
1781       get_red_format_for_rgb_format(info->view.format);
1782 
1783    assert(isl_format_get_layout(red_format)->channels.r.type ==
1784           isl_format_get_layout(info->view.format)->channels.r.type);
1785    assert(isl_format_get_layout(red_format)->channels.r.bits ==
1786           isl_format_get_layout(info->view.format)->channels.r.bits);
1787 
1788    info->surf.format = info->view.format = red_format;
1789 }
1790 
1791 enum blit_shrink_status {
1792    BLIT_NO_SHRINK = 0,
1793    BLIT_SRC_WIDTH_SHRINK   = (1 << 0),
1794    BLIT_DST_WIDTH_SHRINK   = (1 << 1),
1795    BLIT_SRC_HEIGHT_SHRINK  = (1 << 2),
1796    BLIT_DST_HEIGHT_SHRINK  = (1 << 3),
1797 };
1798 
1799 /* Try to blit. If the surface parameters exceed the size allowed by hardware,
1800  * then enum blit_shrink_status will be returned. If BLIT_NO_SHRINK is
1801  * returned, then the blit was successful.
1802  */
1803 static enum blit_shrink_status
try_blorp_blit(struct blorp_batch * batch,struct blorp_params * params,struct brw_blorp_blit_prog_key * wm_prog_key,struct blt_coords * coords)1804 try_blorp_blit(struct blorp_batch *batch,
1805                struct blorp_params *params,
1806                struct brw_blorp_blit_prog_key *wm_prog_key,
1807                struct blt_coords *coords)
1808 {
1809    const struct gen_device_info *devinfo = batch->blorp->isl_dev->info;
1810 
1811    if (params->dst.surf.usage & ISL_SURF_USAGE_DEPTH_BIT) {
1812       if (devinfo->gen >= 7) {
1813          /* We can render as depth on Gen5 but there's no real advantage since
1814           * it doesn't support MSAA or HiZ.  On Gen4, we can't always render
1815           * to depth due to issues with depth buffers and mip-mapping.  On
1816           * Gen6, we can do everything but we have weird offsetting for HiZ
1817           * and stencil.  It's easier to just render using the color pipe
1818           * on those platforms.
1819           */
1820          wm_prog_key->dst_usage = ISL_SURF_USAGE_DEPTH_BIT;
1821       } else {
1822          wm_prog_key->dst_usage = ISL_SURF_USAGE_RENDER_TARGET_BIT;
1823       }
1824    } else if (params->dst.surf.usage & ISL_SURF_USAGE_STENCIL_BIT) {
1825       assert(params->dst.surf.format == ISL_FORMAT_R8_UINT);
1826       if (devinfo->gen >= 9) {
1827          wm_prog_key->dst_usage = ISL_SURF_USAGE_STENCIL_BIT;
1828       } else {
1829          wm_prog_key->dst_usage = ISL_SURF_USAGE_RENDER_TARGET_BIT;
1830       }
1831    } else {
1832       wm_prog_key->dst_usage = ISL_SURF_USAGE_RENDER_TARGET_BIT;
1833    }
1834 
1835    if (isl_format_has_sint_channel(params->src.view.format)) {
1836       wm_prog_key->texture_data_type = nir_type_int;
1837    } else if (isl_format_has_uint_channel(params->src.view.format)) {
1838       wm_prog_key->texture_data_type = nir_type_uint;
1839    } else {
1840       wm_prog_key->texture_data_type = nir_type_float;
1841    }
1842 
1843    /* src_samples and dst_samples are the true sample counts */
1844    wm_prog_key->src_samples = params->src.surf.samples;
1845    wm_prog_key->dst_samples = params->dst.surf.samples;
1846 
1847    wm_prog_key->tex_aux_usage = params->src.aux_usage;
1848 
1849    /* src_layout and dst_layout indicate the true MSAA layout used by src and
1850     * dst.
1851     */
1852    wm_prog_key->src_layout = params->src.surf.msaa_layout;
1853    wm_prog_key->dst_layout = params->dst.surf.msaa_layout;
1854 
1855    /* Round floating point values to nearest integer to avoid "off by one texel"
1856     * kind of errors when blitting.
1857     */
1858    params->x0 = params->wm_inputs.discard_rect.x0 = round(coords->x.dst0);
1859    params->y0 = params->wm_inputs.discard_rect.y0 = round(coords->y.dst0);
1860    params->x1 = params->wm_inputs.discard_rect.x1 = round(coords->x.dst1);
1861    params->y1 = params->wm_inputs.discard_rect.y1 = round(coords->y.dst1);
1862 
1863    brw_blorp_setup_coord_transform(&params->wm_inputs.coord_transform[0],
1864                                    coords->x.src0, coords->x.src1,
1865                                    coords->x.dst0, coords->x.dst1,
1866                                    coords->x.mirror);
1867    brw_blorp_setup_coord_transform(&params->wm_inputs.coord_transform[1],
1868                                    coords->y.src0, coords->y.src1,
1869                                    coords->y.dst0, coords->y.dst1,
1870                                    coords->y.mirror);
1871 
1872 
1873    if (devinfo->gen == 4) {
1874       /* The MinLOD and MinimumArrayElement don't work properly for cube maps.
1875        * Convert them to a single slice on gen4.
1876        */
1877       if (params->dst.surf.usage & ISL_SURF_USAGE_CUBE_BIT) {
1878          blorp_surf_convert_to_single_slice(batch->blorp->isl_dev, &params->dst);
1879          wm_prog_key->need_dst_offset = true;
1880       }
1881 
1882       if (params->src.surf.usage & ISL_SURF_USAGE_CUBE_BIT) {
1883          blorp_surf_convert_to_single_slice(batch->blorp->isl_dev, &params->src);
1884          wm_prog_key->need_src_offset = true;
1885       }
1886    }
1887 
1888    if (devinfo->gen > 6 &&
1889        !isl_surf_usage_is_depth_or_stencil(wm_prog_key->dst_usage) &&
1890        params->dst.surf.msaa_layout == ISL_MSAA_LAYOUT_INTERLEAVED) {
1891       assert(params->dst.surf.samples > 1);
1892 
1893       /* We must expand the rectangle we send through the rendering pipeline,
1894        * to account for the fact that we are mapping the destination region as
1895        * single-sampled when it is in fact multisampled.  We must also align
1896        * it to a multiple of the multisampling pattern, because the
1897        * differences between multisampled and single-sampled surface formats
1898        * will mean that pixels are scrambled within the multisampling pattern.
1899        * TODO: what if this makes the coordinates too large?
1900        *
1901        * Note: this only works if the destination surface uses the IMS layout.
1902        * If it's UMS, then we have no choice but to set up the rendering
1903        * pipeline as multisampled.
1904        */
1905       struct isl_extent2d px_size_sa =
1906          isl_get_interleaved_msaa_px_size_sa(params->dst.surf.samples);
1907       params->x0 = ROUND_DOWN_TO(params->x0, 2) * px_size_sa.width;
1908       params->y0 = ROUND_DOWN_TO(params->y0, 2) * px_size_sa.height;
1909       params->x1 = ALIGN(params->x1, 2) * px_size_sa.width;
1910       params->y1 = ALIGN(params->y1, 2) * px_size_sa.height;
1911 
1912       blorp_surf_fake_interleaved_msaa(batch->blorp->isl_dev, &params->dst);
1913 
1914       wm_prog_key->use_kill = true;
1915       wm_prog_key->need_dst_offset = true;
1916    }
1917 
1918    if (params->dst.surf.tiling == ISL_TILING_W &&
1919        wm_prog_key->dst_usage != ISL_SURF_USAGE_STENCIL_BIT) {
1920       /* We must modify the rectangle we send through the rendering pipeline
1921        * (and the size and x/y offset of the destination surface), to account
1922        * for the fact that we are mapping it as Y-tiled when it is in fact
1923        * W-tiled.
1924        *
1925        * Both Y tiling and W tiling can be understood as organizations of
1926        * 32-byte sub-tiles; within each 32-byte sub-tile, the layout of pixels
1927        * is different, but the layout of the 32-byte sub-tiles within the 4k
1928        * tile is the same (8 sub-tiles across by 16 sub-tiles down, in
1929        * column-major order).  In Y tiling, the sub-tiles are 16 bytes wide
1930        * and 2 rows high; in W tiling, they are 8 bytes wide and 4 rows high.
1931        *
1932        * Therefore, to account for the layout differences within the 32-byte
1933        * sub-tiles, we must expand the rectangle so the X coordinates of its
1934        * edges are multiples of 8 (the W sub-tile width), and its Y
1935        * coordinates of its edges are multiples of 4 (the W sub-tile height).
1936        * Then we need to scale the X and Y coordinates of the rectangle to
1937        * account for the differences in aspect ratio between the Y and W
1938        * sub-tiles.  We need to modify the layer width and height similarly.
1939        *
1940        * A correction needs to be applied when MSAA is in use: since
1941        * INTEL_MSAA_LAYOUT_IMS uses an interleaving pattern whose height is 4,
1942        * we need to align the Y coordinates to multiples of 8, so that when
1943        * they are divided by two they are still multiples of 4.
1944        *
1945        * Note: Since the x/y offset of the surface will be applied using the
1946        * SURFACE_STATE command packet, it will be invisible to the swizzling
1947        * code in the shader; therefore it needs to be in a multiple of the
1948        * 32-byte sub-tile size.  Fortunately it is, since the sub-tile is 8
1949        * pixels wide and 4 pixels high (when viewed as a W-tiled stencil
1950        * buffer), and the miplevel alignment used for stencil buffers is 8
1951        * pixels horizontally and either 4 or 8 pixels vertically (see
1952        * intel_horizontal_texture_alignment_unit() and
1953        * intel_vertical_texture_alignment_unit()).
1954        *
1955        * Note: Also, since the SURFACE_STATE command packet can only apply
1956        * offsets that are multiples of 4 pixels horizontally and 2 pixels
1957        * vertically, it is important that the offsets will be multiples of
1958        * these sizes after they are converted into Y-tiled coordinates.
1959        * Fortunately they will be, since we know from above that the offsets
1960        * are a multiple of the 32-byte sub-tile size, and in Y-tiled
1961        * coordinates the sub-tile is 16 pixels wide and 2 pixels high.
1962        *
1963        * TODO: what if this makes the coordinates (or the texture size) too
1964        * large?
1965        */
1966       const unsigned x_align = 8;
1967       const unsigned y_align = params->dst.surf.samples != 0 ? 8 : 4;
1968       params->x0 = ROUND_DOWN_TO(params->x0, x_align) * 2;
1969       params->y0 = ROUND_DOWN_TO(params->y0, y_align) / 2;
1970       params->x1 = ALIGN(params->x1, x_align) * 2;
1971       params->y1 = ALIGN(params->y1, y_align) / 2;
1972 
1973       /* Retile the surface to Y-tiled */
1974       blorp_surf_retile_w_to_y(batch->blorp->isl_dev, &params->dst);
1975 
1976       wm_prog_key->dst_tiled_w = true;
1977       wm_prog_key->use_kill = true;
1978       wm_prog_key->need_dst_offset = true;
1979 
1980       if (params->dst.surf.samples > 1) {
1981          /* If the destination surface is a W-tiled multisampled stencil
1982           * buffer that we're mapping as Y tiled, then we need to arrange for
1983           * the WM program to run once per sample rather than once per pixel,
1984           * because the memory layout of related samples doesn't match between
1985           * W and Y tiling.
1986           */
1987          wm_prog_key->persample_msaa_dispatch = true;
1988       }
1989    }
1990 
1991    if (devinfo->gen < 8 && params->src.surf.tiling == ISL_TILING_W) {
1992       /* On Haswell and earlier, we have to fake W-tiled sources as Y-tiled.
1993        * Broadwell adds support for sampling from stencil.
1994        *
1995        * See the comments above concerning x/y offset alignment for the
1996        * destination surface.
1997        *
1998        * TODO: what if this makes the texture size too large?
1999        */
2000       blorp_surf_retile_w_to_y(batch->blorp->isl_dev, &params->src);
2001 
2002       wm_prog_key->src_tiled_w = true;
2003       wm_prog_key->need_src_offset = true;
2004    }
2005 
2006    /* tex_samples and rt_samples are the sample counts that are set up in
2007     * SURFACE_STATE.
2008     */
2009    wm_prog_key->tex_samples = params->src.surf.samples;
2010    wm_prog_key->rt_samples  = params->dst.surf.samples;
2011 
2012    /* tex_layout and rt_layout indicate the MSAA layout the GPU pipeline will
2013     * use to access the source and destination surfaces.
2014     */
2015    wm_prog_key->tex_layout = params->src.surf.msaa_layout;
2016    wm_prog_key->rt_layout = params->dst.surf.msaa_layout;
2017 
2018    if (params->src.surf.samples > 0 && params->dst.surf.samples > 1) {
2019       /* We are blitting from a multisample buffer to a multisample buffer, so
2020        * we must preserve samples within a pixel.  This means we have to
2021        * arrange for the WM program to run once per sample rather than once
2022        * per pixel.
2023        */
2024       wm_prog_key->persample_msaa_dispatch = true;
2025    }
2026 
2027    params->num_samples = params->dst.surf.samples;
2028 
2029    if ((wm_prog_key->filter == BLORP_FILTER_AVERAGE ||
2030         wm_prog_key->filter == BLORP_FILTER_BILINEAR) &&
2031        batch->blorp->isl_dev->info->gen <= 6) {
2032       /* Gen4-5 don't support non-normalized texture coordinates */
2033       wm_prog_key->src_coords_normalized = true;
2034       params->wm_inputs.src_inv_size[0] =
2035          1.0f / minify(params->src.surf.logical_level0_px.width,
2036                        params->src.view.base_level);
2037       params->wm_inputs.src_inv_size[1] =
2038          1.0f / minify(params->src.surf.logical_level0_px.height,
2039                        params->src.view.base_level);
2040    }
2041 
2042    if (isl_format_get_layout(params->dst.view.format)->bpb % 3 == 0) {
2043       /* We can't render to  RGB formats natively because they aren't a
2044        * power-of-two size.  Instead, we fake them by using a red format
2045        * with the same channel type and size and emitting shader code to
2046        * only write one channel at a time.
2047        */
2048       params->x0 *= 3;
2049       params->x1 *= 3;
2050 
2051       /* If it happens to be sRGB, we need to force a conversion */
2052       if (params->dst.view.format == ISL_FORMAT_R8G8B8_UNORM_SRGB)
2053          wm_prog_key->dst_format = ISL_FORMAT_R8G8B8_UNORM_SRGB;
2054 
2055       surf_fake_rgb_with_red(batch->blorp->isl_dev, &params->dst);
2056 
2057       wm_prog_key->dst_rgb = true;
2058       wm_prog_key->need_dst_offset = true;
2059    } else if (isl_format_is_rgbx(params->dst.view.format)) {
2060       /* We can handle RGBX formats easily enough by treating them as RGBA */
2061       params->dst.view.format =
2062          isl_format_rgbx_to_rgba(params->dst.view.format);
2063    } else if (params->dst.view.format == ISL_FORMAT_R24_UNORM_X8_TYPELESS &&
2064               wm_prog_key->dst_usage != ISL_SURF_USAGE_DEPTH_BIT) {
2065       wm_prog_key->dst_format = params->dst.view.format;
2066       params->dst.view.format = ISL_FORMAT_R32_UINT;
2067    } else if (params->dst.view.format == ISL_FORMAT_A4B4G4R4_UNORM) {
2068       params->dst.view.swizzle =
2069          isl_swizzle_compose(params->dst.view.swizzle,
2070                              ISL_SWIZZLE(ALPHA, RED, GREEN, BLUE));
2071       params->dst.view.format = ISL_FORMAT_B4G4R4A4_UNORM;
2072    } else if (params->dst.view.format == ISL_FORMAT_L8_UNORM_SRGB) {
2073       wm_prog_key->dst_format = params->dst.view.format;
2074       params->dst.view.format = ISL_FORMAT_R8_UNORM;
2075    } else if (params->dst.view.format == ISL_FORMAT_R9G9B9E5_SHAREDEXP) {
2076       wm_prog_key->dst_format = params->dst.view.format;
2077       params->dst.view.format = ISL_FORMAT_R32_UINT;
2078    }
2079 
2080    if (devinfo->gen <= 7 && !devinfo->is_haswell &&
2081        !isl_swizzle_is_identity(params->src.view.swizzle)) {
2082       wm_prog_key->src_swizzle = params->src.view.swizzle;
2083       params->src.view.swizzle = ISL_SWIZZLE_IDENTITY;
2084    } else {
2085       wm_prog_key->src_swizzle = ISL_SWIZZLE_IDENTITY;
2086    }
2087 
2088    if (!isl_swizzle_supports_rendering(devinfo, params->dst.view.swizzle)) {
2089       wm_prog_key->dst_swizzle = params->dst.view.swizzle;
2090       params->dst.view.swizzle = ISL_SWIZZLE_IDENTITY;
2091    } else {
2092       wm_prog_key->dst_swizzle = ISL_SWIZZLE_IDENTITY;
2093    }
2094 
2095    if (params->src.tile_x_sa || params->src.tile_y_sa) {
2096       assert(wm_prog_key->need_src_offset);
2097       surf_get_intratile_offset_px(&params->src,
2098                                    &params->wm_inputs.src_offset.x,
2099                                    &params->wm_inputs.src_offset.y);
2100    }
2101 
2102    if (params->dst.tile_x_sa || params->dst.tile_y_sa) {
2103       assert(wm_prog_key->need_dst_offset);
2104       surf_get_intratile_offset_px(&params->dst,
2105                                    &params->wm_inputs.dst_offset.x,
2106                                    &params->wm_inputs.dst_offset.y);
2107       params->x0 += params->wm_inputs.dst_offset.x;
2108       params->y0 += params->wm_inputs.dst_offset.y;
2109       params->x1 += params->wm_inputs.dst_offset.x;
2110       params->y1 += params->wm_inputs.dst_offset.y;
2111    }
2112 
2113    /* For some texture types, we need to pass the layer through the sampler. */
2114    params->wm_inputs.src_z = params->src.z_offset;
2115 
2116    if (!brw_blorp_get_blit_kernel(batch, params, wm_prog_key))
2117       return 0;
2118 
2119    if (!blorp_ensure_sf_program(batch, params))
2120       return 0;
2121 
2122    unsigned result = 0;
2123    unsigned max_src_surface_size = get_max_surface_size(devinfo, &params->src);
2124    if (params->src.surf.logical_level0_px.width > max_src_surface_size)
2125       result |= BLIT_SRC_WIDTH_SHRINK;
2126    if (params->src.surf.logical_level0_px.height > max_src_surface_size)
2127       result |= BLIT_SRC_HEIGHT_SHRINK;
2128 
2129    unsigned max_dst_surface_size = get_max_surface_size(devinfo, &params->dst);
2130    if (params->dst.surf.logical_level0_px.width > max_dst_surface_size)
2131       result |= BLIT_DST_WIDTH_SHRINK;
2132    if (params->dst.surf.logical_level0_px.height > max_dst_surface_size)
2133       result |= BLIT_DST_HEIGHT_SHRINK;
2134 
2135    if (result == 0) {
2136       if (wm_prog_key->dst_usage == ISL_SURF_USAGE_DEPTH_BIT) {
2137          params->depth = params->dst;
2138          memset(&params->dst, 0, sizeof(params->dst));
2139       } else if (wm_prog_key->dst_usage == ISL_SURF_USAGE_STENCIL_BIT) {
2140          params->stencil = params->dst;
2141          params->stencil_mask = 0xff;
2142          memset(&params->dst, 0, sizeof(params->dst));
2143       }
2144 
2145       batch->blorp->exec(batch, params);
2146    }
2147 
2148    return result;
2149 }
2150 
2151 /* Adjust split blit source coordinates for the current destination
2152  * coordinates.
2153  */
2154 static void
adjust_split_source_coords(const struct blt_axis * orig,struct blt_axis * split_coords,double scale)2155 adjust_split_source_coords(const struct blt_axis *orig,
2156                            struct blt_axis *split_coords,
2157                            double scale)
2158 {
2159    /* When scale is greater than 0, then we are growing from the start, so
2160     * src0 uses delta0, and src1 uses delta1. When scale is less than 0, the
2161     * source range shrinks from the end. In that case src0 is adjusted by
2162     * delta1, and src1 is adjusted by delta0.
2163     */
2164    double delta0 = scale * (split_coords->dst0 - orig->dst0);
2165    double delta1 = scale * (split_coords->dst1 - orig->dst1);
2166    split_coords->src0 = orig->src0 + (scale >= 0.0 ? delta0 : delta1);
2167    split_coords->src1 = orig->src1 + (scale >= 0.0 ? delta1 : delta0);
2168 }
2169 
2170 static struct isl_extent2d
get_px_size_sa(const struct isl_surf * surf)2171 get_px_size_sa(const struct isl_surf *surf)
2172 {
2173    static const struct isl_extent2d one_to_one = { .w = 1, .h = 1 };
2174 
2175    if (surf->msaa_layout != ISL_MSAA_LAYOUT_INTERLEAVED)
2176       return one_to_one;
2177    else
2178       return isl_get_interleaved_msaa_px_size_sa(surf->samples);
2179 }
2180 
2181 static void
shrink_surface_params(const struct isl_device * dev,struct brw_blorp_surface_info * info,double * x0,double * x1,double * y0,double * y1)2182 shrink_surface_params(const struct isl_device *dev,
2183                       struct brw_blorp_surface_info *info,
2184                       double *x0, double *x1, double *y0, double *y1)
2185 {
2186    uint32_t byte_offset, x_offset_sa, y_offset_sa, size;
2187    struct isl_extent2d px_size_sa;
2188    int adjust;
2189 
2190    blorp_surf_convert_to_single_slice(dev, info);
2191 
2192    px_size_sa = get_px_size_sa(&info->surf);
2193 
2194    /* Because this gets called after we lower compressed images, the tile
2195     * offsets may be non-zero and we need to incorporate them in our
2196     * calculations.
2197     */
2198    x_offset_sa = (uint32_t)*x0 * px_size_sa.w + info->tile_x_sa;
2199    y_offset_sa = (uint32_t)*y0 * px_size_sa.h + info->tile_y_sa;
2200    isl_tiling_get_intratile_offset_sa(info->surf.tiling,
2201                                       info->surf.format, info->surf.row_pitch_B,
2202                                       x_offset_sa, y_offset_sa,
2203                                       &byte_offset,
2204                                       &info->tile_x_sa, &info->tile_y_sa);
2205 
2206    info->addr.offset += byte_offset;
2207 
2208    adjust = (int)info->tile_x_sa / px_size_sa.w - (int)*x0;
2209    *x0 += adjust;
2210    *x1 += adjust;
2211    info->tile_x_sa = 0;
2212 
2213    adjust = (int)info->tile_y_sa / px_size_sa.h - (int)*y0;
2214    *y0 += adjust;
2215    *y1 += adjust;
2216    info->tile_y_sa = 0;
2217 
2218    size = MIN2((uint32_t)ceil(*x1), info->surf.logical_level0_px.width);
2219    info->surf.logical_level0_px.width = size;
2220    info->surf.phys_level0_sa.width = size * px_size_sa.w;
2221 
2222    size = MIN2((uint32_t)ceil(*y1), info->surf.logical_level0_px.height);
2223    info->surf.logical_level0_px.height = size;
2224    info->surf.phys_level0_sa.height = size * px_size_sa.h;
2225 }
2226 
2227 static void
do_blorp_blit(struct blorp_batch * batch,const struct blorp_params * orig_params,struct brw_blorp_blit_prog_key * wm_prog_key,const struct blt_coords * orig)2228 do_blorp_blit(struct blorp_batch *batch,
2229               const struct blorp_params *orig_params,
2230               struct brw_blorp_blit_prog_key *wm_prog_key,
2231               const struct blt_coords *orig)
2232 {
2233    struct blorp_params params;
2234    struct blt_coords blit_coords;
2235    struct blt_coords split_coords = *orig;
2236    double w = orig->x.dst1 - orig->x.dst0;
2237    double h = orig->y.dst1 - orig->y.dst0;
2238    double x_scale = (orig->x.src1 - orig->x.src0) / w;
2239    double y_scale = (orig->y.src1 - orig->y.src0) / h;
2240    if (orig->x.mirror)
2241       x_scale = -x_scale;
2242    if (orig->y.mirror)
2243       y_scale = -y_scale;
2244 
2245    enum blit_shrink_status shrink = BLIT_NO_SHRINK;
2246    if (split_blorp_blit_debug) {
2247       if (can_shrink_surface(&orig_params->src))
2248          shrink |= BLIT_SRC_WIDTH_SHRINK | BLIT_SRC_HEIGHT_SHRINK;
2249       if (can_shrink_surface(&orig_params->dst))
2250          shrink |= BLIT_DST_WIDTH_SHRINK | BLIT_DST_HEIGHT_SHRINK;
2251    }
2252 
2253    bool x_done, y_done;
2254    do {
2255       params = *orig_params;
2256       blit_coords = split_coords;
2257 
2258       if (shrink & (BLIT_SRC_WIDTH_SHRINK | BLIT_SRC_HEIGHT_SHRINK)) {
2259          shrink_surface_params(batch->blorp->isl_dev, &params.src,
2260                                &blit_coords.x.src0, &blit_coords.x.src1,
2261                                &blit_coords.y.src0, &blit_coords.y.src1);
2262          wm_prog_key->need_src_offset = false;
2263       }
2264 
2265       if (shrink & (BLIT_DST_WIDTH_SHRINK | BLIT_DST_HEIGHT_SHRINK)) {
2266          shrink_surface_params(batch->blorp->isl_dev, &params.dst,
2267                                &blit_coords.x.dst0, &blit_coords.x.dst1,
2268                                &blit_coords.y.dst0, &blit_coords.y.dst1);
2269          wm_prog_key->need_dst_offset = false;
2270       }
2271 
2272       enum blit_shrink_status result =
2273          try_blorp_blit(batch, &params, wm_prog_key, &blit_coords);
2274 
2275       if (result & (BLIT_SRC_WIDTH_SHRINK | BLIT_SRC_HEIGHT_SHRINK))
2276          assert(can_shrink_surface(&orig_params->src));
2277 
2278       if (result & (BLIT_DST_WIDTH_SHRINK | BLIT_DST_HEIGHT_SHRINK))
2279          assert(can_shrink_surface(&orig_params->dst));
2280 
2281       if (result & (BLIT_SRC_WIDTH_SHRINK | BLIT_DST_WIDTH_SHRINK)) {
2282          w /= 2.0;
2283          assert(w >= 1.0);
2284          split_coords.x.dst1 = MIN2(split_coords.x.dst0 + w, orig->x.dst1);
2285          adjust_split_source_coords(&orig->x, &split_coords.x, x_scale);
2286       }
2287       if (result & (BLIT_SRC_HEIGHT_SHRINK | BLIT_DST_HEIGHT_SHRINK)) {
2288          h /= 2.0;
2289          assert(h >= 1.0);
2290          split_coords.y.dst1 = MIN2(split_coords.y.dst0 + h, orig->y.dst1);
2291          adjust_split_source_coords(&orig->y, &split_coords.y, y_scale);
2292       }
2293 
2294       if (result) {
2295          /* We may get less bits set on result than we had already, so make
2296           * sure we remember all the ways in which a resize is required.
2297           */
2298          shrink |= result;
2299          continue;
2300       }
2301 
2302       y_done = (orig->y.dst1 - split_coords.y.dst1 < 0.5);
2303       x_done = y_done && (orig->x.dst1 - split_coords.x.dst1 < 0.5);
2304       if (x_done) {
2305          break;
2306       } else if (y_done) {
2307          split_coords.x.dst0 += w;
2308          split_coords.x.dst1 = MIN2(split_coords.x.dst0 + w, orig->x.dst1);
2309          split_coords.y.dst0 = orig->y.dst0;
2310          split_coords.y.dst1 = MIN2(split_coords.y.dst0 + h, orig->y.dst1);
2311          adjust_split_source_coords(&orig->x, &split_coords.x, x_scale);
2312       } else {
2313          split_coords.y.dst0 += h;
2314          split_coords.y.dst1 = MIN2(split_coords.y.dst0 + h, orig->y.dst1);
2315          adjust_split_source_coords(&orig->y, &split_coords.y, y_scale);
2316       }
2317    } while (true);
2318 }
2319 
2320 void
blorp_blit(struct blorp_batch * batch,const struct blorp_surf * src_surf,unsigned src_level,float src_layer,enum isl_format src_format,struct isl_swizzle src_swizzle,const struct blorp_surf * dst_surf,unsigned dst_level,unsigned dst_layer,enum isl_format dst_format,struct isl_swizzle dst_swizzle,float src_x0,float src_y0,float src_x1,float src_y1,float dst_x0,float dst_y0,float dst_x1,float dst_y1,enum blorp_filter filter,bool mirror_x,bool mirror_y)2321 blorp_blit(struct blorp_batch *batch,
2322            const struct blorp_surf *src_surf,
2323            unsigned src_level, float src_layer,
2324            enum isl_format src_format, struct isl_swizzle src_swizzle,
2325            const struct blorp_surf *dst_surf,
2326            unsigned dst_level, unsigned dst_layer,
2327            enum isl_format dst_format, struct isl_swizzle dst_swizzle,
2328            float src_x0, float src_y0,
2329            float src_x1, float src_y1,
2330            float dst_x0, float dst_y0,
2331            float dst_x1, float dst_y1,
2332            enum blorp_filter filter,
2333            bool mirror_x, bool mirror_y)
2334 {
2335    struct blorp_params params;
2336    blorp_params_init(&params);
2337 
2338    /* We cannot handle combined depth and stencil. */
2339    if (src_surf->surf->usage & ISL_SURF_USAGE_STENCIL_BIT)
2340       assert(src_surf->surf->format == ISL_FORMAT_R8_UINT);
2341    if (dst_surf->surf->usage & ISL_SURF_USAGE_STENCIL_BIT)
2342       assert(dst_surf->surf->format == ISL_FORMAT_R8_UINT);
2343 
2344    if (dst_surf->surf->usage & ISL_SURF_USAGE_STENCIL_BIT) {
2345       assert(src_surf->surf->usage & ISL_SURF_USAGE_STENCIL_BIT);
2346       /* Prior to Broadwell, we can't render to R8_UINT */
2347       if (batch->blorp->isl_dev->info->gen < 8) {
2348          src_format = ISL_FORMAT_R8_UNORM;
2349          dst_format = ISL_FORMAT_R8_UNORM;
2350       }
2351    }
2352 
2353    brw_blorp_surface_info_init(batch->blorp, &params.src, src_surf, src_level,
2354                                src_layer, src_format, false);
2355    brw_blorp_surface_info_init(batch->blorp, &params.dst, dst_surf, dst_level,
2356                                dst_layer, dst_format, true);
2357 
2358    params.src.view.swizzle = src_swizzle;
2359    params.dst.view.swizzle = dst_swizzle;
2360 
2361    const struct isl_format_layout *src_fmtl =
2362       isl_format_get_layout(params.src.view.format);
2363 
2364    struct brw_blorp_blit_prog_key wm_prog_key = {
2365       .shader_type = BLORP_SHADER_TYPE_BLIT,
2366       .filter = filter,
2367       .sint32_to_uint = src_fmtl->channels.r.bits == 32 &&
2368                         isl_format_has_sint_channel(params.src.view.format) &&
2369                         isl_format_has_uint_channel(params.dst.view.format),
2370       .uint32_to_sint = src_fmtl->channels.r.bits == 32 &&
2371                         isl_format_has_uint_channel(params.src.view.format) &&
2372                         isl_format_has_sint_channel(params.dst.view.format),
2373    };
2374 
2375    /* Scaling factors used for bilinear filtering in multisample scaled
2376     * blits.
2377     */
2378    if (params.src.surf.samples == 16)
2379       wm_prog_key.x_scale = 4.0f;
2380    else
2381       wm_prog_key.x_scale = 2.0f;
2382    wm_prog_key.y_scale = params.src.surf.samples / wm_prog_key.x_scale;
2383 
2384    params.wm_inputs.rect_grid.x1 =
2385       minify(params.src.surf.logical_level0_px.width, src_level) *
2386       wm_prog_key.x_scale - 1.0f;
2387    params.wm_inputs.rect_grid.y1 =
2388       minify(params.src.surf.logical_level0_px.height, src_level) *
2389       wm_prog_key.y_scale - 1.0f;
2390 
2391    struct blt_coords coords = {
2392       .x = {
2393          .src0 = src_x0,
2394          .src1 = src_x1,
2395          .dst0 = dst_x0,
2396          .dst1 = dst_x1,
2397          .mirror = mirror_x
2398       },
2399       .y = {
2400          .src0 = src_y0,
2401          .src1 = src_y1,
2402          .dst0 = dst_y0,
2403          .dst1 = dst_y1,
2404          .mirror = mirror_y
2405       }
2406    };
2407 
2408    do_blorp_blit(batch, &params, &wm_prog_key, &coords);
2409 }
2410 
2411 static enum isl_format
get_copy_format_for_bpb(const struct isl_device * isl_dev,unsigned bpb)2412 get_copy_format_for_bpb(const struct isl_device *isl_dev, unsigned bpb)
2413 {
2414    /* The choice of UNORM and UINT formats is very intentional here.  Most
2415     * of the time, we want to use a UINT format to avoid any rounding error
2416     * in the blit.  For stencil blits, R8_UINT is required by the hardware.
2417     * (It's the only format allowed in conjunction with W-tiling.)  Also we
2418     * intentionally use the 4-channel formats whenever we can.  This is so
2419     * that, when we do a RGB <-> RGBX copy, the two formats will line up
2420     * even though one of them is 3/4 the size of the other.  The choice of
2421     * UNORM vs. UINT is also very intentional because we don't have 8 or
2422     * 16-bit RGB UINT formats until Sky Lake so we have to use UNORM there.
2423     * Fortunately, the only time we should ever use two different formats in
2424     * the table below is for RGB -> RGBA blits and so we will never have any
2425     * UNORM/UINT mismatch.
2426     */
2427    if (ISL_DEV_GEN(isl_dev) >= 9) {
2428       switch (bpb) {
2429       case 8:  return ISL_FORMAT_R8_UINT;
2430       case 16: return ISL_FORMAT_R8G8_UINT;
2431       case 24: return ISL_FORMAT_R8G8B8_UINT;
2432       case 32: return ISL_FORMAT_R8G8B8A8_UINT;
2433       case 48: return ISL_FORMAT_R16G16B16_UINT;
2434       case 64: return ISL_FORMAT_R16G16B16A16_UINT;
2435       case 96: return ISL_FORMAT_R32G32B32_UINT;
2436       case 128:return ISL_FORMAT_R32G32B32A32_UINT;
2437       default:
2438          unreachable("Unknown format bpb");
2439       }
2440    } else {
2441       switch (bpb) {
2442       case 8:  return ISL_FORMAT_R8_UINT;
2443       case 16: return ISL_FORMAT_R8G8_UINT;
2444       case 24: return ISL_FORMAT_R8G8B8_UNORM;
2445       case 32: return ISL_FORMAT_R8G8B8A8_UNORM;
2446       case 48: return ISL_FORMAT_R16G16B16_UNORM;
2447       case 64: return ISL_FORMAT_R16G16B16A16_UNORM;
2448       case 96: return ISL_FORMAT_R32G32B32_UINT;
2449       case 128:return ISL_FORMAT_R32G32B32A32_UINT;
2450       default:
2451          unreachable("Unknown format bpb");
2452       }
2453    }
2454 }
2455 
2456 /** Returns a UINT format that is CCS-compatible with the given format
2457  *
2458  * The PRM's say absolutely nothing about how render compression works.  The
2459  * only thing they provide is a list of formats on which it is and is not
2460  * supported.  Empirical testing indicates that the compression is only based
2461  * on the bit-layout of the format and the channel encoding doesn't matter.
2462  * So, while texture views don't work in general, you can create a view as
2463  * long as the bit-layout of the formats are the same.
2464  *
2465  * Fortunately, for every render compression capable format, the UINT format
2466  * with the same bit layout also supports render compression.  This means that
2467  * we only need to handle UINT formats for copy operations.  In order to do
2468  * copies between formats with different bit layouts, we attach both with a
2469  * UINT format and use bit_cast_color() to generate code to do the bit-cast
2470  * operation between the two bit layouts.
2471  */
2472 static enum isl_format
get_ccs_compatible_copy_format(const struct isl_format_layout * fmtl)2473 get_ccs_compatible_copy_format(const struct isl_format_layout *fmtl)
2474 {
2475    switch (fmtl->format) {
2476    case ISL_FORMAT_R32G32B32A32_FLOAT:
2477    case ISL_FORMAT_R32G32B32A32_SINT:
2478    case ISL_FORMAT_R32G32B32A32_UINT:
2479    case ISL_FORMAT_R32G32B32A32_UNORM:
2480    case ISL_FORMAT_R32G32B32A32_SNORM:
2481    case ISL_FORMAT_R32G32B32X32_FLOAT:
2482       return ISL_FORMAT_R32G32B32A32_UINT;
2483 
2484    case ISL_FORMAT_R16G16B16A16_UNORM:
2485    case ISL_FORMAT_R16G16B16A16_SNORM:
2486    case ISL_FORMAT_R16G16B16A16_SINT:
2487    case ISL_FORMAT_R16G16B16A16_UINT:
2488    case ISL_FORMAT_R16G16B16A16_FLOAT:
2489    case ISL_FORMAT_R16G16B16X16_UNORM:
2490    case ISL_FORMAT_R16G16B16X16_FLOAT:
2491       return ISL_FORMAT_R16G16B16A16_UINT;
2492 
2493    case ISL_FORMAT_R32G32_FLOAT:
2494    case ISL_FORMAT_R32G32_SINT:
2495    case ISL_FORMAT_R32G32_UINT:
2496    case ISL_FORMAT_R32G32_UNORM:
2497    case ISL_FORMAT_R32G32_SNORM:
2498       return ISL_FORMAT_R32G32_UINT;
2499 
2500    case ISL_FORMAT_B8G8R8A8_UNORM:
2501    case ISL_FORMAT_B8G8R8A8_UNORM_SRGB:
2502    case ISL_FORMAT_R8G8B8A8_UNORM:
2503    case ISL_FORMAT_R8G8B8A8_UNORM_SRGB:
2504    case ISL_FORMAT_R8G8B8A8_SNORM:
2505    case ISL_FORMAT_R8G8B8A8_SINT:
2506    case ISL_FORMAT_R8G8B8A8_UINT:
2507    case ISL_FORMAT_B8G8R8X8_UNORM:
2508    case ISL_FORMAT_B8G8R8X8_UNORM_SRGB:
2509    case ISL_FORMAT_R8G8B8X8_UNORM:
2510    case ISL_FORMAT_R8G8B8X8_UNORM_SRGB:
2511       return ISL_FORMAT_R8G8B8A8_UINT;
2512 
2513    case ISL_FORMAT_R16G16_UNORM:
2514    case ISL_FORMAT_R16G16_SNORM:
2515    case ISL_FORMAT_R16G16_SINT:
2516    case ISL_FORMAT_R16G16_UINT:
2517    case ISL_FORMAT_R16G16_FLOAT:
2518       return ISL_FORMAT_R16G16_UINT;
2519 
2520    case ISL_FORMAT_R32_SINT:
2521    case ISL_FORMAT_R32_UINT:
2522    case ISL_FORMAT_R32_FLOAT:
2523    case ISL_FORMAT_R32_UNORM:
2524    case ISL_FORMAT_R32_SNORM:
2525       return ISL_FORMAT_R32_UINT;
2526 
2527    case ISL_FORMAT_B10G10R10A2_UNORM:
2528    case ISL_FORMAT_B10G10R10A2_UNORM_SRGB:
2529    case ISL_FORMAT_R10G10B10A2_UNORM:
2530    case ISL_FORMAT_R10G10B10A2_UNORM_SRGB:
2531    case ISL_FORMAT_R10G10B10_FLOAT_A2_UNORM:
2532    case ISL_FORMAT_R10G10B10A2_UINT:
2533       return ISL_FORMAT_R10G10B10A2_UINT;
2534 
2535    case ISL_FORMAT_R16_UNORM:
2536    case ISL_FORMAT_R16_SNORM:
2537    case ISL_FORMAT_R16_SINT:
2538    case ISL_FORMAT_R16_UINT:
2539    case ISL_FORMAT_R16_FLOAT:
2540       return ISL_FORMAT_R16_UINT;
2541 
2542    case ISL_FORMAT_R8G8_UNORM:
2543    case ISL_FORMAT_R8G8_SNORM:
2544    case ISL_FORMAT_R8G8_SINT:
2545    case ISL_FORMAT_R8G8_UINT:
2546       return ISL_FORMAT_R8G8_UINT;
2547 
2548    case ISL_FORMAT_B5G5R5X1_UNORM:
2549    case ISL_FORMAT_B5G5R5X1_UNORM_SRGB:
2550    case ISL_FORMAT_B5G5R5A1_UNORM:
2551    case ISL_FORMAT_B5G5R5A1_UNORM_SRGB:
2552       return ISL_FORMAT_B5G5R5A1_UNORM;
2553 
2554    case ISL_FORMAT_A4B4G4R4_UNORM:
2555    case ISL_FORMAT_B4G4R4A4_UNORM:
2556    case ISL_FORMAT_B4G4R4A4_UNORM_SRGB:
2557       return ISL_FORMAT_B4G4R4A4_UNORM;
2558 
2559    case ISL_FORMAT_B5G6R5_UNORM:
2560    case ISL_FORMAT_B5G6R5_UNORM_SRGB:
2561       return ISL_FORMAT_B5G6R5_UNORM;
2562 
2563    case ISL_FORMAT_A1B5G5R5_UNORM:
2564       return ISL_FORMAT_A1B5G5R5_UNORM;
2565 
2566    case ISL_FORMAT_A8_UNORM:
2567    case ISL_FORMAT_R8_UNORM:
2568    case ISL_FORMAT_R8_SNORM:
2569    case ISL_FORMAT_R8_SINT:
2570    case ISL_FORMAT_R8_UINT:
2571       return ISL_FORMAT_R8_UINT;
2572 
2573    default:
2574       unreachable("Not a compressible format");
2575    }
2576 }
2577 
2578 void
blorp_surf_convert_to_uncompressed(const struct isl_device * isl_dev,struct brw_blorp_surface_info * info,uint32_t * x,uint32_t * y,uint32_t * width,uint32_t * height)2579 blorp_surf_convert_to_uncompressed(const struct isl_device *isl_dev,
2580                                    struct brw_blorp_surface_info *info,
2581                                    uint32_t *x, uint32_t *y,
2582                                    uint32_t *width, uint32_t *height)
2583 {
2584    const struct isl_format_layout *fmtl =
2585       isl_format_get_layout(info->surf.format);
2586 
2587    assert(fmtl->bw > 1 || fmtl->bh > 1);
2588 
2589    /* This is a compressed surface.  We need to convert it to a single
2590     * slice (because compressed layouts don't perfectly match uncompressed
2591     * ones with the same bpb) and divide x, y, width, and height by the
2592     * block size.
2593     */
2594    blorp_surf_convert_to_single_slice(isl_dev, info);
2595 
2596    if (width && height) {
2597 #ifndef NDEBUG
2598       uint32_t right_edge_px = info->tile_x_sa + *x + *width;
2599       uint32_t bottom_edge_px = info->tile_y_sa + *y + *height;
2600       assert(*width % fmtl->bw == 0 ||
2601              right_edge_px == info->surf.logical_level0_px.width);
2602       assert(*height % fmtl->bh == 0 ||
2603              bottom_edge_px == info->surf.logical_level0_px.height);
2604 #endif
2605       *width = DIV_ROUND_UP(*width, fmtl->bw);
2606       *height = DIV_ROUND_UP(*height, fmtl->bh);
2607    }
2608 
2609    if (x && y) {
2610       assert(*x % fmtl->bw == 0);
2611       assert(*y % fmtl->bh == 0);
2612       *x /= fmtl->bw;
2613       *y /= fmtl->bh;
2614    }
2615 
2616    info->surf.logical_level0_px = isl_surf_get_logical_level0_el(&info->surf);
2617    info->surf.phys_level0_sa = isl_surf_get_phys_level0_el(&info->surf);
2618 
2619    assert(info->tile_x_sa % fmtl->bw == 0);
2620    assert(info->tile_y_sa % fmtl->bh == 0);
2621    info->tile_x_sa /= fmtl->bw;
2622    info->tile_y_sa /= fmtl->bh;
2623 
2624    /* It's now an uncompressed surface so we need an uncompressed format */
2625    info->surf.format = get_copy_format_for_bpb(isl_dev, fmtl->bpb);
2626 }
2627 
2628 void
blorp_copy(struct blorp_batch * batch,const struct blorp_surf * src_surf,unsigned src_level,unsigned src_layer,const struct blorp_surf * dst_surf,unsigned dst_level,unsigned dst_layer,uint32_t src_x,uint32_t src_y,uint32_t dst_x,uint32_t dst_y,uint32_t src_width,uint32_t src_height)2629 blorp_copy(struct blorp_batch *batch,
2630            const struct blorp_surf *src_surf,
2631            unsigned src_level, unsigned src_layer,
2632            const struct blorp_surf *dst_surf,
2633            unsigned dst_level, unsigned dst_layer,
2634            uint32_t src_x, uint32_t src_y,
2635            uint32_t dst_x, uint32_t dst_y,
2636            uint32_t src_width, uint32_t src_height)
2637 {
2638    const struct isl_device *isl_dev = batch->blorp->isl_dev;
2639    struct blorp_params params;
2640 
2641    if (src_width == 0 || src_height == 0)
2642       return;
2643 
2644    blorp_params_init(&params);
2645    brw_blorp_surface_info_init(batch->blorp, &params.src, src_surf, src_level,
2646                                src_layer, ISL_FORMAT_UNSUPPORTED, false);
2647    brw_blorp_surface_info_init(batch->blorp, &params.dst, dst_surf, dst_level,
2648                                dst_layer, ISL_FORMAT_UNSUPPORTED, true);
2649 
2650    struct brw_blorp_blit_prog_key wm_prog_key = {
2651       .shader_type = BLORP_SHADER_TYPE_COPY,
2652       .filter = BLORP_FILTER_NONE,
2653       .need_src_offset = src_surf->tile_x_sa || src_surf->tile_y_sa,
2654       .need_dst_offset = dst_surf->tile_x_sa || dst_surf->tile_y_sa,
2655    };
2656 
2657    const struct isl_format_layout *src_fmtl =
2658       isl_format_get_layout(params.src.surf.format);
2659    const struct isl_format_layout *dst_fmtl =
2660       isl_format_get_layout(params.dst.surf.format);
2661 
2662    assert(params.src.aux_usage == ISL_AUX_USAGE_NONE ||
2663           params.src.aux_usage == ISL_AUX_USAGE_HIZ ||
2664           params.src.aux_usage == ISL_AUX_USAGE_HIZ_CCS_WT ||
2665           params.src.aux_usage == ISL_AUX_USAGE_MCS ||
2666           params.src.aux_usage == ISL_AUX_USAGE_MCS_CCS ||
2667           params.src.aux_usage == ISL_AUX_USAGE_CCS_E ||
2668           params.src.aux_usage == ISL_AUX_USAGE_GEN12_CCS_E ||
2669           params.src.aux_usage == ISL_AUX_USAGE_STC_CCS);
2670 
2671    if (isl_aux_usage_has_hiz(params.src.aux_usage)) {
2672       /* In order to use HiZ, we have to use the real format for the source.
2673        * Depth <-> Color copies are not allowed.
2674        */
2675       params.src.view.format = params.src.surf.format;
2676       params.dst.view.format = params.src.surf.format;
2677    } else if ((params.dst.surf.usage & ISL_SURF_USAGE_DEPTH_BIT) &&
2678               isl_dev->info->gen >= 7) {
2679       /* On Gen7 and higher, we use actual depth writes for blits into depth
2680        * buffers so we need the real format.
2681        */
2682       params.src.view.format = params.dst.surf.format;
2683       params.dst.view.format = params.dst.surf.format;
2684    } else if (params.dst.aux_usage == ISL_AUX_USAGE_CCS_E ||
2685               params.dst.aux_usage == ISL_AUX_USAGE_GEN12_CCS_E) {
2686       params.dst.view.format = get_ccs_compatible_copy_format(dst_fmtl);
2687       if (params.src.aux_usage == ISL_AUX_USAGE_CCS_E ||
2688           params.src.aux_usage == ISL_AUX_USAGE_GEN12_CCS_E) {
2689          params.src.view.format = get_ccs_compatible_copy_format(src_fmtl);
2690       } else if (src_fmtl->bpb == dst_fmtl->bpb) {
2691          params.src.view.format = params.dst.view.format;
2692       } else {
2693          params.src.view.format =
2694             get_copy_format_for_bpb(isl_dev, src_fmtl->bpb);
2695       }
2696    } else if (params.src.aux_usage == ISL_AUX_USAGE_CCS_E ||
2697               params.src.aux_usage == ISL_AUX_USAGE_GEN12_CCS_E) {
2698       params.src.view.format = get_ccs_compatible_copy_format(src_fmtl);
2699       if (src_fmtl->bpb == dst_fmtl->bpb) {
2700          params.dst.view.format = params.src.view.format;
2701       } else {
2702          params.dst.view.format =
2703             get_copy_format_for_bpb(isl_dev, dst_fmtl->bpb);
2704       }
2705    } else {
2706       params.dst.view.format = get_copy_format_for_bpb(isl_dev, dst_fmtl->bpb);
2707       params.src.view.format = get_copy_format_for_bpb(isl_dev, src_fmtl->bpb);
2708    }
2709 
2710    if (params.src.aux_usage == ISL_AUX_USAGE_CCS_E) {
2711       /* It's safe to do a blorp_copy between things which are sRGB with CCS_E
2712        * enabled even though CCS_E doesn't technically do sRGB on SKL because
2713        * we stomp everything to UINT anyway.  The one thing we have to be
2714        * careful of is clear colors.  Because fast clear colors for sRGB on
2715        * gen9 are encoded as the float values between format conversion and
2716        * sRGB curve application, a given clear color float will convert to the
2717        * same bits regardless of whether the format is UNORM or sRGB.
2718        * Therefore, we can handle sRGB without any special cases.
2719        */
2720       UNUSED enum isl_format linear_src_format =
2721          isl_format_srgb_to_linear(src_surf->surf->format);
2722       assert(isl_formats_are_ccs_e_compatible(batch->blorp->isl_dev->info,
2723                                               linear_src_format,
2724                                               params.src.view.format));
2725       uint32_t packed[4];
2726       isl_color_value_pack(&params.src.clear_color,
2727                            linear_src_format, packed);
2728       isl_color_value_unpack(&params.src.clear_color,
2729                              params.src.view.format, packed);
2730    }
2731 
2732    if (params.dst.aux_usage == ISL_AUX_USAGE_CCS_E) {
2733       /* See above where we handle linear_src_format */
2734       UNUSED enum isl_format linear_dst_format =
2735          isl_format_srgb_to_linear(dst_surf->surf->format);
2736       assert(isl_formats_are_ccs_e_compatible(batch->blorp->isl_dev->info,
2737                                               linear_dst_format,
2738                                               params.dst.view.format));
2739       uint32_t packed[4];
2740       isl_color_value_pack(&params.dst.clear_color,
2741                            linear_dst_format, packed);
2742       isl_color_value_unpack(&params.dst.clear_color,
2743                              params.dst.view.format, packed);
2744    }
2745 
2746    if (params.src.view.format != params.dst.view.format) {
2747       enum isl_format src_cast_format = params.src.view.format;
2748       enum isl_format dst_cast_format = params.dst.view.format;
2749 
2750       /* The BLORP bitcast code gets confused by RGB formats.  Just treat them
2751        * as RGBA and then everything will be happy.  This is perfectly safe
2752        * because BLORP likes to treat things as if they have vec4 colors all
2753        * the time anyway.
2754        */
2755       if (isl_format_get_layout(src_cast_format)->bpb % 3 == 0)
2756          src_cast_format = isl_format_rgb_to_rgba(src_cast_format);
2757       if (isl_format_get_layout(dst_cast_format)->bpb % 3 == 0)
2758          dst_cast_format = isl_format_rgb_to_rgba(dst_cast_format);
2759 
2760       if (src_cast_format != dst_cast_format) {
2761          wm_prog_key.format_bit_cast = true;
2762          wm_prog_key.src_format = src_cast_format;
2763          wm_prog_key.dst_format = dst_cast_format;
2764       }
2765    }
2766 
2767    if (src_fmtl->bw > 1 || src_fmtl->bh > 1) {
2768       blorp_surf_convert_to_uncompressed(batch->blorp->isl_dev, &params.src,
2769                                          &src_x, &src_y,
2770                                          &src_width, &src_height);
2771       wm_prog_key.need_src_offset = true;
2772    }
2773 
2774    if (dst_fmtl->bw > 1 || dst_fmtl->bh > 1) {
2775       blorp_surf_convert_to_uncompressed(batch->blorp->isl_dev, &params.dst,
2776                                          &dst_x, &dst_y, NULL, NULL);
2777       wm_prog_key.need_dst_offset = true;
2778    }
2779 
2780    /* Once both surfaces are stompped to uncompressed as needed, the
2781     * destination size is the same as the source size.
2782     */
2783    uint32_t dst_width = src_width;
2784    uint32_t dst_height = src_height;
2785 
2786    struct blt_coords coords = {
2787       .x = {
2788          .src0 = src_x,
2789          .src1 = src_x + src_width,
2790          .dst0 = dst_x,
2791          .dst1 = dst_x + dst_width,
2792          .mirror = false
2793       },
2794       .y = {
2795          .src0 = src_y,
2796          .src1 = src_y + src_height,
2797          .dst0 = dst_y,
2798          .dst1 = dst_y + dst_height,
2799          .mirror = false
2800       }
2801    };
2802 
2803    do_blorp_blit(batch, &params, &wm_prog_key, &coords);
2804 }
2805 
2806 static enum isl_format
isl_format_for_size(unsigned size_B)2807 isl_format_for_size(unsigned size_B)
2808 {
2809    switch (size_B) {
2810    case 1:  return ISL_FORMAT_R8_UINT;
2811    case 2:  return ISL_FORMAT_R8G8_UINT;
2812    case 4:  return ISL_FORMAT_R8G8B8A8_UINT;
2813    case 8:  return ISL_FORMAT_R16G16B16A16_UINT;
2814    case 16: return ISL_FORMAT_R32G32B32A32_UINT;
2815    default:
2816       unreachable("Not a power-of-two format size");
2817    }
2818 }
2819 
2820 /**
2821  * Returns the greatest common divisor of a and b that is a power of two.
2822  */
2823 static uint64_t
gcd_pow2_u64(uint64_t a,uint64_t b)2824 gcd_pow2_u64(uint64_t a, uint64_t b)
2825 {
2826    assert(a > 0 || b > 0);
2827 
2828    unsigned a_log2 = ffsll(a) - 1;
2829    unsigned b_log2 = ffsll(b) - 1;
2830 
2831    /* If either a or b is 0, then a_log2 or b_log2 till be UINT_MAX in which
2832     * case, the MIN2() will take the other one.  If both are 0 then we will
2833     * hit the assert above.
2834     */
2835    return 1 << MIN2(a_log2, b_log2);
2836 }
2837 
2838 static void
do_buffer_copy(struct blorp_batch * batch,struct blorp_address * src,struct blorp_address * dst,int width,int height,int block_size)2839 do_buffer_copy(struct blorp_batch *batch,
2840                struct blorp_address *src,
2841                struct blorp_address *dst,
2842                int width, int height, int block_size)
2843 {
2844    /* The actual format we pick doesn't matter as blorp will throw it away.
2845     * The only thing that actually matters is the size.
2846     */
2847    enum isl_format format = isl_format_for_size(block_size);
2848 
2849    UNUSED bool ok;
2850    struct isl_surf surf;
2851    ok = isl_surf_init(batch->blorp->isl_dev, &surf,
2852                       .dim = ISL_SURF_DIM_2D,
2853                       .format = format,
2854                       .width = width,
2855                       .height = height,
2856                       .depth = 1,
2857                       .levels = 1,
2858                       .array_len = 1,
2859                       .samples = 1,
2860                       .row_pitch_B = width * block_size,
2861                       .usage = ISL_SURF_USAGE_TEXTURE_BIT |
2862                                ISL_SURF_USAGE_RENDER_TARGET_BIT,
2863                       .tiling_flags = ISL_TILING_LINEAR_BIT);
2864    assert(ok);
2865 
2866    struct blorp_surf src_blorp_surf = {
2867       .surf = &surf,
2868       .addr = *src,
2869    };
2870 
2871    struct blorp_surf dst_blorp_surf = {
2872       .surf = &surf,
2873       .addr = *dst,
2874    };
2875 
2876    blorp_copy(batch, &src_blorp_surf, 0, 0, &dst_blorp_surf, 0, 0,
2877               0, 0, 0, 0, width, height);
2878 }
2879 
2880 void
blorp_buffer_copy(struct blorp_batch * batch,struct blorp_address src,struct blorp_address dst,uint64_t size)2881 blorp_buffer_copy(struct blorp_batch *batch,
2882                   struct blorp_address src,
2883                   struct blorp_address dst,
2884                   uint64_t size)
2885 {
2886    const struct gen_device_info *devinfo = batch->blorp->isl_dev->info;
2887    uint64_t copy_size = size;
2888 
2889    /* This is maximum possible width/height our HW can handle */
2890    uint64_t max_surface_dim = 1 << (devinfo->gen >= 7 ? 14 : 13);
2891 
2892    /* First, we compute the biggest format that can be used with the
2893     * given offsets and size.
2894     */
2895    int bs = 16;
2896    bs = gcd_pow2_u64(bs, src.offset);
2897    bs = gcd_pow2_u64(bs, dst.offset);
2898    bs = gcd_pow2_u64(bs, size);
2899 
2900    /* First, we make a bunch of max-sized copies */
2901    uint64_t max_copy_size = max_surface_dim * max_surface_dim * bs;
2902    while (copy_size >= max_copy_size) {
2903       do_buffer_copy(batch, &src, &dst, max_surface_dim, max_surface_dim, bs);
2904       copy_size -= max_copy_size;
2905       src.offset += max_copy_size;
2906       dst.offset += max_copy_size;
2907    }
2908 
2909    /* Now make a max-width copy */
2910    uint64_t height = copy_size / (max_surface_dim * bs);
2911    assert(height < max_surface_dim);
2912    if (height != 0) {
2913       uint64_t rect_copy_size = height * max_surface_dim * bs;
2914       do_buffer_copy(batch, &src, &dst, max_surface_dim, height, bs);
2915       copy_size -= rect_copy_size;
2916       src.offset += rect_copy_size;
2917       dst.offset += rect_copy_size;
2918    }
2919 
2920    /* Finally, make a small copy to finish it off */
2921    if (copy_size != 0) {
2922       do_buffer_copy(batch, &src, &dst, copy_size / bs, 1, bs);
2923    }
2924 }
2925