1 /*
2  * Copyright 2006 VMware, Inc.
3  * All Rights Reserved.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the
7  * "Software"), to deal in the Software without restriction, including
8  * without limitation the rights to use, copy, modify, merge, publish,
9  * distribute, sublicense, and/or sell copies of the Software, and to
10  * permit persons to whom the Software is furnished to do so, subject to
11  * the following conditions:
12  *
13  * The above copyright notice and this permission notice (including the
14  * next paragraph) shall be included in all copies or substantial portions
15  * of the Software.
16  *
17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
18  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
19  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
20  * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
21  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
22  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
23  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
24  */
25 
26 #include <GL/gl.h>
27 #include <GL/internal/dri_interface.h>
28 #include "drm-uapi/drm_fourcc.h"
29 
30 #include "intel_batchbuffer.h"
31 #include "intel_image.h"
32 #include "intel_mipmap_tree.h"
33 #include "intel_tex.h"
34 #include "intel_blit.h"
35 #include "intel_fbo.h"
36 
37 #include "brw_blorp.h"
38 #include "brw_context.h"
39 #include "brw_state.h"
40 
41 #include "main/enums.h"
42 #include "main/fbobject.h"
43 #include "main/formats.h"
44 #include "main/glformats.h"
45 #include "main/texcompress_etc.h"
46 #include "main/teximage.h"
47 #include "main/streaming-load-memcpy.h"
48 
49 #include "util/format_srgb.h"
50 #include "util/u_memory.h"
51 
52 #include "x86/common_x86_asm.h"
53 
54 #define FILE_DEBUG_FLAG DEBUG_MIPTREE
55 
56 static void *intel_miptree_map_raw(struct brw_context *brw,
57                                    struct intel_mipmap_tree *mt,
58                                    GLbitfield mode);
59 
60 static void intel_miptree_unmap_raw(struct intel_mipmap_tree *mt);
61 
62 /**
63  * Return true if the format that will be used to access the miptree is
64  * CCS_E-compatible with the miptree's linear/non-sRGB format.
65  *
66  * Why use the linear format? Well, although the miptree may be specified with
67  * an sRGB format, the usage of that color space/format can be toggled. Since
68  * our HW tends to support more linear formats than sRGB ones, we use this
69  * format variant for check for CCS_E compatibility.
70  */
71 static bool
format_ccs_e_compat_with_miptree(const struct gen_device_info * devinfo,const struct intel_mipmap_tree * mt,enum isl_format access_format)72 format_ccs_e_compat_with_miptree(const struct gen_device_info *devinfo,
73                                  const struct intel_mipmap_tree *mt,
74                                  enum isl_format access_format)
75 {
76    assert(mt->aux_usage == ISL_AUX_USAGE_CCS_E);
77 
78    mesa_format linear_format = _mesa_get_srgb_format_linear(mt->format);
79    enum isl_format isl_format = brw_isl_format_for_mesa_format(linear_format);
80    return isl_formats_are_ccs_e_compatible(devinfo, isl_format, access_format);
81 }
82 
83 /* Determine if CCS_E is supported for a given platform and mesa format. */
84 static bool
format_supports_ccs_e(const struct brw_context * brw,mesa_format format)85 format_supports_ccs_e(const struct brw_context *brw, mesa_format format)
86 {
87    /* For now compression is only enabled for integer formats even though
88     * there exist supported floating point formats also. This is a heuristic
89     * decision based on current public benchmarks. In none of the cases these
90     * formats provided any improvement but a few cases were seen to regress.
91     * Hence these are left to to be enabled in the future when they are known
92     * to improve things.
93     */
94    if (_mesa_get_format_datatype(format) == GL_FLOAT)
95       return false;
96 
97    /* Many window system buffers are sRGB even if they are never rendered as
98     * sRGB.  For those, we want CCS_E for when sRGBEncode is false.  When the
99     * surface is used as sRGB, we fall back to CCS_D.
100     */
101    mesa_format linear_format = _mesa_get_srgb_format_linear(format);
102    enum isl_format isl_format = brw_isl_format_for_mesa_format(linear_format);
103    return isl_format_supports_ccs_e(&brw->screen->devinfo, isl_format);
104 }
105 
106 /**
107  * Determine depth format corresponding to a depth+stencil format,
108  * for separate stencil.
109  */
110 mesa_format
intel_depth_format_for_depthstencil_format(mesa_format format)111 intel_depth_format_for_depthstencil_format(mesa_format format) {
112    switch (format) {
113    case MESA_FORMAT_Z24_UNORM_S8_UINT:
114       return MESA_FORMAT_Z24_UNORM_X8_UINT;
115    case MESA_FORMAT_Z32_FLOAT_S8X24_UINT:
116       return MESA_FORMAT_Z_FLOAT32;
117    default:
118       return format;
119    }
120 }
121 
122 static bool
create_mapping_table(GLenum target,unsigned first_level,unsigned last_level,unsigned depth0,struct intel_mipmap_level * table)123 create_mapping_table(GLenum target, unsigned first_level, unsigned last_level,
124                      unsigned depth0, struct intel_mipmap_level *table)
125 {
126    for (unsigned level = first_level; level <= last_level; level++) {
127       const unsigned d =
128          target == GL_TEXTURE_3D ? minify(depth0, level) : depth0;
129 
130       table[level].slice = calloc(d, sizeof(*table[0].slice));
131       if (!table[level].slice)
132          goto unwind;
133    }
134 
135    return true;
136 
137 unwind:
138    for (unsigned level = first_level; level <= last_level; level++)
139       free(table[level].slice);
140 
141    return false;
142 }
143 
144 static bool
needs_separate_stencil(const struct brw_context * brw,struct intel_mipmap_tree * mt,mesa_format format)145 needs_separate_stencil(const struct brw_context *brw,
146                        struct intel_mipmap_tree *mt,
147                        mesa_format format)
148 {
149    const struct gen_device_info *devinfo = &brw->screen->devinfo;
150 
151    if (_mesa_get_format_base_format(format) != GL_DEPTH_STENCIL)
152       return false;
153 
154    if (devinfo->must_use_separate_stencil)
155       return true;
156 
157    return brw->has_separate_stencil && brw->has_hiz;
158 }
159 
160 /**
161  * Choose the aux usage for this miptree.  This function must be called fairly
162  * late in the miptree create process after we have a tiling.
163  */
164 static void
intel_miptree_choose_aux_usage(struct brw_context * brw,struct intel_mipmap_tree * mt)165 intel_miptree_choose_aux_usage(struct brw_context *brw,
166                                struct intel_mipmap_tree *mt)
167 {
168    assert(mt->aux_usage == ISL_AUX_USAGE_NONE);
169 
170    if (_mesa_is_format_color_format(mt->format)) {
171       if (mt->surf.samples > 1) {
172          mt->aux_usage = ISL_AUX_USAGE_MCS;
173       } else if (!(INTEL_DEBUG & DEBUG_NO_RBC) &&
174                  format_supports_ccs_e(brw, mt->format)) {
175          mt->aux_usage = ISL_AUX_USAGE_CCS_E;
176       } else if (brw->mesa_format_supports_render[mt->format]) {
177          mt->aux_usage = ISL_AUX_USAGE_CCS_D;
178       }
179    } else if (isl_surf_usage_is_depth(mt->surf.usage) && brw->has_hiz) {
180       mt->aux_usage = ISL_AUX_USAGE_HIZ;
181    }
182 
183    /* We can do fast-clear on all auxiliary surface types that are
184     * allocated through the normal texture creation paths.
185     */
186    if (mt->aux_usage != ISL_AUX_USAGE_NONE)
187       mt->supports_fast_clear = true;
188 }
189 
190 
191 /**
192  * Choose an appropriate uncompressed format for a requested
193  * compressed format, if unsupported.
194  */
195 mesa_format
intel_lower_compressed_format(struct brw_context * brw,mesa_format format)196 intel_lower_compressed_format(struct brw_context *brw, mesa_format format)
197 {
198    const struct gen_device_info *devinfo = &brw->screen->devinfo;
199 
200    /* No need to lower ETC formats on these platforms,
201     * they are supported natively.
202     */
203    if (devinfo->gen >= 8 || devinfo->is_baytrail)
204       return format;
205 
206    switch (format) {
207    case MESA_FORMAT_ETC1_RGB8:
208       return MESA_FORMAT_R8G8B8X8_UNORM;
209    case MESA_FORMAT_ETC2_RGB8:
210       return MESA_FORMAT_R8G8B8X8_UNORM;
211    case MESA_FORMAT_ETC2_SRGB8:
212    case MESA_FORMAT_ETC2_SRGB8_ALPHA8_EAC:
213    case MESA_FORMAT_ETC2_SRGB8_PUNCHTHROUGH_ALPHA1:
214       return MESA_FORMAT_B8G8R8A8_SRGB;
215    case MESA_FORMAT_ETC2_RGBA8_EAC:
216    case MESA_FORMAT_ETC2_RGB8_PUNCHTHROUGH_ALPHA1:
217       return MESA_FORMAT_R8G8B8A8_UNORM;
218    case MESA_FORMAT_ETC2_R11_EAC:
219       return MESA_FORMAT_R_UNORM16;
220    case MESA_FORMAT_ETC2_SIGNED_R11_EAC:
221       return MESA_FORMAT_R_SNORM16;
222    case MESA_FORMAT_ETC2_RG11_EAC:
223       return MESA_FORMAT_RG_UNORM16;
224    case MESA_FORMAT_ETC2_SIGNED_RG11_EAC:
225       return MESA_FORMAT_RG_SNORM16;
226    default:
227       /* Non ETC1 / ETC2 format */
228       return format;
229    }
230 }
231 
232 unsigned
brw_get_num_logical_layers(const struct intel_mipmap_tree * mt,unsigned level)233 brw_get_num_logical_layers(const struct intel_mipmap_tree *mt, unsigned level)
234 {
235    if (mt->surf.dim == ISL_SURF_DIM_3D)
236       return minify(mt->surf.logical_level0_px.depth, level);
237    else
238       return mt->surf.logical_level0_px.array_len;
239 }
240 
241 UNUSED static unsigned
get_num_phys_layers(const struct isl_surf * surf,unsigned level)242 get_num_phys_layers(const struct isl_surf *surf, unsigned level)
243 {
244    /* In case of physical dimensions one needs to consider also the layout.
245     * See isl_calc_phys_level0_extent_sa().
246     */
247    if (surf->dim != ISL_SURF_DIM_3D)
248       return surf->phys_level0_sa.array_len;
249 
250    if (surf->dim_layout == ISL_DIM_LAYOUT_GEN4_2D)
251       return minify(surf->phys_level0_sa.array_len, level);
252 
253    return minify(surf->phys_level0_sa.depth, level);
254 }
255 
256 /** \brief Assert that the level and layer are valid for the miptree. */
257 void
intel_miptree_check_level_layer(const struct intel_mipmap_tree * mt,uint32_t level,uint32_t layer)258 intel_miptree_check_level_layer(const struct intel_mipmap_tree *mt,
259                                 uint32_t level,
260                                 uint32_t layer)
261 {
262    (void) mt;
263    (void) level;
264    (void) layer;
265 
266    assert(level >= mt->first_level);
267    assert(level <= mt->last_level);
268    assert(layer < get_num_phys_layers(&mt->surf, level));
269 }
270 
271 static enum isl_aux_state **
create_aux_state_map(struct intel_mipmap_tree * mt,enum isl_aux_state initial)272 create_aux_state_map(struct intel_mipmap_tree *mt,
273                      enum isl_aux_state initial)
274 {
275    const uint32_t levels = mt->last_level + 1;
276 
277    uint32_t total_slices = 0;
278    for (uint32_t level = 0; level < levels; level++)
279       total_slices += brw_get_num_logical_layers(mt, level);
280 
281    const size_t per_level_array_size = levels * sizeof(enum isl_aux_state *);
282 
283    /* We're going to allocate a single chunk of data for both the per-level
284     * reference array and the arrays of aux_state.  This makes cleanup
285     * significantly easier.
286     */
287    const size_t total_size = per_level_array_size +
288                              total_slices * sizeof(enum isl_aux_state);
289    void *data = malloc(total_size);
290    if (data == NULL)
291       return NULL;
292 
293    enum isl_aux_state **per_level_arr = data;
294    enum isl_aux_state *s = data + per_level_array_size;
295    for (uint32_t level = 0; level < levels; level++) {
296       per_level_arr[level] = s;
297       const unsigned level_layers = brw_get_num_logical_layers(mt, level);
298       for (uint32_t a = 0; a < level_layers; a++)
299          *(s++) = initial;
300    }
301    assert((void *)s == data + total_size);
302 
303    return per_level_arr;
304 }
305 
306 static void
free_aux_state_map(enum isl_aux_state ** state)307 free_aux_state_map(enum isl_aux_state **state)
308 {
309    free(state);
310 }
311 
312 static bool
need_to_retile_as_linear(struct brw_context * brw,unsigned blt_pitch,enum isl_tiling tiling,unsigned samples)313 need_to_retile_as_linear(struct brw_context *brw, unsigned blt_pitch,
314                          enum isl_tiling tiling, unsigned samples)
315 {
316    if (samples > 1)
317       return false;
318 
319    if (tiling == ISL_TILING_LINEAR)
320       return false;
321 
322    if (blt_pitch >= 32768) {
323       perf_debug("blt pitch %u too large to blit, falling back to untiled",
324                  blt_pitch);
325       return true;
326    }
327 
328    return false;
329 }
330 
331 static bool
need_to_retile_as_x(const struct brw_context * brw,uint64_t size,enum isl_tiling tiling)332 need_to_retile_as_x(const struct brw_context *brw, uint64_t size,
333                     enum isl_tiling tiling)
334 {
335    const struct gen_device_info *devinfo = &brw->screen->devinfo;
336 
337    /* If the BO is too large to fit in the aperture, we need to use the
338     * BLT engine to support it.  Prior to Sandybridge, the BLT paths can't
339     * handle Y-tiling, so we need to fall back to X.
340     */
341    if (devinfo->gen < 6 && size >= brw->max_gtt_map_object_size &&
342        tiling == ISL_TILING_Y0)
343       return true;
344 
345    return false;
346 }
347 
348 static struct intel_mipmap_tree *
make_surface(struct brw_context * brw,GLenum target,mesa_format format,unsigned first_level,unsigned last_level,unsigned width0,unsigned height0,unsigned depth0,unsigned num_samples,isl_tiling_flags_t tiling_flags,isl_surf_usage_flags_t isl_usage_flags,uint32_t alloc_flags,unsigned row_pitch_B,struct brw_bo * bo)349 make_surface(struct brw_context *brw, GLenum target, mesa_format format,
350              unsigned first_level, unsigned last_level,
351              unsigned width0, unsigned height0, unsigned depth0,
352              unsigned num_samples, isl_tiling_flags_t tiling_flags,
353              isl_surf_usage_flags_t isl_usage_flags, uint32_t alloc_flags,
354              unsigned row_pitch_B, struct brw_bo *bo)
355 {
356    struct intel_mipmap_tree *mt = calloc(sizeof(*mt), 1);
357    if (!mt)
358       return NULL;
359 
360    if (!create_mapping_table(target, first_level, last_level, depth0,
361                              mt->level)) {
362       free(mt);
363       return NULL;
364    }
365 
366    mt->refcount = 1;
367 
368    if (target == GL_TEXTURE_CUBE_MAP ||
369        target == GL_TEXTURE_CUBE_MAP_ARRAY)
370       isl_usage_flags |= ISL_SURF_USAGE_CUBE_BIT;
371 
372    DBG("%s: %s %s %ux %u:%u:%u %d..%d <-- %p\n",
373         __func__,
374        _mesa_enum_to_string(target),
375        _mesa_get_format_name(format),
376        num_samples, width0, height0, depth0,
377        first_level, last_level, mt);
378 
379    struct isl_surf_init_info init_info = {
380       .dim = get_isl_surf_dim(target),
381       .format = translate_tex_format(brw, format, false),
382       .width = width0,
383       .height = height0,
384       .depth = target == GL_TEXTURE_3D ? depth0 : 1,
385       .levels = last_level - first_level + 1,
386       .array_len = target == GL_TEXTURE_3D ? 1 : depth0,
387       .samples = num_samples,
388       .row_pitch_B = row_pitch_B,
389       .usage = isl_usage_flags,
390       .tiling_flags = tiling_flags,
391    };
392 
393    if (!isl_surf_init_s(&brw->isl_dev, &mt->surf, &init_info))
394       goto fail;
395 
396    /* Depth surfaces are always Y-tiled and stencil is always W-tiled, although
397     * on gen7 platforms we also need to create Y-tiled copies of stencil for
398     * texturing since the hardware can't sample from W-tiled surfaces. For
399     * everything else, check for corner cases needing special treatment.
400     */
401    bool is_depth_stencil =
402       mt->surf.usage & (ISL_SURF_USAGE_STENCIL_BIT | ISL_SURF_USAGE_DEPTH_BIT);
403    if (!is_depth_stencil) {
404       if (need_to_retile_as_linear(brw, intel_miptree_blt_pitch(mt),
405                                    mt->surf.tiling, mt->surf.samples)) {
406          init_info.tiling_flags = 1u << ISL_TILING_LINEAR;
407          if (!isl_surf_init_s(&brw->isl_dev, &mt->surf, &init_info))
408             goto fail;
409       } else if (need_to_retile_as_x(brw, mt->surf.size_B, mt->surf.tiling)) {
410          init_info.tiling_flags = 1u << ISL_TILING_X;
411          if (!isl_surf_init_s(&brw->isl_dev, &mt->surf, &init_info))
412             goto fail;
413       }
414    }
415 
416    /* In case of linear the buffer gets padded by fixed 64 bytes and therefore
417     * the size may not be multiple of row_pitch.
418     * See isl_apply_surface_padding().
419     */
420    if (mt->surf.tiling != ISL_TILING_LINEAR)
421       assert(mt->surf.size_B % mt->surf.row_pitch_B == 0);
422 
423    if (!bo) {
424       mt->bo = brw_bo_alloc_tiled(brw->bufmgr, "isl-miptree",
425                                   mt->surf.size_B,
426                                   BRW_MEMZONE_OTHER,
427                                   isl_tiling_to_i915_tiling(
428                                      mt->surf.tiling),
429                                   mt->surf.row_pitch_B, alloc_flags);
430       if (!mt->bo)
431          goto fail;
432    } else {
433       mt->bo = bo;
434    }
435 
436    mt->first_level = first_level;
437    mt->last_level = last_level;
438    mt->target = target;
439    mt->format = format;
440    mt->aux_state = NULL;
441    mt->cpp = isl_format_get_layout(mt->surf.format)->bpb / 8;
442    mt->compressed = _mesa_is_format_compressed(format);
443    mt->drm_modifier = DRM_FORMAT_MOD_INVALID;
444 
445    return mt;
446 
447 fail:
448    intel_miptree_release(&mt);
449    return NULL;
450 }
451 
452 /* Return the usual surface usage flags for the given format. */
453 static isl_surf_usage_flags_t
mt_surf_usage(mesa_format format)454 mt_surf_usage(mesa_format format)
455 {
456    switch(_mesa_get_format_base_format(format)) {
457    case GL_DEPTH_COMPONENT:
458       return ISL_SURF_USAGE_DEPTH_BIT | ISL_SURF_USAGE_TEXTURE_BIT;
459    case GL_DEPTH_STENCIL:
460       return ISL_SURF_USAGE_DEPTH_BIT | ISL_SURF_USAGE_STENCIL_BIT |
461              ISL_SURF_USAGE_TEXTURE_BIT;
462    case GL_STENCIL_INDEX:
463       return ISL_SURF_USAGE_STENCIL_BIT | ISL_SURF_USAGE_TEXTURE_BIT;
464    default:
465       return ISL_SURF_USAGE_RENDER_TARGET_BIT | ISL_SURF_USAGE_TEXTURE_BIT;
466    }
467 }
468 
469 static struct intel_mipmap_tree *
miptree_create(struct brw_context * brw,GLenum target,mesa_format format,GLuint first_level,GLuint last_level,GLuint width0,GLuint height0,GLuint depth0,GLuint num_samples,enum intel_miptree_create_flags flags)470 miptree_create(struct brw_context *brw,
471                GLenum target,
472                mesa_format format,
473                GLuint first_level,
474                GLuint last_level,
475                GLuint width0,
476                GLuint height0,
477                GLuint depth0,
478                GLuint num_samples,
479                enum intel_miptree_create_flags flags)
480 {
481    const struct gen_device_info *devinfo = &brw->screen->devinfo;
482    const uint32_t alloc_flags =
483       (flags & MIPTREE_CREATE_BUSY || num_samples > 1) ? BO_ALLOC_BUSY : 0;
484    isl_tiling_flags_t tiling_flags = ISL_TILING_ANY_MASK;
485 
486    /* TODO: This used to be because there wasn't BLORP to handle Y-tiling. */
487    if (devinfo->gen < 6 && _mesa_is_format_color_format(format))
488       tiling_flags &= ~ISL_TILING_Y0_BIT;
489 
490    mesa_format mt_fmt = format;
491    if (!_mesa_is_format_color_format(format) && devinfo->gen >= 6) {
492       /* Fix up the Z miptree format for how we're splitting out separate
493        * stencil. Gen7 expects there to be no stencil bits in its depth buffer.
494        */
495       mt_fmt = intel_depth_format_for_depthstencil_format(format);
496    }
497 
498    struct intel_mipmap_tree *mt =
499       make_surface(brw, target, mt_fmt, first_level, last_level,
500                    width0, height0, depth0, num_samples,
501                    tiling_flags, mt_surf_usage(mt_fmt),
502                    alloc_flags, 0, NULL);
503 
504    if (mt == NULL)
505       return NULL;
506 
507    if (intel_miptree_needs_fake_etc(brw, mt)) {
508       mesa_format decomp_format = intel_lower_compressed_format(brw, format);
509       mt->shadow_mt = make_surface(brw, target, decomp_format, first_level,
510                                    last_level, width0, height0, depth0,
511                                    num_samples, tiling_flags,
512                                    mt_surf_usage(decomp_format),
513                                    alloc_flags, 0, NULL);
514 
515       if (mt->shadow_mt == NULL) {
516          intel_miptree_release(&mt);
517          return NULL;
518       }
519    }
520 
521    if (needs_separate_stencil(brw, mt, format)) {
522       mt->stencil_mt =
523          make_surface(brw, target, MESA_FORMAT_S_UINT8, first_level, last_level,
524                       width0, height0, depth0, num_samples,
525                       ISL_TILING_W_BIT, mt_surf_usage(MESA_FORMAT_S_UINT8),
526                       alloc_flags, 0, NULL);
527       if (mt->stencil_mt == NULL) {
528          intel_miptree_release(&mt);
529          return NULL;
530       }
531    }
532 
533    if (!(flags & MIPTREE_CREATE_NO_AUX))
534       intel_miptree_choose_aux_usage(brw, mt);
535 
536    return mt;
537 }
538 
539 struct intel_mipmap_tree *
intel_miptree_create(struct brw_context * brw,GLenum target,mesa_format format,GLuint first_level,GLuint last_level,GLuint width0,GLuint height0,GLuint depth0,GLuint num_samples,enum intel_miptree_create_flags flags)540 intel_miptree_create(struct brw_context *brw,
541                      GLenum target,
542                      mesa_format format,
543                      GLuint first_level,
544                      GLuint last_level,
545                      GLuint width0,
546                      GLuint height0,
547                      GLuint depth0,
548                      GLuint num_samples,
549                      enum intel_miptree_create_flags flags)
550 {
551    assert(num_samples > 0);
552 
553    struct intel_mipmap_tree *mt = miptree_create(
554                                      brw, target, format,
555                                      first_level, last_level,
556                                      width0, height0, depth0, num_samples,
557                                      flags);
558    if (!mt)
559       return NULL;
560 
561    mt->offset = 0;
562 
563    /* Create the auxiliary surface up-front. CCS_D, on the other hand, can only
564     * compress clear color so we wait until an actual fast-clear to allocate
565     * it.
566     */
567    if (mt->aux_usage != ISL_AUX_USAGE_CCS_D &&
568        !intel_miptree_alloc_aux(brw, mt)) {
569       mt->aux_usage = ISL_AUX_USAGE_NONE;
570       mt->supports_fast_clear = false;
571    }
572 
573    return mt;
574 }
575 
576 struct intel_mipmap_tree *
intel_miptree_create_for_bo(struct brw_context * brw,struct brw_bo * bo,mesa_format format,uint32_t offset,uint32_t width,uint32_t height,uint32_t depth,int pitch,enum isl_tiling tiling,enum intel_miptree_create_flags flags)577 intel_miptree_create_for_bo(struct brw_context *brw,
578                             struct brw_bo *bo,
579                             mesa_format format,
580                             uint32_t offset,
581                             uint32_t width,
582                             uint32_t height,
583                             uint32_t depth,
584                             int pitch,
585                             enum isl_tiling tiling,
586                             enum intel_miptree_create_flags flags)
587 {
588    const struct gen_device_info *devinfo = &brw->screen->devinfo;
589    struct intel_mipmap_tree *mt;
590    const GLenum target = depth > 1 ? GL_TEXTURE_2D_ARRAY : GL_TEXTURE_2D;
591    const GLenum base_format = _mesa_get_format_base_format(format);
592 
593    if ((base_format == GL_DEPTH_COMPONENT ||
594         base_format == GL_DEPTH_STENCIL)) {
595       const mesa_format mt_fmt = (devinfo->gen < 6) ? format :
596          intel_depth_format_for_depthstencil_format(format);
597       mt = make_surface(brw, target, mt_fmt,
598                         0, 0, width, height, depth, 1, ISL_TILING_Y0_BIT,
599                         mt_surf_usage(mt_fmt),
600                         0, pitch, bo);
601       if (!mt)
602          return NULL;
603 
604       brw_bo_reference(bo);
605 
606       if (!(flags & MIPTREE_CREATE_NO_AUX))
607          intel_miptree_choose_aux_usage(brw, mt);
608 
609       return mt;
610    } else if (format == MESA_FORMAT_S_UINT8) {
611       mt = make_surface(brw, target, MESA_FORMAT_S_UINT8,
612                         0, 0, width, height, depth, 1,
613                         ISL_TILING_W_BIT,
614                         mt_surf_usage(MESA_FORMAT_S_UINT8),
615                         0, pitch, bo);
616       if (!mt)
617          return NULL;
618 
619       assert(bo->size >= mt->surf.size_B);
620 
621       brw_bo_reference(bo);
622       return mt;
623    }
624 
625    /* Nothing will be able to use this miptree with the BO if the offset isn't
626     * aligned.
627     */
628    if (tiling != ISL_TILING_LINEAR)
629       assert(offset % 4096 == 0);
630 
631    /* miptrees can't handle negative pitch.  If you need flipping of images,
632     * that's outside of the scope of the mt.
633     */
634    assert(pitch >= 0);
635 
636    mt = make_surface(brw, target, format,
637                      0, 0, width, height, depth, 1,
638                      1lu << tiling,
639                      mt_surf_usage(format),
640                      0, pitch, bo);
641    if (!mt)
642       return NULL;
643 
644    brw_bo_reference(bo);
645    mt->bo = bo;
646    mt->offset = offset;
647 
648    if (!(flags & MIPTREE_CREATE_NO_AUX)) {
649       intel_miptree_choose_aux_usage(brw, mt);
650 
651       /* Create the auxiliary surface up-front. CCS_D, on the other hand, can
652        * only compress clear color so we wait until an actual fast-clear to
653        * allocate it.
654        */
655       if (mt->aux_usage != ISL_AUX_USAGE_CCS_D &&
656           !intel_miptree_alloc_aux(brw, mt)) {
657          mt->aux_usage = ISL_AUX_USAGE_NONE;
658          mt->supports_fast_clear = false;
659       }
660    }
661 
662    return mt;
663 }
664 
665 static struct intel_mipmap_tree *
miptree_create_for_planar_image(struct brw_context * brw,__DRIimage * image,GLenum target,enum isl_tiling tiling)666 miptree_create_for_planar_image(struct brw_context *brw,
667                                 __DRIimage *image, GLenum target,
668                                 enum isl_tiling tiling)
669 {
670    const struct intel_image_format *f = image->planar_format;
671    struct intel_mipmap_tree *planar_mt = NULL;
672 
673    for (int i = 0; i < f->nplanes; i++) {
674       const int index = f->planes[i].buffer_index;
675       const uint32_t dri_format = f->planes[i].dri_format;
676       const mesa_format format = driImageFormatToGLFormat(dri_format);
677       const uint32_t width = image->width >> f->planes[i].width_shift;
678       const uint32_t height = image->height >> f->planes[i].height_shift;
679 
680       /* Disable creation of the texture's aux buffers because the driver
681        * exposes no EGL API to manage them. That is, there is no API for
682        * resolving the aux buffer's content to the main buffer nor for
683        * invalidating the aux buffer's content.
684        */
685       struct intel_mipmap_tree *mt =
686          intel_miptree_create_for_bo(brw, image->bo, format,
687                                      image->offsets[index],
688                                      width, height, 1,
689                                      image->strides[index],
690                                      tiling,
691                                      MIPTREE_CREATE_NO_AUX);
692       if (mt == NULL) {
693          intel_miptree_release(&planar_mt);
694          return NULL;
695       }
696 
697       mt->target = target;
698 
699       if (i == 0)
700          planar_mt = mt;
701       else
702          planar_mt->plane[i - 1] = mt;
703    }
704 
705    planar_mt->drm_modifier = image->modifier;
706 
707    return planar_mt;
708 }
709 
710 static bool
create_ccs_buf_for_image(struct brw_context * brw,__DRIimage * image,struct intel_mipmap_tree * mt,enum isl_aux_state initial_state)711 create_ccs_buf_for_image(struct brw_context *brw,
712                          __DRIimage *image,
713                          struct intel_mipmap_tree *mt,
714                          enum isl_aux_state initial_state)
715 {
716    struct isl_surf temp_ccs_surf = {0,};
717 
718    /* CCS is only supported for very simple miptrees */
719    assert(image->aux_offset != 0 && image->aux_pitch != 0);
720    assert(image->tile_x == 0 && image->tile_y == 0);
721    assert(mt->surf.samples == 1);
722    assert(mt->surf.levels == 1);
723    assert(mt->surf.logical_level0_px.depth == 1);
724    assert(mt->surf.logical_level0_px.array_len == 1);
725    assert(mt->first_level == 0);
726    assert(mt->last_level == 0);
727 
728    /* We shouldn't already have a CCS */
729    assert(!mt->aux_buf);
730 
731    if (!isl_surf_get_ccs_surf(&brw->isl_dev, &mt->surf, &temp_ccs_surf, NULL,
732                               image->aux_pitch))
733       return false;
734 
735    assert(image->aux_offset < image->bo->size);
736    assert(temp_ccs_surf.size_B <= image->bo->size - image->aux_offset);
737 
738    mt->aux_buf = calloc(sizeof(*mt->aux_buf), 1);
739    if (mt->aux_buf == NULL)
740       return false;
741 
742    mt->aux_state = create_aux_state_map(mt, initial_state);
743    if (!mt->aux_state) {
744       free(mt->aux_buf);
745       mt->aux_buf = NULL;
746       return false;
747    }
748 
749    /* On gen10+ we start using an extra space in the aux buffer to store the
750     * indirect clear color. However, if we imported an image from the window
751     * system with CCS, we don't have the extra space at the end of the aux
752     * buffer. So create a new bo here that will store that clear color.
753     */
754    if (brw->isl_dev.ss.clear_color_state_size > 0) {
755       mt->aux_buf->clear_color_bo =
756          brw_bo_alloc_tiled(brw->bufmgr, "clear_color_bo",
757                             brw->isl_dev.ss.clear_color_state_size,
758                             BRW_MEMZONE_OTHER, I915_TILING_NONE, 0,
759                             BO_ALLOC_ZEROED);
760       if (!mt->aux_buf->clear_color_bo) {
761          free(mt->aux_buf);
762          mt->aux_buf = NULL;
763          return false;
764       }
765    }
766 
767    mt->aux_buf->bo = image->bo;
768    brw_bo_reference(image->bo);
769 
770    mt->aux_buf->offset = image->aux_offset;
771    mt->aux_buf->surf = temp_ccs_surf;
772 
773    return true;
774 }
775 
776 struct intel_mipmap_tree *
intel_miptree_create_for_dri_image(struct brw_context * brw,__DRIimage * image,GLenum target,mesa_format format,bool allow_internal_aux)777 intel_miptree_create_for_dri_image(struct brw_context *brw,
778                                    __DRIimage *image, GLenum target,
779                                    mesa_format format,
780                                    bool allow_internal_aux)
781 {
782    uint32_t bo_tiling, bo_swizzle;
783    brw_bo_get_tiling(image->bo, &bo_tiling, &bo_swizzle);
784 
785    const struct isl_drm_modifier_info *mod_info =
786       isl_drm_modifier_get_info(image->modifier);
787 
788    const enum isl_tiling tiling =
789       mod_info ? mod_info->tiling : isl_tiling_from_i915_tiling(bo_tiling);
790 
791    if (image->planar_format && image->planar_format->nplanes > 1)
792       return miptree_create_for_planar_image(brw, image, target, tiling);
793 
794    if (image->planar_format)
795       assert(image->planar_format->planes[0].dri_format == image->dri_format);
796 
797    if (!brw->ctx.TextureFormatSupported[format]) {
798       /* The texture storage paths in core Mesa detect if the driver does not
799        * support the user-requested format, and then searches for a
800        * fallback format. The DRIimage code bypasses core Mesa, though. So we
801        * do the fallbacks here for important formats.
802        *
803        * We must support DRM_FOURCC_XBGR8888 textures because the Android
804        * framework produces HAL_PIXEL_FORMAT_RGBX8888 winsys surfaces, which
805        * the Chrome OS compositor consumes as dma_buf EGLImages.
806        */
807       format = _mesa_format_fallback_rgbx_to_rgba(format);
808    }
809 
810    if (!brw->ctx.TextureFormatSupported[format])
811       return NULL;
812 
813    enum intel_miptree_create_flags mt_create_flags = 0;
814 
815    /* If this image comes in from a window system, we have different
816     * requirements than if it comes in via an EGL import operation.  Window
817     * system images can use any form of auxiliary compression we wish because
818     * they get "flushed" before being handed off to the window system and we
819     * have the opportunity to do resolves.  Non window-system images, on the
820     * other hand, have no resolve point so we can't have aux without a
821     * modifier.
822     */
823    if (!allow_internal_aux)
824       mt_create_flags |= MIPTREE_CREATE_NO_AUX;
825 
826    /* If we have a modifier which specifies aux, don't create one yet */
827    if (mod_info && mod_info->aux_usage != ISL_AUX_USAGE_NONE)
828       mt_create_flags |= MIPTREE_CREATE_NO_AUX;
829 
830    /* Disable creation of the texture's aux buffers because the driver exposes
831     * no EGL API to manage them. That is, there is no API for resolving the aux
832     * buffer's content to the main buffer nor for invalidating the aux buffer's
833     * content.
834     */
835    struct intel_mipmap_tree *mt =
836       intel_miptree_create_for_bo(brw, image->bo, format,
837                                   image->offset, image->width, image->height, 1,
838                                   image->pitch, tiling, mt_create_flags);
839    if (mt == NULL)
840       return NULL;
841 
842    mt->target = target;
843    mt->level[0].level_x = image->tile_x;
844    mt->level[0].level_y = image->tile_y;
845    mt->drm_modifier = image->modifier;
846 
847    /* From "OES_EGL_image" error reporting. We report GL_INVALID_OPERATION
848     * for EGL images from non-tile aligned sufaces in gen4 hw and earlier which has
849     * trouble resolving back to destination image due to alignment issues.
850     */
851    const struct gen_device_info *devinfo = &brw->screen->devinfo;
852    if (!devinfo->has_surface_tile_offset) {
853       uint32_t draw_x, draw_y;
854       intel_miptree_get_tile_offsets(mt, 0, 0, &draw_x, &draw_y);
855 
856       if (draw_x != 0 || draw_y != 0) {
857          _mesa_error(&brw->ctx, GL_INVALID_OPERATION, __func__);
858          intel_miptree_release(&mt);
859          return NULL;
860       }
861    }
862 
863    if (mod_info && mod_info->aux_usage != ISL_AUX_USAGE_NONE) {
864       assert(mod_info->aux_usage == ISL_AUX_USAGE_CCS_E);
865 
866       mt->aux_usage = mod_info->aux_usage;
867       /* If we are a window system buffer, then we can support fast-clears
868        * even if the modifier doesn't support them by doing a partial resolve
869        * as part of the flush operation.
870        */
871       mt->supports_fast_clear =
872          allow_internal_aux || mod_info->supports_clear_color;
873 
874       /* We don't know the actual state of the surface when we get it but we
875        * can make a pretty good guess based on the modifier.  What we do know
876        * for sure is that it isn't in the AUX_INVALID state, so we just assume
877        * a worst case of compression.
878        */
879       enum isl_aux_state initial_state =
880          isl_drm_modifier_get_default_aux_state(image->modifier);
881 
882       if (!create_ccs_buf_for_image(brw, image, mt, initial_state)) {
883          intel_miptree_release(&mt);
884          return NULL;
885       }
886    }
887 
888    /* Don't assume coherency for imported EGLimages.  We don't know what
889     * external clients are going to do with it.  They may scan it out.
890     */
891    image->bo->cache_coherent = false;
892 
893    return mt;
894 }
895 
896 /**
897  * For a singlesample renderbuffer, this simply wraps the given BO with a
898  * miptree.
899  *
900  * For a multisample renderbuffer, this wraps the window system's
901  * (singlesample) BO with a singlesample miptree attached to the
902  * intel_renderbuffer, then creates a multisample miptree attached to irb->mt
903  * that will contain the actual rendering (which is lazily resolved to
904  * irb->singlesample_mt).
905  */
906 bool
intel_update_winsys_renderbuffer_miptree(struct brw_context * intel,struct intel_renderbuffer * irb,struct intel_mipmap_tree * singlesample_mt,uint32_t width,uint32_t height,uint32_t pitch)907 intel_update_winsys_renderbuffer_miptree(struct brw_context *intel,
908                                          struct intel_renderbuffer *irb,
909                                          struct intel_mipmap_tree *singlesample_mt,
910                                          uint32_t width, uint32_t height,
911                                          uint32_t pitch)
912 {
913    struct intel_mipmap_tree *multisample_mt = NULL;
914    struct gl_renderbuffer *rb = &irb->Base.Base;
915    mesa_format format = rb->Format;
916    const unsigned num_samples = MAX2(rb->NumSamples, 1);
917 
918    /* Only the front and back buffers, which are color buffers, are allocated
919     * through the image loader.
920     */
921    assert(_mesa_get_format_base_format(format) == GL_RGB ||
922           _mesa_get_format_base_format(format) == GL_RGBA);
923 
924    assert(singlesample_mt);
925 
926    if (num_samples == 1) {
927       intel_miptree_release(&irb->mt);
928       irb->mt = singlesample_mt;
929 
930       assert(!irb->singlesample_mt);
931    } else {
932       intel_miptree_release(&irb->singlesample_mt);
933       irb->singlesample_mt = singlesample_mt;
934 
935       if (!irb->mt ||
936           irb->mt->surf.logical_level0_px.width != width ||
937           irb->mt->surf.logical_level0_px.height != height) {
938          multisample_mt = intel_miptree_create_for_renderbuffer(intel,
939                                                                 format,
940                                                                 width,
941                                                                 height,
942                                                                 num_samples);
943          if (!multisample_mt)
944             goto fail;
945 
946          irb->need_downsample = false;
947          intel_miptree_release(&irb->mt);
948          irb->mt = multisample_mt;
949       }
950    }
951    return true;
952 
953 fail:
954    intel_miptree_release(&irb->mt);
955    return false;
956 }
957 
958 struct intel_mipmap_tree*
intel_miptree_create_for_renderbuffer(struct brw_context * brw,mesa_format format,uint32_t width,uint32_t height,uint32_t num_samples)959 intel_miptree_create_for_renderbuffer(struct brw_context *brw,
960                                       mesa_format format,
961                                       uint32_t width,
962                                       uint32_t height,
963                                       uint32_t num_samples)
964 {
965    struct intel_mipmap_tree *mt;
966    uint32_t depth = 1;
967    GLenum target = num_samples > 1 ? GL_TEXTURE_2D_MULTISAMPLE : GL_TEXTURE_2D;
968 
969    mt = intel_miptree_create(brw, target, format, 0, 0,
970                              width, height, depth, num_samples,
971                              MIPTREE_CREATE_BUSY);
972    if (!mt)
973       goto fail;
974 
975    return mt;
976 
977 fail:
978    intel_miptree_release(&mt);
979    return NULL;
980 }
981 
982 void
intel_miptree_reference(struct intel_mipmap_tree ** dst,struct intel_mipmap_tree * src)983 intel_miptree_reference(struct intel_mipmap_tree **dst,
984                         struct intel_mipmap_tree *src)
985 {
986    if (*dst == src)
987       return;
988 
989    intel_miptree_release(dst);
990 
991    if (src) {
992       src->refcount++;
993       DBG("%s %p refcount now %d\n", __func__, src, src->refcount);
994    }
995 
996    *dst = src;
997 }
998 
999 static void
intel_miptree_aux_buffer_free(struct intel_miptree_aux_buffer * aux_buf)1000 intel_miptree_aux_buffer_free(struct intel_miptree_aux_buffer *aux_buf)
1001 {
1002    if (aux_buf == NULL)
1003       return;
1004 
1005    brw_bo_unreference(aux_buf->bo);
1006    brw_bo_unreference(aux_buf->clear_color_bo);
1007 
1008    free(aux_buf);
1009 }
1010 
1011 void
intel_miptree_release(struct intel_mipmap_tree ** mt)1012 intel_miptree_release(struct intel_mipmap_tree **mt)
1013 {
1014    if (!*mt)
1015       return;
1016 
1017    DBG("%s %p refcount will be %d\n", __func__, *mt, (*mt)->refcount - 1);
1018    if (--(*mt)->refcount <= 0) {
1019       GLuint i;
1020 
1021       DBG("%s deleting %p\n", __func__, *mt);
1022 
1023       brw_bo_unreference((*mt)->bo);
1024       intel_miptree_release(&(*mt)->stencil_mt);
1025       intel_miptree_release(&(*mt)->shadow_mt);
1026       intel_miptree_aux_buffer_free((*mt)->aux_buf);
1027       free_aux_state_map((*mt)->aux_state);
1028 
1029       intel_miptree_release(&(*mt)->plane[0]);
1030       intel_miptree_release(&(*mt)->plane[1]);
1031 
1032       for (i = 0; i < MAX_TEXTURE_LEVELS; i++) {
1033 	 free((*mt)->level[i].slice);
1034       }
1035 
1036       free(*mt);
1037    }
1038    *mt = NULL;
1039 }
1040 
1041 
1042 void
intel_get_image_dims(struct gl_texture_image * image,int * width,int * height,int * depth)1043 intel_get_image_dims(struct gl_texture_image *image,
1044                      int *width, int *height, int *depth)
1045 {
1046    switch (image->TexObject->Target) {
1047    case GL_TEXTURE_1D_ARRAY:
1048       /* For a 1D Array texture the OpenGL API will treat the image height as
1049        * the number of array slices. For Intel hardware, we treat the 1D array
1050        * as a 2D Array with a height of 1. So, here we want to swap image
1051        * height and depth.
1052        */
1053       assert(image->Depth == 1);
1054       *width = image->Width;
1055       *height = 1;
1056       *depth = image->Height;
1057       break;
1058    case GL_TEXTURE_CUBE_MAP:
1059       /* For Cube maps, the mesa/main api layer gives us a depth of 1 even
1060        * though we really have 6 slices.
1061        */
1062       assert(image->Depth == 1);
1063       *width = image->Width;
1064       *height = image->Height;
1065       *depth = 6;
1066       break;
1067    default:
1068       *width = image->Width;
1069       *height = image->Height;
1070       *depth = image->Depth;
1071       break;
1072    }
1073 }
1074 
1075 /**
1076  * Can the image be pulled into a unified mipmap tree?  This mirrors
1077  * the completeness test in a lot of ways.
1078  *
1079  * Not sure whether I want to pass gl_texture_image here.
1080  */
1081 bool
intel_miptree_match_image(struct intel_mipmap_tree * mt,struct gl_texture_image * image)1082 intel_miptree_match_image(struct intel_mipmap_tree *mt,
1083                           struct gl_texture_image *image)
1084 {
1085    struct intel_texture_image *intelImage = intel_texture_image(image);
1086    GLuint level = intelImage->base.Base.Level;
1087    int width, height, depth;
1088 
1089    /* glTexImage* choose the texture object based on the target passed in, and
1090     * objects can't change targets over their lifetimes, so this should be
1091     * true.
1092     */
1093    assert(image->TexObject->Target == mt->target);
1094 
1095    mesa_format mt_format = mt->format;
1096    if (mt->format == MESA_FORMAT_Z24_UNORM_X8_UINT && mt->stencil_mt)
1097       mt_format = MESA_FORMAT_Z24_UNORM_S8_UINT;
1098    if (mt->format == MESA_FORMAT_Z_FLOAT32 && mt->stencil_mt)
1099       mt_format = MESA_FORMAT_Z32_FLOAT_S8X24_UINT;
1100 
1101    if (_mesa_get_srgb_format_linear(image->TexFormat) !=
1102        _mesa_get_srgb_format_linear(mt_format))
1103       return false;
1104 
1105    intel_get_image_dims(image, &width, &height, &depth);
1106 
1107    if (mt->target == GL_TEXTURE_CUBE_MAP)
1108       depth = 6;
1109 
1110    if (level >= mt->surf.levels)
1111       return false;
1112 
1113    const unsigned level_depth =
1114       mt->surf.dim == ISL_SURF_DIM_3D ?
1115          minify(mt->surf.logical_level0_px.depth, level) :
1116          mt->surf.logical_level0_px.array_len;
1117 
1118    return width == minify(mt->surf.logical_level0_px.width, level) &&
1119           height == minify(mt->surf.logical_level0_px.height, level) &&
1120           depth == level_depth &&
1121           MAX2(image->NumSamples, 1) == mt->surf.samples;
1122 }
1123 
1124 void
intel_miptree_get_image_offset(const struct intel_mipmap_tree * mt,GLuint level,GLuint slice,GLuint * x,GLuint * y)1125 intel_miptree_get_image_offset(const struct intel_mipmap_tree *mt,
1126 			       GLuint level, GLuint slice,
1127 			       GLuint *x, GLuint *y)
1128 {
1129    if (level == 0 && slice == 0) {
1130       *x = mt->level[0].level_x;
1131       *y = mt->level[0].level_y;
1132       return;
1133    }
1134 
1135    uint32_t x_offset_sa, y_offset_sa;
1136 
1137    /* Miptree itself can have an offset only if it represents a single
1138     * slice in an imported buffer object.
1139     * See intel_miptree_create_for_dri_image().
1140     */
1141    assert(mt->level[0].level_x == 0);
1142    assert(mt->level[0].level_y == 0);
1143 
1144    /* Given level is relative to level zero while the miptree may be
1145     * represent just a subset of all levels starting from 'first_level'.
1146     */
1147    assert(level >= mt->first_level);
1148    level -= mt->first_level;
1149 
1150    const unsigned z = mt->surf.dim == ISL_SURF_DIM_3D ? slice : 0;
1151    slice = mt->surf.dim == ISL_SURF_DIM_3D ? 0 : slice;
1152    isl_surf_get_image_offset_el(&mt->surf, level, slice, z,
1153                                 &x_offset_sa, &y_offset_sa);
1154 
1155    *x = x_offset_sa;
1156    *y = y_offset_sa;
1157 }
1158 
1159 
1160 /**
1161  * This function computes the tile_w (in bytes) and tile_h (in rows) of
1162  * different tiling patterns. If the BO is untiled, tile_w is set to cpp
1163  * and tile_h is set to 1.
1164  */
1165 void
intel_get_tile_dims(enum isl_tiling tiling,uint32_t cpp,uint32_t * tile_w,uint32_t * tile_h)1166 intel_get_tile_dims(enum isl_tiling tiling, uint32_t cpp,
1167                     uint32_t *tile_w, uint32_t *tile_h)
1168 {
1169    switch (tiling) {
1170    case ISL_TILING_X:
1171       *tile_w = 512;
1172       *tile_h = 8;
1173       break;
1174    case ISL_TILING_Y0:
1175       *tile_w = 128;
1176       *tile_h = 32;
1177       break;
1178    case ISL_TILING_LINEAR:
1179       *tile_w = cpp;
1180       *tile_h = 1;
1181       break;
1182    default:
1183       unreachable("not reached");
1184    }
1185 }
1186 
1187 
1188 /**
1189  * This function computes masks that may be used to select the bits of the X
1190  * and Y coordinates that indicate the offset within a tile.  If the BO is
1191  * untiled, the masks are set to 0.
1192  */
1193 void
intel_get_tile_masks(enum isl_tiling tiling,uint32_t cpp,uint32_t * mask_x,uint32_t * mask_y)1194 intel_get_tile_masks(enum isl_tiling tiling, uint32_t cpp,
1195                      uint32_t *mask_x, uint32_t *mask_y)
1196 {
1197    uint32_t tile_w_bytes, tile_h;
1198 
1199    intel_get_tile_dims(tiling, cpp, &tile_w_bytes, &tile_h);
1200 
1201    *mask_x = tile_w_bytes / cpp - 1;
1202    *mask_y = tile_h - 1;
1203 }
1204 
1205 /**
1206  * Compute the offset (in bytes) from the start of the BO to the given x
1207  * and y coordinate.  For tiled BOs, caller must ensure that x and y are
1208  * multiples of the tile size.
1209  */
1210 uint32_t
intel_miptree_get_aligned_offset(const struct intel_mipmap_tree * mt,uint32_t x,uint32_t y)1211 intel_miptree_get_aligned_offset(const struct intel_mipmap_tree *mt,
1212                                  uint32_t x, uint32_t y)
1213 {
1214    int cpp = mt->cpp;
1215    uint32_t pitch = mt->surf.row_pitch_B;
1216 
1217    switch (mt->surf.tiling) {
1218    default:
1219       unreachable("not reached");
1220    case ISL_TILING_LINEAR:
1221       return y * pitch + x * cpp;
1222    case ISL_TILING_X:
1223       assert((x % (512 / cpp)) == 0);
1224       assert((y % 8) == 0);
1225       return y * pitch + x / (512 / cpp) * 4096;
1226    case ISL_TILING_Y0:
1227       assert((x % (128 / cpp)) == 0);
1228       assert((y % 32) == 0);
1229       return y * pitch + x / (128 / cpp) * 4096;
1230    }
1231 }
1232 
1233 /**
1234  * Rendering with tiled buffers requires that the base address of the buffer
1235  * be aligned to a page boundary.  For renderbuffers, and sometimes with
1236  * textures, we may want the surface to point at a texture image level that
1237  * isn't at a page boundary.
1238  *
1239  * This function returns an appropriately-aligned base offset
1240  * according to the tiling restrictions, plus any required x/y offset
1241  * from there.
1242  */
1243 uint32_t
intel_miptree_get_tile_offsets(const struct intel_mipmap_tree * mt,GLuint level,GLuint slice,uint32_t * tile_x,uint32_t * tile_y)1244 intel_miptree_get_tile_offsets(const struct intel_mipmap_tree *mt,
1245                                GLuint level, GLuint slice,
1246                                uint32_t *tile_x,
1247                                uint32_t *tile_y)
1248 {
1249    uint32_t x, y;
1250    uint32_t mask_x, mask_y;
1251 
1252    intel_get_tile_masks(mt->surf.tiling, mt->cpp, &mask_x, &mask_y);
1253    intel_miptree_get_image_offset(mt, level, slice, &x, &y);
1254 
1255    *tile_x = x & mask_x;
1256    *tile_y = y & mask_y;
1257 
1258    return intel_miptree_get_aligned_offset(mt, x & ~mask_x, y & ~mask_y);
1259 }
1260 
1261 static void
intel_miptree_copy_slice_sw(struct brw_context * brw,struct intel_mipmap_tree * src_mt,unsigned src_level,unsigned src_layer,struct intel_mipmap_tree * dst_mt,unsigned dst_level,unsigned dst_layer,unsigned width,unsigned height)1262 intel_miptree_copy_slice_sw(struct brw_context *brw,
1263                             struct intel_mipmap_tree *src_mt,
1264                             unsigned src_level, unsigned src_layer,
1265                             struct intel_mipmap_tree *dst_mt,
1266                             unsigned dst_level, unsigned dst_layer,
1267                             unsigned width, unsigned height)
1268 {
1269    void *src, *dst;
1270    ptrdiff_t src_stride, dst_stride;
1271    const unsigned cpp = (isl_format_get_layout(dst_mt->surf.format)->bpb / 8);
1272 
1273    intel_miptree_map(brw, src_mt,
1274                      src_level, src_layer,
1275                      0, 0,
1276                      width, height,
1277                      GL_MAP_READ_BIT | BRW_MAP_DIRECT_BIT,
1278                      &src, &src_stride);
1279 
1280    intel_miptree_map(brw, dst_mt,
1281                      dst_level, dst_layer,
1282                      0, 0,
1283                      width, height,
1284                      GL_MAP_WRITE_BIT | GL_MAP_INVALIDATE_RANGE_BIT |
1285                      BRW_MAP_DIRECT_BIT,
1286                      &dst, &dst_stride);
1287 
1288    DBG("sw blit %s mt %p %p/%"PRIdPTR" -> %s mt %p %p/%"PRIdPTR" (%dx%d)\n",
1289        _mesa_get_format_name(src_mt->format),
1290        src_mt, src, src_stride,
1291        _mesa_get_format_name(dst_mt->format),
1292        dst_mt, dst, dst_stride,
1293        width, height);
1294 
1295    int row_size = cpp * width;
1296    if (src_stride == row_size &&
1297        dst_stride == row_size) {
1298       memcpy(dst, src, row_size * height);
1299    } else {
1300       for (int i = 0; i < height; i++) {
1301          memcpy(dst, src, row_size);
1302          dst += dst_stride;
1303          src += src_stride;
1304       }
1305    }
1306 
1307    intel_miptree_unmap(brw, dst_mt, dst_level, dst_layer);
1308    intel_miptree_unmap(brw, src_mt, src_level, src_layer);
1309 
1310    /* Don't forget to copy the stencil data over, too.  We could have skipped
1311     * passing BRW_MAP_DIRECT_BIT, but that would have meant intel_miptree_map
1312     * shuffling the two data sources in/out of temporary storage instead of
1313     * the direct mapping we get this way.
1314     */
1315    if (dst_mt->stencil_mt) {
1316       assert(src_mt->stencil_mt);
1317       intel_miptree_copy_slice_sw(brw,
1318                                   src_mt->stencil_mt, src_level, src_layer,
1319                                   dst_mt->stencil_mt, dst_level, dst_layer,
1320                                   width, height);
1321    }
1322 }
1323 
1324 void
intel_miptree_copy_slice(struct brw_context * brw,struct intel_mipmap_tree * src_mt,unsigned src_level,unsigned src_layer,struct intel_mipmap_tree * dst_mt,unsigned dst_level,unsigned dst_layer)1325 intel_miptree_copy_slice(struct brw_context *brw,
1326                          struct intel_mipmap_tree *src_mt,
1327                          unsigned src_level, unsigned src_layer,
1328                          struct intel_mipmap_tree *dst_mt,
1329                          unsigned dst_level, unsigned dst_layer)
1330 
1331 {
1332    const struct gen_device_info *devinfo = &brw->screen->devinfo;
1333    mesa_format format = src_mt->format;
1334    unsigned width = minify(src_mt->surf.phys_level0_sa.width,
1335                            src_level - src_mt->first_level);
1336    unsigned height = minify(src_mt->surf.phys_level0_sa.height,
1337                             src_level - src_mt->first_level);
1338 
1339    assert(src_layer < get_num_phys_layers(&src_mt->surf,
1340                                           src_level - src_mt->first_level));
1341 
1342    assert(_mesa_get_srgb_format_linear(src_mt->format) ==
1343           _mesa_get_srgb_format_linear(dst_mt->format));
1344 
1345    DBG("validate blit mt %s %p %d,%d -> mt %s %p %d,%d (%dx%d)\n",
1346        _mesa_get_format_name(src_mt->format),
1347        src_mt, src_level, src_layer,
1348        _mesa_get_format_name(dst_mt->format),
1349        dst_mt, dst_level, dst_layer,
1350        width, height);
1351 
1352    if (devinfo->gen >= 6) {
1353       /* On gen6 and above, we just use blorp.  It's faster than the blitter
1354        * and can handle everything without software fallbacks.
1355        */
1356       brw_blorp_copy_miptrees(brw,
1357                               src_mt, src_level, src_layer,
1358                               dst_mt, dst_level, dst_layer,
1359                               0, 0, 0, 0, width, height);
1360 
1361       if (src_mt->stencil_mt) {
1362          assert(dst_mt->stencil_mt);
1363          brw_blorp_copy_miptrees(brw,
1364                                  src_mt->stencil_mt, src_level, src_layer,
1365                                  dst_mt->stencil_mt, dst_level, dst_layer,
1366                                  0, 0, 0, 0, width, height);
1367       }
1368       return;
1369    }
1370 
1371    if (dst_mt->compressed) {
1372       unsigned int i, j;
1373       _mesa_get_format_block_size(dst_mt->format, &i, &j);
1374       height = ALIGN_NPOT(height, j) / j;
1375       width = ALIGN_NPOT(width, i) / i;
1376    }
1377 
1378    /* Gen4-5 doesn't support separate stencil */
1379    assert(!src_mt->stencil_mt);
1380 
1381    uint32_t dst_x, dst_y, src_x, src_y;
1382    intel_miptree_get_image_offset(dst_mt, dst_level, dst_layer,
1383                                   &dst_x, &dst_y);
1384    intel_miptree_get_image_offset(src_mt, src_level, src_layer,
1385                                   &src_x, &src_y);
1386 
1387    DBG("validate blit mt %s %p %d,%d/%d -> mt %s %p %d,%d/%d (%dx%d)\n",
1388        _mesa_get_format_name(src_mt->format),
1389        src_mt, src_x, src_y, src_mt->surf.row_pitch_B,
1390        _mesa_get_format_name(dst_mt->format),
1391        dst_mt, dst_x, dst_y, dst_mt->surf.row_pitch_B,
1392        width, height);
1393 
1394    if (!intel_miptree_blit(brw,
1395                            src_mt, src_level, src_layer, 0, 0, false,
1396                            dst_mt, dst_level, dst_layer, 0, 0, false,
1397                            width, height, COLOR_LOGICOP_COPY)) {
1398       perf_debug("miptree validate blit for %s failed\n",
1399                  _mesa_get_format_name(format));
1400 
1401       intel_miptree_copy_slice_sw(brw,
1402                                   src_mt, src_level, src_layer,
1403                                   dst_mt, dst_level, dst_layer,
1404                                   width, height);
1405    }
1406 }
1407 
1408 /**
1409  * Copies the image's current data to the given miptree, and associates that
1410  * miptree with the image.
1411  */
1412 void
intel_miptree_copy_teximage(struct brw_context * brw,struct intel_texture_image * intelImage,struct intel_mipmap_tree * dst_mt)1413 intel_miptree_copy_teximage(struct brw_context *brw,
1414 			    struct intel_texture_image *intelImage,
1415 			    struct intel_mipmap_tree *dst_mt)
1416 {
1417    struct intel_mipmap_tree *src_mt = intelImage->mt;
1418    struct intel_texture_object *intel_obj =
1419       intel_texture_object(intelImage->base.Base.TexObject);
1420    int level = intelImage->base.Base.Level;
1421    const unsigned face = intelImage->base.Base.Face;
1422    unsigned start_layer, end_layer;
1423 
1424    if (intel_obj->base.Target == GL_TEXTURE_1D_ARRAY) {
1425       assert(face == 0);
1426       assert(intelImage->base.Base.Height);
1427       start_layer = 0;
1428       end_layer = intelImage->base.Base.Height - 1;
1429    } else if (face > 0) {
1430       start_layer = face;
1431       end_layer = face;
1432    } else {
1433       assert(intelImage->base.Base.Depth);
1434       start_layer = 0;
1435       end_layer = intelImage->base.Base.Depth - 1;
1436    }
1437 
1438    for (unsigned i = start_layer; i <= end_layer; i++) {
1439       intel_miptree_copy_slice(brw,
1440                                src_mt, level, i,
1441                                dst_mt, level, i);
1442    }
1443 
1444    intel_miptree_reference(&intelImage->mt, dst_mt);
1445    intel_obj->needs_validate = true;
1446 }
1447 
1448 static struct intel_miptree_aux_buffer *
intel_alloc_aux_buffer(struct brw_context * brw,const struct isl_surf * aux_surf,bool wants_memset,uint8_t memset_value)1449 intel_alloc_aux_buffer(struct brw_context *brw,
1450                        const struct isl_surf *aux_surf,
1451                        bool wants_memset,
1452                        uint8_t memset_value)
1453 {
1454    struct intel_miptree_aux_buffer *buf = calloc(sizeof(*buf), 1);
1455    if (!buf)
1456       return false;
1457 
1458    uint64_t size = aux_surf->size_B;
1459 
1460    const bool has_indirect_clear = brw->isl_dev.ss.clear_color_state_size > 0;
1461    if (has_indirect_clear) {
1462       /* On CNL+, instead of setting the clear color in the SURFACE_STATE, we
1463        * will set a pointer to a dword somewhere that contains the color. So,
1464        * allocate the space for the clear color value here on the aux buffer.
1465        */
1466       buf->clear_color_offset = size;
1467       size += brw->isl_dev.ss.clear_color_state_size;
1468    }
1469 
1470    /* If the buffer needs to be initialised (requiring the buffer to be
1471     * immediately mapped to cpu space for writing), do not use the gpu access
1472     * flag which can cause an unnecessary delay if the backing pages happened
1473     * to be just used by the GPU.
1474     */
1475    const bool alloc_zeroed = wants_memset && memset_value == 0;
1476    const bool needs_memset =
1477       !alloc_zeroed && (wants_memset || has_indirect_clear);
1478    const uint32_t alloc_flags =
1479       alloc_zeroed ? BO_ALLOC_ZEROED : (needs_memset ? 0 : BO_ALLOC_BUSY);
1480 
1481    /* ISL has stricter set of alignment rules then the drm allocator.
1482     * Therefore one can pass the ISL dimensions in terms of bytes instead of
1483     * trying to recalculate based on different format block sizes.
1484     */
1485    buf->bo = brw_bo_alloc_tiled(brw->bufmgr, "aux-miptree", size,
1486                                 BRW_MEMZONE_OTHER, I915_TILING_Y,
1487                                 aux_surf->row_pitch_B, alloc_flags);
1488    if (!buf->bo) {
1489       free(buf);
1490       return NULL;
1491    }
1492 
1493    /* Initialize the bo to the desired value */
1494    if (needs_memset) {
1495       assert(!(alloc_flags & BO_ALLOC_BUSY));
1496 
1497       void *map = brw_bo_map(brw, buf->bo, MAP_WRITE | MAP_RAW);
1498       if (map == NULL) {
1499          intel_miptree_aux_buffer_free(buf);
1500          return NULL;
1501       }
1502 
1503       /* Memset the aux_surf portion of the BO. */
1504       if (wants_memset)
1505          memset(map, memset_value, aux_surf->size_B);
1506 
1507       /* Zero the indirect clear color to match ::fast_clear_color. */
1508       if (has_indirect_clear) {
1509          memset((char *)map + buf->clear_color_offset, 0,
1510                 brw->isl_dev.ss.clear_color_state_size);
1511       }
1512 
1513       brw_bo_unmap(buf->bo);
1514    }
1515 
1516    if (has_indirect_clear) {
1517       buf->clear_color_bo = buf->bo;
1518       brw_bo_reference(buf->clear_color_bo);
1519    }
1520 
1521    buf->surf = *aux_surf;
1522 
1523    return buf;
1524 }
1525 
1526 
1527 /**
1528  * Helper for intel_miptree_alloc_aux() that sets
1529  * \c mt->level[level].has_hiz. Return true if and only if
1530  * \c has_hiz was set.
1531  */
1532 static bool
intel_miptree_level_enable_hiz(struct brw_context * brw,struct intel_mipmap_tree * mt,uint32_t level)1533 intel_miptree_level_enable_hiz(struct brw_context *brw,
1534                                struct intel_mipmap_tree *mt,
1535                                uint32_t level)
1536 {
1537    const struct gen_device_info *devinfo = &brw->screen->devinfo;
1538 
1539    assert(mt->aux_buf);
1540    assert(mt->surf.size_B > 0);
1541 
1542    if (devinfo->gen >= 8 || devinfo->is_haswell) {
1543       uint32_t width = minify(mt->surf.phys_level0_sa.width, level);
1544       uint32_t height = minify(mt->surf.phys_level0_sa.height, level);
1545 
1546       /* Disable HiZ for LOD > 0 unless the width is 8 aligned
1547        * and the height is 4 aligned. This allows our HiZ support
1548        * to fulfill Haswell restrictions for HiZ ops. For LOD == 0,
1549        * we can grow the width & height to allow the HiZ op to
1550        * force the proper size alignments.
1551        */
1552       if (level > 0 && ((width & 7) || (height & 3))) {
1553          DBG("mt %p level %d: HiZ DISABLED\n", mt, level);
1554          return false;
1555       }
1556    }
1557 
1558    DBG("mt %p level %d: HiZ enabled\n", mt, level);
1559    mt->level[level].has_hiz = true;
1560    return true;
1561 }
1562 
1563 
1564 /**
1565  * Allocate the initial aux surface for a miptree based on mt->aux_usage
1566  *
1567  * Since MCS, HiZ, and CCS_E can compress more than just clear color, we
1568  * create the auxiliary surfaces up-front.  CCS_D, on the other hand, can only
1569  * compress clear color so we wait until an actual fast-clear to allocate it.
1570  */
1571 bool
intel_miptree_alloc_aux(struct brw_context * brw,struct intel_mipmap_tree * mt)1572 intel_miptree_alloc_aux(struct brw_context *brw,
1573                         struct intel_mipmap_tree *mt)
1574 {
1575    assert(mt->aux_buf == NULL);
1576 
1577    /* Get the aux buf allocation parameters for this miptree. */
1578    enum isl_aux_state initial_state;
1579    uint8_t memset_value;
1580    struct isl_surf aux_surf = {0,};
1581    bool aux_surf_ok = false;
1582 
1583    switch (mt->aux_usage) {
1584    case ISL_AUX_USAGE_NONE:
1585       aux_surf.size_B = 0;
1586       aux_surf_ok = true;
1587       break;
1588    case ISL_AUX_USAGE_HIZ:
1589       initial_state = ISL_AUX_STATE_AUX_INVALID;
1590       memset_value = 0;
1591       aux_surf_ok = isl_surf_get_hiz_surf(&brw->isl_dev, &mt->surf, &aux_surf);
1592       break;
1593    case ISL_AUX_USAGE_MCS:
1594       /* From the Ivy Bridge PRM, Vol 2 Part 1 p326:
1595        *
1596        *     When MCS buffer is enabled and bound to MSRT, it is required that
1597        *     it is cleared prior to any rendering.
1598        *
1599        * Since we don't use the MCS buffer for any purpose other than
1600        * rendering, it makes sense to just clear it immediately upon
1601        * allocation.
1602        *
1603        * Note: the clear value for MCS buffers is all 1's, so we memset to
1604        * 0xff.
1605        */
1606       initial_state = ISL_AUX_STATE_CLEAR;
1607       memset_value = 0xFF;
1608       aux_surf_ok = isl_surf_get_mcs_surf(&brw->isl_dev, &mt->surf, &aux_surf);
1609       break;
1610    case ISL_AUX_USAGE_CCS_D:
1611    case ISL_AUX_USAGE_CCS_E:
1612       /* When CCS_E is used, we need to ensure that the CCS starts off in a
1613        * valid state.  From the Sky Lake PRM, "MCS Buffer for Render
1614        * Target(s)":
1615        *
1616        *    "If Software wants to enable Color Compression without Fast
1617        *    clear, Software needs to initialize MCS with zeros."
1618        *
1619        * A CCS value of 0 indicates that the corresponding block is in the
1620        * pass-through state which is what we want.
1621        *
1622        * For CCS_D, do the same thing. On gen9+, this avoids having any
1623        * undefined bits in the aux buffer.
1624        */
1625       initial_state = ISL_AUX_STATE_PASS_THROUGH;
1626       memset_value = 0;
1627       aux_surf_ok =
1628          isl_surf_get_ccs_surf(&brw->isl_dev, &mt->surf, &aux_surf, NULL, 0);
1629       break;
1630 
1631    default:
1632       unreachable("Invalid aux usage");
1633    }
1634 
1635    /* We should have a valid aux_surf. */
1636    if (!aux_surf_ok)
1637       return false;
1638 
1639    /* No work is needed for a zero-sized auxiliary buffer. */
1640    if (aux_surf.size_B == 0)
1641       return true;
1642 
1643    /* Create the aux_state for the auxiliary buffer. */
1644    mt->aux_state = create_aux_state_map(mt, initial_state);
1645    if (mt->aux_state == NULL)
1646       return false;
1647 
1648    /* Allocate the auxiliary buffer. */
1649    const bool needs_memset = initial_state != ISL_AUX_STATE_AUX_INVALID;
1650    mt->aux_buf = intel_alloc_aux_buffer(brw, &aux_surf, needs_memset,
1651                                         memset_value);
1652    if (mt->aux_buf == NULL) {
1653       free_aux_state_map(mt->aux_state);
1654       mt->aux_state = NULL;
1655       return false;
1656    }
1657 
1658    /* Perform aux_usage-specific initialization. */
1659    if (mt->aux_usage == ISL_AUX_USAGE_HIZ) {
1660       for (unsigned level = mt->first_level; level <= mt->last_level; ++level)
1661          intel_miptree_level_enable_hiz(brw, mt, level);
1662    }
1663 
1664    return true;
1665 }
1666 
1667 
1668 /**
1669  * Can the miptree sample using the hiz buffer?
1670  */
1671 bool
intel_miptree_sample_with_hiz(struct brw_context * brw,struct intel_mipmap_tree * mt)1672 intel_miptree_sample_with_hiz(struct brw_context *brw,
1673                               struct intel_mipmap_tree *mt)
1674 {
1675    const struct gen_device_info *devinfo = &brw->screen->devinfo;
1676 
1677    if (!devinfo->has_sample_with_hiz) {
1678       return false;
1679    }
1680 
1681    if (!mt->aux_buf) {
1682       return false;
1683    }
1684 
1685    /* It seems the hardware won't fallback to the depth buffer if some of the
1686     * mipmap levels aren't available in the HiZ buffer. So we need all levels
1687     * of the texture to be HiZ enabled.
1688     */
1689    for (unsigned level = 0; level < mt->surf.levels; ++level) {
1690       if (!intel_miptree_level_has_hiz(mt, level))
1691          return false;
1692    }
1693 
1694    /* If compressed multisampling is enabled, then we use it for the auxiliary
1695     * buffer instead.
1696     *
1697     * From the BDW PRM (Volume 2d: Command Reference: Structures
1698     *                   RENDER_SURFACE_STATE.AuxiliarySurfaceMode):
1699     *
1700     *  "If this field is set to AUX_HIZ, Number of Multisamples must be
1701     *   MULTISAMPLECOUNT_1, and Surface Type cannot be SURFTYPE_3D.
1702     *
1703     * There is no such blurb for 1D textures, but there is sufficient evidence
1704     * that this is broken on SKL+.
1705     */
1706    return (mt->surf.samples == 1 &&
1707            mt->target != GL_TEXTURE_3D &&
1708            mt->target != GL_TEXTURE_1D /* gen9+ restriction */);
1709 }
1710 
1711 static bool
level_has_aux(const struct intel_mipmap_tree * mt,uint32_t level)1712 level_has_aux(const struct intel_mipmap_tree *mt, uint32_t level)
1713 {
1714    return isl_aux_usage_has_hiz(mt->aux_usage) ?
1715           intel_miptree_level_has_hiz(mt, level) :
1716           mt->aux_usage != ISL_AUX_USAGE_NONE && mt->aux_buf;
1717 }
1718 
1719 /**
1720  * Does the miptree slice have hiz enabled?
1721  */
1722 bool
intel_miptree_level_has_hiz(const struct intel_mipmap_tree * mt,uint32_t level)1723 intel_miptree_level_has_hiz(const struct intel_mipmap_tree *mt, uint32_t level)
1724 {
1725    intel_miptree_check_level_layer(mt, level, 0);
1726    return mt->level[level].has_hiz;
1727 }
1728 
1729 static inline uint32_t
miptree_level_range_length(const struct intel_mipmap_tree * mt,uint32_t start_level,uint32_t num_levels)1730 miptree_level_range_length(const struct intel_mipmap_tree *mt,
1731                            uint32_t start_level, uint32_t num_levels)
1732 {
1733    assert(start_level >= mt->first_level);
1734    assert(start_level <= mt->last_level);
1735 
1736    if (num_levels == INTEL_REMAINING_LAYERS)
1737       num_levels = mt->last_level - start_level + 1;
1738    /* Check for overflow */
1739    assert(start_level + num_levels >= start_level);
1740    assert(start_level + num_levels <= mt->last_level + 1);
1741 
1742    return num_levels;
1743 }
1744 
1745 static inline uint32_t
miptree_layer_range_length(const struct intel_mipmap_tree * mt,uint32_t level,uint32_t start_layer,uint32_t num_layers)1746 miptree_layer_range_length(const struct intel_mipmap_tree *mt, uint32_t level,
1747                            uint32_t start_layer, uint32_t num_layers)
1748 {
1749    assert(level <= mt->last_level);
1750 
1751    const uint32_t total_num_layers = brw_get_num_logical_layers(mt, level);
1752    assert(start_layer < total_num_layers);
1753    if (num_layers == INTEL_REMAINING_LAYERS)
1754       num_layers = total_num_layers - start_layer;
1755    /* Check for overflow */
1756    assert(start_layer + num_layers >= start_layer);
1757    assert(start_layer + num_layers <= total_num_layers);
1758 
1759    return num_layers;
1760 }
1761 
1762 bool
intel_miptree_has_color_unresolved(const struct intel_mipmap_tree * mt,unsigned start_level,unsigned num_levels,unsigned start_layer,unsigned num_layers)1763 intel_miptree_has_color_unresolved(const struct intel_mipmap_tree *mt,
1764                                    unsigned start_level, unsigned num_levels,
1765                                    unsigned start_layer, unsigned num_layers)
1766 {
1767    assert(_mesa_is_format_color_format(mt->format));
1768 
1769    if (!mt->aux_buf)
1770       return false;
1771 
1772    /* Clamp the level range to fit the miptree */
1773    num_levels = miptree_level_range_length(mt, start_level, num_levels);
1774 
1775    for (uint32_t l = 0; l < num_levels; l++) {
1776       const uint32_t level = start_level + l;
1777       const uint32_t level_layers =
1778          miptree_layer_range_length(mt, level, start_layer, num_layers);
1779       for (unsigned a = 0; a < level_layers; a++) {
1780          enum isl_aux_state aux_state =
1781             intel_miptree_get_aux_state(mt, level, start_layer + a);
1782          assert(aux_state != ISL_AUX_STATE_AUX_INVALID);
1783          if (aux_state != ISL_AUX_STATE_PASS_THROUGH)
1784             return true;
1785       }
1786    }
1787 
1788    return false;
1789 }
1790 
1791 static void
intel_miptree_check_color_resolve(const struct brw_context * brw,const struct intel_mipmap_tree * mt,unsigned level,unsigned layer)1792 intel_miptree_check_color_resolve(const struct brw_context *brw,
1793                                   const struct intel_mipmap_tree *mt,
1794                                   unsigned level, unsigned layer)
1795 {
1796    if (!mt->aux_buf)
1797       return;
1798 
1799    /* Fast color clear is supported for mipmapped surfaces only on Gen8+. */
1800    assert(brw->screen->devinfo.gen >= 8 ||
1801           (level == 0 && mt->first_level == 0 && mt->last_level == 0));
1802 
1803    /* Compression of arrayed msaa surfaces is supported. */
1804    if (mt->surf.samples > 1)
1805       return;
1806 
1807    /* Fast color clear is supported for non-msaa arrays only on Gen8+. */
1808    assert(brw->screen->devinfo.gen >= 8 ||
1809           (layer == 0 &&
1810            mt->surf.logical_level0_px.depth == 1 &&
1811            mt->surf.logical_level0_px.array_len == 1));
1812 
1813    (void)level;
1814    (void)layer;
1815 }
1816 
1817 void
intel_miptree_prepare_access(struct brw_context * brw,struct intel_mipmap_tree * mt,uint32_t start_level,uint32_t num_levels,uint32_t start_layer,uint32_t num_layers,enum isl_aux_usage aux_usage,bool fast_clear_supported)1818 intel_miptree_prepare_access(struct brw_context *brw,
1819                              struct intel_mipmap_tree *mt,
1820                              uint32_t start_level, uint32_t num_levels,
1821                              uint32_t start_layer, uint32_t num_layers,
1822                              enum isl_aux_usage aux_usage,
1823                              bool fast_clear_supported)
1824 {
1825    const uint32_t clamped_levels =
1826       miptree_level_range_length(mt, start_level, num_levels);
1827    for (uint32_t l = 0; l < clamped_levels; l++) {
1828       const uint32_t level = start_level + l;
1829       if (!level_has_aux(mt, level))
1830          continue;
1831 
1832       const uint32_t level_layers =
1833          miptree_layer_range_length(mt, level, start_layer, num_layers);
1834       for (uint32_t a = 0; a < level_layers; a++) {
1835          const uint32_t layer = start_layer + a;
1836          const enum isl_aux_state aux_state =
1837             intel_miptree_get_aux_state(mt, level, layer);
1838          const enum isl_aux_op aux_op =
1839             isl_aux_prepare_access(aux_state, aux_usage, fast_clear_supported);
1840 
1841          if (aux_op == ISL_AUX_OP_NONE) {
1842             /* Nothing to do here. */
1843          } else if (isl_aux_usage_has_mcs(mt->aux_usage)) {
1844             assert(aux_op == ISL_AUX_OP_PARTIAL_RESOLVE);
1845             brw_blorp_mcs_partial_resolve(brw, mt, layer, 1);
1846          } else if (isl_aux_usage_has_hiz(mt->aux_usage)) {
1847             intel_hiz_exec(brw, mt, level, layer, 1, aux_op);
1848          } else {
1849             assert(isl_aux_usage_has_ccs(mt->aux_usage));
1850             intel_miptree_check_color_resolve(brw, mt, level, layer);
1851             brw_blorp_resolve_color(brw, mt, level, layer, aux_op);
1852          }
1853 
1854          const enum isl_aux_state new_state =
1855             isl_aux_state_transition_aux_op(aux_state, mt->aux_usage, aux_op);
1856          intel_miptree_set_aux_state(brw, mt, level, layer, 1, new_state);
1857       }
1858    }
1859 }
1860 
1861 void
intel_miptree_finish_write(struct brw_context * brw,struct intel_mipmap_tree * mt,uint32_t level,uint32_t start_layer,uint32_t num_layers,enum isl_aux_usage aux_usage)1862 intel_miptree_finish_write(struct brw_context *brw,
1863                            struct intel_mipmap_tree *mt, uint32_t level,
1864                            uint32_t start_layer, uint32_t num_layers,
1865                            enum isl_aux_usage aux_usage)
1866 {
1867    const struct gen_device_info *devinfo = &brw->screen->devinfo;
1868 
1869    if (mt->format == MESA_FORMAT_S_UINT8 && devinfo->gen <= 7) {
1870       mt->shadow_needs_update = true;
1871    } else if (intel_miptree_has_etc_shadow(brw, mt)) {
1872       mt->shadow_needs_update = true;
1873    }
1874 
1875    if (!level_has_aux(mt, level))
1876       return;
1877 
1878    const uint32_t level_layers =
1879       miptree_layer_range_length(mt, level, start_layer, num_layers);
1880 
1881    for (uint32_t a = 0; a < level_layers; a++) {
1882       const uint32_t layer = start_layer + a;
1883       const enum isl_aux_state aux_state =
1884          intel_miptree_get_aux_state(mt, level, layer);
1885       const enum isl_aux_state new_aux_state =
1886          isl_aux_state_transition_write(aux_state, aux_usage, false);
1887       intel_miptree_set_aux_state(brw, mt, level, layer, 1, new_aux_state);
1888    }
1889 }
1890 
1891 enum isl_aux_state
intel_miptree_get_aux_state(const struct intel_mipmap_tree * mt,uint32_t level,uint32_t layer)1892 intel_miptree_get_aux_state(const struct intel_mipmap_tree *mt,
1893                             uint32_t level, uint32_t layer)
1894 {
1895    intel_miptree_check_level_layer(mt, level, layer);
1896 
1897    if (_mesa_is_format_color_format(mt->format)) {
1898       assert(mt->aux_buf != NULL);
1899       assert(mt->surf.samples == 1 ||
1900              mt->surf.msaa_layout == ISL_MSAA_LAYOUT_ARRAY);
1901    } else if (mt->format == MESA_FORMAT_S_UINT8) {
1902       unreachable("Cannot get aux state for stencil");
1903    } else {
1904       assert(intel_miptree_level_has_hiz(mt, level));
1905    }
1906 
1907    return mt->aux_state[level][layer];
1908 }
1909 
1910 void
intel_miptree_set_aux_state(struct brw_context * brw,struct intel_mipmap_tree * mt,uint32_t level,uint32_t start_layer,uint32_t num_layers,enum isl_aux_state aux_state)1911 intel_miptree_set_aux_state(struct brw_context *brw,
1912                             struct intel_mipmap_tree *mt, uint32_t level,
1913                             uint32_t start_layer, uint32_t num_layers,
1914                             enum isl_aux_state aux_state)
1915 {
1916    num_layers = miptree_layer_range_length(mt, level, start_layer, num_layers);
1917 
1918    if (_mesa_is_format_color_format(mt->format)) {
1919       assert(mt->aux_buf != NULL);
1920       assert(mt->surf.samples == 1 ||
1921              mt->surf.msaa_layout == ISL_MSAA_LAYOUT_ARRAY);
1922    } else if (mt->format == MESA_FORMAT_S_UINT8) {
1923       unreachable("Cannot get aux state for stencil");
1924    } else {
1925       assert(intel_miptree_level_has_hiz(mt, level));
1926    }
1927 
1928    for (unsigned a = 0; a < num_layers; a++) {
1929       if (mt->aux_state[level][start_layer + a] != aux_state) {
1930          mt->aux_state[level][start_layer + a] = aux_state;
1931          brw->ctx.NewDriverState |= BRW_NEW_AUX_STATE;
1932       }
1933    }
1934 }
1935 
1936 /* On Gen9 color buffers may be compressed by the hardware (lossless
1937  * compression). There are, however, format restrictions and care needs to be
1938  * taken that the sampler engine is capable for re-interpreting a buffer with
1939  * format different the buffer was originally written with.
1940  *
1941  * For example, SRGB formats are not compressible and the sampler engine isn't
1942  * capable of treating RGBA_UNORM as SRGB_ALPHA. In such a case the underlying
1943  * color buffer needs to be resolved so that the sampling surface can be
1944  * sampled as non-compressed (i.e., without the auxiliary MCS buffer being
1945  * set).
1946  */
1947 static bool
can_texture_with_ccs(struct brw_context * brw,struct intel_mipmap_tree * mt,enum isl_format view_format)1948 can_texture_with_ccs(struct brw_context *brw,
1949                      struct intel_mipmap_tree *mt,
1950                      enum isl_format view_format)
1951 {
1952    if (mt->aux_usage != ISL_AUX_USAGE_CCS_E)
1953       return false;
1954 
1955    if (!format_ccs_e_compat_with_miptree(&brw->screen->devinfo,
1956                                          mt, view_format)) {
1957       perf_debug("Incompatible sampling format (%s) for rbc (%s)\n",
1958                  isl_format_get_layout(view_format)->name,
1959                  _mesa_get_format_name(mt->format));
1960       return false;
1961    }
1962 
1963    return true;
1964 }
1965 
1966 enum isl_aux_usage
intel_miptree_texture_aux_usage(struct brw_context * brw,struct intel_mipmap_tree * mt,enum isl_format view_format,enum gen9_astc5x5_wa_tex_type astc5x5_wa_bits)1967 intel_miptree_texture_aux_usage(struct brw_context *brw,
1968                                 struct intel_mipmap_tree *mt,
1969                                 enum isl_format view_format,
1970                                 enum gen9_astc5x5_wa_tex_type astc5x5_wa_bits)
1971 {
1972    assert(brw->screen->devinfo.gen == 9 || astc5x5_wa_bits == 0);
1973 
1974    /* On gen9, ASTC 5x5 textures cannot live in the sampler cache along side
1975     * CCS or HiZ compressed textures.  See gen9_apply_astc5x5_wa_flush() for
1976     * details.
1977     */
1978    if ((astc5x5_wa_bits & GEN9_ASTC5X5_WA_TEX_TYPE_ASTC5x5) &&
1979        mt->aux_usage != ISL_AUX_USAGE_MCS)
1980       return ISL_AUX_USAGE_NONE;
1981 
1982    switch (mt->aux_usage) {
1983    case ISL_AUX_USAGE_HIZ:
1984       if (intel_miptree_sample_with_hiz(brw, mt))
1985          return ISL_AUX_USAGE_HIZ;
1986       break;
1987 
1988    case ISL_AUX_USAGE_MCS:
1989       return ISL_AUX_USAGE_MCS;
1990 
1991    case ISL_AUX_USAGE_CCS_D:
1992    case ISL_AUX_USAGE_CCS_E:
1993       if (!mt->aux_buf) {
1994          assert(mt->aux_usage == ISL_AUX_USAGE_CCS_D);
1995          return ISL_AUX_USAGE_NONE;
1996       }
1997 
1998       /* If we don't have any unresolved color, report an aux usage of
1999        * ISL_AUX_USAGE_NONE.  This way, texturing won't even look at the
2000        * aux surface and we can save some bandwidth.
2001        */
2002       if (!intel_miptree_has_color_unresolved(mt, 0, INTEL_REMAINING_LEVELS,
2003                                               0, INTEL_REMAINING_LAYERS))
2004          return ISL_AUX_USAGE_NONE;
2005 
2006       if (can_texture_with_ccs(brw, mt, view_format))
2007          return ISL_AUX_USAGE_CCS_E;
2008       break;
2009 
2010    default:
2011       break;
2012    }
2013 
2014    return ISL_AUX_USAGE_NONE;
2015 }
2016 
2017 static bool
isl_formats_are_fast_clear_compatible(enum isl_format a,enum isl_format b)2018 isl_formats_are_fast_clear_compatible(enum isl_format a, enum isl_format b)
2019 {
2020    /* On gen8 and earlier, the hardware was only capable of handling 0/1 clear
2021     * values so sRGB curve application was a no-op for all fast-clearable
2022     * formats.
2023     *
2024     * On gen9+, the hardware supports arbitrary clear values.  For sRGB clear
2025     * values, the hardware interprets the floats, not as what would be
2026     * returned from the sampler (or written by the shader), but as being
2027     * between format conversion and sRGB curve application.  This means that
2028     * we can switch between sRGB and UNORM without having to whack the clear
2029     * color.
2030     */
2031    return isl_format_srgb_to_linear(a) == isl_format_srgb_to_linear(b);
2032 }
2033 
2034 void
intel_miptree_prepare_texture(struct brw_context * brw,struct intel_mipmap_tree * mt,enum isl_format view_format,uint32_t start_level,uint32_t num_levels,uint32_t start_layer,uint32_t num_layers,enum gen9_astc5x5_wa_tex_type astc5x5_wa_bits)2035 intel_miptree_prepare_texture(struct brw_context *brw,
2036                               struct intel_mipmap_tree *mt,
2037                               enum isl_format view_format,
2038                               uint32_t start_level, uint32_t num_levels,
2039                               uint32_t start_layer, uint32_t num_layers,
2040                               enum gen9_astc5x5_wa_tex_type astc5x5_wa_bits)
2041 {
2042    enum isl_aux_usage aux_usage =
2043       intel_miptree_texture_aux_usage(brw, mt, view_format, astc5x5_wa_bits);
2044 
2045    bool clear_supported = aux_usage != ISL_AUX_USAGE_NONE;
2046 
2047    /* Clear color is specified as ints or floats and the conversion is done by
2048     * the sampler.  If we have a texture view, we would have to perform the
2049     * clear color conversion manually.  Just disable clear color.
2050     */
2051    if (!isl_formats_are_fast_clear_compatible(mt->surf.format, view_format))
2052       clear_supported = false;
2053 
2054    intel_miptree_prepare_access(brw, mt, start_level, num_levels,
2055                                 start_layer, num_layers,
2056                                 aux_usage, clear_supported);
2057 }
2058 
2059 void
intel_miptree_prepare_image(struct brw_context * brw,struct intel_mipmap_tree * mt)2060 intel_miptree_prepare_image(struct brw_context *brw,
2061                             struct intel_mipmap_tree *mt)
2062 {
2063    /* The data port doesn't understand any compression */
2064    intel_miptree_prepare_access(brw, mt, 0, INTEL_REMAINING_LEVELS,
2065                                 0, INTEL_REMAINING_LAYERS,
2066                                 ISL_AUX_USAGE_NONE, false);
2067 }
2068 
2069 enum isl_aux_usage
intel_miptree_render_aux_usage(struct brw_context * brw,struct intel_mipmap_tree * mt,enum isl_format render_format,bool blend_enabled,bool draw_aux_disabled)2070 intel_miptree_render_aux_usage(struct brw_context *brw,
2071                                struct intel_mipmap_tree *mt,
2072                                enum isl_format render_format,
2073                                bool blend_enabled,
2074                                bool draw_aux_disabled)
2075 {
2076    struct gen_device_info *devinfo = &brw->screen->devinfo;
2077 
2078    if (draw_aux_disabled)
2079       return ISL_AUX_USAGE_NONE;
2080 
2081    switch (mt->aux_usage) {
2082    case ISL_AUX_USAGE_MCS:
2083       assert(mt->aux_buf);
2084       return ISL_AUX_USAGE_MCS;
2085 
2086    case ISL_AUX_USAGE_CCS_D:
2087    case ISL_AUX_USAGE_CCS_E:
2088       if (!mt->aux_buf) {
2089          assert(mt->aux_usage == ISL_AUX_USAGE_CCS_D);
2090          return ISL_AUX_USAGE_NONE;
2091       }
2092 
2093       /* gen9+ hardware technically supports non-0/1 clear colors with sRGB
2094        * formats.  However, there are issues with blending where it doesn't
2095        * properly apply the sRGB curve to the clear color when blending.
2096        */
2097       if (devinfo->gen >= 9 && blend_enabled &&
2098           isl_format_is_srgb(render_format) &&
2099           !isl_color_value_is_zero_one(mt->fast_clear_color, render_format))
2100          return ISL_AUX_USAGE_NONE;
2101 
2102       if (mt->aux_usage == ISL_AUX_USAGE_CCS_E &&
2103           format_ccs_e_compat_with_miptree(&brw->screen->devinfo,
2104                                            mt, render_format))
2105          return ISL_AUX_USAGE_CCS_E;
2106 
2107       /* Otherwise, we have to fall back to CCS_D */
2108       return ISL_AUX_USAGE_CCS_D;
2109 
2110    default:
2111       return ISL_AUX_USAGE_NONE;
2112    }
2113 }
2114 
2115 void
intel_miptree_prepare_render(struct brw_context * brw,struct intel_mipmap_tree * mt,uint32_t level,uint32_t start_layer,uint32_t layer_count,enum isl_aux_usage aux_usage)2116 intel_miptree_prepare_render(struct brw_context *brw,
2117                              struct intel_mipmap_tree *mt, uint32_t level,
2118                              uint32_t start_layer, uint32_t layer_count,
2119                              enum isl_aux_usage aux_usage)
2120 {
2121    intel_miptree_prepare_access(brw, mt, level, 1, start_layer, layer_count,
2122                                 aux_usage, aux_usage != ISL_AUX_USAGE_NONE);
2123 }
2124 
2125 void
intel_miptree_finish_render(struct brw_context * brw,struct intel_mipmap_tree * mt,uint32_t level,uint32_t start_layer,uint32_t layer_count,enum isl_aux_usage aux_usage)2126 intel_miptree_finish_render(struct brw_context *brw,
2127                             struct intel_mipmap_tree *mt, uint32_t level,
2128                             uint32_t start_layer, uint32_t layer_count,
2129                             enum isl_aux_usage aux_usage)
2130 {
2131    assert(_mesa_is_format_color_format(mt->format));
2132 
2133    intel_miptree_finish_write(brw, mt, level, start_layer, layer_count,
2134                               aux_usage);
2135 }
2136 
2137 void
intel_miptree_prepare_depth(struct brw_context * brw,struct intel_mipmap_tree * mt,uint32_t level,uint32_t start_layer,uint32_t layer_count)2138 intel_miptree_prepare_depth(struct brw_context *brw,
2139                             struct intel_mipmap_tree *mt, uint32_t level,
2140                             uint32_t start_layer, uint32_t layer_count)
2141 {
2142    intel_miptree_prepare_access(brw, mt, level, 1, start_layer, layer_count,
2143                                 mt->aux_usage, mt->aux_buf != NULL);
2144 }
2145 
2146 void
intel_miptree_finish_depth(struct brw_context * brw,struct intel_mipmap_tree * mt,uint32_t level,uint32_t start_layer,uint32_t layer_count,bool depth_written)2147 intel_miptree_finish_depth(struct brw_context *brw,
2148                            struct intel_mipmap_tree *mt, uint32_t level,
2149                            uint32_t start_layer, uint32_t layer_count,
2150                            bool depth_written)
2151 {
2152    if (depth_written) {
2153       intel_miptree_finish_write(brw, mt, level, start_layer, layer_count,
2154                                  mt->aux_usage);
2155    }
2156 }
2157 
2158 void
intel_miptree_prepare_external(struct brw_context * brw,struct intel_mipmap_tree * mt)2159 intel_miptree_prepare_external(struct brw_context *brw,
2160                                struct intel_mipmap_tree *mt)
2161 {
2162    enum isl_aux_usage aux_usage = ISL_AUX_USAGE_NONE;
2163    bool supports_fast_clear = false;
2164 
2165    const struct isl_drm_modifier_info *mod_info =
2166       isl_drm_modifier_get_info(mt->drm_modifier);
2167 
2168    if (mod_info && mod_info->aux_usage != ISL_AUX_USAGE_NONE) {
2169       /* CCS_E is the only supported aux for external images and it's only
2170        * supported on very simple images.
2171        */
2172       assert(mod_info->aux_usage == ISL_AUX_USAGE_CCS_E);
2173       assert(_mesa_is_format_color_format(mt->format));
2174       assert(mt->first_level == 0 && mt->last_level == 0);
2175       assert(mt->surf.logical_level0_px.depth == 1);
2176       assert(mt->surf.logical_level0_px.array_len == 1);
2177       assert(mt->surf.samples == 1);
2178       assert(mt->aux_buf != NULL);
2179 
2180       aux_usage = mod_info->aux_usage;
2181       supports_fast_clear = mod_info->supports_clear_color;
2182    }
2183 
2184    intel_miptree_prepare_access(brw, mt, 0, INTEL_REMAINING_LEVELS,
2185                                 0, INTEL_REMAINING_LAYERS,
2186                                 aux_usage, supports_fast_clear);
2187 }
2188 
2189 void
intel_miptree_finish_external(struct brw_context * brw,struct intel_mipmap_tree * mt)2190 intel_miptree_finish_external(struct brw_context *brw,
2191                               struct intel_mipmap_tree *mt)
2192 {
2193    if (!mt->aux_buf)
2194       return;
2195 
2196    /* We don't know the actual aux state of the aux surface.  The previous
2197     * owner could have given it to us in a number of different states.
2198     * Because we don't know the aux state, we reset the aux state to the
2199     * least common denominator of possible valid states.
2200     */
2201    enum isl_aux_state default_aux_state =
2202       isl_drm_modifier_get_default_aux_state(mt->drm_modifier);
2203    assert(mt->last_level == mt->first_level);
2204    intel_miptree_set_aux_state(brw, mt, 0, 0, INTEL_REMAINING_LAYERS,
2205                                default_aux_state);
2206 }
2207 
2208 /**
2209  * Make it possible to share the BO backing the given miptree with another
2210  * process or another miptree.
2211  *
2212  * Fast color clears are unsafe with shared buffers, so we need to resolve and
2213  * then discard the MCS buffer, if present.  We also set the no_ccs flag to
2214  * ensure that no MCS buffer gets allocated in the future.
2215  *
2216  * HiZ is similarly unsafe with shared buffers.
2217  */
2218 void
intel_miptree_make_shareable(struct brw_context * brw,struct intel_mipmap_tree * mt)2219 intel_miptree_make_shareable(struct brw_context *brw,
2220                              struct intel_mipmap_tree *mt)
2221 {
2222    /* MCS buffers are also used for multisample buffers, but we can't resolve
2223     * away a multisample MCS buffer because it's an integral part of how the
2224     * pixel data is stored.  Fortunately this code path should never be
2225     * reached for multisample buffers.
2226     */
2227    assert(mt->surf.msaa_layout == ISL_MSAA_LAYOUT_NONE ||
2228           mt->surf.samples == 1);
2229 
2230    intel_miptree_prepare_access(brw, mt, 0, INTEL_REMAINING_LEVELS,
2231                                 0, INTEL_REMAINING_LAYERS,
2232                                 ISL_AUX_USAGE_NONE, false);
2233 
2234    if (mt->aux_buf) {
2235       intel_miptree_aux_buffer_free(mt->aux_buf);
2236       mt->aux_buf = NULL;
2237 
2238       /* Make future calls of intel_miptree_level_has_hiz() return false. */
2239       for (uint32_t l = mt->first_level; l <= mt->last_level; ++l) {
2240          mt->level[l].has_hiz = false;
2241       }
2242 
2243       free(mt->aux_state);
2244       mt->aux_state = NULL;
2245       brw->ctx.NewDriverState |= BRW_NEW_AUX_STATE;
2246    }
2247 
2248    mt->aux_usage = ISL_AUX_USAGE_NONE;
2249    mt->supports_fast_clear = false;
2250 }
2251 
2252 
2253 /**
2254  * \brief Get pointer offset into stencil buffer.
2255  *
2256  * The stencil buffer is W tiled. Since the GTT is incapable of W fencing, we
2257  * must decode the tile's layout in software.
2258  *
2259  * See
2260  *   - PRM, 2011 Sandy Bridge, Volume 1, Part 2, Section 4.5.2.1 W-Major Tile
2261  *     Format.
2262  *   - PRM, 2011 Sandy Bridge, Volume 1, Part 2, Section 4.5.3 Tiling Algorithm
2263  *
2264  * Even though the returned offset is always positive, the return type is
2265  * signed due to
2266  *    commit e8b1c6d6f55f5be3bef25084fdd8b6127517e137
2267  *    mesa: Fix return type of  _mesa_get_format_bytes() (#37351)
2268  */
2269 static intptr_t
intel_offset_S8(uint32_t stride,uint32_t x,uint32_t y,bool swizzled)2270 intel_offset_S8(uint32_t stride, uint32_t x, uint32_t y, bool swizzled)
2271 {
2272    uint32_t tile_size = 4096;
2273    uint32_t tile_width = 64;
2274    uint32_t tile_height = 64;
2275    uint32_t row_size = 64 * stride / 2; /* Two rows are interleaved. */
2276 
2277    uint32_t tile_x = x / tile_width;
2278    uint32_t tile_y = y / tile_height;
2279 
2280    /* The byte's address relative to the tile's base addres. */
2281    uint32_t byte_x = x % tile_width;
2282    uint32_t byte_y = y % tile_height;
2283 
2284    uintptr_t u = tile_y * row_size
2285                + tile_x * tile_size
2286                + 512 * (byte_x / 8)
2287                +  64 * (byte_y / 8)
2288                +  32 * ((byte_y / 4) % 2)
2289                +  16 * ((byte_x / 4) % 2)
2290                +   8 * ((byte_y / 2) % 2)
2291                +   4 * ((byte_x / 2) % 2)
2292                +   2 * (byte_y % 2)
2293                +   1 * (byte_x % 2);
2294 
2295    if (swizzled) {
2296       /* adjust for bit6 swizzling */
2297       if (((byte_x / 8) % 2) == 1) {
2298 	 if (((byte_y / 8) % 2) == 0) {
2299 	    u += 64;
2300 	 } else {
2301 	    u -= 64;
2302 	 }
2303       }
2304    }
2305 
2306    return u;
2307 }
2308 
2309 void
intel_miptree_updownsample(struct brw_context * brw,struct intel_mipmap_tree * src,struct intel_mipmap_tree * dst)2310 intel_miptree_updownsample(struct brw_context *brw,
2311                            struct intel_mipmap_tree *src,
2312                            struct intel_mipmap_tree *dst)
2313 {
2314    unsigned src_w = src->surf.logical_level0_px.width;
2315    unsigned src_h = src->surf.logical_level0_px.height;
2316    unsigned dst_w = dst->surf.logical_level0_px.width;
2317    unsigned dst_h = dst->surf.logical_level0_px.height;
2318 
2319    brw_blorp_blit_miptrees(brw,
2320                            src, 0 /* level */, 0 /* layer */,
2321                            src->format, SWIZZLE_XYZW,
2322                            dst, 0 /* level */, 0 /* layer */, dst->format,
2323                            0, 0, src_w, src_h,
2324                            0, 0, dst_w, dst_h,
2325                            GL_NEAREST, false, false /*mirror x, y*/,
2326                            false, false);
2327 
2328    if (src->stencil_mt) {
2329       src_w = src->stencil_mt->surf.logical_level0_px.width;
2330       src_h = src->stencil_mt->surf.logical_level0_px.height;
2331       dst_w = dst->stencil_mt->surf.logical_level0_px.width;
2332       dst_h = dst->stencil_mt->surf.logical_level0_px.height;
2333 
2334       brw_blorp_blit_miptrees(brw,
2335                               src->stencil_mt, 0 /* level */, 0 /* layer */,
2336                               src->stencil_mt->format, SWIZZLE_XYZW,
2337                               dst->stencil_mt, 0 /* level */, 0 /* layer */,
2338                               dst->stencil_mt->format,
2339                               0, 0, src_w, src_h,
2340                               0, 0, dst_w, dst_h,
2341                               GL_NEAREST, false, false /*mirror x, y*/,
2342                               false, false /* decode/encode srgb */);
2343    }
2344 }
2345 
2346 void
intel_update_r8stencil(struct brw_context * brw,struct intel_mipmap_tree * mt)2347 intel_update_r8stencil(struct brw_context *brw,
2348                        struct intel_mipmap_tree *mt)
2349 {
2350    const struct gen_device_info *devinfo = &brw->screen->devinfo;
2351 
2352    assert(devinfo->gen >= 7);
2353    struct intel_mipmap_tree *src =
2354       mt->format == MESA_FORMAT_S_UINT8 ? mt : mt->stencil_mt;
2355    if (!src || devinfo->gen >= 8)
2356       return;
2357 
2358    assert(src->surf.size_B > 0);
2359 
2360    if (!mt->shadow_mt) {
2361       assert(devinfo->gen > 6); /* Handle MIPTREE_LAYOUT_GEN6_HIZ_STENCIL */
2362       mt->shadow_mt = make_surface(
2363                             brw,
2364                             src->target,
2365                             MESA_FORMAT_R_UINT8,
2366                             src->first_level, src->last_level,
2367                             src->surf.logical_level0_px.width,
2368                             src->surf.logical_level0_px.height,
2369                             src->surf.dim == ISL_SURF_DIM_3D ?
2370                                src->surf.logical_level0_px.depth :
2371                                src->surf.logical_level0_px.array_len,
2372                             src->surf.samples,
2373                             ISL_TILING_Y0_BIT,
2374                             ISL_SURF_USAGE_TEXTURE_BIT,
2375                             BO_ALLOC_BUSY, 0, NULL);
2376       assert(mt->shadow_mt);
2377    }
2378 
2379    if (src->shadow_needs_update == false)
2380       return;
2381 
2382    struct intel_mipmap_tree *dst = mt->shadow_mt;
2383 
2384    for (int level = src->first_level; level <= src->last_level; level++) {
2385       const unsigned depth = src->surf.dim == ISL_SURF_DIM_3D ?
2386          minify(src->surf.phys_level0_sa.depth, level) :
2387          src->surf.phys_level0_sa.array_len;
2388 
2389       for (unsigned layer = 0; layer < depth; layer++) {
2390          brw_blorp_copy_miptrees(brw,
2391                                  src, level, layer,
2392                                  dst, level, layer,
2393                                  0, 0, 0, 0,
2394                                  minify(src->surf.logical_level0_px.width,
2395                                         level),
2396                                  minify(src->surf.logical_level0_px.height,
2397                                         level));
2398       }
2399    }
2400 
2401    brw_cache_flush_for_read(brw, dst->bo);
2402    src->shadow_needs_update = false;
2403 }
2404 
2405 static void *
intel_miptree_map_raw(struct brw_context * brw,struct intel_mipmap_tree * mt,GLbitfield mode)2406 intel_miptree_map_raw(struct brw_context *brw,
2407                       struct intel_mipmap_tree *mt,
2408                       GLbitfield mode)
2409 {
2410    struct brw_bo *bo = mt->bo;
2411 
2412    if (brw_batch_references(&brw->batch, bo))
2413       intel_batchbuffer_flush(brw);
2414 
2415    return brw_bo_map(brw, bo, mode);
2416 }
2417 
2418 static void
intel_miptree_unmap_raw(struct intel_mipmap_tree * mt)2419 intel_miptree_unmap_raw(struct intel_mipmap_tree *mt)
2420 {
2421    brw_bo_unmap(mt->bo);
2422 }
2423 
2424 static void
intel_miptree_unmap_map(struct brw_context * brw,struct intel_mipmap_tree * mt,struct intel_miptree_map * map,unsigned int level,unsigned int slice)2425 intel_miptree_unmap_map(struct brw_context *brw,
2426                         struct intel_mipmap_tree *mt,
2427                         struct intel_miptree_map *map,
2428                         unsigned int level, unsigned int slice)
2429 {
2430    intel_miptree_unmap_raw(mt);
2431 }
2432 
2433 static void
intel_miptree_map_map(struct brw_context * brw,struct intel_mipmap_tree * mt,struct intel_miptree_map * map,unsigned int level,unsigned int slice)2434 intel_miptree_map_map(struct brw_context *brw,
2435 		      struct intel_mipmap_tree *mt,
2436 		      struct intel_miptree_map *map,
2437 		      unsigned int level, unsigned int slice)
2438 {
2439    unsigned int bw, bh;
2440    void *base;
2441    unsigned int image_x, image_y;
2442    intptr_t x = map->x;
2443    intptr_t y = map->y;
2444 
2445    /* For compressed formats, the stride is the number of bytes per
2446     * row of blocks.  intel_miptree_get_image_offset() already does
2447     * the divide.
2448     */
2449    _mesa_get_format_block_size(mt->format, &bw, &bh);
2450    assert(y % bh == 0);
2451    assert(x % bw == 0);
2452    y /= bh;
2453    x /= bw;
2454 
2455    intel_miptree_access_raw(brw, mt, level, slice,
2456                             map->mode & GL_MAP_WRITE_BIT);
2457 
2458    base = intel_miptree_map_raw(brw, mt, map->mode);
2459 
2460    if (base == NULL)
2461       map->ptr = NULL;
2462    else {
2463       base += mt->offset;
2464 
2465       /* Note that in the case of cube maps, the caller must have passed the
2466        * slice number referencing the face.
2467       */
2468       intel_miptree_get_image_offset(mt, level, slice, &image_x, &image_y);
2469       x += image_x;
2470       y += image_y;
2471 
2472       map->stride = mt->surf.row_pitch_B;
2473       map->ptr = base + y * map->stride + x * mt->cpp;
2474    }
2475 
2476    DBG("%s: %d,%d %dx%d from mt %p (%s) "
2477        "%"PRIiPTR",%"PRIiPTR" = %p/%d\n", __func__,
2478        map->x, map->y, map->w, map->h,
2479        mt, _mesa_get_format_name(mt->format),
2480        x, y, map->ptr, map->stride);
2481 
2482    map->unmap = intel_miptree_unmap_map;
2483 }
2484 
2485 static void
intel_miptree_unmap_blit(struct brw_context * brw,struct intel_mipmap_tree * mt,struct intel_miptree_map * map,unsigned int level,unsigned int slice)2486 intel_miptree_unmap_blit(struct brw_context *brw,
2487 			 struct intel_mipmap_tree *mt,
2488 			 struct intel_miptree_map *map,
2489 			 unsigned int level,
2490 			 unsigned int slice)
2491 {
2492    const struct gen_device_info *devinfo = &brw->screen->devinfo;
2493    struct gl_context *ctx = &brw->ctx;
2494 
2495    intel_miptree_unmap_raw(map->linear_mt);
2496 
2497    if (map->mode & GL_MAP_WRITE_BIT) {
2498       if (devinfo->gen >= 6) {
2499          brw_blorp_copy_miptrees(brw, map->linear_mt, 0, 0,
2500                                  mt, level, slice,
2501                                  0, 0, map->x, map->y, map->w, map->h);
2502       } else {
2503          bool ok = intel_miptree_copy(brw,
2504                                       map->linear_mt, 0, 0, 0, 0,
2505                                       mt, level, slice, map->x, map->y,
2506                                       map->w, map->h);
2507          WARN_ONCE(!ok, "Failed to blit from linear temporary mapping");
2508       }
2509    }
2510 
2511    intel_miptree_release(&map->linear_mt);
2512 }
2513 
2514 /* Compute extent parameters for use with tiled_memcpy functions.
2515  * xs are in units of bytes and ys are in units of strides.
2516  */
2517 static inline void
tile_extents(struct intel_mipmap_tree * mt,struct intel_miptree_map * map,unsigned int level,unsigned int slice,unsigned int * x1_B,unsigned int * x2_B,unsigned int * y1_el,unsigned int * y2_el)2518 tile_extents(struct intel_mipmap_tree *mt, struct intel_miptree_map *map,
2519              unsigned int level, unsigned int slice, unsigned int *x1_B,
2520              unsigned int *x2_B, unsigned int *y1_el, unsigned int *y2_el)
2521 {
2522    unsigned int block_width, block_height;
2523    unsigned int x0_el, y0_el;
2524 
2525    _mesa_get_format_block_size(mt->format, &block_width, &block_height);
2526 
2527    assert(map->x % block_width == 0);
2528    assert(map->y % block_height == 0);
2529 
2530    intel_miptree_get_image_offset(mt, level, slice, &x0_el, &y0_el);
2531    *x1_B = (map->x / block_width + x0_el) * mt->cpp;
2532    *y1_el = map->y / block_height + y0_el;
2533    *x2_B = (DIV_ROUND_UP(map->x + map->w, block_width) + x0_el) * mt->cpp;
2534    *y2_el = DIV_ROUND_UP(map->y + map->h, block_height) + y0_el;
2535 }
2536 
2537 static void
intel_miptree_unmap_tiled_memcpy(struct brw_context * brw,struct intel_mipmap_tree * mt,struct intel_miptree_map * map,unsigned int level,unsigned int slice)2538 intel_miptree_unmap_tiled_memcpy(struct brw_context *brw,
2539                                  struct intel_mipmap_tree *mt,
2540                                  struct intel_miptree_map *map,
2541                                  unsigned int level,
2542                                  unsigned int slice)
2543 {
2544    if (map->mode & GL_MAP_WRITE_BIT) {
2545       unsigned int x1, x2, y1, y2;
2546       tile_extents(mt, map, level, slice, &x1, &x2, &y1, &y2);
2547 
2548       char *dst = intel_miptree_map_raw(brw, mt, map->mode | MAP_RAW);
2549       dst += mt->offset;
2550 
2551       isl_memcpy_linear_to_tiled(
2552          x1, x2, y1, y2, dst, map->ptr, mt->surf.row_pitch_B, map->stride,
2553          brw->has_swizzling, mt->surf.tiling, ISL_MEMCPY);
2554 
2555       intel_miptree_unmap_raw(mt);
2556    }
2557    align_free(map->buffer);
2558    map->buffer = map->ptr = NULL;
2559 }
2560 
2561 /**
2562  * Determine which copy function to use for the given format combination
2563  *
2564  * The only two possible copy functions which are ever returned are a
2565  * direct memcpy and a RGBA <-> BGRA copy function.  Since RGBA -> BGRA and
2566  * BGRA -> RGBA are exactly the same operation (and memcpy is obviously
2567  * symmetric), it doesn't matter whether the copy is from the tiled image
2568  * to the untiled or vice versa.  The copy function required is the same in
2569  * either case so this function can be used.
2570  *
2571  * \param[in]  tiledFormat The format of the tiled image
2572  * \param[in]  format      The GL format of the client data
2573  * \param[in]  type        The GL type of the client data
2574  * \param[out] mem_copy    Will be set to one of either the standard
2575  *                         library's memcpy or a different copy function
2576  *                         that performs an RGBA to BGRA conversion
2577  * \param[out] cpp         Number of bytes per channel
2578  *
2579  * \return true if the format and type combination are valid
2580  */
2581 isl_memcpy_type
intel_miptree_get_memcpy_type(mesa_format tiledFormat,GLenum format,GLenum type,uint32_t * cpp)2582 intel_miptree_get_memcpy_type(mesa_format tiledFormat, GLenum format, GLenum type,
2583                               uint32_t *cpp)
2584 {
2585    if (type == GL_UNSIGNED_INT_8_8_8_8_REV &&
2586        !(format == GL_RGBA || format == GL_BGRA))
2587       return ISL_MEMCPY_INVALID; /* Invalid type/format combination */
2588 
2589    if ((tiledFormat == MESA_FORMAT_L_UNORM8 && format == GL_LUMINANCE) ||
2590        (tiledFormat == MESA_FORMAT_A_UNORM8 && format == GL_ALPHA)) {
2591       *cpp = 1;
2592       return ISL_MEMCPY;
2593    } else if ((tiledFormat == MESA_FORMAT_B8G8R8A8_UNORM) ||
2594               (tiledFormat == MESA_FORMAT_B8G8R8X8_UNORM) ||
2595               (tiledFormat == MESA_FORMAT_B8G8R8A8_SRGB) ||
2596               (tiledFormat == MESA_FORMAT_B8G8R8X8_SRGB)) {
2597       *cpp = 4;
2598       if (format == GL_BGRA) {
2599          return ISL_MEMCPY;
2600       } else if (format == GL_RGBA) {
2601          return ISL_MEMCPY_BGRA8;
2602       }
2603    } else if ((tiledFormat == MESA_FORMAT_R8G8B8A8_UNORM) ||
2604               (tiledFormat == MESA_FORMAT_R8G8B8X8_UNORM) ||
2605               (tiledFormat == MESA_FORMAT_R8G8B8A8_SRGB) ||
2606               (tiledFormat == MESA_FORMAT_R8G8B8X8_SRGB)) {
2607       *cpp = 4;
2608       if (format == GL_BGRA) {
2609          /* Copying from RGBA to BGRA is the same as BGRA to RGBA so we can
2610           * use the same function.
2611           */
2612          return ISL_MEMCPY_BGRA8;
2613       } else if (format == GL_RGBA) {
2614          return ISL_MEMCPY;
2615       }
2616    }
2617 
2618    return ISL_MEMCPY_INVALID;
2619 }
2620 
2621 static void
intel_miptree_map_tiled_memcpy(struct brw_context * brw,struct intel_mipmap_tree * mt,struct intel_miptree_map * map,unsigned int level,unsigned int slice)2622 intel_miptree_map_tiled_memcpy(struct brw_context *brw,
2623                                struct intel_mipmap_tree *mt,
2624                                struct intel_miptree_map *map,
2625                                unsigned int level, unsigned int slice)
2626 {
2627    intel_miptree_access_raw(brw, mt, level, slice,
2628                             map->mode & GL_MAP_WRITE_BIT);
2629 
2630    unsigned int x1, x2, y1, y2;
2631    tile_extents(mt, map, level, slice, &x1, &x2, &y1, &y2);
2632    map->stride = ALIGN(_mesa_format_row_stride(mt->format, map->w), 16);
2633 
2634    /* The tiling and detiling functions require that the linear buffer
2635     * has proper 16-byte alignment (that is, its `x0` is 16-byte
2636     * aligned). Here we over-allocate the linear buffer by enough
2637     * bytes to get the proper alignment.
2638     */
2639    map->buffer = align_malloc(map->stride * (y2 - y1) + (x1 & 0xf), 16);
2640    map->ptr = (char *)map->buffer + (x1 & 0xf);
2641    assert(map->buffer);
2642 
2643    if (!(map->mode & GL_MAP_INVALIDATE_RANGE_BIT)) {
2644       char *src = intel_miptree_map_raw(brw, mt, map->mode | MAP_RAW);
2645       src += mt->offset;
2646 
2647       const isl_memcpy_type copy_type =
2648 #if defined(USE_SSE41)
2649          cpu_has_sse4_1 ? ISL_MEMCPY_STREAMING_LOAD :
2650 #endif
2651          ISL_MEMCPY;
2652 
2653       isl_memcpy_tiled_to_linear(
2654          x1, x2, y1, y2, map->ptr, src, map->stride,
2655          mt->surf.row_pitch_B, brw->has_swizzling, mt->surf.tiling,
2656          copy_type);
2657 
2658       intel_miptree_unmap_raw(mt);
2659    }
2660 
2661    map->unmap = intel_miptree_unmap_tiled_memcpy;
2662 }
2663 
2664 static void
intel_miptree_map_blit(struct brw_context * brw,struct intel_mipmap_tree * mt,struct intel_miptree_map * map,unsigned int level,unsigned int slice)2665 intel_miptree_map_blit(struct brw_context *brw,
2666 		       struct intel_mipmap_tree *mt,
2667 		       struct intel_miptree_map *map,
2668 		       unsigned int level, unsigned int slice)
2669 {
2670    const struct gen_device_info *devinfo = &brw->screen->devinfo;
2671    map->linear_mt = make_surface(brw, GL_TEXTURE_2D, mt->format,
2672                                  0, 0, map->w, map->h, 1, 1,
2673                                  ISL_TILING_LINEAR_BIT,
2674                                  ISL_SURF_USAGE_RENDER_TARGET_BIT |
2675                                  ISL_SURF_USAGE_TEXTURE_BIT,
2676                                  0, 0, NULL);
2677 
2678    if (!map->linear_mt) {
2679       fprintf(stderr, "Failed to allocate blit temporary\n");
2680       goto fail;
2681    }
2682    map->stride = map->linear_mt->surf.row_pitch_B;
2683 
2684    /* One of either READ_BIT or WRITE_BIT or both is set.  READ_BIT implies no
2685     * INVALIDATE_RANGE_BIT.  WRITE_BIT needs the original values read in unless
2686     * invalidate is set, since we'll be writing the whole rectangle from our
2687     * temporary buffer back out.
2688     */
2689    if (!(map->mode & GL_MAP_INVALIDATE_RANGE_BIT)) {
2690       if (devinfo->gen >= 6) {
2691          brw_blorp_copy_miptrees(brw, mt, level, slice,
2692                                  map->linear_mt, 0, 0,
2693                                  map->x, map->y, 0, 0, map->w, map->h);
2694       } else {
2695          if (!intel_miptree_copy(brw,
2696                                  mt, level, slice, map->x, map->y,
2697                                  map->linear_mt, 0, 0, 0, 0,
2698                                  map->w, map->h)) {
2699             fprintf(stderr, "Failed to blit\n");
2700             goto fail;
2701          }
2702       }
2703    }
2704 
2705    map->ptr = intel_miptree_map_raw(brw, map->linear_mt, map->mode);
2706 
2707    DBG("%s: %d,%d %dx%d from mt %p (%s) %d,%d = %p/%d\n", __func__,
2708        map->x, map->y, map->w, map->h,
2709        mt, _mesa_get_format_name(mt->format),
2710        level, slice, map->ptr, map->stride);
2711 
2712    map->unmap = intel_miptree_unmap_blit;
2713    return;
2714 
2715 fail:
2716    intel_miptree_release(&map->linear_mt);
2717    map->ptr = NULL;
2718    map->stride = 0;
2719 }
2720 
2721 /**
2722  * "Map" a buffer by copying it to an untiled temporary using MOVNTDQA.
2723  */
2724 #if defined(USE_SSE41)
2725 static void
intel_miptree_unmap_movntdqa(struct brw_context * brw,struct intel_mipmap_tree * mt,struct intel_miptree_map * map,unsigned int level,unsigned int slice)2726 intel_miptree_unmap_movntdqa(struct brw_context *brw,
2727                              struct intel_mipmap_tree *mt,
2728                              struct intel_miptree_map *map,
2729                              unsigned int level,
2730                              unsigned int slice)
2731 {
2732    align_free(map->buffer);
2733    map->buffer = NULL;
2734    map->ptr = NULL;
2735 }
2736 
2737 static void
intel_miptree_map_movntdqa(struct brw_context * brw,struct intel_mipmap_tree * mt,struct intel_miptree_map * map,unsigned int level,unsigned int slice)2738 intel_miptree_map_movntdqa(struct brw_context *brw,
2739                            struct intel_mipmap_tree *mt,
2740                            struct intel_miptree_map *map,
2741                            unsigned int level, unsigned int slice)
2742 {
2743    assert(map->mode & GL_MAP_READ_BIT);
2744    assert(!(map->mode & GL_MAP_WRITE_BIT));
2745 
2746    intel_miptree_access_raw(brw, mt, level, slice, false);
2747 
2748    DBG("%s: %d,%d %dx%d from mt %p (%s) %d,%d = %p/%d\n", __func__,
2749        map->x, map->y, map->w, map->h,
2750        mt, _mesa_get_format_name(mt->format),
2751        level, slice, map->ptr, map->stride);
2752 
2753    /* Map the original image */
2754    uint32_t image_x;
2755    uint32_t image_y;
2756    intel_miptree_get_image_offset(mt, level, slice, &image_x, &image_y);
2757    image_x += map->x;
2758    image_y += map->y;
2759 
2760    void *src = intel_miptree_map_raw(brw, mt, map->mode);
2761    if (!src)
2762       return;
2763 
2764    src += mt->offset;
2765 
2766    src += image_y * mt->surf.row_pitch_B;
2767    src += image_x * mt->cpp;
2768 
2769    /* Due to the pixel offsets for the particular image being mapped, our
2770     * src pointer may not be 16-byte aligned.  However, if the pitch is
2771     * divisible by 16, then the amount by which it's misaligned will remain
2772     * consistent from row to row.
2773     */
2774    assert((mt->surf.row_pitch_B % 16) == 0);
2775    const int misalignment = ((uintptr_t) src) & 15;
2776 
2777    /* Create an untiled temporary buffer for the mapping. */
2778    const unsigned width_bytes = _mesa_format_row_stride(mt->format, map->w);
2779 
2780    map->stride = ALIGN(misalignment + width_bytes, 16);
2781 
2782    map->buffer = align_malloc(map->stride * map->h, 16);
2783    /* Offset the destination so it has the same misalignment as src. */
2784    map->ptr = map->buffer + misalignment;
2785 
2786    assert((((uintptr_t) map->ptr) & 15) == misalignment);
2787 
2788    for (uint32_t y = 0; y < map->h; y++) {
2789       void *dst_ptr = map->ptr + y * map->stride;
2790       void *src_ptr = src + y * mt->surf.row_pitch_B;
2791 
2792       _mesa_streaming_load_memcpy(dst_ptr, src_ptr, width_bytes);
2793    }
2794 
2795    intel_miptree_unmap_raw(mt);
2796 
2797    map->unmap = intel_miptree_unmap_movntdqa;
2798 }
2799 #endif
2800 
2801 static void
intel_miptree_unmap_s8(struct brw_context * brw,struct intel_mipmap_tree * mt,struct intel_miptree_map * map,unsigned int level,unsigned int slice)2802 intel_miptree_unmap_s8(struct brw_context *brw,
2803 		       struct intel_mipmap_tree *mt,
2804 		       struct intel_miptree_map *map,
2805 		       unsigned int level,
2806 		       unsigned int slice)
2807 {
2808    if (map->mode & GL_MAP_WRITE_BIT) {
2809       unsigned int image_x, image_y;
2810       uint8_t *untiled_s8_map = map->ptr;
2811       uint8_t *tiled_s8_map = intel_miptree_map_raw(brw, mt, GL_MAP_WRITE_BIT);
2812 
2813       intel_miptree_get_image_offset(mt, level, slice, &image_x, &image_y);
2814 
2815       for (uint32_t y = 0; y < map->h; y++) {
2816 	 for (uint32_t x = 0; x < map->w; x++) {
2817 	    ptrdiff_t offset = intel_offset_S8(mt->surf.row_pitch_B,
2818 	                                       image_x + x + map->x,
2819 	                                       image_y + y + map->y,
2820 					       brw->has_swizzling);
2821 	    tiled_s8_map[offset] = untiled_s8_map[y * map->w + x];
2822 	 }
2823       }
2824 
2825       intel_miptree_unmap_raw(mt);
2826    }
2827 
2828    free(map->buffer);
2829 }
2830 
2831 static void
intel_miptree_map_s8(struct brw_context * brw,struct intel_mipmap_tree * mt,struct intel_miptree_map * map,unsigned int level,unsigned int slice)2832 intel_miptree_map_s8(struct brw_context *brw,
2833 		     struct intel_mipmap_tree *mt,
2834 		     struct intel_miptree_map *map,
2835 		     unsigned int level, unsigned int slice)
2836 {
2837    map->stride = map->w;
2838    map->buffer = map->ptr = malloc(map->stride * map->h);
2839    if (!map->buffer)
2840       return;
2841 
2842    intel_miptree_access_raw(brw, mt, level, slice,
2843                             map->mode & GL_MAP_WRITE_BIT);
2844 
2845    /* One of either READ_BIT or WRITE_BIT or both is set.  READ_BIT implies no
2846     * INVALIDATE_RANGE_BIT.  WRITE_BIT needs the original values read in unless
2847     * invalidate is set, since we'll be writing the whole rectangle from our
2848     * temporary buffer back out.
2849     */
2850    if (!(map->mode & GL_MAP_INVALIDATE_RANGE_BIT)) {
2851       uint8_t *untiled_s8_map = map->ptr;
2852       uint8_t *tiled_s8_map = intel_miptree_map_raw(brw, mt, GL_MAP_READ_BIT);
2853       unsigned int image_x, image_y;
2854 
2855       intel_miptree_get_image_offset(mt, level, slice, &image_x, &image_y);
2856 
2857       for (uint32_t y = 0; y < map->h; y++) {
2858 	 for (uint32_t x = 0; x < map->w; x++) {
2859 	    ptrdiff_t offset = intel_offset_S8(mt->surf.row_pitch_B,
2860 	                                       x + image_x + map->x,
2861 	                                       y + image_y + map->y,
2862 					       brw->has_swizzling);
2863 	    untiled_s8_map[y * map->w + x] = tiled_s8_map[offset];
2864 	 }
2865       }
2866 
2867       intel_miptree_unmap_raw(mt);
2868 
2869       DBG("%s: %d,%d %dx%d from mt %p %d,%d = %p/%d\n", __func__,
2870 	  map->x, map->y, map->w, map->h,
2871 	  mt, map->x + image_x, map->y + image_y, map->ptr, map->stride);
2872    } else {
2873       DBG("%s: %d,%d %dx%d from mt %p = %p/%d\n", __func__,
2874 	  map->x, map->y, map->w, map->h,
2875 	  mt, map->ptr, map->stride);
2876    }
2877 
2878    map->unmap = intel_miptree_unmap_s8;
2879 }
2880 
2881 /**
2882  * Mapping functions for packed depth/stencil miptrees backed by real separate
2883  * miptrees for depth and stencil.
2884  *
2885  * On gen7, and to support HiZ pre-gen7, we have to have the stencil buffer
2886  * separate from the depth buffer.  Yet at the GL API level, we have to expose
2887  * packed depth/stencil textures and FBO attachments, and Mesa core expects to
2888  * be able to map that memory for texture storage and glReadPixels-type
2889  * operations.  We give Mesa core that access by mallocing a temporary and
2890  * copying the data between the actual backing store and the temporary.
2891  */
2892 static void
intel_miptree_unmap_depthstencil(struct brw_context * brw,struct intel_mipmap_tree * mt,struct intel_miptree_map * map,unsigned int level,unsigned int slice)2893 intel_miptree_unmap_depthstencil(struct brw_context *brw,
2894 				 struct intel_mipmap_tree *mt,
2895 				 struct intel_miptree_map *map,
2896 				 unsigned int level,
2897 				 unsigned int slice)
2898 {
2899    struct intel_mipmap_tree *z_mt = mt;
2900    struct intel_mipmap_tree *s_mt = mt->stencil_mt;
2901    bool map_z32f_x24s8 = mt->format == MESA_FORMAT_Z_FLOAT32;
2902 
2903    if (map->mode & GL_MAP_WRITE_BIT) {
2904       uint32_t *packed_map = map->ptr;
2905       uint8_t *s_map = intel_miptree_map_raw(brw, s_mt, GL_MAP_WRITE_BIT);
2906       uint32_t *z_map = intel_miptree_map_raw(brw, z_mt, GL_MAP_WRITE_BIT);
2907       unsigned int s_image_x, s_image_y;
2908       unsigned int z_image_x, z_image_y;
2909 
2910       intel_miptree_get_image_offset(s_mt, level, slice,
2911 				     &s_image_x, &s_image_y);
2912       intel_miptree_get_image_offset(z_mt, level, slice,
2913 				     &z_image_x, &z_image_y);
2914 
2915       for (uint32_t y = 0; y < map->h; y++) {
2916 	 for (uint32_t x = 0; x < map->w; x++) {
2917 	    ptrdiff_t s_offset = intel_offset_S8(s_mt->surf.row_pitch_B,
2918 						 x + s_image_x + map->x,
2919 						 y + s_image_y + map->y,
2920 						 brw->has_swizzling);
2921 	    ptrdiff_t z_offset = ((y + z_image_y + map->y) *
2922                                   (z_mt->surf.row_pitch_B / 4) +
2923 				  (x + z_image_x + map->x));
2924 
2925 	    if (map_z32f_x24s8) {
2926 	       z_map[z_offset] = packed_map[(y * map->w + x) * 2 + 0];
2927 	       s_map[s_offset] = packed_map[(y * map->w + x) * 2 + 1];
2928 	    } else {
2929 	       uint32_t packed = packed_map[y * map->w + x];
2930 	       s_map[s_offset] = packed >> 24;
2931 	       z_map[z_offset] = packed;
2932 	    }
2933 	 }
2934       }
2935 
2936       intel_miptree_unmap_raw(s_mt);
2937       intel_miptree_unmap_raw(z_mt);
2938 
2939       DBG("%s: %d,%d %dx%d from z mt %p (%s) %d,%d, s mt %p %d,%d = %p/%d\n",
2940 	  __func__,
2941 	  map->x, map->y, map->w, map->h,
2942 	  z_mt, _mesa_get_format_name(z_mt->format),
2943 	  map->x + z_image_x, map->y + z_image_y,
2944 	  s_mt, map->x + s_image_x, map->y + s_image_y,
2945 	  map->ptr, map->stride);
2946    }
2947 
2948    free(map->buffer);
2949 }
2950 
2951 static void
intel_miptree_map_depthstencil(struct brw_context * brw,struct intel_mipmap_tree * mt,struct intel_miptree_map * map,unsigned int level,unsigned int slice)2952 intel_miptree_map_depthstencil(struct brw_context *brw,
2953 			       struct intel_mipmap_tree *mt,
2954 			       struct intel_miptree_map *map,
2955 			       unsigned int level, unsigned int slice)
2956 {
2957    struct intel_mipmap_tree *z_mt = mt;
2958    struct intel_mipmap_tree *s_mt = mt->stencil_mt;
2959    bool map_z32f_x24s8 = mt->format == MESA_FORMAT_Z_FLOAT32;
2960    int packed_bpp = map_z32f_x24s8 ? 8 : 4;
2961 
2962    map->stride = map->w * packed_bpp;
2963    map->buffer = map->ptr = malloc(map->stride * map->h);
2964    if (!map->buffer)
2965       return;
2966 
2967    intel_miptree_access_raw(brw, z_mt, level, slice,
2968                             map->mode & GL_MAP_WRITE_BIT);
2969    intel_miptree_access_raw(brw, s_mt, level, slice,
2970                             map->mode & GL_MAP_WRITE_BIT);
2971 
2972    /* One of either READ_BIT or WRITE_BIT or both is set.  READ_BIT implies no
2973     * INVALIDATE_RANGE_BIT.  WRITE_BIT needs the original values read in unless
2974     * invalidate is set, since we'll be writing the whole rectangle from our
2975     * temporary buffer back out.
2976     */
2977    if (!(map->mode & GL_MAP_INVALIDATE_RANGE_BIT)) {
2978       uint32_t *packed_map = map->ptr;
2979       uint8_t *s_map = intel_miptree_map_raw(brw, s_mt, GL_MAP_READ_BIT);
2980       uint32_t *z_map = intel_miptree_map_raw(brw, z_mt, GL_MAP_READ_BIT);
2981       unsigned int s_image_x, s_image_y;
2982       unsigned int z_image_x, z_image_y;
2983 
2984       intel_miptree_get_image_offset(s_mt, level, slice,
2985 				     &s_image_x, &s_image_y);
2986       intel_miptree_get_image_offset(z_mt, level, slice,
2987 				     &z_image_x, &z_image_y);
2988 
2989       for (uint32_t y = 0; y < map->h; y++) {
2990 	 for (uint32_t x = 0; x < map->w; x++) {
2991 	    int map_x = map->x + x, map_y = map->y + y;
2992 	    ptrdiff_t s_offset = intel_offset_S8(s_mt->surf.row_pitch_B,
2993 						 map_x + s_image_x,
2994 						 map_y + s_image_y,
2995 						 brw->has_swizzling);
2996 	    ptrdiff_t z_offset = ((map_y + z_image_y) *
2997                                   (z_mt->surf.row_pitch_B / 4) +
2998 				  (map_x + z_image_x));
2999 	    uint8_t s = s_map[s_offset];
3000 	    uint32_t z = z_map[z_offset];
3001 
3002 	    if (map_z32f_x24s8) {
3003 	       packed_map[(y * map->w + x) * 2 + 0] = z;
3004 	       packed_map[(y * map->w + x) * 2 + 1] = s;
3005 	    } else {
3006 	       packed_map[y * map->w + x] = (s << 24) | (z & 0x00ffffff);
3007 	    }
3008 	 }
3009       }
3010 
3011       intel_miptree_unmap_raw(s_mt);
3012       intel_miptree_unmap_raw(z_mt);
3013 
3014       DBG("%s: %d,%d %dx%d from z mt %p %d,%d, s mt %p %d,%d = %p/%d\n",
3015 	  __func__,
3016 	  map->x, map->y, map->w, map->h,
3017 	  z_mt, map->x + z_image_x, map->y + z_image_y,
3018 	  s_mt, map->x + s_image_x, map->y + s_image_y,
3019 	  map->ptr, map->stride);
3020    } else {
3021       DBG("%s: %d,%d %dx%d from mt %p = %p/%d\n", __func__,
3022 	  map->x, map->y, map->w, map->h,
3023 	  mt, map->ptr, map->stride);
3024    }
3025 
3026    map->unmap = intel_miptree_unmap_depthstencil;
3027 }
3028 
3029 /**
3030  * Create and attach a map to the miptree at (level, slice). Return the
3031  * attached map.
3032  */
3033 static struct intel_miptree_map*
intel_miptree_attach_map(struct intel_mipmap_tree * mt,unsigned int level,unsigned int slice,unsigned int x,unsigned int y,unsigned int w,unsigned int h,GLbitfield mode)3034 intel_miptree_attach_map(struct intel_mipmap_tree *mt,
3035                          unsigned int level,
3036                          unsigned int slice,
3037                          unsigned int x,
3038                          unsigned int y,
3039                          unsigned int w,
3040                          unsigned int h,
3041                          GLbitfield mode)
3042 {
3043    struct intel_miptree_map *map = calloc(1, sizeof(*map));
3044 
3045    if (!map)
3046       return NULL;
3047 
3048    assert(mt->level[level].slice[slice].map == NULL);
3049    mt->level[level].slice[slice].map = map;
3050 
3051    map->mode = mode;
3052    map->x = x;
3053    map->y = y;
3054    map->w = w;
3055    map->h = h;
3056 
3057    return map;
3058 }
3059 
3060 /**
3061  * Release the map at (level, slice).
3062  */
3063 static void
intel_miptree_release_map(struct intel_mipmap_tree * mt,unsigned int level,unsigned int slice)3064 intel_miptree_release_map(struct intel_mipmap_tree *mt,
3065                          unsigned int level,
3066                          unsigned int slice)
3067 {
3068    struct intel_miptree_map **map;
3069 
3070    map = &mt->level[level].slice[slice].map;
3071    free(*map);
3072    *map = NULL;
3073 }
3074 
3075 static bool
can_blit_slice(struct intel_mipmap_tree * mt,const struct intel_miptree_map * map)3076 can_blit_slice(struct intel_mipmap_tree *mt,
3077                const struct intel_miptree_map *map)
3078 {
3079    /* See intel_miptree_blit() for details on the 32k pitch limit. */
3080    const unsigned src_blt_pitch = intel_miptree_blt_pitch(mt);
3081    const unsigned dst_blt_pitch = ALIGN(map->w * mt->cpp, 64);
3082    return src_blt_pitch < 32768 && dst_blt_pitch < 32768;
3083 }
3084 
3085 static bool
use_intel_mipree_map_blit(struct brw_context * brw,struct intel_mipmap_tree * mt,const struct intel_miptree_map * map)3086 use_intel_mipree_map_blit(struct brw_context *brw,
3087                           struct intel_mipmap_tree *mt,
3088                           const struct intel_miptree_map *map)
3089 {
3090    const struct gen_device_info *devinfo = &brw->screen->devinfo;
3091 
3092    if (devinfo->has_llc &&
3093       /* It's probably not worth swapping to the blit ring because of
3094        * all the overhead involved.
3095        */
3096        !(map->mode & GL_MAP_WRITE_BIT) &&
3097        !mt->compressed &&
3098        (mt->surf.tiling == ISL_TILING_X ||
3099         /* Prior to Sandybridge, the blitter can't handle Y tiling */
3100         (devinfo->gen >= 6 && mt->surf.tiling == ISL_TILING_Y0) ||
3101         /* Fast copy blit on skl+ supports all tiling formats. */
3102         devinfo->gen >= 9) &&
3103        can_blit_slice(mt, map))
3104       return true;
3105 
3106    if (mt->surf.tiling != ISL_TILING_LINEAR &&
3107        mt->bo->size >= brw->max_gtt_map_object_size) {
3108       assert(can_blit_slice(mt, map));
3109       return true;
3110    }
3111 
3112    return false;
3113 }
3114 
3115 /**
3116  * Parameter \a out_stride has type ptrdiff_t not because the buffer stride may
3117  * exceed 32 bits but to diminish the likelihood subtle bugs in pointer
3118  * arithmetic overflow.
3119  *
3120  * If you call this function and use \a out_stride, then you're doing pointer
3121  * arithmetic on \a out_ptr. The type of \a out_stride doesn't prevent all
3122  * bugs.  The caller must still take care to avoid 32-bit overflow errors in
3123  * all arithmetic expressions that contain buffer offsets and pixel sizes,
3124  * which usually have type uint32_t or GLuint.
3125  */
3126 void
intel_miptree_map(struct brw_context * brw,struct intel_mipmap_tree * mt,unsigned int level,unsigned int slice,unsigned int x,unsigned int y,unsigned int w,unsigned int h,GLbitfield mode,void ** out_ptr,ptrdiff_t * out_stride)3127 intel_miptree_map(struct brw_context *brw,
3128                   struct intel_mipmap_tree *mt,
3129                   unsigned int level,
3130                   unsigned int slice,
3131                   unsigned int x,
3132                   unsigned int y,
3133                   unsigned int w,
3134                   unsigned int h,
3135                   GLbitfield mode,
3136                   void **out_ptr,
3137                   ptrdiff_t *out_stride)
3138 {
3139    const struct gen_device_info *devinfo = &brw->screen->devinfo;
3140    struct intel_miptree_map *map;
3141 
3142    assert(mt->surf.samples == 1);
3143 
3144    map = intel_miptree_attach_map(mt, level, slice, x, y, w, h, mode);
3145    if (!map){
3146       *out_ptr = NULL;
3147       *out_stride = 0;
3148       return;
3149    }
3150 
3151    if (mt->format == MESA_FORMAT_S_UINT8) {
3152       intel_miptree_map_s8(brw, mt, map, level, slice);
3153    } else if (mt->stencil_mt && !(mode & BRW_MAP_DIRECT_BIT)) {
3154       intel_miptree_map_depthstencil(brw, mt, map, level, slice);
3155    } else if (use_intel_mipree_map_blit(brw, mt, map)) {
3156       intel_miptree_map_blit(brw, mt, map, level, slice);
3157    } else if (mt->surf.tiling != ISL_TILING_LINEAR && devinfo->gen > 4) {
3158       intel_miptree_map_tiled_memcpy(brw, mt, map, level, slice);
3159 #if defined(USE_SSE41)
3160    } else if (!(mode & GL_MAP_WRITE_BIT) &&
3161               !mt->compressed && cpu_has_sse4_1 &&
3162               (mt->surf.row_pitch_B % 16 == 0)) {
3163       intel_miptree_map_movntdqa(brw, mt, map, level, slice);
3164 #endif
3165    } else {
3166       if (mt->surf.tiling != ISL_TILING_LINEAR)
3167          perf_debug("intel_miptree_map: mapping via gtt");
3168       intel_miptree_map_map(brw, mt, map, level, slice);
3169    }
3170 
3171    *out_ptr = map->ptr;
3172    *out_stride = map->stride;
3173 
3174    if (map->ptr == NULL)
3175       intel_miptree_release_map(mt, level, slice);
3176 }
3177 
3178 void
intel_miptree_unmap(struct brw_context * brw,struct intel_mipmap_tree * mt,unsigned int level,unsigned int slice)3179 intel_miptree_unmap(struct brw_context *brw,
3180                     struct intel_mipmap_tree *mt,
3181                     unsigned int level,
3182                     unsigned int slice)
3183 {
3184    struct intel_miptree_map *map = mt->level[level].slice[slice].map;
3185 
3186    assert(mt->surf.samples == 1);
3187 
3188    if (!map)
3189       return;
3190 
3191    DBG("%s: mt %p (%s) level %d slice %d\n", __func__,
3192        mt, _mesa_get_format_name(mt->format), level, slice);
3193 
3194    if (map->unmap)
3195 	   map->unmap(brw, mt, map, level, slice);
3196 
3197    intel_miptree_release_map(mt, level, slice);
3198 }
3199 
3200 enum isl_surf_dim
get_isl_surf_dim(GLenum target)3201 get_isl_surf_dim(GLenum target)
3202 {
3203    switch (target) {
3204    case GL_TEXTURE_1D:
3205    case GL_TEXTURE_1D_ARRAY:
3206       return ISL_SURF_DIM_1D;
3207 
3208    case GL_TEXTURE_2D:
3209    case GL_TEXTURE_2D_ARRAY:
3210    case GL_TEXTURE_RECTANGLE:
3211    case GL_TEXTURE_CUBE_MAP:
3212    case GL_TEXTURE_CUBE_MAP_ARRAY:
3213    case GL_TEXTURE_2D_MULTISAMPLE:
3214    case GL_TEXTURE_2D_MULTISAMPLE_ARRAY:
3215    case GL_TEXTURE_EXTERNAL_OES:
3216       return ISL_SURF_DIM_2D;
3217 
3218    case GL_TEXTURE_3D:
3219       return ISL_SURF_DIM_3D;
3220    }
3221 
3222    unreachable("Invalid texture target");
3223 }
3224 
3225 enum isl_dim_layout
get_isl_dim_layout(const struct gen_device_info * devinfo,enum isl_tiling tiling,GLenum target)3226 get_isl_dim_layout(const struct gen_device_info *devinfo,
3227                    enum isl_tiling tiling, GLenum target)
3228 {
3229    switch (target) {
3230    case GL_TEXTURE_1D:
3231    case GL_TEXTURE_1D_ARRAY:
3232       return (devinfo->gen >= 9 && tiling == ISL_TILING_LINEAR ?
3233               ISL_DIM_LAYOUT_GEN9_1D : ISL_DIM_LAYOUT_GEN4_2D);
3234 
3235    case GL_TEXTURE_2D:
3236    case GL_TEXTURE_2D_ARRAY:
3237    case GL_TEXTURE_RECTANGLE:
3238    case GL_TEXTURE_2D_MULTISAMPLE:
3239    case GL_TEXTURE_2D_MULTISAMPLE_ARRAY:
3240    case GL_TEXTURE_EXTERNAL_OES:
3241       return ISL_DIM_LAYOUT_GEN4_2D;
3242 
3243    case GL_TEXTURE_CUBE_MAP:
3244    case GL_TEXTURE_CUBE_MAP_ARRAY:
3245       return (devinfo->gen == 4 ? ISL_DIM_LAYOUT_GEN4_3D :
3246               ISL_DIM_LAYOUT_GEN4_2D);
3247 
3248    case GL_TEXTURE_3D:
3249       return (devinfo->gen >= 9 ?
3250               ISL_DIM_LAYOUT_GEN4_2D : ISL_DIM_LAYOUT_GEN4_3D);
3251    }
3252 
3253    unreachable("Invalid texture target");
3254 }
3255 
3256 bool
intel_miptree_set_clear_color(struct brw_context * brw,struct intel_mipmap_tree * mt,union isl_color_value clear_color)3257 intel_miptree_set_clear_color(struct brw_context *brw,
3258                               struct intel_mipmap_tree *mt,
3259                               union isl_color_value clear_color)
3260 {
3261    if (memcmp(&mt->fast_clear_color, &clear_color, sizeof(clear_color)) != 0) {
3262       mt->fast_clear_color = clear_color;
3263       if (mt->aux_buf->clear_color_bo) {
3264          /* We can't update the clear color while the hardware is still using
3265           * the previous one for a resolve or sampling from it. Make sure that
3266           * there are no pending commands at this point.
3267           */
3268          brw_emit_pipe_control_flush(brw, PIPE_CONTROL_CS_STALL);
3269          for (int i = 0; i < 4; i++) {
3270             brw_store_data_imm32(brw, mt->aux_buf->clear_color_bo,
3271                                  mt->aux_buf->clear_color_offset + i * 4,
3272                                  mt->fast_clear_color.u32[i]);
3273          }
3274          brw_emit_pipe_control_flush(brw, PIPE_CONTROL_STATE_CACHE_INVALIDATE);
3275       }
3276       brw->ctx.NewDriverState |= BRW_NEW_AUX_STATE;
3277       return true;
3278    }
3279    return false;
3280 }
3281 
3282 union isl_color_value
intel_miptree_get_clear_color(const struct intel_mipmap_tree * mt,struct brw_bo ** clear_color_bo,uint64_t * clear_color_offset)3283 intel_miptree_get_clear_color(const struct intel_mipmap_tree *mt,
3284                               struct brw_bo **clear_color_bo,
3285                               uint64_t *clear_color_offset)
3286 {
3287    assert(mt->aux_buf);
3288 
3289    *clear_color_bo = mt->aux_buf->clear_color_bo;
3290    *clear_color_offset = mt->aux_buf->clear_color_offset;
3291    return mt->fast_clear_color;
3292 }
3293 
3294 static void
intel_miptree_update_etc_shadow(struct brw_context * brw,struct intel_mipmap_tree * mt,unsigned int level,unsigned int slice,int level_w,int level_h)3295 intel_miptree_update_etc_shadow(struct brw_context *brw,
3296                                 struct intel_mipmap_tree *mt,
3297                                 unsigned int level,
3298                                 unsigned int slice,
3299                                 int level_w,
3300                                 int level_h)
3301 {
3302    ptrdiff_t etc_stride, shadow_stride;
3303    void *mptr, *sptr;
3304    struct intel_mipmap_tree *smt = mt->shadow_mt;
3305 
3306    assert(intel_miptree_has_etc_shadow(brw, mt));
3307 
3308    intel_miptree_map(brw, mt, level, slice, 0, 0, level_w, level_h,
3309                      GL_MAP_READ_BIT, &mptr, &etc_stride);
3310    intel_miptree_map(brw, smt, level, slice, 0, 0, level_w, level_h,
3311                      GL_MAP_WRITE_BIT | GL_MAP_INVALIDATE_RANGE_BIT,
3312                      &sptr, &shadow_stride);
3313 
3314    if (mt->format == MESA_FORMAT_ETC1_RGB8) {
3315       _mesa_etc1_unpack_rgba8888(sptr, shadow_stride, mptr, etc_stride,
3316                                  level_w, level_h);
3317    } else {
3318       /* destination and source images must have the same swizzle */
3319       bool is_bgra = (smt->format == MESA_FORMAT_B8G8R8A8_SRGB);
3320       _mesa_unpack_etc2_format(sptr, shadow_stride, mptr, etc_stride,
3321                                level_w, level_h, mt->format, is_bgra);
3322    }
3323 
3324    intel_miptree_unmap(brw, mt, level, slice);
3325    intel_miptree_unmap(brw, smt, level, slice);
3326 }
3327 
3328 void
intel_miptree_update_etc_shadow_levels(struct brw_context * brw,struct intel_mipmap_tree * mt)3329 intel_miptree_update_etc_shadow_levels(struct brw_context *brw,
3330                                        struct intel_mipmap_tree *mt)
3331 {
3332    struct intel_mipmap_tree *smt;
3333    int num_slices;
3334 
3335    assert(mt);
3336    assert(mt->surf.size_B > 0);
3337    assert(intel_miptree_has_etc_shadow(brw, mt));
3338 
3339    smt = mt->shadow_mt;
3340    num_slices = smt->surf.logical_level0_px.array_len;
3341 
3342    for (int level = smt->first_level; level <= smt->last_level; level++) {
3343       int level_w = minify(smt->surf.logical_level0_px.width,
3344                            level - smt->first_level);
3345       int level_h = minify(smt->surf.logical_level0_px.height,
3346                            level - smt->first_level);
3347 
3348       for (unsigned int slice = 0; slice < num_slices; slice++) {
3349          intel_miptree_update_etc_shadow(brw, mt, level, slice, level_w,
3350                                          level_h);
3351       }
3352    }
3353 
3354    mt->shadow_needs_update = false;
3355 }
3356