1 /*
2 * Copyright © 2020 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "nir_builder.h"
25
26 #include <string.h>
27
28 /** Returns the type to use for a copy of the given size.
29 *
30 * The actual type doesn't matter here all that much as we're just going to do
31 * a load/store on it and never any arithmetic.
32 */
33 static const struct glsl_type *
copy_type_for_byte_size(unsigned size)34 copy_type_for_byte_size(unsigned size)
35 {
36 switch (size) {
37 case 1: return glsl_vector_type(GLSL_TYPE_UINT8, 1);
38 case 2: return glsl_vector_type(GLSL_TYPE_UINT16, 1);
39 case 4: return glsl_vector_type(GLSL_TYPE_UINT, 1);
40 case 8: return glsl_vector_type(GLSL_TYPE_UINT, 2);
41 case 16: return glsl_vector_type(GLSL_TYPE_UINT, 4);
42 default:
43 unreachable("Unsupported size");
44 }
45 }
46
47 static nir_ssa_def *
memcpy_load_deref_elem(nir_builder * b,nir_deref_instr * parent,nir_ssa_def * index)48 memcpy_load_deref_elem(nir_builder *b, nir_deref_instr *parent,
49 nir_ssa_def *index)
50 {
51 nir_deref_instr *deref;
52
53 index = nir_i2i(b, index, nir_dest_bit_size(parent->dest));
54 assert(parent->deref_type == nir_deref_type_cast);
55 deref = nir_build_deref_ptr_as_array(b, parent, index);
56
57 return nir_load_deref(b, deref);
58 }
59
60 static nir_ssa_def *
memcpy_load_deref_elem_imm(nir_builder * b,nir_deref_instr * parent,uint64_t index)61 memcpy_load_deref_elem_imm(nir_builder *b, nir_deref_instr *parent,
62 uint64_t index)
63 {
64 nir_ssa_def *idx = nir_imm_intN_t(b, index, parent->dest.ssa.bit_size);
65 return memcpy_load_deref_elem(b, parent, idx);
66 }
67
68 static void
memcpy_store_deref_elem(nir_builder * b,nir_deref_instr * parent,nir_ssa_def * index,nir_ssa_def * value)69 memcpy_store_deref_elem(nir_builder *b, nir_deref_instr *parent,
70 nir_ssa_def *index, nir_ssa_def *value)
71 {
72 nir_deref_instr *deref;
73
74 index = nir_i2i(b, index, nir_dest_bit_size(parent->dest));
75 assert(parent->deref_type == nir_deref_type_cast);
76 deref = nir_build_deref_ptr_as_array(b, parent, index);
77 nir_store_deref(b, deref, value, ~0);
78 }
79
80 static void
memcpy_store_deref_elem_imm(nir_builder * b,nir_deref_instr * parent,uint64_t index,nir_ssa_def * value)81 memcpy_store_deref_elem_imm(nir_builder *b, nir_deref_instr *parent,
82 uint64_t index, nir_ssa_def *value)
83 {
84 nir_ssa_def *idx = nir_imm_intN_t(b, index, parent->dest.ssa.bit_size);
85 memcpy_store_deref_elem(b, parent, idx, value);
86 }
87
88 static bool
lower_memcpy_impl(nir_function_impl * impl)89 lower_memcpy_impl(nir_function_impl *impl)
90 {
91 nir_builder b;
92 nir_builder_init(&b, impl);
93
94 bool found_const_memcpy = false, found_non_const_memcpy = false;
95
96 nir_foreach_block_safe(block, impl) {
97 nir_foreach_instr_safe(instr, block) {
98 if (instr->type != nir_instr_type_intrinsic)
99 continue;
100
101 nir_intrinsic_instr *cpy = nir_instr_as_intrinsic(instr);
102 if (cpy->intrinsic != nir_intrinsic_memcpy_deref)
103 continue;
104
105 b.cursor = nir_instr_remove(&cpy->instr);
106
107 nir_deref_instr *dst = nir_src_as_deref(cpy->src[0]);
108 nir_deref_instr *src = nir_src_as_deref(cpy->src[1]);
109 if (nir_src_is_const(cpy->src[2])) {
110 found_const_memcpy = true;
111 uint64_t size = nir_src_as_uint(cpy->src[2]);
112 uint64_t offset = 0;
113 while (offset < size) {
114 uint64_t remaining = offset - size;
115 /* For our chunk size, we choose the largest power-of-two that
116 * divides size with a maximum of 16B (a vec4).
117 */
118 unsigned copy_size = 1u << MIN2(ffsll(remaining) - 1, 4);
119 const struct glsl_type *copy_type =
120 copy_type_for_byte_size(copy_size);
121
122 nir_deref_instr *copy_dst =
123 nir_build_deref_cast(&b, &dst->dest.ssa, dst->modes,
124 copy_type, copy_size);
125 nir_deref_instr *copy_src =
126 nir_build_deref_cast(&b, &src->dest.ssa, src->modes,
127 copy_type, copy_size);
128
129 uint64_t index = offset / copy_size;
130 nir_ssa_def *value =
131 memcpy_load_deref_elem_imm(&b, copy_src, index);
132 memcpy_store_deref_elem_imm(&b, copy_dst, index, value);
133 offset += copy_size;
134 }
135 } else {
136 found_non_const_memcpy = true;
137 assert(cpy->src[2].is_ssa);
138 nir_ssa_def *size = cpy->src[2].ssa;
139
140 /* In this case, we don't have any idea what the size is so we
141 * emit a loop which copies one byte at a time.
142 */
143 nir_deref_instr *copy_dst =
144 nir_build_deref_cast(&b, &dst->dest.ssa, dst->modes,
145 glsl_uint8_t_type(), 1);
146 nir_deref_instr *copy_src =
147 nir_build_deref_cast(&b, &src->dest.ssa, src->modes,
148 glsl_uint8_t_type(), 1);
149
150 nir_variable *i = nir_local_variable_create(impl,
151 glsl_uintN_t_type(size->bit_size), NULL);
152 nir_store_var(&b, i, nir_imm_intN_t(&b, 0, size->bit_size), ~0);
153 nir_push_loop(&b);
154 {
155 nir_ssa_def *index = nir_load_var(&b, i);
156 nir_push_if(&b, nir_uge(&b, index, size));
157 {
158 nir_jump(&b, nir_jump_break);
159 }
160 nir_pop_if(&b, NULL);
161
162 nir_ssa_def *value =
163 memcpy_load_deref_elem(&b, copy_src, index);
164 memcpy_store_deref_elem(&b, copy_dst, index, value);
165 nir_store_var(&b, i, nir_iadd_imm(&b, index, 1), ~0);
166 }
167 nir_pop_loop(&b, NULL);
168 }
169 }
170 }
171
172 if (found_non_const_memcpy) {
173 nir_metadata_preserve(impl, nir_metadata_none);
174 } else if (found_const_memcpy) {
175 nir_metadata_preserve(impl, nir_metadata_block_index |
176 nir_metadata_dominance);
177 } else {
178 nir_metadata_preserve(impl, nir_metadata_all);
179 }
180
181 return found_const_memcpy || found_non_const_memcpy;
182 }
183
184 bool
nir_lower_memcpy(nir_shader * shader)185 nir_lower_memcpy(nir_shader *shader)
186 {
187 bool progress = false;
188
189 nir_foreach_function(function, shader) {
190 if (function->impl && lower_memcpy_impl(function->impl))
191 progress = true;
192 }
193
194 return progress;
195 }
196