1
2 #include "util/u_format.h"
3
4 #include "nvc0_context.h"
5
6 #include "nv50/nv50_defs.xml.h"
7
8 struct nvc0_transfer {
9 struct pipe_transfer base;
10 struct nv50_m2mf_rect rect[2];
11 uint32_t nblocksx;
12 uint16_t nblocksy;
13 uint16_t nlayers;
14 };
15
16 static void
nvc0_m2mf_transfer_rect(struct nvc0_context * nvc0,const struct nv50_m2mf_rect * dst,const struct nv50_m2mf_rect * src,uint32_t nblocksx,uint32_t nblocksy)17 nvc0_m2mf_transfer_rect(struct nvc0_context *nvc0,
18 const struct nv50_m2mf_rect *dst,
19 const struct nv50_m2mf_rect *src,
20 uint32_t nblocksx, uint32_t nblocksy)
21 {
22 struct nouveau_pushbuf *push = nvc0->base.pushbuf;
23 struct nouveau_bufctx *bctx = nvc0->bufctx;
24 const int cpp = dst->cpp;
25 uint32_t src_ofst = src->base;
26 uint32_t dst_ofst = dst->base;
27 uint32_t height = nblocksy;
28 uint32_t sy = src->y;
29 uint32_t dy = dst->y;
30 uint32_t exec = (1 << 20);
31
32 assert(dst->cpp == src->cpp);
33
34 nouveau_bufctx_refn(bctx, 0, src->bo, src->domain | NOUVEAU_BO_RD);
35 nouveau_bufctx_refn(bctx, 0, dst->bo, dst->domain | NOUVEAU_BO_WR);
36 nouveau_pushbuf_bufctx(push, bctx);
37 nouveau_pushbuf_validate(push);
38
39 if (nouveau_bo_memtype(src->bo)) {
40 BEGIN_NVC0(push, NVC0_M2MF(TILING_MODE_IN), 5);
41 PUSH_DATA (push, src->tile_mode);
42 PUSH_DATA (push, src->width * cpp);
43 PUSH_DATA (push, src->height);
44 PUSH_DATA (push, src->depth);
45 PUSH_DATA (push, src->z);
46 } else {
47 src_ofst += src->y * src->pitch + src->x * cpp;
48
49 BEGIN_NVC0(push, NVC0_M2MF(PITCH_IN), 1);
50 PUSH_DATA (push, src->width * cpp);
51
52 exec |= NVC0_M2MF_EXEC_LINEAR_IN;
53 }
54
55 if (nouveau_bo_memtype(dst->bo)) {
56 BEGIN_NVC0(push, NVC0_M2MF(TILING_MODE_OUT), 5);
57 PUSH_DATA (push, dst->tile_mode);
58 PUSH_DATA (push, dst->width * cpp);
59 PUSH_DATA (push, dst->height);
60 PUSH_DATA (push, dst->depth);
61 PUSH_DATA (push, dst->z);
62 } else {
63 dst_ofst += dst->y * dst->pitch + dst->x * cpp;
64
65 BEGIN_NVC0(push, NVC0_M2MF(PITCH_OUT), 1);
66 PUSH_DATA (push, dst->width * cpp);
67
68 exec |= NVC0_M2MF_EXEC_LINEAR_OUT;
69 }
70
71 while (height) {
72 int line_count = height > 2047 ? 2047 : height;
73
74 BEGIN_NVC0(push, NVC0_M2MF(OFFSET_IN_HIGH), 2);
75 PUSH_DATAh(push, src->bo->offset + src_ofst);
76 PUSH_DATA (push, src->bo->offset + src_ofst);
77
78 BEGIN_NVC0(push, NVC0_M2MF(OFFSET_OUT_HIGH), 2);
79 PUSH_DATAh(push, dst->bo->offset + dst_ofst);
80 PUSH_DATA (push, dst->bo->offset + dst_ofst);
81
82 if (!(exec & NVC0_M2MF_EXEC_LINEAR_IN)) {
83 BEGIN_NVC0(push, NVC0_M2MF(TILING_POSITION_IN_X), 2);
84 PUSH_DATA (push, src->x * cpp);
85 PUSH_DATA (push, sy);
86 } else {
87 src_ofst += line_count * src->pitch;
88 }
89 if (!(exec & NVC0_M2MF_EXEC_LINEAR_OUT)) {
90 BEGIN_NVC0(push, NVC0_M2MF(TILING_POSITION_OUT_X), 2);
91 PUSH_DATA (push, dst->x * cpp);
92 PUSH_DATA (push, dy);
93 } else {
94 dst_ofst += line_count * dst->pitch;
95 }
96
97 BEGIN_NVC0(push, NVC0_M2MF(LINE_LENGTH_IN), 2);
98 PUSH_DATA (push, nblocksx * cpp);
99 PUSH_DATA (push, line_count);
100 BEGIN_NVC0(push, NVC0_M2MF(EXEC), 1);
101 PUSH_DATA (push, exec);
102
103 height -= line_count;
104 sy += line_count;
105 dy += line_count;
106 }
107
108 nouveau_bufctx_reset(bctx, 0);
109 }
110
111 static void
nve4_m2mf_transfer_rect(struct nvc0_context * nvc0,const struct nv50_m2mf_rect * dst,const struct nv50_m2mf_rect * src,uint32_t nblocksx,uint32_t nblocksy)112 nve4_m2mf_transfer_rect(struct nvc0_context *nvc0,
113 const struct nv50_m2mf_rect *dst,
114 const struct nv50_m2mf_rect *src,
115 uint32_t nblocksx, uint32_t nblocksy)
116 {
117 struct nouveau_pushbuf *push = nvc0->base.pushbuf;
118 struct nouveau_bufctx *bctx = nvc0->bufctx;
119 uint32_t exec;
120 uint32_t src_base = src->base;
121 uint32_t dst_base = dst->base;
122 const int cpp = dst->cpp;
123
124 assert(dst->cpp == src->cpp);
125
126 nouveau_bufctx_refn(bctx, 0, dst->bo, dst->domain | NOUVEAU_BO_WR);
127 nouveau_bufctx_refn(bctx, 0, src->bo, src->domain | NOUVEAU_BO_RD);
128 nouveau_pushbuf_bufctx(push, bctx);
129 nouveau_pushbuf_validate(push);
130
131 exec = 0x200 /* 2D_ENABLE */ | 0x6 /* UNK */;
132
133 if (!nouveau_bo_memtype(dst->bo)) {
134 assert(!dst->z);
135 dst_base += dst->y * dst->pitch + dst->x * cpp;
136 exec |= 0x100; /* DST_MODE_2D_LINEAR */
137 }
138 if (!nouveau_bo_memtype(src->bo)) {
139 assert(!src->z);
140 src_base += src->y * src->pitch + src->x * cpp;
141 exec |= 0x080; /* SRC_MODE_2D_LINEAR */
142 }
143
144 BEGIN_NVC0(push, SUBC_COPY(0x070c), 6);
145 PUSH_DATA (push, 0x1000 | dst->tile_mode);
146 PUSH_DATA (push, dst->pitch);
147 PUSH_DATA (push, dst->height);
148 PUSH_DATA (push, dst->depth);
149 PUSH_DATA (push, dst->z);
150 PUSH_DATA (push, (dst->y << 16) | (dst->x * cpp));
151
152 BEGIN_NVC0(push, SUBC_COPY(0x0728), 6);
153 PUSH_DATA (push, 0x1000 | src->tile_mode);
154 PUSH_DATA (push, src->pitch);
155 PUSH_DATA (push, src->height);
156 PUSH_DATA (push, src->depth);
157 PUSH_DATA (push, src->z);
158 PUSH_DATA (push, (src->y << 16) | (src->x * cpp));
159
160 BEGIN_NVC0(push, SUBC_COPY(0x0400), 8);
161 PUSH_DATAh(push, src->bo->offset + src_base);
162 PUSH_DATA (push, src->bo->offset + src_base);
163 PUSH_DATAh(push, dst->bo->offset + dst_base);
164 PUSH_DATA (push, dst->bo->offset + dst_base);
165 PUSH_DATA (push, src->pitch);
166 PUSH_DATA (push, dst->pitch);
167 PUSH_DATA (push, nblocksx * cpp);
168 PUSH_DATA (push, nblocksy);
169
170 BEGIN_NVC0(push, SUBC_COPY(0x0300), 1);
171 PUSH_DATA (push, exec);
172
173 nouveau_bufctx_reset(bctx, 0);
174 }
175
176 void
nvc0_m2mf_push_linear(struct nouveau_context * nv,struct nouveau_bo * dst,unsigned offset,unsigned domain,unsigned size,const void * data)177 nvc0_m2mf_push_linear(struct nouveau_context *nv,
178 struct nouveau_bo *dst, unsigned offset, unsigned domain,
179 unsigned size, const void *data)
180 {
181 struct nvc0_context *nvc0 = nvc0_context(&nv->pipe);
182 struct nouveau_pushbuf *push = nv->pushbuf;
183 uint32_t *src = (uint32_t *)data;
184 unsigned count = (size + 3) / 4;
185
186 nouveau_bufctx_refn(nvc0->bufctx, 0, dst, domain | NOUVEAU_BO_WR);
187 nouveau_pushbuf_bufctx(push, nvc0->bufctx);
188 nouveau_pushbuf_validate(push);
189
190 while (count) {
191 unsigned nr;
192
193 if (!PUSH_SPACE(push, 16))
194 break;
195 nr = PUSH_AVAIL(push);
196 assert(nr >= 16);
197 nr = MIN2(count, nr - 9);
198 nr = MIN2(nr, NV04_PFIFO_MAX_PACKET_LEN);
199
200 BEGIN_NVC0(push, NVC0_M2MF(OFFSET_OUT_HIGH), 2);
201 PUSH_DATAh(push, dst->offset + offset);
202 PUSH_DATA (push, dst->offset + offset);
203 BEGIN_NVC0(push, NVC0_M2MF(LINE_LENGTH_IN), 2);
204 PUSH_DATA (push, nr * 4);
205 PUSH_DATA (push, 1);
206 BEGIN_NVC0(push, NVC0_M2MF(EXEC), 1);
207 PUSH_DATA (push, 0x100111);
208
209 /* must not be interrupted (trap on QUERY fence, 0x50 works however) */
210 BEGIN_NIC0(push, NVC0_M2MF(DATA), nr);
211 PUSH_DATAp(push, src, nr);
212
213 count -= nr;
214 src += nr;
215 offset += nr * 4;
216 }
217
218 nouveau_bufctx_reset(nvc0->bufctx, 0);
219 }
220
221 void
nve4_p2mf_push_linear(struct nouveau_context * nv,struct nouveau_bo * dst,unsigned offset,unsigned domain,unsigned size,const void * data)222 nve4_p2mf_push_linear(struct nouveau_context *nv,
223 struct nouveau_bo *dst, unsigned offset, unsigned domain,
224 unsigned size, const void *data)
225 {
226 struct nvc0_context *nvc0 = nvc0_context(&nv->pipe);
227 struct nouveau_pushbuf *push = nv->pushbuf;
228 uint32_t *src = (uint32_t *)data;
229 unsigned count = (size + 3) / 4;
230
231 nouveau_bufctx_refn(nvc0->bufctx, 0, dst, domain | NOUVEAU_BO_WR);
232 nouveau_pushbuf_bufctx(push, nvc0->bufctx);
233 nouveau_pushbuf_validate(push);
234
235 while (count) {
236 unsigned nr;
237
238 if (!PUSH_SPACE(push, 16))
239 break;
240 nr = PUSH_AVAIL(push);
241 assert(nr >= 16);
242 nr = MIN2(count, nr - 8);
243 nr = MIN2(nr, (NV04_PFIFO_MAX_PACKET_LEN - 1));
244
245 BEGIN_NVC0(push, NVE4_P2MF(DST_ADDRESS_HIGH), 2);
246 PUSH_DATAh(push, dst->offset + offset);
247 PUSH_DATA (push, dst->offset + offset);
248 BEGIN_NVC0(push, NVE4_P2MF(LINE_LENGTH_IN), 2);
249 PUSH_DATA (push, nr * 4);
250 PUSH_DATA (push, 1);
251 /* must not be interrupted (trap on QUERY fence, 0x50 works however) */
252 BEGIN_1IC0(push, NVE4_P2MF(EXEC), nr + 1);
253 PUSH_DATA (push, 0x1001);
254 PUSH_DATAp(push, src, nr);
255
256 count -= nr;
257 src += nr;
258 offset += nr * 4;
259 }
260
261 nouveau_bufctx_reset(nvc0->bufctx, 0);
262 }
263
264 static void
nvc0_m2mf_copy_linear(struct nouveau_context * nv,struct nouveau_bo * dst,unsigned dstoff,unsigned dstdom,struct nouveau_bo * src,unsigned srcoff,unsigned srcdom,unsigned size)265 nvc0_m2mf_copy_linear(struct nouveau_context *nv,
266 struct nouveau_bo *dst, unsigned dstoff, unsigned dstdom,
267 struct nouveau_bo *src, unsigned srcoff, unsigned srcdom,
268 unsigned size)
269 {
270 struct nouveau_pushbuf *push = nv->pushbuf;
271 struct nouveau_bufctx *bctx = nvc0_context(&nv->pipe)->bufctx;
272
273 nouveau_bufctx_refn(bctx, 0, src, srcdom | NOUVEAU_BO_RD);
274 nouveau_bufctx_refn(bctx, 0, dst, dstdom | NOUVEAU_BO_WR);
275 nouveau_pushbuf_bufctx(push, bctx);
276 nouveau_pushbuf_validate(push);
277
278 while (size) {
279 unsigned bytes = MIN2(size, 1 << 17);
280
281 BEGIN_NVC0(push, NVC0_M2MF(OFFSET_OUT_HIGH), 2);
282 PUSH_DATAh(push, dst->offset + dstoff);
283 PUSH_DATA (push, dst->offset + dstoff);
284 BEGIN_NVC0(push, NVC0_M2MF(OFFSET_IN_HIGH), 2);
285 PUSH_DATAh(push, src->offset + srcoff);
286 PUSH_DATA (push, src->offset + srcoff);
287 BEGIN_NVC0(push, NVC0_M2MF(LINE_LENGTH_IN), 2);
288 PUSH_DATA (push, bytes);
289 PUSH_DATA (push, 1);
290 BEGIN_NVC0(push, NVC0_M2MF(EXEC), 1);
291 PUSH_DATA (push, (1 << NVC0_M2MF_EXEC_INC__SHIFT) |
292 NVC0_M2MF_EXEC_LINEAR_IN | NVC0_M2MF_EXEC_LINEAR_OUT);
293
294 srcoff += bytes;
295 dstoff += bytes;
296 size -= bytes;
297 }
298
299 nouveau_bufctx_reset(bctx, 0);
300 }
301
302 static void
nve4_m2mf_copy_linear(struct nouveau_context * nv,struct nouveau_bo * dst,unsigned dstoff,unsigned dstdom,struct nouveau_bo * src,unsigned srcoff,unsigned srcdom,unsigned size)303 nve4_m2mf_copy_linear(struct nouveau_context *nv,
304 struct nouveau_bo *dst, unsigned dstoff, unsigned dstdom,
305 struct nouveau_bo *src, unsigned srcoff, unsigned srcdom,
306 unsigned size)
307 {
308 struct nouveau_pushbuf *push = nv->pushbuf;
309 struct nouveau_bufctx *bctx = nvc0_context(&nv->pipe)->bufctx;
310
311 nouveau_bufctx_refn(bctx, 0, src, srcdom | NOUVEAU_BO_RD);
312 nouveau_bufctx_refn(bctx, 0, dst, dstdom | NOUVEAU_BO_WR);
313 nouveau_pushbuf_bufctx(push, bctx);
314 nouveau_pushbuf_validate(push);
315
316 BEGIN_NVC0(push, SUBC_COPY(0x0400), 4);
317 PUSH_DATAh(push, src->offset + srcoff);
318 PUSH_DATA (push, src->offset + srcoff);
319 PUSH_DATAh(push, dst->offset + dstoff);
320 PUSH_DATA (push, dst->offset + dstoff);
321 BEGIN_NVC0(push, SUBC_COPY(0x0418), 1);
322 PUSH_DATA (push, size);
323 BEGIN_NVC0(push, SUBC_COPY(0x0300), 1);
324 PUSH_DATA (push, 0x186);
325
326 nouveau_bufctx_reset(bctx, 0);
327 }
328
329 struct pipe_transfer *
nvc0_miptree_transfer_new(struct pipe_context * pctx,struct pipe_resource * res,unsigned level,unsigned usage,const struct pipe_box * box)330 nvc0_miptree_transfer_new(struct pipe_context *pctx,
331 struct pipe_resource *res,
332 unsigned level,
333 unsigned usage,
334 const struct pipe_box *box)
335 {
336 struct nvc0_context *nvc0 = nvc0_context(pctx);
337 struct nouveau_device *dev = nvc0->screen->base.device;
338 struct nv50_miptree *mt = nv50_miptree(res);
339 struct nvc0_transfer *tx;
340 uint32_t size;
341 int ret;
342
343 if (usage & PIPE_TRANSFER_MAP_DIRECTLY)
344 return NULL;
345
346 tx = CALLOC_STRUCT(nvc0_transfer);
347 if (!tx)
348 return NULL;
349
350 pipe_resource_reference(&tx->base.resource, res);
351
352 tx->base.level = level;
353 tx->base.usage = usage;
354 tx->base.box = *box;
355
356 if (util_format_is_plain(res->format)) {
357 tx->nblocksx = box->width << mt->ms_x;
358 tx->nblocksy = box->height << mt->ms_y;
359 } else {
360 tx->nblocksx = util_format_get_nblocksx(res->format, box->width);
361 tx->nblocksy = util_format_get_nblocksy(res->format, box->height);
362 }
363 tx->nlayers = box->depth;
364
365 tx->base.stride = tx->nblocksx * util_format_get_blocksize(res->format);
366 tx->base.layer_stride = tx->nblocksy * tx->base.stride;
367
368 nv50_m2mf_rect_setup(&tx->rect[0], res, level, box->x, box->y, box->z);
369
370 size = tx->base.layer_stride;
371
372 ret = nouveau_bo_new(dev, NOUVEAU_BO_GART | NOUVEAU_BO_MAP, 0,
373 size * tx->nlayers, NULL, &tx->rect[1].bo);
374 if (ret) {
375 FREE(tx);
376 return NULL;
377 }
378
379 tx->rect[1].cpp = tx->rect[0].cpp;
380 tx->rect[1].width = tx->nblocksx;
381 tx->rect[1].height = tx->nblocksy;
382 tx->rect[1].depth = 1;
383 tx->rect[1].pitch = tx->base.stride;
384 tx->rect[1].domain = NOUVEAU_BO_GART;
385
386 if (usage & PIPE_TRANSFER_READ) {
387 unsigned base = tx->rect[0].base;
388 unsigned z = tx->rect[0].z;
389 unsigned i;
390 for (i = 0; i < tx->nlayers; ++i) {
391 nvc0->m2mf_copy_rect(nvc0, &tx->rect[1], &tx->rect[0],
392 tx->nblocksx, tx->nblocksy);
393 if (mt->layout_3d)
394 tx->rect[0].z++;
395 else
396 tx->rect[0].base += mt->layer_stride;
397 tx->rect[1].base += size;
398 }
399 tx->rect[0].z = z;
400 tx->rect[0].base = base;
401 tx->rect[1].base = 0;
402 }
403
404 return &tx->base;
405 }
406
407 void
nvc0_miptree_transfer_del(struct pipe_context * pctx,struct pipe_transfer * transfer)408 nvc0_miptree_transfer_del(struct pipe_context *pctx,
409 struct pipe_transfer *transfer)
410 {
411 struct nvc0_context *nvc0 = nvc0_context(pctx);
412 struct nvc0_transfer *tx = (struct nvc0_transfer *)transfer;
413 struct nv50_miptree *mt = nv50_miptree(tx->base.resource);
414 unsigned i;
415
416 if (tx->base.usage & PIPE_TRANSFER_WRITE) {
417 for (i = 0; i < tx->nlayers; ++i) {
418 nvc0->m2mf_copy_rect(nvc0, &tx->rect[0], &tx->rect[1],
419 tx->nblocksx, tx->nblocksy);
420 if (mt->layout_3d)
421 tx->rect[0].z++;
422 else
423 tx->rect[0].base += mt->layer_stride;
424 tx->rect[1].base += tx->nblocksy * tx->base.stride;
425 }
426 }
427
428 nouveau_bo_ref(NULL, &tx->rect[1].bo);
429 pipe_resource_reference(&transfer->resource, NULL);
430
431 FREE(tx);
432 }
433
434 void *
nvc0_miptree_transfer_map(struct pipe_context * pctx,struct pipe_transfer * transfer)435 nvc0_miptree_transfer_map(struct pipe_context *pctx,
436 struct pipe_transfer *transfer)
437 {
438 struct nvc0_context *nvc0 = nvc0_context(pctx);
439 struct nvc0_transfer *tx = (struct nvc0_transfer *)transfer;
440 int ret;
441 unsigned flags = 0;
442
443 if (tx->rect[1].bo->map)
444 return tx->rect[1].bo->map;
445
446 if (transfer->usage & PIPE_TRANSFER_READ)
447 flags = NOUVEAU_BO_RD;
448 if (transfer->usage & PIPE_TRANSFER_WRITE)
449 flags |= NOUVEAU_BO_WR;
450
451 ret = nouveau_bo_map(tx->rect[1].bo, flags, nvc0->screen->base.client);
452 if (ret)
453 return NULL;
454 return tx->rect[1].bo->map;
455 }
456
457 void
nvc0_miptree_transfer_unmap(struct pipe_context * pctx,struct pipe_transfer * transfer)458 nvc0_miptree_transfer_unmap(struct pipe_context *pctx,
459 struct pipe_transfer *transfer)
460 {
461 }
462
463 void
nvc0_cb_push(struct nouveau_context * nv,struct nouveau_bo * bo,unsigned domain,unsigned base,unsigned size,unsigned offset,unsigned words,const uint32_t * data)464 nvc0_cb_push(struct nouveau_context *nv,
465 struct nouveau_bo *bo, unsigned domain,
466 unsigned base, unsigned size,
467 unsigned offset, unsigned words, const uint32_t *data)
468 {
469 struct nouveau_bufctx *bctx = nvc0_context(&nv->pipe)->bufctx;
470 struct nouveau_pushbuf *push = nv->pushbuf;
471
472 assert(!(offset & 3));
473 size = align(size, 0x100);
474
475 nouveau_bufctx_refn(bctx, 0, bo, NOUVEAU_BO_WR | domain);
476 nouveau_pushbuf_bufctx(push, bctx);
477 nouveau_pushbuf_validate(push);
478
479 BEGIN_NVC0(push, NVC0_3D(CB_SIZE), 3);
480 PUSH_DATA (push, size);
481 PUSH_DATAh(push, bo->offset + base);
482 PUSH_DATA (push, bo->offset + base);
483
484 while (words) {
485 unsigned nr = PUSH_AVAIL(push);
486 nr = MIN2(nr, words);
487 nr = MIN2(nr, NV04_PFIFO_MAX_PACKET_LEN - 1);
488
489 BEGIN_1IC0(push, NVC0_3D(CB_POS), nr + 1);
490 PUSH_DATA (push, offset);
491 PUSH_DATAp(push, data, nr);
492
493 words -= nr;
494 data += nr;
495 offset += nr * 4;
496 }
497
498 nouveau_bufctx_reset(bctx, 0);
499 }
500
501 void
nvc0_init_transfer_functions(struct nvc0_context * nvc0)502 nvc0_init_transfer_functions(struct nvc0_context *nvc0)
503 {
504 if (nvc0->screen->base.class_3d >= NVE4_3D_CLASS) {
505 nvc0->m2mf_copy_rect = nve4_m2mf_transfer_rect;
506 nvc0->base.copy_data = nve4_m2mf_copy_linear;
507 nvc0->base.push_data = nve4_p2mf_push_linear;
508 } else {
509 nvc0->m2mf_copy_rect = nvc0_m2mf_transfer_rect;
510 nvc0->base.copy_data = nvc0_m2mf_copy_linear;
511 nvc0->base.push_data = nvc0_m2mf_push_linear;
512 }
513 nvc0->base.push_cb = nvc0_cb_push;
514 }
515