1 /*
2  * Copyright 2013 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21  * SOFTWARE.
22  *
23  * Authors: Marek Olšák <maraeo@gmail.com>
24  *
25  */
26 
27 /**
28  * This file contains common screen and context structures and functions
29  * for r600g and radeonsi.
30  */
31 
32 #ifndef R600_PIPE_COMMON_H
33 #define R600_PIPE_COMMON_H
34 
35 #include <stdio.h>
36 
37 #include "radeon/radeon_winsys.h"
38 
39 #include "util/disk_cache.h"
40 #include "util/u_blitter.h"
41 #include "util/list.h"
42 #include "util/u_range.h"
43 #include "util/slab.h"
44 #include "util/u_suballoc.h"
45 #include "util/u_transfer.h"
46 #include "util/u_threaded_context.h"
47 
48 struct u_log_context;
49 #define ATI_VENDOR_ID 0x1002
50 
51 #define R600_RESOURCE_FLAG_TRANSFER		(PIPE_RESOURCE_FLAG_DRV_PRIV << 0)
52 #define R600_RESOURCE_FLAG_FLUSHED_DEPTH	(PIPE_RESOURCE_FLAG_DRV_PRIV << 1)
53 #define R600_RESOURCE_FLAG_FORCE_TILING		(PIPE_RESOURCE_FLAG_DRV_PRIV << 2)
54 #define R600_RESOURCE_FLAG_UNMAPPABLE		(PIPE_RESOURCE_FLAG_DRV_PRIV << 4)
55 
56 #define R600_CONTEXT_STREAMOUT_FLUSH		(1u << 0)
57 /* Pipeline & streamout query controls. */
58 #define R600_CONTEXT_START_PIPELINE_STATS	(1u << 1)
59 #define R600_CONTEXT_STOP_PIPELINE_STATS	(1u << 2)
60 #define R600_CONTEXT_FLUSH_FOR_RENDER_COND	(1u << 3)
61 #define R600_CONTEXT_PRIVATE_FLAG		(1u << 4)
62 
63 /* special primitive types */
64 #define R600_PRIM_RECTANGLE_LIST	PIPE_PRIM_MAX
65 
66 #define R600_NOT_QUERY		0xffffffff
67 
68 /* Debug flags. */
69 #define DBG_VS			(1 << PIPE_SHADER_VERTEX)
70 #define DBG_PS			(1 << PIPE_SHADER_FRAGMENT)
71 #define DBG_GS			(1 << PIPE_SHADER_GEOMETRY)
72 #define DBG_TCS			(1 << PIPE_SHADER_TESS_CTRL)
73 #define DBG_TES			(1 << PIPE_SHADER_TESS_EVAL)
74 #define DBG_CS			(1 << PIPE_SHADER_COMPUTE)
75 #define DBG_ALL_SHADERS		(DBG_FS - 1)
76 #define DBG_FS			(1 << 6) /* fetch shader */
77 #define DBG_TEX			(1 << 7)
78 #define DBG_NIR			(1 << 8)
79 #define DBG_COMPUTE		(1 << 9)
80 /* gap */
81 #define DBG_VM			(1 << 11)
82 #define DBG_NO_IR		(1 << 12)
83 #define DBG_NO_TGSI		(1 << 13)
84 #define DBG_NO_ASM		(1 << 14)
85 #define DBG_PREOPT_IR		(1 << 15)
86 #define DBG_CHECK_IR		(1 << 16)
87 #define DBG_NO_OPT_VARIANT	(1 << 17)
88 #define DBG_FS_CORRECT_DERIVS_AFTER_KILL (1 << 18)
89 /* gaps */
90 #define DBG_TEST_DMA		(1 << 20)
91 /* Bits 21-31 are reserved for the r600g driver. */
92 /* features */
93 #define DBG_NO_ASYNC_DMA	(1ull << 32)
94 #define DBG_NO_HYPERZ		(1ull << 33)
95 #define DBG_NO_DISCARD_RANGE	(1ull << 34)
96 #define DBG_NO_2D_TILING	(1ull << 35)
97 #define DBG_NO_TILING		(1ull << 36)
98 #define DBG_SWITCH_ON_EOP	(1ull << 37)
99 #define DBG_FORCE_DMA		(1ull << 38)
100 #define DBG_PRECOMPILE		(1ull << 39)
101 #define DBG_INFO		(1ull << 40)
102 #define DBG_NO_WC		(1ull << 41)
103 #define DBG_CHECK_VM		(1ull << 42)
104 /* gap */
105 #define DBG_UNSAFE_MATH		(1ull << 49)
106 #define DBG_TEST_VMFAULT_CP	(1ull << 51)
107 #define DBG_TEST_VMFAULT_SDMA	(1ull << 52)
108 #define DBG_TEST_VMFAULT_SHADER	(1ull << 53)
109 
110 #define R600_MAP_BUFFER_ALIGNMENT 64
111 #define R600_MAX_VIEWPORTS        16
112 
113 #define SI_MAX_VARIABLE_THREADS_PER_BLOCK 1024
114 
115 enum r600_coherency {
116 	R600_COHERENCY_NONE, /* no cache flushes needed */
117 	R600_COHERENCY_SHADER,
118 	R600_COHERENCY_CB_META,
119 };
120 
121 #if UTIL_ARCH_BIG_ENDIAN
122 #define R600_BIG_ENDIAN 1
123 #else
124 #define R600_BIG_ENDIAN 0
125 #endif
126 
127 struct r600_common_context;
128 struct r600_perfcounters;
129 struct tgsi_shader_info;
130 struct r600_qbo_state;
131 
132 /* Only 32-bit buffer allocations are supported, gallium doesn't support more
133  * at the moment.
134  */
135 struct r600_resource {
136 	struct threaded_resource	b;
137 
138 	/* Winsys objects. */
139 	struct pb_buffer		*buf;
140 	uint64_t			gpu_address;
141 	/* Memory usage if the buffer placement is optimal. */
142 	uint64_t			vram_usage;
143 	uint64_t			gart_usage;
144 
145 	/* Resource properties. */
146 	uint64_t			bo_size;
147 	unsigned			bo_alignment;
148 	enum radeon_bo_domain		domains;
149 	enum radeon_bo_flag		flags;
150 	unsigned			bind_history;
151 
152 	/* The buffer range which is initialized (with a write transfer,
153 	 * streamout, DMA, or as a random access target). The rest of
154 	 * the buffer is considered invalid and can be mapped unsynchronized.
155 	 *
156 	 * This allows unsychronized mapping of a buffer range which hasn't
157 	 * been used yet. It's for applications which forget to use
158 	 * the unsynchronized map flag and expect the driver to figure it out.
159          */
160 	struct util_range		valid_buffer_range;
161 
162 	/* Whether the resource has been exported via resource_get_handle. */
163 	unsigned			external_usage; /* PIPE_HANDLE_USAGE_* */
164 
165 	/* Whether this resource is referenced by bindless handles. */
166 	bool				texture_handle_allocated;
167 	bool				image_handle_allocated;
168 
169 	/*
170 	 * EG/Cayman only - for RAT operations hw need an immediate buffer
171 	 * to store results in.
172 	 */
173 	struct r600_resource            *immed_buffer;
174 };
175 
176 struct r600_transfer {
177 	struct threaded_transfer	b;
178 	struct r600_resource		*staging;
179 	unsigned			offset;
180 };
181 
182 struct r600_fmask_info {
183 	uint64_t offset;
184 	uint64_t size;
185 	unsigned alignment;
186 	unsigned pitch_in_pixels;
187 	unsigned bank_height;
188 	unsigned slice_tile_max;
189 	unsigned tile_mode_index;
190 	unsigned tile_swizzle;
191 };
192 
193 struct r600_cmask_info {
194 	uint64_t offset;
195 	uint64_t size;
196 	unsigned alignment;
197 	unsigned slice_tile_max;
198 	uint64_t base_address_reg;
199 };
200 
201 struct r600_texture {
202 	struct r600_resource		resource;
203 
204 	uint64_t			size;
205 	unsigned			num_level0_transfers;
206 	enum pipe_format		db_render_format;
207 	bool				is_depth;
208 	bool				db_compatible;
209 	bool				can_sample_z;
210 	bool				can_sample_s;
211 	unsigned			dirty_level_mask; /* each bit says if that mipmap is compressed */
212 	unsigned			stencil_dirty_level_mask; /* each bit says if that mipmap is compressed */
213 	struct r600_texture		*flushed_depth_texture;
214 	struct radeon_surf		surface;
215 
216 	/* Colorbuffer compression and fast clear. */
217 	struct r600_fmask_info		fmask;
218 	struct r600_cmask_info		cmask;
219 	struct r600_resource		*cmask_buffer;
220 	unsigned			cb_color_info; /* fast clear enable bit */
221 	unsigned			color_clear_value[2];
222 	unsigned			last_msaa_resolve_target_micro_mode;
223 
224 	/* Depth buffer compression and fast clear. */
225 	uint64_t			htile_offset;
226 	bool				depth_cleared; /* if it was cleared at least once */
227 	float				depth_clear_value;
228 	bool				stencil_cleared; /* if it was cleared at least once */
229 	uint8_t				stencil_clear_value;
230 
231 	bool				non_disp_tiling; /* R600-Cayman only */
232 
233 	/* Counter that should be non-zero if the texture is bound to a
234 	 * framebuffer. Implemented in radeonsi only.
235 	 */
236 	uint32_t			framebuffers_bound;
237 };
238 
239 struct r600_surface {
240 	struct pipe_surface		base;
241 
242 	/* These can vary with block-compressed textures. */
243 	unsigned width0;
244 	unsigned height0;
245 
246 	bool color_initialized;
247 	bool depth_initialized;
248 
249 	/* Misc. color flags. */
250 	bool alphatest_bypass;
251 	bool export_16bpc;
252 	bool color_is_int8;
253 	bool color_is_int10;
254 
255 	/* Color registers. */
256 	unsigned cb_color_info;
257 	unsigned cb_color_base;
258 	unsigned cb_color_view;
259 	unsigned cb_color_size;		/* R600 only */
260 	unsigned cb_color_dim;		/* EG only */
261 	unsigned cb_color_pitch;	/* EG and later */
262 	unsigned cb_color_slice;	/* EG and later */
263 	unsigned cb_color_attrib;	/* EG and later */
264 	unsigned cb_color_fmask;	/* CB_COLORn_FMASK (EG and later) or CB_COLORn_FRAG (r600) */
265 	unsigned cb_color_fmask_slice;	/* EG and later */
266 	unsigned cb_color_cmask;	/* CB_COLORn_TILE (r600 only) */
267 	unsigned cb_color_mask;		/* R600 only */
268 	struct r600_resource *cb_buffer_fmask; /* Used for FMASK relocations. R600 only */
269 	struct r600_resource *cb_buffer_cmask; /* Used for CMASK relocations. R600 only */
270 
271 	/* DB registers. */
272 	uint64_t db_depth_base;		/* DB_Z_READ/WRITE_BASE (EG and later) or DB_DEPTH_BASE (r600) */
273 	uint64_t db_stencil_base;	/* EG and later */
274 	uint64_t db_htile_data_base;
275 	unsigned db_depth_info;		/* R600 only, then SI and later */
276 	unsigned db_z_info;		/* EG and later */
277 	unsigned db_depth_view;
278 	unsigned db_depth_size;
279 	unsigned db_depth_slice;	/* EG and later */
280 	unsigned db_stencil_info;	/* EG and later */
281 	unsigned db_prefetch_limit;	/* R600 only */
282 	unsigned db_htile_surface;
283 	unsigned db_preload_control;	/* EG and later */
284 };
285 
286 struct r600_mmio_counter {
287 	unsigned busy;
288 	unsigned idle;
289 };
290 
291 union r600_mmio_counters {
292 	struct r600_mmio_counters_named {
293 		/* For global GPU load including SDMA. */
294 		struct r600_mmio_counter gpu;
295 
296 		/* GRBM_STATUS */
297 		struct r600_mmio_counter spi;
298 		struct r600_mmio_counter gui;
299 		struct r600_mmio_counter ta;
300 		struct r600_mmio_counter gds;
301 		struct r600_mmio_counter vgt;
302 		struct r600_mmio_counter ia;
303 		struct r600_mmio_counter sx;
304 		struct r600_mmio_counter wd;
305 		struct r600_mmio_counter bci;
306 		struct r600_mmio_counter sc;
307 		struct r600_mmio_counter pa;
308 		struct r600_mmio_counter db;
309 		struct r600_mmio_counter cp;
310 		struct r600_mmio_counter cb;
311 
312 		/* SRBM_STATUS2 */
313 		struct r600_mmio_counter sdma;
314 
315 		/* CP_STAT */
316 		struct r600_mmio_counter pfp;
317 		struct r600_mmio_counter meq;
318 		struct r600_mmio_counter me;
319 		struct r600_mmio_counter surf_sync;
320 		struct r600_mmio_counter cp_dma;
321 		struct r600_mmio_counter scratch_ram;
322 	} named;
323 	unsigned array[sizeof(struct r600_mmio_counters_named) / sizeof(unsigned)];
324 };
325 
326 struct r600_memory_object {
327 	struct pipe_memory_object	b;
328 	struct pb_buffer		*buf;
329 	uint32_t			stride;
330 	uint32_t			offset;
331 };
332 
333 struct r600_common_screen {
334 	struct pipe_screen		b;
335 	struct radeon_winsys		*ws;
336 	enum radeon_family		family;
337 	enum chip_class			chip_class;
338 	struct radeon_info		info;
339 	uint64_t			debug_flags;
340 	bool				has_cp_dma;
341 	bool				has_streamout;
342 
343 	struct disk_cache		*disk_shader_cache;
344 
345 	struct slab_parent_pool		pool_transfers;
346 
347 	/* Texture filter settings. */
348 	int				force_aniso; /* -1 = disabled */
349 
350 	/* Auxiliary context. Mainly used to initialize resources.
351 	 * It must be locked prior to using and flushed before unlocking. */
352 	struct pipe_context		*aux_context;
353 	mtx_t				aux_context_lock;
354 
355 	/* This must be in the screen, because UE4 uses one context for
356 	 * compilation and another one for rendering.
357 	 */
358 	unsigned			num_compilations;
359 	/* Along with ST_DEBUG=precompile, this should show if applications
360 	 * are loading shaders on demand. This is a monotonic counter.
361 	 */
362 	unsigned			num_shaders_created;
363 	unsigned			num_shader_cache_hits;
364 
365 	/* GPU load thread. */
366 	mtx_t				gpu_load_mutex;
367 	thrd_t				gpu_load_thread;
368 	union r600_mmio_counters	mmio_counters;
369 	volatile unsigned		gpu_load_stop_thread; /* bool */
370 
371 	char				renderer_string[100];
372 
373 	/* Performance counters. */
374 	struct r600_perfcounters	*perfcounters;
375 
376 	/* If pipe_screen wants to recompute and re-emit the framebuffer,
377 	 * sampler, and image states of all contexts, it should atomically
378 	 * increment this.
379 	 *
380 	 * Each context will compare this with its own last known value of
381 	 * the counter before drawing and re-emit the states accordingly.
382 	 */
383 	unsigned			dirty_tex_counter;
384 
385 	/* Atomically increment this counter when an existing texture's
386 	 * metadata is enabled or disabled in a way that requires changing
387 	 * contexts' compressed texture binding masks.
388 	 */
389 	unsigned			compressed_colortex_counter;
390 
391 	struct {
392 		/* Context flags to set so that all writes from earlier jobs
393 		 * in the CP are seen by L2 clients.
394 		 */
395 		unsigned cp_to_L2;
396 
397 		/* Context flags to set so that all writes from earlier jobs
398 		 * that end in L2 are seen by CP.
399 		 */
400 		unsigned L2_to_cp;
401 
402 		/* Context flags to set so that all writes from earlier
403 		 * compute jobs are seen by L2 clients.
404 		 */
405 		unsigned compute_to_L2;
406 	} barrier_flags;
407 };
408 
409 /* This encapsulates a state or an operation which can emitted into the GPU
410  * command stream. */
411 struct r600_atom {
412 	void (*emit)(struct r600_common_context *ctx, struct r600_atom *state);
413 	unsigned		num_dw;
414 	unsigned short		id;
415 };
416 
417 struct r600_so_target {
418 	struct pipe_stream_output_target b;
419 
420 	/* The buffer where BUFFER_FILLED_SIZE is stored. */
421 	struct r600_resource	*buf_filled_size;
422 	unsigned		buf_filled_size_offset;
423 	bool			buf_filled_size_valid;
424 
425 	unsigned		stride_in_dw;
426 };
427 
428 struct r600_streamout {
429 	struct r600_atom		begin_atom;
430 	bool				begin_emitted;
431 	unsigned			num_dw_for_end;
432 
433 	unsigned			enabled_mask;
434 	unsigned			num_targets;
435 	struct r600_so_target		*targets[PIPE_MAX_SO_BUFFERS];
436 
437 	unsigned			append_bitmask;
438 	bool				suspended;
439 
440 	/* External state which comes from the vertex shader,
441 	 * it must be set explicitly when binding a shader. */
442 	uint16_t			*stride_in_dw;
443 	unsigned			enabled_stream_buffers_mask; /* stream0 buffers0-3 in 4 LSB */
444 
445 	/* The state of VGT_STRMOUT_BUFFER_(CONFIG|EN). */
446 	unsigned			hw_enabled_mask;
447 
448 	/* The state of VGT_STRMOUT_(CONFIG|EN). */
449 	struct r600_atom		enable_atom;
450 	bool				streamout_enabled;
451 	bool				prims_gen_query_enabled;
452 	int				num_prims_gen_queries;
453 };
454 
455 struct r600_signed_scissor {
456 	int minx;
457 	int miny;
458 	int maxx;
459 	int maxy;
460 };
461 
462 struct r600_scissors {
463 	struct r600_atom		atom;
464 	unsigned			dirty_mask;
465 	struct pipe_scissor_state	states[R600_MAX_VIEWPORTS];
466 };
467 
468 struct r600_viewports {
469 	struct r600_atom		atom;
470 	unsigned			dirty_mask;
471 	unsigned			depth_range_dirty_mask;
472 	struct pipe_viewport_state	states[R600_MAX_VIEWPORTS];
473 	struct r600_signed_scissor	as_scissor[R600_MAX_VIEWPORTS];
474 };
475 
476 struct r600_ring {
477 	struct radeon_cmdbuf		*cs;
478 	void (*flush)(void *ctx, unsigned flags,
479 		      struct pipe_fence_handle **fence);
480 };
481 
482 /* Saved CS data for debugging features. */
483 struct radeon_saved_cs {
484 	uint32_t			*ib;
485 	unsigned			num_dw;
486 
487 	struct radeon_bo_list_item	*bo_list;
488 	unsigned			bo_count;
489 };
490 
491 struct r600_common_context {
492 	struct pipe_context b; /* base class */
493 
494 	struct r600_common_screen	*screen;
495 	struct radeon_winsys		*ws;
496 	struct radeon_winsys_ctx	*ctx;
497 	enum radeon_family		family;
498 	enum chip_class			chip_class;
499 	struct r600_ring		gfx;
500 	struct r600_ring		dma;
501 	struct pipe_fence_handle	*last_gfx_fence;
502 	struct pipe_fence_handle	*last_sdma_fence;
503 	struct r600_resource		*eop_bug_scratch;
504 	unsigned			num_gfx_cs_flushes;
505 	unsigned			initial_gfx_cs_size;
506 	unsigned			last_dirty_tex_counter;
507 	unsigned			last_compressed_colortex_counter;
508 	unsigned			last_num_draw_calls;
509 
510 	struct threaded_context		*tc;
511 	struct u_suballocator		*allocator_zeroed_memory;
512 	struct slab_child_pool		pool_transfers;
513 	struct slab_child_pool		pool_transfers_unsync; /* for threaded_context */
514 
515 	/* Current unaccounted memory usage. */
516 	uint64_t			vram;
517 	uint64_t			gtt;
518 
519 	/* States. */
520 	struct r600_streamout		streamout;
521 	struct r600_scissors		scissors;
522 	struct r600_viewports		viewports;
523 	bool				scissor_enabled;
524 	bool				clip_halfz;
525 	bool				vs_writes_viewport_index;
526 	bool				vs_disables_clipping_viewport;
527 
528 	/* Additional context states. */
529 	unsigned flags; /* flush flags */
530 
531 	/* Queries. */
532 	/* Maintain the list of active queries for pausing between IBs. */
533 	int				num_occlusion_queries;
534 	int				num_perfect_occlusion_queries;
535 	struct list_head		active_queries;
536 	unsigned			num_cs_dw_queries_suspend;
537 	/* Misc stats. */
538 	unsigned			num_draw_calls;
539 	unsigned			num_decompress_calls;
540 	unsigned			num_mrt_draw_calls;
541 	unsigned			num_prim_restart_calls;
542 	unsigned			num_spill_draw_calls;
543 	unsigned			num_compute_calls;
544 	unsigned			num_spill_compute_calls;
545 	unsigned			num_dma_calls;
546 	unsigned			num_cp_dma_calls;
547 	unsigned			num_vs_flushes;
548 	unsigned			num_ps_flushes;
549 	unsigned			num_cs_flushes;
550 	unsigned			num_cb_cache_flushes;
551 	unsigned			num_db_cache_flushes;
552 	unsigned			num_resident_handles;
553 	uint64_t			num_alloc_tex_transfer_bytes;
554 
555 	/* Render condition. */
556 	struct r600_atom		render_cond_atom;
557 	struct pipe_query		*render_cond;
558 	unsigned			render_cond_mode;
559 	bool				render_cond_invert;
560 	bool				render_cond_force_off; /* for u_blitter */
561 
562 	/* MSAA sample locations.
563 	 * The first index is the sample index.
564 	 * The second index is the coordinate: X, Y. */
565 	float				sample_locations_1x[1][2];
566 	float				sample_locations_2x[2][2];
567 	float				sample_locations_4x[4][2];
568 	float				sample_locations_8x[8][2];
569 	float				sample_locations_16x[16][2];
570 
571 	struct pipe_debug_callback	debug;
572 	struct pipe_device_reset_callback device_reset_callback;
573 	struct u_log_context		*log;
574 
575 	void				*query_result_shader;
576 
577 	/* Copy one resource to another using async DMA. */
578 	void (*dma_copy)(struct pipe_context *ctx,
579 			 struct pipe_resource *dst,
580 			 unsigned dst_level,
581 			 unsigned dst_x, unsigned dst_y, unsigned dst_z,
582 			 struct pipe_resource *src,
583 			 unsigned src_level,
584 			 const struct pipe_box *src_box);
585 
586 	void (*dma_clear_buffer)(struct pipe_context *ctx, struct pipe_resource *dst,
587 				 uint64_t offset, uint64_t size, unsigned value);
588 
589 	void (*clear_buffer)(struct pipe_context *ctx, struct pipe_resource *dst,
590 			     uint64_t offset, uint64_t size, unsigned value,
591 			     enum r600_coherency coher);
592 
593 	void (*blit_decompress_depth)(struct pipe_context *ctx,
594 				      struct r600_texture *texture,
595 				      struct r600_texture *staging,
596 				      unsigned first_level, unsigned last_level,
597 				      unsigned first_layer, unsigned last_layer,
598 				      unsigned first_sample, unsigned last_sample);
599 
600 	/* Reallocate the buffer and update all resource bindings where
601 	 * the buffer is bound, including all resource descriptors. */
602 	void (*invalidate_buffer)(struct pipe_context *ctx, struct pipe_resource *buf);
603 
604 	/* Update all resource bindings where the buffer is bound, including
605 	 * all resource descriptors. This is invalidate_buffer without
606 	 * the invalidation. */
607 	void (*rebind_buffer)(struct pipe_context *ctx, struct pipe_resource *buf,
608 			      uint64_t old_gpu_address);
609 
610 	void (*save_qbo_state)(struct pipe_context *ctx, struct r600_qbo_state *st);
611 
612 	/* This ensures there is enough space in the command stream. */
613 	void (*need_gfx_cs_space)(struct pipe_context *ctx, unsigned num_dw,
614 				  bool include_draw_vbo);
615 
616 	void (*set_atom_dirty)(struct r600_common_context *ctx,
617 			       struct r600_atom *atom, bool dirty);
618 
619 	void (*check_vm_faults)(struct r600_common_context *ctx,
620 				struct radeon_saved_cs *saved,
621 				enum ring_type ring);
622 };
623 
624 /* r600_buffer_common.c */
625 bool r600_rings_is_buffer_referenced(struct r600_common_context *ctx,
626 				     struct pb_buffer *buf,
627 				     enum radeon_bo_usage usage);
628 void *r600_buffer_map_sync_with_rings(struct r600_common_context *ctx,
629                                       struct r600_resource *resource,
630                                       unsigned usage);
631 void r600_buffer_subdata(struct pipe_context *ctx,
632 			 struct pipe_resource *buffer,
633 			 unsigned usage, unsigned offset,
634 			 unsigned size, const void *data);
635 void r600_init_resource_fields(struct r600_common_screen *rscreen,
636 			       struct r600_resource *res,
637 			       uint64_t size, unsigned alignment);
638 bool r600_alloc_resource(struct r600_common_screen *rscreen,
639 			 struct r600_resource *res);
640 struct pipe_resource *r600_buffer_create(struct pipe_screen *screen,
641 					 const struct pipe_resource *templ,
642 					 unsigned alignment);
643 struct pipe_resource * r600_aligned_buffer_create(struct pipe_screen *screen,
644 						  unsigned flags,
645 						  unsigned usage,
646 						  unsigned size,
647 						  unsigned alignment);
648 struct pipe_resource *
649 r600_buffer_from_user_memory(struct pipe_screen *screen,
650 			     const struct pipe_resource *templ,
651 			     void *user_memory);
652 void
653 r600_invalidate_resource(struct pipe_context *ctx,
654 			 struct pipe_resource *resource);
655 void r600_replace_buffer_storage(struct pipe_context *ctx,
656 				 struct pipe_resource *dst,
657 				 struct pipe_resource *src);
658 
659 /* r600_common_pipe.c */
660 void r600_gfx_write_event_eop(struct r600_common_context *ctx,
661 			      unsigned event, unsigned event_flags,
662 			      unsigned data_sel,
663 			      struct r600_resource *buf, uint64_t va,
664 			      uint32_t new_fence, unsigned query_type);
665 unsigned r600_gfx_write_fence_dwords(struct r600_common_screen *screen);
666 void r600_gfx_wait_fence(struct r600_common_context *ctx,
667 			 struct r600_resource *buf,
668 			 uint64_t va, uint32_t ref, uint32_t mask);
669 void r600_draw_rectangle(struct blitter_context *blitter,
670 			 void *vertex_elements_cso,
671 			 blitter_get_vs_func get_vs,
672 			 int x1, int y1, int x2, int y2,
673 			 float depth, unsigned num_instances,
674 			 enum blitter_attrib_type type,
675 			 const union blitter_attrib *attrib);
676 bool r600_common_screen_init(struct r600_common_screen *rscreen,
677 			     struct radeon_winsys *ws);
678 void r600_destroy_common_screen(struct r600_common_screen *rscreen);
679 void r600_preflush_suspend_features(struct r600_common_context *ctx);
680 void r600_postflush_resume_features(struct r600_common_context *ctx);
681 bool r600_common_context_init(struct r600_common_context *rctx,
682 			      struct r600_common_screen *rscreen,
683 			      unsigned context_flags);
684 void r600_common_context_cleanup(struct r600_common_context *rctx);
685 bool r600_can_dump_shader(struct r600_common_screen *rscreen,
686 			  unsigned processor);
687 bool r600_extra_shader_checks(struct r600_common_screen *rscreen,
688 			      unsigned processor);
689 void r600_screen_clear_buffer(struct r600_common_screen *rscreen, struct pipe_resource *dst,
690 			      uint64_t offset, uint64_t size, unsigned value);
691 struct pipe_resource *r600_resource_create_common(struct pipe_screen *screen,
692 						  const struct pipe_resource *templ);
693 const char *r600_get_llvm_processor_name(enum radeon_family family);
694 void r600_need_dma_space(struct r600_common_context *ctx, unsigned num_dw,
695 			 struct r600_resource *dst, struct r600_resource *src);
696 void radeon_save_cs(struct radeon_winsys *ws, struct radeon_cmdbuf *cs,
697 		    struct radeon_saved_cs *saved, bool get_buffer_list);
698 void radeon_clear_saved_cs(struct radeon_saved_cs *saved);
699 bool r600_check_device_reset(struct r600_common_context *rctx);
700 
701 /* r600_gpu_load.c */
702 void r600_gpu_load_kill_thread(struct r600_common_screen *rscreen);
703 uint64_t r600_begin_counter(struct r600_common_screen *rscreen, unsigned type);
704 unsigned r600_end_counter(struct r600_common_screen *rscreen, unsigned type,
705 			  uint64_t begin);
706 
707 /* r600_perfcounters.c */
708 void r600_perfcounters_destroy(struct r600_common_screen *rscreen);
709 
710 /* r600_query.c */
711 void r600_init_screen_query_functions(struct r600_common_screen *rscreen);
712 void r600_query_init(struct r600_common_context *rctx);
713 void r600_suspend_queries(struct r600_common_context *ctx);
714 void r600_resume_queries(struct r600_common_context *ctx);
715 void r600_query_fix_enabled_rb_mask(struct r600_common_screen *rscreen);
716 
717 /* r600_streamout.c */
718 void r600_streamout_buffers_dirty(struct r600_common_context *rctx);
719 void r600_set_streamout_targets(struct pipe_context *ctx,
720 				unsigned num_targets,
721 				struct pipe_stream_output_target **targets,
722 				const unsigned *offset);
723 void r600_emit_streamout_end(struct r600_common_context *rctx);
724 void r600_update_prims_generated_query_state(struct r600_common_context *rctx,
725 					     unsigned type, int diff);
726 void r600_streamout_init(struct r600_common_context *rctx);
727 
728 /* r600_test_dma.c */
729 void r600_test_dma(struct r600_common_screen *rscreen);
730 
731 /* r600_texture.c */
732 bool r600_prepare_for_dma_blit(struct r600_common_context *rctx,
733 				struct r600_texture *rdst,
734 				unsigned dst_level, unsigned dstx,
735 				unsigned dsty, unsigned dstz,
736 				struct r600_texture *rsrc,
737 				unsigned src_level,
738 				const struct pipe_box *src_box);
739 void r600_texture_get_fmask_info(struct r600_common_screen *rscreen,
740 				 struct r600_texture *rtex,
741 				 unsigned nr_samples,
742 				 struct r600_fmask_info *out);
743 void r600_texture_get_cmask_info(struct r600_common_screen *rscreen,
744 				 struct r600_texture *rtex,
745 				 struct r600_cmask_info *out);
746 bool r600_init_flushed_depth_texture(struct pipe_context *ctx,
747 				     struct pipe_resource *texture,
748 				     struct r600_texture **staging);
749 void r600_print_texture_info(struct r600_common_screen *rscreen,
750 			     struct r600_texture *rtex, struct u_log_context *log);
751 struct pipe_resource *r600_texture_create(struct pipe_screen *screen,
752 					const struct pipe_resource *templ);
753 struct pipe_surface *r600_create_surface_custom(struct pipe_context *pipe,
754 						struct pipe_resource *texture,
755 						const struct pipe_surface *templ,
756 						unsigned width0, unsigned height0,
757 						unsigned width, unsigned height);
758 unsigned r600_translate_colorswap(enum pipe_format format, bool do_endian_swap);
759 void evergreen_do_fast_color_clear(struct r600_common_context *rctx,
760 				   struct pipe_framebuffer_state *fb,
761 				   struct r600_atom *fb_state,
762 				   unsigned *buffers, ubyte *dirty_cbufs,
763 				   const union pipe_color_union *color);
764 void r600_init_screen_texture_functions(struct r600_common_screen *rscreen);
765 void r600_init_context_texture_functions(struct r600_common_context *rctx);
766 void eg_resource_alloc_immed(struct r600_common_screen *rscreen,
767 			     struct r600_resource *res,
768 			     unsigned immed_size);
769 
770 /* r600_viewport.c */
771 void evergreen_apply_scissor_bug_workaround(struct r600_common_context *rctx,
772 					    struct pipe_scissor_state *scissor);
773 void r600_viewport_set_rast_deps(struct r600_common_context *rctx,
774 				 bool scissor_enable, bool clip_halfz);
775 void r600_update_vs_writes_viewport_index(struct r600_common_context *rctx,
776 					  struct tgsi_shader_info *info);
777 void r600_init_viewport_functions(struct r600_common_context *rctx);
778 
779 /* cayman_msaa.c */
780 extern const uint32_t eg_sample_locs_2x[4];
781 extern const unsigned eg_max_dist_2x;
782 extern const uint32_t eg_sample_locs_4x[4];
783 extern const unsigned eg_max_dist_4x;
784 void cayman_get_sample_position(struct pipe_context *ctx, unsigned sample_count,
785 				unsigned sample_index, float *out_value);
786 void cayman_init_msaa(struct pipe_context *ctx);
787 void cayman_emit_msaa_state(struct radeon_cmdbuf *cs, int nr_samples,
788 			    int ps_iter_samples, int overrast_samples);
789 
790 
791 /* Inline helpers. */
792 
r600_resource(struct pipe_resource * r)793 static inline struct r600_resource *r600_resource(struct pipe_resource *r)
794 {
795 	return (struct r600_resource*)r;
796 }
797 
798 static inline void
r600_resource_reference(struct r600_resource ** ptr,struct r600_resource * res)799 r600_resource_reference(struct r600_resource **ptr, struct r600_resource *res)
800 {
801 	pipe_resource_reference((struct pipe_resource **)ptr,
802 				(struct pipe_resource *)res);
803 }
804 
805 static inline void
r600_texture_reference(struct r600_texture ** ptr,struct r600_texture * res)806 r600_texture_reference(struct r600_texture **ptr, struct r600_texture *res)
807 {
808 	pipe_resource_reference((struct pipe_resource **)ptr, &res->resource.b.b);
809 }
810 
811 static inline void
r600_context_add_resource_size(struct pipe_context * ctx,struct pipe_resource * r)812 r600_context_add_resource_size(struct pipe_context *ctx, struct pipe_resource *r)
813 {
814 	struct r600_common_context *rctx = (struct r600_common_context *)ctx;
815 	struct r600_resource *res = (struct r600_resource *)r;
816 
817 	if (res) {
818 		/* Add memory usage for need_gfx_cs_space */
819 		rctx->vram += res->vram_usage;
820 		rctx->gtt += res->gart_usage;
821 	}
822 }
823 
r600_get_strmout_en(struct r600_common_context * rctx)824 static inline bool r600_get_strmout_en(struct r600_common_context *rctx)
825 {
826 	return rctx->streamout.streamout_enabled ||
827 	       rctx->streamout.prims_gen_query_enabled;
828 }
829 
830 #define     SQ_TEX_XY_FILTER_POINT                         0x00
831 #define     SQ_TEX_XY_FILTER_BILINEAR                      0x01
832 #define     SQ_TEX_XY_FILTER_ANISO_POINT                   0x02
833 #define     SQ_TEX_XY_FILTER_ANISO_BILINEAR                0x03
834 
eg_tex_filter(unsigned filter,unsigned max_aniso)835 static inline unsigned eg_tex_filter(unsigned filter, unsigned max_aniso)
836 {
837 	if (filter == PIPE_TEX_FILTER_LINEAR)
838 		return max_aniso > 1 ? SQ_TEX_XY_FILTER_ANISO_BILINEAR
839 				     : SQ_TEX_XY_FILTER_BILINEAR;
840 	else
841 		return max_aniso > 1 ? SQ_TEX_XY_FILTER_ANISO_POINT
842 				     : SQ_TEX_XY_FILTER_POINT;
843 }
844 
r600_tex_aniso_filter(unsigned filter)845 static inline unsigned r600_tex_aniso_filter(unsigned filter)
846 {
847 	if (filter < 2)
848 		return 0;
849 	if (filter < 4)
850 		return 1;
851 	if (filter < 8)
852 		return 2;
853 	if (filter < 16)
854 		return 3;
855 	return 4;
856 }
857 
r600_wavefront_size(enum radeon_family family)858 static inline unsigned r600_wavefront_size(enum radeon_family family)
859 {
860 	switch (family) {
861 	case CHIP_RV610:
862 	case CHIP_RS780:
863 	case CHIP_RV620:
864 	case CHIP_RS880:
865 		return 16;
866 	case CHIP_RV630:
867 	case CHIP_RV635:
868 	case CHIP_RV730:
869 	case CHIP_RV710:
870 	case CHIP_PALM:
871 	case CHIP_CEDAR:
872 		return 32;
873 	default:
874 		return 64;
875 	}
876 }
877 
878 static inline enum radeon_bo_priority
r600_get_sampler_view_priority(struct r600_resource * res)879 r600_get_sampler_view_priority(struct r600_resource *res)
880 {
881 	if (res->b.b.target == PIPE_BUFFER)
882 		return RADEON_PRIO_SAMPLER_BUFFER;
883 
884 	if (res->b.b.nr_samples > 1)
885 		return RADEON_PRIO_SAMPLER_TEXTURE_MSAA;
886 
887 	return RADEON_PRIO_SAMPLER_TEXTURE;
888 }
889 
890 static inline bool
r600_can_sample_zs(struct r600_texture * tex,bool stencil_sampler)891 r600_can_sample_zs(struct r600_texture *tex, bool stencil_sampler)
892 {
893 	return (stencil_sampler && tex->can_sample_s) ||
894 	       (!stencil_sampler && tex->can_sample_z);
895 }
896 
897 static inline bool
r600_htile_enabled(struct r600_texture * tex,unsigned level)898 r600_htile_enabled(struct r600_texture *tex, unsigned level)
899 {
900 	return tex->htile_offset && level == 0;
901 }
902 
903 #define COMPUTE_DBG(rscreen, fmt, args...) \
904 	do { \
905 		if ((rscreen->b.debug_flags & DBG_COMPUTE)) fprintf(stderr, fmt, ##args); \
906 	} while (0);
907 
908 #define R600_ERR(fmt, args...) \
909 	fprintf(stderr, "EE %s:%d %s - " fmt, __FILE__, __LINE__, __func__, ##args)
910 
911 /* For MSAA sample positions. */
912 #define FILL_SREG(s0x, s0y, s1x, s1y, s2x, s2y, s3x, s3y)  \
913 	(((s0x) & 0xf) | (((unsigned)(s0y) & 0xf) << 4) |		   \
914 	(((unsigned)(s1x) & 0xf) << 8) | (((unsigned)(s1y) & 0xf) << 12) |	   \
915 	(((unsigned)(s2x) & 0xf) << 16) | (((unsigned)(s2y) & 0xf) << 20) |	   \
916 	 (((unsigned)(s3x) & 0xf) << 24) | (((unsigned)(s3y) & 0xf) << 28))
917 
S_FIXED(float value,unsigned frac_bits)918 static inline int S_FIXED(float value, unsigned frac_bits)
919 {
920 	return value * (1 << frac_bits);
921 }
922 
923 #endif
924