1 /*
2 * Copyright 2015 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 * SOFTWARE.
22 */
23
24 #include "util/u_memory.h"
25 #include "r600_query.h"
26 #include "radeonsi/si_pipe.h"
27 #include "amd/common/sid.h"
28
29 /* Max counters per HW block */
30 #define R600_QUERY_MAX_COUNTERS 16
31
32 static struct r600_perfcounter_block *
lookup_counter(struct r600_perfcounters * pc,unsigned index,unsigned * base_gid,unsigned * sub_index)33 lookup_counter(struct r600_perfcounters *pc, unsigned index,
34 unsigned *base_gid, unsigned *sub_index)
35 {
36 struct r600_perfcounter_block *block = pc->blocks;
37 unsigned bid;
38
39 *base_gid = 0;
40 for (bid = 0; bid < pc->num_blocks; ++bid, ++block) {
41 unsigned total = block->num_groups * block->num_selectors;
42
43 if (index < total) {
44 *sub_index = index;
45 return block;
46 }
47
48 index -= total;
49 *base_gid += block->num_groups;
50 }
51
52 return NULL;
53 }
54
55 static struct r600_perfcounter_block *
lookup_group(struct r600_perfcounters * pc,unsigned * index)56 lookup_group(struct r600_perfcounters *pc, unsigned *index)
57 {
58 unsigned bid;
59 struct r600_perfcounter_block *block = pc->blocks;
60
61 for (bid = 0; bid < pc->num_blocks; ++bid, ++block) {
62 if (*index < block->num_groups)
63 return block;
64 *index -= block->num_groups;
65 }
66
67 return NULL;
68 }
69
70 struct r600_pc_group {
71 struct r600_pc_group *next;
72 struct r600_perfcounter_block *block;
73 unsigned sub_gid; /* only used during init */
74 unsigned result_base; /* only used during init */
75 int se;
76 int instance;
77 unsigned num_counters;
78 unsigned selectors[R600_QUERY_MAX_COUNTERS];
79 };
80
81 struct r600_pc_counter {
82 unsigned base;
83 unsigned qwords;
84 unsigned stride; /* in uint64s */
85 };
86
87 #define R600_PC_SHADERS_WINDOWING (1 << 31)
88
89 struct r600_query_pc {
90 struct r600_query_hw b;
91
92 unsigned shaders;
93 unsigned num_counters;
94 struct r600_pc_counter *counters;
95 struct r600_pc_group *groups;
96 };
97
r600_pc_query_destroy(struct si_screen * sscreen,struct r600_query * rquery)98 static void r600_pc_query_destroy(struct si_screen *sscreen,
99 struct r600_query *rquery)
100 {
101 struct r600_query_pc *query = (struct r600_query_pc *)rquery;
102
103 while (query->groups) {
104 struct r600_pc_group *group = query->groups;
105 query->groups = group->next;
106 FREE(group);
107 }
108
109 FREE(query->counters);
110
111 si_query_hw_destroy(sscreen, rquery);
112 }
113
r600_pc_query_prepare_buffer(struct si_screen * screen,struct r600_query_hw * hwquery,struct r600_resource * buffer)114 static bool r600_pc_query_prepare_buffer(struct si_screen *screen,
115 struct r600_query_hw *hwquery,
116 struct r600_resource *buffer)
117 {
118 /* no-op */
119 return true;
120 }
121
r600_pc_query_emit_start(struct r600_common_context * ctx,struct r600_query_hw * hwquery,struct r600_resource * buffer,uint64_t va)122 static void r600_pc_query_emit_start(struct r600_common_context *ctx,
123 struct r600_query_hw *hwquery,
124 struct r600_resource *buffer, uint64_t va)
125 {
126 struct r600_perfcounters *pc = ctx->screen->perfcounters;
127 struct r600_query_pc *query = (struct r600_query_pc *)hwquery;
128 struct r600_pc_group *group;
129 int current_se = -1;
130 int current_instance = -1;
131
132 if (query->shaders)
133 pc->emit_shaders(ctx, query->shaders);
134
135 for (group = query->groups; group; group = group->next) {
136 struct r600_perfcounter_block *block = group->block;
137
138 if (group->se != current_se || group->instance != current_instance) {
139 current_se = group->se;
140 current_instance = group->instance;
141 pc->emit_instance(ctx, group->se, group->instance);
142 }
143
144 pc->emit_select(ctx, block, group->num_counters, group->selectors);
145 }
146
147 if (current_se != -1 || current_instance != -1)
148 pc->emit_instance(ctx, -1, -1);
149
150 pc->emit_start(ctx, buffer, va);
151 }
152
r600_pc_query_emit_stop(struct r600_common_context * ctx,struct r600_query_hw * hwquery,struct r600_resource * buffer,uint64_t va)153 static void r600_pc_query_emit_stop(struct r600_common_context *ctx,
154 struct r600_query_hw *hwquery,
155 struct r600_resource *buffer, uint64_t va)
156 {
157 struct r600_perfcounters *pc = ctx->screen->perfcounters;
158 struct r600_query_pc *query = (struct r600_query_pc *)hwquery;
159 struct r600_pc_group *group;
160
161 pc->emit_stop(ctx, buffer, va);
162
163 for (group = query->groups; group; group = group->next) {
164 struct r600_perfcounter_block *block = group->block;
165 unsigned se = group->se >= 0 ? group->se : 0;
166 unsigned se_end = se + 1;
167
168 if ((block->flags & R600_PC_BLOCK_SE) && (group->se < 0))
169 se_end = ctx->screen->info.max_se;
170
171 do {
172 unsigned instance = group->instance >= 0 ? group->instance : 0;
173
174 do {
175 pc->emit_instance(ctx, se, instance);
176 pc->emit_read(ctx, block,
177 group->num_counters, group->selectors,
178 buffer, va);
179 va += sizeof(uint64_t) * group->num_counters;
180 } while (group->instance < 0 && ++instance < block->num_instances);
181 } while (++se < se_end);
182 }
183
184 pc->emit_instance(ctx, -1, -1);
185 }
186
r600_pc_query_clear_result(struct r600_query_hw * hwquery,union pipe_query_result * result)187 static void r600_pc_query_clear_result(struct r600_query_hw *hwquery,
188 union pipe_query_result *result)
189 {
190 struct r600_query_pc *query = (struct r600_query_pc *)hwquery;
191
192 memset(result, 0, sizeof(result->batch[0]) * query->num_counters);
193 }
194
r600_pc_query_add_result(struct si_screen * sscreen,struct r600_query_hw * hwquery,void * buffer,union pipe_query_result * result)195 static void r600_pc_query_add_result(struct si_screen *sscreen,
196 struct r600_query_hw *hwquery,
197 void *buffer,
198 union pipe_query_result *result)
199 {
200 struct r600_query_pc *query = (struct r600_query_pc *)hwquery;
201 uint64_t *results = buffer;
202 unsigned i, j;
203
204 for (i = 0; i < query->num_counters; ++i) {
205 struct r600_pc_counter *counter = &query->counters[i];
206
207 for (j = 0; j < counter->qwords; ++j) {
208 uint32_t value = results[counter->base + j * counter->stride];
209 result->batch[i].u64 += value;
210 }
211 }
212 }
213
214 static struct r600_query_ops batch_query_ops = {
215 .destroy = r600_pc_query_destroy,
216 .begin = si_query_hw_begin,
217 .end = si_query_hw_end,
218 .get_result = si_query_hw_get_result
219 };
220
221 static struct r600_query_hw_ops batch_query_hw_ops = {
222 .prepare_buffer = r600_pc_query_prepare_buffer,
223 .emit_start = r600_pc_query_emit_start,
224 .emit_stop = r600_pc_query_emit_stop,
225 .clear_result = r600_pc_query_clear_result,
226 .add_result = r600_pc_query_add_result,
227 };
228
get_group_state(struct si_screen * screen,struct r600_query_pc * query,struct r600_perfcounter_block * block,unsigned sub_gid)229 static struct r600_pc_group *get_group_state(struct si_screen *screen,
230 struct r600_query_pc *query,
231 struct r600_perfcounter_block *block,
232 unsigned sub_gid)
233 {
234 struct r600_pc_group *group = query->groups;
235
236 while (group) {
237 if (group->block == block && group->sub_gid == sub_gid)
238 return group;
239 group = group->next;
240 }
241
242 group = CALLOC_STRUCT(r600_pc_group);
243 if (!group)
244 return NULL;
245
246 group->block = block;
247 group->sub_gid = sub_gid;
248
249 if (block->flags & R600_PC_BLOCK_SHADER) {
250 unsigned sub_gids = block->num_instances;
251 unsigned shader_id;
252 unsigned shaders;
253 unsigned query_shaders;
254
255 if (block->flags & R600_PC_BLOCK_SE_GROUPS)
256 sub_gids = sub_gids * screen->info.max_se;
257 shader_id = sub_gid / sub_gids;
258 sub_gid = sub_gid % sub_gids;
259
260 shaders = screen->perfcounters->shader_type_bits[shader_id];
261
262 query_shaders = query->shaders & ~R600_PC_SHADERS_WINDOWING;
263 if (query_shaders && query_shaders != shaders) {
264 fprintf(stderr, "r600_perfcounter: incompatible shader groups\n");
265 FREE(group);
266 return NULL;
267 }
268 query->shaders = shaders;
269 }
270
271 if (block->flags & R600_PC_BLOCK_SHADER_WINDOWED && !query->shaders) {
272 // A non-zero value in query->shaders ensures that the shader
273 // masking is reset unless the user explicitly requests one.
274 query->shaders = R600_PC_SHADERS_WINDOWING;
275 }
276
277 if (block->flags & R600_PC_BLOCK_SE_GROUPS) {
278 group->se = sub_gid / block->num_instances;
279 sub_gid = sub_gid % block->num_instances;
280 } else {
281 group->se = -1;
282 }
283
284 if (block->flags & R600_PC_BLOCK_INSTANCE_GROUPS) {
285 group->instance = sub_gid;
286 } else {
287 group->instance = -1;
288 }
289
290 group->next = query->groups;
291 query->groups = group;
292
293 return group;
294 }
295
si_create_batch_query(struct pipe_context * ctx,unsigned num_queries,unsigned * query_types)296 struct pipe_query *si_create_batch_query(struct pipe_context *ctx,
297 unsigned num_queries,
298 unsigned *query_types)
299 {
300 struct si_screen *screen =
301 (struct si_screen *)ctx->screen;
302 struct r600_perfcounters *pc = screen->perfcounters;
303 struct r600_perfcounter_block *block;
304 struct r600_pc_group *group;
305 struct r600_query_pc *query;
306 unsigned base_gid, sub_gid, sub_index;
307 unsigned i, j;
308
309 if (!pc)
310 return NULL;
311
312 query = CALLOC_STRUCT(r600_query_pc);
313 if (!query)
314 return NULL;
315
316 query->b.b.ops = &batch_query_ops;
317 query->b.ops = &batch_query_hw_ops;
318
319 query->num_counters = num_queries;
320
321 /* Collect selectors per group */
322 for (i = 0; i < num_queries; ++i) {
323 unsigned sub_gid;
324
325 if (query_types[i] < R600_QUERY_FIRST_PERFCOUNTER)
326 goto error;
327
328 block = lookup_counter(pc, query_types[i] - R600_QUERY_FIRST_PERFCOUNTER,
329 &base_gid, &sub_index);
330 if (!block)
331 goto error;
332
333 sub_gid = sub_index / block->num_selectors;
334 sub_index = sub_index % block->num_selectors;
335
336 group = get_group_state(screen, query, block, sub_gid);
337 if (!group)
338 goto error;
339
340 if (group->num_counters >= block->num_counters) {
341 fprintf(stderr,
342 "perfcounter group %s: too many selected\n",
343 block->basename);
344 goto error;
345 }
346 group->selectors[group->num_counters] = sub_index;
347 ++group->num_counters;
348 }
349
350 /* Compute result bases and CS size per group */
351 query->b.num_cs_dw_begin = pc->num_start_cs_dwords;
352 query->b.num_cs_dw_end = pc->num_stop_cs_dwords;
353
354 query->b.num_cs_dw_begin += pc->num_instance_cs_dwords; /* conservative */
355 query->b.num_cs_dw_end += pc->num_instance_cs_dwords;
356
357 i = 0;
358 for (group = query->groups; group; group = group->next) {
359 struct r600_perfcounter_block *block = group->block;
360 unsigned select_dw, read_dw;
361 unsigned instances = 1;
362
363 if ((block->flags & R600_PC_BLOCK_SE) && group->se < 0)
364 instances = screen->info.max_se;
365 if (group->instance < 0)
366 instances *= block->num_instances;
367
368 group->result_base = i;
369 query->b.result_size += sizeof(uint64_t) * instances * group->num_counters;
370 i += instances * group->num_counters;
371
372 pc->get_size(block, group->num_counters, group->selectors,
373 &select_dw, &read_dw);
374 query->b.num_cs_dw_begin += select_dw;
375 query->b.num_cs_dw_end += instances * read_dw;
376 query->b.num_cs_dw_begin += pc->num_instance_cs_dwords; /* conservative */
377 query->b.num_cs_dw_end += instances * pc->num_instance_cs_dwords;
378 }
379
380 if (query->shaders) {
381 if (query->shaders == R600_PC_SHADERS_WINDOWING)
382 query->shaders = 0xffffffff;
383 query->b.num_cs_dw_begin += pc->num_shaders_cs_dwords;
384 }
385
386 /* Map user-supplied query array to result indices */
387 query->counters = CALLOC(num_queries, sizeof(*query->counters));
388 for (i = 0; i < num_queries; ++i) {
389 struct r600_pc_counter *counter = &query->counters[i];
390 struct r600_perfcounter_block *block;
391
392 block = lookup_counter(pc, query_types[i] - R600_QUERY_FIRST_PERFCOUNTER,
393 &base_gid, &sub_index);
394
395 sub_gid = sub_index / block->num_selectors;
396 sub_index = sub_index % block->num_selectors;
397
398 group = get_group_state(screen, query, block, sub_gid);
399 assert(group != NULL);
400
401 for (j = 0; j < group->num_counters; ++j) {
402 if (group->selectors[j] == sub_index)
403 break;
404 }
405
406 counter->base = group->result_base + j;
407 counter->stride = group->num_counters;
408
409 counter->qwords = 1;
410 if ((block->flags & R600_PC_BLOCK_SE) && group->se < 0)
411 counter->qwords = screen->info.max_se;
412 if (group->instance < 0)
413 counter->qwords *= block->num_instances;
414 }
415
416 if (!si_query_hw_init(screen, &query->b))
417 goto error;
418
419 return (struct pipe_query *)query;
420
421 error:
422 r600_pc_query_destroy(screen, &query->b.b);
423 return NULL;
424 }
425
r600_init_block_names(struct si_screen * screen,struct r600_perfcounter_block * block)426 static bool r600_init_block_names(struct si_screen *screen,
427 struct r600_perfcounter_block *block)
428 {
429 unsigned i, j, k;
430 unsigned groups_shader = 1, groups_se = 1, groups_instance = 1;
431 unsigned namelen;
432 char *groupname;
433 char *p;
434
435 if (block->flags & R600_PC_BLOCK_INSTANCE_GROUPS)
436 groups_instance = block->num_instances;
437 if (block->flags & R600_PC_BLOCK_SE_GROUPS)
438 groups_se = screen->info.max_se;
439 if (block->flags & R600_PC_BLOCK_SHADER)
440 groups_shader = screen->perfcounters->num_shader_types;
441
442 namelen = strlen(block->basename);
443 block->group_name_stride = namelen + 1;
444 if (block->flags & R600_PC_BLOCK_SHADER)
445 block->group_name_stride += 3;
446 if (block->flags & R600_PC_BLOCK_SE_GROUPS) {
447 assert(groups_se <= 10);
448 block->group_name_stride += 1;
449
450 if (block->flags & R600_PC_BLOCK_INSTANCE_GROUPS)
451 block->group_name_stride += 1;
452 }
453 if (block->flags & R600_PC_BLOCK_INSTANCE_GROUPS) {
454 assert(groups_instance <= 100);
455 block->group_name_stride += 2;
456 }
457
458 block->group_names = MALLOC(block->num_groups * block->group_name_stride);
459 if (!block->group_names)
460 return false;
461
462 groupname = block->group_names;
463 for (i = 0; i < groups_shader; ++i) {
464 const char *shader_suffix = screen->perfcounters->shader_type_suffixes[i];
465 unsigned shaderlen = strlen(shader_suffix);
466 for (j = 0; j < groups_se; ++j) {
467 for (k = 0; k < groups_instance; ++k) {
468 strcpy(groupname, block->basename);
469 p = groupname + namelen;
470
471 if (block->flags & R600_PC_BLOCK_SHADER) {
472 strcpy(p, shader_suffix);
473 p += shaderlen;
474 }
475
476 if (block->flags & R600_PC_BLOCK_SE_GROUPS) {
477 p += sprintf(p, "%d", j);
478 if (block->flags & R600_PC_BLOCK_INSTANCE_GROUPS)
479 *p++ = '_';
480 }
481
482 if (block->flags & R600_PC_BLOCK_INSTANCE_GROUPS)
483 p += sprintf(p, "%d", k);
484
485 groupname += block->group_name_stride;
486 }
487 }
488 }
489
490 assert(block->num_selectors <= 1000);
491 block->selector_name_stride = block->group_name_stride + 4;
492 block->selector_names = MALLOC(block->num_groups * block->num_selectors *
493 block->selector_name_stride);
494 if (!block->selector_names)
495 return false;
496
497 groupname = block->group_names;
498 p = block->selector_names;
499 for (i = 0; i < block->num_groups; ++i) {
500 for (j = 0; j < block->num_selectors; ++j) {
501 sprintf(p, "%s_%03d", groupname, j);
502 p += block->selector_name_stride;
503 }
504 groupname += block->group_name_stride;
505 }
506
507 return true;
508 }
509
si_get_perfcounter_info(struct si_screen * screen,unsigned index,struct pipe_driver_query_info * info)510 int si_get_perfcounter_info(struct si_screen *screen,
511 unsigned index,
512 struct pipe_driver_query_info *info)
513 {
514 struct r600_perfcounters *pc = screen->perfcounters;
515 struct r600_perfcounter_block *block;
516 unsigned base_gid, sub;
517
518 if (!pc)
519 return 0;
520
521 if (!info) {
522 unsigned bid, num_queries = 0;
523
524 for (bid = 0; bid < pc->num_blocks; ++bid) {
525 num_queries += pc->blocks[bid].num_selectors *
526 pc->blocks[bid].num_groups;
527 }
528
529 return num_queries;
530 }
531
532 block = lookup_counter(pc, index, &base_gid, &sub);
533 if (!block)
534 return 0;
535
536 if (!block->selector_names) {
537 if (!r600_init_block_names(screen, block))
538 return 0;
539 }
540 info->name = block->selector_names + sub * block->selector_name_stride;
541 info->query_type = R600_QUERY_FIRST_PERFCOUNTER + index;
542 info->max_value.u64 = 0;
543 info->type = PIPE_DRIVER_QUERY_TYPE_UINT64;
544 info->result_type = PIPE_DRIVER_QUERY_RESULT_TYPE_AVERAGE;
545 info->group_id = base_gid + sub / block->num_selectors;
546 info->flags = PIPE_DRIVER_QUERY_FLAG_BATCH;
547 if (sub > 0 && sub + 1 < block->num_selectors * block->num_groups)
548 info->flags |= PIPE_DRIVER_QUERY_FLAG_DONT_LIST;
549 return 1;
550 }
551
si_get_perfcounter_group_info(struct si_screen * screen,unsigned index,struct pipe_driver_query_group_info * info)552 int si_get_perfcounter_group_info(struct si_screen *screen,
553 unsigned index,
554 struct pipe_driver_query_group_info *info)
555 {
556 struct r600_perfcounters *pc = screen->perfcounters;
557 struct r600_perfcounter_block *block;
558
559 if (!pc)
560 return 0;
561
562 if (!info)
563 return pc->num_groups;
564
565 block = lookup_group(pc, &index);
566 if (!block)
567 return 0;
568
569 if (!block->group_names) {
570 if (!r600_init_block_names(screen, block))
571 return 0;
572 }
573 info->name = block->group_names + index * block->group_name_stride;
574 info->num_queries = block->num_selectors;
575 info->max_active_queries = block->num_counters;
576 return 1;
577 }
578
si_perfcounters_destroy(struct si_screen * sscreen)579 void si_perfcounters_destroy(struct si_screen *sscreen)
580 {
581 if (sscreen->perfcounters)
582 sscreen->perfcounters->cleanup(sscreen);
583 }
584
si_perfcounters_init(struct r600_perfcounters * pc,unsigned num_blocks)585 bool si_perfcounters_init(struct r600_perfcounters *pc,
586 unsigned num_blocks)
587 {
588 pc->blocks = CALLOC(num_blocks, sizeof(struct r600_perfcounter_block));
589 if (!pc->blocks)
590 return false;
591
592 pc->separate_se = debug_get_bool_option("RADEON_PC_SEPARATE_SE", false);
593 pc->separate_instance = debug_get_bool_option("RADEON_PC_SEPARATE_INSTANCE", false);
594
595 return true;
596 }
597
si_perfcounters_add_block(struct si_screen * sscreen,struct r600_perfcounters * pc,const char * name,unsigned flags,unsigned counters,unsigned selectors,unsigned instances,void * data)598 void si_perfcounters_add_block(struct si_screen *sscreen,
599 struct r600_perfcounters *pc,
600 const char *name, unsigned flags,
601 unsigned counters, unsigned selectors,
602 unsigned instances, void *data)
603 {
604 struct r600_perfcounter_block *block = &pc->blocks[pc->num_blocks];
605
606 assert(counters <= R600_QUERY_MAX_COUNTERS);
607
608 block->basename = name;
609 block->flags = flags;
610 block->num_counters = counters;
611 block->num_selectors = selectors;
612 block->num_instances = MAX2(instances, 1);
613 block->data = data;
614
615 if (pc->separate_se && (block->flags & R600_PC_BLOCK_SE))
616 block->flags |= R600_PC_BLOCK_SE_GROUPS;
617 if (pc->separate_instance && block->num_instances > 1)
618 block->flags |= R600_PC_BLOCK_INSTANCE_GROUPS;
619
620 if (block->flags & R600_PC_BLOCK_INSTANCE_GROUPS) {
621 block->num_groups = block->num_instances;
622 } else {
623 block->num_groups = 1;
624 }
625
626 if (block->flags & R600_PC_BLOCK_SE_GROUPS)
627 block->num_groups *= sscreen->info.max_se;
628 if (block->flags & R600_PC_BLOCK_SHADER)
629 block->num_groups *= pc->num_shader_types;
630
631 ++pc->num_blocks;
632 pc->num_groups += block->num_groups;
633 }
634
si_perfcounters_do_destroy(struct r600_perfcounters * pc)635 void si_perfcounters_do_destroy(struct r600_perfcounters *pc)
636 {
637 unsigned i;
638
639 for (i = 0; i < pc->num_blocks; ++i) {
640 FREE(pc->blocks[i].group_names);
641 FREE(pc->blocks[i].selector_names);
642 }
643 FREE(pc->blocks);
644 FREE(pc);
645 }
646