Lines Matching refs:th
160 stack_size = (size_t)TCR_PTR(thr->th.th_info.ds.ds_stacksize); in __kmp_get_global_thread_id()
161 stack_base = (char *)TCR_PTR(thr->th.th_info.ds.ds_stackbase); in __kmp_get_global_thread_id()
191 if (!TCR_4(other_threads[i]->th.th_info.ds.ds_stackgrow)) { in __kmp_get_global_thread_id()
195 stack_base = (char *)other_threads[i]->th.th_info.ds.ds_stackbase; in __kmp_get_global_thread_id()
197 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stackbase, stack_addr); in __kmp_get_global_thread_id()
198 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize, in __kmp_get_global_thread_id()
199 other_threads[i]->th.th_info.ds.ds_stacksize + stack_addr - in __kmp_get_global_thread_id()
202 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize, in __kmp_get_global_thread_id()
208 char *stack_end = (char *)other_threads[i]->th.th_info.ds.ds_stackbase; in __kmp_get_global_thread_id()
209 char *stack_beg = stack_end - other_threads[i]->th.th_info.ds.ds_stacksize; in __kmp_get_global_thread_id()
211 other_threads[i]->th.th_info.ds.ds_stacksize, in __kmp_get_global_thread_id()
260 void __kmp_check_stack_overlap(kmp_info_t *th) { in __kmp_check_stack_overlap() argument
268 stack_end = (char *)th->th.th_info.ds.ds_stackbase; in __kmp_check_stack_overlap()
269 stack_beg = stack_end - th->th.th_info.ds.ds_stacksize; in __kmp_check_stack_overlap()
271 gtid = __kmp_gtid_from_thread(th); in __kmp_check_stack_overlap()
275 gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize, in __kmp_check_stack_overlap()
277 (th->th.th_info.ds.ds_stackgrow) ? "initial" : "actual"); in __kmp_check_stack_overlap()
280 gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize, in __kmp_check_stack_overlap()
282 (th->th.th_info.ds.ds_stackgrow) ? "initial" : "actual"); in __kmp_check_stack_overlap()
288 gtid = __kmp_gtid_from_thread(th); in __kmp_check_stack_overlap()
293 stack_end = (char *)th->th.th_info.ds.ds_stackbase; in __kmp_check_stack_overlap()
294 stack_beg = stack_end - th->th.th_info.ds.ds_stacksize; in __kmp_check_stack_overlap()
300 if (f_th && f_th != th) { in __kmp_check_stack_overlap()
302 (char *)TCR_PTR(f_th->th.th_info.ds.ds_stackbase); in __kmp_check_stack_overlap()
304 other_stack_end - (size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize); in __kmp_check_stack_overlap()
312 (size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize), in __kmp_check_stack_overlap()
465 __kmp_print_storage_map_gtid(gtid, &thr->th.th_info, &thr->th.th_team, in __kmp_print_thread_storage_map()
468 __kmp_print_storage_map_gtid(gtid, &thr->th.th_local, &thr->th.th_pri_head, in __kmp_print_thread_storage_map()
472 gtid, &thr->th.th_bar[0], &thr->th.th_bar[bs_last_barrier], in __kmp_print_thread_storage_map()
475 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_plain_barrier], in __kmp_print_thread_storage_map()
476 &thr->th.th_bar[bs_plain_barrier + 1], in __kmp_print_thread_storage_map()
480 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_forkjoin_barrier], in __kmp_print_thread_storage_map()
481 &thr->th.th_bar[bs_forkjoin_barrier + 1], in __kmp_print_thread_storage_map()
486 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_reduction_barrier], in __kmp_print_thread_storage_map()
487 &thr->th.th_bar[bs_reduction_barrier + 1], in __kmp_print_thread_storage_map()
572 kmp_info_t *th = __kmp_threads[i]; in __kmp_reset_locks_on_process_detach() local
573 if (th == NULL) in __kmp_reset_locks_on_process_detach()
575 int gtid = th->th.th_info.ds.ds_gtid; in __kmp_reset_locks_on_process_detach()
581 int alive = __kmp_is_thread_alive(th, &exit_val); in __kmp_reset_locks_on_process_detach()
672 if (__kmp_threads[gtid]->th.th_root->r.r_active) in __kmp_parallel_deo()
698 if (__kmp_threads[gtid]->th.th_root->r.r_active) in __kmp_parallel_dxo()
719 kmp_info_t *th; in __kmp_enter_single() local
726 th = __kmp_threads[gtid]; in __kmp_enter_single()
727 team = th->th.th_team; in __kmp_enter_single()
730 th->th.th_ident = id_ref; in __kmp_enter_single()
735 kmp_int32 old_this = th->th.th_local.this_construct; in __kmp_enter_single()
737 ++th->th.th_local.this_construct; in __kmp_enter_single()
743 th->th.th_local.this_construct); in __kmp_enter_single()
747 KMP_MASTER_GTID(gtid) && th->th.th_teams_microtask == NULL && in __kmp_enter_single()
881 int cg_nthreads = this_thr->th.th_cg_roots->cg_nthreads; in __kmp_reserve_threads()
882 int max_cg_threads = this_thr->th.th_cg_roots->cg_thread_limit; in __kmp_reserve_threads()
976 master_th->th.th_info.ds.ds_tid = 0; in __kmp_fork_team_threads()
977 master_th->th.th_team = team; in __kmp_fork_team_threads()
978 master_th->th.th_team_nproc = team->t.t_nproc; in __kmp_fork_team_threads()
979 master_th->th.th_team_master = master_th; in __kmp_fork_team_threads()
980 master_th->th.th_team_serialized = FALSE; in __kmp_fork_team_threads()
981 master_th->th.th_dispatch = &team->t.t_dispatch[0]; in __kmp_fork_team_threads()
986 kmp_hot_team_ptr_t *hot_teams = master_th->th.th_hot_teams; in __kmp_fork_team_threads()
990 if (master_th->th.th_teams_microtask) { // are we inside the teams? in __kmp_fork_team_threads()
991 if (master_th->th.th_teams_size.nteams > 1) { in __kmp_fork_team_threads()
996 master_th->th.th_teams_level == team->t.t_level) { in __kmp_fork_team_threads()
1031 KMP_DEBUG_ASSERT(thr->th.th_team == team); in __kmp_fork_team_threads()
1039 thr->th.th_teams_microtask = master_th->th.th_teams_microtask; in __kmp_fork_team_threads()
1040 thr->th.th_teams_level = master_th->th.th_teams_level; in __kmp_fork_team_threads()
1041 thr->th.th_teams_size = master_th->th.th_teams_size; in __kmp_fork_team_threads()
1044 kmp_balign_t *balign = team->t.t_threads[i]->th.th_bar; in __kmp_fork_team_threads()
1063 if (thr->th.th_prev_num_threads != team->t.t_nproc || in __kmp_fork_team_threads()
1064 thr->th.th_prev_level != team->t.t_level) { in __kmp_fork_team_threads()
1157 serial_team = this_thr->th.th_serial_team; in __kmp_serialized_parallel()
1165 this_thr->th.th_task_team == in __kmp_serialized_parallel()
1166 this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state]); in __kmp_serialized_parallel()
1167 KMP_DEBUG_ASSERT(serial_team->t.t_task_team[this_thr->th.th_task_state] == in __kmp_serialized_parallel()
1171 global_tid, this_thr->th.th_task_team, this_thr->th.th_team)); in __kmp_serialized_parallel()
1172 this_thr->th.th_task_team = NULL; in __kmp_serialized_parallel()
1175 kmp_proc_bind_t proc_bind = this_thr->th.th_set_proc_bind; in __kmp_serialized_parallel()
1176 if (this_thr->th.th_current_task->td_icvs.proc_bind == proc_bind_false) { in __kmp_serialized_parallel()
1181 proc_bind = this_thr->th.th_current_task->td_icvs.proc_bind; in __kmp_serialized_parallel()
1184 this_thr->th.th_set_proc_bind = proc_bind_default; in __kmp_serialized_parallel()
1191 this_thr->th.ompt_thread_info.state != ompt_state_overhead) { in __kmp_serialized_parallel()
1208 if (this_thr->th.th_team != serial_team) { in __kmp_serialized_parallel()
1210 int level = this_thr->th.th_team->t.t_level; in __kmp_serialized_parallel()
1220 __kmp_allocate_team(this_thr->th.th_root, 1, 1, in __kmp_serialized_parallel()
1224 proc_bind, &this_thr->th.th_current_task->td_icvs, in __kmp_serialized_parallel()
1231 new_team->t.t_parent = this_thr->th.th_team; in __kmp_serialized_parallel()
1233 this_thr->th.th_serial_team = serial_team; in __kmp_serialized_parallel()
1253 KMP_DEBUG_ASSERT(this_thr->th.th_team != serial_team); in __kmp_serialized_parallel()
1257 serial_team->t.t_parent = this_thr->th.th_team; in __kmp_serialized_parallel()
1258 serial_team->t.t_sched.sched = this_thr->th.th_team->t.t_sched.sched; in __kmp_serialized_parallel()
1259 this_thr->th.th_team = serial_team; in __kmp_serialized_parallel()
1260 serial_team->t.t_master_tid = this_thr->th.th_info.ds.ds_tid; in __kmp_serialized_parallel()
1263 this_thr->th.th_current_task)); in __kmp_serialized_parallel()
1264 KMP_ASSERT(this_thr->th.th_current_task->td_flags.executing == 1); in __kmp_serialized_parallel()
1265 this_thr->th.th_current_task->td_flags.executing = 0; in __kmp_serialized_parallel()
1272 copy_icvs(&this_thr->th.th_current_task->td_icvs, in __kmp_serialized_parallel()
1273 &this_thr->th.th_current_task->td_parent->td_icvs); in __kmp_serialized_parallel()
1278 this_thr->th.th_current_task->td_icvs.nproc = in __kmp_serialized_parallel()
1284 this_thr->th.th_current_task->td_icvs.proc_bind = in __kmp_serialized_parallel()
1291 this_thr->th.th_info.ds.ds_tid = 0; in __kmp_serialized_parallel()
1294 this_thr->th.th_team_nproc = 1; in __kmp_serialized_parallel()
1295 this_thr->th.th_team_master = this_thr; in __kmp_serialized_parallel()
1296 this_thr->th.th_team_serialized = 1; in __kmp_serialized_parallel()
1300 serial_team->t.t_def_allocator = this_thr->th.th_def_allocator; // save in __kmp_serialized_parallel()
1311 this_thr->th.th_dispatch = serial_team->t.t_dispatch; in __kmp_serialized_parallel()
1318 KMP_DEBUG_ASSERT(this_thr->th.th_team == serial_team); in __kmp_serialized_parallel()
1322 this_thr->th.th_team_serialized = serial_team->t.t_serialized; in __kmp_serialized_parallel()
1325 int level = this_thr->th.th_team->t.t_level; in __kmp_serialized_parallel()
1329 this_thr->th.th_current_task->td_icvs.nproc = in __kmp_serialized_parallel()
1346 this_thr->th.th_dispatch = serial_team->t.t_dispatch; in __kmp_serialized_parallel()
1355 if (this_thr->th.th_prev_level != serial_team->t.t_level || in __kmp_serialized_parallel()
1356 this_thr->th.th_prev_num_threads != 1) { in __kmp_serialized_parallel()
1359 this_thr->th.th_prev_level = serial_team->t.t_level; in __kmp_serialized_parallel()
1360 this_thr->th.th_prev_num_threads = 1; in __kmp_serialized_parallel()
1369 this_thr->th.ompt_thread_info.state != ompt_state_overhead) { in __kmp_serialized_parallel()
1390 this_thr->th.ompt_thread_info.state = ompt_state_work_parallel; in __kmp_serialized_parallel()
1443 parent_team = master_th->th.th_team; in __kmp_fork_call()
1444 master_tid = master_th->th.th_info.ds.ds_tid; in __kmp_fork_call()
1445 master_this_cons = master_th->th.th_local.this_construct; in __kmp_fork_call()
1446 root = master_th->th.th_root; in __kmp_fork_call()
1448 master_set_numthreads = master_th->th.th_set_nproc; in __kmp_fork_call()
1469 teams_level = master_th->th.th_teams_level; in __kmp_fork_call()
1471 p_hot_teams = &master_th->th.th_hot_teams; in __kmp_fork_call()
1495 master_th->th.ompt_thread_info.state = ompt_state_overhead; in __kmp_fork_call()
1499 master_th->th.th_ident = loc; in __kmp_fork_call()
1501 if (master_th->th.th_teams_microtask && ap && in __kmp_fork_call()
1513 if (parent_team == master_th->th.th_serial_team) { in __kmp_fork_call()
1551 master_th->th.ompt_thread_info.state = ompt_state_work_parallel; in __kmp_fork_call()
1588 master_th->th.ompt_thread_info.state = ompt_state_overhead; in __kmp_fork_call()
1599 parent_team->t.t_def_allocator = master_th->th.th_def_allocator; // save in __kmp_fork_call()
1612 if (master_set_numthreads < master_th->th.th_teams_size.nth) { in __kmp_fork_call()
1617 other_threads[i]->th.th_team_nproc = master_set_numthreads; in __kmp_fork_call()
1621 master_th->th.th_set_nproc = 0; in __kmp_fork_call()
1638 && master_th->th.th_teams_size.nteams == 1) { in __kmp_fork_call()
1640 master_th->th.th_frame_time = tmp_time; in __kmp_fork_call()
1678 KMP_DEBUG_ASSERT(master_th->th.th_task_team == in __kmp_fork_call()
1679 parent_team->t.t_task_team[master_th->th.th_task_state]); in __kmp_fork_call()
1684 master_th->th.th_current_task->td_icvs.max_active_levels) { in __kmp_fork_call()
1729 master_th->th.th_set_nproc = 0; in __kmp_fork_call()
1749 master_th->th.th_serial_team->t.t_ident = loc; in __kmp_fork_call()
1752 master_th->th.th_serial_team->t.t_level--; in __kmp_fork_call()
1782 master_th->th.ompt_thread_info.state = ompt_state_work_parallel; in __kmp_fork_call()
1817 master_th->th.ompt_thread_info.state = ompt_state_overhead; in __kmp_fork_call()
1821 KMP_DEBUG_ASSERT(master_th->th.th_team == in __kmp_fork_call()
1822 master_th->th.th_serial_team); in __kmp_fork_call()
1823 team = master_th->th.th_team; in __kmp_fork_call()
1856 master_th->th.ompt_thread_info.state = ompt_state_overhead; in __kmp_fork_call()
1892 master_th->th.ompt_thread_info.state = ompt_state_work_parallel; in __kmp_fork_call()
1927 master_th->th.ompt_thread_info.state = ompt_state_overhead; in __kmp_fork_call()
1960 master_th->th.th_current_task, in __kmp_fork_call()
1961 master_th->th.th_current_task->td_icvs.max_active_levels)); in __kmp_fork_call()
1965 master_th->th.th_current_task->td_flags.executing = 0; in __kmp_fork_call()
1967 if (!master_th->th.th_teams_microtask || level > teams_level) { in __kmp_fork_call()
1973 int nthreads_icv = master_th->th.th_current_task->td_icvs.nproc; in __kmp_fork_call()
1982 kmp_proc_bind_t proc_bind = master_th->th.th_set_proc_bind; in __kmp_fork_call()
1985 if (master_th->th.th_current_task->td_icvs.proc_bind == proc_bind_false) { in __kmp_fork_call()
1991 proc_bind = master_th->th.th_current_task->td_icvs.proc_bind; in __kmp_fork_call()
1999 master_th->th.th_current_task->td_icvs.proc_bind)) { in __kmp_fork_call()
2005 master_th->th.th_set_proc_bind = proc_bind_default; in __kmp_fork_call()
2009 copy_icvs(&new_icvs, &master_th->th.th_current_task->td_icvs); in __kmp_fork_call()
2034 &master_th->th.th_current_task->td_icvs, in __kmp_fork_call()
2052 if (!master_th->th.th_teams_microtask || level > teams_level) { in __kmp_fork_call()
2069 KMP_CHECK_UPDATE(team->t.t_def_allocator, master_th->th.th_def_allocator); in __kmp_fork_call()
2077 KMP_DEBUG_ASSERT(master_th->th.th_task_team == in __kmp_fork_call()
2078 parent_team->t.t_task_team[master_th->th.th_task_state]); in __kmp_fork_call()
2082 master_th->th.th_task_team, parent_team, in __kmp_fork_call()
2083 team->t.t_task_team[master_th->th.th_task_state], team)); in __kmp_fork_call()
2085 if (active_level || master_th->th.th_task_team) { in __kmp_fork_call()
2087 KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack); in __kmp_fork_call()
2088 if (master_th->th.th_task_state_top >= in __kmp_fork_call()
2089 master_th->th.th_task_state_stack_sz) { // increase size in __kmp_fork_call()
2090 kmp_uint32 new_size = 2 * master_th->th.th_task_state_stack_sz; in __kmp_fork_call()
2094 for (i = 0; i < master_th->th.th_task_state_stack_sz; ++i) { in __kmp_fork_call()
2095 new_stack[i] = master_th->th.th_task_state_memo_stack[i]; in __kmp_fork_call()
2097 for (i = master_th->th.th_task_state_stack_sz; i < new_size; in __kmp_fork_call()
2101 old_stack = master_th->th.th_task_state_memo_stack; in __kmp_fork_call()
2102 master_th->th.th_task_state_memo_stack = new_stack; in __kmp_fork_call()
2103 master_th->th.th_task_state_stack_sz = new_size; in __kmp_fork_call()
2107 master_th->th in __kmp_fork_call()
2108 .th_task_state_memo_stack[master_th->th.th_task_state_top] = in __kmp_fork_call()
2109 master_th->th.th_task_state; in __kmp_fork_call()
2110 master_th->th.th_task_state_top++; in __kmp_fork_call()
2112 if (master_th->th.th_hot_teams && in __kmp_fork_call()
2114 team == master_th->th.th_hot_teams[active_level].hot_team) { in __kmp_fork_call()
2116 master_th->th.th_task_state = in __kmp_fork_call()
2117 master_th->th in __kmp_fork_call()
2118 .th_task_state_memo_stack[master_th->th.th_task_state_top]; in __kmp_fork_call()
2121 master_th->th.th_task_state = 0; in __kmp_fork_call()
2127 KMP_DEBUG_ASSERT((master_th->th.th_task_team == NULL) || in __kmp_fork_call()
2165 &master_th->th.th_current_task->td_icvs, loc); in __kmp_fork_call()
2168 master_th->th.ompt_thread_info.state = ompt_state_work_parallel; in __kmp_fork_call()
2175 && !master_th->th.th_teams_microtask) { // not in teams construct in __kmp_fork_call()
2184 master_th->th.th_frame_time = tmp_time; in __kmp_fork_call()
2199 KMP_DEBUG_ASSERT(team == __kmp_threads[gtid]->th.th_team); in __kmp_fork_call()
2259 master_th->th.ompt_thread_info.state = ompt_state_overhead; in __kmp_fork_call()
2270 thread->th.ompt_thread_info.state = in __kmp_join_restore_state()
2307 root = master_th->th.th_root; in __kmp_join_call()
2308 team = master_th->th.th_team; in __kmp_join_call()
2311 master_th->th.th_ident = loc; in __kmp_join_call()
2320 master_th->th.ompt_thread_info.state = ompt_state_overhead; in __kmp_join_call()
2329 team->t.t_task_team[master_th->th.th_task_state], in __kmp_join_call()
2330 master_th->th.th_task_team)); in __kmp_join_call()
2331 KMP_DEBUG_ASSERT(master_th->th.th_task_team == in __kmp_join_call()
2332 team->t.t_task_team[master_th->th.th_task_state]); in __kmp_join_call()
2337 if (master_th->th.th_teams_microtask) { in __kmp_join_call()
2340 int tlevel = master_th->th.th_teams_level; in __kmp_join_call()
2370 master_th->th.th_task_state = in __kmp_join_call()
2388 (!master_th->th.th_teams_microtask || /* not in teams construct */ in __kmp_join_call()
2389 master_th->th.th_teams_size.nteams == 1)) { in __kmp_join_call()
2390 master_th->th.th_ident = loc; in __kmp_join_call()
2396 master_th->th.th_frame_time, 0, loc, in __kmp_join_call()
2397 master_th->th.th_team_nproc, 1); in __kmp_join_call()
2404 if (master_th->th.th_teams_microtask && !exit_teams && in __kmp_join_call()
2406 team->t.t_level == master_th->th.th_teams_level + 1) { in __kmp_join_call()
2435 if (master_th->th.th_team_nproc < master_th->th.th_teams_size.nth) { in __kmp_join_call()
2436 int old_num = master_th->th.th_team_nproc; in __kmp_join_call()
2437 int new_num = master_th->th.th_teams_size.nth; in __kmp_join_call()
2441 other_threads[i]->th.th_team_nproc = new_num; in __kmp_join_call()
2447 kmp_balign_t *balign = other_threads[i]->th.th_bar; in __kmp_join_call()
2457 other_threads[i]->th.th_task_state = master_th->th.th_task_state; in __kmp_join_call()
2473 master_th->th.th_info.ds.ds_tid = team->t.t_master_tid; in __kmp_join_call()
2474 master_th->th.th_local.this_construct = team->t.t_master_this_cons; in __kmp_join_call()
2476 master_th->th.th_dispatch = &parent_team->t.t_dispatch[team->t.t_master_tid]; in __kmp_join_call()
2483 if (!master_th->th.th_teams_microtask || in __kmp_join_call()
2484 team->t.t_level > master_th->th.th_teams_level) { in __kmp_join_call()
2513 master_th->th.th_first_place = team->t.t_first_place; in __kmp_join_call()
2514 master_th->th.th_last_place = team->t.t_last_place; in __kmp_join_call()
2516 master_th->th.th_def_allocator = team->t.t_def_allocator; in __kmp_join_call()
2531 master_th->th.th_team = parent_team; in __kmp_join_call()
2532 master_th->th.th_team_nproc = parent_team->t.t_nproc; in __kmp_join_call()
2533 master_th->th.th_team_master = parent_team->t.t_threads[0]; in __kmp_join_call()
2534 master_th->th.th_team_serialized = parent_team->t.t_serialized; in __kmp_join_call()
2538 parent_team != master_th->th.th_serial_team && in __kmp_join_call()
2541 master_th->th.th_serial_team USE_NESTED_HOT_ARG(NULL)); in __kmp_join_call()
2542 master_th->th.th_serial_team = parent_team; in __kmp_join_call()
2546 if (master_th->th.th_task_state_top > in __kmp_join_call()
2548 KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack); in __kmp_join_call()
2550 master_th->th.th_task_state_memo_stack[master_th->th.th_task_state_top] = in __kmp_join_call()
2551 master_th->th.th_task_state; in __kmp_join_call()
2552 --master_th->th.th_task_state_top; // pop in __kmp_join_call()
2554 master_th->th.th_task_state = in __kmp_join_call()
2555 master_th->th in __kmp_join_call()
2556 .th_task_state_memo_stack[master_th->th.th_task_state_top]; in __kmp_join_call()
2559 master_th->th.th_task_team = in __kmp_join_call()
2560 parent_team->t.t_task_team[master_th->th.th_task_state]; in __kmp_join_call()
2563 __kmp_gtid_from_thread(master_th), master_th->th.th_task_team, in __kmp_join_call()
2570 master_th->th.th_current_task->td_flags.executing = 1; in __kmp_join_call()
2593 if (thread->th.th_team != thread->th.th_serial_team) { in __kmp_save_internal_controls()
2596 if (thread->th.th_team->t.t_serialized > 1) { in __kmp_save_internal_controls()
2599 if (thread->th.th_team->t.t_control_stack_top == NULL) { in __kmp_save_internal_controls()
2602 if (thread->th.th_team->t.t_control_stack_top->serial_nesting_level != in __kmp_save_internal_controls()
2603 thread->th.th_team->t.t_serialized) { in __kmp_save_internal_controls()
2612 copy_icvs(control, &thread->th.th_current_task->td_icvs); in __kmp_save_internal_controls()
2614 control->serial_nesting_level = thread->th.th_team->t.t_serialized; in __kmp_save_internal_controls()
2616 control->next = thread->th.th_team->t.t_control_stack_top; in __kmp_save_internal_controls()
2617 thread->th.th_team->t.t_control_stack_top = control; in __kmp_save_internal_controls()
2637 if (thread->th.th_current_task->td_icvs.nproc == new_nth) in __kmp_set_num_threads()
2647 root = thread->th.th_root; in __kmp_set_num_threads()
2665 hot_team->t.t_threads[f]->th.th_task_team = NULL; in __kmp_set_num_threads()
2672 if (thread->th.th_hot_teams) { in __kmp_set_num_threads()
2673 KMP_DEBUG_ASSERT(hot_team == thread->th.th_hot_teams[0].hot_team); in __kmp_set_num_threads()
2674 thread->th.th_hot_teams[0].hot_team_nth = new_nth; in __kmp_set_num_threads()
2683 hot_team->t.t_threads[f]->th.th_team_nproc = new_nth; in __kmp_set_num_threads()
2743 KMP_DEBUG_ASSERT(thread->th.th_current_task); in __kmp_get_max_active_levels()
2746 gtid, thread->th.th_current_task, in __kmp_get_max_active_levels()
2747 thread->th.th_current_task->td_icvs.max_active_levels)); in __kmp_get_max_active_levels()
2748 return thread->th.th_current_task->td_icvs.max_active_levels; in __kmp_get_max_active_levels()
2789 thread->th.th_current_task->td_icvs.sched.r_sched_type = kmp_sch_static; in __kmp_set_schedule()
2791 thread->th.th_current_task->td_icvs.sched.r_sched_type = in __kmp_set_schedule()
2797 thread->th.th_current_task->td_icvs.sched.r_sched_type = in __kmp_set_schedule()
2802 orig_kind, &(thread->th.th_current_task->td_icvs.sched.r_sched_type)); in __kmp_set_schedule()
2805 thread->th.th_current_task->td_icvs.sched.chunk = KMP_DEFAULT_CHUNK; in __kmp_set_schedule()
2807 thread->th.th_current_task->td_icvs.sched.chunk = chunk; in __kmp_set_schedule()
2821 th_type = thread->th.th_current_task->td_icvs.sched.r_sched_type; in __kmp_get_schedule()
2857 *chunk = thread->th.th_current_task->td_icvs.sched.chunk; in __kmp_get_schedule()
2875 team = thr->th.th_team; in __kmp_get_ancestor_thread_num()
2880 if (thr->th.th_teams_microtask) { in __kmp_get_ancestor_thread_num()
2882 int tlevel = thr->th.th_teams_level; // the level of the teams construct in __kmp_get_ancestor_thread_num()
2933 team = thr->th.th_team; in __kmp_get_team_size()
2938 if (thr->th.th_teams_microtask) { in __kmp_get_team_size()
2940 int tlevel = thr->th.th_teams_level; // the level of the teams construct in __kmp_get_team_size()
3147 copy_icvs(&gx_icvs, &team->t.t_threads[0]->th.th_current_task->td_icvs); in __kmp_get_x_global_icvs()
3310 __kmp_printf("%2d %p\n", thread->th.th_info.ds.ds_gtid, thread); in __kmp_print_structure_thread()
3351 __kmp_printf(" Our Root: %p\n", thread->th.th_root); in __kmp_print_structure()
3352 __kmp_print_structure_team(" Our Team: ", thread->th.th_team); in __kmp_print_structure()
3354 thread->th.th_serial_team); in __kmp_print_structure()
3355 __kmp_printf(" Threads: %2d\n", thread->th.th_team_nproc); in __kmp_print_structure()
3357 thread->th.th_team_master); in __kmp_print_structure()
3358 __kmp_printf(" Serialized?: %2d\n", thread->th.th_team_serialized); in __kmp_print_structure()
3359 __kmp_printf(" Set NProc: %2d\n", thread->th.th_set_nproc); in __kmp_print_structure()
3360 __kmp_printf(" Set Proc Bind: %2d\n", thread->th.th_set_proc_bind); in __kmp_print_structure()
3362 thread->th.th_next_pool); in __kmp_print_structure()
3364 __kmp_print_structure_team_accum(list, thread->th.th_team); in __kmp_print_structure()
3365 __kmp_print_structure_team_accum(list, thread->th.th_serial_team); in __kmp_print_structure()
3455 unsigned x = thread->th.th_x; in __kmp_get_random()
3458 thread->th.th_x = x * thread->th.th_a + 1; in __kmp_get_random()
3461 thread->th.th_info.ds.ds_tid, r)); in __kmp_get_random()
3468 unsigned seed = thread->th.th_info.ds.ds_tid; in __kmp_init_random()
3470 thread->th.th_a = in __kmp_init_random()
3472 thread->th.th_x = (seed + 1) * thread->th.th_a + 1; in __kmp_init_random()
3474 ("__kmp_init_random: THREAD: %u; A: %u\n", seed, thread->th.th_a)); in __kmp_init_random()
3703 root_thread->th.th_info.ds.ds_gtid = gtid; in __kmp_register_root()
3705 root_thread->th.ompt_thread_info.thread_data = ompt_data_none; in __kmp_register_root()
3707 root_thread->th.th_root = root; in __kmp_register_root()
3709 root_thread->th.th_cons = __kmp_allocate_cons_stack(gtid); in __kmp_register_root()
3716 KMP_DEBUG_ASSERT(root_thread->th.th_local.bget_data == NULL); in __kmp_register_root()
3723 if (!root_thread->th.th_serial_team) { in __kmp_register_root()
3726 root_thread->th.th_serial_team = __kmp_allocate_team( in __kmp_register_root()
3733 KMP_ASSERT(root_thread->th.th_serial_team); in __kmp_register_root()
3735 root_thread->th.th_serial_team)); in __kmp_register_root()
3742 root_thread->th.th_serial_team->t.t_threads[0] = root_thread; in __kmp_register_root()
3744 root_thread->th.th_serial_team->t.t_serialized = 0; in __kmp_register_root()
3772 root_thread->th.th_bar[b].bb.b_arrived = KMP_INIT_BARRIER_STATE; in __kmp_register_root()
3774 root_thread->th.th_bar[b].bb.b_worker_arrived = 0; in __kmp_register_root()
3782 root_thread->th.th_current_place = KMP_PLACE_UNDEFINED; in __kmp_register_root()
3783 root_thread->th.th_new_place = KMP_PLACE_UNDEFINED; in __kmp_register_root()
3784 root_thread->th.th_first_place = KMP_PLACE_UNDEFINED; in __kmp_register_root()
3785 root_thread->th.th_last_place = KMP_PLACE_UNDEFINED; in __kmp_register_root()
3790 root_thread->th.th_def_allocator = __kmp_def_allocator; in __kmp_register_root()
3791 root_thread->th.th_prev_level = 0; in __kmp_register_root()
3792 root_thread->th.th_prev_num_threads = 1; in __kmp_register_root()
3802 root_thread->th.th_cg_roots = tmp; in __kmp_register_root()
3839 kmp_hot_team_ptr_t *hot_teams = thr->th.th_hot_teams; in __kmp_free_hot_teams()
3849 kmp_info_t *th = team->t.t_threads[i]; in __kmp_free_hot_teams() local
3850 n += __kmp_free_hot_teams(root, th, level + 1, max_level); in __kmp_free_hot_teams()
3851 if (i > 0 && th->th.th_hot_teams) { in __kmp_free_hot_teams()
3852 __kmp_free(th->th.th_hot_teams); in __kmp_free_hot_teams()
3853 th->th.th_hot_teams = NULL; in __kmp_free_hot_teams()
3881 kmp_info_t *th = hot_team->t.t_threads[i]; in __kmp_reset_root() local
3883 n += __kmp_free_hot_teams(root, th, 1, __kmp_hot_teams_max_level); in __kmp_reset_root()
3885 if (th->th.th_hot_teams) { in __kmp_reset_root()
3886 __kmp_free(th->th.th_hot_teams); in __kmp_reset_root()
3887 th->th.th_hot_teams = NULL; in __kmp_reset_root()
3906 (LPVOID) & (root->r.r_uber_thread->th), in __kmp_reset_root()
3907 root->r.r_uber_thread->th.th_info.ds.ds_thread)); in __kmp_reset_root()
3908 __kmp_free_handle(root->r.r_uber_thread->th.th_info.ds.ds_thread); in __kmp_reset_root()
3921 &(root->r.r_uber_thread->th.ompt_thread_info.thread_data)); in __kmp_reset_root()
3927 i = root->r.r_uber_thread->th.th_cg_roots->cg_nthreads--; in __kmp_reset_root()
3930 root->r.r_uber_thread, root->r.r_uber_thread->th.th_cg_roots, in __kmp_reset_root()
3931 root->r.r_uber_thread->th.th_cg_roots->cg_nthreads)); in __kmp_reset_root()
3935 root->r.r_uber_thread->th.th_cg_roots->cg_root); in __kmp_reset_root()
3936 KMP_DEBUG_ASSERT(root->r.r_uber_thread->th.th_cg_roots->up == NULL); in __kmp_reset_root()
3937 __kmp_free(root->r.r_uber_thread->th.th_cg_roots); in __kmp_reset_root()
3938 root->r.r_uber_thread->th.th_cg_roots = NULL; in __kmp_reset_root()
3968 KMP_ASSERT(root == __kmp_threads[gtid]->th.th_root); in __kmp_unregister_root_current_thread()
3974 kmp_team_t *team = thread->th.th_team; in __kmp_unregister_root_current_thread()
3975 kmp_task_team_t *task_team = thread->th.th_task_team; in __kmp_unregister_root_current_thread()
3981 thread->th.ompt_thread_info.state = ompt_state_undefined; in __kmp_unregister_root_current_thread()
4006 KMP_ASSERT(root == __kmp_threads[gtid]->th.th_root); in __kmp_unregister_root_other_thread()
4022 kmp_team_t *steam = this_thr->th.th_serial_team; in __kmp_task_info()
4023 kmp_team_t *team = this_thr->th.th_team; in __kmp_task_info()
4028 gtid, tid, this_thr, team, steam, this_thr->th.th_current_task, in __kmp_task_info()
4043 KMP_DEBUG_ASSERT(this_thr->th.th_serial_team); in __kmp_initialize_info()
4048 KMP_DEBUG_ASSERT(master->th.th_root); in __kmp_initialize_info()
4052 TCW_SYNC_PTR(this_thr->th.th_team, team); in __kmp_initialize_info()
4054 this_thr->th.th_info.ds.ds_tid = tid; in __kmp_initialize_info()
4055 this_thr->th.th_set_nproc = 0; in __kmp_initialize_info()
4059 this_thr->th.th_reap_state = KMP_NOT_SAFE_TO_REAP; in __kmp_initialize_info()
4061 this_thr->th.th_reap_state = KMP_SAFE_TO_REAP; in __kmp_initialize_info()
4062 this_thr->th.th_set_proc_bind = proc_bind_default; in __kmp_initialize_info()
4064 this_thr->th.th_new_place = this_thr->th.th_current_place; in __kmp_initialize_info()
4066 this_thr->th.th_root = master->th.th_root; in __kmp_initialize_info()
4069 this_thr->th.th_team_nproc = team->t.t_nproc; in __kmp_initialize_info()
4070 this_thr->th.th_team_master = master; in __kmp_initialize_info()
4071 this_thr->th.th_team_serialized = team->t.t_serialized; in __kmp_initialize_info()
4072 TCW_PTR(this_thr->th.th_sleep_loc, NULL); in __kmp_initialize_info()
4077 tid, gtid, this_thr, this_thr->th.th_current_task)); in __kmp_initialize_info()
4079 __kmp_init_implicit_task(this_thr->th.th_team_master->th.th_ident, this_thr, in __kmp_initialize_info()
4083 tid, gtid, this_thr, this_thr->th.th_current_task)); in __kmp_initialize_info()
4088 this_thr->th.th_dispatch = &team->t.t_dispatch[tid]; in __kmp_initialize_info()
4090 this_thr->th.th_local.this_construct = 0; in __kmp_initialize_info()
4092 if (!this_thr->th.th_pri_common) { in __kmp_initialize_info()
4093 this_thr->th.th_pri_common = in __kmp_initialize_info()
4097 gtid, this_thr->th.th_pri_common, this_thr->th.th_pri_common + 1, in __kmp_initialize_info()
4100 this_thr->th.th_pri_head = NULL; in __kmp_initialize_info()
4104 this_thr->th.th_cg_roots != master->th.th_cg_roots) { // CG root not set in __kmp_initialize_info()
4106 KMP_DEBUG_ASSERT(master->th.th_cg_roots); in __kmp_initialize_info()
4107 kmp_cg_root_t *tmp = this_thr->th.th_cg_roots; in __kmp_initialize_info()
4118 this_thr->th.th_cg_roots = master->th.th_cg_roots; in __kmp_initialize_info()
4120 this_thr->th.th_cg_roots->cg_nthreads++; in __kmp_initialize_info()
4123 this_thr, this_thr->th.th_cg_roots, in __kmp_initialize_info()
4124 this_thr->th.th_cg_roots->cg_root, in __kmp_initialize_info()
4125 this_thr->th.th_cg_roots->cg_nthreads)); in __kmp_initialize_info()
4126 this_thr->th.th_current_task->td_icvs.thread_limit = in __kmp_initialize_info()
4127 this_thr->th.th_cg_roots->cg_thread_limit; in __kmp_initialize_info()
4132 volatile kmp_disp_t *dispatch = this_thr->th.th_dispatch; in __kmp_initialize_info()
4170 this_thr->th.th_next_pool = NULL; in __kmp_initialize_info()
4172 if (!this_thr->th.th_task_state_memo_stack) { in __kmp_initialize_info()
4174 this_thr->th.th_task_state_memo_stack = in __kmp_initialize_info()
4176 this_thr->th.th_task_state_top = 0; in __kmp_initialize_info()
4177 this_thr->th.th_task_state_stack_sz = 4; in __kmp_initialize_info()
4178 for (i = 0; i < this_thr->th.th_task_state_stack_sz; in __kmp_initialize_info()
4180 this_thr->th.th_task_state_memo_stack[i] = 0; in __kmp_initialize_info()
4183 KMP_DEBUG_ASSERT(!this_thr->th.th_spin_here); in __kmp_initialize_info()
4184 KMP_DEBUG_ASSERT(this_thr->th.th_next_waiting == 0); in __kmp_initialize_info()
4210 __kmp_thread_pool = (volatile kmp_info_t *)new_thr->th.th_next_pool; in __kmp_allocate_thread()
4214 TCW_4(new_thr->th.th_in_pool, FALSE); in __kmp_allocate_thread()
4217 if (new_thr->th.th_active_in_pool == TRUE) { in __kmp_allocate_thread()
4218 KMP_DEBUG_ASSERT(new_thr->th.th_active == TRUE); in __kmp_allocate_thread()
4220 new_thr->th.th_active_in_pool = FALSE; in __kmp_allocate_thread()
4225 __kmp_get_gtid(), new_thr->th.th_info.ds.ds_gtid)); in __kmp_allocate_thread()
4226 KMP_ASSERT(!new_thr->th.th_team); in __kmp_allocate_thread()
4231 new_thr->th.th_info.ds.ds_gtid); in __kmp_allocate_thread()
4232 KMP_DEBUG_ASSERT(new_thr->th.th_serial_team); in __kmp_allocate_thread()
4236 new_thr->th.th_task_state = 0; in __kmp_allocate_thread()
4237 new_thr->th.th_task_state_top = 0; in __kmp_allocate_thread()
4238 new_thr->th.th_task_state_stack_sz = 4; in __kmp_allocate_thread()
4254 kmp_balign_t *balign = new_thr->th.th_bar; in __kmp_allocate_thread()
4260 __kmp_get_gtid(), new_thr, new_thr->th.th_info.ds.ds_gtid)); in __kmp_allocate_thread()
4315 &new_thr->th.th_sleep_loc, sizeof(new_thr->th.th_sleep_loc)); in __kmp_allocate_thread()
4318 &new_thr->th.th_reap_state, sizeof(new_thr->th.th_reap_state)); in __kmp_allocate_thread()
4322 &new_thr->th.th_suspend_init, sizeof(new_thr->th.th_suspend_init)); in __kmp_allocate_thread()
4326 &new_thr->th.th_suspend_init_count, in __kmp_allocate_thread()
4327 sizeof(new_thr->th.th_suspend_init_count)); in __kmp_allocate_thread()
4332 CCAST(kmp_uint64 *, &new_thr->th.th_bar[0].bb.b_go), in __kmp_allocate_thread()
4333 sizeof(new_thr->th.th_bar[0].bb.b_go)); in __kmp_allocate_thread()
4336 CCAST(kmp_uint64 *, &new_thr->th.th_bar[1].bb.b_go), in __kmp_allocate_thread()
4337 sizeof(new_thr->th.th_bar[1].bb.b_go)); in __kmp_allocate_thread()
4340 CCAST(kmp_uint64 *, &new_thr->th.th_bar[2].bb.b_go), in __kmp_allocate_thread()
4341 sizeof(new_thr->th.th_bar[2].bb.b_go)); in __kmp_allocate_thread()
4351 new_thr->th.th_serial_team = serial_team = in __kmp_allocate_thread()
4375 KMP_DEBUG_ASSERT(new_thr->th.th_local.bget_data == NULL); in __kmp_allocate_thread()
4387 kmp_balign_t *balign = new_thr->th.th_bar; in __kmp_allocate_thread()
4395 new_thr->th.th_spin_here = FALSE; in __kmp_allocate_thread()
4396 new_thr->th.th_next_waiting = 0; in __kmp_allocate_thread()
4398 new_thr->th.th_blocking = false; in __kmp_allocate_thread()
4402 new_thr->th.th_current_place = KMP_PLACE_UNDEFINED; in __kmp_allocate_thread()
4403 new_thr->th.th_new_place = KMP_PLACE_UNDEFINED; in __kmp_allocate_thread()
4404 new_thr->th.th_first_place = KMP_PLACE_UNDEFINED; in __kmp_allocate_thread()
4405 new_thr->th.th_last_place = KMP_PLACE_UNDEFINED; in __kmp_allocate_thread()
4407 new_thr->th.th_def_allocator = __kmp_def_allocator; in __kmp_allocate_thread()
4408 new_thr->th.th_prev_level = 0; in __kmp_allocate_thread()
4409 new_thr->th.th_prev_num_threads = 1; in __kmp_allocate_thread()
4411 TCW_4(new_thr->th.th_in_pool, FALSE); in __kmp_allocate_thread()
4412 new_thr->th.th_active_in_pool = FALSE; in __kmp_allocate_thread()
4413 TCW_4(new_thr->th.th_active, TRUE); in __kmp_allocate_thread()
4565 int first_place = master_th->th.th_first_place; in __kmp_partition_places()
4566 int last_place = master_th->th.th_last_place; in __kmp_partition_places()
4567 int masters_place = master_th->th.th_current_place; in __kmp_partition_places()
4588 kmp_info_t *th = team->t.t_threads[f]; in __kmp_partition_places() local
4589 KMP_DEBUG_ASSERT(th != NULL); in __kmp_partition_places()
4590 th->th.th_first_place = first_place; in __kmp_partition_places()
4591 th->th.th_last_place = last_place; in __kmp_partition_places()
4592 th->th.th_new_place = masters_place; in __kmp_partition_places()
4593 if (__kmp_display_affinity && masters_place != th->th.th_current_place && in __kmp_partition_places()
4617 kmp_info_t *th = team->t.t_threads[f]; in __kmp_partition_places() local
4618 KMP_DEBUG_ASSERT(th != NULL); in __kmp_partition_places()
4627 th->th.th_first_place = first_place; in __kmp_partition_places()
4628 th->th.th_last_place = last_place; in __kmp_partition_places()
4629 th->th.th_new_place = place; in __kmp_partition_places()
4630 if (__kmp_display_affinity && place != th->th.th_current_place && in __kmp_partition_places()
4649 kmp_info_t *th = team->t.t_threads[f]; in __kmp_partition_places() local
4650 KMP_DEBUG_ASSERT(th != NULL); in __kmp_partition_places()
4652 th->th.th_first_place = first_place; in __kmp_partition_places()
4653 th->th.th_last_place = last_place; in __kmp_partition_places()
4654 th->th.th_new_place = place; in __kmp_partition_places()
4655 if (__kmp_display_affinity && place != th->th.th_current_place && in __kmp_partition_places()
4691 th->th.th_new_place, first_place, last_place)); in __kmp_partition_places()
4722 kmp_info_t *th = team->t.t_threads[f]; in __kmp_partition_places() local
4723 KMP_DEBUG_ASSERT(th != NULL); in __kmp_partition_places()
4725 th->th.th_first_place = place; in __kmp_partition_places()
4726 th->th.th_new_place = place; in __kmp_partition_places()
4727 if (__kmp_display_affinity && place != th->th.th_current_place && in __kmp_partition_places()
4753 th->th.th_last_place = place; in __kmp_partition_places()
4768 f, th->th.th_new_place, th->th.th_first_place, in __kmp_partition_places()
4769 th->th.th_last_place, __kmp_affinity_num_masks)); in __kmp_partition_places()
4779 kmp_info_t *th; in __kmp_partition_places() local
4817 th = team->t.t_threads[f]; in __kmp_partition_places()
4818 KMP_DEBUG_ASSERT(th); in __kmp_partition_places()
4819 th->th.th_first_place = first; in __kmp_partition_places()
4820 th->th.th_new_place = place; in __kmp_partition_places()
4821 th->th.th_last_place = last; in __kmp_partition_places()
4822 if (__kmp_display_affinity && place != th->th.th_current_place && in __kmp_partition_places()
4830 team->t.t_id, f, th->th.th_new_place, in __kmp_partition_places()
4831 th->th.th_first_place, th->th.th_last_place, spacing)); in __kmp_partition_places()
4848 kmp_info_t *th = team->t.t_threads[f]; in __kmp_partition_places() local
4849 KMP_DEBUG_ASSERT(th != NULL); in __kmp_partition_places()
4851 th->th.th_first_place = place; in __kmp_partition_places()
4852 th->th.th_last_place = place; in __kmp_partition_places()
4853 th->th.th_new_place = place; in __kmp_partition_places()
4854 if (__kmp_display_affinity && place != th->th.th_current_place && in __kmp_partition_places()
4889 team->t.t_id, f, th->th.th_new_place, in __kmp_partition_places()
4890 th->th.th_first_place, th->th.th_last_place)); in __kmp_partition_places()
4929 team = master->th.th_team; in __kmp_allocate_team()
4931 if (master->th.th_teams_microtask) { // in teams construct? in __kmp_allocate_team()
4932 if (master->th.th_teams_size.nteams > 1 && in __kmp_allocate_team()
4936 master->th.th_teams_level < in __kmp_allocate_team()
4942 hot_teams = master->th.th_hot_teams; in __kmp_allocate_team()
4990 root->r.r_uber_thread->th.th_ident); in __kmp_allocate_team()
5033 team->t.t_threads[f]->th.th_task_team = NULL; in __kmp_allocate_team()
5045 kmp_balign_t *balign = team->t.t_threads[f]->th.th_bar; in __kmp_allocate_team()
5059 root->r.r_uber_thread->th.th_ident); in __kmp_allocate_team()
5063 team->t.t_threads[f]->th.th_team_nproc = new_nproc; in __kmp_allocate_team()
5076 team->t.t_threads[f]->th.th_team_nproc == in __kmp_allocate_team()
5108 kmp_balign_t *balign = other_threads[f]->th.th_bar; in __kmp_allocate_team()
5158 kmp_balign_t *balign = new_worker->th.th_bar; in __kmp_allocate_team()
5184 root->r.r_uber_thread->th.th_ident); in __kmp_allocate_team()
5199 team->t.t_threads[f]->th.th_task_state = in __kmp_allocate_team()
5200 team->t.t_threads[0]->th.th_task_state_memo_stack[level]; in __kmp_allocate_team()
5203 team->t.t_threads[0]->th.th_task_state; // copy master's state in __kmp_allocate_team()
5205 team->t.t_threads[f]->th.th_task_state = old_state; in __kmp_allocate_team()
5211 team->t.t_threads[f]->th.th_team_nproc == in __kmp_allocate_team()
5223 if (master->th.th_teams_microtask) { in __kmp_allocate_team()
5227 thr->th.th_teams_microtask = master->th.th_teams_microtask; in __kmp_allocate_team()
5228 thr->th.th_teams_level = master->th.th_teams_level; in __kmp_allocate_team()
5229 thr->th.th_teams_size = master->th.th_teams_size; in __kmp_allocate_team()
5239 kmp_balign_t *balign = thr->th.th_bar; in __kmp_allocate_team()
5415 if (master->th.th_teams_microtask) { // in teams construct? in __kmp_free_team()
5416 if (master->th.th_teams_size.nteams > 1) { in __kmp_free_team()
5421 master->th.th_teams_level == team->t.t_level) { in __kmp_free_team()
5426 hot_teams = master->th.th_hot_teams; in __kmp_free_team()
5448 kmp_info_t *th = team->t.t_threads[f]; in __kmp_free_team() local
5449 volatile kmp_uint32 *state = &th->th.th_reap_state; in __kmp_free_team()
5454 if (!__kmp_is_thread_alive(th, &ecode)) { in __kmp_free_team()
5460 kmp_flag_64<> fl(&th->th.th_bar[bs_forkjoin_barrier].bb.b_go, th); in __kmp_free_team()
5462 fl.resume(__kmp_gtid_from_thread(th)); in __kmp_free_team()
5474 team->t.t_threads[f]->th.th_task_team = NULL; in __kmp_free_team()
5507 team->t.t_threads[1]->th.th_cg_roots); in __kmp_free_team()
5508 if (team->t.t_threads[1]->th.th_cg_roots->cg_root == team->t.t_threads[1]) { in __kmp_free_team()
5512 KMP_DEBUG_ASSERT(thr && thr->th.th_cg_roots && in __kmp_free_team()
5513 thr->th.th_cg_roots->cg_root == thr); in __kmp_free_team()
5515 kmp_cg_root_t *tmp = thr->th.th_cg_roots; in __kmp_free_team()
5516 thr->th.th_cg_roots = tmp->up; in __kmp_free_team()
5519 thr, tmp, thr->th.th_cg_roots, tmp->cg_nthreads)); in __kmp_free_team()
5525 if (thr->th.th_cg_roots) in __kmp_free_team()
5526 thr->th.th_current_task->td_icvs.thread_limit = in __kmp_free_team()
5527 thr->th.th_cg_roots->cg_thread_limit; in __kmp_free_team()
5587 __kmp_get_gtid(), this_th->th.th_info.ds.ds_gtid)); in __kmp_free_thread()
5594 kmp_balign_t *balign = this_th->th.th_bar; in __kmp_free_thread()
5601 this_th->th.th_task_state = 0; in __kmp_free_thread()
5602 this_th->th.th_reap_state = KMP_SAFE_TO_REAP; in __kmp_free_thread()
5605 TCW_PTR(this_th->th.th_team, NULL); in __kmp_free_thread()
5606 TCW_PTR(this_th->th.th_root, NULL); in __kmp_free_thread()
5607 TCW_PTR(this_th->th.th_dispatch, NULL); /* NOT NEEDED */ in __kmp_free_thread()
5609 while (this_th->th.th_cg_roots) { in __kmp_free_thread()
5610 this_th->th.th_cg_roots->cg_nthreads--; in __kmp_free_thread()
5613 this_th, this_th->th.th_cg_roots, in __kmp_free_thread()
5614 this_th->th.th_cg_roots->cg_root, in __kmp_free_thread()
5615 this_th->th.th_cg_roots->cg_nthreads)); in __kmp_free_thread()
5616 kmp_cg_root_t *tmp = this_th->th.th_cg_roots; in __kmp_free_thread()
5621 this_th->th.th_cg_roots = tmp->up; in __kmp_free_thread()
5627 this_th->th.th_cg_roots = NULL; in __kmp_free_thread()
5638 this_th->th.th_current_task = NULL; in __kmp_free_thread()
5642 gtid = this_th->th.th_info.ds.ds_gtid; in __kmp_free_thread()
5645 if (__kmp_thread_pool_insert_pt->th.th_info.ds.ds_gtid > gtid) { in __kmp_free_thread()
5656 scan = &(__kmp_thread_pool_insert_pt->th.th_next_pool); in __kmp_free_thread()
5660 for (; (*scan != NULL) && ((*scan)->th.th_info.ds.ds_gtid < gtid); in __kmp_free_thread()
5661 scan = &((*scan)->th.th_next_pool)) in __kmp_free_thread()
5666 TCW_PTR(this_th->th.th_next_pool, *scan); in __kmp_free_thread()
5668 KMP_DEBUG_ASSERT((this_th->th.th_next_pool == NULL) || in __kmp_free_thread()
5669 (this_th->th.th_info.ds.ds_gtid < in __kmp_free_thread()
5670 this_th->th.th_next_pool->th.th_info.ds.ds_gtid)); in __kmp_free_thread()
5671 TCW_4(this_th->th.th_in_pool, TRUE); in __kmp_free_thread()
5674 if (this_th->th.th_active == TRUE) { in __kmp_free_thread()
5676 this_th->th.th_active_in_pool = TRUE; in __kmp_free_thread()
5680 KMP_DEBUG_ASSERT(this_th->th.th_active_in_pool == FALSE); in __kmp_free_thread()
5704 int gtid = this_thr->th.th_info.ds.ds_gtid; in __kmp_launch_thread()
5712 this_thr->th.th_cons = __kmp_allocate_cons_stack(gtid); // ATT: Memory leak? in __kmp_launch_thread()
5718 thread_data = &(this_thr->th.ompt_thread_info.thread_data); in __kmp_launch_thread()
5721 this_thr->th.ompt_thread_info.state = ompt_state_overhead; in __kmp_launch_thread()
5722 this_thr->th.ompt_thread_info.wait_id = 0; in __kmp_launch_thread()
5723 this_thr->th.ompt_thread_info.idle_frame = OMPT_GET_FRAME_ADDRESS(0); in __kmp_launch_thread()
5724 this_thr->th.ompt_thread_info.parallel_flags = 0; in __kmp_launch_thread()
5729 this_thr->th.ompt_thread_info.state = ompt_state_idle; in __kmp_launch_thread()
5746 this_thr->th.ompt_thread_info.state = ompt_state_overhead; in __kmp_launch_thread()
5750 pteam = &this_thr->th.th_team; in __kmp_launch_thread()
5766 this_thr->th.ompt_thread_info.state = ompt_state_work_parallel; in __kmp_launch_thread()
5783 this_thr->th.ompt_thread_info.state = ompt_state_overhead; in __kmp_launch_thread()
5798 this_thr->th.th_task_team = NULL; in __kmp_launch_thread()
5876 gtid = thread->th.th_info.ds.ds_gtid; in __kmp_reap_thread()
5887 kmp_flag_64<> flag(&thread->th.th_bar[bs_forkjoin_barrier].bb.b_go, in __kmp_reap_thread()
5905 if (thread->th.th_active_in_pool) { in __kmp_reap_thread()
5906 thread->th.th_active_in_pool = FALSE; in __kmp_reap_thread()
5940 if (thread->th.th_cons) { in __kmp_reap_thread()
5941 __kmp_free_cons_stack(thread->th.th_cons); in __kmp_reap_thread()
5942 thread->th.th_cons = NULL; in __kmp_reap_thread()
5946 if (thread->th.th_pri_common != NULL) { in __kmp_reap_thread()
5947 __kmp_free(thread->th.th_pri_common); in __kmp_reap_thread()
5948 thread->th.th_pri_common = NULL; in __kmp_reap_thread()
5951 if (thread->th.th_task_state_memo_stack != NULL) { in __kmp_reap_thread()
5952 __kmp_free(thread->th.th_task_state_memo_stack); in __kmp_reap_thread()
5953 thread->th.th_task_state_memo_stack = NULL; in __kmp_reap_thread()
5957 if (thread->th.th_local.bget_data != NULL) { in __kmp_reap_thread()
5963 if (thread->th.th_affin_mask != NULL) { in __kmp_reap_thread()
5964 KMP_CPU_FREE(thread->th.th_affin_mask); in __kmp_reap_thread()
5965 thread->th.th_affin_mask = NULL; in __kmp_reap_thread()
5970 if (thread->th.th_hier_bar_data != NULL) { in __kmp_reap_thread()
5971 __kmp_free(thread->th.th_hier_bar_data); in __kmp_reap_thread()
5972 thread->th.th_hier_bar_data = NULL; in __kmp_reap_thread()
5976 __kmp_reap_team(thread->th.th_serial_team); in __kmp_reap_thread()
5977 thread->th.th_serial_team = NULL; in __kmp_reap_thread()
6046 __kmp_thread_pool = thread->th.th_next_pool; in __kmp_internal_end()
6048 KMP_DEBUG_ASSERT(thread->th.th_reap_state == KMP_SAFE_TO_REAP); in __kmp_internal_end()
6049 thread->th.th_next_pool = NULL; in __kmp_internal_end()
6050 thread->th.th_in_pool = FALSE; in __kmp_internal_end()
6074 while (thr && KMP_ATOMIC_LD_ACQ(&thr->th.th_blocking)) in __kmp_internal_end()
6282 __kmp_threads[gtid]->th.th_task_team = NULL; in __kmp_internal_end_thread()
7006 if (thread->th.th_current_task->td_icvs.nproc != 0) in __kmp_do_middle_initialize()
7127 this_thr->th.th_local.this_construct = 0; in __kmp_run_before_invoked_task()
7129 KMP_CACHE_PREFETCH(&this_thr->th.th_bar[bs_forkjoin_barrier].bb.b_arrived); in __kmp_run_before_invoked_task()
7131 dispatch = (kmp_disp_t *)TCR_PTR(this_thr->th.th_dispatch); in __kmp_run_before_invoked_task()
7157 kmp_team_t *team = this_thr->th.th_team; in __kmp_invoke_task_func()
7216 this_thr->th.ompt_thread_info.parallel_flags |= ompt_parallel_team; in __kmp_invoke_task_func()
7241 kmp_team_t *team = thr->th.th_team; in __kmp_teams_master()
7243 thr->th.th_set_nproc = thr->th.th_teams_size.nth; in __kmp_teams_master()
7244 KMP_DEBUG_ASSERT(thr->th.th_teams_microtask); in __kmp_teams_master()
7245 KMP_DEBUG_ASSERT(thr->th.th_set_nproc); in __kmp_teams_master()
7247 __kmp_tid_from_gtid(gtid), thr->th.th_teams_microtask)); in __kmp_teams_master()
7253 tmp->cg_thread_limit = thr->th.th_current_task->td_icvs.thread_limit; in __kmp_teams_master()
7258 tmp->up = thr->th.th_cg_roots; in __kmp_teams_master()
7259 thr->th.th_cg_roots = tmp; in __kmp_teams_master()
7267 (microtask_t)thr->th.th_teams_microtask, // "wrapped" task in __kmp_teams_master()
7273 if (thr->th.th_team_nproc < thr->th.th_teams_size.nth) in __kmp_teams_master()
7274 thr->th.th_teams_size.nth = thr->th.th_team_nproc; in __kmp_teams_master()
7288 kmp_team_t *team = this_thr->th.th_team; in __kmp_invoke_teams_master()
7290 if (!__kmp_threads[gtid]->th.th_team->t.t_serialized) in __kmp_invoke_teams_master()
7291 KMP_DEBUG_ASSERT((void *)__kmp_threads[gtid]->th.th_team->t.t_pkfn == in __kmp_invoke_teams_master()
7309 this_thr->th.ompt_thread_info.parallel_flags |= ompt_parallel_league; in __kmp_invoke_teams_master()
7324 thr->th.th_set_nproc = num_threads; in __kmp_push_num_threads()
7348 thr->th.th_set_nproc = thr->th.th_teams_size.nteams = num_teams; in __kmp_push_num_teams()
7363 if (num_threads > thr->th.th_current_task->td_icvs.thread_limit) { in __kmp_push_num_teams()
7364 num_threads = thr->th.th_current_task->td_icvs.thread_limit; in __kmp_push_num_teams()
7372 thr->th.th_current_task->td_icvs.thread_limit = num_threads; in __kmp_push_num_teams()
7388 thr->th.th_teams_size.nth = num_threads; in __kmp_push_num_teams()
7394 thr->th.th_set_proc_bind = proc_bind; in __kmp_push_proc_bind()
7407 KMP_DEBUG_ASSERT(this_thr->th.th_team == team); in __kmp_internal_fork()
7429 KMP_ASSERT(this_thr->th.th_team == team); in __kmp_internal_fork()
7434 team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc); in __kmp_internal_fork()
7446 KMP_DEBUG_ASSERT(this_thr->th.th_team == team); in __kmp_internal_join()
7454 __kmp_threads[gtid]->th.th_team_nproc != team->t.t_nproc) { in __kmp_internal_join()
7459 gtid, __kmp_threads[gtid]->th.th_team_nproc, team, in __kmp_internal_join()
7464 __kmp_threads[gtid]->th.th_team_nproc == team->t.t_nproc); in __kmp_internal_join()
7470 this_thr->th.ompt_thread_info.state == ompt_state_wait_barrier_implicit) { in __kmp_internal_join()
7471 int ds_tid = this_thr->th.th_info.ds.ds_tid; in __kmp_internal_join()
7473 this_thr->th.ompt_thread_info.state = ompt_state_overhead; in __kmp_internal_join()
7500 KMP_ASSERT(this_thr->th.th_team == team); in __kmp_internal_join()
7525 if (hot_team->t.t_threads[i]->th.th_active) { in __kmp_active_hot_team_nproc()
7545 ->th.th_current_task->td_icvs.dynamic == TRUE); in __kmp_load_balance_nproc()
7728 root = __kmp_threads[gtid]->th.th_root; in __kmp_internal_begin()
7756 root = thread->th.th_root; in __kmp_user_set_library()
7768 thread->th.th_set_nproc = 0; in __kmp_user_set_library()
7772 thread->th.th_set_nproc = 0; in __kmp_user_set_library()
7777 thread->th.th_set_nproc = 0; in __kmp_user_set_library()
7845 if (thr->th.th_teams_microtask) { in __kmp_aux_get_team_info()
7846 kmp_team_t *team = thr->th.th_team; in __kmp_aux_get_team_info()
7847 int tlevel = thr->th.th_teams_level; // the level of the teams construct in __kmp_aux_get_team_info()
7951 static int __kmp_aux_capture_affinity_field(int gtid, const kmp_info_t *th, in __kmp_aux_capture_affinity_field() argument
7962 KMP_DEBUG_ASSERT(th); in __kmp_aux_capture_affinity_field()
8062 rc = __kmp_str_buf_print(field_buffer, format, th->th.th_team->t.t_level); in __kmp_aux_capture_affinity_field()
8080 rc = __kmp_str_buf_print(field_buffer, format, th->th.th_team->t.t_nproc); in __kmp_aux_capture_affinity_field()
8084 __kmp_get_ancestor_thread_num(gtid, th->th.th_team->t.t_level - 1); in __kmp_aux_capture_affinity_field()
8091 __kmp_affinity_str_buf_mask(&buf, th->th.th_affin_mask); in __kmp_aux_capture_affinity_field()
8124 const kmp_info_t *th; in __kmp_aux_capture_affinity() local
8133 th = __kmp_threads[gtid]; in __kmp_aux_capture_affinity()
8148 int rc = __kmp_aux_capture_affinity_field(gtid, th, &parse_ptr, &field); in __kmp_aux_capture_affinity()
8188 set__blocktime_team(thread->th.th_team, tid, blocktime); in __kmp_aux_set_blocktime()
8189 set__blocktime_team(thread->th.th_serial_team, 0, blocktime); in __kmp_aux_set_blocktime()
8195 set__bt_intervals_team(thread->th.th_team, tid, bt_intervals); in __kmp_aux_set_blocktime()
8196 set__bt_intervals_team(thread->th.th_serial_team, 0, bt_intervals); in __kmp_aux_set_blocktime()
8202 set__bt_set_team(thread->th.th_team, tid, bt_set); in __kmp_aux_set_blocktime()
8203 set__bt_set_team(thread->th.th_serial_team, 0, bt_set); in __kmp_aux_set_blocktime()
8207 __kmp_gtid_from_tid(tid, thread->th.th_team), in __kmp_aux_set_blocktime()
8208 thread->th.th_team->t.t_id, tid, blocktime, bt_intervals, in __kmp_aux_set_blocktime()
8212 __kmp_gtid_from_tid(tid, thread->th.th_team), in __kmp_aux_set_blocktime()
8213 thread->th.th_team->t.t_id, tid, blocktime)); in __kmp_aux_set_blocktime()
8384 return ((__kmp_entry_thread()->th.th_local.packed_reduction_method) >> 8); in __kmp_get_reduce_method()
8406 kmp_flag_64<> fl(&thread->th.th_bar[bs_forkjoin_barrier].bb.b_go, in __kmp_resume_if_soft_paused()