1 // Copyright 2017 The Abseil Authors.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // https://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14 //
15 // Each active thread has an ThreadIdentity that may represent the thread in
16 // various level interfaces. ThreadIdentity objects are never deallocated.
17 // When a thread terminates, its ThreadIdentity object may be reused for a
18 // thread created later.
19
20 #ifndef ABSL_BASE_INTERNAL_THREAD_IDENTITY_H_
21 #define ABSL_BASE_INTERNAL_THREAD_IDENTITY_H_
22
23 #ifndef _WIN32
24 #include <pthread.h>
25 // Defines __GOOGLE_GRTE_VERSION__ (via glibc-specific features.h) when
26 // supported.
27 #include <unistd.h>
28 #endif
29
30 #include <atomic>
31 #include <cstdint>
32
33 #include "absl/base/config.h"
34 #include "absl/base/internal/per_thread_tls.h"
35
36 namespace absl {
37 ABSL_NAMESPACE_BEGIN
38
39 struct SynchLocksHeld;
40 struct SynchWaitParams;
41
42 namespace base_internal {
43
44 class SpinLock;
45 struct ThreadIdentity;
46
47 // Used by the implementation of absl::Mutex and absl::CondVar.
48 struct PerThreadSynch {
49 // The internal representation of absl::Mutex and absl::CondVar rely
50 // on the alignment of PerThreadSynch. Both store the address of the
51 // PerThreadSynch in the high-order bits of their internal state,
52 // which means the low kLowZeroBits of the address of PerThreadSynch
53 // must be zero.
54 static constexpr int kLowZeroBits = 8;
55 static constexpr int kAlignment = 1 << kLowZeroBits;
56
57 // Returns the associated ThreadIdentity.
58 // This can be implemented as a cast because we guarantee
59 // PerThreadSynch is the first element of ThreadIdentity.
thread_identityPerThreadSynch60 ThreadIdentity* thread_identity() {
61 return reinterpret_cast<ThreadIdentity*>(this);
62 }
63
64 PerThreadSynch *next; // Circular waiter queue; initialized to 0.
65 PerThreadSynch *skip; // If non-zero, all entries in Mutex queue
66 // up to and including "skip" have same
67 // condition as this, and will be woken later
68 bool may_skip; // if false while on mutex queue, a mutex unlocker
69 // is using this PerThreadSynch as a terminator. Its
70 // skip field must not be filled in because the loop
71 // might then skip over the terminator.
72
73 // The wait parameters of the current wait. waitp is null if the
74 // thread is not waiting. Transitions from null to non-null must
75 // occur before the enqueue commit point (state = kQueued in
76 // Enqueue() and CondVarEnqueue()). Transitions from non-null to
77 // null must occur after the wait is finished (state = kAvailable in
78 // Mutex::Block() and CondVar::WaitCommon()). This field may be
79 // changed only by the thread that describes this PerThreadSynch. A
80 // special case is Fer(), which calls Enqueue() on another thread,
81 // but with an identical SynchWaitParams pointer, thus leaving the
82 // pointer unchanged.
83 SynchWaitParams *waitp;
84
85 bool suppress_fatal_errors; // If true, try to proceed even in the face of
86 // broken invariants. This is used within fatal
87 // signal handlers to improve the chances of
88 // debug logging information being output
89 // successfully.
90
91 intptr_t readers; // Number of readers in mutex.
92 int priority; // Priority of thread (updated every so often).
93
94 // When priority will next be read (cycles).
95 int64_t next_priority_read_cycles;
96
97 // State values:
98 // kAvailable: This PerThreadSynch is available.
99 // kQueued: This PerThreadSynch is unavailable, it's currently queued on a
100 // Mutex or CondVar waistlist.
101 //
102 // Transitions from kQueued to kAvailable require a release
103 // barrier. This is needed as a waiter may use "state" to
104 // independently observe that it's no longer queued.
105 //
106 // Transitions from kAvailable to kQueued require no barrier, they
107 // are externally ordered by the Mutex.
108 enum State {
109 kAvailable,
110 kQueued
111 };
112 std::atomic<State> state;
113
114 bool maybe_unlocking; // Valid at head of Mutex waiter queue;
115 // true if UnlockSlow could be searching
116 // for a waiter to wake. Used for an optimization
117 // in Enqueue(). true is always a valid value.
118 // Can be reset to false when the unlocker or any
119 // writer releases the lock, or a reader fully releases
120 // the lock. It may not be set to false by a reader
121 // that decrements the count to non-zero.
122 // protected by mutex spinlock
123
124 bool wake; // This thread is to be woken from a Mutex.
125
126 // If "x" is on a waiter list for a mutex, "x->cond_waiter" is true iff the
127 // waiter is waiting on the mutex as part of a CV Wait or Mutex Await.
128 //
129 // The value of "x->cond_waiter" is meaningless if "x" is not on a
130 // Mutex waiter list.
131 bool cond_waiter;
132
133 // Locks held; used during deadlock detection.
134 // Allocated in Synch_GetAllLocks() and freed in ReclaimThreadIdentity().
135 SynchLocksHeld *all_locks;
136 };
137
138 struct ThreadIdentity {
139 // Must be the first member. The Mutex implementation requires that
140 // the PerThreadSynch object associated with each thread is
141 // PerThreadSynch::kAlignment aligned. We provide this alignment on
142 // ThreadIdentity itself.
143 PerThreadSynch per_thread_synch;
144
145 // Private: Reserved for absl::synchronization_internal::Waiter.
146 struct WaiterState {
147 char data[128];
148 } waiter_state;
149
150 // Used by PerThreadSem::{Get,Set}ThreadBlockedCounter().
151 std::atomic<int>* blocked_count_ptr;
152
153 // The following variables are mostly read/written just by the
154 // thread itself. The only exception is that these are read by
155 // a ticker thread as a hint.
156 std::atomic<int> ticker; // Tick counter, incremented once per second.
157 std::atomic<int> wait_start; // Ticker value when thread started waiting.
158 std::atomic<bool> is_idle; // Has thread become idle yet?
159
160 ThreadIdentity* next;
161 };
162
163 // Returns the ThreadIdentity object representing the calling thread; guaranteed
164 // to be unique for its lifetime. The returned object will remain valid for the
165 // program's lifetime; although it may be re-assigned to a subsequent thread.
166 // If one does not exist, return nullptr instead.
167 //
168 // Does not malloc(*), and is async-signal safe.
169 // [*] Technically pthread_setspecific() does malloc on first use; however this
170 // is handled internally within tcmalloc's initialization already.
171 //
172 // New ThreadIdentity objects can be constructed and associated with a thread
173 // by calling GetOrCreateCurrentThreadIdentity() in per-thread-sem.h.
174 ThreadIdentity* CurrentThreadIdentityIfPresent();
175
176 using ThreadIdentityReclaimerFunction = void (*)(void*);
177
178 // Sets the current thread identity to the given value. 'reclaimer' is a
179 // pointer to the global function for cleaning up instances on thread
180 // destruction.
181 void SetCurrentThreadIdentity(ThreadIdentity* identity,
182 ThreadIdentityReclaimerFunction reclaimer);
183
184 // Removes the currently associated ThreadIdentity from the running thread.
185 // This must be called from inside the ThreadIdentityReclaimerFunction, and only
186 // from that function.
187 void ClearCurrentThreadIdentity();
188
189 // May be chosen at compile time via: -DABSL_FORCE_THREAD_IDENTITY_MODE=<mode
190 // index>
191 #ifdef ABSL_THREAD_IDENTITY_MODE_USE_POSIX_SETSPECIFIC
192 #error ABSL_THREAD_IDENTITY_MODE_USE_POSIX_SETSPECIFIC cannot be direcly set
193 #else
194 #define ABSL_THREAD_IDENTITY_MODE_USE_POSIX_SETSPECIFIC 0
195 #endif
196
197 #ifdef ABSL_THREAD_IDENTITY_MODE_USE_TLS
198 #error ABSL_THREAD_IDENTITY_MODE_USE_TLS cannot be direcly set
199 #else
200 #define ABSL_THREAD_IDENTITY_MODE_USE_TLS 1
201 #endif
202
203 #ifdef ABSL_THREAD_IDENTITY_MODE_USE_CPP11
204 #error ABSL_THREAD_IDENTITY_MODE_USE_CPP11 cannot be direcly set
205 #else
206 #define ABSL_THREAD_IDENTITY_MODE_USE_CPP11 2
207 #endif
208
209 #ifdef ABSL_THREAD_IDENTITY_MODE
210 #error ABSL_THREAD_IDENTITY_MODE cannot be direcly set
211 #elif defined(ABSL_FORCE_THREAD_IDENTITY_MODE)
212 #define ABSL_THREAD_IDENTITY_MODE ABSL_FORCE_THREAD_IDENTITY_MODE
213 #elif defined(_WIN32) && !defined(__MINGW32__)
214 #define ABSL_THREAD_IDENTITY_MODE ABSL_THREAD_IDENTITY_MODE_USE_CPP11
215 #elif ABSL_PER_THREAD_TLS && defined(__GOOGLE_GRTE_VERSION__) && \
216 (__GOOGLE_GRTE_VERSION__ >= 20140228L)
217 // Support for async-safe TLS was specifically added in GRTEv4. It's not
218 // present in the upstream eglibc.
219 // Note: Current default for production systems.
220 #define ABSL_THREAD_IDENTITY_MODE ABSL_THREAD_IDENTITY_MODE_USE_TLS
221 #else
222 #define ABSL_THREAD_IDENTITY_MODE \
223 ABSL_THREAD_IDENTITY_MODE_USE_POSIX_SETSPECIFIC
224 #endif
225
226 #if ABSL_THREAD_IDENTITY_MODE == ABSL_THREAD_IDENTITY_MODE_USE_TLS || \
227 ABSL_THREAD_IDENTITY_MODE == ABSL_THREAD_IDENTITY_MODE_USE_CPP11
228
229 #if ABSL_PER_THREAD_TLS
230 ABSL_CONST_INIT extern ABSL_PER_THREAD_TLS_KEYWORD ThreadIdentity*
231 thread_identity_ptr;
232 #elif defined(ABSL_HAVE_THREAD_LOCAL)
233 ABSL_CONST_INIT extern thread_local ThreadIdentity* thread_identity_ptr;
234 #else
235 #error Thread-local storage not detected on this platform
236 #endif
237
238 // thread_local variables cannot be in headers exposed by DLLs. However, it is
239 // important for performance reasons in general that
240 // `CurrentThreadIdentityIfPresent` be inlined. This is not possible across a
241 // DLL boundary so, with DLLs, we opt to have the function not be inlined. Note
242 // that `CurrentThreadIdentityIfPresent` is declared above so we can exclude
243 // this entire inline definition when compiling as a DLL.
244 #if !defined(ABSL_BUILD_DLL) && !defined(ABSL_CONSUME_DLL)
CurrentThreadIdentityIfPresent()245 inline ThreadIdentity* CurrentThreadIdentityIfPresent() {
246 return thread_identity_ptr;
247 }
248 #endif
249
250 #elif ABSL_THREAD_IDENTITY_MODE != \
251 ABSL_THREAD_IDENTITY_MODE_USE_POSIX_SETSPECIFIC
252 #error Unknown ABSL_THREAD_IDENTITY_MODE
253 #endif
254
255 } // namespace base_internal
256 ABSL_NAMESPACE_END
257 } // namespace absl
258
259 #endif // ABSL_BASE_INTERNAL_THREAD_IDENTITY_H_
260