1 /*
2 * Copyright (C) 2019 The Android Open Source Project
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * * Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * * Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in
12 * the documentation and/or other materials provided with the
13 * distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
16 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
17 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
18 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
19 * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
21 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
22 * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
23 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
25 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29 #pragma once
30
31 #include <link.h>
32 #include <pthread.h>
33 #include <stdatomic.h>
34 #include <stdint.h>
35 #include <sys/cdefs.h>
36
37 __LIBC_HIDDEN__ extern _Atomic(size_t) __libc_tls_generation_copy;
38
39 struct TlsSegment {
40 size_t size = 0;
41 size_t alignment = 1;
42 const void* init_ptr = ""; // Field is non-null even when init_size is 0.
43 size_t init_size = 0;
44 };
45
46 __LIBC_HIDDEN__ bool __bionic_get_tls_segment(const ElfW(Phdr)* phdr_table, size_t phdr_count,
47 ElfW(Addr) load_bias, TlsSegment* out);
48
49 __LIBC_HIDDEN__ bool __bionic_check_tls_alignment(size_t* alignment);
50
51 struct StaticTlsLayout {
StaticTlsLayoutStaticTlsLayout52 constexpr StaticTlsLayout() {}
53
54 private:
55 size_t offset_ = 0;
56 size_t alignment_ = 1;
57 bool overflowed_ = false;
58
59 // Offsets to various Bionic TLS structs from the beginning of static TLS.
60 size_t offset_bionic_tcb_ = SIZE_MAX;
61 size_t offset_bionic_tls_ = SIZE_MAX;
62
63 public:
offset_bionic_tcbStaticTlsLayout64 size_t offset_bionic_tcb() const { return offset_bionic_tcb_; }
offset_bionic_tlsStaticTlsLayout65 size_t offset_bionic_tls() const { return offset_bionic_tls_; }
66 size_t offset_thread_pointer() const;
67
sizeStaticTlsLayout68 size_t size() const { return offset_; }
alignmentStaticTlsLayout69 size_t alignment() const { return alignment_; }
overflowedStaticTlsLayout70 bool overflowed() const { return overflowed_; }
71
72 size_t reserve_exe_segment_and_tcb(const TlsSegment* exe_segment, const char* progname);
73 void reserve_bionic_tls();
reserve_solib_segmentStaticTlsLayout74 size_t reserve_solib_segment(const TlsSegment& segment) {
75 return reserve(segment.size, segment.alignment);
76 }
77 void finish_layout();
78
79 private:
80 size_t reserve(size_t size, size_t alignment);
81
reserve_typeStaticTlsLayout82 template <typename T> size_t reserve_type() {
83 return reserve(sizeof(T), alignof(T));
84 }
85
86 size_t round_up_with_overflow_check(size_t value, size_t alignment);
87 };
88
89 static constexpr size_t kTlsGenerationNone = 0;
90 static constexpr size_t kTlsGenerationFirst = 1;
91
92 // The first ELF TLS module has ID 1. Zero is reserved for the first word of
93 // the DTV, a generation count. Unresolved weak symbols also use module ID 0.
94 static constexpr size_t kTlsUninitializedModuleId = 0;
95
__tls_module_id_to_idx(size_t id)96 static inline size_t __tls_module_id_to_idx(size_t id) { return id - 1; }
__tls_module_idx_to_id(size_t idx)97 static inline size_t __tls_module_idx_to_id(size_t idx) { return idx + 1; }
98
99 // A descriptor for a single ELF TLS module.
100 struct TlsModule {
101 TlsSegment segment;
102
103 // Offset into the static TLS block or SIZE_MAX for a dynamic module.
104 size_t static_offset = SIZE_MAX;
105
106 // The generation in which this module was loaded. Dynamic TLS lookups use
107 // this field to detect when a module has been unloaded.
108 size_t first_generation = kTlsGenerationNone;
109
110 // Used by the dynamic linker to track the associated soinfo* object.
111 void* soinfo_ptr = nullptr;
112 };
113
114 // Signature of the callbacks that will be called after DTLS creation and
115 // before DTLS destruction.
116 typedef void (*dtls_listener_t)(void* dynamic_tls_begin, void* dynamic_tls_end);
117
118 // Signature of the thread-exit callbacks.
119 typedef void (*thread_exit_cb_t)(void);
120
121 struct CallbackHolder {
122 thread_exit_cb_t cb;
123 CallbackHolder* prev;
124 };
125
126 // Table of the ELF TLS modules. Either the dynamic linker or the static
127 // initialization code prepares this table, and it's then used during thread
128 // creation and for dynamic TLS lookups.
129 struct TlsModules {
TlsModulesTlsModules130 constexpr TlsModules() {}
131
132 // A pointer to the TLS generation counter in libc.so. The counter is
133 // incremented each time an solib is loaded or unloaded.
134 _Atomic(size_t) generation = kTlsGenerationFirst;
135 _Atomic(size_t) *generation_libc_so = nullptr;
136
137 // Access to the TlsModule[] table requires taking this lock.
138 pthread_rwlock_t rwlock = PTHREAD_RWLOCK_INITIALIZER;
139
140 // Pointer to a block of TlsModule objects. The first module has ID 1 and
141 // is stored at index 0 in this table.
142 size_t module_count = 0;
143 size_t static_module_count = 0;
144 TlsModule* module_table = nullptr;
145
146 // Callback to be invoked after a dynamic TLS allocation.
147 dtls_listener_t on_creation_cb = nullptr;
148
149 // Callback to be invoked before a dynamic TLS deallocation.
150 dtls_listener_t on_destruction_cb = nullptr;
151
152 // The first thread-exit callback; inlined to avoid allocation.
153 thread_exit_cb_t first_thread_exit_callback = nullptr;
154
155 // The additional callbacks, if any.
156 CallbackHolder* thread_exit_callback_tail_node = nullptr;
157 };
158
159 void __init_static_tls(void* static_tls);
160
161 // Dynamic Thread Vector. Each thread has a different DTV. For each module
162 // (executable or solib), the DTV has a pointer to that module's TLS memory. The
163 // DTV is initially empty and is allocated on-demand. It grows as more modules
164 // are dlopen'ed. See https://www.akkadia.org/drepper/tls.pdf.
165 //
166 // The layout of the DTV is specified in various documents, but it is not part
167 // of Bionic's public ABI. A compiler can't generate code to access it directly,
168 // because it can't access libc's global generation counter.
169 struct TlsDtv {
170 // Number of elements in this object's modules field.
171 size_t count;
172
173 // A pointer to an older TlsDtv object that should be freed when the thread
174 // exits. The objects aren't immediately freed because a DTV could be
175 // reallocated by a signal handler that interrupted __tls_get_addr's fast
176 // path.
177 TlsDtv* next;
178
179 // The DTV slot points at this field, which allows omitting an add instruction
180 // on the fast path for a TLS lookup. The arm64 tlsdesc_resolver.S depends on
181 // the layout of fields past this point.
182 size_t generation;
183 void* modules[];
184 };
185
186 struct TlsIndex {
187 size_t module_id;
188 size_t offset;
189 };
190
191 #if defined(__i386__)
192 #define TLS_GET_ADDR_CCONV __attribute__((regparm(1)))
193 #define TLS_GET_ADDR ___tls_get_addr
194 #else
195 #define TLS_GET_ADDR_CCONV
196 #define TLS_GET_ADDR __tls_get_addr
197 #endif
198
199 extern "C" void* TLS_GET_ADDR(const TlsIndex* ti) TLS_GET_ADDR_CCONV;
200
201 struct bionic_tcb;
202 void __free_dynamic_tls(bionic_tcb* tcb);
203 void __notify_thread_exit_callbacks();
204