1 /*
2 * Copyright (C) 2021 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 *
16 * rebalance-interrupts:
17 *
18 * One-shot distribution of unassigned* IRQs to CPU cores.
19 * Useful for devices with the ARM-GIC-v3, as the Linux driver will take any
20 * interrupt assigned with an all-cores mask and always have it run on core 0.
21 *
22 * This should be run once, long enough after boot that all drivers have
23 * registered their interrupts.
24 *
25 * This program is configured to spread the load across all the cores in
26 * CPUFREQ policy 0. This is because other cores may be hotplugged in
27 * or out, and if hotplugged out the interrupts would be sent to core0 always.
28 *
29 * It might be wise to avoid core0 so that any later-added IRQs don't overcrowd
30 * core 0.
31 *
32 * Any program that has an actual IRQ related performance constraint should
33 * override any settings assigned by this and assign the IRQ to the same
34 * core as the code whose performance is impacted by the IRQ.
35 *
36 */
37
38 #include <sys/types.h>
39 #include <dirent.h>
40
41 #include <iostream>
42 #include <list>
43 #include <map>
44 #include <vector>
45
46 #define LOG_TAG "rebalance_interrupts"
47
48 #include <android-base/file.h>
49 #include <android-base/format.h>
50 #include <android-base/logging.h>
51 #include <android-base/parseint.h>
52 #include <android-base/strings.h>
53
54
55 #define POLICY0_CORES_PATH "/sys/devices/system/cpu/cpufreq/policy0/affected_cpus"
56 #define SYSFS_IRQDIR "/sys/kernel/irq"
57 #define PROC_IRQDIR "/proc/irq"
58
59 using android::base::ParseInt;
60 using android::base::ParseUint;
61 using android::base::ReadFileToString;
62 using android::base::Trim;
63 using android::base::WriteStringToFile;
64 using std::list;
65 using std::map;
66 using std::pair;
67 using std::string;
68 using std::vector;
69
70 // Return a vector of strings describing the affected CPUs for cpufreq
71 // Policy 0.
Policy0AffectedCpus()72 vector<int> Policy0AffectedCpus() {
73 string policy0_cores_unparsed;
74 if (!ReadFileToString(POLICY0_CORES_PATH, &policy0_cores_unparsed))
75 return vector<int>();
76 string policy0_trimmed = android::base::Trim(policy0_cores_unparsed);
77 vector<string> cpus_as_string = android::base::Split(policy0_trimmed, " ");
78
79 vector<int> cpus_as_int;
80 for (int i = 0; i < cpus_as_string.size(); ++i) {
81 int cpu;
82 if (!ParseInt(cpus_as_string[i].c_str(), &cpu))
83 return vector<int>();
84 cpus_as_int.push_back(cpu);
85 }
86 return cpus_as_int;
87 }
88
89 // Return a vector of strings describing the CPU masks for cpufreq Policy 0.
Policy0CpuMasks()90 vector<string> Policy0CpuMasks() {
91 vector<int> cpus = Policy0AffectedCpus();
92 vector<string> cpu_masks;
93 for (int i = 0; i < cpus.size(); ++i)
94 cpu_masks.push_back(fmt::format("{0:02x}", 1 << cpus[i]));
95 return cpu_masks;
96 }
97
98 // Read the actions for the given irq# from sysfs, and add it to action_to_irq
AddEntryToIrqmap(const char * irq,map<string,list<string>> & action_to_irqs)99 bool AddEntryToIrqmap(const char* irq,
100 map<string, list<string>>& action_to_irqs) {
101 const string irq_base(SYSFS_IRQDIR "/");
102 string irq_actions_path = irq_base + irq + "/actions";
103
104 string irq_actions;
105 if (!ReadFileToString(irq_actions_path, &irq_actions))
106 return false;
107
108 irq_actions = Trim(irq_actions);
109
110 if (irq_actions == "(null)")
111 irq_actions = "";
112
113 action_to_irqs[irq_actions].push_back(irq);
114
115 return true;
116 }
117
118 // Get a mapping of driver "action" to IRQ#s for each IRQ# in
119 // SYSFS_IRQDIR.
GetIrqmap(map<string,list<string>> & action_to_irqs)120 bool GetIrqmap(map<string, list<string>>& action_to_irqs) {
121 bool some_success = false;
122 std::unique_ptr<DIR, decltype(&closedir)> irq_dir(opendir(SYSFS_IRQDIR), closedir);
123 if (!irq_dir) {
124 PLOG(ERROR) << "opening dir " SYSFS_IRQDIR;
125 return false;
126 }
127
128 struct dirent* entry;
129 while ((entry = readdir(irq_dir.get()))) {
130
131 // If the directory entry isn't a parsable number, skip it.
132 // . and .. get skipped here.
133 unsigned throwaway;
134 if (!ParseUint(entry->d_name, &throwaway))
135 continue;
136
137 some_success |= AddEntryToIrqmap(entry->d_name, action_to_irqs);
138 }
139 return some_success;
140 }
141
142 // Given a map of irq actions -> IRQs,
143 // find out which ones haven't been assigned and add those to
144 // rebalance_actions.
FindUnassignedIrqs(const map<string,list<string>> & action_to_irqs,list<pair<string,list<string>>> & rebalance_actions)145 void FindUnassignedIrqs(const map<string, list<string>>& action_to_irqs,
146 list<pair<string, list<string>>>& rebalance_actions) {
147 for (const auto &action_to_irqs_entry: action_to_irqs) {
148 bool rebalance = true;
149 for (const auto& irq: action_to_irqs_entry.second) {
150 string smp_affinity;
151 string proc_path(PROC_IRQDIR "/");
152 proc_path += irq + "/smp_affinity";
153 ReadFileToString(proc_path, &smp_affinity);
154 smp_affinity = Trim(smp_affinity);
155
156 // Try to respect previoulsy set IRQ affinities.
157 // On ARM interrupt controllers under Linux, if an IRQ is assigned
158 // to more than one core it will only be assigned to the lowest core.
159 // Assume any IRQ which is set to more than one core in the lowest four
160 // CPUs hasn't been assigned and needs to be rebalanced.
161 if (smp_affinity.back() == '0' ||
162 smp_affinity.back() == '1' ||
163 smp_affinity.back() == '2' ||
164 smp_affinity.back() == '4' ||
165 smp_affinity.back() == '8') {
166 rebalance = false;
167 }
168
169 // Treat each unnamed action IRQ as independent.
170 if (action_to_irqs_entry.first.empty()) {
171 if (rebalance) {
172 pair<string, list<string>> empty_action_irq;
173 empty_action_irq.first = "";
174 empty_action_irq.second.push_back(irq);
175 rebalance_actions.push_back(empty_action_irq);
176 }
177 rebalance = true;
178 }
179 }
180 if (rebalance && !action_to_irqs_entry.first.empty()) {
181 rebalance_actions.push_back(std::make_pair(action_to_irqs_entry.first,
182 action_to_irqs_entry.second));
183 }
184 }
185 }
186
187 // Read the file at `path`, Trim whitespace, see if it matches `expected_value`.
188 // Print the results to stdout.
ReportIfAffinityUpdated(const std::string expected_value,const std::string path)189 void ReportIfAffinityUpdated(const std::string expected_value,
190 const std::string path) {
191 std::string readback, report;
192 ReadFileToString(path, &readback);
193 readback = Trim(readback);
194 if (readback != expected_value) {
195 report += "Unable to set ";
196 } else {
197 report += "Success setting ";
198 }
199 report += path;
200 report += ": found " + readback + " vs " + expected_value + "\n";
201 LOG(DEBUG) << report;
202 }
203
204 // Evenly distribute the IRQ actions across all the Policy0 CPUs.
205 // Assign all the IRQs of an action to a single CPU core.
RebalanceIrqs(const list<pair<string,list<string>>> & action_to_irqs)206 bool RebalanceIrqs(const list<pair<string, list<string>>>& action_to_irqs) {
207 int mask_index = 0;
208 std::vector<std::string> affinity_masks = Policy0CpuMasks();
209
210 if (affinity_masks.empty()) {
211 LOG(ERROR) << "Unable to find Policy0 CPUs for IRQ assignment.";
212 return false;
213 }
214
215 for (const auto &action_to_irq: action_to_irqs) {
216 for (const auto& irq: action_to_irq.second) {
217 std::string affinity_path(PROC_IRQDIR "/");
218 affinity_path += irq + "/smp_affinity";
219 WriteStringToFile(affinity_masks[mask_index], affinity_path);
220 ReportIfAffinityUpdated(affinity_masks[mask_index], affinity_path);
221 }
222 mask_index = (mask_index + 1) % affinity_masks.size();
223 }
224 return true;
225 }
226
ChownIrqAffinity()227 void ChownIrqAffinity() {
228 std::unique_ptr<DIR, decltype(&closedir)> irq_dir(opendir(PROC_IRQDIR), closedir);
229 if (!irq_dir) {
230 PLOG(ERROR) << "opening dir " PROC_IRQDIR;
231 return;
232 }
233
234 struct dirent *entry;
235 while ((entry = readdir(irq_dir.get()))) {
236 // If the directory entry isn't a parsable number, skip it.
237 // . and .. get skipped here.
238 unsigned throwaway;
239 if (!ParseUint(entry->d_name, &throwaway))
240 continue;
241
242 string affinity_path(PROC_IRQDIR "/");
243 affinity_path += entry->d_name;
244 affinity_path += "/smp_affinity";
245 chown(affinity_path.c_str(), 1000, 1000);
246
247 string affinity_list_path(PROC_IRQDIR "/");
248 affinity_list_path += entry->d_name;
249 affinity_list_path += "/smp_affinity_list";
250 chown(affinity_list_path.c_str(), 1000, 1000);
251 }
252 }
253
main(int,char * [])254 int main(int /* argc */, char* /* argv */[]) {
255 map<string, list<string>> irq_mapping;
256 list<pair<string, list<string>>> action_to_irqs;
257
258 // Find the mapping of "irq actions" to IRQs.
259 // Each IRQ has an assocatied irq_actions field, showing the actions
260 // associated with it. Multiple IRQs have the same actions.
261 // Generate the mapping of actions to IRQs with that action,
262 // as these IRQs should all be mapped to the same cores.
263 if (!GetIrqmap(irq_mapping)) {
264 LOG(ERROR) << "Unable to read IRQ mappings. Are you root?";
265 return 1;
266 }
267
268 // Change ownership of smp_affinity and smp_affinity_list handles
269 // from root to system.
270 ChownIrqAffinity();
271
272 // Some IRQs are already assigned to a subset of cores, usually for
273 // good reason (like some drivers have an IRQ per core, for per-core
274 // queues.) Find the set of IRQs that haven't been mapped to specific
275 // cores.
276 FindUnassignedIrqs(irq_mapping, action_to_irqs);
277
278 // Distribute the rebalancable IRQs across all cores.
279 return RebalanceIrqs(action_to_irqs) ? 0 : 1;
280 }
281
282