1 // Copyright (C) 2016 The Android Open Source Project
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 //      http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14 
15 #include "linker/module_merger.h"
16 #include "repr/ir_dumper.h"
17 #include "repr/ir_reader.h"
18 #include "repr/ir_representation.h"
19 #include "repr/symbol/so_file_parser.h"
20 #include "repr/symbol/version_script_parser.h"
21 #include "utils/command_line_utils.h"
22 #include "utils/header_abi_util.h"
23 
24 #include <llvm/ADT/Optional.h>
25 #include <llvm/Support/CommandLine.h>
26 #include <llvm/Support/raw_ostream.h>
27 
28 #include <fstream>
29 #include <functional>
30 #include <iostream>
31 #include <memory>
32 #include <string>
33 #include <thread>
34 #include <vector>
35 
36 #include <stdlib.h>
37 
38 
39 using namespace header_checker;
40 using header_checker::repr::TextFormatIR;
41 using header_checker::utils::CollectAllExportedHeaders;
42 using header_checker::utils::GetCwd;
43 using header_checker::utils::HideIrrelevantCommandLineOptions;
44 
45 
46 static llvm::cl::OptionCategory header_linker_category(
47     "header-abi-linker options");
48 
49 static llvm::cl::list<std::string> dump_files(
50     llvm::cl::Positional, llvm::cl::desc("<dump-files>"), llvm::cl::ZeroOrMore,
51     llvm::cl::cat(header_linker_category));
52 
53 static llvm::cl::opt<std::string> linked_dump(
54     "o", llvm::cl::desc("<linked dump>"), llvm::cl::Required,
55     llvm::cl::cat(header_linker_category));
56 
57 static llvm::cl::list<std::string> exported_header_dirs(
58     "I", llvm::cl::desc("<export_include_dirs>"), llvm::cl::Prefix,
59     llvm::cl::ZeroOrMore, llvm::cl::cat(header_linker_category));
60 
61 static llvm::cl::opt<std::string> root_dir(
62     "root-dir",
63     llvm::cl::desc("Specify the directory that the paths in the dump files are "
64                    "relative to. Default to current working directory"),
65     llvm::cl::Optional, llvm::cl::cat(header_linker_category));
66 
67 static llvm::cl::opt<std::string> version_script(
68     "v", llvm::cl::desc("<version_script>"), llvm::cl::Optional,
69     llvm::cl::cat(header_linker_category));
70 
71 static llvm::cl::list<std::string> excluded_symbol_versions(
72     "exclude-symbol-version", llvm::cl::Optional,
73     llvm::cl::cat(header_linker_category));
74 
75 static llvm::cl::list<std::string> excluded_symbol_tags(
76     "exclude-symbol-tag", llvm::cl::Optional,
77     llvm::cl::cat(header_linker_category));
78 
79 static llvm::cl::opt<std::string> api(
80     "api", llvm::cl::desc("<api>"), llvm::cl::Optional,
81     llvm::cl::init("current"),
82     llvm::cl::cat(header_linker_category));
83 
84 static llvm::cl::opt<std::string> arch(
85     "arch", llvm::cl::desc("<arch>"), llvm::cl::Optional,
86     llvm::cl::cat(header_linker_category));
87 
88 static llvm::cl::opt<bool> no_filter(
89     "no-filter", llvm::cl::desc("Do not filter any abi"), llvm::cl::Optional,
90     llvm::cl::cat(header_linker_category));
91 
92 static llvm::cl::opt<std::string> so_file(
93     "so", llvm::cl::desc("<path to so file>"), llvm::cl::Optional,
94     llvm::cl::cat(header_linker_category));
95 
96 static llvm::cl::opt<TextFormatIR> input_format(
97     "input-format", llvm::cl::desc("Specify format of input dump files"),
98     llvm::cl::values(clEnumValN(TextFormatIR::ProtobufTextFormat,
99                                 "ProtobufTextFormat", "ProtobufTextFormat"),
100                      clEnumValN(TextFormatIR::Json, "Json", "JSON")),
101     llvm::cl::init(TextFormatIR::Json),
102     llvm::cl::cat(header_linker_category));
103 
104 static llvm::cl::opt<TextFormatIR> output_format(
105     "output-format", llvm::cl::desc("Specify format of output dump file"),
106     llvm::cl::values(clEnumValN(TextFormatIR::ProtobufTextFormat,
107                                 "ProtobufTextFormat", "ProtobufTextFormat"),
108                      clEnumValN(TextFormatIR::Json, "Json", "JSON")),
109     llvm::cl::init(TextFormatIR::Json),
110     llvm::cl::cat(header_linker_category));
111 
112 static llvm::cl::opt<std::size_t> sources_per_thread(
113     "sources-per-thread",
114     llvm::cl::desc("Specify number of input dump files each thread parses, for "
115                    "debugging merging types"),
116     llvm::cl::init(7), llvm::cl::Hidden);
117 
118 class HeaderAbiLinker {
119  public:
HeaderAbiLinker(const std::vector<std::string> & dump_files,const std::vector<std::string> & exported_header_dirs,const std::string & version_script,const std::string & so_file,const std::string & linked_dump,const std::string & arch,const std::string & api,const std::vector<std::string> & excluded_symbol_versions,const std::vector<std::string> & excluded_symbol_tags)120   HeaderAbiLinker(
121       const std::vector<std::string> &dump_files,
122       const std::vector<std::string> &exported_header_dirs,
123       const std::string &version_script,
124       const std::string &so_file,
125       const std::string &linked_dump,
126       const std::string &arch,
127       const std::string &api,
128       const std::vector<std::string> &excluded_symbol_versions,
129       const std::vector<std::string> &excluded_symbol_tags)
130       : dump_files_(dump_files), exported_header_dirs_(exported_header_dirs),
131         version_script_(version_script), so_file_(so_file),
132         out_dump_name_(linked_dump), arch_(arch), api_(api),
133         excluded_symbol_versions_(excluded_symbol_versions),
134         excluded_symbol_tags_(excluded_symbol_tags) {}
135 
136   bool LinkAndDump();
137 
138  private:
139   template <typename T>
140   bool LinkDecl(repr::ModuleIR *dst,
141                 const repr::AbiElementMap<T> &src,
142                 const std::function<bool(const std::string &)> &symbol_filter);
143 
144   std::unique_ptr<linker::ModuleMerger> ReadInputDumpFiles();
145 
146   bool ReadExportedSymbols();
147 
148   bool ReadExportedSymbolsFromVersionScript();
149 
150   bool ReadExportedSymbolsFromSharedObjectFile();
151 
152   bool LinkTypes(const repr::ModuleIR &module, repr::ModuleIR *linked_module);
153 
154   bool LinkFunctions(const repr::ModuleIR &module,
155                      repr::ModuleIR *linked_module);
156 
157   bool LinkGlobalVars(const repr::ModuleIR &module,
158                       repr::ModuleIR *linked_module);
159 
160   bool LinkExportedSymbols(repr::ModuleIR *linked_module);
161 
162   bool LinkExportedSymbols(repr::ModuleIR *linked_module,
163                            const repr::ExportedSymbolSet &exported_symbols);
164 
165   template <typename SymbolMap>
166   bool LinkExportedSymbols(repr::ModuleIR *linked_module,
167                            const SymbolMap &symbols);
168 
169   // Check whether a symbol name is considered as exported.  If both
170   // `shared_object_symbols_` and `version_script_symbols_` exists, the symbol
171   // name must pass the `HasSymbol()` test in both cases.
172   bool IsSymbolExported(const std::string &name) const;
173 
174  private:
175   const std::vector<std::string> &dump_files_;
176   const std::vector<std::string> &exported_header_dirs_;
177   const std::string &version_script_;
178   const std::string &so_file_;
179   const std::string &out_dump_name_;
180   const std::string &arch_;
181   const std::string &api_;
182   const std::vector<std::string> &excluded_symbol_versions_;
183   const std::vector<std::string> &excluded_symbol_tags_;
184 
185   std::set<std::string> exported_headers_;
186 
187   // Exported symbols
188   std::unique_ptr<repr::ExportedSymbolSet> shared_object_symbols_;
189 
190   std::unique_ptr<repr::ExportedSymbolSet> version_script_symbols_;
191 };
192 
DeDuplicateAbiElementsThread(std::vector<std::string>::const_iterator dump_files_begin,std::vector<std::string>::const_iterator dump_files_end,const std::set<std::string> * exported_headers,linker::ModuleMerger * merger)193 static void DeDuplicateAbiElementsThread(
194     std::vector<std::string>::const_iterator dump_files_begin,
195     std::vector<std::string>::const_iterator dump_files_end,
196     const std::set<std::string> *exported_headers,
197     linker::ModuleMerger *merger) {
198   for (auto it = dump_files_begin; it != dump_files_end; it++) {
199     std::unique_ptr<repr::IRReader> reader =
200         repr::IRReader::CreateIRReader(input_format, exported_headers);
201     assert(reader != nullptr);
202     if (!reader->ReadDump(*it)) {
203       llvm::errs() << "ReadDump failed\n";
204       ::exit(1);
205     }
206     merger->MergeGraphs(reader->GetModule());
207   }
208 }
209 
ReadInputDumpFiles()210 std::unique_ptr<linker::ModuleMerger> HeaderAbiLinker::ReadInputDumpFiles() {
211   std::unique_ptr<linker::ModuleMerger> merger(
212       new linker::ModuleMerger(&exported_headers_));
213   std::size_t max_threads = std::thread::hardware_concurrency();
214   std::size_t num_threads = std::max<std::size_t>(
215       std::min(dump_files_.size() / sources_per_thread, max_threads), 1);
216   std::vector<std::thread> threads;
217   std::vector<linker::ModuleMerger> thread_mergers;
218   thread_mergers.reserve(num_threads - 1);
219 
220   std::size_t dump_files_index = 0;
221   std::size_t first_end_index = 0;
222   for (std::size_t i = 0; i < num_threads; i++) {
223     std::size_t cnt = dump_files_.size() / num_threads +
224                       (i < dump_files_.size() % num_threads ? 1 : 0);
225     if (i == 0) {
226       first_end_index = cnt;
227     } else {
228       thread_mergers.emplace_back(&exported_headers_);
229       threads.emplace_back(DeDuplicateAbiElementsThread,
230                            dump_files_.begin() + dump_files_index,
231                            dump_files_.begin() + dump_files_index + cnt,
232                            &exported_headers_, &thread_mergers.back());
233     }
234     dump_files_index += cnt;
235   }
236   assert(dump_files_index == dump_files_.size());
237 
238   DeDuplicateAbiElementsThread(dump_files_.begin(),
239                                dump_files_.begin() + first_end_index,
240                                &exported_headers_, merger.get());
241 
242   for (std::size_t i = 0; i < threads.size(); i++) {
243     threads[i].join();
244     merger->MergeGraphs(thread_mergers[i].GetModule());
245   }
246 
247   return merger;
248 }
249 
LinkAndDump()250 bool HeaderAbiLinker::LinkAndDump() {
251   // Extract exported functions and variables from a shared lib or a version
252   // script.
253   if (!ReadExportedSymbols()) {
254     return false;
255   }
256 
257   // Construct the list of exported headers for source location filtering.
258   exported_headers_ = CollectAllExportedHeaders(
259       exported_header_dirs_, root_dir.empty() ? GetCwd() : root_dir);
260 
261   // Read all input ABI dumps.
262   auto merger = ReadInputDumpFiles();
263 
264   const repr::ModuleIR &module = merger->GetModule();
265 
266   // Link input ABI dumps.
267   std::unique_ptr<repr::ModuleIR> linked_module(
268       new repr::ModuleIR(&exported_headers_));
269 
270   if (!LinkExportedSymbols(linked_module.get())) {
271     return false;
272   }
273 
274   if (!LinkTypes(module, linked_module.get()) ||
275       !LinkFunctions(module, linked_module.get()) ||
276       !LinkGlobalVars(module, linked_module.get())) {
277     llvm::errs() << "Failed to link elements\n";
278     return false;
279   }
280 
281   // Dump the linked module.
282   std::unique_ptr<repr::IRDumper> ir_dumper =
283       repr::IRDumper::CreateIRDumper(output_format, out_dump_name_);
284   assert(ir_dumper != nullptr);
285   if (!ir_dumper->Dump(*linked_module)) {
286     llvm::errs() << "Failed to serialize the linked output to ostream\n";
287     return false;
288   }
289 
290   return true;
291 }
292 
293 template <typename T>
LinkDecl(repr::ModuleIR * dst,const repr::AbiElementMap<T> & src,const std::function<bool (const std::string &)> & symbol_filter)294 bool HeaderAbiLinker::LinkDecl(
295     repr::ModuleIR *dst, const repr::AbiElementMap<T> &src,
296     const std::function<bool(const std::string &)> &symbol_filter) {
297   assert(dst != nullptr);
298   for (auto &&element : src) {
299     // If we are not using a version script and exported headers are available,
300     // filter out unexported abi.
301     std::string source_file = element.second.GetSourceFile();
302     // Builtin types will not have source file information.
303     if (!exported_headers_.empty() && !source_file.empty() &&
304         exported_headers_.find(source_file) == exported_headers_.end()) {
305       continue;
306     }
307     // Check for the existence of the element in version script / symbol file.
308     if (!symbol_filter(element.first)) {
309       continue;
310     }
311     if (!dst->AddLinkableMessage(element.second)) {
312       llvm::errs() << "Failed to add element to linked dump\n";
313       return false;
314     }
315   }
316   return true;
317 }
318 
LinkTypes(const repr::ModuleIR & module,repr::ModuleIR * linked_module)319 bool HeaderAbiLinker::LinkTypes(const repr::ModuleIR &module,
320                                 repr::ModuleIR *linked_module) {
321   auto no_filter = [](const std::string &symbol) { return true; };
322   return LinkDecl(linked_module, module.GetRecordTypes(), no_filter) &&
323          LinkDecl(linked_module, module.GetEnumTypes(), no_filter) &&
324          LinkDecl(linked_module, module.GetFunctionTypes(), no_filter) &&
325          LinkDecl(linked_module, module.GetBuiltinTypes(), no_filter) &&
326          LinkDecl(linked_module, module.GetPointerTypes(), no_filter) &&
327          LinkDecl(linked_module, module.GetRvalueReferenceTypes(), no_filter) &&
328          LinkDecl(linked_module, module.GetLvalueReferenceTypes(), no_filter) &&
329          LinkDecl(linked_module, module.GetArrayTypes(), no_filter) &&
330          LinkDecl(linked_module, module.GetQualifiedTypes(), no_filter);
331 }
332 
IsSymbolExported(const std::string & name) const333 bool HeaderAbiLinker::IsSymbolExported(const std::string &name) const {
334   if (shared_object_symbols_ && !shared_object_symbols_->HasSymbol(name)) {
335     return false;
336   }
337   if (version_script_symbols_ && !version_script_symbols_->HasSymbol(name)) {
338     return false;
339   }
340   return true;
341 }
342 
LinkFunctions(const repr::ModuleIR & module,repr::ModuleIR * linked_module)343 bool HeaderAbiLinker::LinkFunctions(const repr::ModuleIR &module,
344                                     repr::ModuleIR *linked_module) {
345   auto symbol_filter = [this](const std::string &linker_set_key) {
346     return IsSymbolExported(linker_set_key);
347   };
348   return LinkDecl(linked_module, module.GetFunctions(), symbol_filter);
349 }
350 
LinkGlobalVars(const repr::ModuleIR & module,repr::ModuleIR * linked_module)351 bool HeaderAbiLinker::LinkGlobalVars(const repr::ModuleIR &module,
352                                      repr::ModuleIR *linked_module) {
353   auto symbol_filter = [this](const std::string &linker_set_key) {
354     return IsSymbolExported(linker_set_key);
355   };
356   return LinkDecl(linked_module, module.GetGlobalVariables(), symbol_filter);
357 }
358 
359 template <typename SymbolMap>
LinkExportedSymbols(repr::ModuleIR * dst,const SymbolMap & symbols)360 bool HeaderAbiLinker::LinkExportedSymbols(repr::ModuleIR *dst,
361                                           const SymbolMap &symbols) {
362   for (auto &&symbol : symbols) {
363     if (!IsSymbolExported(symbol.first)) {
364       continue;
365     }
366     if (!dst->AddElfSymbol(symbol.second)) {
367       return false;
368     }
369   }
370   return true;
371 }
372 
LinkExportedSymbols(repr::ModuleIR * linked_module,const repr::ExportedSymbolSet & exported_symbols)373 bool HeaderAbiLinker::LinkExportedSymbols(
374     repr::ModuleIR *linked_module,
375     const repr::ExportedSymbolSet &exported_symbols) {
376   return (LinkExportedSymbols(linked_module, exported_symbols.GetFunctions()) &&
377           LinkExportedSymbols(linked_module, exported_symbols.GetVars()));
378 }
379 
LinkExportedSymbols(repr::ModuleIR * linked_module)380 bool HeaderAbiLinker::LinkExportedSymbols(repr::ModuleIR *linked_module) {
381   if (shared_object_symbols_) {
382     return LinkExportedSymbols(linked_module, *shared_object_symbols_);
383   }
384 
385   if (version_script_symbols_) {
386     return LinkExportedSymbols(linked_module, *version_script_symbols_);
387   }
388 
389   return false;
390 }
391 
ReadExportedSymbols()392 bool HeaderAbiLinker::ReadExportedSymbols() {
393   if (so_file_.empty() && version_script_.empty()) {
394     llvm::errs() << "Either shared lib or version script must be specified.\n";
395     return false;
396   }
397 
398   if (!so_file_.empty()) {
399     if (!ReadExportedSymbolsFromSharedObjectFile()) {
400       llvm::errs() << "Failed to parse the shared library (.so file): "
401                    << so_file_ << "\n";
402       return false;
403     }
404   }
405 
406   if (!version_script_.empty()) {
407     if (!ReadExportedSymbolsFromVersionScript()) {
408       llvm::errs() << "Failed to parse the version script: " << version_script_
409                    << "\n";
410       return false;
411     }
412   }
413 
414   return true;
415 }
416 
ReadExportedSymbolsFromVersionScript()417 bool HeaderAbiLinker::ReadExportedSymbolsFromVersionScript() {
418   llvm::Optional<utils::ApiLevel> api_level = utils::ParseApiLevel(api_);
419   if (!api_level) {
420     llvm::errs() << "-api must be either \"current\" or an integer (e.g. 21)\n";
421     return false;
422   }
423 
424   std::ifstream stream(version_script_, std::ios_base::in);
425   if (!stream) {
426     llvm::errs() << "Failed to open version script file\n";
427     return false;
428   }
429 
430   repr::VersionScriptParser parser;
431   parser.SetArch(arch_);
432   parser.SetApiLevel(api_level.getValue());
433   for (auto &&version : excluded_symbol_versions_) {
434     parser.AddExcludedSymbolVersion(version);
435   }
436   for (auto &&tag : excluded_symbol_tags_) {
437     parser.AddExcludedSymbolTag(tag);
438   }
439 
440   version_script_symbols_ = parser.Parse(stream);
441   if (!version_script_symbols_) {
442     llvm::errs() << "Failed to parse version script file\n";
443     return false;
444   }
445 
446   return true;
447 }
448 
ReadExportedSymbolsFromSharedObjectFile()449 bool HeaderAbiLinker::ReadExportedSymbolsFromSharedObjectFile() {
450   std::unique_ptr<repr::SoFileParser> so_parser =
451       repr::SoFileParser::Create(so_file_);
452   if (!so_parser) {
453     return false;
454   }
455 
456   shared_object_symbols_ = so_parser->Parse();
457   if (!shared_object_symbols_) {
458     llvm::errs() << "Failed to parse shared object file\n";
459     return false;
460   }
461 
462   return true;
463 }
464 
main(int argc,const char ** argv)465 int main(int argc, const char **argv) {
466   HideIrrelevantCommandLineOptions(header_linker_category);
467   llvm::cl::ParseCommandLineOptions(argc, argv, "header-linker");
468 
469   if (so_file.empty() && version_script.empty()) {
470     llvm::errs() << "One of -so or -v needs to be specified\n";
471     return -1;
472   }
473 
474   if (no_filter) {
475     static_cast<std::vector<std::string> &>(exported_header_dirs).clear();
476   }
477 
478   HeaderAbiLinker Linker(dump_files, exported_header_dirs, version_script,
479                          so_file, linked_dump, arch, api,
480                          excluded_symbol_versions,
481                          excluded_symbol_tags);
482 
483   if (!Linker.LinkAndDump()) {
484     llvm::errs() << "Failed to link and dump elements\n";
485     return -1;
486   }
487 
488   return 0;
489 }
490