1 /*
2 * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "test/test_main_lib.h"
12
13 #include <fstream>
14 #include <memory>
15 #include <string>
16
17 #include "absl/flags/flag.h"
18 #include "absl/flags/parse.h"
19 #include "absl/memory/memory.h"
20 #include "absl/strings/match.h"
21 #include "absl/types/optional.h"
22 #include "rtc_base/checks.h"
23 #include "rtc_base/event_tracer.h"
24 #include "rtc_base/logging.h"
25 #include "rtc_base/ssl_adapter.h"
26 #include "rtc_base/ssl_stream_adapter.h"
27 #include "rtc_base/thread.h"
28 #include "system_wrappers/include/field_trial.h"
29 #include "system_wrappers/include/metrics.h"
30 #include "test/field_trial.h"
31 #include "test/gmock.h"
32 #include "test/gtest.h"
33 #include "test/testsupport/perf_test.h"
34 #include "test/testsupport/resources_dir_flag.h"
35
36 #if defined(WEBRTC_WIN)
37 #include "rtc_base/win32_socket_init.h"
38 #endif
39
40 #if defined(WEBRTC_IOS)
41 #include "test/ios/test_support.h"
42
43 ABSL_FLAG(std::string,
44 NSTreatUnknownArgumentsAsOpen,
45 "",
46 "Intentionally ignored flag intended for iOS simulator.");
47 ABSL_FLAG(std::string,
48 ApplePersistenceIgnoreState,
49 "",
50 "Intentionally ignored flag intended for iOS simulator.");
51
52 // This is the cousin of isolated_script_test_perf_output, but we can't dictate
53 // where to write on iOS so the semantics of this flag are a bit different.
54 ABSL_FLAG(
55 bool,
56 write_perf_output_on_ios,
57 false,
58 "Store the perf results in Documents/perftest_result.pb in the format "
59 "described by histogram.proto in "
60 "https://chromium.googlesource.com/catapult/.");
61
62 #else
63
64 ABSL_FLAG(
65 std::string,
66 isolated_script_test_perf_output,
67 "",
68 "Path where the perf results should be stored in proto format described "
69 "described by histogram.proto in "
70 "https://chromium.googlesource.com/catapult/.");
71
72 #endif
73
74 constexpr char kPlotAllMetrics[] = "all";
75 ABSL_FLAG(std::vector<std::string>,
76 plot,
77 {},
78 "List of metrics that should be exported for plotting (if they are "
79 "available). Example: psnr,ssim,encode_time. To plot all available "
80 " metrics pass 'all' as flag value");
81
82 ABSL_FLAG(bool, logs, true, "print logs to stderr");
83 ABSL_FLAG(bool, verbose, false, "verbose logs to stderr");
84
85 ABSL_FLAG(std::string,
86 trace_event,
87 "",
88 "Path to collect trace events (json file) for chrome://tracing. "
89 "If not set, events aren't captured.");
90
91 ABSL_FLAG(std::string,
92 force_fieldtrials,
93 "",
94 "Field trials control experimental feature code which can be forced. "
95 "E.g. running with --force_fieldtrials=WebRTC-FooFeature/Enable/"
96 " will assign the group Enable to field trial WebRTC-FooFeature.");
97
98 namespace webrtc {
99
100 namespace {
101
102 class TestMainImpl : public TestMain {
103 public:
104 // In order to set up a fresh rtc::Thread state for each test and avoid
105 // accidentally carrying over pending tasks that might be sent from one test
106 // and executed while another test is running, we inject a TestListener
107 // that sets up a new rtc::Thread instance for the main thread, per test.
108 class TestListener : public ::testing::EmptyTestEventListener {
109 public:
110 TestListener() = default;
111
112 private:
IsDeathTest(const char * test_case_name,const char * test_name)113 bool IsDeathTest(const char* test_case_name, const char* test_name) {
114 // Workaround to avoid wrapping the main thread when we run death tests.
115 // The approach we take for detecting death tests is essentially the same
116 // as gtest does internally. Gtest does this:
117 //
118 // static const char kDeathTestCaseFilter[] = "*DeathTest:*DeathTest/*";
119 // ::testing::internal::UnitTestOptions::MatchesFilter(
120 // test_case_name, kDeathTestCaseFilter);
121 //
122 // Our approach is a little more straight forward.
123 if (absl::EndsWith(test_case_name, "DeathTest"))
124 return true;
125
126 return absl::EndsWith(test_name, "DeathTest");
127 }
128
OnTestStart(const::testing::TestInfo & test_info)129 void OnTestStart(const ::testing::TestInfo& test_info) override {
130 if (!IsDeathTest(test_info.test_suite_name(), test_info.name())) {
131 // Ensure that main thread gets wrapped as an rtc::Thread.
132 // TODO(bugs.webrtc.org/9714): It might be better to avoid wrapping the
133 // main thread, or leave it to individual tests that need it. But as
134 // long as we have automatic thread wrapping, we need this to avoid that
135 // some other random thread (which one depending on which tests are run)
136 // gets automatically wrapped.
137 thread_ = rtc::Thread::CreateWithSocketServer();
138 thread_->WrapCurrent();
139 RTC_DCHECK_EQ(rtc::Thread::Current(), thread_.get());
140 } else {
141 RTC_LOG(LS_INFO) << "No thread auto wrap for death test.";
142 }
143 }
144
OnTestEnd(const::testing::TestInfo & test_info)145 void OnTestEnd(const ::testing::TestInfo& test_info) override {
146 // Terminate the message loop. Note that if the test failed to clean
147 // up pending messages, this may execute part of the test. Ideally we
148 // should print a warning message here, or even fail the test if it leaks.
149 if (thread_) {
150 thread_->Quit(); // Signal quit.
151 thread_->Run(); // Flush + process Quit signal.
152 thread_->UnwrapCurrent();
153 thread_ = nullptr;
154 }
155 }
156
157 std::unique_ptr<rtc::Thread> thread_;
158 };
159
Init(int * argc,char * argv[])160 int Init(int* argc, char* argv[]) override {
161 ::testing::InitGoogleMock(argc, argv);
162 absl::ParseCommandLine(*argc, argv);
163
164 // Make sure we always pull in the --resources_dir flag, even if the test
165 // binary doesn't link with fileutils (downstream expects all test mains to
166 // have this flag).
167 (void)absl::GetFlag(FLAGS_resources_dir);
168
169 // Default to LS_INFO, even for release builds to provide better test
170 // logging.
171 if (rtc::LogMessage::GetLogToDebug() > rtc::LS_INFO)
172 rtc::LogMessage::LogToDebug(rtc::LS_INFO);
173
174 if (absl::GetFlag(FLAGS_verbose))
175 rtc::LogMessage::LogToDebug(rtc::LS_VERBOSE);
176
177 rtc::LogMessage::SetLogToStderr(absl::GetFlag(FLAGS_logs) ||
178 absl::GetFlag(FLAGS_verbose));
179
180 // InitFieldTrialsFromString stores the char*, so the char array must
181 // outlive the application.
182 field_trials_ = absl::GetFlag(FLAGS_force_fieldtrials);
183 webrtc::field_trial::InitFieldTrialsFromString(field_trials_.c_str());
184 webrtc::metrics::Enable();
185
186 #if defined(WEBRTC_WIN)
187 winsock_init_ = std::make_unique<rtc::WinsockInitializer>();
188 #endif
189
190 // Initialize SSL which are used by several tests.
191 rtc::InitializeSSL();
192 rtc::SSLStreamAdapter::EnableTimeCallbackForTesting();
193
194 ::testing::UnitTest::GetInstance()->listeners().Append(new TestListener());
195
196 return 0;
197 }
198
Run(int argc,char * argv[])199 int Run(int argc, char* argv[]) override {
200 std::string trace_event_path = absl::GetFlag(FLAGS_trace_event);
201 const bool capture_events = !trace_event_path.empty();
202 if (capture_events) {
203 rtc::tracing::SetupInternalTracer();
204 rtc::tracing::StartInternalCapture(trace_event_path.c_str());
205 }
206
207 absl::optional<std::vector<std::string>> metrics_to_plot =
208 absl::GetFlag(FLAGS_plot);
209
210 if (metrics_to_plot->empty()) {
211 metrics_to_plot = absl::nullopt;
212 } else {
213 if (metrics_to_plot->size() == 1 &&
214 (*metrics_to_plot)[0] == kPlotAllMetrics) {
215 metrics_to_plot->clear();
216 }
217 }
218
219 #if defined(WEBRTC_IOS)
220 rtc::test::InitTestSuite(RUN_ALL_TESTS, argc, argv,
221 absl::GetFlag(FLAGS_write_perf_output_on_ios),
222 metrics_to_plot);
223 rtc::test::RunTestsFromIOSApp();
224 int exit_code = 0;
225 #else
226 int exit_code = RUN_ALL_TESTS();
227
228 std::string perf_output_file =
229 absl::GetFlag(FLAGS_isolated_script_test_perf_output);
230 if (!perf_output_file.empty()) {
231 if (!webrtc::test::WritePerfResults(perf_output_file)) {
232 return 1;
233 }
234 }
235 if (metrics_to_plot) {
236 webrtc::test::PrintPlottableResults(*metrics_to_plot);
237 }
238 #endif
239
240 if (capture_events) {
241 rtc::tracing::StopInternalCapture();
242 }
243
244 #if defined(ADDRESS_SANITIZER) || defined(LEAK_SANITIZER) || \
245 defined(MEMORY_SANITIZER) || defined(THREAD_SANITIZER) || \
246 defined(UNDEFINED_SANITIZER)
247 // We want the test flagged as failed only for sanitizer defects,
248 // in which case the sanitizer will override exit code with 66.
249 exit_code = 0;
250 #endif
251
252 return exit_code;
253 }
254
255 ~TestMainImpl() override = default;
256
257 private:
258 #if defined(WEBRTC_WIN)
259 std::unique_ptr<rtc::WinsockInitializer> winsock_init_;
260 #endif
261 };
262
263 } // namespace
264
Create()265 std::unique_ptr<TestMain> TestMain::Create() {
266 return std::make_unique<TestMainImpl>();
267 }
268
269 } // namespace webrtc
270