1 /*
2  *  Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #include "test/test_main_lib.h"
12 
13 #include <fstream>
14 #include <memory>
15 #include <string>
16 
17 #include "absl/flags/flag.h"
18 #include "absl/flags/parse.h"
19 #include "absl/memory/memory.h"
20 #include "absl/strings/match.h"
21 #include "absl/types/optional.h"
22 #include "rtc_base/checks.h"
23 #include "rtc_base/event_tracer.h"
24 #include "rtc_base/logging.h"
25 #include "rtc_base/ssl_adapter.h"
26 #include "rtc_base/ssl_stream_adapter.h"
27 #include "rtc_base/thread.h"
28 #include "system_wrappers/include/field_trial.h"
29 #include "system_wrappers/include/metrics.h"
30 #include "test/field_trial.h"
31 #include "test/gmock.h"
32 #include "test/gtest.h"
33 #include "test/testsupport/perf_test.h"
34 #include "test/testsupport/resources_dir_flag.h"
35 
36 #if defined(WEBRTC_WIN)
37 #include "rtc_base/win32_socket_init.h"
38 #endif
39 
40 #if defined(WEBRTC_IOS)
41 #include "test/ios/test_support.h"
42 
43 ABSL_FLAG(std::string,
44           NSTreatUnknownArgumentsAsOpen,
45           "",
46           "Intentionally ignored flag intended for iOS simulator.");
47 ABSL_FLAG(std::string,
48           ApplePersistenceIgnoreState,
49           "",
50           "Intentionally ignored flag intended for iOS simulator.");
51 
52 // This is the cousin of isolated_script_test_perf_output, but we can't dictate
53 // where to write on iOS so the semantics of this flag are a bit different.
54 ABSL_FLAG(
55     bool,
56     write_perf_output_on_ios,
57     false,
58     "Store the perf results in Documents/perftest_result.pb in the format "
59     "described by histogram.proto in "
60     "https://chromium.googlesource.com/catapult/.");
61 
62 #else
63 
64 ABSL_FLAG(
65     std::string,
66     isolated_script_test_perf_output,
67     "",
68     "Path where the perf results should be stored in proto format described "
69     "described by histogram.proto in "
70     "https://chromium.googlesource.com/catapult/.");
71 
72 #endif
73 
74 constexpr char kPlotAllMetrics[] = "all";
75 ABSL_FLAG(std::vector<std::string>,
76           plot,
77           {},
78           "List of metrics that should be exported for plotting (if they are "
79           "available). Example: psnr,ssim,encode_time. To plot all available "
80           " metrics pass 'all' as flag value");
81 
82 ABSL_FLAG(bool, logs, true, "print logs to stderr");
83 ABSL_FLAG(bool, verbose, false, "verbose logs to stderr");
84 
85 ABSL_FLAG(std::string,
86           trace_event,
87           "",
88           "Path to collect trace events (json file) for chrome://tracing. "
89           "If not set, events aren't captured.");
90 
91 ABSL_FLAG(std::string,
92           force_fieldtrials,
93           "",
94           "Field trials control experimental feature code which can be forced. "
95           "E.g. running with --force_fieldtrials=WebRTC-FooFeature/Enable/"
96           " will assign the group Enable to field trial WebRTC-FooFeature.");
97 
98 namespace webrtc {
99 
100 namespace {
101 
102 class TestMainImpl : public TestMain {
103  public:
104   // In order to set up a fresh rtc::Thread state for each test and avoid
105   // accidentally carrying over pending tasks that might be sent from one test
106   // and executed while another test is running, we inject a TestListener
107   // that sets up a new rtc::Thread instance for the main thread, per test.
108   class TestListener : public ::testing::EmptyTestEventListener {
109    public:
110     TestListener() = default;
111 
112    private:
IsDeathTest(const char * test_case_name,const char * test_name)113     bool IsDeathTest(const char* test_case_name, const char* test_name) {
114       // Workaround to avoid wrapping the main thread when we run death tests.
115       // The approach we take for detecting death tests is essentially the same
116       // as gtest does internally. Gtest does this:
117       //
118       // static const char kDeathTestCaseFilter[] = "*DeathTest:*DeathTest/*";
119       // ::testing::internal::UnitTestOptions::MatchesFilter(
120       //     test_case_name, kDeathTestCaseFilter);
121       //
122       // Our approach is a little more straight forward.
123       if (absl::EndsWith(test_case_name, "DeathTest"))
124         return true;
125 
126       return absl::EndsWith(test_name, "DeathTest");
127     }
128 
OnTestStart(const::testing::TestInfo & test_info)129     void OnTestStart(const ::testing::TestInfo& test_info) override {
130       if (!IsDeathTest(test_info.test_suite_name(), test_info.name())) {
131         // Ensure that main thread gets wrapped as an rtc::Thread.
132         // TODO(bugs.webrtc.org/9714): It might be better to avoid wrapping the
133         // main thread, or leave it to individual tests that need it. But as
134         // long as we have automatic thread wrapping, we need this to avoid that
135         // some other random thread (which one depending on which tests are run)
136         // gets automatically wrapped.
137         thread_ = rtc::Thread::CreateWithSocketServer();
138         thread_->WrapCurrent();
139         RTC_DCHECK_EQ(rtc::Thread::Current(), thread_.get());
140       } else {
141         RTC_LOG(LS_INFO) << "No thread auto wrap for death test.";
142       }
143     }
144 
OnTestEnd(const::testing::TestInfo & test_info)145     void OnTestEnd(const ::testing::TestInfo& test_info) override {
146       // Terminate the message loop. Note that if the test failed to clean
147       // up pending messages, this may execute part of the test. Ideally we
148       // should print a warning message here, or even fail the test if it leaks.
149       if (thread_) {
150         thread_->Quit();  // Signal quit.
151         thread_->Run();   // Flush + process Quit signal.
152         thread_->UnwrapCurrent();
153         thread_ = nullptr;
154       }
155     }
156 
157     std::unique_ptr<rtc::Thread> thread_;
158   };
159 
Init(int * argc,char * argv[])160   int Init(int* argc, char* argv[]) override {
161     ::testing::InitGoogleMock(argc, argv);
162     absl::ParseCommandLine(*argc, argv);
163 
164     // Make sure we always pull in the --resources_dir flag, even if the test
165     // binary doesn't link with fileutils (downstream expects all test mains to
166     // have this flag).
167     (void)absl::GetFlag(FLAGS_resources_dir);
168 
169     // Default to LS_INFO, even for release builds to provide better test
170     // logging.
171     if (rtc::LogMessage::GetLogToDebug() > rtc::LS_INFO)
172       rtc::LogMessage::LogToDebug(rtc::LS_INFO);
173 
174     if (absl::GetFlag(FLAGS_verbose))
175       rtc::LogMessage::LogToDebug(rtc::LS_VERBOSE);
176 
177     rtc::LogMessage::SetLogToStderr(absl::GetFlag(FLAGS_logs) ||
178                                     absl::GetFlag(FLAGS_verbose));
179 
180     // InitFieldTrialsFromString stores the char*, so the char array must
181     // outlive the application.
182     field_trials_ = absl::GetFlag(FLAGS_force_fieldtrials);
183     webrtc::field_trial::InitFieldTrialsFromString(field_trials_.c_str());
184     webrtc::metrics::Enable();
185 
186 #if defined(WEBRTC_WIN)
187     winsock_init_ = std::make_unique<rtc::WinsockInitializer>();
188 #endif
189 
190     // Initialize SSL which are used by several tests.
191     rtc::InitializeSSL();
192     rtc::SSLStreamAdapter::EnableTimeCallbackForTesting();
193 
194     ::testing::UnitTest::GetInstance()->listeners().Append(new TestListener());
195 
196     return 0;
197   }
198 
Run(int argc,char * argv[])199   int Run(int argc, char* argv[]) override {
200     std::string trace_event_path = absl::GetFlag(FLAGS_trace_event);
201     const bool capture_events = !trace_event_path.empty();
202     if (capture_events) {
203       rtc::tracing::SetupInternalTracer();
204       rtc::tracing::StartInternalCapture(trace_event_path.c_str());
205     }
206 
207     absl::optional<std::vector<std::string>> metrics_to_plot =
208         absl::GetFlag(FLAGS_plot);
209 
210     if (metrics_to_plot->empty()) {
211       metrics_to_plot = absl::nullopt;
212     } else {
213       if (metrics_to_plot->size() == 1 &&
214           (*metrics_to_plot)[0] == kPlotAllMetrics) {
215         metrics_to_plot->clear();
216       }
217     }
218 
219 #if defined(WEBRTC_IOS)
220     rtc::test::InitTestSuite(RUN_ALL_TESTS, argc, argv,
221                              absl::GetFlag(FLAGS_write_perf_output_on_ios),
222                              metrics_to_plot);
223     rtc::test::RunTestsFromIOSApp();
224     int exit_code = 0;
225 #else
226     int exit_code = RUN_ALL_TESTS();
227 
228     std::string perf_output_file =
229         absl::GetFlag(FLAGS_isolated_script_test_perf_output);
230     if (!perf_output_file.empty()) {
231       if (!webrtc::test::WritePerfResults(perf_output_file)) {
232         return 1;
233       }
234     }
235     if (metrics_to_plot) {
236       webrtc::test::PrintPlottableResults(*metrics_to_plot);
237     }
238 #endif
239 
240     if (capture_events) {
241       rtc::tracing::StopInternalCapture();
242     }
243 
244 #if defined(ADDRESS_SANITIZER) || defined(LEAK_SANITIZER) ||  \
245     defined(MEMORY_SANITIZER) || defined(THREAD_SANITIZER) || \
246     defined(UNDEFINED_SANITIZER)
247     // We want the test flagged as failed only for sanitizer defects,
248     // in which case the sanitizer will override exit code with 66.
249     exit_code = 0;
250 #endif
251 
252     return exit_code;
253   }
254 
255   ~TestMainImpl() override = default;
256 
257  private:
258 #if defined(WEBRTC_WIN)
259   std::unique_ptr<rtc::WinsockInitializer> winsock_init_;
260 #endif
261 };
262 
263 }  // namespace
264 
Create()265 std::unique_ptr<TestMain> TestMain::Create() {
266   return std::make_unique<TestMainImpl>();
267 }
268 
269 }  // namespace webrtc
270