1 /*
2  *  Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #include "modules/audio_coding/neteq/tools/neteq_test.h"
12 
13 #include <iomanip>
14 #include <iostream>
15 
16 #include "modules/audio_coding/neteq/default_neteq_factory.h"
17 #include "modules/rtp_rtcp/source/byte_io.h"
18 #include "system_wrappers/include/clock.h"
19 
20 namespace webrtc {
21 namespace test {
22 namespace {
23 
ActionToOperations(absl::optional<NetEqSimulator::Action> a)24 absl::optional<NetEq::Operation> ActionToOperations(
25     absl::optional<NetEqSimulator::Action> a) {
26   if (!a) {
27     return absl::nullopt;
28   }
29   switch (*a) {
30     case NetEqSimulator::Action::kAccelerate:
31       return absl::make_optional(NetEq::Operation::kAccelerate);
32     case NetEqSimulator::Action::kExpand:
33       return absl::make_optional(NetEq::Operation::kExpand);
34     case NetEqSimulator::Action::kNormal:
35       return absl::make_optional(NetEq::Operation::kNormal);
36     case NetEqSimulator::Action::kPreemptiveExpand:
37       return absl::make_optional(NetEq::Operation::kPreemptiveExpand);
38   }
39 }
40 
CreateNetEq(const NetEq::Config & config,Clock * clock,const rtc::scoped_refptr<AudioDecoderFactory> & decoder_factory)41 std::unique_ptr<NetEq> CreateNetEq(
42     const NetEq::Config& config,
43     Clock* clock,
44     const rtc::scoped_refptr<AudioDecoderFactory>& decoder_factory) {
45   return DefaultNetEqFactory().CreateNetEq(config, decoder_factory, clock);
46 }
47 
48 }  // namespace
49 
OnInsertPacketError(const NetEqInput::PacketData & packet)50 void DefaultNetEqTestErrorCallback::OnInsertPacketError(
51     const NetEqInput::PacketData& packet) {
52   std::cerr << "InsertPacket returned an error." << std::endl;
53   std::cerr << "Packet data: " << packet.ToString() << std::endl;
54   FATAL();
55 }
56 
OnGetAudioError()57 void DefaultNetEqTestErrorCallback::OnGetAudioError() {
58   std::cerr << "GetAudio returned an error." << std::endl;
59   FATAL();
60 }
61 
NetEqTest(const NetEq::Config & config,rtc::scoped_refptr<AudioDecoderFactory> decoder_factory,const DecoderMap & codecs,std::unique_ptr<std::ofstream> text_log,NetEqFactory * neteq_factory,std::unique_ptr<NetEqInput> input,std::unique_ptr<AudioSink> output,Callbacks callbacks)62 NetEqTest::NetEqTest(const NetEq::Config& config,
63                      rtc::scoped_refptr<AudioDecoderFactory> decoder_factory,
64                      const DecoderMap& codecs,
65                      std::unique_ptr<std::ofstream> text_log,
66                      NetEqFactory* neteq_factory,
67                      std::unique_ptr<NetEqInput> input,
68                      std::unique_ptr<AudioSink> output,
69                      Callbacks callbacks)
70     : clock_(0),
71       neteq_(neteq_factory
72                  ? neteq_factory->CreateNetEq(config, decoder_factory, &clock_)
73                  : CreateNetEq(config, &clock_, decoder_factory)),
74       input_(std::move(input)),
75       output_(std::move(output)),
76       callbacks_(callbacks),
77       sample_rate_hz_(config.sample_rate_hz),
78       text_log_(std::move(text_log)) {
79   RTC_CHECK(!config.enable_muted_state)
80       << "The code does not handle enable_muted_state";
81   RegisterDecoders(codecs);
82 }
83 
84 NetEqTest::~NetEqTest() = default;
85 
Run()86 int64_t NetEqTest::Run() {
87   int64_t simulation_time = 0;
88   SimulationStepResult step_result;
89   do {
90     step_result = RunToNextGetAudio();
91     simulation_time += step_result.simulation_step_ms;
92   } while (!step_result.is_simulation_finished);
93   if (callbacks_.simulation_ended_callback) {
94     callbacks_.simulation_ended_callback->SimulationEnded(simulation_time,
95                                                           neteq_.get());
96   }
97   return simulation_time;
98 }
99 
RunToNextGetAudio()100 NetEqTest::SimulationStepResult NetEqTest::RunToNextGetAudio() {
101   SimulationStepResult result;
102   const int64_t start_time_ms = *input_->NextEventTime();
103   int64_t time_now_ms = start_time_ms;
104   current_state_.packet_iat_ms.clear();
105 
106   while (!input_->ended()) {
107     // Advance time to next event.
108     RTC_DCHECK(input_->NextEventTime());
109     clock_.AdvanceTimeMilliseconds(*input_->NextEventTime() - time_now_ms);
110     time_now_ms = *input_->NextEventTime();
111     // Check if it is time to insert packet.
112     if (input_->NextPacketTime() && time_now_ms >= *input_->NextPacketTime()) {
113       std::unique_ptr<NetEqInput::PacketData> packet_data = input_->PopPacket();
114       RTC_CHECK(packet_data);
115       const size_t payload_data_length =
116           packet_data->payload.size() - packet_data->header.paddingLength;
117       if (payload_data_length != 0) {
118         int error = neteq_->InsertPacket(
119             packet_data->header,
120             rtc::ArrayView<const uint8_t>(packet_data->payload));
121         if (error != NetEq::kOK && callbacks_.error_callback) {
122           callbacks_.error_callback->OnInsertPacketError(*packet_data);
123         }
124         if (callbacks_.post_insert_packet) {
125           callbacks_.post_insert_packet->AfterInsertPacket(*packet_data,
126                                                            neteq_.get());
127         }
128       } else {
129         neteq_->InsertEmptyPacket(packet_data->header);
130       }
131       if (last_packet_time_ms_) {
132         current_state_.packet_iat_ms.push_back(time_now_ms -
133                                                *last_packet_time_ms_);
134       }
135       if (text_log_) {
136         const auto ops_state = neteq_->GetOperationsAndState();
137         const auto delta_wallclock =
138             last_packet_time_ms_ ? (time_now_ms - *last_packet_time_ms_) : -1;
139         const auto delta_timestamp =
140             last_packet_timestamp_
141                 ? (static_cast<int64_t>(packet_data->header.timestamp) -
142                    *last_packet_timestamp_) *
143                       1000 / sample_rate_hz_
144                 : -1;
145         const auto packet_size_bytes =
146             packet_data->payload.size() == 12
147                 ? ByteReader<uint32_t>::ReadLittleEndian(
148                       &packet_data->payload[8])
149                 : -1;
150         *text_log_ << "Packet   - wallclock: " << std::setw(5) << time_now_ms
151                    << ", delta wc: " << std::setw(4) << delta_wallclock
152                    << ", seq_no: " << packet_data->header.sequenceNumber
153                    << ", timestamp: " << std::setw(10)
154                    << packet_data->header.timestamp
155                    << ", delta ts: " << std::setw(4) << delta_timestamp
156                    << ", size: " << std::setw(5) << packet_size_bytes
157                    << ", frame size: " << std::setw(3)
158                    << ops_state.current_frame_size_ms
159                    << ", buffer size: " << std::setw(4)
160                    << ops_state.current_buffer_size_ms << std::endl;
161       }
162       last_packet_time_ms_ = absl::make_optional<int>(time_now_ms);
163       last_packet_timestamp_ =
164           absl::make_optional<uint32_t>(packet_data->header.timestamp);
165     }
166 
167     // Check if it is time to get output audio.
168     if (input_->NextOutputEventTime() &&
169         time_now_ms >= *input_->NextOutputEventTime()) {
170       if (callbacks_.get_audio_callback) {
171         callbacks_.get_audio_callback->BeforeGetAudio(neteq_.get());
172       }
173       AudioFrame out_frame;
174       bool muted;
175       int error = neteq_->GetAudio(&out_frame, &muted,
176                                    ActionToOperations(next_action_));
177       next_action_ = absl::nullopt;
178       RTC_CHECK(!muted) << "The code does not handle enable_muted_state";
179       if (error != NetEq::kOK) {
180         if (callbacks_.error_callback) {
181           callbacks_.error_callback->OnGetAudioError();
182         }
183       } else {
184         sample_rate_hz_ = out_frame.sample_rate_hz_;
185       }
186       if (callbacks_.get_audio_callback) {
187         callbacks_.get_audio_callback->AfterGetAudio(time_now_ms, out_frame,
188                                                      muted, neteq_.get());
189       }
190 
191       if (output_) {
192         RTC_CHECK(output_->WriteArray(
193             out_frame.data(),
194             out_frame.samples_per_channel_ * out_frame.num_channels_));
195       }
196 
197       input_->AdvanceOutputEvent();
198       result.simulation_step_ms =
199           input_->NextEventTime().value_or(time_now_ms) - start_time_ms;
200       const auto operations_state = neteq_->GetOperationsAndState();
201       current_state_.current_delay_ms = operations_state.current_buffer_size_ms;
202       current_state_.packet_size_ms = operations_state.current_frame_size_ms;
203       current_state_.next_packet_available =
204           operations_state.next_packet_available;
205       current_state_.packet_buffer_flushed =
206           operations_state.packet_buffer_flushes >
207           prev_ops_state_.packet_buffer_flushes;
208       // TODO(ivoc): Add more accurate reporting by tracking the origin of
209       // samples in the sync buffer.
210       result.action_times_ms[Action::kExpand] = 0;
211       result.action_times_ms[Action::kAccelerate] = 0;
212       result.action_times_ms[Action::kPreemptiveExpand] = 0;
213       result.action_times_ms[Action::kNormal] = 0;
214 
215       if (out_frame.speech_type_ == AudioFrame::SpeechType::kPLC ||
216           out_frame.speech_type_ == AudioFrame::SpeechType::kPLCCNG) {
217         // Consider the whole frame to be the result of expansion.
218         result.action_times_ms[Action::kExpand] = 10;
219       } else if (operations_state.accelerate_samples -
220                      prev_ops_state_.accelerate_samples >
221                  0) {
222         // Consider the whole frame to be the result of acceleration.
223         result.action_times_ms[Action::kAccelerate] = 10;
224       } else if (operations_state.preemptive_samples -
225                      prev_ops_state_.preemptive_samples >
226                  0) {
227         // Consider the whole frame to be the result of preemptive expansion.
228         result.action_times_ms[Action::kPreemptiveExpand] = 10;
229       } else {
230         // Consider the whole frame to be the result of normal playout.
231         result.action_times_ms[Action::kNormal] = 10;
232       }
233       auto lifetime_stats = LifetimeStats();
234       if (text_log_) {
235         const bool plc =
236             (out_frame.speech_type_ == AudioFrame::SpeechType::kPLC) ||
237             (out_frame.speech_type_ == AudioFrame::SpeechType::kPLCCNG);
238         const bool cng = out_frame.speech_type_ == AudioFrame::SpeechType::kCNG;
239         const bool voice_concealed =
240             (lifetime_stats.concealed_samples -
241              lifetime_stats.silent_concealed_samples) >
242             (prev_lifetime_stats_.concealed_samples -
243              prev_lifetime_stats_.silent_concealed_samples);
244         *text_log_ << "GetAudio - wallclock: " << std::setw(5) << time_now_ms
245                    << ", delta wc: " << std::setw(4)
246                    << (input_->NextEventTime().value_or(time_now_ms) -
247                        start_time_ms)
248                    << ", CNG: " << cng << ", PLC: " << plc
249                    << ", voice concealed: " << voice_concealed
250                    << ", buffer size: " << std::setw(4)
251                    << current_state_.current_delay_ms << std::endl;
252         if (operations_state.discarded_primary_packets >
253             prev_ops_state_.discarded_primary_packets) {
254           *text_log_ << "Discarded "
255                      << (operations_state.discarded_primary_packets -
256                          prev_ops_state_.discarded_primary_packets)
257                      << " primary packets." << std::endl;
258         }
259         if (operations_state.packet_buffer_flushes >
260             prev_ops_state_.packet_buffer_flushes) {
261           *text_log_ << "Flushed packet buffer "
262                      << (operations_state.packet_buffer_flushes -
263                          prev_ops_state_.packet_buffer_flushes)
264                      << " times." << std::endl;
265         }
266       }
267       prev_lifetime_stats_ = lifetime_stats;
268       const bool no_more_packets_to_decode =
269           !input_->NextPacketTime() && !operations_state.next_packet_available;
270       result.is_simulation_finished =
271           no_more_packets_to_decode || input_->ended();
272       prev_ops_state_ = operations_state;
273       return result;
274     }
275   }
276   result.simulation_step_ms =
277       input_->NextEventTime().value_or(time_now_ms) - start_time_ms;
278   result.is_simulation_finished = true;
279   return result;
280 }
281 
SetNextAction(NetEqTest::Action next_operation)282 void NetEqTest::SetNextAction(NetEqTest::Action next_operation) {
283   next_action_ = absl::optional<Action>(next_operation);
284 }
285 
GetNetEqState()286 NetEqTest::NetEqState NetEqTest::GetNetEqState() {
287   return current_state_;
288 }
289 
SimulationStats()290 NetEqNetworkStatistics NetEqTest::SimulationStats() {
291   NetEqNetworkStatistics stats;
292   RTC_CHECK_EQ(neteq_->NetworkStatistics(&stats), 0);
293   return stats;
294 }
295 
LifetimeStats() const296 NetEqLifetimeStatistics NetEqTest::LifetimeStats() const {
297   return neteq_->GetLifetimeStatistics();
298 }
299 
StandardDecoderMap()300 NetEqTest::DecoderMap NetEqTest::StandardDecoderMap() {
301   DecoderMap codecs = {
302     {0, SdpAudioFormat("pcmu", 8000, 1)},
303     {8, SdpAudioFormat("pcma", 8000, 1)},
304 #ifdef WEBRTC_CODEC_ILBC
305     {102, SdpAudioFormat("ilbc", 8000, 1)},
306 #endif
307     {103, SdpAudioFormat("isac", 16000, 1)},
308 #if !defined(WEBRTC_ANDROID)
309     {104, SdpAudioFormat("isac", 32000, 1)},
310 #endif
311 #ifdef WEBRTC_CODEC_OPUS
312     {111, SdpAudioFormat("opus", 48000, 2)},
313 #endif
314     {93, SdpAudioFormat("l16", 8000, 1)},
315     {94, SdpAudioFormat("l16", 16000, 1)},
316     {95, SdpAudioFormat("l16", 32000, 1)},
317     {96, SdpAudioFormat("l16", 48000, 1)},
318     {9, SdpAudioFormat("g722", 8000, 1)},
319     {106, SdpAudioFormat("telephone-event", 8000, 1)},
320     {114, SdpAudioFormat("telephone-event", 16000, 1)},
321     {115, SdpAudioFormat("telephone-event", 32000, 1)},
322     {116, SdpAudioFormat("telephone-event", 48000, 1)},
323     {117, SdpAudioFormat("red", 8000, 1)},
324     {13, SdpAudioFormat("cn", 8000, 1)},
325     {98, SdpAudioFormat("cn", 16000, 1)},
326     {99, SdpAudioFormat("cn", 32000, 1)},
327     {100, SdpAudioFormat("cn", 48000, 1)}
328   };
329   return codecs;
330 }
331 
RegisterDecoders(const DecoderMap & codecs)332 void NetEqTest::RegisterDecoders(const DecoderMap& codecs) {
333   for (const auto& c : codecs) {
334     RTC_CHECK(neteq_->RegisterPayloadType(c.first, c.second))
335         << "Cannot register " << c.second.name << " to payload type "
336         << c.first;
337   }
338 }
339 
340 }  // namespace test
341 }  // namespace webrtc
342