1 /*
2 * Copyright 2017 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "NuPlayer2CCDecoder"
19 #include <utils/Log.h>
20 #include <inttypes.h>
21
22 #include "NuPlayer2CCDecoder.h"
23
24 #include <media/NdkMediaFormat.h>
25 #include <media/stagefright/foundation/ABitReader.h>
26 #include <media/stagefright/foundation/ABuffer.h>
27 #include <media/stagefright/foundation/ADebug.h>
28 #include <media/stagefright/foundation/AMessage.h>
29 #include <media/stagefright/foundation/avc_utils.h>
30 #include <media/stagefright/MediaDefs.h>
31
32 namespace android {
33
34 // In CEA-708B, the maximum bandwidth of CC is set to 9600bps.
35 static const size_t kMaxBandwithSizeBytes = 9600 / 8;
36
37 struct CCData {
CCDataandroid::CCData38 CCData(uint8_t type, uint8_t data1, uint8_t data2)
39 : mType(type), mData1(data1), mData2(data2) {
40 }
getChannelandroid::CCData41 bool getChannel(size_t *channel) const {
42 if (mData1 >= 0x10 && mData1 <= 0x1f) {
43 *channel = (mData1 >= 0x18 ? 1 : 0) + (mType ? 2 : 0);
44 return true;
45 }
46 return false;
47 }
48
49 uint8_t mType;
50 uint8_t mData1;
51 uint8_t mData2;
52 };
53
isNullPad(CCData * cc)54 static bool isNullPad(CCData *cc) {
55 return cc->mData1 < 0x10 && cc->mData2 < 0x10;
56 }
57
58 static void dumpBytePair(const sp<ABuffer> &ccBuf) __attribute__ ((unused));
dumpBytePair(const sp<ABuffer> & ccBuf)59 static void dumpBytePair(const sp<ABuffer> &ccBuf) {
60 size_t offset = 0;
61 AString out;
62
63 while (offset < ccBuf->size()) {
64 char tmp[128];
65
66 CCData *cc = (CCData *) (ccBuf->data() + offset);
67
68 if (isNullPad(cc)) {
69 // 1 null pad or XDS metadata, ignore
70 offset += sizeof(CCData);
71 continue;
72 }
73
74 if (cc->mData1 >= 0x20 && cc->mData1 <= 0x7f) {
75 // 2 basic chars
76 snprintf(tmp, sizeof(tmp), "[%d]Basic: %c %c", cc->mType, cc->mData1, cc->mData2);
77 } else if ((cc->mData1 == 0x11 || cc->mData1 == 0x19)
78 && cc->mData2 >= 0x30 && cc->mData2 <= 0x3f) {
79 // 1 special char
80 snprintf(tmp, sizeof(tmp), "[%d]Special: %02x %02x", cc->mType, cc->mData1, cc->mData2);
81 } else if ((cc->mData1 == 0x12 || cc->mData1 == 0x1A)
82 && cc->mData2 >= 0x20 && cc->mData2 <= 0x3f){
83 // 1 Spanish/French char
84 snprintf(tmp, sizeof(tmp), "[%d]Spanish: %02x %02x", cc->mType, cc->mData1, cc->mData2);
85 } else if ((cc->mData1 == 0x13 || cc->mData1 == 0x1B)
86 && cc->mData2 >= 0x20 && cc->mData2 <= 0x3f){
87 // 1 Portuguese/German/Danish char
88 snprintf(tmp, sizeof(tmp), "[%d]German: %02x %02x", cc->mType, cc->mData1, cc->mData2);
89 } else if ((cc->mData1 == 0x11 || cc->mData1 == 0x19)
90 && cc->mData2 >= 0x20 && cc->mData2 <= 0x2f){
91 // Mid-Row Codes (Table 69)
92 snprintf(tmp, sizeof(tmp), "[%d]Mid-row: %02x %02x", cc->mType, cc->mData1, cc->mData2);
93 } else if (((cc->mData1 == 0x14 || cc->mData1 == 0x1c)
94 && cc->mData2 >= 0x20 && cc->mData2 <= 0x2f)
95 ||
96 ((cc->mData1 == 0x17 || cc->mData1 == 0x1f)
97 && cc->mData2 >= 0x21 && cc->mData2 <= 0x23)){
98 // Misc Control Codes (Table 70)
99 snprintf(tmp, sizeof(tmp), "[%d]Ctrl: %02x %02x", cc->mType, cc->mData1, cc->mData2);
100 } else if ((cc->mData1 & 0x70) == 0x10
101 && (cc->mData2 & 0x40) == 0x40
102 && ((cc->mData1 & 0x07) || !(cc->mData2 & 0x20)) ) {
103 // Preamble Address Codes (Table 71)
104 snprintf(tmp, sizeof(tmp), "[%d]PAC: %02x %02x", cc->mType, cc->mData1, cc->mData2);
105 } else {
106 snprintf(tmp, sizeof(tmp), "[%d]Invalid: %02x %02x", cc->mType, cc->mData1, cc->mData2);
107 }
108
109 if (out.size() > 0) {
110 out.append(", ");
111 }
112
113 out.append(tmp);
114
115 offset += sizeof(CCData);
116 }
117
118 ALOGI("%s", out.c_str());
119 }
120
CCDecoder(const sp<AMessage> & notify)121 NuPlayer2::CCDecoder::CCDecoder(const sp<AMessage> ¬ify)
122 : mNotify(notify),
123 mSelectedTrack(-1),
124 mDTVCCPacket(new ABuffer(kMaxBandwithSizeBytes)) {
125 mDTVCCPacket->setRange(0, 0);
126
127 // In CEA-608, streams from packets which have the value 0 of cc_type contain CC1 and CC2, and
128 // streams from packets which have the value 1 of cc_type contain CC3 and CC4.
129 // The following array indicates the current transmitting channels for each value of cc_type.
130 mLine21Channels[0] = 0; // CC1
131 mLine21Channels[1] = 2; // CC3
132 }
133
getTrackCount() const134 size_t NuPlayer2::CCDecoder::getTrackCount() const {
135 return mTracks.size();
136 }
137
getTrackInfo(size_t index) const138 sp<AMessage> NuPlayer2::CCDecoder::getTrackInfo(size_t index) const {
139 if (!isTrackValid(index)) {
140 return NULL;
141 }
142
143 sp<AMessage> format = new AMessage();
144
145 CCTrack track = mTracks[index];
146
147 format->setInt32("type", MEDIA_TRACK_TYPE_SUBTITLE);
148 format->setString("language", "und");
149
150 switch (track.mTrackType) {
151 case kTrackTypeCEA608:
152 format->setString("mime", MEDIA_MIMETYPE_TEXT_CEA_608);
153 break;
154 case kTrackTypeCEA708:
155 format->setString("mime", MEDIA_MIMETYPE_TEXT_CEA_708);
156 break;
157 default:
158 ALOGE("Unknown track type: %d", track.mTrackType);
159 return NULL;
160 }
161
162 // For CEA-608 CC1, field 0 channel 0
163 bool isDefaultAuto = track.mTrackType == kTrackTypeCEA608
164 && track.mTrackChannel == 0;
165 // For CEA-708, Primary Caption Service.
166 bool isDefaultOnly = track.mTrackType == kTrackTypeCEA708
167 && track.mTrackChannel == 1;
168 format->setInt32("auto", isDefaultAuto);
169 format->setInt32("default", isDefaultAuto || isDefaultOnly);
170 format->setInt32("forced", 0);
171
172 return format;
173 }
174
selectTrack(size_t index,bool select)175 status_t NuPlayer2::CCDecoder::selectTrack(size_t index, bool select) {
176 if (!isTrackValid(index)) {
177 return BAD_VALUE;
178 }
179
180 if (select) {
181 if (mSelectedTrack == (ssize_t)index) {
182 ALOGE("track %zu already selected", index);
183 return BAD_VALUE;
184 }
185 ALOGV("selected track %zu", index);
186 mSelectedTrack = index;
187 } else {
188 if (mSelectedTrack != (ssize_t)index) {
189 ALOGE("track %zu is not selected", index);
190 return BAD_VALUE;
191 }
192 ALOGV("unselected track %zu", index);
193 mSelectedTrack = -1;
194 }
195
196 // Clear the previous track payloads
197 mCCMap.clear();
198
199 return OK;
200 }
201
getSelectedTrack(media_track_type type) const202 ssize_t NuPlayer2::CCDecoder::getSelectedTrack(media_track_type type) const {
203 if (mSelectedTrack != -1) {
204 CCTrack track = mTracks[mSelectedTrack];
205 if (track.mTrackType == kTrackTypeCEA608 || track.mTrackType == kTrackTypeCEA708) {
206 return (type == MEDIA_TRACK_TYPE_SUBTITLE ? mSelectedTrack : -1);
207 }
208 return (type == MEDIA_TRACK_TYPE_UNKNOWN ? mSelectedTrack : -1);
209 }
210
211 return -1;
212 }
213
isSelected() const214 bool NuPlayer2::CCDecoder::isSelected() const {
215 return mSelectedTrack >= 0 && mSelectedTrack < (int32_t)getTrackCount();
216 }
217
isTrackValid(size_t index) const218 bool NuPlayer2::CCDecoder::isTrackValid(size_t index) const {
219 return index < getTrackCount();
220 }
221
222 // returns true if a new CC track is found
extractFromSEI(const sp<ABuffer> & accessUnit)223 bool NuPlayer2::CCDecoder::extractFromSEI(const sp<ABuffer> &accessUnit) {
224 sp<ABuffer> sei;
225 if (!accessUnit->meta()->findBuffer("sei", &sei) || sei == NULL) {
226 return false;
227 }
228
229 int64_t timeUs;
230 CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs));
231
232 bool trackAdded = false;
233
234 const NALPosition *nal = (NALPosition *)sei->data();
235
236 for (size_t i = 0; i < sei->size() / sizeof(NALPosition); ++i, ++nal) {
237 trackAdded |= parseSEINalUnit(
238 timeUs, accessUnit->data() + nal->nalOffset, nal->nalSize);
239 }
240
241 return trackAdded;
242 }
243
244 // returns true if a new CC track is found
parseSEINalUnit(int64_t timeUs,const uint8_t * data,size_t size)245 bool NuPlayer2::CCDecoder::parseSEINalUnit(int64_t timeUs, const uint8_t *data, size_t size) {
246 unsigned nalType = data[0] & 0x1f;
247
248 // the buffer should only have SEI in it
249 if (nalType != 6) {
250 return false;
251 }
252
253 bool trackAdded = false;
254 NALBitReader br(data + 1, size - 1);
255
256 // sei_message()
257 while (br.atLeastNumBitsLeft(16)) { // at least 16-bit for sei_message()
258 uint32_t payload_type = 0;
259 size_t payload_size = 0;
260 uint8_t last_byte;
261
262 do {
263 last_byte = br.getBits(8);
264 payload_type += last_byte;
265 } while (last_byte == 0xFF);
266
267 do {
268 last_byte = br.getBits(8);
269 payload_size += last_byte;
270 } while (last_byte == 0xFF);
271
272 if (payload_size > SIZE_MAX / 8
273 || !br.atLeastNumBitsLeft(payload_size * 8)) {
274 ALOGV("Malformed SEI payload");
275 break;
276 }
277
278 // sei_payload()
279 if (payload_type == 4) {
280 bool isCC = false;
281 if (payload_size > 1 + 2 + 4 + 1) {
282 // user_data_registered_itu_t_t35()
283
284 // ATSC A/72: 6.4.2
285 uint8_t itu_t_t35_country_code = br.getBits(8);
286 uint16_t itu_t_t35_provider_code = br.getBits(16);
287 uint32_t user_identifier = br.getBits(32);
288 uint8_t user_data_type_code = br.getBits(8);
289
290 payload_size -= 1 + 2 + 4 + 1;
291
292 isCC = itu_t_t35_country_code == 0xB5
293 && itu_t_t35_provider_code == 0x0031
294 && user_identifier == 'GA94'
295 && user_data_type_code == 0x3;
296 }
297
298 if (isCC && payload_size > 2) {
299 trackAdded |= parseMPEGCCData(timeUs, br.data(), br.numBitsLeft() / 8);
300 } else {
301 ALOGV("Malformed SEI payload type 4");
302 }
303 } else {
304 ALOGV("Unsupported SEI payload type %d", payload_type);
305 }
306
307 // skipping remaining bits of this payload
308 br.skipBits(payload_size * 8);
309 }
310
311 return trackAdded;
312 }
313
314 // returns true if a new CC track is found
extractFromMPEGUserData(const sp<ABuffer> & accessUnit)315 bool NuPlayer2::CCDecoder::extractFromMPEGUserData(const sp<ABuffer> &accessUnit) {
316 sp<ABuffer> mpegUserData;
317 if (!accessUnit->meta()->findBuffer(AMEDIAFORMAT_KEY_MPEG_USER_DATA, &mpegUserData)
318 || mpegUserData == NULL) {
319 return false;
320 }
321
322 int64_t timeUs;
323 CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs));
324
325 bool trackAdded = false;
326
327 const size_t *userData = (size_t *)mpegUserData->data();
328
329 for (size_t i = 0; i < mpegUserData->size() / sizeof(size_t); ++i) {
330 trackAdded |= parseMPEGUserDataUnit(
331 timeUs, accessUnit->data() + userData[i], accessUnit->size() - userData[i]);
332 }
333
334 return trackAdded;
335 }
336
337 // returns true if a new CC track is found
parseMPEGUserDataUnit(int64_t timeUs,const uint8_t * data,size_t size)338 bool NuPlayer2::CCDecoder::parseMPEGUserDataUnit(int64_t timeUs, const uint8_t *data, size_t size) {
339 ABitReader br(data + 4, 5);
340
341 uint32_t user_identifier = br.getBits(32);
342 uint8_t user_data_type = br.getBits(8);
343
344 if (user_identifier == 'GA94' && user_data_type == 0x3) {
345 return parseMPEGCCData(timeUs, data + 9, size - 9);
346 }
347
348 return false;
349 }
350
351 // returns true if a new CC track is found
parseMPEGCCData(int64_t timeUs,const uint8_t * data,size_t size)352 bool NuPlayer2::CCDecoder::parseMPEGCCData(int64_t timeUs, const uint8_t *data, size_t size) {
353 bool trackAdded = false;
354
355 // MPEG_cc_data()
356 // ATSC A/53 Part 4: 6.2.3.1
357 ABitReader br(data, size);
358
359 if (br.numBitsLeft() <= 16) {
360 return false;
361 }
362
363 br.skipBits(1);
364 bool process_cc_data_flag = br.getBits(1);
365 br.skipBits(1);
366 size_t cc_count = br.getBits(5);
367 br.skipBits(8);
368
369 if (!process_cc_data_flag || 3 * 8 * cc_count >= br.numBitsLeft()) {
370 return false;
371 }
372
373 sp<ABuffer> line21CCBuf = NULL;
374
375 for (size_t i = 0; i < cc_count; ++i) {
376 br.skipBits(5);
377 bool cc_valid = br.getBits(1);
378 uint8_t cc_type = br.getBits(2);
379
380 if (cc_valid) {
381 if (cc_type == 3) {
382 if (mDTVCCPacket->size() > 0) {
383 trackAdded |= parseDTVCCPacket(
384 timeUs, mDTVCCPacket->data(), mDTVCCPacket->size());
385 mDTVCCPacket->setRange(0, 0);
386 }
387 if (mDTVCCPacket->size() + 2 > mDTVCCPacket->capacity()) {
388 return false;
389 }
390 memcpy(mDTVCCPacket->data() + mDTVCCPacket->size(), br.data(), 2);
391 mDTVCCPacket->setRange(0, mDTVCCPacket->size() + 2);
392 br.skipBits(16);
393 } else if (mDTVCCPacket->size() > 0 && cc_type == 2) {
394 if (mDTVCCPacket->size() + 2 > mDTVCCPacket->capacity()) {
395 return false;
396 }
397 memcpy(mDTVCCPacket->data() + mDTVCCPacket->size(), br.data(), 2);
398 mDTVCCPacket->setRange(0, mDTVCCPacket->size() + 2);
399 br.skipBits(16);
400 } else if (cc_type == 0 || cc_type == 1) {
401 uint8_t cc_data_1 = br.getBits(8) & 0x7f;
402 uint8_t cc_data_2 = br.getBits(8) & 0x7f;
403
404 CCData cc(cc_type, cc_data_1, cc_data_2);
405
406 if (isNullPad(&cc)) {
407 continue;
408 }
409
410 size_t channel;
411 if (cc.getChannel(&channel)) {
412 mLine21Channels[cc_type] = channel;
413
414 // create a new track if it does not exist.
415 getTrackIndex(kTrackTypeCEA608, channel, &trackAdded);
416 }
417
418 if (isSelected() && mTracks[mSelectedTrack].mTrackType == kTrackTypeCEA608
419 && mTracks[mSelectedTrack].mTrackChannel == mLine21Channels[cc_type]) {
420 if (line21CCBuf == NULL) {
421 line21CCBuf = new ABuffer((cc_count - i) * sizeof(CCData));
422 line21CCBuf->setRange(0, 0);
423 }
424 if (line21CCBuf->size() + sizeof(cc) > line21CCBuf->capacity()) {
425 return false;
426 }
427 memcpy(line21CCBuf->data() + line21CCBuf->size(), &cc, sizeof(cc));
428 line21CCBuf->setRange(0, line21CCBuf->size() + sizeof(CCData));
429 }
430 } else {
431 br.skipBits(16);
432 }
433 } else {
434 if ((cc_type == 3 || cc_type == 2) && mDTVCCPacket->size() > 0) {
435 trackAdded |= parseDTVCCPacket(timeUs, mDTVCCPacket->data(), mDTVCCPacket->size());
436 mDTVCCPacket->setRange(0, 0);
437 }
438 br.skipBits(16);
439 }
440 }
441
442 if (isSelected() && mTracks[mSelectedTrack].mTrackType == kTrackTypeCEA608
443 && line21CCBuf != NULL && line21CCBuf->size() > 0) {
444 mCCMap.add(timeUs, line21CCBuf);
445 }
446
447 return trackAdded;
448 }
449
450 // returns true if a new CC track is found
parseDTVCCPacket(int64_t timeUs,const uint8_t * data,size_t size)451 bool NuPlayer2::CCDecoder::parseDTVCCPacket(int64_t timeUs, const uint8_t *data, size_t size) {
452 // CEA-708B 5 DTVCC Packet Layer.
453 ABitReader br(data, size);
454 br.skipBits(2);
455
456 size_t packet_size = br.getBits(6);
457 if (packet_size == 0) packet_size = 64;
458 packet_size *= 2;
459
460 if (size != packet_size) {
461 return false;
462 }
463
464 bool trackAdded = false;
465
466 while (br.numBitsLeft() >= 16) {
467 // CEA-708B Figure 5 and 6.
468 uint8_t service_number = br.getBits(3);
469 size_t block_size = br.getBits(5);
470
471 if (service_number == 64) {
472 br.skipBits(2);
473 service_number = br.getBits(6);
474
475 if (service_number < 64) {
476 return trackAdded;
477 }
478 }
479
480 if (br.numBitsLeft() < block_size * 8) {
481 return trackAdded;
482 }
483
484 if (block_size > 0) {
485 size_t trackIndex = getTrackIndex(kTrackTypeCEA708, service_number, &trackAdded);
486 if (mSelectedTrack == (ssize_t)trackIndex) {
487 sp<ABuffer> ccPacket = new ABuffer(block_size);
488 if (ccPacket->capacity() == 0) {
489 return false;
490 }
491 memcpy(ccPacket->data(), br.data(), block_size);
492 mCCMap.add(timeUs, ccPacket);
493 }
494 }
495 br.skipBits(block_size * 8);
496 }
497
498 return trackAdded;
499 }
500
501 // return the track index for a given type and channel.
502 // if the track does not exist, creates a new one.
getTrackIndex(int32_t trackType,size_t channel,bool * trackAdded)503 size_t NuPlayer2::CCDecoder::getTrackIndex(
504 int32_t trackType, size_t channel, bool *trackAdded) {
505 CCTrack track(trackType, channel);
506 ssize_t index = mTrackIndices.indexOfKey(track);
507
508 if (index < 0) {
509 // A new track is added.
510 index = mTracks.size();
511 mTrackIndices.add(track, index);
512 mTracks.add(track);
513 *trackAdded = true;
514 return index;
515 }
516
517 return mTrackIndices.valueAt(index);
518 }
519
decode(const sp<ABuffer> & accessUnit)520 void NuPlayer2::CCDecoder::decode(const sp<ABuffer> &accessUnit) {
521 if (extractFromMPEGUserData(accessUnit) || extractFromSEI(accessUnit)) {
522 sp<AMessage> msg = mNotify->dup();
523 msg->setInt32("what", kWhatTrackAdded);
524 msg->post();
525 }
526 // TODO: extract CC from other sources
527 }
528
display(int64_t timeUs)529 void NuPlayer2::CCDecoder::display(int64_t timeUs) {
530 if (!isSelected()) {
531 return;
532 }
533
534 ssize_t index = mCCMap.indexOfKey(timeUs);
535 if (index < 0) {
536 ALOGV("cc for timestamp %" PRId64 " not found", timeUs);
537 return;
538 }
539
540 sp<ABuffer> ccBuf;
541
542 if (index == 0) {
543 ccBuf = mCCMap.valueAt(index);
544 } else {
545 size_t size = 0;
546
547 for (ssize_t i = 0; i <= index; ++i) {
548 size += mCCMap.valueAt(i)->size();
549 }
550
551 ccBuf = new ABuffer(size);
552 ccBuf->setRange(0, 0);
553
554 if (ccBuf->capacity() > 0) {
555 for (ssize_t i = 0; i <= index; ++i) {
556 sp<ABuffer> buf = mCCMap.valueAt(i);
557 memcpy(ccBuf->data() + ccBuf->size(), buf->data(), buf->size());
558 ccBuf->setRange(0, ccBuf->size() + buf->size());
559 }
560 }
561 }
562
563 if (ccBuf->size() > 0) {
564 #if 0
565 dumpBytePair(ccBuf);
566 #endif
567
568 ccBuf->meta()->setInt32(AMEDIAFORMAT_KEY_TRACK_INDEX, mSelectedTrack);
569 ccBuf->meta()->setInt64("timeUs", timeUs);
570 ccBuf->meta()->setInt64("durationUs", 0LL);
571
572 sp<AMessage> msg = mNotify->dup();
573 msg->setInt32("what", kWhatClosedCaptionData);
574 msg->setBuffer("buffer", ccBuf);
575 msg->post();
576 }
577
578 // remove all entries before timeUs
579 mCCMap.removeItemsAt(0, index + 1);
580 }
581
flush()582 void NuPlayer2::CCDecoder::flush() {
583 mCCMap.clear();
584 mDTVCCPacket->setRange(0, 0);
585 }
586
compare(const NuPlayer2::CCDecoder::CCTrack & rhs) const587 int32_t NuPlayer2::CCDecoder::CCTrack::compare(const NuPlayer2::CCDecoder::CCTrack& rhs) const {
588 int32_t cmp = mTrackType - rhs.mTrackType;
589 if (cmp != 0) return cmp;
590 return mTrackChannel - rhs.mTrackChannel;
591 }
592
operator <(const NuPlayer2::CCDecoder::CCTrack & rhs) const593 bool NuPlayer2::CCDecoder::CCTrack::operator<(const NuPlayer2::CCDecoder::CCTrack& rhs) const {
594 return compare(rhs) < 0;
595 }
596
operator ==(const NuPlayer2::CCDecoder::CCTrack & rhs) const597 bool NuPlayer2::CCDecoder::CCTrack::operator==(const NuPlayer2::CCDecoder::CCTrack& rhs) const {
598 return compare(rhs) == 0;
599 }
600
operator !=(const NuPlayer2::CCDecoder::CCTrack & rhs) const601 bool NuPlayer2::CCDecoder::CCTrack::operator!=(const NuPlayer2::CCDecoder::CCTrack& rhs) const {
602 return compare(rhs) != 0;
603 }
604
605 } // namespace android
606
607