1 /*
2  * Copyright 2017 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "NuPlayer2CCDecoder"
19 #include <utils/Log.h>
20 #include <inttypes.h>
21 
22 #include "NuPlayer2CCDecoder.h"
23 
24 #include <media/NdkMediaFormat.h>
25 #include <media/stagefright/foundation/ABitReader.h>
26 #include <media/stagefright/foundation/ABuffer.h>
27 #include <media/stagefright/foundation/ADebug.h>
28 #include <media/stagefright/foundation/AMessage.h>
29 #include <media/stagefright/foundation/avc_utils.h>
30 #include <media/stagefright/MediaDefs.h>
31 
32 namespace android {
33 
34 // In CEA-708B, the maximum bandwidth of CC is set to 9600bps.
35 static const size_t kMaxBandwithSizeBytes = 9600 / 8;
36 
37 struct CCData {
CCDataandroid::CCData38     CCData(uint8_t type, uint8_t data1, uint8_t data2)
39         : mType(type), mData1(data1), mData2(data2) {
40     }
getChannelandroid::CCData41     bool getChannel(size_t *channel) const {
42         if (mData1 >= 0x10 && mData1 <= 0x1f) {
43             *channel = (mData1 >= 0x18 ? 1 : 0) + (mType ? 2 : 0);
44             return true;
45         }
46         return false;
47     }
48 
49     uint8_t mType;
50     uint8_t mData1;
51     uint8_t mData2;
52 };
53 
isNullPad(CCData * cc)54 static bool isNullPad(CCData *cc) {
55     return cc->mData1 < 0x10 && cc->mData2 < 0x10;
56 }
57 
58 static void dumpBytePair(const sp<ABuffer> &ccBuf) __attribute__ ((unused));
dumpBytePair(const sp<ABuffer> & ccBuf)59 static void dumpBytePair(const sp<ABuffer> &ccBuf) {
60     size_t offset = 0;
61     AString out;
62 
63     while (offset < ccBuf->size()) {
64         char tmp[128];
65 
66         CCData *cc = (CCData *) (ccBuf->data() + offset);
67 
68         if (isNullPad(cc)) {
69             // 1 null pad or XDS metadata, ignore
70             offset += sizeof(CCData);
71             continue;
72         }
73 
74         if (cc->mData1 >= 0x20 && cc->mData1 <= 0x7f) {
75             // 2 basic chars
76             snprintf(tmp, sizeof(tmp), "[%d]Basic: %c %c", cc->mType, cc->mData1, cc->mData2);
77         } else if ((cc->mData1 == 0x11 || cc->mData1 == 0x19)
78                  && cc->mData2 >= 0x30 && cc->mData2 <= 0x3f) {
79             // 1 special char
80             snprintf(tmp, sizeof(tmp), "[%d]Special: %02x %02x", cc->mType, cc->mData1, cc->mData2);
81         } else if ((cc->mData1 == 0x12 || cc->mData1 == 0x1A)
82                  && cc->mData2 >= 0x20 && cc->mData2 <= 0x3f){
83             // 1 Spanish/French char
84             snprintf(tmp, sizeof(tmp), "[%d]Spanish: %02x %02x", cc->mType, cc->mData1, cc->mData2);
85         } else if ((cc->mData1 == 0x13 || cc->mData1 == 0x1B)
86                  && cc->mData2 >= 0x20 && cc->mData2 <= 0x3f){
87             // 1 Portuguese/German/Danish char
88             snprintf(tmp, sizeof(tmp), "[%d]German: %02x %02x", cc->mType, cc->mData1, cc->mData2);
89         } else if ((cc->mData1 == 0x11 || cc->mData1 == 0x19)
90                  && cc->mData2 >= 0x20 && cc->mData2 <= 0x2f){
91             // Mid-Row Codes (Table 69)
92             snprintf(tmp, sizeof(tmp), "[%d]Mid-row: %02x %02x", cc->mType, cc->mData1, cc->mData2);
93         } else if (((cc->mData1 == 0x14 || cc->mData1 == 0x1c)
94                   && cc->mData2 >= 0x20 && cc->mData2 <= 0x2f)
95                   ||
96                    ((cc->mData1 == 0x17 || cc->mData1 == 0x1f)
97                   && cc->mData2 >= 0x21 && cc->mData2 <= 0x23)){
98             // Misc Control Codes (Table 70)
99             snprintf(tmp, sizeof(tmp), "[%d]Ctrl: %02x %02x", cc->mType, cc->mData1, cc->mData2);
100         } else if ((cc->mData1 & 0x70) == 0x10
101                 && (cc->mData2 & 0x40) == 0x40
102                 && ((cc->mData1 & 0x07) || !(cc->mData2 & 0x20)) ) {
103             // Preamble Address Codes (Table 71)
104             snprintf(tmp, sizeof(tmp), "[%d]PAC: %02x %02x", cc->mType, cc->mData1, cc->mData2);
105         } else {
106             snprintf(tmp, sizeof(tmp), "[%d]Invalid: %02x %02x", cc->mType, cc->mData1, cc->mData2);
107         }
108 
109         if (out.size() > 0) {
110             out.append(", ");
111         }
112 
113         out.append(tmp);
114 
115         offset += sizeof(CCData);
116     }
117 
118     ALOGI("%s", out.c_str());
119 }
120 
CCDecoder(const sp<AMessage> & notify)121 NuPlayer2::CCDecoder::CCDecoder(const sp<AMessage> &notify)
122     : mNotify(notify),
123       mSelectedTrack(-1),
124       mDTVCCPacket(new ABuffer(kMaxBandwithSizeBytes)) {
125     mDTVCCPacket->setRange(0, 0);
126 
127     // In CEA-608, streams from packets which have the value 0 of cc_type contain CC1 and CC2, and
128     // streams from packets which have the value 1 of cc_type contain CC3 and CC4.
129     // The following array indicates the current transmitting channels for each value of cc_type.
130     mLine21Channels[0] = 0; // CC1
131     mLine21Channels[1] = 2; // CC3
132 }
133 
getTrackCount() const134 size_t NuPlayer2::CCDecoder::getTrackCount() const {
135     return mTracks.size();
136 }
137 
getTrackInfo(size_t index) const138 sp<AMessage> NuPlayer2::CCDecoder::getTrackInfo(size_t index) const {
139     if (!isTrackValid(index)) {
140         return NULL;
141     }
142 
143     sp<AMessage> format = new AMessage();
144 
145     CCTrack track = mTracks[index];
146 
147     format->setInt32("type", MEDIA_TRACK_TYPE_SUBTITLE);
148     format->setString("language", "und");
149 
150     switch (track.mTrackType) {
151         case kTrackTypeCEA608:
152             format->setString("mime", MEDIA_MIMETYPE_TEXT_CEA_608);
153             break;
154         case kTrackTypeCEA708:
155             format->setString("mime", MEDIA_MIMETYPE_TEXT_CEA_708);
156             break;
157         default:
158             ALOGE("Unknown track type: %d", track.mTrackType);
159             return NULL;
160     }
161 
162     // For CEA-608 CC1, field 0 channel 0
163     bool isDefaultAuto = track.mTrackType == kTrackTypeCEA608
164             && track.mTrackChannel == 0;
165     // For CEA-708, Primary Caption Service.
166     bool isDefaultOnly = track.mTrackType == kTrackTypeCEA708
167             && track.mTrackChannel == 1;
168     format->setInt32("auto", isDefaultAuto);
169     format->setInt32("default", isDefaultAuto || isDefaultOnly);
170     format->setInt32("forced", 0);
171 
172     return format;
173 }
174 
selectTrack(size_t index,bool select)175 status_t NuPlayer2::CCDecoder::selectTrack(size_t index, bool select) {
176     if (!isTrackValid(index)) {
177         return BAD_VALUE;
178     }
179 
180     if (select) {
181         if (mSelectedTrack == (ssize_t)index) {
182             ALOGE("track %zu already selected", index);
183             return BAD_VALUE;
184         }
185         ALOGV("selected track %zu", index);
186         mSelectedTrack = index;
187     } else {
188         if (mSelectedTrack != (ssize_t)index) {
189             ALOGE("track %zu is not selected", index);
190             return BAD_VALUE;
191         }
192         ALOGV("unselected track %zu", index);
193         mSelectedTrack = -1;
194     }
195 
196     // Clear the previous track payloads
197     mCCMap.clear();
198 
199     return OK;
200 }
201 
isSelected() const202 bool NuPlayer2::CCDecoder::isSelected() const {
203     return mSelectedTrack >= 0 && mSelectedTrack < (int32_t)getTrackCount();
204 }
205 
isTrackValid(size_t index) const206 bool NuPlayer2::CCDecoder::isTrackValid(size_t index) const {
207     return index < getTrackCount();
208 }
209 
210 // returns true if a new CC track is found
extractFromSEI(const sp<ABuffer> & accessUnit)211 bool NuPlayer2::CCDecoder::extractFromSEI(const sp<ABuffer> &accessUnit) {
212     sp<ABuffer> sei;
213     if (!accessUnit->meta()->findBuffer("sei", &sei) || sei == NULL) {
214         return false;
215     }
216 
217     int64_t timeUs;
218     CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs));
219 
220     bool trackAdded = false;
221 
222     const NALPosition *nal = (NALPosition *)sei->data();
223 
224     for (size_t i = 0; i < sei->size() / sizeof(NALPosition); ++i, ++nal) {
225         trackAdded |= parseSEINalUnit(
226                 timeUs, accessUnit->data() + nal->nalOffset, nal->nalSize);
227     }
228 
229     return trackAdded;
230 }
231 
232 // returns true if a new CC track is found
parseSEINalUnit(int64_t timeUs,const uint8_t * data,size_t size)233 bool NuPlayer2::CCDecoder::parseSEINalUnit(int64_t timeUs, const uint8_t *data, size_t size) {
234     unsigned nalType = data[0] & 0x1f;
235 
236     // the buffer should only have SEI in it
237     if (nalType != 6) {
238         return false;
239     }
240 
241     bool trackAdded = false;
242     NALBitReader br(data + 1, size - 1);
243 
244     // sei_message()
245     while (br.atLeastNumBitsLeft(16)) { // at least 16-bit for sei_message()
246         uint32_t payload_type = 0;
247         size_t payload_size = 0;
248         uint8_t last_byte;
249 
250         do {
251             last_byte = br.getBits(8);
252             payload_type += last_byte;
253         } while (last_byte == 0xFF);
254 
255         do {
256             last_byte = br.getBits(8);
257             payload_size += last_byte;
258         } while (last_byte == 0xFF);
259 
260         if (payload_size > SIZE_MAX / 8
261                 || !br.atLeastNumBitsLeft(payload_size * 8)) {
262             ALOGV("Malformed SEI payload");
263             break;
264         }
265 
266         // sei_payload()
267         if (payload_type == 4) {
268             bool isCC = false;
269             if (payload_size > 1 + 2 + 4 + 1) {
270                 // user_data_registered_itu_t_t35()
271 
272                 // ATSC A/72: 6.4.2
273                 uint8_t itu_t_t35_country_code = br.getBits(8);
274                 uint16_t itu_t_t35_provider_code = br.getBits(16);
275                 uint32_t user_identifier = br.getBits(32);
276                 uint8_t user_data_type_code = br.getBits(8);
277 
278                 payload_size -= 1 + 2 + 4 + 1;
279 
280                 isCC = itu_t_t35_country_code == 0xB5
281                         && itu_t_t35_provider_code == 0x0031
282                         && user_identifier == 'GA94'
283                         && user_data_type_code == 0x3;
284             }
285 
286             if (isCC && payload_size > 2) {
287                 trackAdded |= parseMPEGCCData(timeUs, br.data(), br.numBitsLeft() / 8);
288             } else {
289                 ALOGV("Malformed SEI payload type 4");
290             }
291         } else {
292             ALOGV("Unsupported SEI payload type %d", payload_type);
293         }
294 
295         // skipping remaining bits of this payload
296         br.skipBits(payload_size * 8);
297     }
298 
299     return trackAdded;
300 }
301 
302 // returns true if a new CC track is found
extractFromMPEGUserData(const sp<ABuffer> & accessUnit)303 bool NuPlayer2::CCDecoder::extractFromMPEGUserData(const sp<ABuffer> &accessUnit) {
304     sp<ABuffer> mpegUserData;
305     if (!accessUnit->meta()->findBuffer(AMEDIAFORMAT_KEY_MPEG_USER_DATA, &mpegUserData)
306             || mpegUserData == NULL) {
307         return false;
308     }
309 
310     int64_t timeUs;
311     CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs));
312 
313     bool trackAdded = false;
314 
315     const size_t *userData = (size_t *)mpegUserData->data();
316 
317     for (size_t i = 0; i < mpegUserData->size() / sizeof(size_t); ++i) {
318         trackAdded |= parseMPEGUserDataUnit(
319                 timeUs, accessUnit->data() + userData[i], accessUnit->size() - userData[i]);
320     }
321 
322     return trackAdded;
323 }
324 
325 // returns true if a new CC track is found
parseMPEGUserDataUnit(int64_t timeUs,const uint8_t * data,size_t size)326 bool NuPlayer2::CCDecoder::parseMPEGUserDataUnit(int64_t timeUs, const uint8_t *data, size_t size) {
327     ABitReader br(data + 4, 5);
328 
329     uint32_t user_identifier = br.getBits(32);
330     uint8_t user_data_type = br.getBits(8);
331 
332     if (user_identifier == 'GA94' && user_data_type == 0x3) {
333         return parseMPEGCCData(timeUs, data + 9, size - 9);
334     }
335 
336     return false;
337 }
338 
339 // returns true if a new CC track is found
parseMPEGCCData(int64_t timeUs,const uint8_t * data,size_t size)340 bool NuPlayer2::CCDecoder::parseMPEGCCData(int64_t timeUs, const uint8_t *data, size_t size) {
341     bool trackAdded = false;
342 
343     // MPEG_cc_data()
344     // ATSC A/53 Part 4: 6.2.3.1
345     ABitReader br(data, size);
346 
347     if (br.numBitsLeft() <= 16) {
348         return false;
349     }
350 
351     br.skipBits(1);
352     bool process_cc_data_flag = br.getBits(1);
353     br.skipBits(1);
354     size_t cc_count = br.getBits(5);
355     br.skipBits(8);
356 
357     if (!process_cc_data_flag || 3 * 8 * cc_count >= br.numBitsLeft()) {
358         return false;
359     }
360 
361     sp<ABuffer> line21CCBuf = NULL;
362 
363     for (size_t i = 0; i < cc_count; ++i) {
364         br.skipBits(5);
365         bool cc_valid = br.getBits(1);
366         uint8_t cc_type = br.getBits(2);
367 
368         if (cc_valid) {
369             if (cc_type == 3) {
370                 if (mDTVCCPacket->size() > 0) {
371                     trackAdded |= parseDTVCCPacket(
372                             timeUs, mDTVCCPacket->data(), mDTVCCPacket->size());
373                     mDTVCCPacket->setRange(0, 0);
374                 }
375                 memcpy(mDTVCCPacket->data() + mDTVCCPacket->size(), br.data(), 2);
376                 mDTVCCPacket->setRange(0, mDTVCCPacket->size() + 2);
377                 br.skipBits(16);
378             } else if (mDTVCCPacket->size() > 0 && cc_type == 2) {
379                 memcpy(mDTVCCPacket->data() + mDTVCCPacket->size(), br.data(), 2);
380                 mDTVCCPacket->setRange(0, mDTVCCPacket->size() + 2);
381                 br.skipBits(16);
382             } else if (cc_type == 0 || cc_type == 1) {
383                 uint8_t cc_data_1 = br.getBits(8) & 0x7f;
384                 uint8_t cc_data_2 = br.getBits(8) & 0x7f;
385 
386                 CCData cc(cc_type, cc_data_1, cc_data_2);
387 
388                 if (isNullPad(&cc)) {
389                     continue;
390                 }
391 
392                 size_t channel;
393                 if (cc.getChannel(&channel)) {
394                     mLine21Channels[cc_type] = channel;
395 
396                     // create a new track if it does not exist.
397                     getTrackIndex(kTrackTypeCEA608, channel, &trackAdded);
398                 }
399 
400                 if (isSelected() && mTracks[mSelectedTrack].mTrackType == kTrackTypeCEA608
401                         && mTracks[mSelectedTrack].mTrackChannel == mLine21Channels[cc_type]) {
402                     if (line21CCBuf == NULL) {
403                         line21CCBuf = new ABuffer((cc_count - i) * sizeof(CCData));
404                         line21CCBuf->setRange(0, 0);
405                     }
406                     memcpy(line21CCBuf->data() + line21CCBuf->size(), &cc, sizeof(cc));
407                     line21CCBuf->setRange(0, line21CCBuf->size() + sizeof(CCData));
408                 }
409             } else {
410                 br.skipBits(16);
411             }
412         } else {
413             if ((cc_type == 3 || cc_type == 2) && mDTVCCPacket->size() > 0) {
414                 trackAdded |= parseDTVCCPacket(timeUs, mDTVCCPacket->data(), mDTVCCPacket->size());
415                 mDTVCCPacket->setRange(0, 0);
416             }
417             br.skipBits(16);
418         }
419     }
420 
421     if (isSelected() && mTracks[mSelectedTrack].mTrackType == kTrackTypeCEA608
422             && line21CCBuf != NULL && line21CCBuf->size() > 0) {
423         mCCMap.add(timeUs, line21CCBuf);
424     }
425 
426     return trackAdded;
427 }
428 
429 // returns true if a new CC track is found
parseDTVCCPacket(int64_t timeUs,const uint8_t * data,size_t size)430 bool NuPlayer2::CCDecoder::parseDTVCCPacket(int64_t timeUs, const uint8_t *data, size_t size) {
431     // CEA-708B 5 DTVCC Packet Layer.
432     ABitReader br(data, size);
433     br.skipBits(2);
434 
435     size_t packet_size = br.getBits(6);
436     if (packet_size == 0) packet_size = 64;
437     packet_size *= 2;
438 
439     if (size != packet_size) {
440         return false;
441     }
442 
443     bool trackAdded = false;
444 
445     while (br.numBitsLeft() >= 16) {
446         // CEA-708B Figure 5 and 6.
447         uint8_t service_number = br.getBits(3);
448         size_t block_size = br.getBits(5);
449 
450         if (service_number == 64) {
451             br.skipBits(2);
452             service_number = br.getBits(6);
453 
454             if (service_number < 64) {
455                 return trackAdded;
456             }
457         }
458 
459         if (br.numBitsLeft() < block_size * 8) {
460             return trackAdded;
461         }
462 
463         if (block_size > 0) {
464             size_t trackIndex = getTrackIndex(kTrackTypeCEA708, service_number, &trackAdded);
465             if (mSelectedTrack == (ssize_t)trackIndex) {
466                 sp<ABuffer> ccPacket = new ABuffer(block_size);
467                 memcpy(ccPacket->data(), br.data(), block_size);
468                 mCCMap.add(timeUs, ccPacket);
469             }
470         }
471         br.skipBits(block_size * 8);
472     }
473 
474     return trackAdded;
475 }
476 
477 // return the track index for a given type and channel.
478 // if the track does not exist, creates a new one.
getTrackIndex(int32_t trackType,size_t channel,bool * trackAdded)479 size_t NuPlayer2::CCDecoder::getTrackIndex(
480         int32_t trackType, size_t channel, bool *trackAdded) {
481     CCTrack track(trackType, channel);
482     ssize_t index = mTrackIndices.indexOfKey(track);
483 
484     if (index < 0) {
485         // A new track is added.
486         index = mTracks.size();
487         mTrackIndices.add(track, index);
488         mTracks.add(track);
489         *trackAdded = true;
490         return index;
491     }
492 
493     return mTrackIndices.valueAt(index);
494 }
495 
decode(const sp<ABuffer> & accessUnit)496 void NuPlayer2::CCDecoder::decode(const sp<ABuffer> &accessUnit) {
497     if (extractFromMPEGUserData(accessUnit) || extractFromSEI(accessUnit)) {
498         sp<AMessage> msg = mNotify->dup();
499         msg->setInt32("what", kWhatTrackAdded);
500         msg->post();
501     }
502     // TODO: extract CC from other sources
503 }
504 
display(int64_t timeUs)505 void NuPlayer2::CCDecoder::display(int64_t timeUs) {
506     if (!isSelected()) {
507         return;
508     }
509 
510     ssize_t index = mCCMap.indexOfKey(timeUs);
511     if (index < 0) {
512         ALOGV("cc for timestamp %" PRId64 " not found", timeUs);
513         return;
514     }
515 
516     sp<ABuffer> ccBuf;
517 
518     if (index == 0) {
519         ccBuf = mCCMap.valueAt(index);
520     } else {
521         size_t size = 0;
522 
523         for (ssize_t i = 0; i <= index; ++i) {
524             size += mCCMap.valueAt(i)->size();
525         }
526 
527         ccBuf = new ABuffer(size);
528         ccBuf->setRange(0, 0);
529 
530         for (ssize_t i = 0; i <= index; ++i) {
531             sp<ABuffer> buf = mCCMap.valueAt(i);
532             memcpy(ccBuf->data() + ccBuf->size(), buf->data(), buf->size());
533             ccBuf->setRange(0, ccBuf->size() + buf->size());
534         }
535     }
536 
537     if (ccBuf->size() > 0) {
538 #if 0
539         dumpBytePair(ccBuf);
540 #endif
541 
542         ccBuf->meta()->setInt32(AMEDIAFORMAT_KEY_TRACK_INDEX, mSelectedTrack);
543         ccBuf->meta()->setInt64("timeUs", timeUs);
544         ccBuf->meta()->setInt64("durationUs", 0ll);
545 
546         sp<AMessage> msg = mNotify->dup();
547         msg->setInt32("what", kWhatClosedCaptionData);
548         msg->setBuffer("buffer", ccBuf);
549         msg->post();
550     }
551 
552     // remove all entries before timeUs
553     mCCMap.removeItemsAt(0, index + 1);
554 }
555 
flush()556 void NuPlayer2::CCDecoder::flush() {
557     mCCMap.clear();
558     mDTVCCPacket->setRange(0, 0);
559 }
560 
compare(const NuPlayer2::CCDecoder::CCTrack & rhs) const561 int32_t NuPlayer2::CCDecoder::CCTrack::compare(const NuPlayer2::CCDecoder::CCTrack& rhs) const {
562     int32_t cmp = mTrackType - rhs.mTrackType;
563     if (cmp != 0) return cmp;
564     return mTrackChannel - rhs.mTrackChannel;
565 }
566 
operator <(const NuPlayer2::CCDecoder::CCTrack & rhs) const567 bool NuPlayer2::CCDecoder::CCTrack::operator<(const NuPlayer2::CCDecoder::CCTrack& rhs) const {
568     return compare(rhs) < 0;
569 }
570 
operator ==(const NuPlayer2::CCDecoder::CCTrack & rhs) const571 bool NuPlayer2::CCDecoder::CCTrack::operator==(const NuPlayer2::CCDecoder::CCTrack& rhs) const {
572     return compare(rhs) == 0;
573 }
574 
operator !=(const NuPlayer2::CCDecoder::CCTrack & rhs) const575 bool NuPlayer2::CCDecoder::CCTrack::operator!=(const NuPlayer2::CCDecoder::CCTrack& rhs) const {
576     return compare(rhs) != 0;
577 }
578 
579 }  // namespace android
580 
581