1 /*
2  * Copyright 2014 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "NuPlayerCCDecoder"
19 #include <utils/Log.h>
20 #include <inttypes.h>
21 
22 #include "avc_utils.h"
23 #include "NuPlayerCCDecoder.h"
24 
25 #include <media/stagefright/foundation/ABitReader.h>
26 #include <media/stagefright/foundation/ABuffer.h>
27 #include <media/stagefright/foundation/ADebug.h>
28 #include <media/stagefright/foundation/AMessage.h>
29 #include <media/stagefright/MediaDefs.h>
30 
31 namespace android {
32 
33 // In CEA-708B, the maximum bandwidth of CC is set to 9600bps.
34 static const size_t kMaxBandwithSizeBytes = 9600 / 8;
35 
36 struct CCData {
CCDataandroid::CCData37     CCData(uint8_t type, uint8_t data1, uint8_t data2)
38         : mType(type), mData1(data1), mData2(data2) {
39     }
getChannelandroid::CCData40     bool getChannel(size_t *channel) const {
41         if (mData1 >= 0x10 && mData1 <= 0x1f) {
42             *channel = (mData1 >= 0x18 ? 1 : 0) + (mType ? 2 : 0);
43             return true;
44         }
45         return false;
46     }
47 
48     uint8_t mType;
49     uint8_t mData1;
50     uint8_t mData2;
51 };
52 
isNullPad(CCData * cc)53 static bool isNullPad(CCData *cc) {
54     return cc->mData1 < 0x10 && cc->mData2 < 0x10;
55 }
56 
57 static void dumpBytePair(const sp<ABuffer> &ccBuf) __attribute__ ((unused));
dumpBytePair(const sp<ABuffer> & ccBuf)58 static void dumpBytePair(const sp<ABuffer> &ccBuf) {
59     size_t offset = 0;
60     AString out;
61 
62     while (offset < ccBuf->size()) {
63         char tmp[128];
64 
65         CCData *cc = (CCData *) (ccBuf->data() + offset);
66 
67         if (isNullPad(cc)) {
68             // 1 null pad or XDS metadata, ignore
69             offset += sizeof(CCData);
70             continue;
71         }
72 
73         if (cc->mData1 >= 0x20 && cc->mData1 <= 0x7f) {
74             // 2 basic chars
75             snprintf(tmp, sizeof(tmp), "[%d]Basic: %c %c", cc->mType, cc->mData1, cc->mData2);
76         } else if ((cc->mData1 == 0x11 || cc->mData1 == 0x19)
77                  && cc->mData2 >= 0x30 && cc->mData2 <= 0x3f) {
78             // 1 special char
79             snprintf(tmp, sizeof(tmp), "[%d]Special: %02x %02x", cc->mType, cc->mData1, cc->mData2);
80         } else if ((cc->mData1 == 0x12 || cc->mData1 == 0x1A)
81                  && cc->mData2 >= 0x20 && cc->mData2 <= 0x3f){
82             // 1 Spanish/French char
83             snprintf(tmp, sizeof(tmp), "[%d]Spanish: %02x %02x", cc->mType, cc->mData1, cc->mData2);
84         } else if ((cc->mData1 == 0x13 || cc->mData1 == 0x1B)
85                  && cc->mData2 >= 0x20 && cc->mData2 <= 0x3f){
86             // 1 Portuguese/German/Danish char
87             snprintf(tmp, sizeof(tmp), "[%d]German: %02x %02x", cc->mType, cc->mData1, cc->mData2);
88         } else if ((cc->mData1 == 0x11 || cc->mData1 == 0x19)
89                  && cc->mData2 >= 0x20 && cc->mData2 <= 0x2f){
90             // Mid-Row Codes (Table 69)
91             snprintf(tmp, sizeof(tmp), "[%d]Mid-row: %02x %02x", cc->mType, cc->mData1, cc->mData2);
92         } else if (((cc->mData1 == 0x14 || cc->mData1 == 0x1c)
93                   && cc->mData2 >= 0x20 && cc->mData2 <= 0x2f)
94                   ||
95                    ((cc->mData1 == 0x17 || cc->mData1 == 0x1f)
96                   && cc->mData2 >= 0x21 && cc->mData2 <= 0x23)){
97             // Misc Control Codes (Table 70)
98             snprintf(tmp, sizeof(tmp), "[%d]Ctrl: %02x %02x", cc->mType, cc->mData1, cc->mData2);
99         } else if ((cc->mData1 & 0x70) == 0x10
100                 && (cc->mData2 & 0x40) == 0x40
101                 && ((cc->mData1 & 0x07) || !(cc->mData2 & 0x20)) ) {
102             // Preamble Address Codes (Table 71)
103             snprintf(tmp, sizeof(tmp), "[%d]PAC: %02x %02x", cc->mType, cc->mData1, cc->mData2);
104         } else {
105             snprintf(tmp, sizeof(tmp), "[%d]Invalid: %02x %02x", cc->mType, cc->mData1, cc->mData2);
106         }
107 
108         if (out.size() > 0) {
109             out.append(", ");
110         }
111 
112         out.append(tmp);
113 
114         offset += sizeof(CCData);
115     }
116 
117     ALOGI("%s", out.c_str());
118 }
119 
CCDecoder(const sp<AMessage> & notify)120 NuPlayer::CCDecoder::CCDecoder(const sp<AMessage> &notify)
121     : mNotify(notify),
122       mSelectedTrack(-1),
123       mDTVCCPacket(new ABuffer(kMaxBandwithSizeBytes)) {
124     mDTVCCPacket->setRange(0, 0);
125 
126     // In CEA-608, streams from packets which have the value 0 of cc_type contain CC1 and CC2, and
127     // streams from packets which have the value 1 of cc_type contain CC3 and CC4.
128     // The following array indicates the current transmitting channels for each value of cc_type.
129     mLine21Channels[0] = 0; // CC1
130     mLine21Channels[1] = 2; // CC3
131 }
132 
getTrackCount() const133 size_t NuPlayer::CCDecoder::getTrackCount() const {
134     return mTracks.size();
135 }
136 
getTrackInfo(size_t index) const137 sp<AMessage> NuPlayer::CCDecoder::getTrackInfo(size_t index) const {
138     if (!isTrackValid(index)) {
139         return NULL;
140     }
141 
142     sp<AMessage> format = new AMessage();
143 
144     CCTrack track = mTracks[index];
145 
146     format->setInt32("type", MEDIA_TRACK_TYPE_SUBTITLE);
147     format->setString("language", "und");
148 
149     switch (track.mTrackType) {
150         case kTrackTypeCEA608:
151             format->setString("mime", MEDIA_MIMETYPE_TEXT_CEA_608);
152             break;
153         case kTrackTypeCEA708:
154             format->setString("mime", MEDIA_MIMETYPE_TEXT_CEA_708);
155             break;
156         default:
157             ALOGE("Unknown track type: %d", track.mTrackType);
158             return NULL;
159     }
160 
161     // For CEA-608 CC1, field 0 channel 0
162     bool isDefaultAuto = track.mTrackType == kTrackTypeCEA608
163             && track.mTrackChannel == 0;
164     // For CEA-708, Primary Caption Service.
165     bool isDefaultOnly = track.mTrackType == kTrackTypeCEA708
166             && track.mTrackChannel == 1;
167     format->setInt32("auto", isDefaultAuto);
168     format->setInt32("default", isDefaultAuto || isDefaultOnly);
169     format->setInt32("forced", 0);
170 
171     return format;
172 }
173 
selectTrack(size_t index,bool select)174 status_t NuPlayer::CCDecoder::selectTrack(size_t index, bool select) {
175     if (!isTrackValid(index)) {
176         return BAD_VALUE;
177     }
178 
179     if (select) {
180         if (mSelectedTrack == (ssize_t)index) {
181             ALOGE("track %zu already selected", index);
182             return BAD_VALUE;
183         }
184         ALOGV("selected track %zu", index);
185         mSelectedTrack = index;
186     } else {
187         if (mSelectedTrack != (ssize_t)index) {
188             ALOGE("track %zu is not selected", index);
189             return BAD_VALUE;
190         }
191         ALOGV("unselected track %zu", index);
192         mSelectedTrack = -1;
193     }
194 
195     // Clear the previous track payloads
196     mCCMap.clear();
197 
198     return OK;
199 }
200 
isSelected() const201 bool NuPlayer::CCDecoder::isSelected() const {
202     return mSelectedTrack >= 0 && mSelectedTrack < (int32_t)getTrackCount();
203 }
204 
isTrackValid(size_t index) const205 bool NuPlayer::CCDecoder::isTrackValid(size_t index) const {
206     return index < getTrackCount();
207 }
208 
209 // returns true if a new CC track is found
extractFromSEI(const sp<ABuffer> & accessUnit)210 bool NuPlayer::CCDecoder::extractFromSEI(const sp<ABuffer> &accessUnit) {
211     sp<ABuffer> sei;
212     if (!accessUnit->meta()->findBuffer("sei", &sei) || sei == NULL) {
213         return false;
214     }
215 
216     int64_t timeUs;
217     CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs));
218 
219     bool trackAdded = false;
220 
221     const NALPosition *nal = (NALPosition *)sei->data();
222 
223     for (size_t i = 0; i < sei->size() / sizeof(NALPosition); ++i, ++nal) {
224         trackAdded |= parseSEINalUnit(
225                 timeUs, accessUnit->data() + nal->nalOffset, nal->nalSize);
226     }
227 
228     return trackAdded;
229 }
230 
231 // returns true if a new CC track is found
parseSEINalUnit(int64_t timeUs,const uint8_t * data,size_t size)232 bool NuPlayer::CCDecoder::parseSEINalUnit(int64_t timeUs, const uint8_t *data, size_t size) {
233     unsigned nalType = data[0] & 0x1f;
234 
235     // the buffer should only have SEI in it
236     if (nalType != 6) {
237         return false;
238     }
239 
240     bool trackAdded = false;
241     NALBitReader br(data + 1, size - 1);
242 
243     // sei_message()
244     while (br.atLeastNumBitsLeft(16)) { // at least 16-bit for sei_message()
245         uint32_t payload_type = 0;
246         size_t payload_size = 0;
247         uint8_t last_byte;
248 
249         do {
250             last_byte = br.getBits(8);
251             payload_type += last_byte;
252         } while (last_byte == 0xFF);
253 
254         do {
255             last_byte = br.getBits(8);
256             payload_size += last_byte;
257         } while (last_byte == 0xFF);
258 
259         if (payload_size > SIZE_MAX / 8
260                 || !br.atLeastNumBitsLeft(payload_size * 8)) {
261             ALOGV("Malformed SEI payload");
262             break;
263         }
264 
265         // sei_payload()
266         if (payload_type == 4) {
267             bool isCC = false;
268             if (payload_size > 1 + 2 + 4 + 1) {
269                 // user_data_registered_itu_t_t35()
270 
271                 // ATSC A/72: 6.4.2
272                 uint8_t itu_t_t35_country_code = br.getBits(8);
273                 uint16_t itu_t_t35_provider_code = br.getBits(16);
274                 uint32_t user_identifier = br.getBits(32);
275                 uint8_t user_data_type_code = br.getBits(8);
276 
277                 payload_size -= 1 + 2 + 4 + 1;
278 
279                 isCC = itu_t_t35_country_code == 0xB5
280                         && itu_t_t35_provider_code == 0x0031
281                         && user_identifier == 'GA94'
282                         && user_data_type_code == 0x3;
283             }
284 
285             if (isCC && payload_size > 2) {
286                 trackAdded |= parseMPEGCCData(timeUs, br.data(), br.numBitsLeft() / 8);
287             } else {
288                 ALOGV("Malformed SEI payload type 4");
289             }
290         } else {
291             ALOGV("Unsupported SEI payload type %d", payload_type);
292         }
293 
294         // skipping remaining bits of this payload
295         br.skipBits(payload_size * 8);
296     }
297 
298     return trackAdded;
299 }
300 
301 // returns true if a new CC track is found
extractFromMPEGUserData(const sp<ABuffer> & accessUnit)302 bool NuPlayer::CCDecoder::extractFromMPEGUserData(const sp<ABuffer> &accessUnit) {
303     sp<ABuffer> mpegUserData;
304     if (!accessUnit->meta()->findBuffer("mpegUserData", &mpegUserData)
305             || mpegUserData == NULL) {
306         return false;
307     }
308 
309     int64_t timeUs;
310     CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs));
311 
312     bool trackAdded = false;
313 
314     const size_t *userData = (size_t *)mpegUserData->data();
315 
316     for (size_t i = 0; i < mpegUserData->size() / sizeof(size_t); ++i) {
317         trackAdded |= parseMPEGUserDataUnit(
318                 timeUs, accessUnit->data() + userData[i], accessUnit->size() - userData[i]);
319     }
320 
321     return trackAdded;
322 }
323 
324 // returns true if a new CC track is found
parseMPEGUserDataUnit(int64_t timeUs,const uint8_t * data,size_t size)325 bool NuPlayer::CCDecoder::parseMPEGUserDataUnit(int64_t timeUs, const uint8_t *data, size_t size) {
326     ABitReader br(data + 4, 5);
327 
328     uint32_t user_identifier = br.getBits(32);
329     uint8_t user_data_type = br.getBits(8);
330 
331     if (user_identifier == 'GA94' && user_data_type == 0x3) {
332         return parseMPEGCCData(timeUs, data + 9, size - 9);
333     }
334 
335     return false;
336 }
337 
338 // returns true if a new CC track is found
parseMPEGCCData(int64_t timeUs,const uint8_t * data,size_t size)339 bool NuPlayer::CCDecoder::parseMPEGCCData(int64_t timeUs, const uint8_t *data, size_t size) {
340     bool trackAdded = false;
341 
342     // MPEG_cc_data()
343     // ATSC A/53 Part 4: 6.2.3.1
344     ABitReader br(data, size);
345 
346     if (br.numBitsLeft() <= 16) {
347         return false;
348     }
349 
350     br.skipBits(1);
351     bool process_cc_data_flag = br.getBits(1);
352     br.skipBits(1);
353     size_t cc_count = br.getBits(5);
354     br.skipBits(8);
355 
356     if (!process_cc_data_flag || 3 * 8 * cc_count >= br.numBitsLeft()) {
357         return false;
358     }
359 
360     sp<ABuffer> line21CCBuf = NULL;
361 
362     for (size_t i = 0; i < cc_count; ++i) {
363         br.skipBits(5);
364         bool cc_valid = br.getBits(1);
365         uint8_t cc_type = br.getBits(2);
366 
367         if (cc_valid) {
368             if (cc_type == 3) {
369                 if (mDTVCCPacket->size() > 0) {
370                     trackAdded |= parseDTVCCPacket(
371                             timeUs, mDTVCCPacket->data(), mDTVCCPacket->size());
372                     mDTVCCPacket->setRange(0, 0);
373                 }
374                 memcpy(mDTVCCPacket->data() + mDTVCCPacket->size(), br.data(), 2);
375                 mDTVCCPacket->setRange(0, mDTVCCPacket->size() + 2);
376                 br.skipBits(16);
377             } else if (mDTVCCPacket->size() > 0 && cc_type == 2) {
378                 memcpy(mDTVCCPacket->data() + mDTVCCPacket->size(), br.data(), 2);
379                 mDTVCCPacket->setRange(0, mDTVCCPacket->size() + 2);
380                 br.skipBits(16);
381             } else if (cc_type == 0 || cc_type == 1) {
382                 uint8_t cc_data_1 = br.getBits(8) & 0x7f;
383                 uint8_t cc_data_2 = br.getBits(8) & 0x7f;
384 
385                 CCData cc(cc_type, cc_data_1, cc_data_2);
386 
387                 if (isNullPad(&cc)) {
388                     continue;
389                 }
390 
391                 size_t channel;
392                 if (cc.getChannel(&channel)) {
393                     mLine21Channels[cc_type] = channel;
394 
395                     // create a new track if it does not exist.
396                     getTrackIndex(kTrackTypeCEA608, channel, &trackAdded);
397                 }
398 
399                 if (isSelected() && mTracks[mSelectedTrack].mTrackType == kTrackTypeCEA608
400                         && mTracks[mSelectedTrack].mTrackChannel == mLine21Channels[cc_type]) {
401                     if (line21CCBuf == NULL) {
402                         line21CCBuf = new ABuffer((cc_count - i) * sizeof(CCData));
403                         line21CCBuf->setRange(0, 0);
404                     }
405                     memcpy(line21CCBuf->data() + line21CCBuf->size(), &cc, sizeof(cc));
406                     line21CCBuf->setRange(0, line21CCBuf->size() + sizeof(CCData));
407                 }
408             } else {
409                 br.skipBits(16);
410             }
411         } else {
412             if ((cc_type == 3 || cc_type == 2) && mDTVCCPacket->size() > 0) {
413                 trackAdded |= parseDTVCCPacket(timeUs, mDTVCCPacket->data(), mDTVCCPacket->size());
414                 mDTVCCPacket->setRange(0, 0);
415             }
416             br.skipBits(16);
417         }
418     }
419 
420     if (isSelected() && mTracks[mSelectedTrack].mTrackType == kTrackTypeCEA608
421             && line21CCBuf != NULL && line21CCBuf->size() > 0) {
422         mCCMap.add(timeUs, line21CCBuf);
423     }
424 
425     return trackAdded;
426 }
427 
428 // returns true if a new CC track is found
parseDTVCCPacket(int64_t timeUs,const uint8_t * data,size_t size)429 bool NuPlayer::CCDecoder::parseDTVCCPacket(int64_t timeUs, const uint8_t *data, size_t size) {
430     // CEA-708B 5 DTVCC Packet Layer.
431     ABitReader br(data, size);
432     br.skipBits(2);
433 
434     size_t packet_size = br.getBits(6);
435     if (packet_size == 0) packet_size = 64;
436     packet_size *= 2;
437 
438     if (size != packet_size) {
439         return false;
440     }
441 
442     bool trackAdded = false;
443 
444     while (br.numBitsLeft() >= 16) {
445         // CEA-708B Figure 5 and 6.
446         uint8_t service_number = br.getBits(3);
447         size_t block_size = br.getBits(5);
448 
449         if (service_number == 64) {
450             br.skipBits(2);
451             service_number = br.getBits(6);
452 
453             if (service_number < 64) {
454                 return trackAdded;
455             }
456         }
457 
458         if (br.numBitsLeft() < block_size * 8) {
459             return trackAdded;
460         }
461 
462         if (block_size > 0) {
463             size_t trackIndex = getTrackIndex(kTrackTypeCEA708, service_number, &trackAdded);
464             if (mSelectedTrack == (ssize_t)trackIndex) {
465                 sp<ABuffer> ccPacket = new ABuffer(block_size);
466                 memcpy(ccPacket->data(), br.data(), block_size);
467                 mCCMap.add(timeUs, ccPacket);
468             }
469         }
470         br.skipBits(block_size * 8);
471     }
472 
473     return trackAdded;
474 }
475 
476 // return the track index for a given type and channel.
477 // if the track does not exist, creates a new one.
getTrackIndex(int32_t trackType,size_t channel,bool * trackAdded)478 size_t NuPlayer::CCDecoder::getTrackIndex(
479         int32_t trackType, size_t channel, bool *trackAdded) {
480     CCTrack track(trackType, channel);
481     ssize_t index = mTrackIndices.indexOfKey(track);
482 
483     if (index < 0) {
484         // A new track is added.
485         index = mTracks.size();
486         mTrackIndices.add(track, index);
487         mTracks.add(track);
488         *trackAdded = true;
489         return index;
490     }
491 
492     return mTrackIndices.valueAt(index);
493 }
494 
decode(const sp<ABuffer> & accessUnit)495 void NuPlayer::CCDecoder::decode(const sp<ABuffer> &accessUnit) {
496     if (extractFromMPEGUserData(accessUnit) || extractFromSEI(accessUnit)) {
497         sp<AMessage> msg = mNotify->dup();
498         msg->setInt32("what", kWhatTrackAdded);
499         msg->post();
500     }
501     // TODO: extract CC from other sources
502 }
503 
display(int64_t timeUs)504 void NuPlayer::CCDecoder::display(int64_t timeUs) {
505     if (!isSelected()) {
506         return;
507     }
508 
509     ssize_t index = mCCMap.indexOfKey(timeUs);
510     if (index < 0) {
511         ALOGV("cc for timestamp %" PRId64 " not found", timeUs);
512         return;
513     }
514 
515     sp<ABuffer> ccBuf;
516 
517     if (index == 0) {
518         ccBuf = mCCMap.valueAt(index);
519     } else {
520         size_t size = 0;
521 
522         for (ssize_t i = 0; i <= index; ++i) {
523             size += mCCMap.valueAt(i)->size();
524         }
525 
526         ccBuf = new ABuffer(size);
527         ccBuf->setRange(0, 0);
528 
529         for (ssize_t i = 0; i <= index; ++i) {
530             sp<ABuffer> buf = mCCMap.valueAt(i);
531             memcpy(ccBuf->data() + ccBuf->size(), buf->data(), buf->size());
532             ccBuf->setRange(0, ccBuf->size() + buf->size());
533         }
534     }
535 
536     if (ccBuf->size() > 0) {
537 #if 0
538         dumpBytePair(ccBuf);
539 #endif
540 
541         ccBuf->meta()->setInt32("trackIndex", mSelectedTrack);
542         ccBuf->meta()->setInt64("timeUs", timeUs);
543         ccBuf->meta()->setInt64("durationUs", 0ll);
544 
545         sp<AMessage> msg = mNotify->dup();
546         msg->setInt32("what", kWhatClosedCaptionData);
547         msg->setBuffer("buffer", ccBuf);
548         msg->post();
549     }
550 
551     // remove all entries before timeUs
552     mCCMap.removeItemsAt(0, index + 1);
553 }
554 
flush()555 void NuPlayer::CCDecoder::flush() {
556     mCCMap.clear();
557     mDTVCCPacket->setRange(0, 0);
558 }
559 
compare(const NuPlayer::CCDecoder::CCTrack & rhs) const560 int32_t NuPlayer::CCDecoder::CCTrack::compare(const NuPlayer::CCDecoder::CCTrack& rhs) const {
561     int32_t cmp = mTrackType - rhs.mTrackType;
562     if (cmp != 0) return cmp;
563     return mTrackChannel - rhs.mTrackChannel;
564 }
565 
operator <(const NuPlayer::CCDecoder::CCTrack & rhs) const566 bool NuPlayer::CCDecoder::CCTrack::operator<(const NuPlayer::CCDecoder::CCTrack& rhs) const {
567     return compare(rhs) < 0;
568 }
569 
operator ==(const NuPlayer::CCDecoder::CCTrack & rhs) const570 bool NuPlayer::CCDecoder::CCTrack::operator==(const NuPlayer::CCDecoder::CCTrack& rhs) const {
571     return compare(rhs) == 0;
572 }
573 
operator !=(const NuPlayer::CCDecoder::CCTrack & rhs) const574 bool NuPlayer::CCDecoder::CCTrack::operator!=(const NuPlayer::CCDecoder::CCTrack& rhs) const {
575     return compare(rhs) != 0;
576 }
577 
578 }  // namespace android
579 
580