1 /*
2  * Copyright 2014 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "NuPlayerCCDecoder"
19 #include <utils/Log.h>
20 #include <inttypes.h>
21 
22 #include "NuPlayerCCDecoder.h"
23 
24 #include <media/stagefright/foundation/ABitReader.h>
25 #include <media/stagefright/foundation/ABuffer.h>
26 #include <media/stagefright/foundation/ADebug.h>
27 #include <media/stagefright/foundation/AMessage.h>
28 #include <media/stagefright/foundation/avc_utils.h>
29 #include <media/stagefright/MediaDefs.h>
30 
31 namespace android {
32 
33 // In CEA-708B, the maximum bandwidth of CC is set to 9600bps.
34 static const size_t kMaxBandwithSizeBytes = 9600 / 8;
35 
36 struct CCData {
CCDataandroid::CCData37     CCData(uint8_t type, uint8_t data1, uint8_t data2)
38         : mType(type), mData1(data1), mData2(data2) {
39     }
getChannelandroid::CCData40     bool getChannel(size_t *channel) const {
41         if (mData1 >= 0x10 && mData1 <= 0x1f) {
42             *channel = (mData1 >= 0x18 ? 1 : 0) + (mType ? 2 : 0);
43             return true;
44         }
45         return false;
46     }
47 
48     uint8_t mType;
49     uint8_t mData1;
50     uint8_t mData2;
51 };
52 
isNullPad(CCData * cc)53 static bool isNullPad(CCData *cc) {
54     return cc->mData1 < 0x10 && cc->mData2 < 0x10;
55 }
56 
57 static void dumpBytePair(const sp<ABuffer> &ccBuf) __attribute__ ((unused));
dumpBytePair(const sp<ABuffer> & ccBuf)58 static void dumpBytePair(const sp<ABuffer> &ccBuf) {
59     size_t offset = 0;
60     AString out;
61 
62     while (offset < ccBuf->size()) {
63         char tmp[128];
64 
65         CCData *cc = (CCData *) (ccBuf->data() + offset);
66 
67         if (isNullPad(cc)) {
68             // 1 null pad or XDS metadata, ignore
69             offset += sizeof(CCData);
70             continue;
71         }
72 
73         if (cc->mData1 >= 0x20 && cc->mData1 <= 0x7f) {
74             // 2 basic chars
75             snprintf(tmp, sizeof(tmp), "[%d]Basic: %c %c", cc->mType, cc->mData1, cc->mData2);
76         } else if ((cc->mData1 == 0x11 || cc->mData1 == 0x19)
77                  && cc->mData2 >= 0x30 && cc->mData2 <= 0x3f) {
78             // 1 special char
79             snprintf(tmp, sizeof(tmp), "[%d]Special: %02x %02x", cc->mType, cc->mData1, cc->mData2);
80         } else if ((cc->mData1 == 0x12 || cc->mData1 == 0x1A)
81                  && cc->mData2 >= 0x20 && cc->mData2 <= 0x3f){
82             // 1 Spanish/French char
83             snprintf(tmp, sizeof(tmp), "[%d]Spanish: %02x %02x", cc->mType, cc->mData1, cc->mData2);
84         } else if ((cc->mData1 == 0x13 || cc->mData1 == 0x1B)
85                  && cc->mData2 >= 0x20 && cc->mData2 <= 0x3f){
86             // 1 Portuguese/German/Danish char
87             snprintf(tmp, sizeof(tmp), "[%d]German: %02x %02x", cc->mType, cc->mData1, cc->mData2);
88         } else if ((cc->mData1 == 0x11 || cc->mData1 == 0x19)
89                  && cc->mData2 >= 0x20 && cc->mData2 <= 0x2f){
90             // Mid-Row Codes (Table 69)
91             snprintf(tmp, sizeof(tmp), "[%d]Mid-row: %02x %02x", cc->mType, cc->mData1, cc->mData2);
92         } else if (((cc->mData1 == 0x14 || cc->mData1 == 0x1c)
93                   && cc->mData2 >= 0x20 && cc->mData2 <= 0x2f)
94                   ||
95                    ((cc->mData1 == 0x17 || cc->mData1 == 0x1f)
96                   && cc->mData2 >= 0x21 && cc->mData2 <= 0x23)){
97             // Misc Control Codes (Table 70)
98             snprintf(tmp, sizeof(tmp), "[%d]Ctrl: %02x %02x", cc->mType, cc->mData1, cc->mData2);
99         } else if ((cc->mData1 & 0x70) == 0x10
100                 && (cc->mData2 & 0x40) == 0x40
101                 && ((cc->mData1 & 0x07) || !(cc->mData2 & 0x20)) ) {
102             // Preamble Address Codes (Table 71)
103             snprintf(tmp, sizeof(tmp), "[%d]PAC: %02x %02x", cc->mType, cc->mData1, cc->mData2);
104         } else {
105             snprintf(tmp, sizeof(tmp), "[%d]Invalid: %02x %02x", cc->mType, cc->mData1, cc->mData2);
106         }
107 
108         if (out.size() > 0) {
109             out.append(", ");
110         }
111 
112         out.append(tmp);
113 
114         offset += sizeof(CCData);
115     }
116 
117     ALOGI("%s", out.c_str());
118 }
119 
CCDecoder(const sp<AMessage> & notify)120 NuPlayer::CCDecoder::CCDecoder(const sp<AMessage> &notify)
121     : mNotify(notify),
122       mSelectedTrack(-1),
123       mDTVCCPacket(new ABuffer(kMaxBandwithSizeBytes)) {
124     mDTVCCPacket->setRange(0, 0);
125 
126     // In CEA-608, streams from packets which have the value 0 of cc_type contain CC1 and CC2, and
127     // streams from packets which have the value 1 of cc_type contain CC3 and CC4.
128     // The following array indicates the current transmitting channels for each value of cc_type.
129     mLine21Channels[0] = 0; // CC1
130     mLine21Channels[1] = 2; // CC3
131 }
132 
getTrackCount() const133 size_t NuPlayer::CCDecoder::getTrackCount() const {
134     return mTracks.size();
135 }
136 
getTrackInfo(size_t index) const137 sp<AMessage> NuPlayer::CCDecoder::getTrackInfo(size_t index) const {
138     if (!isTrackValid(index)) {
139         return NULL;
140     }
141 
142     sp<AMessage> format = new AMessage();
143 
144     CCTrack track = mTracks[index];
145 
146     format->setInt32("type", MEDIA_TRACK_TYPE_SUBTITLE);
147     format->setString("language", "und");
148 
149     switch (track.mTrackType) {
150         case kTrackTypeCEA608:
151             format->setString("mime", MEDIA_MIMETYPE_TEXT_CEA_608);
152             break;
153         case kTrackTypeCEA708:
154             format->setString("mime", MEDIA_MIMETYPE_TEXT_CEA_708);
155             break;
156         default:
157             ALOGE("Unknown track type: %d", track.mTrackType);
158             format->setInt32("type", MEDIA_TRACK_TYPE_UNKNOWN);
159             format->setString("mime", "application/octet-stream");
160             return format;
161     }
162 
163     // For CEA-608 CC1, field 0 channel 0
164     bool isDefaultAuto = track.mTrackType == kTrackTypeCEA608
165             && track.mTrackChannel == 0;
166     // For CEA-708, Primary Caption Service.
167     bool isDefaultOnly = track.mTrackType == kTrackTypeCEA708
168             && track.mTrackChannel == 1;
169     format->setInt32("auto", isDefaultAuto);
170     format->setInt32("default", isDefaultAuto || isDefaultOnly);
171     format->setInt32("forced", 0);
172 
173     return format;
174 }
175 
selectTrack(size_t index,bool select)176 status_t NuPlayer::CCDecoder::selectTrack(size_t index, bool select) {
177     if (!isTrackValid(index)) {
178         return BAD_VALUE;
179     }
180 
181     if (select) {
182         if (mSelectedTrack == (ssize_t)index) {
183             ALOGE("track %zu already selected", index);
184             return BAD_VALUE;
185         }
186         ALOGV("selected track %zu", index);
187         mSelectedTrack = index;
188     } else {
189         if (mSelectedTrack != (ssize_t)index) {
190             ALOGE("track %zu is not selected", index);
191             return BAD_VALUE;
192         }
193         ALOGV("unselected track %zu", index);
194         mSelectedTrack = -1;
195     }
196 
197     // Clear the previous track payloads
198     mCCMap.clear();
199 
200     return OK;
201 }
202 
isSelected() const203 bool NuPlayer::CCDecoder::isSelected() const {
204     return mSelectedTrack >= 0 && mSelectedTrack < (int32_t)getTrackCount();
205 }
206 
isTrackValid(size_t index) const207 bool NuPlayer::CCDecoder::isTrackValid(size_t index) const {
208     return index < getTrackCount();
209 }
210 
211 // returns true if a new CC track is found
extractFromSEI(const sp<ABuffer> & accessUnit)212 bool NuPlayer::CCDecoder::extractFromSEI(const sp<ABuffer> &accessUnit) {
213     sp<ABuffer> sei;
214     if (!accessUnit->meta()->findBuffer("sei", &sei) || sei == NULL) {
215         return false;
216     }
217 
218     int64_t timeUs;
219     CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs));
220 
221     bool trackAdded = false;
222 
223     const NALPosition *nal = (NALPosition *)sei->data();
224 
225     for (size_t i = 0; i < sei->size() / sizeof(NALPosition); ++i, ++nal) {
226         trackAdded |= parseSEINalUnit(
227                 timeUs, accessUnit->data() + nal->nalOffset, nal->nalSize);
228     }
229 
230     return trackAdded;
231 }
232 
233 // returns true if a new CC track is found
parseSEINalUnit(int64_t timeUs,const uint8_t * data,size_t size)234 bool NuPlayer::CCDecoder::parseSEINalUnit(int64_t timeUs, const uint8_t *data, size_t size) {
235     unsigned nalType = data[0] & 0x1f;
236 
237     // the buffer should only have SEI in it
238     if (nalType != 6) {
239         return false;
240     }
241 
242     bool trackAdded = false;
243     NALBitReader br(data + 1, size - 1);
244 
245     // sei_message()
246     while (br.atLeastNumBitsLeft(16)) { // at least 16-bit for sei_message()
247         uint32_t payload_type = 0;
248         size_t payload_size = 0;
249         uint8_t last_byte;
250 
251         do {
252             last_byte = br.getBits(8);
253             payload_type += last_byte;
254         } while (last_byte == 0xFF);
255 
256         do {
257             last_byte = br.getBits(8);
258             payload_size += last_byte;
259         } while (last_byte == 0xFF);
260 
261         if (payload_size > SIZE_MAX / 8
262                 || !br.atLeastNumBitsLeft(payload_size * 8)) {
263             ALOGV("Malformed SEI payload");
264             break;
265         }
266 
267         // sei_payload()
268         if (payload_type == 4) {
269             bool isCC = false;
270             if (payload_size > 1 + 2 + 4 + 1) {
271                 // user_data_registered_itu_t_t35()
272 
273                 // ATSC A/72: 6.4.2
274                 uint8_t itu_t_t35_country_code = br.getBits(8);
275                 uint16_t itu_t_t35_provider_code = br.getBits(16);
276                 uint32_t user_identifier = br.getBits(32);
277                 uint8_t user_data_type_code = br.getBits(8);
278 
279                 payload_size -= 1 + 2 + 4 + 1;
280 
281                 isCC = itu_t_t35_country_code == 0xB5
282                         && itu_t_t35_provider_code == 0x0031
283                         && user_identifier == 'GA94'
284                         && user_data_type_code == 0x3;
285             }
286 
287             if (isCC && payload_size > 2) {
288                 trackAdded |= parseMPEGCCData(timeUs, br.data(), br.numBitsLeft() / 8);
289             } else {
290                 ALOGV("Malformed SEI payload type 4");
291             }
292         } else {
293             ALOGV("Unsupported SEI payload type %d", payload_type);
294         }
295 
296         // skipping remaining bits of this payload
297         br.skipBits(payload_size * 8);
298     }
299 
300     return trackAdded;
301 }
302 
303 // returns true if a new CC track is found
extractFromMPEGUserData(const sp<ABuffer> & accessUnit)304 bool NuPlayer::CCDecoder::extractFromMPEGUserData(const sp<ABuffer> &accessUnit) {
305     sp<ABuffer> mpegUserData;
306     if (!accessUnit->meta()->findBuffer("mpeg-user-data", &mpegUserData)
307             || mpegUserData == NULL) {
308         return false;
309     }
310 
311     int64_t timeUs;
312     CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs));
313 
314     bool trackAdded = false;
315 
316     const size_t *userData = (size_t *)mpegUserData->data();
317 
318     for (size_t i = 0; i < mpegUserData->size() / sizeof(size_t); ++i) {
319         trackAdded |= parseMPEGUserDataUnit(
320                 timeUs, accessUnit->data() + userData[i], accessUnit->size() - userData[i]);
321     }
322 
323     return trackAdded;
324 }
325 
326 // returns true if a new CC track is found
parseMPEGUserDataUnit(int64_t timeUs,const uint8_t * data,size_t size)327 bool NuPlayer::CCDecoder::parseMPEGUserDataUnit(int64_t timeUs, const uint8_t *data, size_t size) {
328     ABitReader br(data + 4, 5);
329 
330     uint32_t user_identifier = br.getBits(32);
331     uint8_t user_data_type = br.getBits(8);
332 
333     if (user_identifier == 'GA94' && user_data_type == 0x3) {
334         return parseMPEGCCData(timeUs, data + 9, size - 9);
335     }
336 
337     return false;
338 }
339 
340 // returns true if a new CC track is found
parseMPEGCCData(int64_t timeUs,const uint8_t * data,size_t size)341 bool NuPlayer::CCDecoder::parseMPEGCCData(int64_t timeUs, const uint8_t *data, size_t size) {
342     bool trackAdded = false;
343 
344     // MPEG_cc_data()
345     // ATSC A/53 Part 4: 6.2.3.1
346     ABitReader br(data, size);
347 
348     if (br.numBitsLeft() <= 16) {
349         return false;
350     }
351 
352     br.skipBits(1);
353     bool process_cc_data_flag = br.getBits(1);
354     br.skipBits(1);
355     size_t cc_count = br.getBits(5);
356     br.skipBits(8);
357 
358     if (!process_cc_data_flag || 3 * 8 * cc_count >= br.numBitsLeft()) {
359         return false;
360     }
361 
362     sp<ABuffer> line21CCBuf = NULL;
363 
364     for (size_t i = 0; i < cc_count; ++i) {
365         br.skipBits(5);
366         bool cc_valid = br.getBits(1);
367         uint8_t cc_type = br.getBits(2);
368 
369         if (cc_valid) {
370             if (cc_type == 3) {
371                 if (mDTVCCPacket->size() > 0) {
372                     trackAdded |= parseDTVCCPacket(
373                             timeUs, mDTVCCPacket->data(), mDTVCCPacket->size());
374                     mDTVCCPacket->setRange(0, 0);
375                 }
376                 memcpy(mDTVCCPacket->data() + mDTVCCPacket->size(), br.data(), 2);
377                 mDTVCCPacket->setRange(0, mDTVCCPacket->size() + 2);
378                 br.skipBits(16);
379             } else if (mDTVCCPacket->size() > 0 && cc_type == 2) {
380                 memcpy(mDTVCCPacket->data() + mDTVCCPacket->size(), br.data(), 2);
381                 mDTVCCPacket->setRange(0, mDTVCCPacket->size() + 2);
382                 br.skipBits(16);
383             } else if (cc_type == 0 || cc_type == 1) {
384                 uint8_t cc_data_1 = br.getBits(8) & 0x7f;
385                 uint8_t cc_data_2 = br.getBits(8) & 0x7f;
386 
387                 CCData cc(cc_type, cc_data_1, cc_data_2);
388 
389                 if (isNullPad(&cc)) {
390                     continue;
391                 }
392 
393                 size_t channel;
394                 if (cc.getChannel(&channel)) {
395                     mLine21Channels[cc_type] = channel;
396 
397                     // create a new track if it does not exist.
398                     getTrackIndex(kTrackTypeCEA608, channel, &trackAdded);
399                 }
400 
401                 if (isSelected() && mTracks[mSelectedTrack].mTrackType == kTrackTypeCEA608
402                         && mTracks[mSelectedTrack].mTrackChannel == mLine21Channels[cc_type]) {
403                     if (line21CCBuf == NULL) {
404                         line21CCBuf = new ABuffer((cc_count - i) * sizeof(CCData));
405                         line21CCBuf->setRange(0, 0);
406                     }
407                     memcpy(line21CCBuf->data() + line21CCBuf->size(), &cc, sizeof(cc));
408                     line21CCBuf->setRange(0, line21CCBuf->size() + sizeof(CCData));
409                 }
410             } else {
411                 br.skipBits(16);
412             }
413         } else {
414             if ((cc_type == 3 || cc_type == 2) && mDTVCCPacket->size() > 0) {
415                 trackAdded |= parseDTVCCPacket(timeUs, mDTVCCPacket->data(), mDTVCCPacket->size());
416                 mDTVCCPacket->setRange(0, 0);
417             }
418             br.skipBits(16);
419         }
420     }
421 
422     if (isSelected() && mTracks[mSelectedTrack].mTrackType == kTrackTypeCEA608
423             && line21CCBuf != NULL && line21CCBuf->size() > 0) {
424         mCCMap.add(timeUs, line21CCBuf);
425     }
426 
427     return trackAdded;
428 }
429 
430 // returns true if a new CC track is found
parseDTVCCPacket(int64_t timeUs,const uint8_t * data,size_t size)431 bool NuPlayer::CCDecoder::parseDTVCCPacket(int64_t timeUs, const uint8_t *data, size_t size) {
432     // CEA-708B 5 DTVCC Packet Layer.
433     ABitReader br(data, size);
434     br.skipBits(2);
435 
436     size_t packet_size = br.getBits(6);
437     if (packet_size == 0) packet_size = 64;
438     packet_size *= 2;
439 
440     if (size != packet_size) {
441         return false;
442     }
443 
444     bool trackAdded = false;
445 
446     while (br.numBitsLeft() >= 16) {
447         // CEA-708B Figure 5 and 6.
448         uint8_t service_number = br.getBits(3);
449         size_t block_size = br.getBits(5);
450 
451         if (service_number == 64) {
452             br.skipBits(2);
453             service_number = br.getBits(6);
454 
455             if (service_number < 64) {
456                 return trackAdded;
457             }
458         }
459 
460         if (br.numBitsLeft() < block_size * 8) {
461             return trackAdded;
462         }
463 
464         if (block_size > 0) {
465             size_t trackIndex = getTrackIndex(kTrackTypeCEA708, service_number, &trackAdded);
466             if (mSelectedTrack == (ssize_t)trackIndex) {
467                 sp<ABuffer> ccPacket = new ABuffer(block_size);
468                 memcpy(ccPacket->data(), br.data(), block_size);
469                 mCCMap.add(timeUs, ccPacket);
470             }
471         }
472         br.skipBits(block_size * 8);
473     }
474 
475     return trackAdded;
476 }
477 
478 // return the track index for a given type and channel.
479 // if the track does not exist, creates a new one.
getTrackIndex(int32_t trackType,size_t channel,bool * trackAdded)480 size_t NuPlayer::CCDecoder::getTrackIndex(
481         int32_t trackType, size_t channel, bool *trackAdded) {
482     CCTrack track(trackType, channel);
483     ssize_t index = mTrackIndices.indexOfKey(track);
484 
485     if (index < 0) {
486         // A new track is added.
487         index = mTracks.size();
488         mTrackIndices.add(track, index);
489         mTracks.add(track);
490         *trackAdded = true;
491         return index;
492     }
493 
494     return mTrackIndices.valueAt(index);
495 }
496 
decode(const sp<ABuffer> & accessUnit)497 void NuPlayer::CCDecoder::decode(const sp<ABuffer> &accessUnit) {
498     if (extractFromMPEGUserData(accessUnit) || extractFromSEI(accessUnit)) {
499         sp<AMessage> msg = mNotify->dup();
500         msg->setInt32("what", kWhatTrackAdded);
501         msg->post();
502     }
503     // TODO: extract CC from other sources
504 }
505 
display(int64_t timeUs)506 void NuPlayer::CCDecoder::display(int64_t timeUs) {
507     if (!isSelected()) {
508         return;
509     }
510 
511     ssize_t index = mCCMap.indexOfKey(timeUs);
512     if (index < 0) {
513         ALOGV("cc for timestamp %" PRId64 " not found", timeUs);
514         return;
515     }
516 
517     sp<ABuffer> ccBuf;
518 
519     if (index == 0) {
520         ccBuf = mCCMap.valueAt(index);
521     } else {
522         size_t size = 0;
523 
524         for (ssize_t i = 0; i <= index; ++i) {
525             size += mCCMap.valueAt(i)->size();
526         }
527 
528         ccBuf = new ABuffer(size);
529         ccBuf->setRange(0, 0);
530 
531         for (ssize_t i = 0; i <= index; ++i) {
532             sp<ABuffer> buf = mCCMap.valueAt(i);
533             memcpy(ccBuf->data() + ccBuf->size(), buf->data(), buf->size());
534             ccBuf->setRange(0, ccBuf->size() + buf->size());
535         }
536     }
537 
538     if (ccBuf->size() > 0) {
539 #if 0
540         dumpBytePair(ccBuf);
541 #endif
542 
543         ccBuf->meta()->setInt32("track-index", mSelectedTrack);
544         ccBuf->meta()->setInt64("timeUs", timeUs);
545         ccBuf->meta()->setInt64("durationUs", 0ll);
546 
547         sp<AMessage> msg = mNotify->dup();
548         msg->setInt32("what", kWhatClosedCaptionData);
549         msg->setBuffer("buffer", ccBuf);
550         msg->post();
551     }
552 
553     // remove all entries before timeUs
554     mCCMap.removeItemsAt(0, index + 1);
555 }
556 
flush()557 void NuPlayer::CCDecoder::flush() {
558     mCCMap.clear();
559     mDTVCCPacket->setRange(0, 0);
560 }
561 
compare(const NuPlayer::CCDecoder::CCTrack & rhs) const562 int32_t NuPlayer::CCDecoder::CCTrack::compare(const NuPlayer::CCDecoder::CCTrack& rhs) const {
563     int32_t cmp = mTrackType - rhs.mTrackType;
564     if (cmp != 0) return cmp;
565     return mTrackChannel - rhs.mTrackChannel;
566 }
567 
operator <(const NuPlayer::CCDecoder::CCTrack & rhs) const568 bool NuPlayer::CCDecoder::CCTrack::operator<(const NuPlayer::CCDecoder::CCTrack& rhs) const {
569     return compare(rhs) < 0;
570 }
571 
operator ==(const NuPlayer::CCDecoder::CCTrack & rhs) const572 bool NuPlayer::CCDecoder::CCTrack::operator==(const NuPlayer::CCDecoder::CCTrack& rhs) const {
573     return compare(rhs) == 0;
574 }
575 
operator !=(const NuPlayer::CCDecoder::CCTrack & rhs) const576 bool NuPlayer::CCDecoder::CCTrack::operator!=(const NuPlayer::CCDecoder::CCTrack& rhs) const {
577     return compare(rhs) != 0;
578 }
579 
580 }  // namespace android
581 
582