1/*
2 * Copyright 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayerCCDecoder"
19#include <utils/Log.h>
20#include <inttypes.h>
21
22#include "avc_utils.h"
23#include "NuPlayerCCDecoder.h"
24
25#include <media/stagefright/foundation/ABitReader.h>
26#include <media/stagefright/foundation/ABuffer.h>
27#include <media/stagefright/foundation/ADebug.h>
28#include <media/stagefright/foundation/AMessage.h>
29#include <media/stagefright/MediaDefs.h>
30
31namespace android {
32
33// In CEA-708B, the maximum bandwidth of CC is set to 9600bps.
34static const size_t kMaxBandwithSizeBytes = 9600 / 8;
35
36struct CCData {
37    CCData(uint8_t type, uint8_t data1, uint8_t data2)
38        : mType(type), mData1(data1), mData2(data2) {
39    }
40    bool getChannel(size_t *channel) const {
41        if (mData1 >= 0x10 && mData1 <= 0x1f) {
42            *channel = (mData1 >= 0x18 ? 1 : 0) + (mType ? 2 : 0);
43            return true;
44        }
45        return false;
46    }
47
48    uint8_t mType;
49    uint8_t mData1;
50    uint8_t mData2;
51};
52
53static bool isNullPad(CCData *cc) {
54    return cc->mData1 < 0x10 && cc->mData2 < 0x10;
55}
56
57static void dumpBytePair(const sp<ABuffer> &ccBuf) __attribute__ ((unused));
58static void dumpBytePair(const sp<ABuffer> &ccBuf) {
59    size_t offset = 0;
60    AString out;
61
62    while (offset < ccBuf->size()) {
63        char tmp[128];
64
65        CCData *cc = (CCData *) (ccBuf->data() + offset);
66
67        if (isNullPad(cc)) {
68            // 1 null pad or XDS metadata, ignore
69            offset += sizeof(CCData);
70            continue;
71        }
72
73        if (cc->mData1 >= 0x20 && cc->mData1 <= 0x7f) {
74            // 2 basic chars
75            sprintf(tmp, "[%d]Basic: %c %c", cc->mType, cc->mData1, cc->mData2);
76        } else if ((cc->mData1 == 0x11 || cc->mData1 == 0x19)
77                 && cc->mData2 >= 0x30 && cc->mData2 <= 0x3f) {
78            // 1 special char
79            sprintf(tmp, "[%d]Special: %02x %02x", cc->mType, cc->mData1, cc->mData2);
80        } else if ((cc->mData1 == 0x12 || cc->mData1 == 0x1A)
81                 && cc->mData2 >= 0x20 && cc->mData2 <= 0x3f){
82            // 1 Spanish/French char
83            sprintf(tmp, "[%d]Spanish: %02x %02x", cc->mType, cc->mData1, cc->mData2);
84        } else if ((cc->mData1 == 0x13 || cc->mData1 == 0x1B)
85                 && cc->mData2 >= 0x20 && cc->mData2 <= 0x3f){
86            // 1 Portuguese/German/Danish char
87            sprintf(tmp, "[%d]German: %02x %02x", cc->mType, cc->mData1, cc->mData2);
88        } else if ((cc->mData1 == 0x11 || cc->mData1 == 0x19)
89                 && cc->mData2 >= 0x20 && cc->mData2 <= 0x2f){
90            // Mid-Row Codes (Table 69)
91            sprintf(tmp, "[%d]Mid-row: %02x %02x", cc->mType, cc->mData1, cc->mData2);
92        } else if (((cc->mData1 == 0x14 || cc->mData1 == 0x1c)
93                  && cc->mData2 >= 0x20 && cc->mData2 <= 0x2f)
94                  ||
95                   ((cc->mData1 == 0x17 || cc->mData1 == 0x1f)
96                  && cc->mData2 >= 0x21 && cc->mData2 <= 0x23)){
97            // Misc Control Codes (Table 70)
98            sprintf(tmp, "[%d]Ctrl: %02x %02x", cc->mType, cc->mData1, cc->mData2);
99        } else if ((cc->mData1 & 0x70) == 0x10
100                && (cc->mData2 & 0x40) == 0x40
101                && ((cc->mData1 & 0x07) || !(cc->mData2 & 0x20)) ) {
102            // Preamble Address Codes (Table 71)
103            sprintf(tmp, "[%d]PAC: %02x %02x", cc->mType, cc->mData1, cc->mData2);
104        } else {
105            sprintf(tmp, "[%d]Invalid: %02x %02x", cc->mType, cc->mData1, cc->mData2);
106        }
107
108        if (out.size() > 0) {
109            out.append(", ");
110        }
111
112        out.append(tmp);
113
114        offset += sizeof(CCData);
115    }
116
117    ALOGI("%s", out.c_str());
118}
119
120NuPlayer::CCDecoder::CCDecoder(const sp<AMessage> &notify)
121    : mNotify(notify),
122      mSelectedTrack(-1),
123      mDTVCCPacket(new ABuffer(kMaxBandwithSizeBytes)) {
124    mDTVCCPacket->setRange(0, 0);
125
126    // In CEA-608, streams from packets which have the value 0 of cc_type contain CC1 and CC2, and
127    // streams from packets which have the value 1 of cc_type contain CC3 and CC4.
128    // The following array indicates the current transmitting channels for each value of cc_type.
129    mLine21Channels[0] = 0; // CC1
130    mLine21Channels[1] = 2; // CC3
131}
132
133size_t NuPlayer::CCDecoder::getTrackCount() const {
134    return mTracks.size();
135}
136
137sp<AMessage> NuPlayer::CCDecoder::getTrackInfo(size_t index) const {
138    if (!isTrackValid(index)) {
139        return NULL;
140    }
141
142    sp<AMessage> format = new AMessage();
143
144    CCTrack track = mTracks[index];
145
146    format->setInt32("type", MEDIA_TRACK_TYPE_SUBTITLE);
147    format->setString("language", "und");
148
149    switch (track.mTrackType) {
150        case kTrackTypeCEA608:
151            format->setString("mime", MEDIA_MIMETYPE_TEXT_CEA_608);
152            break;
153        case kTrackTypeCEA708:
154            format->setString("mime", MEDIA_MIMETYPE_TEXT_CEA_708);
155            break;
156        default:
157            ALOGE("Unknown track type: %d", track.mTrackType);
158            return NULL;
159    }
160
161    // For CEA-608 CC1, field 0 channel 0
162    bool isDefaultAuto = track.mTrackType == kTrackTypeCEA608
163            && track.mTrackChannel == 0;
164    // For CEA-708, Primary Caption Service.
165    bool isDefaultOnly = track.mTrackType == kTrackTypeCEA708
166            && track.mTrackChannel == 1;
167    format->setInt32("auto", isDefaultAuto);
168    format->setInt32("default", isDefaultAuto || isDefaultOnly);
169    format->setInt32("forced", 0);
170
171    return format;
172}
173
174status_t NuPlayer::CCDecoder::selectTrack(size_t index, bool select) {
175    if (!isTrackValid(index)) {
176        return BAD_VALUE;
177    }
178
179    if (select) {
180        if (mSelectedTrack == (ssize_t)index) {
181            ALOGE("track %zu already selected", index);
182            return BAD_VALUE;
183        }
184        ALOGV("selected track %zu", index);
185        mSelectedTrack = index;
186    } else {
187        if (mSelectedTrack != (ssize_t)index) {
188            ALOGE("track %zu is not selected", index);
189            return BAD_VALUE;
190        }
191        ALOGV("unselected track %zu", index);
192        mSelectedTrack = -1;
193    }
194
195    // Clear the previous track payloads
196    mCCMap.clear();
197
198    return OK;
199}
200
201bool NuPlayer::CCDecoder::isSelected() const {
202    return mSelectedTrack >= 0 && mSelectedTrack < (int32_t)getTrackCount();
203}
204
205bool NuPlayer::CCDecoder::isTrackValid(size_t index) const {
206    return index < getTrackCount();
207}
208
209// returns true if a new CC track is found
210bool NuPlayer::CCDecoder::extractFromSEI(const sp<ABuffer> &accessUnit) {
211    sp<ABuffer> sei;
212    if (!accessUnit->meta()->findBuffer("sei", &sei) || sei == NULL) {
213        return false;
214    }
215
216    int64_t timeUs;
217    CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs));
218
219    bool trackAdded = false;
220
221    const NALPosition *nal = (NALPosition *)sei->data();
222
223    for (size_t i = 0; i < sei->size() / sizeof(NALPosition); ++i, ++nal) {
224        trackAdded |= parseSEINalUnit(
225                timeUs, accessUnit->data() + nal->nalOffset, nal->nalSize);
226    }
227
228    return trackAdded;
229}
230
231// returns true if a new CC track is found
232bool NuPlayer::CCDecoder::parseSEINalUnit(int64_t timeUs, const uint8_t *data, size_t size) {
233    unsigned nalType = data[0] & 0x1f;
234
235    // the buffer should only have SEI in it
236    if (nalType != 6) {
237        return false;
238    }
239
240    bool trackAdded = false;
241    NALBitReader br(data + 1, size - 1);
242
243    // sei_message()
244    while (br.atLeastNumBitsLeft(16)) { // at least 16-bit for sei_message()
245        uint32_t payload_type = 0;
246        size_t payload_size = 0;
247        uint8_t last_byte;
248
249        do {
250            last_byte = br.getBits(8);
251            payload_type += last_byte;
252        } while (last_byte == 0xFF);
253
254        do {
255            last_byte = br.getBits(8);
256            payload_size += last_byte;
257        } while (last_byte == 0xFF);
258
259        // sei_payload()
260        if (payload_type == 4) {
261            bool isCC = false;
262            if (payload_size > 1 + 2 + 4 + 1) {
263                // user_data_registered_itu_t_t35()
264
265                // ATSC A/72: 6.4.2
266                uint8_t itu_t_t35_country_code = br.getBits(8);
267                uint16_t itu_t_t35_provider_code = br.getBits(16);
268                uint32_t user_identifier = br.getBits(32);
269                uint8_t user_data_type_code = br.getBits(8);
270
271                payload_size -= 1 + 2 + 4 + 1;
272
273                isCC = itu_t_t35_country_code == 0xB5
274                        && itu_t_t35_provider_code == 0x0031
275                        && user_identifier == 'GA94'
276                        && user_data_type_code == 0x3;
277            }
278
279            if (isCC && payload_size > 2) {
280                trackAdded |= parseMPEGCCData(timeUs, br.data(), br.numBitsLeft() / 8);
281            } else {
282                ALOGV("Malformed SEI payload type 4");
283            }
284        } else {
285            ALOGV("Unsupported SEI payload type %d", payload_type);
286        }
287
288        // skipping remaining bits of this payload
289        br.skipBits(payload_size * 8);
290    }
291
292    return trackAdded;
293}
294
295// returns true if a new CC track is found
296bool NuPlayer::CCDecoder::extractFromMPEGUserData(const sp<ABuffer> &accessUnit) {
297    sp<ABuffer> mpegUserData;
298    if (!accessUnit->meta()->findBuffer("mpegUserData", &mpegUserData)
299            || mpegUserData == NULL) {
300        return false;
301    }
302
303    int64_t timeUs;
304    CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs));
305
306    bool trackAdded = false;
307
308    const size_t *userData = (size_t *)mpegUserData->data();
309
310    for (size_t i = 0; i < mpegUserData->size() / sizeof(size_t); ++i) {
311        trackAdded |= parseMPEGUserDataUnit(
312                timeUs, accessUnit->data() + userData[i], accessUnit->size() - userData[i]);
313    }
314
315    return trackAdded;
316}
317
318// returns true if a new CC track is found
319bool NuPlayer::CCDecoder::parseMPEGUserDataUnit(int64_t timeUs, const uint8_t *data, size_t size) {
320    ABitReader br(data + 4, 5);
321
322    uint32_t user_identifier = br.getBits(32);
323    uint8_t user_data_type = br.getBits(8);
324
325    if (user_identifier == 'GA94' && user_data_type == 0x3) {
326        return parseMPEGCCData(timeUs, data + 9, size - 9);
327    }
328
329    return false;
330}
331
332// returns true if a new CC track is found
333bool NuPlayer::CCDecoder::parseMPEGCCData(int64_t timeUs, const uint8_t *data, size_t size) {
334    bool trackAdded = false;
335
336    // MPEG_cc_data()
337    // ATSC A/53 Part 4: 6.2.3.1
338    ABitReader br(data, size);
339
340    if (br.numBitsLeft() <= 16) {
341        return false;
342    }
343
344    br.skipBits(1);
345    bool process_cc_data_flag = br.getBits(1);
346    br.skipBits(1);
347    size_t cc_count = br.getBits(5);
348    br.skipBits(8);
349
350    if (!process_cc_data_flag || 3 * 8 * cc_count >= br.numBitsLeft()) {
351        return false;
352    }
353
354    sp<ABuffer> line21CCBuf = NULL;
355
356    for (size_t i = 0; i < cc_count; ++i) {
357        br.skipBits(5);
358        bool cc_valid = br.getBits(1);
359        uint8_t cc_type = br.getBits(2);
360
361        if (cc_valid) {
362            if (cc_type == 3) {
363                if (mDTVCCPacket->size() > 0) {
364                    trackAdded |= parseDTVCCPacket(
365                            timeUs, mDTVCCPacket->data(), mDTVCCPacket->size());
366                    mDTVCCPacket->setRange(0, 0);
367                }
368                memcpy(mDTVCCPacket->data() + mDTVCCPacket->size(), br.data(), 2);
369                mDTVCCPacket->setRange(0, mDTVCCPacket->size() + 2);
370                br.skipBits(16);
371            } else if (mDTVCCPacket->size() > 0 && cc_type == 2) {
372                memcpy(mDTVCCPacket->data() + mDTVCCPacket->size(), br.data(), 2);
373                mDTVCCPacket->setRange(0, mDTVCCPacket->size() + 2);
374                br.skipBits(16);
375            } else if (cc_type == 0 || cc_type == 1) {
376                uint8_t cc_data_1 = br.getBits(8) & 0x7f;
377                uint8_t cc_data_2 = br.getBits(8) & 0x7f;
378
379                CCData cc(cc_type, cc_data_1, cc_data_2);
380
381                if (isNullPad(&cc)) {
382                    continue;
383                }
384
385                size_t channel;
386                if (cc.getChannel(&channel)) {
387                    mLine21Channels[cc_type] = channel;
388
389                    // create a new track if it does not exist.
390                    getTrackIndex(kTrackTypeCEA608, channel, &trackAdded);
391                }
392
393                if (isSelected() && mTracks[mSelectedTrack].mTrackType == kTrackTypeCEA608
394                        && mTracks[mSelectedTrack].mTrackChannel == mLine21Channels[cc_type]) {
395                    if (line21CCBuf == NULL) {
396                        line21CCBuf = new ABuffer((cc_count - i) * sizeof(CCData));
397                        line21CCBuf->setRange(0, 0);
398                    }
399                    memcpy(line21CCBuf->data() + line21CCBuf->size(), &cc, sizeof(cc));
400                    line21CCBuf->setRange(0, line21CCBuf->size() + sizeof(CCData));
401                }
402            } else {
403                br.skipBits(16);
404            }
405        } else {
406            if ((cc_type == 3 || cc_type == 2) && mDTVCCPacket->size() > 0) {
407                trackAdded |= parseDTVCCPacket(timeUs, mDTVCCPacket->data(), mDTVCCPacket->size());
408                mDTVCCPacket->setRange(0, 0);
409            }
410            br.skipBits(16);
411        }
412    }
413
414    if (isSelected() && mTracks[mSelectedTrack].mTrackType == kTrackTypeCEA608
415            && line21CCBuf != NULL && line21CCBuf->size() > 0) {
416        mCCMap.add(timeUs, line21CCBuf);
417    }
418
419    return trackAdded;
420}
421
422// returns true if a new CC track is found
423bool NuPlayer::CCDecoder::parseDTVCCPacket(int64_t timeUs, const uint8_t *data, size_t size) {
424    // CEA-708B 5 DTVCC Packet Layer.
425    ABitReader br(data, size);
426    br.skipBits(2);
427
428    size_t packet_size = br.getBits(6);
429    if (packet_size == 0) packet_size = 64;
430    packet_size *= 2;
431
432    if (size != packet_size) {
433        return false;
434    }
435
436    bool trackAdded = false;
437
438    while (br.numBitsLeft() >= 16) {
439        // CEA-708B Figure 5 and 6.
440        uint8_t service_number = br.getBits(3);
441        size_t block_size = br.getBits(5);
442
443        if (service_number == 64) {
444            br.skipBits(2);
445            service_number = br.getBits(6);
446
447            if (service_number < 64) {
448                return trackAdded;
449            }
450        }
451
452        if (br.numBitsLeft() < block_size * 8) {
453            return trackAdded;
454        }
455
456        if (block_size > 0) {
457            size_t trackIndex = getTrackIndex(kTrackTypeCEA708, service_number, &trackAdded);
458            if (mSelectedTrack == (ssize_t)trackIndex) {
459                sp<ABuffer> ccPacket = new ABuffer(block_size);
460                memcpy(ccPacket->data(), br.data(), block_size);
461                mCCMap.add(timeUs, ccPacket);
462            }
463        }
464        br.skipBits(block_size * 8);
465    }
466
467    return trackAdded;
468}
469
470// return the track index for a given type and channel.
471// if the track does not exist, creates a new one.
472size_t NuPlayer::CCDecoder::getTrackIndex(
473        int32_t trackType, size_t channel, bool *trackAdded) {
474    CCTrack track(trackType, channel);
475    ssize_t index = mTrackIndices.indexOfKey(track);
476
477    if (index < 0) {
478        // A new track is added.
479        index = mTracks.size();
480        mTrackIndices.add(track, index);
481        mTracks.add(track);
482        *trackAdded = true;
483        return index;
484    }
485
486    return mTrackIndices.valueAt(index);
487}
488
489void NuPlayer::CCDecoder::decode(const sp<ABuffer> &accessUnit) {
490    if (extractFromMPEGUserData(accessUnit) || extractFromSEI(accessUnit)) {
491        sp<AMessage> msg = mNotify->dup();
492        msg->setInt32("what", kWhatTrackAdded);
493        msg->post();
494    }
495    // TODO: extract CC from other sources
496}
497
498void NuPlayer::CCDecoder::display(int64_t timeUs) {
499    if (!isSelected()) {
500        return;
501    }
502
503    ssize_t index = mCCMap.indexOfKey(timeUs);
504    if (index < 0) {
505        ALOGV("cc for timestamp %" PRId64 " not found", timeUs);
506        return;
507    }
508
509    sp<ABuffer> ccBuf;
510
511    if (index == 0) {
512        ccBuf = mCCMap.valueAt(index);
513    } else {
514        size_t size = 0;
515
516        for (ssize_t i = 0; i <= index; ++i) {
517            size += mCCMap.valueAt(i)->size();
518        }
519
520        ccBuf = new ABuffer(size);
521        ccBuf->setRange(0, 0);
522
523        for (ssize_t i = 0; i <= index; ++i) {
524            sp<ABuffer> buf = mCCMap.valueAt(i);
525            memcpy(ccBuf->data() + ccBuf->size(), buf->data(), buf->size());
526            ccBuf->setRange(0, ccBuf->size() + buf->size());
527        }
528    }
529
530    if (ccBuf->size() > 0) {
531#if 0
532        dumpBytePair(ccBuf);
533#endif
534
535        ccBuf->meta()->setInt32("trackIndex", mSelectedTrack);
536        ccBuf->meta()->setInt64("timeUs", timeUs);
537        ccBuf->meta()->setInt64("durationUs", 0ll);
538
539        sp<AMessage> msg = mNotify->dup();
540        msg->setInt32("what", kWhatClosedCaptionData);
541        msg->setBuffer("buffer", ccBuf);
542        msg->post();
543    }
544
545    // remove all entries before timeUs
546    mCCMap.removeItemsAt(0, index + 1);
547}
548
549void NuPlayer::CCDecoder::flush() {
550    mCCMap.clear();
551    mDTVCCPacket->setRange(0, 0);
552}
553
554int32_t NuPlayer::CCDecoder::CCTrack::compare(const NuPlayer::CCDecoder::CCTrack& rhs) const {
555    int32_t cmp = mTrackType - rhs.mTrackType;
556    if (cmp != 0) return cmp;
557    return mTrackChannel - rhs.mTrackChannel;
558}
559
560bool NuPlayer::CCDecoder::CCTrack::operator<(const NuPlayer::CCDecoder::CCTrack& rhs) const {
561    return compare(rhs) < 0;
562}
563
564bool NuPlayer::CCDecoder::CCTrack::operator==(const NuPlayer::CCDecoder::CCTrack& rhs) const {
565    return compare(rhs) == 0;
566}
567
568bool NuPlayer::CCDecoder::CCTrack::operator!=(const NuPlayer::CCDecoder::CCTrack& rhs) const {
569    return compare(rhs) != 0;
570}
571
572}  // namespace android
573
574