1/*
2 * Copyright (C) 2009 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17/*------------------------------------------------------------------------------
18
19    Table of contents
20
21     1. Include headers
22     2. External compiler flags
23     3. Module defines
24     4. Local function prototypes
25     5. Functions
26          h264bsdDecodeMacroblockLayer
27          h264bsdMbPartPredMode
28          h264bsdNumMbPart
29          h264bsdNumSubMbPart
30          DecodeMbPred
31          DecodeSubMbPred
32          DecodeResidual
33          DetermineNc
34          CbpIntra16x16
35          h264bsdPredModeIntra16x16
36          h264bsdDecodeMacroblock
37          ProcessResidual
38          h264bsdSubMbPartMode
39
40------------------------------------------------------------------------------*/
41
42/*------------------------------------------------------------------------------
43    1. Include headers
44------------------------------------------------------------------------------*/
45
46#include "h264bsd_macroblock_layer.h"
47#include "h264bsd_slice_header.h"
48#include "h264bsd_util.h"
49#include "h264bsd_vlc.h"
50#include "h264bsd_cavlc.h"
51#include "h264bsd_nal_unit.h"
52#include "h264bsd_neighbour.h"
53#include "h264bsd_transform.h"
54#include "h264bsd_intra_prediction.h"
55#include "h264bsd_inter_prediction.h"
56
57#ifdef H264DEC_OMXDL
58#include "omxtypes.h"
59#include "omxVC.h"
60#include "armVC.h"
61#endif /* H264DEC_OMXDL */
62
63/*------------------------------------------------------------------------------
64    2. External compiler flags
65--------------------------------------------------------------------------------
66
67--------------------------------------------------------------------------------
68    3. Module defines
69------------------------------------------------------------------------------*/
70#ifdef H264DEC_OMXDL
71static const u32 chromaIndex[8] = { 256, 260, 288, 292, 320, 324, 352, 356 };
72static const u32 lumaIndex[16] = {   0,   4,  64,  68,
73                                     8,  12,  72,  76,
74                                   128, 132, 192, 196,
75                                   136, 140, 200, 204 };
76#endif
77/* mapping of dc coefficients array to luma blocks */
78static const u32 dcCoeffIndex[16] =
79    {0, 1, 4, 5, 2, 3, 6, 7, 8, 9, 12, 13, 10, 11, 14, 15};
80
81/*------------------------------------------------------------------------------
82    4. Local function prototypes
83------------------------------------------------------------------------------*/
84
85static u32 DecodeMbPred(strmData_t *pStrmData, mbPred_t *pMbPred,
86    mbType_e mbType, u32 numRefIdxActive);
87static u32 DecodeSubMbPred(strmData_t *pStrmData, subMbPred_t *pSubMbPred,
88    mbType_e mbType, u32 numRefIdxActive);
89static u32 DecodeResidual(strmData_t *pStrmData, residual_t *pResidual,
90    mbStorage_t *pMb, mbType_e mbType, u32 codedBlockPattern);
91
92#ifdef H264DEC_OMXDL
93static u32 DetermineNc(mbStorage_t *pMb, u32 blockIndex, u8 *pTotalCoeff);
94#else
95static u32 DetermineNc(mbStorage_t *pMb, u32 blockIndex, i16 *pTotalCoeff);
96#endif
97
98static u32 CbpIntra16x16(mbType_e mbType);
99#ifdef H264DEC_OMXDL
100static u32 ProcessIntra4x4Residual(mbStorage_t *pMb, u8 *data, u32 constrainedIntraPred,
101                    macroblockLayer_t *mbLayer, const u8 **pSrc, image_t *image);
102static u32 ProcessChromaResidual(mbStorage_t *pMb, u8 *data, const u8 **pSrc );
103static u32 ProcessIntra16x16Residual(mbStorage_t *pMb, u8 *data, u32 constrainedIntraPred,
104                    u32 intraChromaPredMode, const u8 **pSrc, image_t *image);
105
106
107#else
108static u32 ProcessResidual(mbStorage_t *pMb, i32 residualLevel[][16], u32 *);
109#endif
110
111/*------------------------------------------------------------------------------
112
113    Function name: h264bsdDecodeMacroblockLayer
114
115        Functional description:
116          Parse macroblock specific information from bit stream.
117
118        Inputs:
119          pStrmData         pointer to stream data structure
120          pMb               pointer to macroblock storage structure
121          sliceType         type of the current slice
122          numRefIdxActive   maximum reference index
123
124        Outputs:
125          pMbLayer          stores the macroblock data parsed from stream
126
127        Returns:
128          HANTRO_OK         success
129          HANTRO_NOK        end of stream or error in stream
130
131------------------------------------------------------------------------------*/
132
133u32 h264bsdDecodeMacroblockLayer(strmData_t *pStrmData,
134    macroblockLayer_t *pMbLayer, mbStorage_t *pMb, u32 sliceType,
135    u32 numRefIdxActive)
136{
137
138/* Variables */
139
140    u32 tmp, i, value;
141    i32 itmp;
142    mbPartPredMode_e partMode;
143
144/* Code */
145
146    ASSERT(pStrmData);
147    ASSERT(pMbLayer);
148
149#ifdef H264DEC_NEON
150    h264bsdClearMbLayer(pMbLayer, ((sizeof(macroblockLayer_t) + 63) & ~0x3F));
151#else
152    H264SwDecMemset(pMbLayer, 0, sizeof(macroblockLayer_t));
153#endif
154
155    tmp = h264bsdDecodeExpGolombUnsigned(pStrmData, &value);
156
157    if (IS_I_SLICE(sliceType))
158    {
159        if ((value + 6) > 31 || tmp != HANTRO_OK)
160            return(HANTRO_NOK);
161        pMbLayer->mbType = (mbType_e)(value + 6);
162    }
163    else
164    {
165        if ((value + 1) > 31 || tmp != HANTRO_OK)
166            return(HANTRO_NOK);
167        pMbLayer->mbType = (mbType_e)(value + 1);
168    }
169
170    if (pMbLayer->mbType == I_PCM)
171    {
172        i32 *level;
173        while( !h264bsdIsByteAligned(pStrmData) )
174        {
175            /* pcm_alignment_zero_bit */
176            tmp = h264bsdGetBits(pStrmData, 1);
177            if (tmp)
178                return(HANTRO_NOK);
179        }
180
181        level = pMbLayer->residual.level[0];
182        for (i = 0; i < 384; i++)
183        {
184            value = h264bsdGetBits(pStrmData, 8);
185            if (value == END_OF_STREAM)
186                return(HANTRO_NOK);
187            *level++ = (i32)value;
188        }
189    }
190    else
191    {
192        partMode = h264bsdMbPartPredMode(pMbLayer->mbType);
193        if ( (partMode == PRED_MODE_INTER) &&
194             (h264bsdNumMbPart(pMbLayer->mbType) == 4) )
195        {
196            tmp = DecodeSubMbPred(pStrmData, &pMbLayer->subMbPred,
197                pMbLayer->mbType, numRefIdxActive);
198        }
199        else
200        {
201            tmp = DecodeMbPred(pStrmData, &pMbLayer->mbPred,
202                pMbLayer->mbType, numRefIdxActive);
203        }
204        if (tmp != HANTRO_OK)
205            return(tmp);
206
207        if (partMode != PRED_MODE_INTRA16x16)
208        {
209            tmp = h264bsdDecodeExpGolombMapped(pStrmData, &value,
210                (u32)(partMode == PRED_MODE_INTRA4x4));
211            if (tmp != HANTRO_OK)
212                return(tmp);
213            pMbLayer->codedBlockPattern = value;
214        }
215        else
216        {
217            pMbLayer->codedBlockPattern = CbpIntra16x16(pMbLayer->mbType);
218        }
219
220        if ( pMbLayer->codedBlockPattern ||
221             (partMode == PRED_MODE_INTRA16x16) )
222        {
223            tmp = h264bsdDecodeExpGolombSigned(pStrmData, &itmp);
224            if (tmp != HANTRO_OK || (itmp < -26) || (itmp > 25) )
225                return(HANTRO_NOK);
226            pMbLayer->mbQpDelta = itmp;
227
228            tmp = DecodeResidual(pStrmData, &pMbLayer->residual, pMb,
229                pMbLayer->mbType, pMbLayer->codedBlockPattern);
230
231            pStrmData->strmBuffReadBits =
232                (u32)(pStrmData->pStrmCurrPos - pStrmData->pStrmBuffStart) * 8 +
233                pStrmData->bitPosInWord;
234
235            if (tmp != HANTRO_OK)
236                return(tmp);
237        }
238    }
239
240    return(HANTRO_OK);
241
242}
243
244/*------------------------------------------------------------------------------
245
246    Function: h264bsdMbPartPredMode
247
248        Functional description:
249          Returns the prediction mode of a macroblock type
250
251------------------------------------------------------------------------------*/
252
253mbPartPredMode_e h264bsdMbPartPredMode(mbType_e mbType)
254{
255
256/* Variables */
257
258
259/* Code */
260
261    ASSERT(mbType <= 31);
262
263    if ((mbType <= P_8x8ref0))
264        return(PRED_MODE_INTER);
265    else if (mbType == I_4x4)
266        return(PRED_MODE_INTRA4x4);
267    else
268        return(PRED_MODE_INTRA16x16);
269
270}
271
272/*------------------------------------------------------------------------------
273
274    Function: h264bsdNumMbPart
275
276        Functional description:
277          Returns the amount of macroblock partitions in a macroblock type
278
279------------------------------------------------------------------------------*/
280
281u32 h264bsdNumMbPart(mbType_e mbType)
282{
283
284/* Variables */
285
286
287/* Code */
288
289    ASSERT(h264bsdMbPartPredMode(mbType) == PRED_MODE_INTER);
290
291    switch (mbType)
292    {
293        case P_L0_16x16:
294        case P_Skip:
295            return(1);
296
297        case P_L0_L0_16x8:
298        case P_L0_L0_8x16:
299            return(2);
300
301        /* P_8x8 or P_8x8ref0 */
302        default:
303            return(4);
304    }
305
306}
307
308/*------------------------------------------------------------------------------
309
310    Function: h264bsdNumSubMbPart
311
312        Functional description:
313          Returns the amount of sub-partitions in a sub-macroblock type
314
315------------------------------------------------------------------------------*/
316
317u32 h264bsdNumSubMbPart(subMbType_e subMbType)
318{
319
320/* Variables */
321
322
323/* Code */
324
325    ASSERT(subMbType <= P_L0_4x4);
326
327    switch (subMbType)
328    {
329        case P_L0_8x8:
330            return(1);
331
332        case P_L0_8x4:
333        case P_L0_4x8:
334            return(2);
335
336        /* P_L0_4x4 */
337        default:
338            return(4);
339    }
340
341}
342
343/*------------------------------------------------------------------------------
344
345    Function: DecodeMbPred
346
347        Functional description:
348          Parse macroblock prediction information from bit stream and store
349          in 'pMbPred'.
350
351------------------------------------------------------------------------------*/
352
353u32 DecodeMbPred(strmData_t *pStrmData, mbPred_t *pMbPred, mbType_e mbType,
354    u32 numRefIdxActive)
355{
356
357/* Variables */
358
359    u32 tmp, i, j, value;
360    i32 itmp;
361
362/* Code */
363
364    ASSERT(pStrmData);
365    ASSERT(pMbPred);
366
367    switch (h264bsdMbPartPredMode(mbType))
368    {
369        case PRED_MODE_INTER: /* PRED_MODE_INTER */
370            if (numRefIdxActive > 1)
371            {
372                for (i = h264bsdNumMbPart(mbType), j = 0; i--;  j++)
373                {
374                    tmp = h264bsdDecodeExpGolombTruncated(pStrmData, &value,
375                        (u32)(numRefIdxActive > 2));
376                    if (tmp != HANTRO_OK || value >= numRefIdxActive)
377                        return(HANTRO_NOK);
378
379                    pMbPred->refIdxL0[j] = value;
380                }
381            }
382
383            for (i = h264bsdNumMbPart(mbType), j = 0; i--;  j++)
384            {
385                tmp = h264bsdDecodeExpGolombSigned(pStrmData, &itmp);
386                if (tmp != HANTRO_OK)
387                    return(tmp);
388                pMbPred->mvdL0[j].hor = (i16)itmp;
389
390                tmp = h264bsdDecodeExpGolombSigned(pStrmData, &itmp);
391                if (tmp != HANTRO_OK)
392                    return(tmp);
393                pMbPred->mvdL0[j].ver = (i16)itmp;
394            }
395            break;
396
397        case PRED_MODE_INTRA4x4:
398            for (itmp = 0, i = 0; itmp < 2; itmp++)
399            {
400                value = h264bsdShowBits32(pStrmData);
401                tmp = 0;
402                for (j = 8; j--; i++)
403                {
404                    pMbPred->prevIntra4x4PredModeFlag[i] =
405                        value & 0x80000000 ? HANTRO_TRUE : HANTRO_FALSE;
406                    value <<= 1;
407                    if (!pMbPred->prevIntra4x4PredModeFlag[i])
408                    {
409                        pMbPred->remIntra4x4PredMode[i] = value>>29;
410                        value <<= 3;
411                        tmp++;
412                    }
413                }
414                if (h264bsdFlushBits(pStrmData, 8 + 3*tmp) == END_OF_STREAM)
415                    return(HANTRO_NOK);
416            }
417            /* fall-through */
418
419        case PRED_MODE_INTRA16x16:
420            tmp = h264bsdDecodeExpGolombUnsigned(pStrmData, &value);
421            if (tmp != HANTRO_OK || value > 3)
422                return(HANTRO_NOK);
423            pMbPred->intraChromaPredMode = value;
424            break;
425    }
426
427    return(HANTRO_OK);
428
429}
430
431/*------------------------------------------------------------------------------
432
433    Function: DecodeSubMbPred
434
435        Functional description:
436          Parse sub-macroblock prediction information from bit stream and
437          store in 'pMbPred'.
438
439------------------------------------------------------------------------------*/
440
441u32 DecodeSubMbPred(strmData_t *pStrmData, subMbPred_t *pSubMbPred,
442    mbType_e mbType, u32 numRefIdxActive)
443{
444
445/* Variables */
446
447    u32 tmp, i, j, value;
448    i32 itmp;
449
450/* Code */
451
452    ASSERT(pStrmData);
453    ASSERT(pSubMbPred);
454    ASSERT(h264bsdMbPartPredMode(mbType) == PRED_MODE_INTER);
455
456    for (i = 0; i < 4; i++)
457    {
458        tmp = h264bsdDecodeExpGolombUnsigned(pStrmData, &value);
459        if (tmp != HANTRO_OK || value > 3)
460            return(HANTRO_NOK);
461        pSubMbPred->subMbType[i] = (subMbType_e)value;
462    }
463
464    if ( (numRefIdxActive > 1) && (mbType != P_8x8ref0) )
465    {
466        for (i = 0; i < 4; i++)
467        {
468            tmp = h264bsdDecodeExpGolombTruncated(pStrmData, &value,
469                (u32)(numRefIdxActive > 2));
470            if (tmp != HANTRO_OK || value >= numRefIdxActive)
471                return(HANTRO_NOK);
472            pSubMbPred->refIdxL0[i] = value;
473        }
474    }
475
476    for (i = 0; i < 4; i++)
477    {
478        j = 0;
479        for (value = h264bsdNumSubMbPart(pSubMbPred->subMbType[i]);
480             value--; j++)
481        {
482            tmp = h264bsdDecodeExpGolombSigned(pStrmData, &itmp);
483            if (tmp != HANTRO_OK)
484                return(tmp);
485            pSubMbPred->mvdL0[i][j].hor = (i16)itmp;
486
487            tmp = h264bsdDecodeExpGolombSigned(pStrmData, &itmp);
488            if (tmp != HANTRO_OK)
489                return(tmp);
490            pSubMbPred->mvdL0[i][j].ver = (i16)itmp;
491        }
492    }
493
494    return(HANTRO_OK);
495
496}
497
498#ifdef H264DEC_OMXDL
499/*------------------------------------------------------------------------------
500
501    Function: DecodeResidual
502
503        Functional description:
504          Parse residual information from bit stream and store in 'pResidual'.
505
506------------------------------------------------------------------------------*/
507
508u32 DecodeResidual(strmData_t *pStrmData, residual_t *pResidual,
509    mbStorage_t *pMb, mbType_e mbType, u32 codedBlockPattern)
510{
511
512/* Variables */
513
514    u32 i, j;
515    u32 blockCoded;
516    u32 blockIndex;
517    u32 is16x16;
518    OMX_INT nc;
519    OMXResult omxRes;
520    OMX_U8 *pPosCoefBuf;
521
522
523/* Code */
524
525    ASSERT(pStrmData);
526    ASSERT(pResidual);
527
528    pPosCoefBuf = pResidual->posCoefBuf;
529
530    /* luma DC is at index 24 */
531    if (h264bsdMbPartPredMode(mbType) == PRED_MODE_INTRA16x16)
532    {
533        nc = (OMX_INT)DetermineNc(pMb, 0, pResidual->totalCoeff);
534#ifndef H264DEC_NEON
535        omxRes =  omxVCM4P10_DecodeCoeffsToPairCAVLC(
536                (const OMX_U8 **) (&pStrmData->pStrmCurrPos),
537                (OMX_S32*) (&pStrmData->bitPosInWord),
538                &pResidual->totalCoeff[24],
539                &pPosCoefBuf,
540                nc,
541                16);
542#else
543        omxRes = armVCM4P10_DecodeCoeffsToPair(
544                (const OMX_U8 **) (&pStrmData->pStrmCurrPos),
545                (OMX_S32*) (&pStrmData->bitPosInWord),
546                &pResidual->totalCoeff[24],
547                &pPosCoefBuf,
548                nc,
549                16);
550#endif
551        if (omxRes != OMX_Sts_NoErr)
552            return(HANTRO_NOK);
553        is16x16 = HANTRO_TRUE;
554    }
555    else
556        is16x16 = HANTRO_FALSE;
557
558    for (i = 4, blockIndex = 0; i--;)
559    {
560        /* luma cbp in bits 0-3 */
561        blockCoded = codedBlockPattern & 0x1;
562        codedBlockPattern >>= 1;
563        if (blockCoded)
564        {
565            for (j = 4; j--; blockIndex++)
566            {
567                nc = (OMX_INT)DetermineNc(pMb,blockIndex,pResidual->totalCoeff);
568                if (is16x16)
569                {
570#ifndef H264DEC_NEON
571                    omxRes =  omxVCM4P10_DecodeCoeffsToPairCAVLC(
572                            (const OMX_U8 **) (&pStrmData->pStrmCurrPos),
573                            (OMX_S32*) (&pStrmData->bitPosInWord),
574                            &pResidual->totalCoeff[blockIndex],
575                            &pPosCoefBuf,
576                            nc,
577                            15);
578#else
579                    omxRes =  armVCM4P10_DecodeCoeffsToPair(
580                            (const OMX_U8 **) (&pStrmData->pStrmCurrPos),
581                            (OMX_S32*) (&pStrmData->bitPosInWord),
582                            &pResidual->totalCoeff[blockIndex],
583                            &pPosCoefBuf,
584                            nc,
585                            15);
586#endif
587                }
588                else
589                {
590#ifndef H264DEC_NEON
591                    omxRes =  omxVCM4P10_DecodeCoeffsToPairCAVLC(
592                            (const OMX_U8 **) (&pStrmData->pStrmCurrPos),
593                            (OMX_S32*) (&pStrmData->bitPosInWord),
594                            &pResidual->totalCoeff[blockIndex],
595                            &pPosCoefBuf,
596                            nc,
597                            16);
598#else
599                    omxRes = armVCM4P10_DecodeCoeffsToPair(
600                            (const OMX_U8 **) (&pStrmData->pStrmCurrPos),
601                            (OMX_S32*) (&pStrmData->bitPosInWord),
602                            &pResidual->totalCoeff[blockIndex],
603                            &pPosCoefBuf,
604                            nc,
605                            16);
606#endif
607                }
608                if (omxRes != OMX_Sts_NoErr)
609                    return(HANTRO_NOK);
610            }
611        }
612        else
613            blockIndex += 4;
614    }
615
616    /* chroma DC block are at indices 25 and 26 */
617    blockCoded = codedBlockPattern & 0x3;
618    if (blockCoded)
619    {
620#ifndef H264DEC_NEON
621        omxRes =  omxVCM4P10_DecodeChromaDcCoeffsToPairCAVLC(
622                (const OMX_U8**) (&pStrmData->pStrmCurrPos),
623                (OMX_S32*) (&pStrmData->bitPosInWord),
624                &pResidual->totalCoeff[25],
625                &pPosCoefBuf);
626#else
627        omxRes = armVCM4P10_DecodeCoeffsToPair(
628                (const OMX_U8**) (&pStrmData->pStrmCurrPos),
629                (OMX_S32*) (&pStrmData->bitPosInWord),
630                &pResidual->totalCoeff[25],
631                &pPosCoefBuf,
632                17,
633                4);
634#endif
635        if (omxRes != OMX_Sts_NoErr)
636            return(HANTRO_NOK);
637#ifndef H264DEC_NEON
638        omxRes =  omxVCM4P10_DecodeChromaDcCoeffsToPairCAVLC(
639                (const OMX_U8**) (&pStrmData->pStrmCurrPos),
640                (OMX_S32*) (&pStrmData->bitPosInWord),
641                &pResidual->totalCoeff[26],
642                &pPosCoefBuf);
643#else
644        omxRes = armVCM4P10_DecodeCoeffsToPair(
645                (const OMX_U8**) (&pStrmData->pStrmCurrPos),
646                (OMX_S32*) (&pStrmData->bitPosInWord),
647                &pResidual->totalCoeff[26],
648                &pPosCoefBuf,
649                17,
650                4);
651#endif
652        if (omxRes != OMX_Sts_NoErr)
653            return(HANTRO_NOK);
654    }
655
656    /* chroma AC */
657    blockCoded = codedBlockPattern & 0x2;
658    if (blockCoded)
659    {
660        for (i = 8; i--;blockIndex++)
661        {
662            nc = (OMX_INT)DetermineNc(pMb, blockIndex, pResidual->totalCoeff);
663#ifndef H264DEC_NEON
664            omxRes =  omxVCM4P10_DecodeCoeffsToPairCAVLC(
665                    (const OMX_U8 **) (&pStrmData->pStrmCurrPos),
666                    (OMX_S32*) (&pStrmData->bitPosInWord),
667                    &pResidual->totalCoeff[blockIndex],
668                    &pPosCoefBuf,
669                    nc,
670                    15);
671#else
672            omxRes =  armVCM4P10_DecodeCoeffsToPair(
673                    (const OMX_U8 **) (&pStrmData->pStrmCurrPos),
674                    (OMX_S32*) (&pStrmData->bitPosInWord),
675                    &pResidual->totalCoeff[blockIndex],
676                    &pPosCoefBuf,
677                    nc,
678                    15);
679#endif
680            if (omxRes != OMX_Sts_NoErr)
681                return(HANTRO_NOK);
682        }
683    }
684
685    return(HANTRO_OK);
686
687}
688
689#else
690/*------------------------------------------------------------------------------
691
692    Function: DecodeResidual
693
694        Functional description:
695          Parse residual information from bit stream and store in 'pResidual'.
696
697------------------------------------------------------------------------------*/
698
699u32 DecodeResidual(strmData_t *pStrmData, residual_t *pResidual,
700    mbStorage_t *pMb, mbType_e mbType, u32 codedBlockPattern)
701{
702
703/* Variables */
704
705    u32 i, j, tmp;
706    i32 nc;
707    u32 blockCoded;
708    u32 blockIndex;
709    u32 is16x16;
710    i32 (*level)[16];
711
712/* Code */
713
714    ASSERT(pStrmData);
715    ASSERT(pResidual);
716
717    level = pResidual->level;
718
719    /* luma DC is at index 24 */
720    if (h264bsdMbPartPredMode(mbType) == PRED_MODE_INTRA16x16)
721    {
722        nc = (i32)DetermineNc(pMb, 0, pResidual->totalCoeff);
723        tmp = h264bsdDecodeResidualBlockCavlc(pStrmData, level[24], nc, 16);
724        if ((tmp & 0xF) != HANTRO_OK)
725            return(tmp);
726        pResidual->totalCoeff[24] = (tmp >> 4) & 0xFF;
727        is16x16 = HANTRO_TRUE;
728    }
729    else
730        is16x16 = HANTRO_FALSE;
731
732    for (i = 4, blockIndex = 0; i--;)
733    {
734        /* luma cbp in bits 0-3 */
735        blockCoded = codedBlockPattern & 0x1;
736        codedBlockPattern >>= 1;
737        if (blockCoded)
738        {
739            for (j = 4; j--; blockIndex++)
740            {
741                nc = (i32)DetermineNc(pMb, blockIndex, pResidual->totalCoeff);
742                if (is16x16)
743                {
744                    tmp = h264bsdDecodeResidualBlockCavlc(pStrmData,
745                        level[blockIndex] + 1, nc, 15);
746                    pResidual->coeffMap[blockIndex] = tmp >> 15;
747                }
748                else
749                {
750                    tmp = h264bsdDecodeResidualBlockCavlc(pStrmData,
751                        level[blockIndex], nc, 16);
752                    pResidual->coeffMap[blockIndex] = tmp >> 16;
753                }
754                if ((tmp & 0xF) != HANTRO_OK)
755                    return(tmp);
756                pResidual->totalCoeff[blockIndex] = (tmp >> 4) & 0xFF;
757            }
758        }
759        else
760            blockIndex += 4;
761    }
762
763    /* chroma DC block are at indices 25 and 26 */
764    blockCoded = codedBlockPattern & 0x3;
765    if (blockCoded)
766    {
767        tmp = h264bsdDecodeResidualBlockCavlc(pStrmData, level[25], -1, 4);
768        if ((tmp & 0xF) != HANTRO_OK)
769            return(tmp);
770        pResidual->totalCoeff[25] = (tmp >> 4) & 0xFF;
771        tmp = h264bsdDecodeResidualBlockCavlc(pStrmData, level[25]+4, -1, 4);
772        if ((tmp & 0xF) != HANTRO_OK)
773            return(tmp);
774        pResidual->totalCoeff[26] = (tmp >> 4) & 0xFF;
775    }
776
777    /* chroma AC */
778    blockCoded = codedBlockPattern & 0x2;
779    if (blockCoded)
780    {
781        for (i = 8; i--;blockIndex++)
782        {
783            nc = (i32)DetermineNc(pMb, blockIndex, pResidual->totalCoeff);
784            tmp = h264bsdDecodeResidualBlockCavlc(pStrmData,
785                level[blockIndex] + 1, nc, 15);
786            if ((tmp & 0xF) != HANTRO_OK)
787                return(tmp);
788            pResidual->totalCoeff[blockIndex] = (tmp >> 4) & 0xFF;
789            pResidual->coeffMap[blockIndex] = (tmp >> 15);
790        }
791    }
792
793    return(HANTRO_OK);
794
795}
796#endif
797
798/*------------------------------------------------------------------------------
799
800    Function: DetermineNc
801
802        Functional description:
803          Returns the nC of a block.
804
805------------------------------------------------------------------------------*/
806#ifdef H264DEC_OMXDL
807u32 DetermineNc(mbStorage_t *pMb, u32 blockIndex, u8 *pTotalCoeff)
808#else
809u32 DetermineNc(mbStorage_t *pMb, u32 blockIndex, i16 *pTotalCoeff)
810#endif
811{
812/*lint -e702 */
813/* Variables */
814
815    u32 tmp;
816    i32 n;
817    const neighbour_t *neighbourA, *neighbourB;
818    u8 neighbourAindex, neighbourBindex;
819
820/* Code */
821
822    ASSERT(blockIndex < 24);
823
824    /* if neighbour block belongs to current macroblock totalCoeff array
825     * mbStorage has not been set/updated yet -> use pTotalCoeff */
826    neighbourA = h264bsdNeighbour4x4BlockA(blockIndex);
827    neighbourB = h264bsdNeighbour4x4BlockB(blockIndex);
828    neighbourAindex = neighbourA->index;
829    neighbourBindex = neighbourB->index;
830    if (neighbourA->mb == MB_CURR && neighbourB->mb == MB_CURR)
831    {
832        n = (pTotalCoeff[neighbourAindex] +
833             pTotalCoeff[neighbourBindex] + 1)>>1;
834    }
835    else if (neighbourA->mb == MB_CURR)
836    {
837        n = pTotalCoeff[neighbourAindex];
838        if (h264bsdIsNeighbourAvailable(pMb, pMb->mbB))
839        {
840            n = (n + pMb->mbB->totalCoeff[neighbourBindex] + 1) >> 1;
841        }
842    }
843    else if (neighbourB->mb == MB_CURR)
844    {
845        n = pTotalCoeff[neighbourBindex];
846        if (h264bsdIsNeighbourAvailable(pMb, pMb->mbA))
847        {
848            n = (n + pMb->mbA->totalCoeff[neighbourAindex] + 1) >> 1;
849        }
850    }
851    else
852    {
853        n = tmp = 0;
854        if (h264bsdIsNeighbourAvailable(pMb, pMb->mbA))
855        {
856            n = pMb->mbA->totalCoeff[neighbourAindex];
857            tmp = 1;
858        }
859        if (h264bsdIsNeighbourAvailable(pMb, pMb->mbB))
860        {
861            if (tmp)
862                n = (n + pMb->mbB->totalCoeff[neighbourBindex] + 1) >> 1;
863            else
864                n = pMb->mbB->totalCoeff[neighbourBindex];
865        }
866    }
867    return((u32)n);
868/*lint +e702 */
869}
870
871/*------------------------------------------------------------------------------
872
873    Function: CbpIntra16x16
874
875        Functional description:
876          Returns the coded block pattern for intra 16x16 macroblock.
877
878------------------------------------------------------------------------------*/
879
880u32 CbpIntra16x16(mbType_e mbType)
881{
882
883/* Variables */
884
885    u32 cbp;
886    u32 tmp;
887
888/* Code */
889
890    ASSERT(mbType >= I_16x16_0_0_0 && mbType <= I_16x16_3_2_1);
891
892    if (mbType >= I_16x16_0_0_1)
893        cbp = 15;
894    else
895        cbp = 0;
896
897    /* tmp is 0 for I_16x16_0_0_0 mb type */
898    /* ignore lint warning on arithmetic on enum's */
899    tmp = /*lint -e(656)*/(mbType - I_16x16_0_0_0) >> 2;
900    if (tmp > 2)
901        tmp -= 3;
902
903    cbp += tmp << 4;
904
905    return(cbp);
906
907}
908
909/*------------------------------------------------------------------------------
910
911    Function: h264bsdPredModeIntra16x16
912
913        Functional description:
914          Returns the prediction mode for intra 16x16 macroblock.
915
916------------------------------------------------------------------------------*/
917
918u32 h264bsdPredModeIntra16x16(mbType_e mbType)
919{
920
921/* Variables */
922
923    u32 tmp;
924
925/* Code */
926
927    ASSERT(mbType >= I_16x16_0_0_0 && mbType <= I_16x16_3_2_1);
928
929    /* tmp is 0 for I_16x16_0_0_0 mb type */
930    /* ignore lint warning on arithmetic on enum's */
931    tmp = /*lint -e(656)*/(mbType - I_16x16_0_0_0);
932
933    return(tmp & 0x3);
934
935}
936
937/*------------------------------------------------------------------------------
938
939    Function: h264bsdDecodeMacroblock
940
941        Functional description:
942          Decode one macroblock and write into output image.
943
944        Inputs:
945          pMb           pointer to macroblock specific information
946          mbLayer       pointer to current macroblock data from stream
947          currImage     pointer to output image
948          dpb           pointer to decoded picture buffer
949          qpY           pointer to slice QP
950          mbNum         current macroblock number
951          constrainedIntraPred  flag specifying if neighbouring inter
952                                macroblocks are used in intra prediction
953
954        Outputs:
955          pMb           structure is updated with current macroblock
956          currImage     decoded macroblock is written into output image
957
958        Returns:
959          HANTRO_OK     success
960          HANTRO_NOK    error in macroblock decoding
961
962------------------------------------------------------------------------------*/
963
964u32 h264bsdDecodeMacroblock(mbStorage_t *pMb, macroblockLayer_t *pMbLayer,
965    image_t *currImage, dpbStorage_t *dpb, i32 *qpY, u32 mbNum,
966    u32 constrainedIntraPredFlag, u8* data)
967{
968
969/* Variables */
970
971    u32 i, tmp;
972    mbType_e mbType;
973#ifdef H264DEC_OMXDL
974    const u8 *pSrc;
975#endif
976/* Code */
977
978    ASSERT(pMb);
979    ASSERT(pMbLayer);
980    ASSERT(currImage);
981    ASSERT(qpY && *qpY < 52);
982    ASSERT(mbNum < currImage->width*currImage->height);
983
984    mbType = pMbLayer->mbType;
985    pMb->mbType = mbType;
986
987    pMb->decoded++;
988
989    h264bsdSetCurrImageMbPointers(currImage, mbNum);
990
991    if (mbType == I_PCM)
992    {
993        u8 *pData = (u8*)data;
994#ifdef H264DEC_OMXDL
995        u8 *tot = pMb->totalCoeff;
996#else
997        i16 *tot = pMb->totalCoeff;
998#endif
999        i32 *lev = pMbLayer->residual.level[0];
1000
1001        pMb->qpY = 0;
1002
1003        /* if decoded flag > 1 -> mb has already been successfully decoded and
1004         * written to output -> do not write again */
1005        if (pMb->decoded > 1)
1006        {
1007            for (i = 24; i--;)
1008                *tot++ = 16;
1009            return HANTRO_OK;
1010        }
1011
1012        for (i = 24; i--;)
1013        {
1014            *tot++ = 16;
1015            for (tmp = 16; tmp--;)
1016                *pData++ = (u8)(*lev++);
1017        }
1018        h264bsdWriteMacroblock(currImage, (u8*)data);
1019
1020        return(HANTRO_OK);
1021    }
1022    else
1023    {
1024#ifdef H264DEC_OMXDL
1025        if (h264bsdMbPartPredMode(mbType) == PRED_MODE_INTER)
1026        {
1027            tmp = h264bsdInterPrediction(pMb, pMbLayer, dpb, mbNum,
1028                currImage, (u8*)data);
1029            if (tmp != HANTRO_OK) return (tmp);
1030        }
1031#endif
1032        if (mbType != P_Skip)
1033        {
1034            H264SwDecMemcpy(pMb->totalCoeff,
1035                            pMbLayer->residual.totalCoeff,
1036                            27*sizeof(*pMb->totalCoeff));
1037
1038            /* update qpY */
1039            if (pMbLayer->mbQpDelta)
1040            {
1041                *qpY = *qpY + pMbLayer->mbQpDelta;
1042                if (*qpY < 0) *qpY += 52;
1043                else if (*qpY >= 52) *qpY -= 52;
1044            }
1045            pMb->qpY = (u32)*qpY;
1046
1047#ifdef H264DEC_OMXDL
1048            pSrc = pMbLayer->residual.posCoefBuf;
1049
1050            if (h264bsdMbPartPredMode(mbType) == PRED_MODE_INTER)
1051            {
1052                OMXResult res;
1053                u8 *p;
1054                u8 *totalCoeff = pMb->totalCoeff;
1055
1056                for (i = 0; i < 16; i++, totalCoeff++)
1057                {
1058                    p = data + lumaIndex[i];
1059                    if (*totalCoeff)
1060                    {
1061                        res = omxVCM4P10_DequantTransformResidualFromPairAndAdd(
1062                                &pSrc, p, 0, p, 16, 16, *qpY, *totalCoeff);
1063                        if (res != OMX_Sts_NoErr)
1064                            return (HANTRO_NOK);
1065                    }
1066                }
1067
1068            }
1069            else if (h264bsdMbPartPredMode(mbType) == PRED_MODE_INTRA4x4)
1070            {
1071                tmp = ProcessIntra4x4Residual(pMb,
1072                                              data,
1073                                              constrainedIntraPredFlag,
1074                                              pMbLayer,
1075                                              &pSrc,
1076                                              currImage);
1077                if (tmp != HANTRO_OK)
1078                    return (tmp);
1079            }
1080            else if (h264bsdMbPartPredMode(mbType) == PRED_MODE_INTRA16x16)
1081            {
1082                tmp = ProcessIntra16x16Residual(pMb,
1083                                        data,
1084                                        constrainedIntraPredFlag,
1085                                        pMbLayer->mbPred.intraChromaPredMode,
1086                                        &pSrc,
1087                                        currImage);
1088                if (tmp != HANTRO_OK)
1089                    return (tmp);
1090            }
1091
1092            tmp = ProcessChromaResidual(pMb, data, &pSrc);
1093
1094#else
1095            tmp = ProcessResidual(pMb, pMbLayer->residual.level,
1096                pMbLayer->residual.coeffMap);
1097#endif
1098            if (tmp != HANTRO_OK)
1099                return (tmp);
1100        }
1101        else
1102        {
1103            H264SwDecMemset(pMb->totalCoeff, 0, 27*sizeof(*pMb->totalCoeff));
1104            pMb->qpY = (u32)*qpY;
1105        }
1106#ifdef H264DEC_OMXDL
1107        /* if decoded flag > 1 -> mb has already been successfully decoded and
1108         * written to output -> do not write again */
1109        if (pMb->decoded > 1)
1110            return HANTRO_OK;
1111
1112        h264bsdWriteMacroblock(currImage, data);
1113#else
1114        if (h264bsdMbPartPredMode(mbType) != PRED_MODE_INTER)
1115        {
1116            tmp = h264bsdIntraPrediction(pMb, pMbLayer, currImage, mbNum,
1117                constrainedIntraPredFlag, (u8*)data);
1118            if (tmp != HANTRO_OK) return (tmp);
1119        }
1120        else
1121        {
1122            tmp = h264bsdInterPrediction(pMb, pMbLayer, dpb, mbNum,
1123                currImage, (u8*)data);
1124            if (tmp != HANTRO_OK) return (tmp);
1125        }
1126#endif
1127    }
1128
1129    return HANTRO_OK;
1130}
1131
1132
1133#ifdef H264DEC_OMXDL
1134
1135/*------------------------------------------------------------------------------
1136
1137    Function: ProcessChromaResidual
1138
1139        Functional description:
1140          Process the residual data of chroma with
1141          inverse quantization and inverse transform.
1142
1143------------------------------------------------------------------------------*/
1144u32 ProcessChromaResidual(mbStorage_t *pMb, u8 *data, const u8 **pSrc )
1145{
1146    u32 i;
1147    u32 chromaQp;
1148    i16 *pDc;
1149    i16 dc[4 + 4] = {0,0,0,0,0,0,0,0};
1150    u8 *totalCoeff;
1151    OMXResult result;
1152    u8 *p;
1153
1154    /* chroma DC processing. First chroma dc block is block with index 25 */
1155    chromaQp =
1156        h264bsdQpC[CLIP3(0, 51, (i32)pMb->qpY + pMb->chromaQpIndexOffset)];
1157
1158    if (pMb->totalCoeff[25])
1159    {
1160        pDc = dc;
1161        result = omxVCM4P10_TransformDequantChromaDCFromPair(
1162                pSrc,
1163                pDc,
1164                (i32)chromaQp);
1165        if (result != OMX_Sts_NoErr)
1166            return (HANTRO_NOK);
1167    }
1168    if (pMb->totalCoeff[26])
1169    {
1170        pDc = dc+4;
1171        result = omxVCM4P10_TransformDequantChromaDCFromPair(
1172                pSrc,
1173                pDc,
1174                (i32)chromaQp);
1175        if (result != OMX_Sts_NoErr)
1176            return (HANTRO_NOK);
1177    }
1178
1179    pDc = dc;
1180    totalCoeff = pMb->totalCoeff + 16;
1181    for (i = 0; i < 8; i++, pDc++, totalCoeff++)
1182    {
1183        /* chroma prediction */
1184        if (*totalCoeff || *pDc)
1185        {
1186            p = data + chromaIndex[i];
1187            result = omxVCM4P10_DequantTransformResidualFromPairAndAdd(
1188                    pSrc,
1189                    p,
1190                    pDc,
1191                    p,
1192                    8,
1193                    8,
1194                    (i32)chromaQp,
1195                    *totalCoeff);
1196            if (result != OMX_Sts_NoErr)
1197                return (HANTRO_NOK);
1198        }
1199    }
1200
1201    return(HANTRO_OK);
1202}
1203
1204/*------------------------------------------------------------------------------
1205
1206    Function: ProcessIntra16x16Residual
1207
1208        Functional description:
1209          Process the residual data of luma with
1210          inverse quantization and inverse transform.
1211
1212------------------------------------------------------------------------------*/
1213u32 ProcessIntra16x16Residual(mbStorage_t *pMb,
1214                              u8 *data,
1215                              u32 constrainedIntraPred,
1216                              u32 intraChromaPredMode,
1217                              const u8** pSrc,
1218                              image_t *image)
1219{
1220    u32 i;
1221    i16 *pDc;
1222    i16 dc[16] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
1223    u8 *totalCoeff;
1224    OMXResult result;
1225    u8 *p;
1226
1227    totalCoeff = pMb->totalCoeff;
1228
1229    if (totalCoeff[24])
1230    {
1231        pDc = dc;
1232        result = omxVCM4P10_TransformDequantLumaDCFromPair(
1233                    pSrc,
1234                    pDc,
1235                    (i32)pMb->qpY);
1236        if (result != OMX_Sts_NoErr)
1237            return (HANTRO_NOK);
1238    }
1239    /* Intra 16x16 pred */
1240    if (h264bsdIntra16x16Prediction(pMb, data, image->luma,
1241                            image->width*16, constrainedIntraPred) != HANTRO_OK)
1242        return(HANTRO_NOK);
1243    for (i = 0; i < 16; i++, totalCoeff++)
1244    {
1245        p = data + lumaIndex[i];
1246        pDc = &dc[dcCoeffIndex[i]];
1247        if (*totalCoeff || *pDc)
1248        {
1249            result = omxVCM4P10_DequantTransformResidualFromPairAndAdd(
1250                    pSrc,
1251                    p,
1252                    pDc,
1253                    p,
1254                    16,
1255                    16,
1256                    (i32)pMb->qpY,
1257                    *totalCoeff);
1258            if (result != OMX_Sts_NoErr)
1259                return (HANTRO_NOK);
1260        }
1261    }
1262
1263    if (h264bsdIntraChromaPrediction(pMb, data + 256,
1264                image,
1265                intraChromaPredMode,
1266                constrainedIntraPred) != HANTRO_OK)
1267        return(HANTRO_NOK);
1268
1269    return HANTRO_OK;
1270}
1271
1272/*------------------------------------------------------------------------------
1273
1274    Function: ProcessIntra4x4Residual
1275
1276        Functional description:
1277          Process the residual data of luma with
1278          inverse quantization and inverse transform.
1279
1280------------------------------------------------------------------------------*/
1281u32 ProcessIntra4x4Residual(mbStorage_t *pMb,
1282                            u8 *data,
1283                            u32 constrainedIntraPred,
1284                            macroblockLayer_t *mbLayer,
1285                            const u8 **pSrc,
1286                            image_t *image)
1287{
1288    u32 i;
1289    u8 *totalCoeff;
1290    OMXResult result;
1291    u8 *p;
1292
1293    totalCoeff = pMb->totalCoeff;
1294
1295    for (i = 0; i < 16; i++, totalCoeff++)
1296    {
1297        p = data + lumaIndex[i];
1298        if (h264bsdIntra4x4Prediction(pMb, p, mbLayer, image->luma,
1299                    image->width*16, constrainedIntraPred, i) != HANTRO_OK)
1300            return(HANTRO_NOK);
1301
1302        if (*totalCoeff)
1303        {
1304            result = omxVCM4P10_DequantTransformResidualFromPairAndAdd(
1305                    pSrc,
1306                    p,
1307                    NULL,
1308                    p,
1309                    16,
1310                    16,
1311                    (i32)pMb->qpY,
1312                    *totalCoeff);
1313            if (result != OMX_Sts_NoErr)
1314                return (HANTRO_NOK);
1315        }
1316    }
1317
1318    if (h264bsdIntraChromaPrediction(pMb, data + 256,
1319                image,
1320                mbLayer->mbPred.intraChromaPredMode,
1321                constrainedIntraPred) != HANTRO_OK)
1322        return(HANTRO_NOK);
1323
1324    return HANTRO_OK;
1325}
1326
1327#else /* H264DEC_OMXDL */
1328
1329/*------------------------------------------------------------------------------
1330
1331    Function: ProcessResidual
1332
1333        Functional description:
1334          Process the residual data of one macroblock with
1335          inverse quantization and inverse transform.
1336
1337------------------------------------------------------------------------------*/
1338
1339u32 ProcessResidual(mbStorage_t *pMb, i32 residualLevel[][16], u32 *coeffMap)
1340{
1341
1342/* Variables */
1343
1344    u32 i;
1345    u32 chromaQp;
1346    i32 (*blockData)[16];
1347    i32 (*blockDc)[16];
1348    i16 *totalCoeff;
1349    i32 *chromaDc;
1350    const u32 *dcCoeffIdx;
1351
1352/* Code */
1353
1354    ASSERT(pMb);
1355    ASSERT(residualLevel);
1356
1357    /* set pointers to DC coefficient blocks */
1358    blockDc = residualLevel + 24;
1359
1360    blockData = residualLevel;
1361    totalCoeff = pMb->totalCoeff;
1362    if (h264bsdMbPartPredMode(pMb->mbType) == PRED_MODE_INTRA16x16)
1363    {
1364        if (totalCoeff[24])
1365        {
1366            h264bsdProcessLumaDc(*blockDc, pMb->qpY);
1367        }
1368        dcCoeffIdx = dcCoeffIndex;
1369
1370        for (i = 16; i--; blockData++, totalCoeff++, coeffMap++)
1371        {
1372            /* set dc coefficient of luma block */
1373            (*blockData)[0] = (*blockDc)[*dcCoeffIdx++];
1374            if ((*blockData)[0] || *totalCoeff)
1375            {
1376                if (h264bsdProcessBlock(*blockData, pMb->qpY, 1, *coeffMap) !=
1377                    HANTRO_OK)
1378                    return(HANTRO_NOK);
1379            }
1380            else
1381                MARK_RESIDUAL_EMPTY(*blockData);
1382        }
1383    }
1384    else
1385    {
1386        for (i = 16; i--; blockData++, totalCoeff++, coeffMap++)
1387        {
1388            if (*totalCoeff)
1389            {
1390                if (h264bsdProcessBlock(*blockData, pMb->qpY, 0, *coeffMap) !=
1391                    HANTRO_OK)
1392                    return(HANTRO_NOK);
1393            }
1394            else
1395                MARK_RESIDUAL_EMPTY(*blockData);
1396        }
1397    }
1398
1399    /* chroma DC processing. First chroma dc block is block with index 25 */
1400    chromaQp =
1401        h264bsdQpC[CLIP3(0, 51, (i32)pMb->qpY + pMb->chromaQpIndexOffset)];
1402    if (pMb->totalCoeff[25] || pMb->totalCoeff[26])
1403        h264bsdProcessChromaDc(residualLevel[25], chromaQp);
1404    chromaDc = residualLevel[25];
1405    for (i = 8; i--; blockData++, totalCoeff++, coeffMap++)
1406    {
1407        /* set dc coefficient of chroma block */
1408        (*blockData)[0] = *chromaDc++;
1409        if ((*blockData)[0] || *totalCoeff)
1410        {
1411            if (h264bsdProcessBlock(*blockData, chromaQp, 1,*coeffMap) !=
1412                HANTRO_OK)
1413                return(HANTRO_NOK);
1414        }
1415        else
1416            MARK_RESIDUAL_EMPTY(*blockData);
1417    }
1418
1419    return(HANTRO_OK);
1420}
1421#endif /* H264DEC_OMXDL */
1422
1423/*------------------------------------------------------------------------------
1424
1425    Function: h264bsdSubMbPartMode
1426
1427        Functional description:
1428          Returns the macroblock's sub-partition mode.
1429
1430------------------------------------------------------------------------------*/
1431
1432subMbPartMode_e h264bsdSubMbPartMode(subMbType_e subMbType)
1433{
1434
1435/* Variables */
1436
1437
1438/* Code */
1439
1440    ASSERT(subMbType < 4);
1441
1442    return((subMbPartMode_e)subMbType);
1443
1444}
1445
1446
1447