1/* 2 * Copyright (C) 2009 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17/*------------------------------------------------------------------------------ 18 19 Table of contents 20 21 1. Include headers 22 2. External compiler flags 23 3. Module defines 24 4. Local function prototypes 25 5. Functions 26 h264bsdConceal 27 ConcealMb 28 Transform 29 30------------------------------------------------------------------------------*/ 31 32/*------------------------------------------------------------------------------ 33 1. Include headers 34------------------------------------------------------------------------------*/ 35 36#include "h264bsd_conceal.h" 37#include "h264bsd_util.h" 38#include "h264bsd_reconstruct.h" 39#include "h264bsd_dpb.h" 40 41/*------------------------------------------------------------------------------ 42 2. External compiler flags 43-------------------------------------------------------------------------------- 44 45-------------------------------------------------------------------------------- 46 3. Module defines 47------------------------------------------------------------------------------*/ 48 49/*lint -e702 disable lint warning on right shift of signed quantity */ 50 51/*------------------------------------------------------------------------------ 52 4. Local function prototypes 53------------------------------------------------------------------------------*/ 54 55static u32 ConcealMb(mbStorage_t *pMb, image_t *currImage, u32 row, u32 col, 56 u32 sliceType, u8 *data); 57 58static void Transform(i32 *data); 59 60/*------------------------------------------------------------------------------ 61 62 Function name: h264bsdConceal 63 64 Functional description: 65 Perform error concealment for a picture. Two types of concealment 66 is performed based on sliceType: 67 1) copy from previous picture for P-slices. 68 2) concealment from neighbour pixels for I-slices 69 70 I-type concealment is based on ideas presented by Jarno Tulkki. 71 The concealment algorithm determines frequency domain coefficients 72 from the neighbour pixels, applies integer transform (the same 73 transform used in the residual processing) and uses the results as 74 pixel values for concealed macroblocks. Transform produces 4x4 75 array and one pixel value has to be used for 4x4 luma blocks and 76 2x2 chroma blocks. 77 78 Similar concealment is performed for whole picture (the choise 79 of the type is based on last successfully decoded slice header of 80 the picture but it is handled by the calling function). It is 81 acknowledged that this may result in wrong type of concealment 82 when a picture contains both types of slices. However, 83 determination of slice type macroblock-by-macroblock cannot 84 be done due to the fact that it is impossible to know to which 85 slice each corrupted (not successfully decoded) macroblock 86 belongs. 87 88 The error concealment is started by searching the first propoerly 89 decoded macroblock and concealing the row containing the macroblock 90 in question. After that all macroblocks above the row in question 91 are concealed. Finally concealment of rows below is performed. 92 The order of concealment for 4x4 picture where macroblock 9 is the 93 first properly decoded one is as follows (properly decoded 94 macroblocks marked with 'x', numbers indicating the order of 95 concealment): 96 97 4 6 8 10 98 3 5 7 9 99 1 x x 2 100 11 12 13 14 101 102 If all macroblocks of the picture are lost, the concealment is 103 copy of previous picture for P-type and setting the image to 104 constant gray (pixel value 128) for I-type. 105 106 Concealment sets quantization parameter of the concealed 107 macroblocks to value 40 and macroblock type to intra to enable 108 deblocking filter to smooth the edges of the concealed areas. 109 110 Inputs: 111 pStorage pointer to storage structure 112 currImage pointer to current image structure 113 sliceType type of the slice 114 115 Outputs: 116 currImage concealed macroblocks will be written here 117 118 Returns: 119 HANTRO_OK 120 121------------------------------------------------------------------------------*/ 122 123u32 h264bsdConceal(storage_t *pStorage, image_t *currImage, u32 sliceType) 124{ 125 126/* Variables */ 127 128 u32 i, j; 129 u32 row, col; 130 u32 width, height; 131 u8 *refData; 132 mbStorage_t *mb; 133 134/* Code */ 135 136 ASSERT(pStorage); 137 ASSERT(currImage); 138 139 DEBUG(("Concealing %s slice\n", IS_I_SLICE(sliceType) ? 140 "intra" : "inter")); 141 142 width = currImage->width; 143 height = currImage->height; 144 refData = NULL; 145 /* use reference picture with smallest available index */ 146 if (IS_P_SLICE(sliceType) || (pStorage->intraConcealmentFlag != 0)) 147 { 148 i = 0; 149 do 150 { 151 refData = h264bsdGetRefPicData(pStorage->dpb, i); 152 i++; 153 if (i >= 16) 154 break; 155 } while (refData == NULL); 156 } 157 158 i = row = col = 0; 159 /* find first properly decoded macroblock -> start point for concealment */ 160 while (i < pStorage->picSizeInMbs && !pStorage->mb[i].decoded) 161 { 162 i++; 163 col++; 164 if (col == width) 165 { 166 row++; 167 col = 0; 168 } 169 } 170 171 /* whole picture lost -> copy previous or set grey */ 172 if (i == pStorage->picSizeInMbs) 173 { 174 if ( (IS_I_SLICE(sliceType) && (pStorage->intraConcealmentFlag == 0)) || 175 refData == NULL) 176 H264SwDecMemset(currImage->data, 128, width*height*384); 177 else 178 H264SwDecMemcpy(currImage->data, refData, width*height*384); 179 180 pStorage->numConcealedMbs = pStorage->picSizeInMbs; 181 182 /* no filtering if whole picture concealed */ 183 for (i = 0; i < pStorage->picSizeInMbs; i++) 184 pStorage->mb[i].disableDeblockingFilterIdc = 1; 185 186 return(HANTRO_OK); 187 } 188 189 /* start from the row containing the first correct macroblock, conceal the 190 * row in question, all rows above that row and then continue downwards */ 191 mb = pStorage->mb + row * width; 192 for (j = col; j--;) 193 { 194 ConcealMb(mb+j, currImage, row, j, sliceType, refData); 195 mb[j].decoded = 1; 196 pStorage->numConcealedMbs++; 197 } 198 for (j = col + 1; j < width; j++) 199 { 200 if (!mb[j].decoded) 201 { 202 ConcealMb(mb+j, currImage, row, j, sliceType, refData); 203 mb[j].decoded = 1; 204 pStorage->numConcealedMbs++; 205 } 206 } 207 /* if previous row(s) could not be concealed -> conceal them now */ 208 if (row) 209 { 210 for (j = 0; j < width; j++) 211 { 212 i = row - 1; 213 mb = pStorage->mb + i*width + j; 214 do 215 { 216 ConcealMb(mb, currImage, i, j, sliceType, refData); 217 mb->decoded = 1; 218 pStorage->numConcealedMbs++; 219 mb -= width; 220 } while(i--); 221 } 222 } 223 224 /* process rows below the one containing the first correct macroblock */ 225 for (i = row + 1; i < height; i++) 226 { 227 mb = pStorage->mb + i * width; 228 229 for (j = 0; j < width; j++) 230 { 231 if (!mb[j].decoded) 232 { 233 ConcealMb(mb+j, currImage, i, j, sliceType, refData); 234 mb[j].decoded = 1; 235 pStorage->numConcealedMbs++; 236 } 237 } 238 } 239 240 return(HANTRO_OK); 241} 242 243/*------------------------------------------------------------------------------ 244 245 Function name: ConcealMb 246 247 Functional description: 248 Perform error concealment for one macroblock, location of the 249 macroblock in the picture indicated by row and col 250 251------------------------------------------------------------------------------*/ 252 253u32 ConcealMb(mbStorage_t *pMb, image_t *currImage, u32 row, u32 col, 254 u32 sliceType, u8 *refData) 255{ 256 257/* Variables */ 258 259 u32 i, j, comp; 260 u32 hor, ver; 261 u32 mbNum; 262 u32 width, height; 263 u8 *mbPos; 264 u8 data[384]; 265 u8 *pData; 266 i32 tmp; 267 i32 firstPhase[16]; 268 i32 *pTmp; 269 /* neighbours above, below, left and right */ 270 i32 a[4] = { 0,0,0,0 }, b[4], l[4] = { 0,0,0,0 }, r[4]; 271 u32 A, B, L, R; 272#ifdef H264DEC_OMXDL 273 u8 fillBuff[32*21 + 15 + 32]; 274 u8 *pFill; 275#endif 276/* Code */ 277 278 ASSERT(pMb); 279 ASSERT(!pMb->decoded); 280 ASSERT(currImage); 281 ASSERT(col < currImage->width); 282 ASSERT(row < currImage->height); 283 284#ifdef H264DEC_OMXDL 285 pFill = ALIGN(fillBuff, 16); 286#endif 287 width = currImage->width; 288 height = currImage->height; 289 mbNum = row * width + col; 290 291 h264bsdSetCurrImageMbPointers(currImage, mbNum); 292 293 mbPos = currImage->data + row * 16 * width * 16 + col * 16; 294 A = B = L = R = HANTRO_FALSE; 295 296 /* set qpY to 40 to enable some filtering in deblocking (stetson value) */ 297 pMb->qpY = 40; 298 pMb->disableDeblockingFilterIdc = 0; 299 /* mbType set to intra to perform filtering despite the values of other 300 * boundary strength determination fields */ 301 pMb->mbType = I_4x4; 302 pMb->filterOffsetA = 0; 303 pMb->filterOffsetB = 0; 304 pMb->chromaQpIndexOffset = 0; 305 306 if (IS_I_SLICE(sliceType)) 307 H264SwDecMemset(data, 0, sizeof(data)); 308 else 309 { 310 mv_t mv = {0,0}; 311 image_t refImage; 312 refImage.width = width; 313 refImage.height = height; 314 refImage.data = refData; 315 if (refImage.data) 316 { 317#ifndef H264DEC_OMXDL 318 h264bsdPredictSamples(data, &mv, &refImage, col*16, row*16, 319 0, 0, 16, 16); 320#else 321 h264bsdPredictSamples(data, &mv, &refImage, 322 ((row*16) + ((col*16)<<16)), 323 0x00001010, pFill); 324#endif 325 h264bsdWriteMacroblock(currImage, data); 326 327 return(HANTRO_OK); 328 } 329 else 330 H264SwDecMemset(data, 0, sizeof(data)); 331 } 332 333 H264SwDecMemset(firstPhase, 0, sizeof(firstPhase)); 334 335 /* counter for number of neighbours used */ 336 j = 0; 337 hor = ver = 0; 338 if (row && (pMb-width)->decoded) 339 { 340 A = HANTRO_TRUE; 341 pData = mbPos - width*16; 342 a[0] = *pData++; a[0] += *pData++; a[0] += *pData++; a[0] += *pData++; 343 a[1] = *pData++; a[1] += *pData++; a[1] += *pData++; a[1] += *pData++; 344 a[2] = *pData++; a[2] += *pData++; a[2] += *pData++; a[2] += *pData++; 345 a[3] = *pData++; a[3] += *pData++; a[3] += *pData++; a[3] += *pData++; 346 j++; 347 hor++; 348 firstPhase[0] += a[0] + a[1] + a[2] + a[3]; 349 firstPhase[1] += a[0] + a[1] - a[2] - a[3]; 350 } 351 if ((row != height - 1) && (pMb+width)->decoded) 352 { 353 B = HANTRO_TRUE; 354 pData = mbPos + 16*width*16; 355 b[0] = *pData++; b[0] += *pData++; b[0] += *pData++; b[0] += *pData++; 356 b[1] = *pData++; b[1] += *pData++; b[1] += *pData++; b[1] += *pData++; 357 b[2] = *pData++; b[2] += *pData++; b[2] += *pData++; b[2] += *pData++; 358 b[3] = *pData++; b[3] += *pData++; b[3] += *pData++; b[3] += *pData++; 359 j++; 360 hor++; 361 firstPhase[0] += b[0] + b[1] + b[2] + b[3]; 362 firstPhase[1] += b[0] + b[1] - b[2] - b[3]; 363 } 364 if (col && (pMb-1)->decoded) 365 { 366 L = HANTRO_TRUE; 367 pData = mbPos - 1; 368 l[0] = pData[0]; l[0] += pData[16*width]; 369 l[0] += pData[32*width]; l[0] += pData[48*width]; 370 pData += 64*width; 371 l[1] = pData[0]; l[1] += pData[16*width]; 372 l[1] += pData[32*width]; l[1] += pData[48*width]; 373 pData += 64*width; 374 l[2] = pData[0]; l[2] += pData[16*width]; 375 l[2] += pData[32*width]; l[2] += pData[48*width]; 376 pData += 64*width; 377 l[3] = pData[0]; l[3] += pData[16*width]; 378 l[3] += pData[32*width]; l[3] += pData[48*width]; 379 j++; 380 ver++; 381 firstPhase[0] += l[0] + l[1] + l[2] + l[3]; 382 firstPhase[4] += l[0] + l[1] - l[2] - l[3]; 383 } 384 if ((col != width - 1) && (pMb+1)->decoded) 385 { 386 R = HANTRO_TRUE; 387 pData = mbPos + 16; 388 r[0] = pData[0]; r[0] += pData[16*width]; 389 r[0] += pData[32*width]; r[0] += pData[48*width]; 390 pData += 64*width; 391 r[1] = pData[0]; r[1] += pData[16*width]; 392 r[1] += pData[32*width]; r[1] += pData[48*width]; 393 pData += 64*width; 394 r[2] = pData[0]; r[2] += pData[16*width]; 395 r[2] += pData[32*width]; r[2] += pData[48*width]; 396 pData += 64*width; 397 r[3] = pData[0]; r[3] += pData[16*width]; 398 r[3] += pData[32*width]; r[3] += pData[48*width]; 399 j++; 400 ver++; 401 firstPhase[0] += r[0] + r[1] + r[2] + r[3]; 402 firstPhase[4] += r[0] + r[1] - r[2] - r[3]; 403 } 404 405 /* at least one properly decoded neighbour available */ 406 ASSERT(j); 407 408 /*lint -esym(644,l,r,a,b) variable initialized above */ 409 if (!hor && L && R) 410 firstPhase[1] = (l[0]+l[1]+l[2]+l[3]-r[0]-r[1]-r[2]-r[3]) >> 5; 411 else if (hor) 412 firstPhase[1] >>= (3+hor); 413 414 if (!ver && A && B) 415 firstPhase[4] = (a[0]+a[1]+a[2]+a[3]-b[0]-b[1]-b[2]-b[3]) >> 5; 416 else if (ver) 417 firstPhase[4] >>= (3+ver); 418 419 switch (j) 420 { 421 case 1: 422 firstPhase[0] >>= 4; 423 break; 424 425 case 2: 426 firstPhase[0] >>= 5; 427 break; 428 429 case 3: 430 /* approximate (firstPhase[0]*4/3)>>6 */ 431 firstPhase[0] = (21 * firstPhase[0]) >> 10; 432 break; 433 434 default: /* 4 */ 435 firstPhase[0] >>= 6; 436 break; 437 438 } 439 440 441 Transform(firstPhase); 442 443 for (i = 0, pData = data, pTmp = firstPhase; i < 256;) 444 { 445 tmp = pTmp[(i & 0xF)>>2]; 446 /*lint -e734 CLIP1 macro results in value that fits into 8 bits */ 447 *pData++ = CLIP1(tmp); 448 /*lint +e734 */ 449 450 i++; 451 if (!(i & 0x3F)) 452 pTmp += 4; 453 } 454 455 /* chroma components */ 456 mbPos = currImage->data + width * height * 256 + 457 row * 8 * width * 8 + col * 8; 458 for (comp = 0; comp < 2; comp++) 459 { 460 461 H264SwDecMemset(firstPhase, 0, sizeof(firstPhase)); 462 463 /* counter for number of neighbours used */ 464 j = 0; 465 hor = ver = 0; 466 if (A) 467 { 468 pData = mbPos - width*8; 469 a[0] = *pData++; a[0] += *pData++; 470 a[1] = *pData++; a[1] += *pData++; 471 a[2] = *pData++; a[2] += *pData++; 472 a[3] = *pData++; a[3] += *pData++; 473 j++; 474 hor++; 475 firstPhase[0] += a[0] + a[1] + a[2] + a[3]; 476 firstPhase[1] += a[0] + a[1] - a[2] - a[3]; 477 } 478 if (B) 479 { 480 pData = mbPos + 8*width*8; 481 b[0] = *pData++; b[0] += *pData++; 482 b[1] = *pData++; b[1] += *pData++; 483 b[2] = *pData++; b[2] += *pData++; 484 b[3] = *pData++; b[3] += *pData++; 485 j++; 486 hor++; 487 firstPhase[0] += b[0] + b[1] + b[2] + b[3]; 488 firstPhase[1] += b[0] + b[1] - b[2] - b[3]; 489 } 490 if (L) 491 { 492 pData = mbPos - 1; 493 l[0] = pData[0]; l[0] += pData[8*width]; 494 pData += 16*width; 495 l[1] = pData[0]; l[1] += pData[8*width]; 496 pData += 16*width; 497 l[2] = pData[0]; l[2] += pData[8*width]; 498 pData += 16*width; 499 l[3] = pData[0]; l[3] += pData[8*width]; 500 j++; 501 ver++; 502 firstPhase[0] += l[0] + l[1] + l[2] + l[3]; 503 firstPhase[4] += l[0] + l[1] - l[2] - l[3]; 504 } 505 if (R) 506 { 507 pData = mbPos + 8; 508 r[0] = pData[0]; r[0] += pData[8*width]; 509 pData += 16*width; 510 r[1] = pData[0]; r[1] += pData[8*width]; 511 pData += 16*width; 512 r[2] = pData[0]; r[2] += pData[8*width]; 513 pData += 16*width; 514 r[3] = pData[0]; r[3] += pData[8*width]; 515 j++; 516 ver++; 517 firstPhase[0] += r[0] + r[1] + r[2] + r[3]; 518 firstPhase[4] += r[0] + r[1] - r[2] - r[3]; 519 } 520 if (!hor && L && R) 521 firstPhase[1] = (l[0]+l[1]+l[2]+l[3]-r[0]-r[1]-r[2]-r[3]) >> 4; 522 else if (hor) 523 firstPhase[1] >>= (2+hor); 524 525 if (!ver && A && B) 526 firstPhase[4] = (a[0]+a[1]+a[2]+a[3]-b[0]-b[1]-b[2]-b[3]) >> 4; 527 else if (ver) 528 firstPhase[4] >>= (2+ver); 529 530 switch (j) 531 { 532 case 1: 533 firstPhase[0] >>= 3; 534 break; 535 536 case 2: 537 firstPhase[0] >>= 4; 538 break; 539 540 case 3: 541 /* approximate (firstPhase[0]*4/3)>>5 */ 542 firstPhase[0] = (21 * firstPhase[0]) >> 9; 543 break; 544 545 default: /* 4 */ 546 firstPhase[0] >>= 5; 547 break; 548 549 } 550 551 Transform(firstPhase); 552 553 pData = data + 256 + comp*64; 554 for (i = 0, pTmp = firstPhase; i < 64;) 555 { 556 tmp = pTmp[(i & 0x7)>>1]; 557 /*lint -e734 CLIP1 macro results in value that fits into 8 bits */ 558 *pData++ = CLIP1(tmp); 559 /*lint +e734 */ 560 561 i++; 562 if (!(i & 0xF)) 563 pTmp += 4; 564 } 565 566 /* increment pointers for cr */ 567 mbPos += width * height * 64; 568 } 569 570 h264bsdWriteMacroblock(currImage, data); 571 572 return(HANTRO_OK); 573 574} 575 576 577/*------------------------------------------------------------------------------ 578 579 Function name: Transform 580 581 Functional description: 582 Simplified transform, assuming that only dc component and lowest 583 horizontal and lowest vertical component may be non-zero 584 585------------------------------------------------------------------------------*/ 586 587void Transform(i32 *data) 588{ 589 590 u32 col; 591 i32 tmp0, tmp1; 592 593 if (!data[1] && !data[4]) 594 { 595 data[1] = data[2] = data[3] = data[4] = data[5] = 596 data[6] = data[7] = data[8] = data[9] = data[10] = 597 data[11] = data[12] = data[13] = data[14] = data[15] = data[0]; 598 return; 599 } 600 /* first horizontal transform for rows 0 and 1 */ 601 tmp0 = data[0]; 602 tmp1 = data[1]; 603 data[0] = tmp0 + tmp1; 604 data[1] = tmp0 + (tmp1>>1); 605 data[2] = tmp0 - (tmp1>>1); 606 data[3] = tmp0 - tmp1; 607 608 tmp0 = data[4]; 609 data[5] = tmp0; 610 data[6] = tmp0; 611 data[7] = tmp0; 612 613 /* then vertical transform */ 614 for (col = 4; col--; data++) 615 { 616 tmp0 = data[0]; 617 tmp1 = data[4]; 618 data[0] = tmp0 + tmp1; 619 data[4] = tmp0 + (tmp1>>1); 620 data[8] = tmp0 - (tmp1>>1); 621 data[12] = tmp0 - tmp1; 622 } 623 624} 625/*lint +e702 */ 626 627