1// Copyright (c) 2012 The Chromium Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4// Note: ported from Chromium commit head: c3bd64c 5 6#include <algorithm> 7#include <limits> 8 9#include "base/bind.h" 10#include "base/bind_helpers.h" 11#include "base/callback_helpers.h" 12#include "base/macros.h" 13#include "base/numerics/safe_conversions.h" 14#include "base/optional.h" 15#include "base/stl_util.h" 16#include "h264_decoder.h" 17 18namespace media { 19 20H264Decoder::H264Accelerator::H264Accelerator() = default; 21 22H264Decoder::H264Accelerator::~H264Accelerator() = default; 23 24H264Decoder::H264Decoder(H264Accelerator* accelerator) 25 : state_(kNeedStreamMetadata), 26 max_frame_num_(0), 27 max_pic_num_(0), 28 max_long_term_frame_idx_(0), 29 max_num_reorder_frames_(0), 30 accelerator_(accelerator) { 31 DCHECK(accelerator_); 32 Reset(); 33} 34 35H264Decoder::~H264Decoder() = default; 36 37void H264Decoder::Reset() { 38 curr_pic_ = nullptr; 39 curr_nalu_ = nullptr; 40 curr_slice_hdr_ = nullptr; 41 curr_sps_id_ = -1; 42 curr_pps_id_ = -1; 43 44 prev_frame_num_ = -1; 45 prev_ref_frame_num_ = -1; 46 prev_frame_num_offset_ = -1; 47 prev_has_memmgmnt5_ = false; 48 49 prev_ref_has_memmgmnt5_ = false; 50 prev_ref_top_field_order_cnt_ = -1; 51 prev_ref_pic_order_cnt_msb_ = -1; 52 prev_ref_pic_order_cnt_lsb_ = -1; 53 prev_ref_field_ = H264Picture::FIELD_NONE; 54 55 ref_pic_list_p0_.clear(); 56 ref_pic_list_b0_.clear(); 57 ref_pic_list_b1_.clear(); 58 dpb_.Clear(); 59 parser_.Reset(); 60 accelerator_->Reset(); 61 last_output_poc_ = std::numeric_limits<int>::min(); 62 63 // If we are in kDecoding, we can resume without processing an SPS. 64 if (state_ == kDecoding) 65 state_ = kAfterReset; 66} 67 68void H264Decoder::PrepareRefPicLists(const H264SliceHeader* slice_hdr) { 69 ConstructReferencePicListsP(slice_hdr); 70 ConstructReferencePicListsB(slice_hdr); 71} 72 73bool H264Decoder::ModifyReferencePicLists(const H264SliceHeader* slice_hdr, 74 H264Picture::Vector* ref_pic_list0, 75 H264Picture::Vector* ref_pic_list1) { 76 ref_pic_list0->clear(); 77 ref_pic_list1->clear(); 78 79 // Fill reference picture lists for B and S/SP slices. 80 if (slice_hdr->IsPSlice() || slice_hdr->IsSPSlice()) { 81 *ref_pic_list0 = ref_pic_list_p0_; 82 return ModifyReferencePicList(slice_hdr, 0, ref_pic_list0); 83 } else if (slice_hdr->IsBSlice()) { 84 *ref_pic_list0 = ref_pic_list_b0_; 85 *ref_pic_list1 = ref_pic_list_b1_; 86 return ModifyReferencePicList(slice_hdr, 0, ref_pic_list0) && 87 ModifyReferencePicList(slice_hdr, 1, ref_pic_list1); 88 } 89 90 return true; 91} 92 93bool H264Decoder::DecodePicture() { 94 DCHECK(curr_pic_.get()); 95 96 DVLOG(4) << "Decoding POC " << curr_pic_->pic_order_cnt; 97 return accelerator_->SubmitDecode(curr_pic_); 98} 99 100bool H264Decoder::InitNonexistingPicture(scoped_refptr<H264Picture> pic, 101 int frame_num) { 102 pic->nonexisting = true; 103 pic->nal_ref_idc = 1; 104 pic->frame_num = pic->pic_num = frame_num; 105 pic->adaptive_ref_pic_marking_mode_flag = false; 106 pic->ref = true; 107 pic->long_term_reference_flag = false; 108 pic->field = H264Picture::FIELD_NONE; 109 110 return CalculatePicOrderCounts(pic); 111} 112 113bool H264Decoder::InitCurrPicture(const H264SliceHeader* slice_hdr) { 114 DCHECK(curr_pic_.get()); 115 116 curr_pic_->idr = slice_hdr->idr_pic_flag; 117 if (curr_pic_->idr) 118 curr_pic_->idr_pic_id = slice_hdr->idr_pic_id; 119 120 if (slice_hdr->field_pic_flag) { 121 curr_pic_->field = slice_hdr->bottom_field_flag ? H264Picture::FIELD_BOTTOM 122 : H264Picture::FIELD_TOP; 123 } else { 124 curr_pic_->field = H264Picture::FIELD_NONE; 125 } 126 127 if (curr_pic_->field != H264Picture::FIELD_NONE) { 128 DVLOG(1) << "Interlaced video not supported."; 129 return false; 130 } 131 132 curr_pic_->nal_ref_idc = slice_hdr->nal_ref_idc; 133 curr_pic_->ref = slice_hdr->nal_ref_idc != 0; 134 // This assumes non-interlaced stream. 135 curr_pic_->frame_num = curr_pic_->pic_num = slice_hdr->frame_num; 136 137 DCHECK_NE(curr_sps_id_, -1); 138 const H264SPS* sps = parser_.GetSPS(curr_sps_id_); 139 if (!sps) 140 return false; 141 142 curr_pic_->pic_order_cnt_type = sps->pic_order_cnt_type; 143 switch (curr_pic_->pic_order_cnt_type) { 144 case 0: 145 curr_pic_->pic_order_cnt_lsb = slice_hdr->pic_order_cnt_lsb; 146 curr_pic_->delta_pic_order_cnt_bottom = 147 slice_hdr->delta_pic_order_cnt_bottom; 148 break; 149 150 case 1: 151 curr_pic_->delta_pic_order_cnt0 = slice_hdr->delta_pic_order_cnt0; 152 curr_pic_->delta_pic_order_cnt1 = slice_hdr->delta_pic_order_cnt1; 153 break; 154 155 case 2: 156 break; 157 158 default: 159 NOTREACHED(); 160 return false; 161 } 162 163 if (!CalculatePicOrderCounts(curr_pic_)) 164 return false; 165 166 curr_pic_->long_term_reference_flag = slice_hdr->long_term_reference_flag; 167 curr_pic_->adaptive_ref_pic_marking_mode_flag = 168 slice_hdr->adaptive_ref_pic_marking_mode_flag; 169 170 // If the slice header indicates we will have to perform reference marking 171 // process after this picture is decoded, store required data for that 172 // purpose. 173 if (slice_hdr->adaptive_ref_pic_marking_mode_flag) { 174 static_assert(sizeof(curr_pic_->ref_pic_marking) == 175 sizeof(slice_hdr->ref_pic_marking), 176 "Array sizes of ref pic marking do not match."); 177 memcpy(curr_pic_->ref_pic_marking, slice_hdr->ref_pic_marking, 178 sizeof(curr_pic_->ref_pic_marking)); 179 } 180 181 curr_pic_->visible_rect = visible_rect_; 182 183 return true; 184} 185 186bool H264Decoder::CalculatePicOrderCounts(scoped_refptr<H264Picture> pic) { 187 const H264SPS* sps = parser_.GetSPS(curr_sps_id_); 188 if (!sps) 189 return false; 190 191 switch (pic->pic_order_cnt_type) { 192 case 0: { 193 // See spec 8.2.1.1. 194 int prev_pic_order_cnt_msb, prev_pic_order_cnt_lsb; 195 196 if (pic->idr) { 197 prev_pic_order_cnt_msb = prev_pic_order_cnt_lsb = 0; 198 } else { 199 if (prev_ref_has_memmgmnt5_) { 200 if (prev_ref_field_ != H264Picture::FIELD_BOTTOM) { 201 prev_pic_order_cnt_msb = 0; 202 prev_pic_order_cnt_lsb = prev_ref_top_field_order_cnt_; 203 } else { 204 prev_pic_order_cnt_msb = 0; 205 prev_pic_order_cnt_lsb = 0; 206 } 207 } else { 208 prev_pic_order_cnt_msb = prev_ref_pic_order_cnt_msb_; 209 prev_pic_order_cnt_lsb = prev_ref_pic_order_cnt_lsb_; 210 } 211 } 212 213 int max_pic_order_cnt_lsb = 214 1 << (sps->log2_max_pic_order_cnt_lsb_minus4 + 4); 215 DCHECK_NE(max_pic_order_cnt_lsb, 0); 216 if ((pic->pic_order_cnt_lsb < prev_pic_order_cnt_lsb) && 217 (prev_pic_order_cnt_lsb - pic->pic_order_cnt_lsb >= 218 max_pic_order_cnt_lsb / 2)) { 219 pic->pic_order_cnt_msb = prev_pic_order_cnt_msb + max_pic_order_cnt_lsb; 220 } else if ((pic->pic_order_cnt_lsb > prev_pic_order_cnt_lsb) && 221 (pic->pic_order_cnt_lsb - prev_pic_order_cnt_lsb > 222 max_pic_order_cnt_lsb / 2)) { 223 pic->pic_order_cnt_msb = prev_pic_order_cnt_msb - max_pic_order_cnt_lsb; 224 } else { 225 pic->pic_order_cnt_msb = prev_pic_order_cnt_msb; 226 } 227 228 if (pic->field != H264Picture::FIELD_BOTTOM) { 229 pic->top_field_order_cnt = 230 pic->pic_order_cnt_msb + pic->pic_order_cnt_lsb; 231 } 232 233 if (pic->field != H264Picture::FIELD_TOP) { 234 if (pic->field == H264Picture::FIELD_NONE) { 235 pic->bottom_field_order_cnt = 236 pic->top_field_order_cnt + pic->delta_pic_order_cnt_bottom; 237 } else { 238 pic->bottom_field_order_cnt = 239 pic->pic_order_cnt_msb + pic->pic_order_cnt_lsb; 240 } 241 } 242 break; 243 } 244 245 case 1: { 246 // See spec 8.2.1.2. 247 if (prev_has_memmgmnt5_) 248 prev_frame_num_offset_ = 0; 249 250 if (pic->idr) 251 pic->frame_num_offset = 0; 252 else if (prev_frame_num_ > pic->frame_num) 253 pic->frame_num_offset = prev_frame_num_offset_ + max_frame_num_; 254 else 255 pic->frame_num_offset = prev_frame_num_offset_; 256 257 int abs_frame_num = 0; 258 if (sps->num_ref_frames_in_pic_order_cnt_cycle != 0) 259 abs_frame_num = pic->frame_num_offset + pic->frame_num; 260 else 261 abs_frame_num = 0; 262 263 if (pic->nal_ref_idc == 0 && abs_frame_num > 0) 264 --abs_frame_num; 265 266 int expected_pic_order_cnt = 0; 267 if (abs_frame_num > 0) { 268 if (sps->num_ref_frames_in_pic_order_cnt_cycle == 0) { 269 DVLOG(1) << "Invalid num_ref_frames_in_pic_order_cnt_cycle " 270 << "in stream"; 271 return false; 272 } 273 274 int pic_order_cnt_cycle_cnt = 275 (abs_frame_num - 1) / sps->num_ref_frames_in_pic_order_cnt_cycle; 276 int frame_num_in_pic_order_cnt_cycle = 277 (abs_frame_num - 1) % sps->num_ref_frames_in_pic_order_cnt_cycle; 278 279 expected_pic_order_cnt = pic_order_cnt_cycle_cnt * 280 sps->expected_delta_per_pic_order_cnt_cycle; 281 // frame_num_in_pic_order_cnt_cycle is verified < 255 in parser 282 for (int i = 0; i <= frame_num_in_pic_order_cnt_cycle; ++i) 283 expected_pic_order_cnt += sps->offset_for_ref_frame[i]; 284 } 285 286 if (!pic->nal_ref_idc) 287 expected_pic_order_cnt += sps->offset_for_non_ref_pic; 288 289 if (pic->field == H264Picture::FIELD_NONE) { 290 pic->top_field_order_cnt = 291 expected_pic_order_cnt + pic->delta_pic_order_cnt0; 292 pic->bottom_field_order_cnt = pic->top_field_order_cnt + 293 sps->offset_for_top_to_bottom_field + 294 pic->delta_pic_order_cnt1; 295 } else if (pic->field != H264Picture::FIELD_BOTTOM) { 296 pic->top_field_order_cnt = 297 expected_pic_order_cnt + pic->delta_pic_order_cnt0; 298 } else { 299 pic->bottom_field_order_cnt = expected_pic_order_cnt + 300 sps->offset_for_top_to_bottom_field + 301 pic->delta_pic_order_cnt0; 302 } 303 break; 304 } 305 306 case 2: { 307 // See spec 8.2.1.3. 308 if (prev_has_memmgmnt5_) 309 prev_frame_num_offset_ = 0; 310 311 if (pic->idr) 312 pic->frame_num_offset = 0; 313 else if (prev_frame_num_ > pic->frame_num) 314 pic->frame_num_offset = prev_frame_num_offset_ + max_frame_num_; 315 else 316 pic->frame_num_offset = prev_frame_num_offset_; 317 318 int temp_pic_order_cnt; 319 if (pic->idr) { 320 temp_pic_order_cnt = 0; 321 } else if (!pic->nal_ref_idc) { 322 temp_pic_order_cnt = 2 * (pic->frame_num_offset + pic->frame_num) - 1; 323 } else { 324 temp_pic_order_cnt = 2 * (pic->frame_num_offset + pic->frame_num); 325 } 326 327 if (pic->field == H264Picture::FIELD_NONE) { 328 pic->top_field_order_cnt = temp_pic_order_cnt; 329 pic->bottom_field_order_cnt = temp_pic_order_cnt; 330 } else if (pic->field == H264Picture::FIELD_BOTTOM) { 331 pic->bottom_field_order_cnt = temp_pic_order_cnt; 332 } else { 333 pic->top_field_order_cnt = temp_pic_order_cnt; 334 } 335 break; 336 } 337 338 default: 339 DVLOG(1) << "Invalid pic_order_cnt_type: " << sps->pic_order_cnt_type; 340 return false; 341 } 342 343 switch (pic->field) { 344 case H264Picture::FIELD_NONE: 345 pic->pic_order_cnt = 346 std::min(pic->top_field_order_cnt, pic->bottom_field_order_cnt); 347 break; 348 case H264Picture::FIELD_TOP: 349 pic->pic_order_cnt = pic->top_field_order_cnt; 350 break; 351 case H264Picture::FIELD_BOTTOM: 352 pic->pic_order_cnt = pic->bottom_field_order_cnt; 353 break; 354 } 355 356 return true; 357} 358 359void H264Decoder::UpdatePicNums(int frame_num) { 360 for (auto& pic : dpb_) { 361 if (!pic->ref) 362 continue; 363 364 // 8.2.4.1. Assumes non-interlaced stream. 365 DCHECK_EQ(pic->field, H264Picture::FIELD_NONE); 366 if (pic->long_term) { 367 pic->long_term_pic_num = pic->long_term_frame_idx; 368 } else { 369 if (pic->frame_num > frame_num) 370 pic->frame_num_wrap = pic->frame_num - max_frame_num_; 371 else 372 pic->frame_num_wrap = pic->frame_num; 373 374 pic->pic_num = pic->frame_num_wrap; 375 } 376 } 377} 378 379struct PicNumDescCompare { 380 bool operator()(const scoped_refptr<H264Picture>& a, 381 const scoped_refptr<H264Picture>& b) const { 382 return a->pic_num > b->pic_num; 383 } 384}; 385 386struct LongTermPicNumAscCompare { 387 bool operator()(const scoped_refptr<H264Picture>& a, 388 const scoped_refptr<H264Picture>& b) const { 389 return a->long_term_pic_num < b->long_term_pic_num; 390 } 391}; 392 393void H264Decoder::ConstructReferencePicListsP( 394 const H264SliceHeader* slice_hdr) { 395 // RefPicList0 (8.2.4.2.1) [[1] [2]], where: 396 // [1] shortterm ref pics sorted by descending pic_num, 397 // [2] longterm ref pics by ascending long_term_pic_num. 398 ref_pic_list_p0_.clear(); 399 400 // First get the short ref pics... 401 dpb_.GetShortTermRefPicsAppending(&ref_pic_list_p0_); 402 size_t num_short_refs = ref_pic_list_p0_.size(); 403 404 // and sort them to get [1]. 405 std::sort(ref_pic_list_p0_.begin(), ref_pic_list_p0_.end(), 406 PicNumDescCompare()); 407 408 // Now get long term pics and sort them by long_term_pic_num to get [2]. 409 dpb_.GetLongTermRefPicsAppending(&ref_pic_list_p0_); 410 std::sort(ref_pic_list_p0_.begin() + num_short_refs, ref_pic_list_p0_.end(), 411 LongTermPicNumAscCompare()); 412} 413 414struct POCAscCompare { 415 bool operator()(const scoped_refptr<H264Picture>& a, 416 const scoped_refptr<H264Picture>& b) const { 417 return a->pic_order_cnt < b->pic_order_cnt; 418 } 419}; 420 421struct POCDescCompare { 422 bool operator()(const scoped_refptr<H264Picture>& a, 423 const scoped_refptr<H264Picture>& b) const { 424 return a->pic_order_cnt > b->pic_order_cnt; 425 } 426}; 427 428void H264Decoder::ConstructReferencePicListsB( 429 const H264SliceHeader* slice_hdr) { 430 // RefPicList0 (8.2.4.2.3) [[1] [2] [3]], where: 431 // [1] shortterm ref pics with POC < curr_pic's POC sorted by descending POC, 432 // [2] shortterm ref pics with POC > curr_pic's POC by ascending POC, 433 // [3] longterm ref pics by ascending long_term_pic_num. 434 ref_pic_list_b0_.clear(); 435 ref_pic_list_b1_.clear(); 436 dpb_.GetShortTermRefPicsAppending(&ref_pic_list_b0_); 437 size_t num_short_refs = ref_pic_list_b0_.size(); 438 439 // First sort ascending, this will put [1] in right place and finish [2]. 440 std::sort(ref_pic_list_b0_.begin(), ref_pic_list_b0_.end(), POCAscCompare()); 441 442 // Find first with POC > curr_pic's POC to get first element in [2]... 443 H264Picture::Vector::iterator iter; 444 iter = std::upper_bound(ref_pic_list_b0_.begin(), ref_pic_list_b0_.end(), 445 curr_pic_.get(), POCAscCompare()); 446 447 // and sort [1] descending, thus finishing sequence [1] [2]. 448 std::sort(ref_pic_list_b0_.begin(), iter, POCDescCompare()); 449 450 // Now add [3] and sort by ascending long_term_pic_num. 451 dpb_.GetLongTermRefPicsAppending(&ref_pic_list_b0_); 452 std::sort(ref_pic_list_b0_.begin() + num_short_refs, ref_pic_list_b0_.end(), 453 LongTermPicNumAscCompare()); 454 455 // RefPicList1 (8.2.4.2.4) [[1] [2] [3]], where: 456 // [1] shortterm ref pics with POC > curr_pic's POC sorted by ascending POC, 457 // [2] shortterm ref pics with POC < curr_pic's POC by descending POC, 458 // [3] longterm ref pics by ascending long_term_pic_num. 459 460 dpb_.GetShortTermRefPicsAppending(&ref_pic_list_b1_); 461 num_short_refs = ref_pic_list_b1_.size(); 462 463 // First sort by descending POC. 464 std::sort(ref_pic_list_b1_.begin(), ref_pic_list_b1_.end(), POCDescCompare()); 465 466 // Find first with POC < curr_pic's POC to get first element in [2]... 467 iter = std::upper_bound(ref_pic_list_b1_.begin(), ref_pic_list_b1_.end(), 468 curr_pic_.get(), POCDescCompare()); 469 470 // and sort [1] ascending. 471 std::sort(ref_pic_list_b1_.begin(), iter, POCAscCompare()); 472 473 // Now add [3] and sort by ascending long_term_pic_num 474 dpb_.GetShortTermRefPicsAppending(&ref_pic_list_b1_); 475 std::sort(ref_pic_list_b1_.begin() + num_short_refs, ref_pic_list_b1_.end(), 476 LongTermPicNumAscCompare()); 477 478 // If lists identical, swap first two entries in RefPicList1 (spec 8.2.4.2.3) 479 if (ref_pic_list_b1_.size() > 1 && 480 std::equal(ref_pic_list_b0_.begin(), ref_pic_list_b0_.end(), 481 ref_pic_list_b1_.begin())) 482 std::swap(ref_pic_list_b1_[0], ref_pic_list_b1_[1]); 483} 484 485// See 8.2.4 486int H264Decoder::PicNumF(const scoped_refptr<H264Picture>& pic) { 487 if (!pic) 488 return -1; 489 490 if (!pic->long_term) 491 return pic->pic_num; 492 else 493 return max_pic_num_; 494} 495 496// See 8.2.4 497int H264Decoder::LongTermPicNumF(const scoped_refptr<H264Picture>& pic) { 498 if (pic->ref && pic->long_term) 499 return pic->long_term_pic_num; 500 else 501 return 2 * (max_long_term_frame_idx_ + 1); 502} 503 504// Shift elements on the |v| starting from |from| to |to|, inclusive, 505// one position to the right and insert pic at |from|. 506static void ShiftRightAndInsert(H264Picture::Vector* v, 507 int from, 508 int to, 509 const scoped_refptr<H264Picture>& pic) { 510 // Security checks, do not disable in Debug mode. 511 CHECK(from <= to); 512 CHECK(to <= std::numeric_limits<int>::max() - 2); 513 // Additional checks. Debug mode ok. 514 DCHECK(v); 515 DCHECK(pic); 516 DCHECK((to + 1 == static_cast<int>(v->size())) || 517 (to + 2 == static_cast<int>(v->size()))); 518 519 v->resize(to + 2); 520 521 for (int i = to + 1; i > from; --i) 522 (*v)[i] = (*v)[i - 1]; 523 524 (*v)[from] = pic; 525} 526 527bool H264Decoder::ModifyReferencePicList(const H264SliceHeader* slice_hdr, 528 int list, 529 H264Picture::Vector* ref_pic_listx) { 530 bool ref_pic_list_modification_flag_lX; 531 int num_ref_idx_lX_active_minus1; 532 const H264ModificationOfPicNum* list_mod; 533 534 // This can process either ref_pic_list0 or ref_pic_list1, depending on 535 // the list argument. Set up pointers to proper list to be processed here. 536 if (list == 0) { 537 ref_pic_list_modification_flag_lX = 538 slice_hdr->ref_pic_list_modification_flag_l0; 539 num_ref_idx_lX_active_minus1 = slice_hdr->num_ref_idx_l0_active_minus1; 540 list_mod = slice_hdr->ref_list_l0_modifications; 541 } else { 542 ref_pic_list_modification_flag_lX = 543 slice_hdr->ref_pic_list_modification_flag_l1; 544 num_ref_idx_lX_active_minus1 = slice_hdr->num_ref_idx_l1_active_minus1; 545 list_mod = slice_hdr->ref_list_l1_modifications; 546 } 547 548 // Resize the list to the size requested in the slice header. 549 // Note that per 8.2.4.2 it's possible for num_ref_idx_lX_active_minus1 to 550 // indicate there should be more ref pics on list than we constructed. 551 // Those superfluous ones should be treated as non-reference and will be 552 // initialized to nullptr, which must be handled by clients. 553 DCHECK_GE(num_ref_idx_lX_active_minus1, 0); 554 ref_pic_listx->resize(num_ref_idx_lX_active_minus1 + 1); 555 556 if (!ref_pic_list_modification_flag_lX) 557 return true; 558 559 // Spec 8.2.4.3: 560 // Reorder pictures on the list in a way specified in the stream. 561 int pic_num_lx_pred = curr_pic_->pic_num; 562 int ref_idx_lx = 0; 563 int pic_num_lx_no_wrap; 564 int pic_num_lx; 565 bool done = false; 566 scoped_refptr<H264Picture> pic; 567 for (int i = 0; i < H264SliceHeader::kRefListModSize && !done; ++i) { 568 switch (list_mod->modification_of_pic_nums_idc) { 569 case 0: 570 case 1: 571 // Modify short reference picture position. 572 if (list_mod->modification_of_pic_nums_idc == 0) { 573 // Subtract given value from predicted PicNum. 574 pic_num_lx_no_wrap = 575 pic_num_lx_pred - 576 (static_cast<int>(list_mod->abs_diff_pic_num_minus1) + 1); 577 // Wrap around max_pic_num_ if it becomes < 0 as result 578 // of subtraction. 579 if (pic_num_lx_no_wrap < 0) 580 pic_num_lx_no_wrap += max_pic_num_; 581 } else { 582 // Add given value to predicted PicNum. 583 pic_num_lx_no_wrap = 584 pic_num_lx_pred + 585 (static_cast<int>(list_mod->abs_diff_pic_num_minus1) + 1); 586 // Wrap around max_pic_num_ if it becomes >= max_pic_num_ as result 587 // of the addition. 588 if (pic_num_lx_no_wrap >= max_pic_num_) 589 pic_num_lx_no_wrap -= max_pic_num_; 590 } 591 592 // For use in next iteration. 593 pic_num_lx_pred = pic_num_lx_no_wrap; 594 595 if (pic_num_lx_no_wrap > curr_pic_->pic_num) 596 pic_num_lx = pic_num_lx_no_wrap - max_pic_num_; 597 else 598 pic_num_lx = pic_num_lx_no_wrap; 599 600 DCHECK_LT(num_ref_idx_lX_active_minus1 + 1, 601 H264SliceHeader::kRefListModSize); 602 pic = dpb_.GetShortRefPicByPicNum(pic_num_lx); 603 if (!pic) { 604 DVLOG(1) << "Malformed stream, no pic num " << pic_num_lx; 605 return false; 606 } 607 ShiftRightAndInsert(ref_pic_listx, ref_idx_lx, 608 num_ref_idx_lX_active_minus1, pic); 609 ref_idx_lx++; 610 611 for (int src = ref_idx_lx, dst = ref_idx_lx; 612 src <= num_ref_idx_lX_active_minus1 + 1; ++src) { 613 if (PicNumF((*ref_pic_listx)[src]) != pic_num_lx) 614 (*ref_pic_listx)[dst++] = (*ref_pic_listx)[src]; 615 } 616 break; 617 618 case 2: 619 // Modify long term reference picture position. 620 DCHECK_LT(num_ref_idx_lX_active_minus1 + 1, 621 H264SliceHeader::kRefListModSize); 622 pic = dpb_.GetLongRefPicByLongTermPicNum(list_mod->long_term_pic_num); 623 if (!pic) { 624 DVLOG(1) << "Malformed stream, no pic num " 625 << list_mod->long_term_pic_num; 626 return false; 627 } 628 ShiftRightAndInsert(ref_pic_listx, ref_idx_lx, 629 num_ref_idx_lX_active_minus1, pic); 630 ref_idx_lx++; 631 632 for (int src = ref_idx_lx, dst = ref_idx_lx; 633 src <= num_ref_idx_lX_active_minus1 + 1; ++src) { 634 if (LongTermPicNumF((*ref_pic_listx)[src]) != 635 static_cast<int>(list_mod->long_term_pic_num)) 636 (*ref_pic_listx)[dst++] = (*ref_pic_listx)[src]; 637 } 638 break; 639 640 case 3: 641 // End of modification list. 642 done = true; 643 break; 644 645 default: 646 // May be recoverable. 647 DVLOG(1) << "Invalid modification_of_pic_nums_idc=" 648 << list_mod->modification_of_pic_nums_idc 649 << " in position " << i; 650 break; 651 } 652 653 ++list_mod; 654 } 655 656 // Per NOTE 2 in 8.2.4.3.2, the ref_pic_listx size in the above loop is 657 // temporarily made one element longer than the required final list. 658 // Resize the list back to its required size. 659 ref_pic_listx->resize(num_ref_idx_lX_active_minus1 + 1); 660 661 return true; 662} 663 664void H264Decoder::OutputPic(scoped_refptr<H264Picture> pic) { 665 DCHECK(!pic->outputted); 666 pic->outputted = true; 667 668 if (pic->nonexisting) { 669 DVLOG(4) << "Skipping output, non-existing frame_num: " << pic->frame_num; 670 return; 671 } 672 673 DVLOG_IF(1, pic->pic_order_cnt < last_output_poc_) 674 << "Outputting out of order, likely a broken stream: " 675 << last_output_poc_ << " -> " << pic->pic_order_cnt; 676 last_output_poc_ = pic->pic_order_cnt; 677 678 DVLOG(4) << "Posting output task for POC: " << pic->pic_order_cnt; 679 accelerator_->OutputPicture(pic); 680} 681 682void H264Decoder::ClearDPB() { 683 // Clear DPB contents, marking the pictures as unused first. 684 dpb_.Clear(); 685 last_output_poc_ = std::numeric_limits<int>::min(); 686} 687 688bool H264Decoder::OutputAllRemainingPics() { 689 // Output all pictures that are waiting to be outputted. 690 FinishPrevFrameIfPresent(); 691 H264Picture::Vector to_output; 692 dpb_.GetNotOutputtedPicsAppending(&to_output); 693 // Sort them by ascending POC to output in order. 694 std::sort(to_output.begin(), to_output.end(), POCAscCompare()); 695 696 for (auto& pic : to_output) 697 OutputPic(pic); 698 699 return true; 700} 701 702bool H264Decoder::Flush() { 703 DVLOG(2) << "Decoder flush"; 704 705 if (!OutputAllRemainingPics()) 706 return false; 707 708 ClearDPB(); 709 DVLOG(2) << "Decoder flush finished"; 710 return true; 711} 712 713bool H264Decoder::StartNewFrame(const H264SliceHeader* slice_hdr) { 714 // TODO posciak: add handling of max_num_ref_frames per spec. 715 CHECK(curr_pic_.get()); 716 DCHECK(slice_hdr); 717 718 curr_pps_id_ = slice_hdr->pic_parameter_set_id; 719 const H264PPS* pps = parser_.GetPPS(curr_pps_id_); 720 if (!pps) 721 return false; 722 723 curr_sps_id_ = pps->seq_parameter_set_id; 724 const H264SPS* sps = parser_.GetSPS(curr_sps_id_); 725 if (!sps) 726 return false; 727 728 max_frame_num_ = 1 << (sps->log2_max_frame_num_minus4 + 4); 729 int frame_num = slice_hdr->frame_num; 730 if (slice_hdr->idr_pic_flag) 731 prev_ref_frame_num_ = 0; 732 733 // 7.4.3 734 if (frame_num != prev_ref_frame_num_ && 735 frame_num != (prev_ref_frame_num_ + 1) % max_frame_num_) { 736 if (!HandleFrameNumGap(frame_num)) 737 return false; 738 } 739 740 if (!InitCurrPicture(slice_hdr)) 741 return false; 742 743 UpdatePicNums(frame_num); 744 PrepareRefPicLists(slice_hdr); 745 746 if (!accelerator_->SubmitFrameMetadata(sps, pps, dpb_, ref_pic_list_p0_, 747 ref_pic_list_b0_, ref_pic_list_b1_, 748 curr_pic_.get())) 749 return false; 750 751 return true; 752} 753 754bool H264Decoder::HandleMemoryManagementOps(scoped_refptr<H264Picture> pic) { 755 // 8.2.5.4 756 for (size_t i = 0; i < arraysize(pic->ref_pic_marking); ++i) { 757 // Code below does not support interlaced stream (per-field pictures). 758 H264DecRefPicMarking* ref_pic_marking = &pic->ref_pic_marking[i]; 759 scoped_refptr<H264Picture> to_mark; 760 int pic_num_x; 761 762 switch (ref_pic_marking->memory_mgmnt_control_operation) { 763 case 0: 764 // Normal end of operations' specification. 765 return true; 766 767 case 1: 768 // Mark a short term reference picture as unused so it can be removed 769 // if outputted. 770 pic_num_x = 771 pic->pic_num - (ref_pic_marking->difference_of_pic_nums_minus1 + 1); 772 to_mark = dpb_.GetShortRefPicByPicNum(pic_num_x); 773 if (to_mark) { 774 to_mark->ref = false; 775 } else { 776 DVLOG(1) << "Invalid short ref pic num to unmark"; 777 return false; 778 } 779 break; 780 781 case 2: 782 // Mark a long term reference picture as unused so it can be removed 783 // if outputted. 784 to_mark = dpb_.GetLongRefPicByLongTermPicNum( 785 ref_pic_marking->long_term_pic_num); 786 if (to_mark) { 787 to_mark->ref = false; 788 } else { 789 DVLOG(1) << "Invalid long term ref pic num to unmark"; 790 return false; 791 } 792 break; 793 794 case 3: 795 // Mark a short term reference picture as long term reference. 796 pic_num_x = 797 pic->pic_num - (ref_pic_marking->difference_of_pic_nums_minus1 + 1); 798 to_mark = dpb_.GetShortRefPicByPicNum(pic_num_x); 799 if (to_mark) { 800 DCHECK(to_mark->ref && !to_mark->long_term); 801 to_mark->long_term = true; 802 to_mark->long_term_frame_idx = ref_pic_marking->long_term_frame_idx; 803 } else { 804 DVLOG(1) << "Invalid short term ref pic num to mark as long ref"; 805 return false; 806 } 807 break; 808 809 case 4: { 810 // Unmark all reference pictures with long_term_frame_idx over new max. 811 max_long_term_frame_idx_ = 812 ref_pic_marking->max_long_term_frame_idx_plus1 - 1; 813 H264Picture::Vector long_terms; 814 dpb_.GetLongTermRefPicsAppending(&long_terms); 815 for (size_t i = 0; i < long_terms.size(); ++i) { 816 scoped_refptr<H264Picture>& long_term_pic = long_terms[i]; 817 DCHECK(long_term_pic->ref && long_term_pic->long_term); 818 // Ok to cast, max_long_term_frame_idx is much smaller than 16bit. 819 if (long_term_pic->long_term_frame_idx > 820 static_cast<int>(max_long_term_frame_idx_)) 821 long_term_pic->ref = false; 822 } 823 break; 824 } 825 826 case 5: 827 // Unmark all reference pictures. 828 dpb_.MarkAllUnusedForRef(); 829 max_long_term_frame_idx_ = -1; 830 pic->mem_mgmt_5 = true; 831 break; 832 833 case 6: { 834 // Replace long term reference pictures with current picture. 835 // First unmark if any existing with this long_term_frame_idx... 836 H264Picture::Vector long_terms; 837 dpb_.GetLongTermRefPicsAppending(&long_terms); 838 for (size_t i = 0; i < long_terms.size(); ++i) { 839 scoped_refptr<H264Picture>& long_term_pic = long_terms[i]; 840 DCHECK(long_term_pic->ref && long_term_pic->long_term); 841 // Ok to cast, long_term_frame_idx is much smaller than 16bit. 842 if (long_term_pic->long_term_frame_idx == 843 static_cast<int>(ref_pic_marking->long_term_frame_idx)) 844 long_term_pic->ref = false; 845 } 846 847 // and mark the current one instead. 848 pic->ref = true; 849 pic->long_term = true; 850 pic->long_term_frame_idx = ref_pic_marking->long_term_frame_idx; 851 break; 852 } 853 854 default: 855 // Would indicate a bug in parser. 856 NOTREACHED(); 857 } 858 } 859 860 return true; 861} 862 863// This method ensures that DPB does not overflow, either by removing 864// reference pictures as specified in the stream, or using a sliding window 865// procedure to remove the oldest one. 866// It also performs marking and unmarking pictures as reference. 867// See spac 8.2.5.1. 868bool H264Decoder::ReferencePictureMarking(scoped_refptr<H264Picture> pic) { 869 // If the current picture is an IDR, all reference pictures are unmarked. 870 if (pic->idr) { 871 dpb_.MarkAllUnusedForRef(); 872 873 if (pic->long_term_reference_flag) { 874 pic->long_term = true; 875 pic->long_term_frame_idx = 0; 876 max_long_term_frame_idx_ = 0; 877 } else { 878 pic->long_term = false; 879 max_long_term_frame_idx_ = -1; 880 } 881 882 return true; 883 } 884 885 // Not an IDR. If the stream contains instructions on how to discard pictures 886 // from DPB and how to mark/unmark existing reference pictures, do so. 887 // Otherwise, fall back to default sliding window process. 888 if (pic->adaptive_ref_pic_marking_mode_flag) { 889 DCHECK(!pic->nonexisting); 890 return HandleMemoryManagementOps(pic); 891 } else { 892 return SlidingWindowPictureMarking(); 893 } 894} 895 896bool H264Decoder::SlidingWindowPictureMarking() { 897 const H264SPS* sps = parser_.GetSPS(curr_sps_id_); 898 if (!sps) 899 return false; 900 901 // 8.2.5.3. Ensure the DPB doesn't overflow by discarding the oldest picture. 902 int num_ref_pics = dpb_.CountRefPics(); 903 DCHECK_LE(num_ref_pics, std::max<int>(sps->max_num_ref_frames, 1)); 904 if (num_ref_pics == std::max<int>(sps->max_num_ref_frames, 1)) { 905 // Max number of reference pics reached, need to remove one of the short 906 // term ones. Find smallest frame_num_wrap short reference picture and mark 907 // it as unused. 908 scoped_refptr<H264Picture> to_unmark = 909 dpb_.GetLowestFrameNumWrapShortRefPic(); 910 if (!to_unmark) { 911 DVLOG(1) << "Couldn't find a short ref picture to unmark"; 912 return false; 913 } 914 915 to_unmark->ref = false; 916 } 917 918 return true; 919} 920 921bool H264Decoder::FinishPicture(scoped_refptr<H264Picture> pic) { 922 // Finish processing the picture. 923 // Start by storing previous picture data for later use. 924 if (pic->ref) { 925 ReferencePictureMarking(pic); 926 prev_ref_has_memmgmnt5_ = pic->mem_mgmt_5; 927 prev_ref_top_field_order_cnt_ = pic->top_field_order_cnt; 928 prev_ref_pic_order_cnt_msb_ = pic->pic_order_cnt_msb; 929 prev_ref_pic_order_cnt_lsb_ = pic->pic_order_cnt_lsb; 930 prev_ref_field_ = pic->field; 931 prev_ref_frame_num_ = pic->frame_num; 932 } 933 prev_frame_num_ = pic->frame_num; 934 prev_has_memmgmnt5_ = pic->mem_mgmt_5; 935 prev_frame_num_offset_ = pic->frame_num_offset; 936 937 // Remove unused (for reference or later output) pictures from DPB, marking 938 // them as such. 939 dpb_.DeleteUnused(); 940 941 DVLOG(4) << "Finishing picture frame_num: " << pic->frame_num 942 << ", entries in DPB: " << dpb_.size(); 943 944 // The ownership of pic will either be transferred to DPB - if the picture is 945 // still needed (for output and/or reference) - or we will release it 946 // immediately if we manage to output it here and won't have to store it for 947 // future reference. 948 949 // Get all pictures that haven't been outputted yet. 950 H264Picture::Vector not_outputted; 951 dpb_.GetNotOutputtedPicsAppending(¬_outputted); 952 // Include the one we've just decoded. 953 not_outputted.push_back(pic); 954 955 // Sort in output order. 956 std::sort(not_outputted.begin(), not_outputted.end(), POCAscCompare()); 957 958 // Try to output as many pictures as we can. A picture can be output, 959 // if the number of decoded and not yet outputted pictures that would remain 960 // in DPB afterwards would at least be equal to max_num_reorder_frames. 961 // If the outputted picture is not a reference picture, it doesn't have 962 // to remain in the DPB and can be removed. 963 H264Picture::Vector::iterator output_candidate = not_outputted.begin(); 964 size_t num_remaining = not_outputted.size(); 965 while (num_remaining > max_num_reorder_frames_ || 966 // If the condition below is used, this is an invalid stream. We should 967 // not be forced to output beyond max_num_reorder_frames in order to 968 // make room in DPB to store the current picture (if we need to do so). 969 // However, if this happens, ignore max_num_reorder_frames and try 970 // to output more. This may cause out-of-order output, but is not 971 // fatal, and better than failing instead. 972 ((dpb_.IsFull() && (!pic->outputted || pic->ref)) && num_remaining)) { 973 DVLOG_IF(1, num_remaining <= max_num_reorder_frames_) 974 << "Invalid stream: max_num_reorder_frames not preserved"; 975 976 OutputPic(*output_candidate); 977 978 if (!(*output_candidate)->ref) { 979 // Current picture hasn't been inserted into DPB yet, so don't remove it 980 // if we managed to output it immediately. 981 int outputted_poc = (*output_candidate)->pic_order_cnt; 982 if (outputted_poc != pic->pic_order_cnt) 983 dpb_.DeleteByPOC(outputted_poc); 984 } 985 986 ++output_candidate; 987 --num_remaining; 988 } 989 990 // If we haven't managed to output the picture that we just decoded, or if 991 // it's a reference picture, we have to store it in DPB. 992 if (!pic->outputted || pic->ref) { 993 if (dpb_.IsFull()) { 994 // If we haven't managed to output anything to free up space in DPB 995 // to store this picture, it's an error in the stream. 996 DVLOG(1) << "Could not free up space in DPB!"; 997 return false; 998 } 999 1000 dpb_.StorePic(pic); 1001 } 1002 1003 return true; 1004} 1005 1006static int LevelToMaxDpbMbs(int level) { 1007 // See table A-1 in spec. 1008 switch (level) { 1009 case 10: 1010 return 396; 1011 case 11: 1012 return 900; 1013 case 12: // fallthrough 1014 case 13: // fallthrough 1015 case 20: 1016 return 2376; 1017 case 21: 1018 return 4752; 1019 case 22: // fallthrough 1020 case 30: 1021 return 8100; 1022 case 31: 1023 return 18000; 1024 case 32: 1025 return 20480; 1026 case 40: // fallthrough 1027 case 41: 1028 return 32768; 1029 case 42: 1030 return 34816; 1031 case 50: 1032 return 110400; 1033 case 51: // fallthrough 1034 case 52: 1035 return 184320; 1036 default: 1037 DVLOG(1) << "Invalid codec level (" << level << ")"; 1038 return 0; 1039 } 1040} 1041 1042bool H264Decoder::UpdateMaxNumReorderFrames(const H264SPS* sps) { 1043 if (sps->vui_parameters_present_flag && sps->bitstream_restriction_flag) { 1044 max_num_reorder_frames_ = 1045 base::checked_cast<size_t>(sps->max_num_reorder_frames); 1046 if (max_num_reorder_frames_ > dpb_.max_num_pics()) { 1047 DVLOG(1) 1048 << "max_num_reorder_frames present, but larger than MaxDpbFrames (" 1049 << max_num_reorder_frames_ << " > " << dpb_.max_num_pics() << ")"; 1050 max_num_reorder_frames_ = 0; 1051 return false; 1052 } 1053 return true; 1054 } 1055 1056 // max_num_reorder_frames not present, infer from profile/constraints 1057 // (see VUI semantics in spec). 1058 if (sps->constraint_set3_flag) { 1059 switch (sps->profile_idc) { 1060 case 44: 1061 case 86: 1062 case 100: 1063 case 110: 1064 case 122: 1065 case 244: 1066 max_num_reorder_frames_ = 0; 1067 break; 1068 default: 1069 max_num_reorder_frames_ = dpb_.max_num_pics(); 1070 break; 1071 } 1072 } else { 1073 max_num_reorder_frames_ = dpb_.max_num_pics(); 1074 } 1075 1076 return true; 1077} 1078 1079bool H264Decoder::ProcessSPS(int sps_id, bool* need_new_buffers) { 1080 DVLOG(4) << "Processing SPS id:" << sps_id; 1081 1082 const H264SPS* sps = parser_.GetSPS(sps_id); 1083 if (!sps) 1084 return false; 1085 1086 *need_new_buffers = false; 1087 1088 if (sps->frame_mbs_only_flag == 0) { 1089 DVLOG(1) << "frame_mbs_only_flag != 1 not supported"; 1090 return false; 1091 } 1092 1093 Size new_pic_size = sps->GetCodedSize().value_or(Size()); 1094 if (new_pic_size.IsEmpty()) { 1095 DVLOG(1) << "Invalid picture size"; 1096 return false; 1097 } 1098 1099 int width_mb = new_pic_size.width() / 16; 1100 int height_mb = new_pic_size.height() / 16; 1101 1102 // Verify that the values are not too large before multiplying. 1103 if (std::numeric_limits<int>::max() / width_mb < height_mb) { 1104 DVLOG(1) << "Picture size is too big: " << new_pic_size.ToString(); 1105 return false; 1106 } 1107 1108 int level = sps->level_idc; 1109 int max_dpb_mbs = LevelToMaxDpbMbs(level); 1110 if (max_dpb_mbs == 0) 1111 return false; 1112 1113 // MaxDpbFrames from level limits per spec. 1114 size_t max_dpb_frames = std::min(max_dpb_mbs / (width_mb * height_mb), 1115 static_cast<int>(H264DPB::kDPBMaxSize)); 1116 DVLOG(1) << "MaxDpbFrames: " << max_dpb_frames 1117 << ", max_num_ref_frames: " << sps->max_num_ref_frames 1118 << ", max_dec_frame_buffering: " << sps->max_dec_frame_buffering; 1119 1120 // Set DPB size to at least the level limit, or what the stream requires. 1121 size_t max_dpb_size = 1122 std::max(static_cast<int>(max_dpb_frames), 1123 std::max(sps->max_num_ref_frames, sps->max_dec_frame_buffering)); 1124 // Some non-conforming streams specify more frames are needed than the current 1125 // level limit. Allow this, but only up to the maximum number of reference 1126 // frames allowed per spec. 1127 DVLOG_IF(1, max_dpb_size > max_dpb_frames) 1128 << "Invalid stream, DPB size > MaxDpbFrames"; 1129 if (max_dpb_size == 0 || max_dpb_size > H264DPB::kDPBMaxSize) { 1130 DVLOG(1) << "Invalid DPB size: " << max_dpb_size; 1131 return false; 1132 } 1133 1134 if ((pic_size_ != new_pic_size) || (dpb_.max_num_pics() != max_dpb_size)) { 1135 if (!Flush()) 1136 return false; 1137 DVLOG(1) << "Codec level: " << level << ", DPB size: " << max_dpb_size 1138 << ", Picture size: " << new_pic_size.ToString(); 1139 *need_new_buffers = true; 1140 pic_size_ = new_pic_size; 1141 dpb_.set_max_num_pics(max_dpb_size); 1142 } 1143 1144 Rect new_visible_rect = sps->GetVisibleRect().value_or(Rect()); 1145 if (visible_rect_ != new_visible_rect) { 1146 DVLOG(2) << "New visible rect: " << new_visible_rect.ToString(); 1147 visible_rect_ = new_visible_rect; 1148 } 1149 1150 if (!UpdateMaxNumReorderFrames(sps)) 1151 return false; 1152 DVLOG(1) << "max_num_reorder_frames: " << max_num_reorder_frames_; 1153 1154 return true; 1155} 1156 1157bool H264Decoder::FinishPrevFrameIfPresent() { 1158 // If we already have a frame waiting to be decoded, decode it and finish. 1159 if (curr_pic_) { 1160 if (!DecodePicture()) 1161 return false; 1162 1163 scoped_refptr<H264Picture> pic = curr_pic_; 1164 curr_pic_ = nullptr; 1165 return FinishPicture(pic); 1166 } 1167 1168 return true; 1169} 1170 1171bool H264Decoder::HandleFrameNumGap(int frame_num) { 1172 const H264SPS* sps = parser_.GetSPS(curr_sps_id_); 1173 if (!sps) 1174 return false; 1175 1176 if (!sps->gaps_in_frame_num_value_allowed_flag) { 1177 DVLOG(1) << "Invalid frame_num: " << frame_num; 1178 return false; 1179 } 1180 1181 DVLOG(2) << "Handling frame_num gap: " << prev_ref_frame_num_ << "->" 1182 << frame_num; 1183 1184 // 7.4.3/7-23 1185 int unused_short_term_frame_num = (prev_ref_frame_num_ + 1) % max_frame_num_; 1186 while (unused_short_term_frame_num != frame_num) { 1187 scoped_refptr<H264Picture> pic = new H264Picture(); 1188 if (!InitNonexistingPicture(pic, unused_short_term_frame_num)) 1189 return false; 1190 1191 UpdatePicNums(unused_short_term_frame_num); 1192 1193 if (!FinishPicture(pic)) 1194 return false; 1195 1196 unused_short_term_frame_num++; 1197 unused_short_term_frame_num %= max_frame_num_; 1198 } 1199 1200 return true; 1201} 1202 1203bool H264Decoder::IsNewPrimaryCodedPicture( 1204 const H264SliceHeader* slice_hdr) const { 1205 if (!curr_pic_) 1206 return true; 1207 1208 // 7.4.1.2.4, assumes non-interlaced. 1209 if (slice_hdr->frame_num != curr_pic_->frame_num || 1210 slice_hdr->pic_parameter_set_id != curr_pps_id_ || 1211 slice_hdr->nal_ref_idc != curr_pic_->nal_ref_idc || 1212 slice_hdr->idr_pic_flag != curr_pic_->idr || 1213 (slice_hdr->idr_pic_flag && 1214 (slice_hdr->idr_pic_id != curr_pic_->idr_pic_id || 1215 // If we have two consecutive IDR slices, and the second one has 1216 // first_mb_in_slice == 0, treat it as a new picture. 1217 // Per spec, idr_pic_id should not be equal in this case (and we should 1218 // have hit the condition above instead, see spec 7.4.3 on idr_pic_id), 1219 // but some encoders neglect changing idr_pic_id for two consecutive 1220 // IDRs. Work around this by checking if the next slice contains the 1221 // zeroth macroblock, i.e. data that belongs to the next picture. 1222 slice_hdr->first_mb_in_slice == 0))) 1223 return true; 1224 1225 const H264SPS* sps = parser_.GetSPS(curr_sps_id_); 1226 if (!sps) 1227 return false; 1228 1229 if (sps->pic_order_cnt_type == curr_pic_->pic_order_cnt_type) { 1230 if (curr_pic_->pic_order_cnt_type == 0) { 1231 if (slice_hdr->pic_order_cnt_lsb != curr_pic_->pic_order_cnt_lsb || 1232 slice_hdr->delta_pic_order_cnt_bottom != 1233 curr_pic_->delta_pic_order_cnt_bottom) 1234 return true; 1235 } else if (curr_pic_->pic_order_cnt_type == 1) { 1236 if (slice_hdr->delta_pic_order_cnt0 != curr_pic_->delta_pic_order_cnt0 || 1237 slice_hdr->delta_pic_order_cnt1 != curr_pic_->delta_pic_order_cnt1) 1238 return true; 1239 } 1240 } 1241 1242 return false; 1243} 1244 1245bool H264Decoder::PreprocessCurrentSlice() { 1246 const H264SliceHeader* slice_hdr = curr_slice_hdr_.get(); 1247 DCHECK(slice_hdr); 1248 1249 if (IsNewPrimaryCodedPicture(slice_hdr)) { 1250 // New picture, so first finish the previous one before processing it. 1251 if (!FinishPrevFrameIfPresent()) 1252 return false; 1253 1254 DCHECK(!curr_pic_); 1255 1256 if (slice_hdr->first_mb_in_slice != 0) { 1257 DVLOG(1) << "ASO/invalid stream, first_mb_in_slice: " 1258 << slice_hdr->first_mb_in_slice; 1259 return false; 1260 } 1261 1262 // If the new picture is an IDR, flush DPB. 1263 if (slice_hdr->idr_pic_flag) { 1264 // Output all remaining pictures, unless we are explicitly instructed 1265 // not to do so. 1266 if (!slice_hdr->no_output_of_prior_pics_flag) { 1267 if (!Flush()) 1268 return false; 1269 } 1270 dpb_.Clear(); 1271 last_output_poc_ = std::numeric_limits<int>::min(); 1272 } 1273 } 1274 1275 return true; 1276} 1277 1278bool H264Decoder::ProcessCurrentSlice() { 1279 DCHECK(curr_pic_); 1280 1281 const H264SliceHeader* slice_hdr = curr_slice_hdr_.get(); 1282 DCHECK(slice_hdr); 1283 1284 if (slice_hdr->field_pic_flag == 0) 1285 max_pic_num_ = max_frame_num_; 1286 else 1287 max_pic_num_ = 2 * max_frame_num_; 1288 1289 H264Picture::Vector ref_pic_list0, ref_pic_list1; 1290 if (!ModifyReferencePicLists(slice_hdr, &ref_pic_list0, &ref_pic_list1)) 1291 return false; 1292 1293 const H264PPS* pps = parser_.GetPPS(curr_pps_id_); 1294 if (!pps) 1295 return false; 1296 1297 if (!accelerator_->SubmitSlice(pps, slice_hdr, ref_pic_list0, ref_pic_list1, 1298 curr_pic_.get(), slice_hdr->nalu_data, 1299 slice_hdr->nalu_size)) 1300 return false; 1301 1302 return true; 1303} 1304 1305#define SET_ERROR_AND_RETURN() \ 1306 do { \ 1307 DVLOG(1) << "Error during decode"; \ 1308 state_ = kError; \ 1309 return H264Decoder::kDecodeError; \ 1310 } while (0) 1311 1312void H264Decoder::SetStream(const uint8_t* ptr, size_t size) { 1313 DCHECK(ptr); 1314 DCHECK(size); 1315 1316 DVLOG(4) << "New input stream at: " << (void*)ptr << " size: " << size; 1317 parser_.SetStream(ptr, size); 1318} 1319 1320H264Decoder::DecodeResult H264Decoder::Decode() { 1321 if (state_ == kError) { 1322 DVLOG(1) << "Decoder in error state"; 1323 return kDecodeError; 1324 } 1325 1326 while (1) { 1327 H264Parser::Result par_res; 1328 1329 if (!curr_nalu_) { 1330 curr_nalu_.reset(new H264NALU()); 1331 par_res = parser_.AdvanceToNextNALU(curr_nalu_.get()); 1332 if (par_res == H264Parser::kEOStream) 1333 return kRanOutOfStreamData; 1334 else if (par_res != H264Parser::kOk) 1335 SET_ERROR_AND_RETURN(); 1336 1337 DVLOG(4) << "New NALU: " << static_cast<int>(curr_nalu_->nal_unit_type); 1338 } 1339 1340 switch (curr_nalu_->nal_unit_type) { 1341 case H264NALU::kNonIDRSlice: 1342 // We can't resume from a non-IDR slice. 1343 if (state_ != kDecoding) 1344 break; 1345 1346 // else fallthrough 1347 case H264NALU::kIDRSlice: { 1348 // TODO(posciak): the IDR may require an SPS that we don't have 1349 // available. For now we'd fail if that happens, but ideally we'd like 1350 // to keep going until the next SPS in the stream. 1351 if (state_ == kNeedStreamMetadata) { 1352 // We need an SPS, skip this IDR and keep looking. 1353 break; 1354 } 1355 1356 // If after reset, we should be able to recover from an IDR. 1357 state_ = kDecoding; 1358 1359 if (!curr_slice_hdr_) { 1360 curr_slice_hdr_.reset(new H264SliceHeader()); 1361 par_res = 1362 parser_.ParseSliceHeader(*curr_nalu_, curr_slice_hdr_.get()); 1363 if (par_res != H264Parser::kOk) 1364 SET_ERROR_AND_RETURN(); 1365 1366 if (!PreprocessCurrentSlice()) 1367 SET_ERROR_AND_RETURN(); 1368 } 1369 1370 if (!curr_pic_) { 1371 // New picture/finished previous one, try to start a new one 1372 // or tell the client we need more surfaces. 1373 curr_pic_ = accelerator_->CreateH264Picture(); 1374 if (!curr_pic_) 1375 return kRanOutOfSurfaces; 1376 1377 if (!StartNewFrame(curr_slice_hdr_.get())) 1378 SET_ERROR_AND_RETURN(); 1379 } 1380 1381 if (!ProcessCurrentSlice()) 1382 SET_ERROR_AND_RETURN(); 1383 1384 curr_slice_hdr_.reset(); 1385 break; 1386 } 1387 1388 case H264NALU::kSPS: { 1389 int sps_id; 1390 1391 if (!FinishPrevFrameIfPresent()) 1392 SET_ERROR_AND_RETURN(); 1393 1394 par_res = parser_.ParseSPS(&sps_id); 1395 if (par_res != H264Parser::kOk) 1396 SET_ERROR_AND_RETURN(); 1397 1398 bool need_new_buffers = false; 1399 if (!ProcessSPS(sps_id, &need_new_buffers)) 1400 SET_ERROR_AND_RETURN(); 1401 1402 if (state_ == kNeedStreamMetadata) 1403 state_ = kAfterReset; 1404 1405 if (need_new_buffers) { 1406 curr_pic_ = nullptr; 1407 curr_nalu_ = nullptr; 1408 ref_pic_list_p0_.clear(); 1409 ref_pic_list_b0_.clear(); 1410 ref_pic_list_b1_.clear(); 1411 1412 return kAllocateNewSurfaces; 1413 } 1414 break; 1415 } 1416 1417 case H264NALU::kPPS: { 1418 int pps_id; 1419 1420 if (!FinishPrevFrameIfPresent()) 1421 SET_ERROR_AND_RETURN(); 1422 1423 par_res = parser_.ParsePPS(&pps_id); 1424 if (par_res != H264Parser::kOk) 1425 SET_ERROR_AND_RETURN(); 1426 1427 break; 1428 } 1429 1430 case H264NALU::kAUD: 1431 case H264NALU::kEOSeq: 1432 case H264NALU::kEOStream: 1433 if (state_ != kDecoding) 1434 break; 1435 1436 if (!FinishPrevFrameIfPresent()) 1437 SET_ERROR_AND_RETURN(); 1438 1439 break; 1440 1441 default: 1442 DVLOG(4) << "Skipping NALU type: " << curr_nalu_->nal_unit_type; 1443 break; 1444 } 1445 1446 DVLOG(4) << "NALU done"; 1447 curr_nalu_.reset(); 1448 } 1449} 1450 1451Size H264Decoder::GetPicSize() const { 1452 return pic_size_; 1453} 1454 1455size_t H264Decoder::GetRequiredNumOfPictures() const { 1456 return dpb_.max_num_pics() + kPicsInPipeline; 1457} 1458 1459} // namespace media 1460