video_denoiser.cc revision 99ab9447d1b48043adf6bfdf72cf820bbda3ee3f 
1/*
2 *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS.  All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 */
10#include "webrtc/common_video/libyuv/include/scaler.h"
11#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
12#include "webrtc/modules/video_processing/video_denoiser.h"
13
14namespace webrtc {
15
16VideoDenoiser::VideoDenoiser()
17    : width_(0), height_(0), filter_(DenoiserFilter::Create()) {}
18
19void VideoDenoiser::TrailingReduction(int mb_rows,
20                                      int mb_cols,
21                                      const uint8_t* y_src,
22                                      int stride_y,
23                                      uint8_t* y_dst) {
24  for (int mb_row = 1; mb_row < mb_rows - 1; ++mb_row) {
25    for (int mb_col = 1; mb_col < mb_cols - 1; ++mb_col) {
26      int mb_index = mb_row * mb_cols + mb_col;
27      uint8_t* mb_dst = y_dst + (mb_row << 4) * stride_y + (mb_col << 4);
28      const uint8_t* mb_src = y_src + (mb_row << 4) * stride_y + (mb_col << 4);
29      // If the number of denoised neighbors is less than a threshold,
30      // do NOT denoise for the block. Set different threshold for skin MB.
31      // The change of denoising status will not propagate.
32      if (metrics_[mb_index].is_skin) {
33        // The threshold is high (more strict) for non-skin MB where the
34        // trailing usually happen.
35        if (metrics_[mb_index].denoise &&
36            metrics_[mb_index + 1].denoise + metrics_[mb_index - 1].denoise +
37                    metrics_[mb_index + mb_cols].denoise +
38                    metrics_[mb_index - mb_cols].denoise <=
39                2) {
40          metrics_[mb_index].denoise = 0;
41          filter_->CopyMem16x16(mb_src, stride_y, mb_dst, stride_y);
42        }
43      } else if (metrics_[mb_index].denoise &&
44                 metrics_[mb_index + 1].denoise +
45                         metrics_[mb_index - 1].denoise +
46                         metrics_[mb_index + mb_cols + 1].denoise +
47                         metrics_[mb_index + mb_cols - 1].denoise +
48                         metrics_[mb_index - mb_cols + 1].denoise +
49                         metrics_[mb_index - mb_cols - 1].denoise +
50                         metrics_[mb_index + mb_cols].denoise +
51                         metrics_[mb_index - mb_cols].denoise <=
52                     7) {
53        filter_->CopyMem16x16(mb_src, stride_y, mb_dst, stride_y);
54      }
55    }
56  }
57}
58
59void VideoDenoiser::DenoiseFrame(const VideoFrame& frame,
60                                 VideoFrame* denoised_frame) {
61  int stride_y = frame.stride(kYPlane);
62  int stride_u = frame.stride(kUPlane);
63  int stride_v = frame.stride(kVPlane);
64  // If previous width and height are different from current frame's, then no
65  // denoising for the current frame.
66  if (width_ != frame.width() || height_ != frame.height()) {
67    width_ = frame.width();
68    height_ = frame.height();
69    denoised_frame->CreateFrame(frame.buffer(kYPlane), frame.buffer(kUPlane),
70                                frame.buffer(kVPlane), width_, height_,
71                                stride_y, stride_u, stride_v);
72    // Setting time parameters to the output frame.
73    denoised_frame->set_timestamp(frame.timestamp());
74    denoised_frame->set_render_time_ms(frame.render_time_ms());
75    return;
76  }
77  // For 16x16 block.
78  int mb_cols = width_ >> 4;
79  int mb_rows = height_ >> 4;
80  if (metrics_.get() == nullptr)
81    metrics_.reset(new DenoiseMetrics[mb_cols * mb_rows]);
82  // Denoise on Y plane.
83  uint8_t* y_dst = denoised_frame->buffer(kYPlane);
84  uint8_t* u_dst = denoised_frame->buffer(kUPlane);
85  uint8_t* v_dst = denoised_frame->buffer(kVPlane);
86  const uint8_t* y_src = frame.buffer(kYPlane);
87  const uint8_t* u_src = frame.buffer(kUPlane);
88  const uint8_t* v_src = frame.buffer(kVPlane);
89  // Temporary buffer to store denoising result.
90  uint8_t y_tmp[16 * 16] = {0};
91  for (int mb_row = 0; mb_row < mb_rows; ++mb_row) {
92    for (int mb_col = 0; mb_col < mb_cols; ++mb_col) {
93      const uint8_t* mb_src = y_src + (mb_row << 4) * stride_y + (mb_col << 4);
94      uint8_t* mb_dst = y_dst + (mb_row << 4) * stride_y + (mb_col << 4);
95      int mb_index = mb_row * mb_cols + mb_col;
96      // Denoise each MB at the very start and save the result to a temporary
97      // buffer.
98      if (filter_->MbDenoise(mb_dst, stride_y, y_tmp, 16, mb_src, stride_y, 0,
99                             1) == FILTER_BLOCK) {
100        uint32_t thr_var = 0;
101        // Save var and sad to the buffer.
102        metrics_[mb_index].var = filter_->Variance16x8(
103            mb_dst, stride_y, y_tmp, 16, &metrics_[mb_index].sad);
104        // Get skin map.
105        metrics_[mb_index].is_skin = MbHasSkinColor(
106            y_src, u_src, v_src, stride_y, stride_u, stride_v, mb_row, mb_col);
107        // Variance threshold for skin/non-skin MB is different.
108        // Skin MB use a small threshold to reduce blockiness.
109        thr_var = metrics_[mb_index].is_skin ? 128 : 12 * 128;
110        if (metrics_[mb_index].var > thr_var) {
111          metrics_[mb_index].denoise = 0;
112          // Use the source MB.
113          filter_->CopyMem16x16(mb_src, stride_y, mb_dst, stride_y);
114        } else {
115          metrics_[mb_index].denoise = 1;
116          // Use the denoised MB.
117          filter_->CopyMem16x16(y_tmp, 16, mb_dst, stride_y);
118        }
119      } else {
120        metrics_[mb_index].denoise = 0;
121        filter_->CopyMem16x16(mb_src, stride_y, mb_dst, stride_y);
122      }
123      // Copy source U/V plane.
124      const uint8_t* mb_src_u =
125          u_src + (mb_row << 3) * stride_u + (mb_col << 3);
126      const uint8_t* mb_src_v =
127          v_src + (mb_row << 3) * stride_v + (mb_col << 3);
128      uint8_t* mb_dst_u = u_dst + (mb_row << 3) * stride_u + (mb_col << 3);
129      uint8_t* mb_dst_v = v_dst + (mb_row << 3) * stride_v + (mb_col << 3);
130      filter_->CopyMem8x8(mb_src_u, stride_u, mb_dst_u, stride_u);
131      filter_->CopyMem8x8(mb_src_v, stride_v, mb_dst_v, stride_v);
132    }
133  }
134  // Second round.
135  // This is to reduce the trailing artifact and blockiness by referring
136  // neighbors' denoising status.
137  TrailingReduction(mb_rows, mb_cols, y_src, stride_y, y_dst);
138
139  // Setting time parameters to the output frame.
140  denoised_frame->set_timestamp(frame.timestamp());
141  denoised_frame->set_render_time_ms(frame.render_time_ms());
142  return;
143}
144
145}  // namespace webrtc
146