1/*
2 *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS.  All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 */
10#include "webrtc/modules/video_processing/main/source/content_analysis.h"
11
12#include <math.h>
13#include <stdlib.h>
14
15#include "webrtc/system_wrappers/interface/cpu_features_wrapper.h"
16#include "webrtc/system_wrappers/interface/tick_util.h"
17
18namespace webrtc {
19
20VPMContentAnalysis::VPMContentAnalysis(bool runtime_cpu_detection)
21    : orig_frame_(NULL),
22      prev_frame_(NULL),
23      width_(0),
24      height_(0),
25      skip_num_(1),
26      border_(8),
27      motion_magnitude_(0.0f),
28      spatial_pred_err_(0.0f),
29      spatial_pred_err_h_(0.0f),
30      spatial_pred_err_v_(0.0f),
31      first_frame_(true),
32      ca_Init_(false),
33      content_metrics_(NULL) {
34  ComputeSpatialMetrics = &VPMContentAnalysis::ComputeSpatialMetrics_C;
35  TemporalDiffMetric = &VPMContentAnalysis::TemporalDiffMetric_C;
36
37  if (runtime_cpu_detection) {
38#if defined(WEBRTC_ARCH_X86_FAMILY)
39    if (WebRtc_GetCPUInfo(kSSE2)) {
40      ComputeSpatialMetrics = &VPMContentAnalysis::ComputeSpatialMetrics_SSE2;
41      TemporalDiffMetric = &VPMContentAnalysis::TemporalDiffMetric_SSE2;
42    }
43#endif
44  }
45  Release();
46}
47
48VPMContentAnalysis::~VPMContentAnalysis() {
49  Release();
50}
51
52
53VideoContentMetrics* VPMContentAnalysis::ComputeContentMetrics(
54  const I420VideoFrame& inputFrame) {
55  if (inputFrame.IsZeroSize())
56    return NULL;
57
58  // Init if needed (native dimension change).
59  if (width_ != inputFrame.width() || height_ != inputFrame.height()) {
60    if (VPM_OK != Initialize(inputFrame.width(), inputFrame.height()))
61      return NULL;
62  }
63  // Only interested in the Y plane.
64  orig_frame_ = inputFrame.buffer(kYPlane);
65
66  // Compute spatial metrics: 3 spatial prediction errors.
67  (this->*ComputeSpatialMetrics)();
68
69  // Compute motion metrics
70  if (first_frame_ == false)
71    ComputeMotionMetrics();
72
73  // Saving current frame as previous one: Y only.
74  memcpy(prev_frame_, orig_frame_, width_ * height_);
75
76  first_frame_ =  false;
77  ca_Init_ = true;
78
79  return ContentMetrics();
80}
81
82int32_t VPMContentAnalysis::Release() {
83  if (content_metrics_ != NULL) {
84    delete content_metrics_;
85    content_metrics_ = NULL;
86  }
87
88  if (prev_frame_ != NULL) {
89    delete [] prev_frame_;
90    prev_frame_ = NULL;
91  }
92
93  width_ = 0;
94  height_ = 0;
95  first_frame_ = true;
96
97  return VPM_OK;
98}
99
100int32_t VPMContentAnalysis::Initialize(int width, int height) {
101  width_ = width;
102  height_ = height;
103  first_frame_ = true;
104
105  // skip parameter: # of skipped rows: for complexity reduction
106  //  temporal also currently uses it for column reduction.
107  skip_num_ = 1;
108
109  // use skipNum = 2 for 4CIF, WHD
110  if ( (height_ >=  576) && (width_ >= 704) ) {
111    skip_num_ = 2;
112  }
113  // use skipNum = 4 for FULLL_HD images
114  if ( (height_ >=  1080) && (width_ >= 1920) ) {
115    skip_num_ = 4;
116  }
117
118  if (content_metrics_ != NULL) {
119    delete content_metrics_;
120  }
121
122  if (prev_frame_ != NULL) {
123    delete [] prev_frame_;
124  }
125
126  // Spatial Metrics don't work on a border of 8. Minimum processing
127  // block size is 16 pixels.  So make sure the width and height support this.
128  if (width_ <= 32 || height_ <= 32) {
129    ca_Init_ = false;
130    return VPM_PARAMETER_ERROR;
131  }
132
133  content_metrics_ = new VideoContentMetrics();
134  if (content_metrics_ == NULL) {
135    return VPM_MEMORY;
136  }
137
138  prev_frame_ = new uint8_t[width_ * height_];  // Y only.
139  if (prev_frame_ == NULL) return VPM_MEMORY;
140
141  return VPM_OK;
142}
143
144
145// Compute motion metrics: magnitude over non-zero motion vectors,
146//  and size of zero cluster
147int32_t VPMContentAnalysis::ComputeMotionMetrics() {
148  // Motion metrics: only one is derived from normalized
149  //  (MAD) temporal difference
150  (this->*TemporalDiffMetric)();
151  return VPM_OK;
152}
153
154// Normalized temporal difference (MAD): used as a motion level metric
155// Normalize MAD by spatial contrast: images with more contrast
156//  (pixel variance) likely have larger temporal difference
157// To reduce complexity, we compute the metric for a reduced set of points.
158int32_t VPMContentAnalysis::TemporalDiffMetric_C() {
159  // size of original frame
160  int sizei = height_;
161  int sizej = width_;
162  uint32_t tempDiffSum = 0;
163  uint32_t pixelSum = 0;
164  uint64_t pixelSqSum = 0;
165
166  uint32_t num_pixels = 0;  // Counter for # of pixels.
167  const int width_end = ((width_ - 2*border_) & -16) + border_;
168
169  for (int i = border_; i < sizei - border_; i += skip_num_) {
170    for (int j = border_; j < width_end; j++) {
171      num_pixels += 1;
172      int ssn =  i * sizej + j;
173
174      uint8_t currPixel  = orig_frame_[ssn];
175      uint8_t prevPixel  = prev_frame_[ssn];
176
177      tempDiffSum += (uint32_t)abs((int16_t)(currPixel - prevPixel));
178      pixelSum += (uint32_t) currPixel;
179      pixelSqSum += (uint64_t) (currPixel * currPixel);
180    }
181  }
182
183  // Default.
184  motion_magnitude_ = 0.0f;
185
186  if (tempDiffSum == 0) return VPM_OK;
187
188  // Normalize over all pixels.
189  float const tempDiffAvg = (float)tempDiffSum / (float)(num_pixels);
190  float const pixelSumAvg = (float)pixelSum / (float)(num_pixels);
191  float const pixelSqSumAvg = (float)pixelSqSum / (float)(num_pixels);
192  float contrast = pixelSqSumAvg - (pixelSumAvg * pixelSumAvg);
193
194  if (contrast > 0.0) {
195    contrast = sqrt(contrast);
196    motion_magnitude_ = tempDiffAvg/contrast;
197  }
198  return VPM_OK;
199}
200
201// Compute spatial metrics:
202// To reduce complexity, we compute the metric for a reduced set of points.
203// The spatial metrics are rough estimates of the prediction error cost for
204//  each QM spatial mode: 2x2,1x2,2x1
205// The metrics are a simple estimate of the up-sampling prediction error,
206// estimated assuming sub-sampling for decimation (no filtering),
207// and up-sampling back up with simple bilinear interpolation.
208int32_t VPMContentAnalysis::ComputeSpatialMetrics_C() {
209  const int sizei = height_;
210  const int sizej = width_;
211
212  // Pixel mean square average: used to normalize the spatial metrics.
213  uint32_t pixelMSA = 0;
214
215  uint32_t spatialErrSum = 0;
216  uint32_t spatialErrVSum = 0;
217  uint32_t spatialErrHSum = 0;
218
219  // make sure work section is a multiple of 16
220  const int width_end = ((sizej - 2*border_) & -16) + border_;
221
222  for (int i = border_; i < sizei - border_; i += skip_num_) {
223    for (int j = border_; j < width_end; j++) {
224      int ssn1=  i * sizej + j;
225      int ssn2 = (i + 1) * sizej + j; // bottom
226      int ssn3 = (i - 1) * sizej + j; // top
227      int ssn4 = i * sizej + j + 1;   // right
228      int ssn5 = i * sizej + j - 1;   // left
229
230      uint16_t refPixel1  = orig_frame_[ssn1] << 1;
231      uint16_t refPixel2  = orig_frame_[ssn1] << 2;
232
233      uint8_t bottPixel = orig_frame_[ssn2];
234      uint8_t topPixel = orig_frame_[ssn3];
235      uint8_t rightPixel = orig_frame_[ssn4];
236      uint8_t leftPixel = orig_frame_[ssn5];
237
238      spatialErrSum  += (uint32_t) abs((int16_t)(refPixel2
239          - (uint16_t)(bottPixel + topPixel + leftPixel + rightPixel)));
240      spatialErrVSum += (uint32_t) abs((int16_t)(refPixel1
241          - (uint16_t)(bottPixel + topPixel)));
242      spatialErrHSum += (uint32_t) abs((int16_t)(refPixel1
243          - (uint16_t)(leftPixel + rightPixel)));
244      pixelMSA += orig_frame_[ssn1];
245    }
246  }
247
248  // Normalize over all pixels.
249  const float spatialErr = (float)(spatialErrSum >> 2);
250  const float spatialErrH = (float)(spatialErrHSum >> 1);
251  const float spatialErrV = (float)(spatialErrVSum >> 1);
252  const float norm = (float)pixelMSA;
253
254  // 2X2:
255  spatial_pred_err_ = spatialErr / norm;
256  // 1X2:
257  spatial_pred_err_h_ = spatialErrH / norm;
258  // 2X1:
259  spatial_pred_err_v_ = spatialErrV / norm;
260  return VPM_OK;
261}
262
263VideoContentMetrics* VPMContentAnalysis::ContentMetrics() {
264  if (ca_Init_ == false) return NULL;
265
266  content_metrics_->spatial_pred_err = spatial_pred_err_;
267  content_metrics_->spatial_pred_err_h = spatial_pred_err_h_;
268  content_metrics_->spatial_pred_err_v = spatial_pred_err_v_;
269  // Motion metric: normalized temporal difference (MAD).
270  content_metrics_->motion_magnitude = motion_magnitude_;
271
272  return content_metrics_;
273}
274
275}  // namespace webrtc
276