1/*
2 *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS.  All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 */
10
11#include "webrtc/modules/video_coding/main/test/quality_modes_test.h"
12
13#include <iostream>
14#include <sstream>
15#include <string>
16#include <time.h>
17
18#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
19#include "webrtc/modules/video_coding/main/interface/video_coding.h"
20#include "webrtc/modules/video_coding/main/test/test_callbacks.h"
21#include "webrtc/modules/video_coding/main/test/test_macros.h"
22#include "webrtc/modules/video_coding/main/test/test_util.h"
23#include "webrtc/system_wrappers/interface/clock.h"
24#include "webrtc/system_wrappers/interface/data_log.h"
25#include "webrtc/system_wrappers/interface/data_log.h"
26#include "webrtc/test/testsupport/fileutils.h"
27#include "webrtc/test/testsupport/metrics/video_metrics.h"
28
29using namespace webrtc;
30
31int qualityModeTest(const CmdArgs& args)
32{
33  SimulatedClock clock(0);
34  NullEventFactory event_factory;
35  VideoCodingModule* vcm = VideoCodingModule::Create(&clock, &event_factory);
36  QualityModesTest QMTest(vcm, &clock);
37  QMTest.Perform(args);
38  VideoCodingModule::Destroy(vcm);
39  return 0;
40}
41
42QualityModesTest::QualityModesTest(VideoCodingModule* vcm,
43                                   Clock* clock):
44NormalTest(vcm, clock),
45_vpm()
46{
47    //
48}
49
50QualityModesTest::~QualityModesTest()
51{
52    //
53}
54
55void
56QualityModesTest::Setup(const CmdArgs& args)
57{
58  NormalTest::Setup(args);
59  _inname = args.inputFile;
60  _outname = args.outputFile;
61  fv_outfilename_ = args.fv_outputfile;
62
63  filename_testvideo_ =
64      _inname.substr(_inname.find_last_of("\\/") + 1,_inname.length());
65
66  _encodedName = test::OutputPath() + "encoded_qmtest.yuv";
67
68  //NATIVE/SOURCE VALUES
69  _nativeWidth = args.width;
70  _nativeHeight = args.height;
71  _nativeFrameRate =args.frameRate;
72
73  //TARGET/ENCODER VALUES
74  _width = args.width;
75  _height = args.height;
76  _frameRate = args.frameRate;
77
78  _bitRate = args.bitRate;
79
80  _flagSSIM = true;
81
82  _lengthSourceFrame  = 3*_nativeWidth*_nativeHeight/2;
83
84  if ((_sourceFile = fopen(_inname.c_str(), "rb")) == NULL)
85  {
86    printf("Cannot read file %s.\n", _inname.c_str());
87    exit(1);
88  }
89  if ((_encodedFile = fopen(_encodedName.c_str(), "wb")) == NULL)
90  {
91    printf("Cannot write encoded file.\n");
92    exit(1);
93  }
94  if ((_decodedFile = fopen(_outname.c_str(),  "wb")) == NULL)
95  {
96    printf("Cannot write file %s.\n", _outname.c_str());
97    exit(1);
98  }
99
100  DataLog::CreateLog();
101
102  feature_table_name_ = fv_outfilename_;
103
104  DataLog::AddTable(feature_table_name_);
105
106  DataLog::AddColumn(feature_table_name_, "motion magnitude", 1);
107  DataLog::AddColumn(feature_table_name_, "spatial prediction error", 1);
108  DataLog::AddColumn(feature_table_name_, "spatial pred err horizontal", 1);
109  DataLog::AddColumn(feature_table_name_, "spatial pred err vertical", 1);
110  DataLog::AddColumn(feature_table_name_, "width", 1);
111  DataLog::AddColumn(feature_table_name_, "height", 1);
112  DataLog::AddColumn(feature_table_name_, "num pixels", 1);
113  DataLog::AddColumn(feature_table_name_, "frame rate", 1);
114  DataLog::AddColumn(feature_table_name_, "num frames since drop", 1);
115
116  _log.open((test::OutputPath() + "TestLog.txt").c_str(),
117            std::fstream::out | std::fstream::app);
118}
119
120void
121QualityModesTest::Print()
122{
123  std::cout << "Quality Modes Test Completed!" << std::endl;
124  (_log) << "Quality Modes Test Completed!" << std::endl;
125  (_log) << "Input file: " << _inname << std::endl;
126  (_log) << "Output file: " << _outname << std::endl;
127  (_log) << "Total run time: " << _testTotalTime << std::endl;
128  printf("Total run time: %f s \n", _testTotalTime);
129  double ActualBitRate = 8.0*( _sumEncBytes / (_frameCnt / _nativeFrameRate));
130  double actualBitRate = ActualBitRate / 1000.0;
131  double avgEncTime = _totalEncodeTime / _frameCnt;
132  double avgDecTime = _totalDecodeTime / _frameCnt;
133  webrtc::test::QualityMetricsResult psnr,ssim;
134  I420PSNRFromFiles(_inname.c_str(), _outname.c_str(), _nativeWidth,
135                    _nativeHeight, &psnr);
136  printf("Actual bitrate: %f kbps\n", actualBitRate);
137  printf("Target bitrate: %f kbps\n", _bitRate);
138  ( _log) << "Actual bitrate: " << actualBitRate<< " kbps\tTarget: " <<
139      _bitRate << " kbps" << std::endl;
140  printf("Average encode time: %f s\n", avgEncTime);
141  ( _log) << "Average encode time: " << avgEncTime << " s" << std::endl;
142  printf("Average decode time: %f s\n", avgDecTime);
143  ( _log) << "Average decode time: " << avgDecTime << " s" << std::endl;
144  printf("PSNR: %f \n", psnr.average);
145  printf("**Number of frames dropped in VPM***%d \n",_numFramesDroppedVPM);
146  ( _log) << "PSNR: " << psnr.average << std::endl;
147  if (_flagSSIM == 1)
148  {
149    printf("***computing SSIM***\n");
150    I420SSIMFromFiles(_inname.c_str(), _outname.c_str(), _nativeWidth,
151                      _nativeHeight, &ssim);
152    printf("SSIM: %f \n", ssim.average);
153  }
154  (_log) << std::endl;
155
156  printf("\nVCM Quality Modes Test: \n\n%i tests completed\n", vcmMacrosTests);
157  if (vcmMacrosErrors > 0)
158  {
159    printf("%i FAILED\n\n", vcmMacrosErrors);
160  }
161  else
162  {
163    printf("ALL PASSED\n\n");
164  }
165}
166void
167QualityModesTest::Teardown()
168{
169  _log.close();
170  fclose(_sourceFile);
171  fclose(_decodedFile);
172  fclose(_encodedFile);
173  return;
174}
175
176int32_t
177QualityModesTest::Perform(const CmdArgs& args)
178{
179  Setup(args);
180  // changing bit/frame rate during the test
181  const float bitRateUpdate[] = {1000};
182  const float frameRateUpdate[] = {30};
183  // frame num at which an update will occur
184  const int updateFrameNum[] = {10000};
185
186  uint32_t numChanges = sizeof(updateFrameNum)/sizeof(*updateFrameNum);
187  uint8_t change = 0;// change counter
188
189  _vpm = VideoProcessingModule::Create(1);
190  EventWrapper* waitEvent = EventWrapper::Create();
191  VideoCodec codec;//both send and receive
192  _vcm->InitializeReceiver();
193  _vcm->InitializeSender();
194  int32_t NumberOfCodecs = _vcm->NumberOfCodecs();
195  for (int i = 0; i < NumberOfCodecs; i++)
196  {
197    _vcm->Codec(i, &codec);
198    if(strncmp(codec.plName,"VP8" , 5) == 0)
199    {
200      codec.startBitrate = (int)_bitRate;
201      codec.maxFramerate = (uint8_t) _frameRate;
202      codec.width = (uint16_t)_width;
203      codec.height = (uint16_t)_height;
204      codec.codecSpecific.VP8.frameDroppingOn = false;
205
206      // Will also set and init the desired codec
207      TEST(_vcm->RegisterSendCodec(&codec, 2, 1440) == VCM_OK);
208      i = NumberOfCodecs;
209    }
210  }
211
212  // register a decoder (same codec for decoder and encoder )
213  TEST(_vcm->RegisterReceiveCodec(&codec, 2) == VCM_OK);
214  /* Callback Settings */
215  VCMQMDecodeCompleCallback  _decodeCallback(
216      _decodedFile, _nativeFrameRate, feature_table_name_);
217  _vcm->RegisterReceiveCallback(&_decodeCallback);
218  VCMNTEncodeCompleteCallback   _encodeCompleteCallback(_encodedFile, *this);
219  _vcm->RegisterTransportCallback(&_encodeCompleteCallback);
220  // encode and decode with the same vcm
221  _encodeCompleteCallback.RegisterReceiverVCM(_vcm);
222
223  //quality modes callback
224  QMTestVideoSettingsCallback QMCallback;
225  QMCallback.RegisterVCM(_vcm);
226  QMCallback.RegisterVPM(_vpm);
227  //_vcm->RegisterVideoQMCallback(&QMCallback);
228
229  ///////////////////////
230  /// Start Test
231  ///////////////////////
232  _vpm->EnableTemporalDecimation(true);
233  _vpm->EnableContentAnalysis(true);
234  _vpm->SetInputFrameResampleMode(kFastRescaling);
235
236  // disabling internal VCM frame dropper
237  _vcm->EnableFrameDropper(false);
238
239  I420VideoFrame sourceFrame;
240  I420VideoFrame *decimatedFrame = NULL;
241  uint8_t* tmpBuffer = new uint8_t[_lengthSourceFrame];
242  double startTime = clock()/(double)CLOCKS_PER_SEC;
243  _vcm->SetChannelParameters(static_cast<uint32_t>(1000 * _bitRate), 0, 0);
244
245  SendStatsTest sendStats;
246  sendStats.set_framerate(static_cast<uint32_t>(_frameRate));
247  sendStats.set_bitrate(1000 * _bitRate);
248  _vcm->RegisterSendStatisticsCallback(&sendStats);
249
250  VideoContentMetrics* contentMetrics = NULL;
251  // setting user frame rate
252  // for starters: keeping native values:
253  _vpm->SetTargetResolution(_width, _height,
254                            (uint32_t)(_frameRate+ 0.5f));
255  _decodeCallback.SetOriginalFrameDimensions(_nativeWidth, _nativeHeight);
256
257  //tmp  - disabling VPM frame dropping
258  _vpm->EnableTemporalDecimation(false);
259
260  int32_t ret = 0;
261  _numFramesDroppedVPM = 0;
262
263  do {
264    if (fread(tmpBuffer, 1, _lengthSourceFrame, _sourceFile) > 0) {
265      _frameCnt++;
266      int size_y = _nativeWidth * _nativeHeight;
267      int size_uv = ((_nativeWidth + 1) / 2) * ((_nativeHeight  + 1) / 2);
268      sourceFrame.CreateFrame(size_y, tmpBuffer,
269                              size_uv, tmpBuffer + size_y,
270                              size_uv, tmpBuffer + size_y + size_uv,
271                              _nativeWidth, _nativeHeight,
272                              _nativeWidth, (_nativeWidth + 1) / 2,
273                              (_nativeWidth + 1) / 2);
274
275      _timeStamp +=
276          (uint32_t)(9e4 / static_cast<float>(codec.maxFramerate));
277      sourceFrame.set_timestamp(_timeStamp);
278
279      ret = _vpm->PreprocessFrame(sourceFrame, &decimatedFrame);
280      if (ret  == 1)
281      {
282        printf("VD: frame drop %d \n",_frameCnt);
283        _numFramesDroppedVPM += 1;
284        continue; // frame drop
285      }
286      else if (ret < 0)
287      {
288        printf("Error in PreprocessFrame: %d\n", ret);
289        //exit(1);
290      }
291      // Frame was not re-sampled => use original.
292      if (decimatedFrame == NULL)
293      {
294        decimatedFrame = &sourceFrame;
295      }
296      contentMetrics = _vpm->ContentMetrics();
297      if (contentMetrics == NULL)
298      {
299        printf("error: contentMetrics = NULL\n");
300      }
301
302      // counting only encoding time
303      _encodeTimes[int(sourceFrame.timestamp())] =
304          clock()/(double)CLOCKS_PER_SEC;
305
306      int32_t ret = _vcm->AddVideoFrame(*decimatedFrame, contentMetrics);
307
308      _totalEncodeTime += clock()/(double)CLOCKS_PER_SEC -
309          _encodeTimes[int(sourceFrame.timestamp())];
310
311      if (ret < 0)
312      {
313        printf("Error in AddFrame: %d\n", ret);
314        //exit(1);
315      }
316
317      // Same timestamp value for encode and decode
318      _decodeTimes[int(sourceFrame.timestamp())] =
319          clock()/(double)CLOCKS_PER_SEC;
320      ret = _vcm->Decode();
321
322      _totalDecodeTime += clock()/(double)CLOCKS_PER_SEC -
323          _decodeTimes[int(sourceFrame.timestamp())];
324
325      if (ret < 0)
326      {
327        printf("Error in Decode: %d\n", ret);
328        //exit(1);
329      }
330      if (_vcm->TimeUntilNextProcess() <= 0)
331      {
332        _vcm->Process();
333      }
334      // mimicking setTargetRates - update every 1 sec
335      // this will trigger QMSelect
336      if (_frameCnt%((int)_frameRate) == 0)
337      {
338        _vcm->SetChannelParameters(static_cast<uint32_t>(1000 * _bitRate), 0,
339                                   1);
340      }
341
342      // check for bit rate update
343      if (change < numChanges && _frameCnt == updateFrameNum[change])
344      {
345        _bitRate = bitRateUpdate[change];
346        _frameRate = frameRateUpdate[change];
347        codec.startBitrate = (int)_bitRate;
348        codec.maxFramerate = (uint8_t) _frameRate;
349        // Will also set and init the desired codec
350        TEST(_vcm->RegisterSendCodec(&codec, 2, 1440) == VCM_OK);
351        change++;
352      }
353
354      DataLog::InsertCell(feature_table_name_, "motion magnitude",
355                          contentMetrics->motion_magnitude);
356      DataLog::InsertCell(feature_table_name_, "spatial prediction error",
357                          contentMetrics->spatial_pred_err);
358      DataLog::InsertCell(feature_table_name_, "spatial pred err horizontal",
359                          contentMetrics->spatial_pred_err_h);
360      DataLog::InsertCell(feature_table_name_, "spatial pred err vertical",
361                          contentMetrics->spatial_pred_err_v);
362
363      DataLog::InsertCell(feature_table_name_, "width", _nativeHeight);
364      DataLog::InsertCell(feature_table_name_, "height", _nativeWidth);
365
366      DataLog::InsertCell(feature_table_name_, "num pixels",
367                          _nativeHeight * _nativeWidth);
368
369      DataLog::InsertCell(feature_table_name_, "frame rate", _nativeFrameRate);
370      DataLog::NextRow(feature_table_name_);
371
372      static_cast<SimulatedClock*>(_clock)->AdvanceTimeMilliseconds(
373          1000 / _nativeFrameRate);
374  }
375
376  } while (feof(_sourceFile) == 0);
377  _decodeCallback.WriteEnd(_frameCnt);
378
379
380  double endTime = clock()/(double)CLOCKS_PER_SEC;
381  _testTotalTime = endTime - startTime;
382  _sumEncBytes = _encodeCompleteCallback.EncodedBytes();
383
384  delete tmpBuffer;
385  delete waitEvent;
386  _vpm->Reset();
387  Teardown();
388  Print();
389  VideoProcessingModule::Destroy(_vpm);
390  return 0;
391}
392
393// implementing callback to be called from
394// VCM to update VPM of frame rate and size
395QMTestVideoSettingsCallback::QMTestVideoSettingsCallback():
396_vpm(NULL),
397_vcm(NULL)
398{
399  //
400}
401
402void
403QMTestVideoSettingsCallback::RegisterVPM(VideoProcessingModule *vpm)
404{
405  _vpm = vpm;
406}
407void
408QMTestVideoSettingsCallback::RegisterVCM(VideoCodingModule *vcm)
409{
410  _vcm = vcm;
411}
412
413bool
414QMTestVideoSettingsCallback::Updated()
415{
416  if (_updated)
417  {
418    _updated = false;
419    return true;
420  }
421  return false;
422}
423
424int32_t
425QMTestVideoSettingsCallback::SetVideoQMSettings(const uint32_t frameRate,
426                                                const uint32_t width,
427                                                const uint32_t height)
428{
429  int32_t retVal = 0;
430  printf("QM updates: W = %d, H = %d, FR = %d, \n", width, height, frameRate);
431  retVal = _vpm->SetTargetResolution(width, height, frameRate);
432  //Initialize codec with new values - is this the best place to do it?
433  if (!retVal)
434  {
435    // first get current settings
436    VideoCodec currentCodec;
437    _vcm->SendCodec(&currentCodec);
438    // now set new values:
439    currentCodec.height = (uint16_t)height;
440    currentCodec.width = (uint16_t)width;
441    currentCodec.maxFramerate = (uint8_t)frameRate;
442
443    // re-register encoder
444    retVal = _vcm->RegisterSendCodec(&currentCodec, 2, 1440);
445    _updated = true;
446  }
447
448  return retVal;
449}
450
451// Decoded Frame Callback Implementation
452VCMQMDecodeCompleCallback::VCMQMDecodeCompleCallback(
453    FILE* decodedFile, int frame_rate, std::string feature_table_name):
454_decodedFile(decodedFile),
455_decodedBytes(0),
456//_test(test),
457_origWidth(0),
458_origHeight(0),
459_decWidth(0),
460_decHeight(0),
461//_interpolator(NULL),
462_decBuffer(NULL),
463_frameCnt(0),
464frame_rate_(frame_rate),
465frames_cnt_since_drop_(0),
466feature_table_name_(feature_table_name)
467{
468    //
469}
470
471VCMQMDecodeCompleCallback::~VCMQMDecodeCompleCallback()
472 {
473//     if (_interpolator != NULL)
474//     {
475//         deleteInterpolator(_interpolator);
476//         _interpolator = NULL;
477//     }
478   if (_decBuffer != NULL)
479   {
480     delete [] _decBuffer;
481     _decBuffer = NULL;
482   }
483 }
484
485int32_t
486VCMQMDecodeCompleCallback::FrameToRender(I420VideoFrame& videoFrame)
487{
488  ++frames_cnt_since_drop_;
489
490  // When receiving the first coded frame the last_frame variable is not set
491  if (last_frame_.IsZeroSize()) {
492    last_frame_.CopyFrame(videoFrame);
493  }
494
495   // Check if there were frames skipped.
496  int num_frames_skipped = static_cast<int>( 0.5f +
497  (videoFrame.timestamp() - (last_frame_.timestamp() + (9e4 / frame_rate_))) /
498  (9e4 / frame_rate_));
499
500  // If so...put the last frames into the encoded stream to make up for the
501  // skipped frame(s)
502  while (num_frames_skipped > 0) {
503    PrintI420VideoFrame(last_frame_, _decodedFile);
504    _frameCnt++;
505    --num_frames_skipped;
506    frames_cnt_since_drop_ = 1; // Reset counter
507
508  }
509
510  DataLog::InsertCell(
511        feature_table_name_,"num frames since drop",frames_cnt_since_drop_);
512
513  if (_origWidth == videoFrame.width() && _origHeight == videoFrame.height())
514  {
515    if (PrintI420VideoFrame(videoFrame, _decodedFile) < 0) {
516      return -1;
517    }
518    _frameCnt++;
519    // no need for interpolator and decBuffer
520    if (_decBuffer != NULL)
521    {
522      delete [] _decBuffer;
523      _decBuffer = NULL;
524    }
525    _decWidth = 0;
526    _decHeight = 0;
527  }
528  else
529  {
530    // TODO(mikhal): Add support for scaling.
531    return -1;
532  }
533
534  _decodedBytes += CalcBufferSize(kI420, videoFrame.width(),
535                                  videoFrame.height());
536  videoFrame.SwapFrame(&last_frame_);
537  return VCM_OK;
538}
539
540int32_t VCMQMDecodeCompleCallback::DecodedBytes()
541{
542  return _decodedBytes;
543}
544
545void VCMQMDecodeCompleCallback::SetOriginalFrameDimensions(int32_t width,
546                                                           int32_t height)
547{
548  _origWidth = width;
549  _origHeight = height;
550}
551
552int32_t VCMQMDecodeCompleCallback::buildInterpolator()
553{
554  uint32_t decFrameLength  = _origWidth*_origHeight*3 >> 1;
555  if (_decBuffer != NULL)
556  {
557    delete [] _decBuffer;
558  }
559  _decBuffer = new uint8_t[decFrameLength];
560  if (_decBuffer == NULL)
561  {
562    return -1;
563  }
564  return 0;
565}
566
567// This function checks if the total number of frames processed in the encoding
568// process is the same as the number of frames rendered. If not,  the last
569// frame (or several consecutive frames from the end) must have been dropped. If
570// this is the case, the last frame is repeated so that there are as many
571// frames rendered as there are number of frames encoded.
572void VCMQMDecodeCompleCallback::WriteEnd(int input_frame_count)
573{
574  int num_missing_frames = input_frame_count - _frameCnt;
575
576  for (int n = num_missing_frames; n > 0; --n) {
577    PrintI420VideoFrame(last_frame_, _decodedFile);
578    _frameCnt++;
579  }
580}
581