1/*
2 *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS.  All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 */
10
11#include "webrtc/modules/video_coding/codecs/test_framework/benchmark.h"
12
13#include <assert.h>
14
15#include <iostream>
16#include <sstream>
17#include <vector>
18#if defined(_WIN32)
19    #include <windows.h>
20#endif
21
22#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
23#include "webrtc/modules/video_coding/codecs/test_framework/video_source.h"
24#include "webrtc/system_wrappers/interface/event_wrapper.h"
25#include "webrtc/test/testsupport/fileutils.h"
26#include "webrtc/test/testsupport/metrics/video_metrics.h"
27
28#define SSIM_CALC 0 // by default, don't compute SSIM
29
30using namespace webrtc;
31
32Benchmark::Benchmark()
33:
34NormalAsyncTest("Benchmark", "Codec benchmark over a range of test cases", 6),
35_resultsFileName(webrtc::test::OutputPath() + "benchmark.txt"),
36_codecName("Default")
37{
38}
39
40Benchmark::Benchmark(std::string name, std::string description)
41:
42NormalAsyncTest(name, description, 6),
43_resultsFileName(webrtc::test::OutputPath() + "benchmark.txt"),
44_codecName("Default")
45{
46}
47
48Benchmark::Benchmark(std::string name, std::string description, std::string resultsFileName, std::string codecName)
49:
50NormalAsyncTest(name, description, 6),
51_resultsFileName(resultsFileName),
52_codecName(codecName)
53{
54}
55
56void
57Benchmark::Perform()
58{
59    std::vector<const VideoSource*> sources;
60    std::vector<const VideoSource*>::iterator it;
61
62    // Configuration --------------------------
63    sources.push_back(new const VideoSource(webrtc::test::ProjectRootPath() +
64                                            "resources/foreman_cif.yuv", kCIF));
65//    sources.push_back(new const VideoSource(webrtc::test::ProjectRootPath() +
66//                                            "resources/akiyo_cif.yuv", kCIF));
67
68    const VideoSize size[] = {kQCIF, kCIF};
69    const int frameRate[] = {10, 15, 30};
70    // Specifies the framerates for which to perform a speed test.
71    const bool speedTestMask[] = {false, false, false};
72    const int bitRate[] = {50, 100, 200, 300, 400, 500, 600, 1000};
73    // Determines the number of iterations to perform to arrive at the speed result.
74    enum { kSpeedTestIterations = 10 };
75    // ----------------------------------------
76
77    const int nFrameRates = sizeof(frameRate)/sizeof(*frameRate);
78    assert(sizeof(speedTestMask)/sizeof(*speedTestMask) == nFrameRates);
79    const int nBitrates = sizeof(bitRate)/sizeof(*bitRate);
80    int testIterations = 10;
81
82    webrtc::test::QualityMetricsResult psnr[nBitrates];
83    webrtc::test::QualityMetricsResult ssim[nBitrates];
84    double fps[nBitrates];
85    double totalEncodeTime[nBitrates];
86    double totalDecodeTime[nBitrates];
87
88    _results.open(_resultsFileName.c_str(), std::fstream::out);
89    _results << GetMagicStr() << std::endl;
90    _results << _codecName << std::endl;
91
92    for (it = sources.begin() ; it < sources.end(); it++)
93    {
94        for (int i = 0; i < static_cast<int>(sizeof(size)/sizeof(*size)); i++)
95        {
96            for (int j = 0; j < nFrameRates; j++)
97            {
98                std::stringstream ss;
99                std::string strFrameRate;
100                std::string outFileName;
101                ss << frameRate[j];
102                ss >> strFrameRate;
103                outFileName = (*it)->GetFilePath() + "/" + (*it)->GetName() + "_" +
104                    VideoSource::GetSizeString(size[i]) + "_" + strFrameRate + ".yuv";
105
106                _target = new const VideoSource(outFileName, size[i], frameRate[j]);
107                (*it)->Convert(*_target);
108                if (VideoSource::FileExists(outFileName.c_str()))
109                {
110                    _inname = outFileName;
111                }
112                else
113                {
114                    _inname = (*it)->GetFileName();
115                }
116
117                std::cout << (*it)->GetName() << ", " << VideoSource::GetSizeString(size[i])
118                    << ", " << frameRate[j] << " fps" << std::endl << "Bitrate [kbps]:";
119                _results << (*it)->GetName() << "," << VideoSource::GetSizeString(size[i])
120                    << "," << frameRate[j] << " fps" << std::endl << "Bitrate [kbps]";
121
122                if (speedTestMask[j])
123                {
124                    testIterations = kSpeedTestIterations;
125                }
126                else
127                {
128                    testIterations = 1;
129                }
130
131                for (int k = 0; k < nBitrates; k++)
132                {
133                    _bitRate = (bitRate[k]);
134                    double avgFps = 0.0;
135                    totalEncodeTime[k] = 0;
136                    totalDecodeTime[k] = 0;
137
138                    for (int l = 0; l < testIterations; l++)
139                    {
140                        PerformNormalTest();
141                        _appendNext = false;
142
143                        avgFps += _framecnt / (_totalEncodeTime + _totalDecodeTime);
144                        totalEncodeTime[k] += _totalEncodeTime;
145                        totalDecodeTime[k] += _totalDecodeTime;
146
147                    }
148                    avgFps /= testIterations;
149                    totalEncodeTime[k] /= testIterations;
150                    totalDecodeTime[k] /= testIterations;
151
152                    double actualBitRate = ActualBitRate(_framecnt) / 1000.0;
153                    std::cout << " " << actualBitRate;
154                    _results << "," << actualBitRate;
155                    webrtc::test::QualityMetricsResult psnr_result;
156                    I420PSNRFromFiles(_inname.c_str(), _outname.c_str(),
157                                      _inst.width, _inst.height, &psnr[k]);
158                    if (SSIM_CALC)
159                    {
160                        webrtc::test::QualityMetricsResult ssim_result;
161                        I420SSIMFromFiles(_inname.c_str(), _outname.c_str(),
162                                          _inst.width, _inst.height, &ssim[k]);
163
164                    }
165                    fps[k] = avgFps;
166                }
167                std::cout << std::endl << "Y-PSNR [dB]:";
168                _results << std::endl << "Y-PSNR [dB]";
169                for (int k = 0; k < nBitrates; k++)
170                {
171                    std::cout << " " << psnr[k].average;
172                    _results << "," << psnr[k].average;
173
174                }
175                if (SSIM_CALC)
176                {
177                    std::cout << std::endl << "SSIM: ";
178                    _results << std::endl << "SSIM ";
179                    for (int k = 0; k < nBitrates; k++)
180                    {
181                        std::cout << " " << ssim[k].average;
182                        _results << "," << ssim[k].average;
183                    }
184
185                }
186
187                std::cout << std::endl << "Encode Time[ms]:";
188                _results << std::endl << "Encode Time[ms]";
189                for (int k = 0; k < nBitrates; k++)
190                {
191                    std::cout << " " << totalEncodeTime[k];
192                    _results << "," << totalEncodeTime[k];
193
194                }
195
196                std::cout << std::endl << "Decode Time[ms]:";
197                _results << std::endl << "Decode Time[ms]";
198                for (int k = 0; k < nBitrates; k++)
199                {
200                    std::cout << " " << totalDecodeTime[k];
201                    _results << "," << totalDecodeTime[k];
202
203                }
204
205                if (speedTestMask[j])
206                {
207                    std::cout << std::endl << "Speed [fps]:";
208                    _results << std::endl << "Speed [fps]";
209                    for (int k = 0; k < nBitrates; k++)
210                    {
211                        std::cout << " " << static_cast<int>(fps[k] + 0.5);
212                        _results << "," << static_cast<int>(fps[k] + 0.5);
213                    }
214                }
215                std::cout << std::endl << std::endl;
216                _results << std::endl << std::endl;
217
218                delete _target;
219            }
220        }
221        delete *it;
222    }
223    _results.close();
224}
225
226void
227Benchmark::PerformNormalTest()
228{
229    _encoder = GetNewEncoder();
230    _decoder = GetNewDecoder();
231    CodecSettings(_target->GetWidth(), _target->GetHeight(), _target->GetFrameRate(), _bitRate);
232    Setup();
233    EventWrapper* waitEvent = EventWrapper::Create();
234    _encoder->InitEncode(&_inst, 4, 1440);
235    CodecSpecific_InitBitrate();
236    _decoder->InitDecode(&_inst,1);
237
238    FrameQueue frameQueue;
239    VideoEncodeCompleteCallback encCallback(_encodedFile, &frameQueue, *this);
240    VideoDecodeCompleteCallback decCallback(_decodedFile, *this);
241    _encoder->RegisterEncodeCompleteCallback(&encCallback);
242    _decoder->RegisterDecodeCompleteCallback(&decCallback);
243
244    SetCodecSpecificParameters();
245
246    _totalEncodeTime = _totalDecodeTime = 0;
247    _totalEncodePipeTime = _totalDecodePipeTime = 0;
248    bool complete = false;
249    _framecnt = 0;
250    _encFrameCnt = 0;
251    _sumEncBytes = 0;
252    _lengthEncFrame = 0;
253    while (!complete)
254    {
255        complete = Encode();
256        if (!frameQueue.Empty() || complete)
257        {
258            while (!frameQueue.Empty())
259            {
260                _frameToDecode = static_cast<FrameQueueTuple *>(frameQueue.PopFrame());
261                DoPacketLoss();
262                int ret = Decode();
263                delete _frameToDecode;
264                _frameToDecode = NULL;
265                if (ret < 0)
266                {
267                    fprintf(stderr,"\n\nError in decoder: %d\n\n", ret);
268                    exit(EXIT_FAILURE);
269                }
270                else if (ret == 0)
271                {
272                    _framecnt++;
273                }
274                else
275                {
276                    fprintf(stderr, "\n\nPositive return value from decode!\n\n");
277                }
278            }
279        }
280        waitEvent->Wait(5);
281    }
282
283    _encodedVideoBuffer.Free();
284
285    _encoder->Release();
286    _decoder->Release();
287    delete waitEvent;
288    delete _encoder;
289    delete _decoder;
290    Teardown();
291}
292
293void
294Benchmark::CodecSpecific_InitBitrate()
295{
296    if (_bitRate == 0)
297    {
298        _encoder->SetRates(600, _inst.maxFramerate);
299    }
300    else
301    {
302        _encoder->SetRates(_bitRate, _inst.maxFramerate);
303    }
304}
305