1/*
2 *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS.  All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 */
10
11#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
12#include "webrtc/engine_configurations.h"
13#include "webrtc/modules/media_file/interface/media_file.h"
14#include "webrtc/modules/utility/source/file_recorder_impl.h"
15#include "webrtc/system_wrappers/interface/logging.h"
16
17#ifdef WEBRTC_MODULE_UTILITY_VIDEO
18    #include "critical_section_wrapper.h"
19    #include "frame_scaler.h"
20    #include "video_coder.h"
21    #include "video_frames_queue.h"
22#endif
23
24namespace webrtc {
25FileRecorder* FileRecorder::CreateFileRecorder(uint32_t instanceID,
26                                               FileFormats fileFormat)
27{
28    switch(fileFormat)
29    {
30    case kFileFormatWavFile:
31    case kFileFormatCompressedFile:
32    case kFileFormatPreencodedFile:
33    case kFileFormatPcm16kHzFile:
34    case kFileFormatPcm8kHzFile:
35    case kFileFormatPcm32kHzFile:
36        return new FileRecorderImpl(instanceID, fileFormat);
37    case kFileFormatAviFile:
38#ifdef WEBRTC_MODULE_UTILITY_VIDEO
39        return new AviRecorder(instanceID, fileFormat);
40#else
41        assert(false);
42        return NULL;
43#endif
44    }
45    assert(false);
46    return NULL;
47}
48
49void FileRecorder::DestroyFileRecorder(FileRecorder* recorder)
50{
51    delete recorder;
52}
53
54FileRecorderImpl::FileRecorderImpl(uint32_t instanceID,
55                                   FileFormats fileFormat)
56    : _instanceID(instanceID),
57      _fileFormat(fileFormat),
58      _moduleFile(MediaFile::CreateMediaFile(_instanceID)),
59      codec_info_(),
60      _amrFormat(AMRFileStorage),
61      _audioBuffer(),
62      _audioEncoder(instanceID),
63      _audioResampler()
64{
65}
66
67FileRecorderImpl::~FileRecorderImpl()
68{
69    MediaFile::DestroyMediaFile(_moduleFile);
70}
71
72FileFormats FileRecorderImpl::RecordingFileFormat() const
73{
74    return _fileFormat;
75}
76
77int32_t FileRecorderImpl::RegisterModuleFileCallback(
78    FileCallback* callback)
79{
80    if(_moduleFile == NULL)
81    {
82        return -1;
83    }
84    return _moduleFile->SetModuleFileCallback(callback);
85}
86
87int32_t FileRecorderImpl::StartRecordingAudioFile(
88    const char* fileName,
89    const CodecInst& codecInst,
90    uint32_t notificationTimeMs,
91    ACMAMRPackingFormat amrFormat)
92{
93    if(_moduleFile == NULL)
94    {
95        return -1;
96    }
97    codec_info_ = codecInst;
98    _amrFormat = amrFormat;
99
100    int32_t retVal = 0;
101    if(_fileFormat != kFileFormatAviFile)
102    {
103        // AVI files should be started using StartRecordingVideoFile(..) all
104        // other formats should use this API.
105        retVal =_moduleFile->StartRecordingAudioFile(fileName, _fileFormat,
106                                                     codecInst,
107                                                     notificationTimeMs);
108    }
109
110    if( retVal == 0)
111    {
112        retVal = SetUpAudioEncoder();
113    }
114    if( retVal != 0)
115    {
116        LOG(LS_WARNING) << "Failed to initialize file " << fileName
117                        << " for recording.";
118
119        if(IsRecording())
120        {
121            StopRecording();
122        }
123    }
124    return retVal;
125}
126
127int32_t FileRecorderImpl::StartRecordingAudioFile(
128    OutStream& destStream,
129    const CodecInst& codecInst,
130    uint32_t notificationTimeMs,
131    ACMAMRPackingFormat amrFormat)
132{
133    codec_info_ = codecInst;
134    _amrFormat = amrFormat;
135
136    int32_t retVal = _moduleFile->StartRecordingAudioStream(
137        destStream,
138        _fileFormat,
139        codecInst,
140        notificationTimeMs);
141
142    if( retVal == 0)
143    {
144        retVal = SetUpAudioEncoder();
145    }
146    if( retVal != 0)
147    {
148        LOG(LS_WARNING) << "Failed to initialize outStream for recording.";
149
150        if(IsRecording())
151        {
152            StopRecording();
153        }
154    }
155    return retVal;
156}
157
158int32_t FileRecorderImpl::StopRecording()
159{
160    memset(&codec_info_, 0, sizeof(CodecInst));
161    return _moduleFile->StopRecording();
162}
163
164bool FileRecorderImpl::IsRecording() const
165{
166    return _moduleFile->IsRecording();
167}
168
169int32_t FileRecorderImpl::RecordAudioToFile(
170    const AudioFrame& incomingAudioFrame,
171    const TickTime* playoutTS)
172{
173    if (codec_info_.plfreq == 0)
174    {
175        LOG(LS_WARNING) << "RecordAudioToFile() recording audio is not "
176                        << "turned on.";
177        return -1;
178    }
179    AudioFrame tempAudioFrame;
180    tempAudioFrame.samples_per_channel_ = 0;
181    if( incomingAudioFrame.num_channels_ == 2 &&
182        !_moduleFile->IsStereo())
183    {
184        // Recording mono but incoming audio is (interleaved) stereo.
185        tempAudioFrame.num_channels_ = 1;
186        tempAudioFrame.sample_rate_hz_ = incomingAudioFrame.sample_rate_hz_;
187        tempAudioFrame.samples_per_channel_ =
188          incomingAudioFrame.samples_per_channel_;
189        for (uint16_t i = 0;
190             i < (incomingAudioFrame.samples_per_channel_); i++)
191        {
192            // Sample value is the average of left and right buffer rounded to
193            // closest integer value. Note samples can be either 1 or 2 byte.
194             tempAudioFrame.data_[i] =
195                 ((incomingAudioFrame.data_[2 * i] +
196                   incomingAudioFrame.data_[(2 * i) + 1] + 1) >> 1);
197        }
198    }
199    else if( incomingAudioFrame.num_channels_ == 1 &&
200        _moduleFile->IsStereo())
201    {
202        // Recording stereo but incoming audio is mono.
203        tempAudioFrame.num_channels_ = 2;
204        tempAudioFrame.sample_rate_hz_ = incomingAudioFrame.sample_rate_hz_;
205        tempAudioFrame.samples_per_channel_ =
206          incomingAudioFrame.samples_per_channel_;
207        for (uint16_t i = 0;
208             i < (incomingAudioFrame.samples_per_channel_); i++)
209        {
210            // Duplicate sample to both channels
211             tempAudioFrame.data_[2*i] =
212               incomingAudioFrame.data_[i];
213             tempAudioFrame.data_[2*i+1] =
214               incomingAudioFrame.data_[i];
215        }
216    }
217
218    const AudioFrame* ptrAudioFrame = &incomingAudioFrame;
219    if(tempAudioFrame.samples_per_channel_ != 0)
220    {
221        // If ptrAudioFrame is not empty it contains the audio to be recorded.
222        ptrAudioFrame = &tempAudioFrame;
223    }
224
225    // Encode the audio data before writing to file. Don't encode if the codec
226    // is PCM.
227    // NOTE: stereo recording is only supported for WAV files.
228    // TODO (hellner): WAV expect PCM in little endian byte order. Not
229    // "encoding" with PCM coder should be a problem for big endian systems.
230    uint32_t encodedLenInBytes = 0;
231    if (_fileFormat == kFileFormatPreencodedFile ||
232        STR_CASE_CMP(codec_info_.plname, "L16") != 0)
233    {
234        if (_audioEncoder.Encode(*ptrAudioFrame, _audioBuffer,
235                                 encodedLenInBytes) == -1)
236        {
237            LOG(LS_WARNING) << "RecordAudioToFile() codec "
238                            << codec_info_.plname
239                            << " not supported or failed to encode stream.";
240            return -1;
241        }
242    } else {
243        int outLen = 0;
244        if(ptrAudioFrame->num_channels_ == 2)
245        {
246            // ptrAudioFrame contains interleaved stereo audio.
247            _audioResampler.ResetIfNeeded(ptrAudioFrame->sample_rate_hz_,
248                                          codec_info_.plfreq,
249                                          kResamplerSynchronousStereo);
250            _audioResampler.Push(ptrAudioFrame->data_,
251                                 ptrAudioFrame->samples_per_channel_ *
252                                 ptrAudioFrame->num_channels_,
253                                 (int16_t*)_audioBuffer,
254                                 MAX_AUDIO_BUFFER_IN_BYTES, outLen);
255        } else {
256            _audioResampler.ResetIfNeeded(ptrAudioFrame->sample_rate_hz_,
257                                          codec_info_.plfreq,
258                                          kResamplerSynchronous);
259            _audioResampler.Push(ptrAudioFrame->data_,
260                                 ptrAudioFrame->samples_per_channel_,
261                                 (int16_t*)_audioBuffer,
262                                 MAX_AUDIO_BUFFER_IN_BYTES, outLen);
263        }
264        encodedLenInBytes = outLen * sizeof(int16_t);
265    }
266
267    // Codec may not be operating at a frame rate of 10 ms. Whenever enough
268    // 10 ms chunks of data has been pushed to the encoder an encoded frame
269    // will be available. Wait until then.
270    if (encodedLenInBytes)
271    {
272        uint16_t msOfData =
273            ptrAudioFrame->samples_per_channel_ /
274            uint16_t(ptrAudioFrame->sample_rate_hz_ / 1000);
275        if (WriteEncodedAudioData(_audioBuffer,
276                                  (uint16_t)encodedLenInBytes,
277                                  msOfData, playoutTS) == -1)
278        {
279            return -1;
280        }
281    }
282    return 0;
283}
284
285int32_t FileRecorderImpl::SetUpAudioEncoder()
286{
287    if (_fileFormat == kFileFormatPreencodedFile ||
288        STR_CASE_CMP(codec_info_.plname, "L16") != 0)
289    {
290        if(_audioEncoder.SetEncodeCodec(codec_info_,_amrFormat) == -1)
291        {
292            LOG(LS_ERROR) << "SetUpAudioEncoder() codec "
293                          << codec_info_.plname << " not supported.";
294            return -1;
295        }
296    }
297    return 0;
298}
299
300int32_t FileRecorderImpl::codec_info(CodecInst& codecInst) const
301{
302    if(codec_info_.plfreq == 0)
303    {
304        return -1;
305    }
306    codecInst = codec_info_;
307    return 0;
308}
309
310int32_t FileRecorderImpl::WriteEncodedAudioData(
311    const int8_t* audioBuffer,
312    uint16_t bufferLength,
313    uint16_t /*millisecondsOfData*/,
314    const TickTime* /*playoutTS*/)
315{
316    return _moduleFile->IncomingAudioData(audioBuffer, bufferLength);
317}
318
319
320#ifdef WEBRTC_MODULE_UTILITY_VIDEO
321AviRecorder::AviRecorder(uint32_t instanceID, FileFormats fileFormat)
322    : FileRecorderImpl(instanceID, fileFormat),
323      _videoOnly(false),
324      _thread( 0),
325      _timeEvent(*EventWrapper::Create()),
326      _critSec(CriticalSectionWrapper::CreateCriticalSection()),
327      _writtenVideoFramesCounter(0),
328      _writtenAudioMS(0),
329      _writtenVideoMS(0)
330{
331    _videoEncoder = new VideoCoder();
332    _frameScaler = new FrameScaler();
333    _videoFramesQueue = new VideoFramesQueue();
334    _thread = ThreadWrapper::CreateThread(Run, this, kNormalPriority,
335                                          "AviRecorder()");
336}
337
338AviRecorder::~AviRecorder( )
339{
340    StopRecording( );
341
342    delete _videoEncoder;
343    delete _frameScaler;
344    delete _videoFramesQueue;
345    delete _thread;
346    delete &_timeEvent;
347    delete _critSec;
348}
349
350int32_t AviRecorder::StartRecordingVideoFile(
351    const char* fileName,
352    const CodecInst& audioCodecInst,
353    const VideoCodec& videoCodecInst,
354    ACMAMRPackingFormat amrFormat,
355    bool videoOnly)
356{
357    _firstAudioFrameReceived = false;
358    _videoCodecInst = videoCodecInst;
359    _videoOnly = videoOnly;
360
361    if(_moduleFile->StartRecordingVideoFile(fileName, _fileFormat,
362                                            audioCodecInst, videoCodecInst,
363                                            videoOnly) != 0)
364    {
365        return -1;
366    }
367
368    if(!videoOnly)
369    {
370        if(FileRecorderImpl::StartRecordingAudioFile(fileName,audioCodecInst, 0,
371                                                     amrFormat) !=0)
372        {
373            StopRecording();
374            return -1;
375        }
376    }
377    if( SetUpVideoEncoder() != 0)
378    {
379        StopRecording();
380        return -1;
381    }
382    if(_videoOnly)
383    {
384        // Writing to AVI file is non-blocking.
385        // Start non-blocking timer if video only. If recording both video and
386        // audio let the pushing of audio frames be the timer.
387        _timeEvent.StartTimer(true, 1000 / _videoCodecInst.maxFramerate);
388    }
389    StartThread();
390    return 0;
391}
392
393int32_t AviRecorder::StopRecording()
394{
395    _timeEvent.StopTimer();
396
397    StopThread();
398    return FileRecorderImpl::StopRecording();
399}
400
401int32_t AviRecorder::CalcI420FrameSize( ) const
402{
403    return 3 * _videoCodecInst.width * _videoCodecInst.height / 2;
404}
405
406int32_t AviRecorder::SetUpVideoEncoder()
407{
408    // Size of unencoded data (I420) should be the largest possible frame size
409    // in a file.
410    _videoMaxPayloadSize = CalcI420FrameSize();
411    _videoEncodedData.VerifyAndAllocate(_videoMaxPayloadSize);
412
413    _videoCodecInst.plType = _videoEncoder->DefaultPayloadType(
414        _videoCodecInst.plName);
415
416    int32_t useNumberOfCores = 1;
417    // Set the max payload size to 16000. This means that the codec will try to
418    // create slices that will fit in 16000 kByte packets. However, the
419    // Encode() call will still generate one full frame.
420    if(_videoEncoder->SetEncodeCodec(_videoCodecInst, useNumberOfCores,
421                                     16000))
422    {
423        return -1;
424    }
425    return 0;
426}
427
428int32_t AviRecorder::RecordVideoToFile(const I420VideoFrame& videoFrame)
429{
430    CriticalSectionScoped lock(_critSec);
431    if(!IsRecording() || videoFrame.IsZeroSize())
432    {
433        return -1;
434    }
435    // The frame is written to file in AviRecorder::Process().
436    int32_t retVal = _videoFramesQueue->AddFrame(videoFrame);
437    if(retVal != 0)
438    {
439        StopRecording();
440    }
441    return retVal;
442}
443
444bool AviRecorder::StartThread()
445{
446    unsigned int id;
447    if( _thread == 0)
448    {
449        return false;
450    }
451
452    return _thread->Start(id);
453}
454
455bool AviRecorder::StopThread()
456{
457    _critSec->Enter();
458
459    if(_thread)
460    {
461        _thread->SetNotAlive();
462
463        ThreadWrapper* thread = _thread;
464        _thread = NULL;
465
466        _timeEvent.Set();
467
468        _critSec->Leave();
469
470        if(thread->Stop())
471        {
472            delete thread;
473        } else {
474            return false;
475        }
476    } else {
477        _critSec->Leave();
478    }
479    return true;
480}
481
482bool AviRecorder::Run( ThreadObj threadObj)
483{
484    return static_cast<AviRecorder*>( threadObj)->Process();
485}
486
487int32_t AviRecorder::ProcessAudio()
488{
489    if (_writtenVideoFramesCounter == 0)
490    {
491        // Get the most recent frame that is due for writing to file. Since
492        // frames are unencoded it's safe to throw away frames if necessary
493        // for synchronizing audio and video.
494        I420VideoFrame* frameToProcess = _videoFramesQueue->FrameToRecord();
495        if(frameToProcess)
496        {
497            // Syncronize audio to the current frame to process by throwing away
498            // audio samples with older timestamp than the video frame.
499            size_t numberOfAudioElements =
500                _audioFramesToWrite.size();
501            for (size_t i = 0; i < numberOfAudioElements; ++i)
502            {
503                AudioFrameFileInfo* frameInfo = _audioFramesToWrite.front();
504                if(TickTime::TicksToMilliseconds(
505                       frameInfo->_playoutTS.Ticks()) <
506                   frameToProcess->render_time_ms())
507                {
508                    delete frameInfo;
509                    _audioFramesToWrite.pop_front();
510                } else
511                {
512                    break;
513                }
514            }
515        }
516    }
517    // Write all audio up to current timestamp.
518    int32_t error = 0;
519    size_t numberOfAudioElements = _audioFramesToWrite.size();
520    for (size_t i = 0; i < numberOfAudioElements; ++i)
521    {
522        AudioFrameFileInfo* frameInfo = _audioFramesToWrite.front();
523        if((TickTime::Now() - frameInfo->_playoutTS).Milliseconds() > 0)
524        {
525            _moduleFile->IncomingAudioData(frameInfo->_audioData,
526                                           frameInfo->_audioSize);
527            _writtenAudioMS += frameInfo->_audioMS;
528            delete frameInfo;
529            _audioFramesToWrite.pop_front();
530        } else {
531            break;
532        }
533    }
534    return error;
535}
536
537bool AviRecorder::Process()
538{
539    switch(_timeEvent.Wait(500))
540    {
541    case kEventSignaled:
542        if(_thread == NULL)
543        {
544            return false;
545        }
546        break;
547    case kEventError:
548        return false;
549    case kEventTimeout:
550        // No events triggered. No work to do.
551        return true;
552    }
553    CriticalSectionScoped lock( _critSec);
554
555    // Get the most recent frame to write to file (if any). Synchronize it with
556    // the audio stream (if any). Synchronization the video based on its render
557    // timestamp (i.e. VideoFrame::RenderTimeMS())
558    I420VideoFrame* frameToProcess = _videoFramesQueue->FrameToRecord();
559    if( frameToProcess == NULL)
560    {
561        return true;
562    }
563    int32_t error = 0;
564    if(!_videoOnly)
565    {
566        if(!_firstAudioFrameReceived)
567        {
568            // Video and audio can only be synchronized if both have been
569            // received.
570            return true;
571        }
572        error = ProcessAudio();
573
574        while (_writtenAudioMS > _writtenVideoMS)
575        {
576            error = EncodeAndWriteVideoToFile( *frameToProcess);
577            if( error != 0)
578            {
579                LOG(LS_ERROR) << "AviRecorder::Process() error writing to "
580                              << "file.";
581                break;
582            } else {
583                uint32_t frameLengthMS = 1000 /
584                    _videoCodecInst.maxFramerate;
585                _writtenVideoFramesCounter++;
586                _writtenVideoMS += frameLengthMS;
587                // A full seconds worth of frames have been written.
588                if(_writtenVideoFramesCounter%_videoCodecInst.maxFramerate == 0)
589                {
590                    // Frame rate is in frames per seconds. Frame length is
591                    // calculated as an integer division which means it may
592                    // be rounded down. Compensate for this every second.
593                    uint32_t rest = 1000 % frameLengthMS;
594                    _writtenVideoMS += rest;
595                }
596            }
597        }
598    } else {
599        // Frame rate is in frames per seconds. Frame length is calculated as an
600        // integer division which means it may be rounded down. This introduces
601        // drift. Once a full frame worth of drift has happened, skip writing
602        // one frame. Note that frame rate is in frames per second so the
603        // drift is completely compensated for.
604        uint32_t frameLengthMS = 1000/_videoCodecInst.maxFramerate;
605        uint32_t restMS = 1000 % frameLengthMS;
606        uint32_t frameSkip = (_videoCodecInst.maxFramerate *
607                              frameLengthMS) / restMS;
608
609        _writtenVideoFramesCounter++;
610        if(_writtenVideoFramesCounter % frameSkip == 0)
611        {
612            _writtenVideoMS += frameLengthMS;
613            return true;
614        }
615
616        error = EncodeAndWriteVideoToFile( *frameToProcess);
617        if(error != 0)
618        {
619            LOG(LS_ERROR) << "AviRecorder::Process() error writing to file.";
620        } else {
621            _writtenVideoMS += frameLengthMS;
622        }
623    }
624    return error == 0;
625}
626
627int32_t AviRecorder::EncodeAndWriteVideoToFile(I420VideoFrame& videoFrame)
628{
629    if (!IsRecording() || videoFrame.IsZeroSize())
630    {
631        return -1;
632    }
633
634    if(_frameScaler->ResizeFrameIfNeeded(&videoFrame, _videoCodecInst.width,
635                                         _videoCodecInst.height) != 0)
636    {
637        return -1;
638    }
639
640    _videoEncodedData.payloadSize = 0;
641
642    if( STR_CASE_CMP(_videoCodecInst.plName, "I420") == 0)
643    {
644       int length  = CalcBufferSize(kI420, videoFrame.width(),
645                                    videoFrame.height());
646        _videoEncodedData.VerifyAndAllocate(length);
647
648        // I420 is raw data. No encoding needed (each sample is represented by
649        // 1 byte so there is no difference depending on endianness).
650        int ret_length = ExtractBuffer(videoFrame, length,
651                                       _videoEncodedData.payloadData);
652        if (ret_length < 0)
653          return -1;
654
655        _videoEncodedData.payloadSize = ret_length;
656        _videoEncodedData.frameType = kVideoFrameKey;
657    }else {
658        if( _videoEncoder->Encode(videoFrame, _videoEncodedData) != 0)
659        {
660            return -1;
661        }
662    }
663
664    if(_videoEncodedData.payloadSize > 0)
665    {
666        if(_moduleFile->IncomingAVIVideoData(
667               (int8_t*)(_videoEncodedData.payloadData),
668               _videoEncodedData.payloadSize))
669        {
670            LOG(LS_ERROR) << "Error writing AVI file.";
671            return -1;
672        }
673    } else {
674        LOG(LS_ERROR) << "FileRecorder::RecordVideoToFile() frame dropped by "
675                      << "encoder, bitrate likely too low.";
676    }
677    return 0;
678}
679
680// Store audio frame in the _audioFramesToWrite buffer. The writing to file
681// happens in AviRecorder::Process().
682int32_t AviRecorder::WriteEncodedAudioData(
683    const int8_t* audioBuffer,
684    uint16_t bufferLength,
685    uint16_t millisecondsOfData,
686    const TickTime* playoutTS)
687{
688    CriticalSectionScoped lock(_critSec);
689
690    if (!IsRecording())
691    {
692        return -1;
693    }
694    if (bufferLength > MAX_AUDIO_BUFFER_IN_BYTES)
695    {
696        return -1;
697    }
698    if (_videoOnly)
699    {
700        return -1;
701    }
702    if (_audioFramesToWrite.size() > kMaxAudioBufferQueueLength)
703    {
704        StopRecording();
705        return -1;
706    }
707    _firstAudioFrameReceived = true;
708
709    if(playoutTS)
710    {
711        _audioFramesToWrite.push_back(new AudioFrameFileInfo(audioBuffer,
712                                                             bufferLength,
713                                                             millisecondsOfData,
714                                                             *playoutTS));
715    } else {
716        _audioFramesToWrite.push_back(new AudioFrameFileInfo(audioBuffer,
717                                                             bufferLength,
718                                                             millisecondsOfData,
719                                                             TickTime::Now()));
720    }
721    _timeEvent.Set();
722    return 0;
723}
724
725#endif // WEBRTC_MODULE_UTILITY_VIDEO
726}  // namespace webrtc
727