19ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdoch// Copyright 2013 The Chromium Authors. All rights reserved. 29ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdoch// Use of this source code is governed by a BSD-style license that can be 39ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdoch// found in the LICENSE file. 49ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdoch 59ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdoch#include "remoting/client/jni/jni_frame_consumer.h" 69ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdoch 79ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdoch#include "base/android/jni_android.h" 89ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdoch#include "base/logging.h" 94e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)#include "base/stl_util.h" 109ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdoch#include "base/synchronization/waitable_event.h" 114e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)#include "remoting/base/util.h" 129ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdoch#include "remoting/client/frame_producer.h" 13f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)#include "remoting/client/jni/chromoting_jni_instance.h" 14bbcdd45c55eb7c4641ab97aef9889b0fc828e7d3Ben Murdoch#include "remoting/client/jni/chromoting_jni_runtime.h" 159ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdoch#include "third_party/webrtc/modules/desktop_capture/desktop_frame.h" 16d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles)#include "third_party/webrtc/modules/desktop_capture/desktop_region.h" 174e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)#include "ui/gfx/android/java_bitmap.h" 189ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdoch 199ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdochnamespace remoting { 209ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdoch 21f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)JniFrameConsumer::JniFrameConsumer( 22f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) ChromotingJniRuntime* jni_runtime, 23f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) scoped_refptr<ChromotingJniInstance> jni_instance) 24bbcdd45c55eb7c4641ab97aef9889b0fc828e7d3Ben Murdoch : jni_runtime_(jni_runtime), 25f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) jni_instance_(jni_instance), 269ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdoch frame_producer_(NULL) { 279ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdoch} 289ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdoch 299ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben MurdochJniFrameConsumer::~JniFrameConsumer() { 304e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) // The producer should now return any pending buffers. At this point, however, 314e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) // ReturnBuffer() tasks scheduled by the producer will not be delivered, 324e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) // so we free all the buffers once the producer's queue is empty. 339ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdoch base::WaitableEvent done_event(true, false); 349ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdoch frame_producer_->RequestReturnBuffers( 359ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdoch base::Bind(&base::WaitableEvent::Signal, base::Unretained(&done_event))); 369ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdoch done_event.Wait(); 374e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 384e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) STLDeleteElements(&buffers_); 399ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdoch} 409ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdoch 419ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdochvoid JniFrameConsumer::set_frame_producer(FrameProducer* producer) { 429ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdoch frame_producer_ = producer; 439ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdoch} 449ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdoch 45d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles)void JniFrameConsumer::ApplyBuffer(const webrtc::DesktopSize& view_size, 46d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) const webrtc::DesktopRect& clip_area, 479ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdoch webrtc::DesktopFrame* buffer, 485d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) const webrtc::DesktopRegion& region, 495d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) const webrtc::DesktopRegion& shape) { 50bbcdd45c55eb7c4641ab97aef9889b0fc828e7d3Ben Murdoch DCHECK(jni_runtime_->display_task_runner()->BelongsToCurrentThread()); 519ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdoch 525d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if (bitmap_->size().width() != buffer->size().width() || 535d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) bitmap_->size().height() != buffer->size().height()) { 544e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) // Drop the frame, since the data belongs to the previous generation, 554e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) // before SetSourceSize() called SetOutputSizeAndClip(). 564e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) FreeBuffer(buffer); 574e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) return; 584e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) } 59bbcdd45c55eb7c4641ab97aef9889b0fc828e7d3Ben Murdoch 604e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) // Copy pixels from |buffer| into the Java Bitmap. 614e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) // TODO(lambroslambrou): Optimize away this copy by having the VideoDecoder 624e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) // decode directly into the Bitmap's pixel memory. This currently doesn't 634e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) // work very well because the VideoDecoder writes the decoded data in BGRA, 644e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) // and then the R/B channels are swapped in place (on the decoding thread). 654e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) // If a repaint is triggered from a Java event handler, the unswapped pixels 664e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) // can sometimes appear on the display. 674e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) uint8* dest_buffer = static_cast<uint8*>(bitmap_->pixels()); 684e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) webrtc::DesktopRect buffer_rect = webrtc::DesktopRect::MakeSize(view_size); 694e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 704e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) for (webrtc::DesktopRegion::Iterator i(region); !i.IsAtEnd(); i.Advance()) { 714e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) const webrtc::DesktopRect& rect(i.rect()); 724e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) CopyRGB32Rect(buffer->data(), buffer->stride(), buffer_rect, dest_buffer, 734e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) bitmap_->stride(), buffer_rect, rect); 749ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdoch } 759ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdoch 764e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) // TODO(lambroslambrou): Optimize this by only repainting the changed pixels. 77f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) base::TimeTicks start_time = base::TimeTicks::Now(); 784e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) jni_runtime_->RedrawCanvas(); 79f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) jni_instance_->RecordPaintTime( 80f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) (base::TimeTicks::Now() - start_time).InMilliseconds()); 814e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 829ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdoch // Supply |frame_producer_| with a buffer to render the next frame into. 834e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) frame_producer_->DrawBuffer(buffer); 849ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdoch} 859ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdoch 869ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdochvoid JniFrameConsumer::ReturnBuffer(webrtc::DesktopFrame* buffer) { 87bbcdd45c55eb7c4641ab97aef9889b0fc828e7d3Ben Murdoch DCHECK(jni_runtime_->display_task_runner()->BelongsToCurrentThread()); 884e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) FreeBuffer(buffer); 899ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdoch} 909ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdoch 91d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles)void JniFrameConsumer::SetSourceSize(const webrtc::DesktopSize& source_size, 92d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) const webrtc::DesktopVector& dpi) { 93bbcdd45c55eb7c4641ab97aef9889b0fc828e7d3Ben Murdoch DCHECK(jni_runtime_->display_task_runner()->BelongsToCurrentThread()); 949ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdoch 959ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdoch // We currently render the desktop 1:1 and perform pan/zoom scaling 969ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdoch // and cropping on the managed canvas. 975d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) clip_area_ = webrtc::DesktopRect::MakeSize(source_size); 985d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) frame_producer_->SetOutputSizeAndClip(source_size, clip_area_); 999ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdoch 1004e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) // Allocate buffer and start drawing frames onto it. 1015d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) AllocateBuffer(source_size); 1024e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)} 1034e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 1044e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)FrameConsumer::PixelFormat JniFrameConsumer::GetPixelFormat() { 1054e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) return FORMAT_RGBA; 1069ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdoch} 1079ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdoch 1085d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)void JniFrameConsumer::AllocateBuffer(const webrtc::DesktopSize& source_size) { 1094e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) DCHECK(jni_runtime_->display_task_runner()->BelongsToCurrentThread()); 1104e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 1115d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) webrtc::DesktopSize size(source_size.width(), source_size.height()); 1124e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 1134e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) // Allocate a new Bitmap, store references here, and pass it to Java. 1144e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) JNIEnv* env = base::android::AttachCurrentThread(); 1154e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 1164e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) // |bitmap_| must be deleted before |bitmap_global_ref_| is released. 1174e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) bitmap_.reset(); 1184e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) bitmap_global_ref_.Reset(env, jni_runtime_->NewBitmap(size).obj()); 1194e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) bitmap_.reset(new gfx::JavaBitmap(bitmap_global_ref_.obj())); 1204e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) jni_runtime_->UpdateFrameBitmap(bitmap_global_ref_.obj()); 1214e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 1224e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) webrtc::DesktopFrame* buffer = new webrtc::BasicDesktopFrame(size); 1234e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) buffers_.push_back(buffer); 1244e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) frame_producer_->DrawBuffer(buffer); 1254e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)} 1264e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 1274e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)void JniFrameConsumer::FreeBuffer(webrtc::DesktopFrame* buffer) { 1284e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) DCHECK(std::find(buffers_.begin(), buffers_.end(), buffer) != buffers_.end()); 1294e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 1304e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) buffers_.remove(buffer); 1314e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) delete buffer; 1329ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdoch} 1339ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdoch 1349ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdoch} // namespace remoting 135