1// Copyright 2014 The Chromium Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5#include <GLES2/gl2.h> 6#include <GLES2/gl2ext.h> 7#include <string.h> 8 9#include <vector> 10 11#include "ppapi/c/pp_errors.h" 12#include "ppapi/c/ppb_opengles2.h" 13#include "ppapi/cpp/completion_callback.h" 14#include "ppapi/cpp/graphics_3d.h" 15#include "ppapi/cpp/graphics_3d_client.h" 16#include "ppapi/cpp/instance.h" 17#include "ppapi/cpp/media_stream_video_track.h" 18#include "ppapi/cpp/module.h" 19#include "ppapi/cpp/rect.h" 20#include "ppapi/cpp/var.h" 21#include "ppapi/cpp/var_dictionary.h" 22#include "ppapi/cpp/video_frame.h" 23#include "ppapi/lib/gl/gles2/gl2ext_ppapi.h" 24#include "ppapi/utility/completion_callback_factory.h" 25 26// When compiling natively on Windows, PostMessage can be #define-d to 27// something else. 28#ifdef PostMessage 29#undef PostMessage 30#endif 31 32// Assert |context_| isn't holding any GL Errors. Done as a macro instead of a 33// function to preserve line number information in the failure message. 34#define AssertNoGLError() \ 35 PP_DCHECK(!glGetError()); 36 37namespace { 38 39// This object is the global object representing this plugin library as long 40// as it is loaded. 41class MediaStreamVideoModule : public pp::Module { 42 public: 43 MediaStreamVideoModule() : pp::Module() {} 44 virtual ~MediaStreamVideoModule() {} 45 46 virtual pp::Instance* CreateInstance(PP_Instance instance); 47}; 48 49class MediaStreamVideoDemoInstance : public pp::Instance, 50 public pp::Graphics3DClient { 51 public: 52 MediaStreamVideoDemoInstance(PP_Instance instance, pp::Module* module); 53 virtual ~MediaStreamVideoDemoInstance(); 54 55 // pp::Instance implementation (see PPP_Instance). 56 virtual void DidChangeView(const pp::Rect& position, 57 const pp::Rect& clip_ignored); 58 virtual void HandleMessage(const pp::Var& message_data); 59 60 // pp::Graphics3DClient implementation. 61 virtual void Graphics3DContextLost() { 62 InitGL(); 63 CreateTextures(); 64 Render(); 65 } 66 67 private: 68 void DrawYUV(); 69 void DrawRGB(); 70 void Render(); 71 72 // GL-related functions. 73 void InitGL(); 74 GLuint CreateTexture(int32_t width, int32_t height, int unit, bool rgba); 75 void CreateGLObjects(); 76 void CreateShader(GLuint program, GLenum type, const char* source); 77 void PaintFinished(int32_t result); 78 void CreateTextures(); 79 void ConfigureTrack(); 80 81 82 // MediaStreamVideoTrack callbacks. 83 void OnConfigure(int32_t result); 84 void OnGetFrame(int32_t result, pp::VideoFrame frame); 85 86 pp::Size position_size_; 87 bool is_painting_; 88 bool needs_paint_; 89 bool is_bgra_; 90 GLuint program_yuv_; 91 GLuint program_rgb_; 92 GLuint buffer_; 93 GLuint texture_y_; 94 GLuint texture_u_; 95 GLuint texture_v_; 96 GLuint texture_rgb_; 97 pp::MediaStreamVideoTrack video_track_; 98 pp::CompletionCallbackFactory<MediaStreamVideoDemoInstance> callback_factory_; 99 std::vector<int32_t> attrib_list_; 100 101 // MediaStreamVideoTrack attributes: 102 bool need_config_; 103 PP_VideoFrame_Format attrib_format_; 104 int32_t attrib_width_; 105 int32_t attrib_height_; 106 107 // Owned data. 108 pp::Graphics3D* context_; 109 110 pp::Size frame_size_; 111}; 112 113MediaStreamVideoDemoInstance::MediaStreamVideoDemoInstance( 114 PP_Instance instance, pp::Module* module) 115 : pp::Instance(instance), 116 pp::Graphics3DClient(this), 117 is_painting_(false), 118 needs_paint_(false), 119 is_bgra_(false), 120 texture_y_(0), 121 texture_u_(0), 122 texture_v_(0), 123 texture_rgb_(0), 124 callback_factory_(this), 125 need_config_(false), 126 attrib_format_(PP_VIDEOFRAME_FORMAT_I420), 127 attrib_width_(0), 128 attrib_height_(0), 129 context_(NULL) { 130 if (!glInitializePPAPI(pp::Module::Get()->get_browser_interface())) { 131 LogToConsole(PP_LOGLEVEL_ERROR, pp::Var("Unable to initialize GL PPAPI!")); 132 assert(false); 133 } 134} 135 136MediaStreamVideoDemoInstance::~MediaStreamVideoDemoInstance() { 137 delete context_; 138} 139 140void MediaStreamVideoDemoInstance::DidChangeView( 141 const pp::Rect& position, const pp::Rect& clip_ignored) { 142 if (position.width() == 0 || position.height() == 0) 143 return; 144 if (position.size() == position_size_) 145 return; 146 147 position_size_ = position.size(); 148 149 // Initialize graphics. 150 InitGL(); 151 Render(); 152} 153 154void MediaStreamVideoDemoInstance::HandleMessage(const pp::Var& var_message) { 155 if (!var_message.is_dictionary()) { 156 LogToConsole(PP_LOGLEVEL_ERROR, pp::Var("Invalid message!")); 157 return; 158 } 159 160 pp::VarDictionary var_dictionary_message(var_message); 161 std::string command = var_dictionary_message.Get("command").AsString(); 162 163 if (command == "init") { 164 pp::Var var_track = var_dictionary_message.Get("track"); 165 if (!var_track.is_resource()) 166 return; 167 pp::Resource resource_track = var_track.AsResource(); 168 video_track_ = pp::MediaStreamVideoTrack(resource_track); 169 ConfigureTrack(); 170 } else if (command == "format") { 171 std::string str_format = var_dictionary_message.Get("format").AsString(); 172 if (str_format == "YV12") { 173 attrib_format_ = PP_VIDEOFRAME_FORMAT_YV12; 174 } else if (str_format == "I420") { 175 attrib_format_ = PP_VIDEOFRAME_FORMAT_I420; 176 } else if (str_format == "BGRA") { 177 attrib_format_ = PP_VIDEOFRAME_FORMAT_BGRA; 178 } else { 179 attrib_format_ = PP_VIDEOFRAME_FORMAT_UNKNOWN; 180 } 181 need_config_ = true; 182 } else if (command == "size") { 183 attrib_width_ = var_dictionary_message.Get("width").AsInt(); 184 attrib_height_ = var_dictionary_message.Get("height").AsInt(); 185 need_config_ = true; 186 } else { 187 LogToConsole(PP_LOGLEVEL_ERROR, pp::Var("Invalid command!")); 188 } 189} 190 191void MediaStreamVideoDemoInstance::InitGL() { 192 PP_DCHECK(position_size_.width() && position_size_.height()); 193 is_painting_ = false; 194 195 delete context_; 196 int32_t attributes[] = { 197 PP_GRAPHICS3DATTRIB_ALPHA_SIZE, 0, 198 PP_GRAPHICS3DATTRIB_BLUE_SIZE, 8, 199 PP_GRAPHICS3DATTRIB_GREEN_SIZE, 8, 200 PP_GRAPHICS3DATTRIB_RED_SIZE, 8, 201 PP_GRAPHICS3DATTRIB_DEPTH_SIZE, 0, 202 PP_GRAPHICS3DATTRIB_STENCIL_SIZE, 0, 203 PP_GRAPHICS3DATTRIB_SAMPLES, 0, 204 PP_GRAPHICS3DATTRIB_SAMPLE_BUFFERS, 0, 205 PP_GRAPHICS3DATTRIB_WIDTH, position_size_.width(), 206 PP_GRAPHICS3DATTRIB_HEIGHT, position_size_.height(), 207 PP_GRAPHICS3DATTRIB_NONE, 208 }; 209 context_ = new pp::Graphics3D(this, attributes); 210 PP_DCHECK(!context_->is_null()); 211 212 glSetCurrentContextPPAPI(context_->pp_resource()); 213 214 // Set viewport window size and clear color bit. 215 glClearColor(1, 0, 0, 1); 216 glClear(GL_COLOR_BUFFER_BIT); 217 glViewport(0, 0, position_size_.width(), position_size_.height()); 218 219 BindGraphics(*context_); 220 AssertNoGLError(); 221 222 CreateGLObjects(); 223} 224 225void MediaStreamVideoDemoInstance::DrawYUV() { 226 static const float kColorMatrix[9] = { 227 1.1643828125f, 1.1643828125f, 1.1643828125f, 228 0.0f, -0.39176171875f, 2.017234375f, 229 1.59602734375f, -0.81296875f, 0.0f 230 }; 231 232 glUseProgram(program_yuv_); 233 glUniform1i(glGetUniformLocation(program_yuv_, "y_texture"), 0); 234 glUniform1i(glGetUniformLocation(program_yuv_, "u_texture"), 1); 235 glUniform1i(glGetUniformLocation(program_yuv_, "v_texture"), 2); 236 glUniformMatrix3fv(glGetUniformLocation(program_yuv_, "color_matrix"), 237 1, GL_FALSE, kColorMatrix); 238 AssertNoGLError(); 239 240 GLint pos_location = glGetAttribLocation(program_yuv_, "a_position"); 241 GLint tc_location = glGetAttribLocation(program_yuv_, "a_texCoord"); 242 AssertNoGLError(); 243 glEnableVertexAttribArray(pos_location); 244 glVertexAttribPointer(pos_location, 2, GL_FLOAT, GL_FALSE, 0, 0); 245 glEnableVertexAttribArray(tc_location); 246 glVertexAttribPointer(tc_location, 2, GL_FLOAT, GL_FALSE, 0, 247 static_cast<float*>(0) + 16); // Skip position coordinates. 248 AssertNoGLError(); 249 250 glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); 251 AssertNoGLError(); 252} 253 254void MediaStreamVideoDemoInstance::DrawRGB() { 255 glUseProgram(program_rgb_); 256 glUniform1i(glGetUniformLocation(program_rgb_, "rgb_texture"), 3); 257 AssertNoGLError(); 258 259 GLint pos_location = glGetAttribLocation(program_rgb_, "a_position"); 260 GLint tc_location = glGetAttribLocation(program_rgb_, "a_texCoord"); 261 AssertNoGLError(); 262 glEnableVertexAttribArray(pos_location); 263 glVertexAttribPointer(pos_location, 2, GL_FLOAT, GL_FALSE, 0, 0); 264 glEnableVertexAttribArray(tc_location); 265 glVertexAttribPointer(tc_location, 2, GL_FLOAT, GL_FALSE, 0, 266 static_cast<float*>(0) + 16); // Skip position coordinates. 267 AssertNoGLError(); 268 269 glDrawArrays(GL_TRIANGLE_STRIP, 4, 4); 270} 271 272void MediaStreamVideoDemoInstance::Render() { 273 PP_DCHECK(!is_painting_); 274 is_painting_ = true; 275 needs_paint_ = false; 276 277 if (texture_y_) { 278 DrawRGB(); 279 DrawYUV(); 280 } else { 281 glClear(GL_COLOR_BUFFER_BIT); 282 } 283 pp::CompletionCallback cb = callback_factory_.NewCallback( 284 &MediaStreamVideoDemoInstance::PaintFinished); 285 context_->SwapBuffers(cb); 286} 287 288void MediaStreamVideoDemoInstance::PaintFinished(int32_t result) { 289 is_painting_ = false; 290 if (needs_paint_) 291 Render(); 292} 293 294GLuint MediaStreamVideoDemoInstance::CreateTexture( 295 int32_t width, int32_t height, int unit, bool rgba) { 296 GLuint texture_id; 297 glGenTextures(1, &texture_id); 298 AssertNoGLError(); 299 300 // Assign parameters. 301 glActiveTexture(GL_TEXTURE0 + unit); 302 glBindTexture(GL_TEXTURE_2D, texture_id); 303 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); 304 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); 305 glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); 306 glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); 307 // Allocate texture. 308 glTexImage2D(GL_TEXTURE_2D, 0, 309 rgba ? GL_BGRA_EXT : GL_LUMINANCE, 310 width, height, 0, 311 rgba ? GL_BGRA_EXT : GL_LUMINANCE, GL_UNSIGNED_BYTE, NULL); 312 AssertNoGLError(); 313 return texture_id; 314} 315 316void MediaStreamVideoDemoInstance::CreateGLObjects() { 317 // Code and constants for shader. 318 static const char kVertexShader[] = 319 "varying vec2 v_texCoord; \n" 320 "attribute vec4 a_position; \n" 321 "attribute vec2 a_texCoord; \n" 322 "void main() \n" 323 "{ \n" 324 " v_texCoord = a_texCoord; \n" 325 " gl_Position = a_position; \n" 326 "}"; 327 328 static const char kFragmentShaderYUV[] = 329 "precision mediump float; \n" 330 "varying vec2 v_texCoord; \n" 331 "uniform sampler2D y_texture; \n" 332 "uniform sampler2D u_texture; \n" 333 "uniform sampler2D v_texture; \n" 334 "uniform mat3 color_matrix; \n" 335 "void main() \n" 336 "{ \n" 337 " vec3 yuv; \n" 338 " yuv.x = texture2D(y_texture, v_texCoord).r; \n" 339 " yuv.y = texture2D(u_texture, v_texCoord).r; \n" 340 " yuv.z = texture2D(v_texture, v_texCoord).r; \n" 341 " vec3 rgb = color_matrix * (yuv - vec3(0.0625, 0.5, 0.5));\n" 342 " gl_FragColor = vec4(rgb, 1.0); \n" 343 "}"; 344 345 static const char kFragmentShaderRGB[] = 346 "precision mediump float; \n" 347 "varying vec2 v_texCoord; \n" 348 "uniform sampler2D rgb_texture; \n" 349 "void main() \n" 350 "{ \n" 351 " gl_FragColor = texture2D(rgb_texture, v_texCoord); \n" 352 "}"; 353 354 // Create shader programs. 355 program_yuv_ = glCreateProgram(); 356 CreateShader(program_yuv_, GL_VERTEX_SHADER, kVertexShader); 357 CreateShader(program_yuv_, GL_FRAGMENT_SHADER, kFragmentShaderYUV); 358 glLinkProgram(program_yuv_); 359 AssertNoGLError(); 360 361 program_rgb_ = glCreateProgram(); 362 CreateShader(program_rgb_, GL_VERTEX_SHADER, kVertexShader); 363 CreateShader(program_rgb_, GL_FRAGMENT_SHADER, kFragmentShaderRGB); 364 glLinkProgram(program_rgb_); 365 AssertNoGLError(); 366 367 // Assign vertex positions and texture coordinates to buffers for use in 368 // shader program. 369 static const float kVertices[] = { 370 -1, 1, -1, -1, 0, 1, 0, -1, // Position coordinates. 371 0, 1, 0, -1, 1, 1, 1, -1, // Position coordinates. 372 0, 0, 0, 1, 1, 0, 1, 1, // Texture coordinates. 373 0, 0, 0, 1, 1, 0, 1, 1, // Texture coordinates. 374 }; 375 376 glGenBuffers(1, &buffer_); 377 glBindBuffer(GL_ARRAY_BUFFER, buffer_); 378 glBufferData(GL_ARRAY_BUFFER, sizeof(kVertices), kVertices, GL_STATIC_DRAW); 379 AssertNoGLError(); 380} 381 382void MediaStreamVideoDemoInstance::CreateShader( 383 GLuint program, GLenum type, const char* source) { 384 GLuint shader = glCreateShader(type); 385 GLint length = strlen(source) + 1; 386 glShaderSource(shader, 1, &source, &length); 387 glCompileShader(shader); 388 glAttachShader(program, shader); 389 glDeleteShader(shader); 390} 391 392void MediaStreamVideoDemoInstance::CreateTextures() { 393 int32_t width = frame_size_.width(); 394 int32_t height = frame_size_.height(); 395 if (width == 0 || height == 0) 396 return; 397 if (texture_y_) 398 glDeleteTextures(1, &texture_y_); 399 if (texture_u_) 400 glDeleteTextures(1, &texture_u_); 401 if (texture_v_) 402 glDeleteTextures(1, &texture_v_); 403 if (texture_rgb_) 404 glDeleteTextures(1, &texture_rgb_); 405 texture_y_ = CreateTexture(width, height, 0, false); 406 407 texture_u_ = CreateTexture(width / 2, height / 2, 1, false); 408 texture_v_ = CreateTexture(width / 2, height / 2, 2, false); 409 texture_rgb_ = CreateTexture(width, height, 3, true); 410} 411 412void MediaStreamVideoDemoInstance::ConfigureTrack() { 413 const int32_t attrib_list[] = { 414 PP_MEDIASTREAMVIDEOTRACK_ATTRIB_FORMAT, attrib_format_, 415 PP_MEDIASTREAMVIDEOTRACK_ATTRIB_WIDTH, attrib_width_, 416 PP_MEDIASTREAMVIDEOTRACK_ATTRIB_HEIGHT, attrib_height_, 417 PP_MEDIASTREAMVIDEOTRACK_ATTRIB_NONE 418 }; 419 video_track_.Configure(attrib_list, callback_factory_.NewCallback( 420 &MediaStreamVideoDemoInstance::OnConfigure)); 421} 422 423void MediaStreamVideoDemoInstance::OnConfigure(int32_t result) { 424 video_track_.GetFrame(callback_factory_.NewCallbackWithOutput( 425 &MediaStreamVideoDemoInstance::OnGetFrame)); 426} 427 428void MediaStreamVideoDemoInstance::OnGetFrame( 429 int32_t result, pp::VideoFrame frame) { 430 if (result != PP_OK) 431 return; 432 const char* data = static_cast<const char*>(frame.GetDataBuffer()); 433 pp::Size size; 434 frame.GetSize(&size); 435 436 if (size != frame_size_) { 437 frame_size_ = size; 438 CreateTextures(); 439 } 440 441 is_bgra_ = (frame.GetFormat() == PP_VIDEOFRAME_FORMAT_BGRA); 442 443 int32_t width = frame_size_.width(); 444 int32_t height = frame_size_.height(); 445 if (!is_bgra_) { 446 glActiveTexture(GL_TEXTURE0); 447 glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, width, height, 448 GL_LUMINANCE, GL_UNSIGNED_BYTE, data); 449 450 data += width * height; 451 width /= 2; 452 height /= 2; 453 454 glActiveTexture(GL_TEXTURE1); 455 glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, width, height, 456 GL_LUMINANCE, GL_UNSIGNED_BYTE, data); 457 458 data += width * height; 459 glActiveTexture(GL_TEXTURE2); 460 glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, width, height, 461 GL_LUMINANCE, GL_UNSIGNED_BYTE, data); 462 } else { 463 glActiveTexture(GL_TEXTURE3); 464 glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, width, height, 465 GL_BGRA_EXT, GL_UNSIGNED_BYTE, data); 466 } 467 468 if (is_painting_) 469 needs_paint_ = true; 470 else 471 Render(); 472 473 video_track_.RecycleFrame(frame); 474 if (need_config_) { 475 ConfigureTrack(); 476 need_config_ = false; 477 } else { 478 video_track_.GetFrame(callback_factory_.NewCallbackWithOutput( 479 &MediaStreamVideoDemoInstance::OnGetFrame)); 480 } 481} 482 483pp::Instance* MediaStreamVideoModule::CreateInstance(PP_Instance instance) { 484 return new MediaStreamVideoDemoInstance(instance, this); 485} 486 487} // anonymous namespace 488 489namespace pp { 490// Factory function for your specialization of the Module object. 491Module* CreateModule() { 492 return new MediaStreamVideoModule(); 493} 494} // namespace pp 495