1// Copyright 2014 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include <GLES2/gl2.h>
6#include <GLES2/gl2ext.h>
7#include <string.h>
8
9#include <vector>
10
11#include "ppapi/c/pp_errors.h"
12#include "ppapi/c/ppb_opengles2.h"
13#include "ppapi/cpp/completion_callback.h"
14#include "ppapi/cpp/graphics_3d.h"
15#include "ppapi/cpp/graphics_3d_client.h"
16#include "ppapi/cpp/instance.h"
17#include "ppapi/cpp/media_stream_video_track.h"
18#include "ppapi/cpp/module.h"
19#include "ppapi/cpp/rect.h"
20#include "ppapi/cpp/var.h"
21#include "ppapi/cpp/var_dictionary.h"
22#include "ppapi/cpp/video_frame.h"
23#include "ppapi/lib/gl/gles2/gl2ext_ppapi.h"
24#include "ppapi/utility/completion_callback_factory.h"
25
26// When compiling natively on Windows, PostMessage can be #define-d to
27// something else.
28#ifdef PostMessage
29#undef PostMessage
30#endif
31
32// Assert |context_| isn't holding any GL Errors.  Done as a macro instead of a
33// function to preserve line number information in the failure message.
34#define AssertNoGLError() \
35  PP_DCHECK(!glGetError());
36
37namespace {
38
39// This object is the global object representing this plugin library as long
40// as it is loaded.
41class MediaStreamVideoModule : public pp::Module {
42 public:
43  MediaStreamVideoModule() : pp::Module() {}
44  virtual ~MediaStreamVideoModule() {}
45
46  virtual pp::Instance* CreateInstance(PP_Instance instance);
47};
48
49class MediaStreamVideoDemoInstance : public pp::Instance,
50                        public pp::Graphics3DClient {
51 public:
52  MediaStreamVideoDemoInstance(PP_Instance instance, pp::Module* module);
53  virtual ~MediaStreamVideoDemoInstance();
54
55  // pp::Instance implementation (see PPP_Instance).
56  virtual void DidChangeView(const pp::Rect& position,
57                             const pp::Rect& clip_ignored);
58  virtual void HandleMessage(const pp::Var& message_data);
59
60  // pp::Graphics3DClient implementation.
61  virtual void Graphics3DContextLost() {
62    InitGL();
63    CreateTextures();
64    Render();
65  }
66
67 private:
68  void DrawYUV();
69  void DrawRGB();
70  void Render();
71
72  // GL-related functions.
73  void InitGL();
74  GLuint CreateTexture(int32_t width, int32_t height, int unit, bool rgba);
75  void CreateGLObjects();
76  void CreateShader(GLuint program, GLenum type, const char* source);
77  void PaintFinished(int32_t result);
78  void CreateTextures();
79  void ConfigureTrack();
80
81
82  // MediaStreamVideoTrack callbacks.
83  void OnConfigure(int32_t result);
84  void OnGetFrame(int32_t result, pp::VideoFrame frame);
85
86  pp::Size position_size_;
87  bool is_painting_;
88  bool needs_paint_;
89  bool is_bgra_;
90  GLuint program_yuv_;
91  GLuint program_rgb_;
92  GLuint buffer_;
93  GLuint texture_y_;
94  GLuint texture_u_;
95  GLuint texture_v_;
96  GLuint texture_rgb_;
97  pp::MediaStreamVideoTrack video_track_;
98  pp::CompletionCallbackFactory<MediaStreamVideoDemoInstance> callback_factory_;
99  std::vector<int32_t> attrib_list_;
100
101  // MediaStreamVideoTrack attributes:
102  bool need_config_;
103  PP_VideoFrame_Format attrib_format_;
104  int32_t attrib_width_;
105  int32_t attrib_height_;
106
107  // Owned data.
108  pp::Graphics3D* context_;
109
110  pp::Size frame_size_;
111};
112
113MediaStreamVideoDemoInstance::MediaStreamVideoDemoInstance(
114    PP_Instance instance, pp::Module* module)
115    : pp::Instance(instance),
116      pp::Graphics3DClient(this),
117      is_painting_(false),
118      needs_paint_(false),
119      is_bgra_(false),
120      texture_y_(0),
121      texture_u_(0),
122      texture_v_(0),
123      texture_rgb_(0),
124      callback_factory_(this),
125      need_config_(false),
126      attrib_format_(PP_VIDEOFRAME_FORMAT_I420),
127      attrib_width_(0),
128      attrib_height_(0),
129      context_(NULL) {
130  if (!glInitializePPAPI(pp::Module::Get()->get_browser_interface())) {
131    LogToConsole(PP_LOGLEVEL_ERROR, pp::Var("Unable to initialize GL PPAPI!"));
132    assert(false);
133  }
134}
135
136MediaStreamVideoDemoInstance::~MediaStreamVideoDemoInstance() {
137  delete context_;
138}
139
140void MediaStreamVideoDemoInstance::DidChangeView(
141    const pp::Rect& position, const pp::Rect& clip_ignored) {
142  if (position.width() == 0 || position.height() == 0)
143    return;
144  if (position.size() == position_size_)
145    return;
146
147  position_size_ = position.size();
148
149  // Initialize graphics.
150  InitGL();
151  Render();
152}
153
154void MediaStreamVideoDemoInstance::HandleMessage(const pp::Var& var_message) {
155  if (!var_message.is_dictionary()) {
156    LogToConsole(PP_LOGLEVEL_ERROR, pp::Var("Invalid message!"));
157    return;
158  }
159
160  pp::VarDictionary var_dictionary_message(var_message);
161  std::string command = var_dictionary_message.Get("command").AsString();
162
163  if (command == "init") {
164    pp::Var var_track = var_dictionary_message.Get("track");
165    if (!var_track.is_resource())
166      return;
167    pp::Resource resource_track = var_track.AsResource();
168    video_track_ = pp::MediaStreamVideoTrack(resource_track);
169    ConfigureTrack();
170  } else if (command == "format") {
171    std::string str_format = var_dictionary_message.Get("format").AsString();
172    if (str_format == "YV12") {
173      attrib_format_ = PP_VIDEOFRAME_FORMAT_YV12;
174    } else if (str_format == "I420") {
175      attrib_format_ = PP_VIDEOFRAME_FORMAT_I420;
176    } else if (str_format == "BGRA") {
177      attrib_format_ = PP_VIDEOFRAME_FORMAT_BGRA;
178    } else {
179      attrib_format_ = PP_VIDEOFRAME_FORMAT_UNKNOWN;
180    }
181    need_config_ = true;
182  } else if (command == "size") {
183    attrib_width_ = var_dictionary_message.Get("width").AsInt();
184    attrib_height_ = var_dictionary_message.Get("height").AsInt();
185    need_config_ = true;
186  } else {
187    LogToConsole(PP_LOGLEVEL_ERROR, pp::Var("Invalid command!"));
188  }
189}
190
191void MediaStreamVideoDemoInstance::InitGL() {
192  PP_DCHECK(position_size_.width() && position_size_.height());
193  is_painting_ = false;
194
195  delete context_;
196  int32_t attributes[] = {
197    PP_GRAPHICS3DATTRIB_ALPHA_SIZE, 0,
198    PP_GRAPHICS3DATTRIB_BLUE_SIZE, 8,
199    PP_GRAPHICS3DATTRIB_GREEN_SIZE, 8,
200    PP_GRAPHICS3DATTRIB_RED_SIZE, 8,
201    PP_GRAPHICS3DATTRIB_DEPTH_SIZE, 0,
202    PP_GRAPHICS3DATTRIB_STENCIL_SIZE, 0,
203    PP_GRAPHICS3DATTRIB_SAMPLES, 0,
204    PP_GRAPHICS3DATTRIB_SAMPLE_BUFFERS, 0,
205    PP_GRAPHICS3DATTRIB_WIDTH, position_size_.width(),
206    PP_GRAPHICS3DATTRIB_HEIGHT, position_size_.height(),
207    PP_GRAPHICS3DATTRIB_NONE,
208  };
209  context_ = new pp::Graphics3D(this, attributes);
210  PP_DCHECK(!context_->is_null());
211
212  glSetCurrentContextPPAPI(context_->pp_resource());
213
214  // Set viewport window size and clear color bit.
215  glClearColor(1, 0, 0, 1);
216  glClear(GL_COLOR_BUFFER_BIT);
217  glViewport(0, 0, position_size_.width(), position_size_.height());
218
219  BindGraphics(*context_);
220  AssertNoGLError();
221
222  CreateGLObjects();
223}
224
225void MediaStreamVideoDemoInstance::DrawYUV() {
226  static const float kColorMatrix[9] = {
227    1.1643828125f, 1.1643828125f, 1.1643828125f,
228    0.0f, -0.39176171875f, 2.017234375f,
229    1.59602734375f, -0.81296875f, 0.0f
230  };
231
232  glUseProgram(program_yuv_);
233  glUniform1i(glGetUniformLocation(program_yuv_, "y_texture"), 0);
234  glUniform1i(glGetUniformLocation(program_yuv_, "u_texture"), 1);
235  glUniform1i(glGetUniformLocation(program_yuv_, "v_texture"), 2);
236  glUniformMatrix3fv(glGetUniformLocation(program_yuv_, "color_matrix"),
237      1, GL_FALSE, kColorMatrix);
238  AssertNoGLError();
239
240  GLint pos_location = glGetAttribLocation(program_yuv_, "a_position");
241  GLint tc_location = glGetAttribLocation(program_yuv_, "a_texCoord");
242  AssertNoGLError();
243  glEnableVertexAttribArray(pos_location);
244  glVertexAttribPointer(pos_location, 2, GL_FLOAT, GL_FALSE, 0, 0);
245  glEnableVertexAttribArray(tc_location);
246  glVertexAttribPointer(tc_location, 2, GL_FLOAT, GL_FALSE, 0,
247      static_cast<float*>(0) + 16);  // Skip position coordinates.
248  AssertNoGLError();
249
250  glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
251  AssertNoGLError();
252}
253
254void MediaStreamVideoDemoInstance::DrawRGB() {
255  glUseProgram(program_rgb_);
256  glUniform1i(glGetUniformLocation(program_rgb_, "rgb_texture"), 3);
257  AssertNoGLError();
258
259  GLint pos_location = glGetAttribLocation(program_rgb_, "a_position");
260  GLint tc_location = glGetAttribLocation(program_rgb_, "a_texCoord");
261  AssertNoGLError();
262  glEnableVertexAttribArray(pos_location);
263  glVertexAttribPointer(pos_location, 2, GL_FLOAT, GL_FALSE, 0, 0);
264  glEnableVertexAttribArray(tc_location);
265  glVertexAttribPointer(tc_location, 2, GL_FLOAT, GL_FALSE, 0,
266      static_cast<float*>(0) + 16);  // Skip position coordinates.
267  AssertNoGLError();
268
269  glDrawArrays(GL_TRIANGLE_STRIP, 4, 4);
270}
271
272void MediaStreamVideoDemoInstance::Render() {
273  PP_DCHECK(!is_painting_);
274  is_painting_ = true;
275  needs_paint_ = false;
276
277  if (texture_y_) {
278    DrawRGB();
279    DrawYUV();
280  } else {
281    glClear(GL_COLOR_BUFFER_BIT);
282  }
283  pp::CompletionCallback cb = callback_factory_.NewCallback(
284      &MediaStreamVideoDemoInstance::PaintFinished);
285  context_->SwapBuffers(cb);
286}
287
288void MediaStreamVideoDemoInstance::PaintFinished(int32_t result) {
289  is_painting_ = false;
290  if (needs_paint_)
291    Render();
292}
293
294GLuint MediaStreamVideoDemoInstance::CreateTexture(
295    int32_t width, int32_t height, int unit, bool rgba) {
296  GLuint texture_id;
297  glGenTextures(1, &texture_id);
298  AssertNoGLError();
299
300  // Assign parameters.
301  glActiveTexture(GL_TEXTURE0 + unit);
302  glBindTexture(GL_TEXTURE_2D, texture_id);
303  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
304  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
305  glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
306  glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
307  // Allocate texture.
308  glTexImage2D(GL_TEXTURE_2D, 0,
309               rgba ? GL_BGRA_EXT : GL_LUMINANCE,
310               width, height, 0,
311               rgba ? GL_BGRA_EXT : GL_LUMINANCE, GL_UNSIGNED_BYTE, NULL);
312  AssertNoGLError();
313  return texture_id;
314}
315
316void MediaStreamVideoDemoInstance::CreateGLObjects() {
317  // Code and constants for shader.
318  static const char kVertexShader[] =
319      "varying vec2 v_texCoord;            \n"
320      "attribute vec4 a_position;          \n"
321      "attribute vec2 a_texCoord;          \n"
322      "void main()                         \n"
323      "{                                   \n"
324      "    v_texCoord = a_texCoord;        \n"
325      "    gl_Position = a_position;       \n"
326      "}";
327
328  static const char kFragmentShaderYUV[] =
329      "precision mediump float;                                   \n"
330      "varying vec2 v_texCoord;                                   \n"
331      "uniform sampler2D y_texture;                               \n"
332      "uniform sampler2D u_texture;                               \n"
333      "uniform sampler2D v_texture;                               \n"
334      "uniform mat3 color_matrix;                                 \n"
335      "void main()                                                \n"
336      "{                                                          \n"
337      "  vec3 yuv;                                                \n"
338      "  yuv.x = texture2D(y_texture, v_texCoord).r;              \n"
339      "  yuv.y = texture2D(u_texture, v_texCoord).r;              \n"
340      "  yuv.z = texture2D(v_texture, v_texCoord).r;              \n"
341      "  vec3 rgb = color_matrix * (yuv - vec3(0.0625, 0.5, 0.5));\n"
342      "  gl_FragColor = vec4(rgb, 1.0);                           \n"
343      "}";
344
345  static const char kFragmentShaderRGB[] =
346      "precision mediump float;                                   \n"
347      "varying vec2 v_texCoord;                                   \n"
348      "uniform sampler2D rgb_texture;                             \n"
349      "void main()                                                \n"
350      "{                                                          \n"
351      "  gl_FragColor = texture2D(rgb_texture, v_texCoord);       \n"
352      "}";
353
354  // Create shader programs.
355  program_yuv_ = glCreateProgram();
356  CreateShader(program_yuv_, GL_VERTEX_SHADER, kVertexShader);
357  CreateShader(program_yuv_, GL_FRAGMENT_SHADER, kFragmentShaderYUV);
358  glLinkProgram(program_yuv_);
359  AssertNoGLError();
360
361  program_rgb_ = glCreateProgram();
362  CreateShader(program_rgb_, GL_VERTEX_SHADER, kVertexShader);
363  CreateShader(program_rgb_, GL_FRAGMENT_SHADER, kFragmentShaderRGB);
364  glLinkProgram(program_rgb_);
365  AssertNoGLError();
366
367  // Assign vertex positions and texture coordinates to buffers for use in
368  // shader program.
369  static const float kVertices[] = {
370    -1, 1, -1, -1, 0, 1, 0, -1,  // Position coordinates.
371    0, 1, 0, -1, 1, 1, 1, -1,  // Position coordinates.
372    0, 0, 0, 1, 1, 0, 1, 1,  // Texture coordinates.
373    0, 0, 0, 1, 1, 0, 1, 1,  // Texture coordinates.
374  };
375
376  glGenBuffers(1, &buffer_);
377  glBindBuffer(GL_ARRAY_BUFFER, buffer_);
378  glBufferData(GL_ARRAY_BUFFER, sizeof(kVertices), kVertices, GL_STATIC_DRAW);
379  AssertNoGLError();
380}
381
382void MediaStreamVideoDemoInstance::CreateShader(
383    GLuint program, GLenum type, const char* source) {
384  GLuint shader = glCreateShader(type);
385  GLint length = static_cast<GLint>(strlen(source) + 1);
386  glShaderSource(shader, 1, &source, &length);
387  glCompileShader(shader);
388  glAttachShader(program, shader);
389  glDeleteShader(shader);
390}
391
392void MediaStreamVideoDemoInstance::CreateTextures() {
393  int32_t width = frame_size_.width();
394  int32_t height = frame_size_.height();
395  if (width == 0 || height == 0)
396    return;
397  if (texture_y_)
398    glDeleteTextures(1, &texture_y_);
399  if (texture_u_)
400    glDeleteTextures(1, &texture_u_);
401  if (texture_v_)
402    glDeleteTextures(1, &texture_v_);
403  if (texture_rgb_)
404    glDeleteTextures(1, &texture_rgb_);
405  texture_y_ = CreateTexture(width, height, 0, false);
406
407  texture_u_ = CreateTexture(width / 2, height / 2, 1, false);
408  texture_v_ = CreateTexture(width / 2, height / 2, 2, false);
409  texture_rgb_ = CreateTexture(width, height, 3, true);
410}
411
412void MediaStreamVideoDemoInstance::ConfigureTrack() {
413  const int32_t attrib_list[] = {
414      PP_MEDIASTREAMVIDEOTRACK_ATTRIB_FORMAT, attrib_format_,
415      PP_MEDIASTREAMVIDEOTRACK_ATTRIB_WIDTH, attrib_width_,
416      PP_MEDIASTREAMVIDEOTRACK_ATTRIB_HEIGHT, attrib_height_,
417      PP_MEDIASTREAMVIDEOTRACK_ATTRIB_NONE
418    };
419  video_track_.Configure(attrib_list, callback_factory_.NewCallback(
420        &MediaStreamVideoDemoInstance::OnConfigure));
421}
422
423void MediaStreamVideoDemoInstance::OnConfigure(int32_t result) {
424  video_track_.GetFrame(callback_factory_.NewCallbackWithOutput(
425      &MediaStreamVideoDemoInstance::OnGetFrame));
426}
427
428void MediaStreamVideoDemoInstance::OnGetFrame(
429    int32_t result, pp::VideoFrame frame) {
430  if (result != PP_OK)
431    return;
432  const char* data = static_cast<const char*>(frame.GetDataBuffer());
433  pp::Size size;
434  frame.GetSize(&size);
435
436  if (size != frame_size_) {
437    frame_size_ = size;
438    CreateTextures();
439  }
440
441  is_bgra_ = (frame.GetFormat() == PP_VIDEOFRAME_FORMAT_BGRA);
442
443  int32_t width = frame_size_.width();
444  int32_t height = frame_size_.height();
445  if (!is_bgra_) {
446    glActiveTexture(GL_TEXTURE0);
447    glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, width, height,
448                    GL_LUMINANCE, GL_UNSIGNED_BYTE, data);
449
450    data += width * height;
451    width /= 2;
452    height /= 2;
453
454    glActiveTexture(GL_TEXTURE1);
455    glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, width, height,
456                    GL_LUMINANCE, GL_UNSIGNED_BYTE, data);
457
458    data += width * height;
459    glActiveTexture(GL_TEXTURE2);
460    glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, width, height,
461                    GL_LUMINANCE, GL_UNSIGNED_BYTE, data);
462  } else {
463    glActiveTexture(GL_TEXTURE3);
464    glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, width, height,
465                    GL_BGRA_EXT, GL_UNSIGNED_BYTE, data);
466  }
467
468  if (is_painting_)
469    needs_paint_ = true;
470  else
471    Render();
472
473  video_track_.RecycleFrame(frame);
474  if (need_config_) {
475    ConfigureTrack();
476    need_config_ = false;
477  } else {
478    video_track_.GetFrame(callback_factory_.NewCallbackWithOutput(
479        &MediaStreamVideoDemoInstance::OnGetFrame));
480  }
481}
482
483pp::Instance* MediaStreamVideoModule::CreateInstance(PP_Instance instance) {
484  return new MediaStreamVideoDemoInstance(instance, this);
485}
486
487}  // anonymous namespace
488
489namespace pp {
490// Factory function for your specialization of the Module object.
491Module* CreateModule() {
492  return new MediaStreamVideoModule();
493}
494}  // namespace pp
495