1// Copyright (c) 2012 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include <stdlib.h>
6#include <string.h>
7
8#include <map>
9#include <vector>
10
11#include "ppapi/c/dev/ppb_video_capture_dev.h"
12#include "ppapi/c/pp_errors.h"
13#include "ppapi/c/ppb_opengles2.h"
14#include "ppapi/cpp/dev/buffer_dev.h"
15#include "ppapi/cpp/dev/device_ref_dev.h"
16#include "ppapi/cpp/dev/video_capture_dev.h"
17#include "ppapi/cpp/dev/video_capture_client_dev.h"
18#include "ppapi/cpp/completion_callback.h"
19#include "ppapi/cpp/graphics_3d_client.h"
20#include "ppapi/cpp/graphics_3d.h"
21#include "ppapi/cpp/instance.h"
22#include "ppapi/cpp/module.h"
23#include "ppapi/cpp/rect.h"
24#include "ppapi/cpp/var.h"
25#include "ppapi/lib/gl/include/GLES2/gl2.h"
26#include "ppapi/utility/completion_callback_factory.h"
27
28// When compiling natively on Windows, PostMessage can be #define-d to
29// something else.
30#ifdef PostMessage
31#undef PostMessage
32#endif
33
34// Assert |context_| isn't holding any GL Errors.  Done as a macro instead of a
35// function to preserve line number information in the failure message.
36#define AssertNoGLError() \
37  PP_DCHECK(!gles2_if_->GetError(context_->pp_resource()));
38
39namespace {
40
41const char* const kDelimiter = "#__#";
42
43// This object is the global object representing this plugin library as long
44// as it is loaded.
45class VCDemoModule : public pp::Module {
46 public:
47  VCDemoModule() : pp::Module() {}
48  virtual ~VCDemoModule() {}
49
50  virtual pp::Instance* CreateInstance(PP_Instance instance);
51};
52
53class VCDemoInstance : public pp::Instance,
54                       public pp::Graphics3DClient,
55                       public pp::VideoCaptureClient_Dev {
56 public:
57  VCDemoInstance(PP_Instance instance, pp::Module* module);
58  virtual ~VCDemoInstance();
59
60  // pp::Instance implementation (see PPP_Instance).
61  virtual void DidChangeView(const pp::Rect& position,
62                             const pp::Rect& clip_ignored);
63  virtual void HandleMessage(const pp::Var& message_data);
64
65  // pp::Graphics3DClient implementation.
66  virtual void Graphics3DContextLost() {
67    InitGL();
68    CreateYUVTextures();
69    Render();
70  }
71
72  virtual void OnDeviceInfo(PP_Resource resource,
73                            const PP_VideoCaptureDeviceInfo_Dev& info,
74                            const std::vector<pp::Buffer_Dev>& buffers) {
75    capture_info_ = info;
76    buffers_ = buffers;
77    CreateYUVTextures();
78  }
79
80  virtual void OnStatus(PP_Resource resource, uint32_t status) {
81  }
82
83  virtual void OnError(PP_Resource resource, uint32_t error) {
84  }
85
86  virtual void OnBufferReady(PP_Resource resource, uint32_t buffer) {
87    const char* data = static_cast<const char*>(buffers_[buffer].data());
88    int32_t width = capture_info_.width;
89    int32_t height = capture_info_.height;
90    gles2_if_->ActiveTexture(context_->pp_resource(), GL_TEXTURE0);
91    gles2_if_->TexSubImage2D(
92        context_->pp_resource(), GL_TEXTURE_2D, 0, 0, 0, width, height,
93        GL_LUMINANCE, GL_UNSIGNED_BYTE, data);
94
95    data += width * height;
96    width /= 2;
97    height /= 2;
98
99    gles2_if_->ActiveTexture(context_->pp_resource(), GL_TEXTURE1);
100    gles2_if_->TexSubImage2D(
101        context_->pp_resource(), GL_TEXTURE_2D, 0, 0, 0, width, height,
102        GL_LUMINANCE, GL_UNSIGNED_BYTE, data);
103
104    data += width * height;
105    gles2_if_->ActiveTexture(context_->pp_resource(), GL_TEXTURE2);
106    gles2_if_->TexSubImage2D(
107        context_->pp_resource(), GL_TEXTURE_2D, 0, 0, 0, width, height,
108        GL_LUMINANCE, GL_UNSIGNED_BYTE, data);
109
110    video_capture_.ReuseBuffer(buffer);
111    if (is_painting_)
112      needs_paint_ = true;
113    else
114      Render();
115  }
116
117 private:
118  void Render();
119
120  // GL-related functions.
121  void InitGL();
122  GLuint CreateTexture(int32_t width, int32_t height, int unit);
123  void CreateGLObjects();
124  void CreateShader(GLuint program, GLenum type, const char* source, int size);
125  void PaintFinished(int32_t result);
126  void CreateYUVTextures();
127
128  void Open(const pp::DeviceRef_Dev& device);
129  void Stop();
130  void Start();
131  void EnumerateDevicesFinished(int32_t result,
132                                std::vector<pp::DeviceRef_Dev>& devices);
133  void OpenFinished(int32_t result);
134
135  static void MonitorDeviceChangeCallback(void* user_data,
136                                          uint32_t device_count,
137                                          const PP_Resource devices[]);
138
139  pp::Size position_size_;
140  bool is_painting_;
141  bool needs_paint_;
142  GLuint texture_y_;
143  GLuint texture_u_;
144  GLuint texture_v_;
145  pp::VideoCapture_Dev video_capture_;
146  PP_VideoCaptureDeviceInfo_Dev capture_info_;
147  std::vector<pp::Buffer_Dev> buffers_;
148  pp::CompletionCallbackFactory<VCDemoInstance> callback_factory_;
149
150  // Unowned pointers.
151  const struct PPB_OpenGLES2* gles2_if_;
152
153  // Owned data.
154  pp::Graphics3D* context_;
155
156  std::vector<pp::DeviceRef_Dev> enumerate_devices_;
157  std::vector<pp::DeviceRef_Dev> monitor_devices_;
158};
159
160VCDemoInstance::VCDemoInstance(PP_Instance instance, pp::Module* module)
161    : pp::Instance(instance),
162      pp::Graphics3DClient(this),
163      pp::VideoCaptureClient_Dev(this),
164      is_painting_(false),
165      needs_paint_(false),
166      texture_y_(0),
167      texture_u_(0),
168      texture_v_(0),
169      video_capture_(this),
170      callback_factory_(this),
171      context_(NULL) {
172  gles2_if_ = static_cast<const struct PPB_OpenGLES2*>(
173      module->GetBrowserInterface(PPB_OPENGLES2_INTERFACE));
174  PP_DCHECK(gles2_if_);
175
176  capture_info_.width = 320;
177  capture_info_.height = 240;
178  capture_info_.frames_per_second = 30;
179}
180
181VCDemoInstance::~VCDemoInstance() {
182  video_capture_.MonitorDeviceChange(NULL, NULL);
183  delete context_;
184}
185
186void VCDemoInstance::DidChangeView(
187    const pp::Rect& position, const pp::Rect& clip_ignored) {
188  if (position.width() == 0 || position.height() == 0)
189    return;
190  if (position.size() == position_size_)
191    return;
192
193  position_size_ = position.size();
194
195  // Initialize graphics.
196  InitGL();
197
198  Render();
199}
200
201void VCDemoInstance::HandleMessage(const pp::Var& message_data) {
202  if (message_data.is_string()) {
203    std::string event = message_data.AsString();
204    if (event == "PageInitialized") {
205      int32_t result = video_capture_.MonitorDeviceChange(
206          &VCDemoInstance::MonitorDeviceChangeCallback, this);
207      if (result != PP_OK)
208        PostMessage(pp::Var("MonitorDeviceChangeFailed"));
209
210      pp::CompletionCallbackWithOutput<std::vector<pp::DeviceRef_Dev> >
211          callback = callback_factory_.NewCallbackWithOutput(
212              &VCDemoInstance::EnumerateDevicesFinished);
213      result = video_capture_.EnumerateDevices(callback);
214      if (result != PP_OK_COMPLETIONPENDING)
215        PostMessage(pp::Var("EnumerationFailed"));
216    } else if (event == "UseDefault") {
217      Open(pp::DeviceRef_Dev());
218    } else if (event == "Stop") {
219      Stop();
220    } else if (event == "Start") {
221      Start();
222    } else if (event.find("Monitor:") == 0) {
223      std::string index_str = event.substr(strlen("Monitor:"));
224      int index = atoi(index_str.c_str());
225      if (index >= 0 && index < static_cast<int>(monitor_devices_.size()))
226        Open(monitor_devices_[index]);
227      else
228        PP_NOTREACHED();
229    } else if (event.find("Enumerate:") == 0) {
230      std::string index_str = event.substr(strlen("Enumerate:"));
231      int index = atoi(index_str.c_str());
232      if (index >= 0 && index < static_cast<int>(enumerate_devices_.size()))
233        Open(enumerate_devices_[index]);
234      else
235        PP_NOTREACHED();
236    }
237  }
238}
239
240void VCDemoInstance::InitGL() {
241  PP_DCHECK(position_size_.width() && position_size_.height());
242  is_painting_ = false;
243
244  delete context_;
245  int32_t attributes[] = {
246    PP_GRAPHICS3DATTRIB_ALPHA_SIZE, 0,
247    PP_GRAPHICS3DATTRIB_BLUE_SIZE, 8,
248    PP_GRAPHICS3DATTRIB_GREEN_SIZE, 8,
249    PP_GRAPHICS3DATTRIB_RED_SIZE, 8,
250    PP_GRAPHICS3DATTRIB_DEPTH_SIZE, 0,
251    PP_GRAPHICS3DATTRIB_STENCIL_SIZE, 0,
252    PP_GRAPHICS3DATTRIB_SAMPLES, 0,
253    PP_GRAPHICS3DATTRIB_SAMPLE_BUFFERS, 0,
254    PP_GRAPHICS3DATTRIB_WIDTH, position_size_.width(),
255    PP_GRAPHICS3DATTRIB_HEIGHT, position_size_.height(),
256    PP_GRAPHICS3DATTRIB_NONE,
257  };
258  context_ = new pp::Graphics3D(this, attributes);
259  PP_DCHECK(!context_->is_null());
260
261  // Set viewport window size and clear color bit.
262  gles2_if_->ClearColor(context_->pp_resource(), 1, 0, 0, 1);
263  gles2_if_->Clear(context_->pp_resource(), GL_COLOR_BUFFER_BIT);
264  gles2_if_->Viewport(context_->pp_resource(), 0, 0,
265                      position_size_.width(), position_size_.height());
266
267  BindGraphics(*context_);
268  AssertNoGLError();
269
270  CreateGLObjects();
271}
272
273void VCDemoInstance::Render() {
274  PP_DCHECK(!is_painting_);
275  is_painting_ = true;
276  needs_paint_ = false;
277  if (texture_y_) {
278    gles2_if_->DrawArrays(context_->pp_resource(), GL_TRIANGLE_STRIP, 0, 4);
279  } else {
280    gles2_if_->Clear(context_->pp_resource(), GL_COLOR_BUFFER_BIT);
281  }
282  pp::CompletionCallback cb = callback_factory_.NewCallback(
283      &VCDemoInstance::PaintFinished);
284  context_->SwapBuffers(cb);
285}
286
287void VCDemoInstance::PaintFinished(int32_t result) {
288  is_painting_ = false;
289  if (needs_paint_)
290    Render();
291}
292
293GLuint VCDemoInstance::CreateTexture(int32_t width, int32_t height, int unit) {
294  GLuint texture_id;
295  gles2_if_->GenTextures(context_->pp_resource(), 1, &texture_id);
296  AssertNoGLError();
297  // Assign parameters.
298  gles2_if_->ActiveTexture(context_->pp_resource(), GL_TEXTURE0 + unit);
299  gles2_if_->BindTexture(context_->pp_resource(), GL_TEXTURE_2D, texture_id);
300  gles2_if_->TexParameteri(
301      context_->pp_resource(), GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER,
302      GL_NEAREST);
303  gles2_if_->TexParameteri(
304      context_->pp_resource(), GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER,
305      GL_NEAREST);
306  gles2_if_->TexParameterf(
307      context_->pp_resource(), GL_TEXTURE_2D, GL_TEXTURE_WRAP_S,
308      GL_CLAMP_TO_EDGE);
309  gles2_if_->TexParameterf(
310      context_->pp_resource(), GL_TEXTURE_2D, GL_TEXTURE_WRAP_T,
311      GL_CLAMP_TO_EDGE);
312
313  // Allocate texture.
314  gles2_if_->TexImage2D(
315      context_->pp_resource(), GL_TEXTURE_2D, 0, GL_LUMINANCE, width, height, 0,
316      GL_LUMINANCE, GL_UNSIGNED_BYTE, NULL);
317  AssertNoGLError();
318  return texture_id;
319}
320
321void VCDemoInstance::CreateGLObjects() {
322  // Code and constants for shader.
323  static const char kVertexShader[] =
324      "varying vec2 v_texCoord;            \n"
325      "attribute vec4 a_position;          \n"
326      "attribute vec2 a_texCoord;          \n"
327      "void main()                         \n"
328      "{                                   \n"
329      "    v_texCoord = a_texCoord;        \n"
330      "    gl_Position = a_position;       \n"
331      "}";
332
333  static const char kFragmentShader[] =
334      "precision mediump float;                                   \n"
335      "varying vec2 v_texCoord;                                   \n"
336      "uniform sampler2D y_texture;                               \n"
337      "uniform sampler2D u_texture;                               \n"
338      "uniform sampler2D v_texture;                               \n"
339      "uniform mat3 color_matrix;                                 \n"
340      "void main()                                                \n"
341      "{                                                          \n"
342      "  vec3 yuv;                                                \n"
343      "  yuv.x = texture2D(y_texture, v_texCoord).r;              \n"
344      "  yuv.y = texture2D(u_texture, v_texCoord).r;              \n"
345      "  yuv.z = texture2D(v_texture, v_texCoord).r;              \n"
346      "  vec3 rgb = color_matrix * (yuv - vec3(0.0625, 0.5, 0.5));\n"
347      "  gl_FragColor = vec4(rgb, 1.0);                           \n"
348      "}";
349
350  static const float kColorMatrix[9] = {
351    1.1643828125f, 1.1643828125f, 1.1643828125f,
352    0.0f, -0.39176171875f, 2.017234375f,
353    1.59602734375f, -0.81296875f, 0.0f
354  };
355
356  PP_Resource context = context_->pp_resource();
357
358  // Create shader program.
359  GLuint program = gles2_if_->CreateProgram(context);
360  CreateShader(program, GL_VERTEX_SHADER, kVertexShader, sizeof(kVertexShader));
361  CreateShader(
362      program, GL_FRAGMENT_SHADER, kFragmentShader, sizeof(kFragmentShader));
363  gles2_if_->LinkProgram(context, program);
364  gles2_if_->UseProgram(context, program);
365  gles2_if_->DeleteProgram(context, program);
366  gles2_if_->Uniform1i(
367      context, gles2_if_->GetUniformLocation(context, program, "y_texture"), 0);
368  gles2_if_->Uniform1i(
369      context, gles2_if_->GetUniformLocation(context, program, "u_texture"), 1);
370  gles2_if_->Uniform1i(
371      context, gles2_if_->GetUniformLocation(context, program, "v_texture"), 2);
372  gles2_if_->UniformMatrix3fv(
373      context,
374      gles2_if_->GetUniformLocation(context, program, "color_matrix"),
375      1, GL_FALSE, kColorMatrix);
376  AssertNoGLError();
377
378  // Assign vertex positions and texture coordinates to buffers for use in
379  // shader program.
380  static const float kVertices[] = {
381    -1, 1, -1, -1, 1, 1, 1, -1,  // Position coordinates.
382    0, 0, 0, 1, 1, 0, 1, 1,  // Texture coordinates.
383  };
384
385  GLuint buffer;
386  gles2_if_->GenBuffers(context, 1, &buffer);
387  gles2_if_->BindBuffer(context, GL_ARRAY_BUFFER, buffer);
388  gles2_if_->BufferData(context, GL_ARRAY_BUFFER,
389                        sizeof(kVertices), kVertices, GL_STATIC_DRAW);
390  AssertNoGLError();
391  GLint pos_location = gles2_if_->GetAttribLocation(
392      context, program, "a_position");
393  GLint tc_location = gles2_if_->GetAttribLocation(
394      context, program, "a_texCoord");
395  AssertNoGLError();
396  gles2_if_->EnableVertexAttribArray(context, pos_location);
397  gles2_if_->VertexAttribPointer(context, pos_location, 2,
398                                 GL_FLOAT, GL_FALSE, 0, 0);
399  gles2_if_->EnableVertexAttribArray(context, tc_location);
400  gles2_if_->VertexAttribPointer(
401      context, tc_location, 2, GL_FLOAT, GL_FALSE, 0,
402      static_cast<float*>(0) + 8);  // Skip position coordinates.
403  AssertNoGLError();
404}
405
406void VCDemoInstance::CreateShader(
407    GLuint program, GLenum type, const char* source, int size) {
408  PP_Resource context = context_->pp_resource();
409  GLuint shader = gles2_if_->CreateShader(context, type);
410  gles2_if_->ShaderSource(context, shader, 1, &source, &size);
411  gles2_if_->CompileShader(context, shader);
412  gles2_if_->AttachShader(context, program, shader);
413  gles2_if_->DeleteShader(context, shader);
414}
415
416void VCDemoInstance::CreateYUVTextures() {
417  int32_t width = capture_info_.width;
418  int32_t height = capture_info_.height;
419  texture_y_ = CreateTexture(width, height, 0);
420
421  width /= 2;
422  height /= 2;
423  texture_u_ = CreateTexture(width, height, 1);
424  texture_v_ = CreateTexture(width, height, 2);
425}
426
427void VCDemoInstance::Open(const pp::DeviceRef_Dev& device) {
428  pp::CompletionCallback callback = callback_factory_.NewCallback(
429      &VCDemoInstance::OpenFinished);
430  int32_t result = video_capture_.Open(device, capture_info_, 4, callback);
431  if (result != PP_OK_COMPLETIONPENDING)
432    PostMessage(pp::Var("OpenFailed"));
433}
434
435void VCDemoInstance::Stop() {
436  if (video_capture_.StopCapture() != PP_OK)
437    PostMessage(pp::Var("StopFailed"));
438}
439
440void VCDemoInstance::Start() {
441  if (video_capture_.StartCapture() != PP_OK)
442    PostMessage(pp::Var("StartFailed"));
443}
444
445void VCDemoInstance::EnumerateDevicesFinished(
446    int32_t result,
447  std::vector<pp::DeviceRef_Dev>& devices) {
448  if (result == PP_OK) {
449    enumerate_devices_.swap(devices);
450    std::string device_names = "Enumerate:";
451    for (size_t index = 0; index < enumerate_devices_.size(); ++index) {
452      pp::Var name = enumerate_devices_[index].GetName();
453      PP_DCHECK(name.is_string());
454
455      if (index != 0)
456        device_names += kDelimiter;
457      device_names += name.AsString();
458    }
459    PostMessage(pp::Var(device_names));
460  } else {
461    PostMessage(pp::Var("EnumerationFailed"));
462  }
463}
464
465void VCDemoInstance::OpenFinished(int32_t result) {
466  if (result == PP_OK)
467    Start();
468  else
469    PostMessage(pp::Var("OpenFailed"));
470}
471
472// static
473void VCDemoInstance::MonitorDeviceChangeCallback(void* user_data,
474                                                 uint32_t device_count,
475                                                 const PP_Resource devices[]) {
476  VCDemoInstance* thiz = static_cast<VCDemoInstance*>(user_data);
477
478  std::string device_names = "Monitor:";
479  thiz->monitor_devices_.clear();
480  thiz->monitor_devices_.reserve(device_count);
481  for (size_t index = 0; index < device_count; ++index) {
482    thiz->monitor_devices_.push_back(pp::DeviceRef_Dev(devices[index]));
483    pp::Var name = thiz->monitor_devices_.back().GetName();
484    PP_DCHECK(name.is_string());
485
486    if (index != 0)
487      device_names += kDelimiter;
488    device_names += name.AsString();
489  }
490  thiz->PostMessage(pp::Var(device_names));
491}
492
493pp::Instance* VCDemoModule::CreateInstance(PP_Instance instance) {
494  return new VCDemoInstance(instance, this);
495}
496
497}  // anonymous namespace
498
499namespace pp {
500// Factory function for your specialization of the Module object.
501Module* CreateModule() {
502  return new VCDemoModule();
503}
504}  // namespace pp
505