1/*
2// Sample demonstrating interoperability of OpenCV UMat with Direct X surface
3// At first, the data obtained from video file or camera and
4// placed onto Direct X surface,
5// following mapping of this Direct X surface to OpenCV UMat and call cv::Blur
6// function. The result is mapped back to Direct X surface and rendered through
7// Direct X API.
8*/
9#define WIN32_LEAN_AND_MEAN
10#include <windows.h>
11#include <d3d9.h>
12
13#include "opencv2/core.hpp"
14#include "opencv2/core/directx.hpp"
15#include "opencv2/core/ocl.hpp"
16#include "opencv2/imgproc.hpp"
17#include "opencv2/videoio.hpp"
18
19#include "d3dsample.hpp"
20
21#pragma comment (lib, "d3d9.lib")
22
23
24using namespace std;
25using namespace cv;
26
27class D3D9WinApp : public D3DSample
28{
29public:
30    D3D9WinApp(int width, int height, std::string& window_name, cv::VideoCapture& cap) :
31        D3DSample(width, height, window_name, cap) {}
32
33    ~D3D9WinApp() {}
34
35    int create(void)
36    {
37        // base initialization
38        D3DSample::create();
39
40        // initialize DirectX
41        HRESULT r;
42
43        m_pD3D9 = ::Direct3DCreate9(D3D_SDK_VERSION);
44        if (NULL == m_pD3D9)
45        {
46            return -1;
47        }
48
49        DWORD flags = D3DCREATE_HARDWARE_VERTEXPROCESSING |
50                      D3DCREATE_PUREDEVICE |
51                      D3DCREATE_NOWINDOWCHANGES |
52                      D3DCREATE_MULTITHREADED |
53                      D3DCREATE_FPU_PRESERVE;
54
55        D3DPRESENT_PARAMETERS d3dpp;
56        ::ZeroMemory(&d3dpp, sizeof(D3DPRESENT_PARAMETERS));
57
58        d3dpp.Windowed                   = true;
59        d3dpp.Flags                      = 0;
60        d3dpp.BackBufferCount            = 0;
61        d3dpp.BackBufferFormat           = D3DFMT_A8R8G8B8;
62        d3dpp.BackBufferHeight           = m_height;
63        d3dpp.BackBufferWidth            = m_width;
64        d3dpp.MultiSampleType            = D3DMULTISAMPLE_NONE;
65        d3dpp.SwapEffect                 = D3DSWAPEFFECT_DISCARD;
66        d3dpp.hDeviceWindow              = m_hWnd;
67        d3dpp.PresentationInterval       = D3DPRESENT_INTERVAL_IMMEDIATE;
68        d3dpp.FullScreen_RefreshRateInHz = D3DPRESENT_RATE_DEFAULT;
69
70        r = m_pD3D9->CreateDevice(D3DADAPTER_DEFAULT, D3DDEVTYPE_HAL, m_hWnd, flags, &d3dpp, &m_pD3D9Dev);
71        if (FAILED(r))
72        {
73            return -1;
74        }
75
76        r = m_pD3D9Dev->GetBackBuffer(0, 0, D3DBACKBUFFER_TYPE_MONO, &m_pBackBuffer);
77        if (FAILED(r))
78        {
79            return -1;
80        }
81
82        r = m_pD3D9Dev->CreateOffscreenPlainSurface(m_width, m_height, D3DFMT_A8R8G8B8, D3DPOOL_DEFAULT, &m_pSurface, NULL);
83        if (FAILED(r))
84        {
85            std::cerr << "Can't create surface for result" << std::endl;
86            return -1;
87        }
88
89        // initialize OpenCL context of OpenCV lib from DirectX
90        if (cv::ocl::haveOpenCL())
91        {
92            m_oclCtx = cv::directx::ocl::initializeContextFromDirect3DDevice9(m_pD3D9Dev);
93        }
94
95        m_oclDevName = cv::ocl::useOpenCL() ?
96            cv::ocl::Context::getDefault().device(0).name() :
97            "No OpenCL device";
98
99        return 0;
100    } // create()
101
102
103    // get media data on DX surface for further processing
104    int get_surface(LPDIRECT3DSURFACE9* ppSurface)
105    {
106        HRESULT r;
107
108        if (!m_cap.read(m_frame_bgr))
109            return -1;
110
111        cv::cvtColor(m_frame_bgr, m_frame_rgba, CV_RGB2RGBA);
112
113        D3DLOCKED_RECT memDesc = { 0, NULL };
114        RECT rc = { 0, 0, m_width, m_height };
115
116        r = m_pSurface->LockRect(&memDesc, &rc, 0);
117        if (FAILED(r))
118        {
119            return r;
120        }
121
122        cv::Mat m(m_height, m_width, CV_8UC4, memDesc.pBits, memDesc.Pitch);
123        // copy video frame data to surface
124        m_frame_rgba.copyTo(m);
125
126        r = m_pSurface->UnlockRect();
127        if (FAILED(r))
128        {
129            return r;
130        }
131
132        *ppSurface = m_pSurface;
133
134        return 0;
135    } // get_surface()
136
137
138    // process and render media data
139    int render()
140    {
141        try
142        {
143            if (m_shutdown)
144                return 0;
145
146            HRESULT r;
147            LPDIRECT3DSURFACE9 pSurface;
148
149            r = get_surface(&pSurface);
150            if (FAILED(r))
151            {
152                return -1;
153            }
154
155            switch (m_mode)
156            {
157                case MODE_NOP:
158                    // no processing
159                    break;
160
161                case MODE_CPU:
162                {
163                    // process video frame on CPU
164                    D3DLOCKED_RECT memDesc = { 0, NULL };
165                    RECT rc = { 0, 0, m_width, m_height };
166
167                    r = pSurface->LockRect(&memDesc, &rc, 0);
168                    if (FAILED(r))
169                    {
170                        return -1;
171                    }
172
173                    cv::Mat m(m_height, m_width, CV_8UC4, memDesc.pBits, memDesc.Pitch);
174
175                    if (!m_disableProcessing)
176                    {
177                        // blur D3D9 surface with OpenCV on CPU
178                        cv::blur(m, m, cv::Size(15, 15), cv::Point(-7, -7));
179                    }
180
181                    r = pSurface->UnlockRect();
182                    if (FAILED(r))
183                    {
184                        return -1;
185                    }
186
187                    break;
188                }
189
190                case MODE_GPU:
191                {
192                    // process video frame on GPU
193                    cv::UMat u;
194
195                    cv::directx::convertFromDirect3DSurface9(pSurface, u);
196
197                    if (!m_disableProcessing)
198                    {
199                        // blur D3D9 surface with OpenCV on GPU with OpenCL
200                        cv::blur(u, u, cv::Size(15, 15), cv::Point(-7, -7));
201                    }
202
203                    cv::directx::convertToDirect3DSurface9(u, pSurface);
204
205                    break;
206                }
207
208            } // switch
209
210            print_info(pSurface, m_mode, getFps(), m_oclDevName);
211
212            // traditional DX render pipeline:
213            //   BitBlt surface to backBuffer and flip backBuffer to frontBuffer
214            r = m_pD3D9Dev->StretchRect(pSurface, NULL, m_pBackBuffer, NULL, D3DTEXF_NONE);
215            if (FAILED(r))
216            {
217                return -1;
218            }
219
220            // present the back buffer contents to the display
221            r = m_pD3D9Dev->Present(NULL, NULL, NULL, NULL);
222            if (FAILED(r))
223            {
224                return -1;
225            }
226        }  // try
227
228        catch (cv::Exception& e)
229        {
230            std::cerr << "Exception: " << e.what() << std::endl;
231            return 10;
232        }
233
234        return 0;
235    } // render()
236
237
238    void print_info(LPDIRECT3DSURFACE9 pSurface, int mode, float fps, cv::String oclDevName)
239    {
240        HDC hDC;
241
242        HRESULT r = pSurface->GetDC(&hDC);
243        if (FAILED(r))
244        {
245            return;
246        }
247
248        HFONT hFont = (HFONT)::GetStockObject(SYSTEM_FONT);
249
250        HFONT hOldFont = (HFONT)::SelectObject(hDC, hFont);
251
252        if (hOldFont)
253        {
254            TEXTMETRIC tm;
255            ::GetTextMetrics(hDC, &tm);
256
257            char buf[256];
258            int  y = 0;
259
260            buf[0] = 0;
261            sprintf(buf, "Mode: %s", m_modeStr[mode].c_str());
262            ::TextOut(hDC, 0, y, buf, (int)strlen(buf));
263
264            y += tm.tmHeight;
265            buf[0] = 0;
266            sprintf(buf, "FPS: %2.1f", fps);
267            ::TextOut(hDC, 0, y, buf, (int)strlen(buf));
268
269            y += tm.tmHeight;
270            buf[0] = 0;
271            sprintf(buf, "OpenCL device: %s", oclDevName.c_str());
272            ::TextOut(hDC, 0, y, buf, (int)strlen(buf));
273
274            ::SelectObject(hDC, hOldFont);
275        }
276
277        r = pSurface->ReleaseDC(hDC);
278
279        return;
280    } // print_info()
281
282
283    int cleanup(void)
284    {
285        SAFE_RELEASE(m_pSurface);
286        SAFE_RELEASE(m_pBackBuffer);
287        SAFE_RELEASE(m_pD3D9Dev);
288        SAFE_RELEASE(m_pD3D9);
289        D3DSample::cleanup();
290        return 0;
291    } // cleanup()
292
293private:
294    LPDIRECT3D9        m_pD3D9;
295    LPDIRECT3DDEVICE9  m_pD3D9Dev;
296    LPDIRECT3DSURFACE9 m_pBackBuffer;
297    LPDIRECT3DSURFACE9 m_pSurface;
298    cv::ocl::Context   m_oclCtx;
299    cv::String         m_oclPlatformName;
300    cv::String         m_oclDevName;
301};
302
303
304// main func
305int main(int argc, char** argv)
306{
307    std::string title = "D3D9 interop sample";
308    return d3d_app<D3D9WinApp>(argc, argv, title);
309}
310