11d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman/* 21d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman * Copyright (C) 2012 Intel Corporation. All Rights Reserved. 31d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman * 41d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman * This is free software; you can redistribute it and/or modify 51d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman * it under the terms of the GNU General Public License as published by 61d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman * the Free Software Foundation; either version 2 of the License, or 71d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman * (at your option) any later version. 81d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman * 91d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman * This software is distributed in the hope that it will be useful, 101d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman * but WITHOUT ANY WARRANTY; without even the implied warranty of 111d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 121d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman * GNU General Public License for more details. 131d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman * 141d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman * You should have received a copy of the GNU General Public License 151d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman * along with this software; if not, write to the Free Software 161d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, 171d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman * USA. 181d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman */ 191d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 201d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman#ifdef LIBVNCSERVER_CONFIG_LIBVA 211d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 221d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman#include <X11/Xlib.h> 231d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman#include <va/va_x11.h> 241d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 251d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartmanenum _slice_types { 261d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman SLICE_TYPE_P = 0, /* Predicted */ 271d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman SLICE_TYPE_B = 1, /* Bi-predicted */ 281d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman SLICE_TYPE_I = 2, /* Intra coded */ 291d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman}; 301d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 311d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman#define SURFACE_NUM 7 321d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 331d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg HartmanVADisplay va_dpy = NULL; 341d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg HartmanVAConfigID va_config_id; 351d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg HartmanVASurfaceID va_surface_id[SURFACE_NUM]; 361d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg HartmanVAContextID va_context_id = 0; 371d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 381d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg HartmanVABufferID va_pic_param_buf_id[SURFACE_NUM]; 391d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg HartmanVABufferID va_mat_param_buf_id[SURFACE_NUM]; 401d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg HartmanVABufferID va_sp_param_buf_id[SURFACE_NUM]; 411d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg HartmanVABufferID va_d_param_buf_id[SURFACE_NUM]; 421d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 431d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartmanstatic int cur_height = 0; 441d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartmanstatic int cur_width = 0; 451d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartmanstatic unsigned int num_frames = 0; 461d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartmanstatic int sid = 0; 471d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartmanstatic unsigned int frame_id = 0; 481d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartmanstatic int field_order_count = 0; 491d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartmanstatic VASurfaceID curr_surface = VA_INVALID_ID; 501d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 511d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg HartmanVAStatus gva_status; 521d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg HartmanVASurfaceStatus gsurface_status; 531d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman#define CHECK_SURF(X) \ 541d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman gva_status = vaQuerySurfaceStatus(va_dpy, X, &gsurface_status); \ 551d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman if (gsurface_status != 4) printf("ss: %d\n", gsurface_status); 561d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 571d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman#ifdef _DEBUG 581d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman#define DebugLog(A) rfbClientLog A 591d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman#else 601d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman#define DebugLog(A) 611d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman#endif 621d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 631d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman#define CHECK_VASTATUS(va_status,func) \ 641d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman if (va_status != VA_STATUS_SUCCESS) { \ 651d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman /*fprintf(stderr,"%s:%s (%d) failed,exit\n", __func__, func, __LINE__);*/ \ 661d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman rfbClientErr("%s:%s:%d failed (0x%x),exit\n", __func__, func, __LINE__, va_status); \ 671d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman exit(1); \ 681d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman } else { \ 691d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman /*fprintf(stderr,">> SUCCESS for: %s:%s (%d)\n", __func__, func, __LINE__);*/ \ 701d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman DebugLog(("%s:%s:%d success\n", __func__, func, __LINE__)); \ 711d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman } 721d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 731d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman/* 741d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman * Forward declarations 751d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman */ 761d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartmanstatic void h264_decode_frame(int f_width, int f_height, char *framedata, int framesize, int slice_type); 771d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartmanstatic void SetVAPictureParameterBufferH264(VAPictureParameterBufferH264 *p, int width, int height); 781d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartmanstatic void SetVASliceParameterBufferH264(VASliceParameterBufferH264 *p); 791d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartmanstatic void SetVASliceParameterBufferH264_Intra(VASliceParameterBufferH264 *p, int first); 801d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 811d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartmanstatic void put_updated_rectangle(rfbClient *client, int x, int y, int width, int height, int f_width, int f_height, int first_for_frame); 821d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartmanstatic void nv12_to_rgba(const VAImage vaImage, rfbClient *client, int ch_x, int ch_y, int ch_w, int ch_h); 831d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 841d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 851d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman/* FIXME: get this value from the server instead of hardcoding 32bit pixels */ 861d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman#define BPP (4 * 8) 871d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 881d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartmanstatic const char *string_of_FOURCC(uint32_t fourcc) 891d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman{ 901d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman static int buf; 911d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman static char str[2][5]; 921d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 931d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman buf ^= 1; 941d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman str[buf][0] = fourcc; 951d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman str[buf][1] = fourcc >> 8; 961d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman str[buf][2] = fourcc >> 16; 971d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman str[buf][3] = fourcc >> 24; 981d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman str[buf][4] = '\0'; 991d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman return str[buf]; 1001d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman} 1011d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 1021d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartmanstatic inline const char *string_of_VAImageFormat(VAImageFormat *imgfmt) 1031d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman{ 1041d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman return string_of_FOURCC(imgfmt->fourcc); 1051d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman} 1061d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 1071d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 1081d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartmanstatic rfbBool 1091d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg HartmanHandleH264 (rfbClient* client, int rx, int ry, int rw, int rh) 1101d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman{ 1111d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman rfbH264Header hdr; 1121d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman char *framedata; 1131d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 1141d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman DebugLog(("Framebuffer update with H264 (x: %d, y: %d, w: %d, h: %d)\n", rx, ry, rw, rh)); 1151d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 1161d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman /* First, read the frame size and allocate buffer to store the data */ 1171d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman if (!ReadFromRFBServer(client, (char *)&hdr, sz_rfbH264Header)) 1181d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman return FALSE; 1191d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 1201d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman hdr.slice_type = rfbClientSwap32IfLE(hdr.slice_type); 1211d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman hdr.nBytes = rfbClientSwap32IfLE(hdr.nBytes); 1221d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman hdr.width = rfbClientSwap32IfLE(hdr.width); 1231d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman hdr.height = rfbClientSwap32IfLE(hdr.height); 1241d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 1251d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman framedata = (char*) malloc(hdr.nBytes); 1261d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 1271d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman /* Obtain frame data from the server */ 1281d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman DebugLog(("Reading %d bytes of frame data (type: %d)\n", hdr.nBytes, hdr.slice_type)); 1291d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman if (!ReadFromRFBServer(client, framedata, hdr.nBytes)) 1301d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman return FALSE; 1311d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 1321d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman /* First make sure we have a large enough raw buffer to hold the 1331d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman * decompressed data. In practice, with a fixed BPP, fixed frame 1341d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman * buffer size and the first update containing the entire frame 1351d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman * buffer, this buffer allocation should only happen once, on the 1361d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman * first update. 1371d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman */ 1381d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman if ( client->raw_buffer_size < (( rw * rh ) * ( BPP / 8 ))) { 1391d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman if ( client->raw_buffer != NULL ) { 1401d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman free( client->raw_buffer ); 1411d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman } 1421d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 1431d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman client->raw_buffer_size = (( rw * rh ) * ( BPP / 8 )); 1441d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman client->raw_buffer = (char*) malloc( client->raw_buffer_size ); 1451d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman rfbClientLog("Allocated raw buffer of %d bytes (%dx%dx%d BPP)\n", client->raw_buffer_size, rw, rh, BPP); 1461d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman } 1471d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 1481d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman /* Decode frame if frame data was sent. Server only sends frame data for the first 1491d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman * framebuffer update message for a particular frame buffer contents. 1501d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman * If more than 1 rectangle is updated, the messages after the first one (with 1511d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman * the H.264 frame) have nBytes == 0. 1521d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman */ 1531d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman if (hdr.nBytes > 0) { 1541d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman DebugLog((" decoding %d bytes of H.264 data\n", hdr.nBytes)); 1551d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman h264_decode_frame(hdr.width, hdr.height, framedata, hdr.nBytes, hdr.slice_type); 1561d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman } 1571d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 1581d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman DebugLog((" updating rectangle (%d, %d)-(%d, %d)\n", rx, ry, rw, rh)); 1591d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman put_updated_rectangle(client, rx, ry, rw, rh, hdr.width, hdr.height, hdr.nBytes != 0); 1601d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 1611d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman free(framedata); 1621d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 1631d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman return TRUE; 1641d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman} 1651d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 1661d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartmanstatic void h264_cleanup_decoder() 1671d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman{ 1681d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman VAStatus va_status; 1691d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 1701d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman rfbClientLog("%s()\n", __FUNCTION__); 1711d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 1721d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman if (va_surface_id[0] != VA_INVALID_ID) { 1731d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman va_status = vaDestroySurfaces(va_dpy, &va_surface_id[0], SURFACE_NUM); 1741d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman CHECK_VASTATUS(va_status, "vaDestroySurfaces"); 1751d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman } 1761d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 1771d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman if (va_context_id) { 1781d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman va_status = vaDestroyContext(va_dpy, va_context_id); 1791d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman CHECK_VASTATUS(va_status, "vaDestroyContext"); 1801d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman va_context_id = 0; 1811d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman } 1821d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 1831d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman num_frames = 0; 1841d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman sid = 0; 1851d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman frame_id = 0; 1861d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman field_order_count = 0; 1871d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman} 1881d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 1891d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartmanstatic void h264_init_decoder(int width, int height) 1901d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman{ 1911d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman VAStatus va_status; 1921d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 1931d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman if (va_context_id) { 1941d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman rfbClientLog("%s: va_dpy already initialized\n", __FUNCTION__); 1951d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman } 1961d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 1971d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman if (va_dpy != NULL) { 1981d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman rfbClientLog("%s: Re-initializing H.264 decoder\n", __FUNCTION__); 1991d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman } 2001d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman else { 2011d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman rfbClientLog("%s: initializing H.264 decoder\n", __FUNCTION__); 2021d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 2031d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman /* Attach VA display to local X display */ 2041d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman Display *win_display = (Display *)XOpenDisplay(":0.0"); 2051d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman if (win_display == NULL) { 2061d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman rfbClientErr("Can't connect to local display\n"); 2071d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman exit(-1); 2081d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman } 2091d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 2101d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman int major_ver, minor_ver; 2111d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman va_dpy = vaGetDisplay(win_display); 2121d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman va_status = vaInitialize(va_dpy, &major_ver, &minor_ver); 2131d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman CHECK_VASTATUS(va_status, "vaInitialize"); 2141d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman rfbClientLog("%s: libva version %d.%d found\n", __FUNCTION__, major_ver, minor_ver); 2151d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman } 2161d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 2171d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman /* Check for VLD entrypoint */ 2181d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman int num_entrypoints; 2191d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman VAEntrypoint entrypoints[5]; 2201d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman int vld_entrypoint_found = 0; 2211d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 2221d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman /* Change VAProfileH264High if needed */ 2231d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman VAProfile profile = VAProfileH264High; 2241d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman va_status = vaQueryConfigEntrypoints(va_dpy, profile, entrypoints, &num_entrypoints); 2251d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman CHECK_VASTATUS(va_status, "vaQueryConfigEntrypoints"); 2261d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman int i; 2271d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman for (i = 0; i < num_entrypoints; ++i) { 2281d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman if (entrypoints[i] == VAEntrypointVLD) { 2291d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman vld_entrypoint_found = 1; 2301d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman break; 2311d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman } 2321d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman } 2331d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 2341d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman if (vld_entrypoint_found == 0) { 2351d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman rfbClientErr("VLD entrypoint not found\n"); 2361d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman exit(1); 2371d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman } 2381d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 2391d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman /* Create configuration for the decode pipeline */ 2401d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman VAConfigAttrib attrib; 2411d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman attrib.type = VAConfigAttribRTFormat; 2421d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman va_status = vaCreateConfig(va_dpy, profile, VAEntrypointVLD, &attrib, 1, &va_config_id); 2431d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman CHECK_VASTATUS(va_status, "vaCreateConfig"); 2441d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 2451d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman /* Create VA surfaces */ 2461d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman for (i = 0; i < SURFACE_NUM; ++i) { 2471d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman va_surface_id[i] = VA_INVALID_ID; 2481d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman va_pic_param_buf_id[i] = VA_INVALID_ID; 2491d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman va_mat_param_buf_id[i] = VA_INVALID_ID; 2501d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman va_sp_param_buf_id[i] = VA_INVALID_ID; 2511d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman va_d_param_buf_id[i] = VA_INVALID_ID; 2521d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman } 2531d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman va_status = vaCreateSurfaces(va_dpy, width, height, VA_RT_FORMAT_YUV420, SURFACE_NUM, &va_surface_id[0]); 2541d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman CHECK_VASTATUS(va_status, "vaCreateSurfaces"); 2551d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman for (i = 0; i < SURFACE_NUM; ++i) { 2561d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman DebugLog(("%s: va_surface_id[%d] = %p\n", __FUNCTION__, i, va_surface_id[i])); 2571d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman } 2581d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 2591d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman /* Create VA context */ 2601d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman va_status = vaCreateContext(va_dpy, va_config_id, width, height, 0/*VA_PROGRESSIVE*/, &va_surface_id[0], SURFACE_NUM, &va_context_id); 2611d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman CHECK_VASTATUS(va_status, "vaCreateContext"); 2621d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman DebugLog(("%s: VA context created (id: %d)\n", __FUNCTION__, va_context_id)); 2631d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 2641d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 2651d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman /* Instantiate decode pipeline */ 2661d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman va_status = vaBeginPicture(va_dpy, va_context_id, va_surface_id[0]); 2671d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman CHECK_VASTATUS(va_status, "vaBeginPicture"); 2681d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 2691d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman rfbClientLog("%s: H.264 decoder initialized\n", __FUNCTION__); 2701d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman} 2711d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 2721d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartmanstatic void h264_decode_frame(int f_width, int f_height, char *framedata, int framesize, int slice_type) 2731d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman{ 2741d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman VAStatus va_status; 2751d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 2761d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman DebugLog(("%s: called for frame of %d bytes (%dx%d) slice_type=%d\n", __FUNCTION__, framesize, width, height, slice_type)); 2771d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 2781d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman /* Initialize decode pipeline if necessary */ 2791d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman if ( (f_width > cur_width) || (f_height > cur_height) ) { 2801d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman if (va_dpy != NULL) 2811d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman h264_cleanup_decoder(); 2821d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman cur_width = f_width; 2831d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman cur_height = f_height; 2841d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 2851d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman h264_init_decoder(f_width, f_height); 2861d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman rfbClientLog("%s: decoder initialized\n", __FUNCTION__); 2871d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman } 2881d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 2891d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman /* Decode frame */ 2901d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman static VAPictureH264 va_picture_h264, va_old_picture_h264; 2911d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 2921d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman /* The server should always send an I-frame when a new client connects 2931d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman * or when the resolution of the framebuffer changes, but we check 2941d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman * just in case. 2951d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman */ 2961d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman if ( (slice_type != SLICE_TYPE_I) && (num_frames == 0) ) { 2971d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman rfbClientLog("First frame is not an I frame !!! Skipping!!!\n"); 2981d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman return; 2991d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman } 3001d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 3011d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman DebugLog(("%s: frame_id=%d va_surface_id[%d]=0x%x field_order_count=%d\n", __FUNCTION__, frame_id, sid, va_surface_id[sid], field_order_count)); 3021d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 3031d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman va_picture_h264.picture_id = va_surface_id[sid]; 3041d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman va_picture_h264.frame_idx = frame_id; 3051d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman va_picture_h264.flags = 0; 3061d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman va_picture_h264.BottomFieldOrderCnt = field_order_count; 3071d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman va_picture_h264.TopFieldOrderCnt = field_order_count; 3081d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 3091d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman /* Set up picture parameter buffer */ 3101d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman if (va_pic_param_buf_id[sid] == VA_INVALID_ID) { 3111d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman va_status = vaCreateBuffer(va_dpy, va_context_id, VAPictureParameterBufferType, sizeof(VAPictureParameterBufferH264), 1, NULL, &va_pic_param_buf_id[sid]); 3121d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman CHECK_VASTATUS(va_status, "vaCreateBuffer(PicParam)"); 3131d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman } 3141d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman CHECK_SURF(va_surface_id[sid]); 3151d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 3161d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman VAPictureParameterBufferH264 *pic_param_buf = NULL; 3171d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman va_status = vaMapBuffer(va_dpy, va_pic_param_buf_id[sid], (void **)&pic_param_buf); 3181d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman CHECK_VASTATUS(va_status, "vaMapBuffer(PicParam)"); 3191d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 3201d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman SetVAPictureParameterBufferH264(pic_param_buf, f_width, f_height); 3211d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman memcpy(&pic_param_buf->CurrPic, &va_picture_h264, sizeof(VAPictureH264)); 3221d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 3231d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman if (slice_type == SLICE_TYPE_P) { 3241d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman memcpy(&pic_param_buf->ReferenceFrames[0], &va_old_picture_h264, sizeof(VAPictureH264)); 3251d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman pic_param_buf->ReferenceFrames[0].flags = 0; 3261d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman } 3271d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman else if (slice_type != SLICE_TYPE_I) { 3281d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman rfbClientLog("Frame type %d not supported!!!\n"); 3291d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman return; 3301d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman } 3311d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman pic_param_buf->frame_num = frame_id; 3321d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 3331d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman va_status = vaUnmapBuffer(va_dpy, va_pic_param_buf_id[sid]); 3341d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman CHECK_VASTATUS(va_status, "vaUnmapBuffer(PicParam)"); 3351d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 3361d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman /* Set up IQ matrix buffer */ 3371d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman if (va_mat_param_buf_id[sid] == VA_INVALID_ID) { 3381d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman va_status = vaCreateBuffer(va_dpy, va_context_id, VAIQMatrixBufferType, sizeof(VAIQMatrixBufferH264), 1, NULL, &va_mat_param_buf_id[sid]); 3391d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman CHECK_VASTATUS(va_status, "vaCreateBuffer(IQMatrix)"); 3401d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman } 3411d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman CHECK_SURF(va_surface_id[sid]); 3421d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 3431d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman VAIQMatrixBufferH264 *iq_matrix_buf = NULL; 3441d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman va_status = vaMapBuffer(va_dpy, va_mat_param_buf_id[sid], (void **)&iq_matrix_buf); 3451d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman CHECK_VASTATUS(va_status, "vaMapBuffer(IQMatrix)"); 3461d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 3471d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman static const unsigned char m_MatrixBufferH264[]= { 3481d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman /* ScalingList4x4[6][16] */ 3491d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10, 3501d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10, 3511d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10, 3521d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10, 3531d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10, 3541d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10, 3551d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman /* ScalingList8x8[2][64] */ 3561d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 3571d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 3581d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 3591d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 3601d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 3611d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 3621d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 3631d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 3641d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 3651d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 3661d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 3671d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 3681d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 3691d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 3701d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 3711d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 3721d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman }; 3731d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 3741d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman memcpy(iq_matrix_buf, m_MatrixBufferH264, 224); 3751d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman va_status = vaUnmapBuffer(va_dpy, va_mat_param_buf_id[sid]); 3761d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman CHECK_VASTATUS(va_status, "vaUnmapBuffer(IQMatrix)"); 3771d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 3781d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman VABufferID buffer_ids[2]; 3791d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman buffer_ids[0] = va_pic_param_buf_id[sid]; 3801d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman buffer_ids[1] = va_mat_param_buf_id[sid]; 3811d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 3821d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman CHECK_SURF(va_surface_id[sid]); 3831d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman va_status = vaRenderPicture(va_dpy, va_context_id, buffer_ids, 2); 3841d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman CHECK_VASTATUS(va_status, "vaRenderPicture"); 3851d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 3861d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman /* Set up slice parameter buffer */ 3871d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman if (va_sp_param_buf_id[sid] == VA_INVALID_ID) { 3881d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman va_status = vaCreateBuffer(va_dpy, va_context_id, VASliceParameterBufferType, sizeof(VASliceParameterBufferH264), 1, NULL, &va_sp_param_buf_id[sid]); 3891d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman CHECK_VASTATUS(va_status, "vaCreateBuffer(SliceParam)"); 3901d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman } 3911d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman CHECK_SURF(va_surface_id[sid]); 3921d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 3931d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman VASliceParameterBufferH264 *slice_param_buf = NULL; 3941d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman va_status = vaMapBuffer(va_dpy, va_sp_param_buf_id[sid], (void **)&slice_param_buf); 3951d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman CHECK_VASTATUS(va_status, "vaMapBuffer(SliceParam)"); 3961d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 3971d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman static int t2_first = 1; 3981d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman if (slice_type == SLICE_TYPE_I) { 3991d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman SetVASliceParameterBufferH264_Intra(slice_param_buf, t2_first); 4001d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman t2_first = 0; 4011d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman } else { 4021d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman SetVASliceParameterBufferH264(slice_param_buf); 4031d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman memcpy(&slice_param_buf->RefPicList0[0], &va_old_picture_h264, sizeof(VAPictureH264)); 4041d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman slice_param_buf->RefPicList0[0].flags = 0; 4051d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman } 4061d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman slice_param_buf->slice_data_bit_offset = 0; 4071d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman slice_param_buf->slice_data_size = framesize; 4081d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 4091d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman va_status = vaUnmapBuffer(va_dpy, va_sp_param_buf_id[sid]); 4101d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman CHECK_VASTATUS(va_status, "vaUnmapBuffer(SliceParam)"); 4111d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman CHECK_SURF(va_surface_id[sid]); 4121d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 4131d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman /* Set up slice data buffer and copy H.264 encoded data */ 4141d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman if (va_d_param_buf_id[sid] == VA_INVALID_ID) { 4151d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman /* TODO use estimation matching framebuffer dimensions instead of this large value */ 4161d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman va_status = vaCreateBuffer(va_dpy, va_context_id, VASliceDataBufferType, 4177920, 1, NULL, &va_d_param_buf_id[sid]); /* 1080p size */ 4171d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman CHECK_VASTATUS(va_status, "vaCreateBuffer(SliceData)"); 4181d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman } 4191d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 4201d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman char *slice_data_buf; 4211d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman va_status = vaMapBuffer(va_dpy, va_d_param_buf_id[sid], (void **)&slice_data_buf); 4221d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman CHECK_VASTATUS(va_status, "vaMapBuffer(SliceData)"); 4231d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman memcpy(slice_data_buf, framedata, framesize); 4241d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 4251d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman CHECK_SURF(va_surface_id[sid]); 4261d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman va_status = vaUnmapBuffer(va_dpy, va_d_param_buf_id[sid]); 4271d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman CHECK_VASTATUS(va_status, "vaUnmapBuffer(SliceData)"); 4281d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 4291d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman buffer_ids[0] = va_sp_param_buf_id[sid]; 4301d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman buffer_ids[1] = va_d_param_buf_id[sid]; 4311d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 4321d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman CHECK_SURF(va_surface_id[sid]); 4331d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman va_status = vaRenderPicture(va_dpy, va_context_id, buffer_ids, 2); 4341d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman CHECK_VASTATUS(va_status, "vaRenderPicture"); 4351d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 4361d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman va_status = vaEndPicture(va_dpy, va_context_id); 4371d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman CHECK_VASTATUS(va_status, "vaEndPicture"); 4381d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 4391d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman /* Prepare next one... */ 4401d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman int sid_new = (sid + 1) % SURFACE_NUM; 4411d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman DebugLog(("%s: new Surface ID = %d\n", __FUNCTION__, sid_new)); 4421d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman va_status = vaBeginPicture(va_dpy, va_context_id, va_surface_id[sid_new]); 4431d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman CHECK_VASTATUS(va_status, "vaBeginPicture"); 4441d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 4451d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman /* Get decoded data */ 4461d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman va_status = vaSyncSurface(va_dpy, va_surface_id[sid]); 4471d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman CHECK_VASTATUS(va_status, "vaSyncSurface"); 4481d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman CHECK_SURF(va_surface_id[sid]); 4491d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 4501d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman curr_surface = va_surface_id[sid]; 4511d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 4521d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman sid = sid_new; 4531d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 4541d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman field_order_count += 2; 4551d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman ++frame_id; 4561d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman if (frame_id > 15) { 4571d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman frame_id = 0; 4581d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman } 4591d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 4601d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman ++num_frames; 4611d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 4621d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman memcpy(&va_old_picture_h264, &va_picture_h264, sizeof(VAPictureH264)); 4631d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman} 4641d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 4651d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartmanstatic void put_updated_rectangle(rfbClient *client, int x, int y, int width, int height, int f_width, int f_height, int first_for_frame) 4661d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman{ 4671d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman if (curr_surface == VA_INVALID_ID) { 4681d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman rfbClientErr("%s: called, but current surface is invalid\n", __FUNCTION__); 4691d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman return; 4701d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman } 4711d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 4721d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman VAStatus va_status; 4731d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 4741d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman if (client->outputWindow) { 4751d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman /* use efficient vaPutSurface() method of putting the framebuffer on the screen */ 4761d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman if (first_for_frame) { 4771d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman /* vaPutSurface() clears window contents outside the given destination rectangle => always update full screen. */ 4781d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman va_status = vaPutSurface(va_dpy, curr_surface, client->outputWindow, 0, 0, f_width, f_height, 0, 0, f_width, f_height, NULL, 0, VA_FRAME_PICTURE); 4791d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman CHECK_VASTATUS(va_status, "vaPutSurface"); 4801d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman } 4811d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman } 4821d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman else if (client->frameBuffer) { 4831d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman /* ... or copy the changed framebuffer region manually as a fallback */ 4841d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman VAImage decoded_image; 4851d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman decoded_image.image_id = VA_INVALID_ID; 4861d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman decoded_image.buf = VA_INVALID_ID; 4871d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman va_status = vaDeriveImage(va_dpy, curr_surface, &decoded_image); 4881d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman CHECK_VASTATUS(va_status, "vaDeriveImage"); 4891d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 4901d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman if ((decoded_image.image_id == VA_INVALID_ID) || (decoded_image.buf == VA_INVALID_ID)) { 4911d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman rfbClientErr("%s: vaDeriveImage() returned success but VA image is invalid (id: %d, buf: %d)\n", __FUNCTION__, decoded_image.image_id, decoded_image.buf); 4921d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman } 4931d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 4941d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman nv12_to_rgba(decoded_image, client, x, y, width, height); 4951d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 4961d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman va_status = vaDestroyImage(va_dpy, decoded_image.image_id); 4971d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman CHECK_VASTATUS(va_status, "vaDestroyImage"); 4981d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman } 4991d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman} 5001d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 5011d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartmanstatic void SetVAPictureParameterBufferH264(VAPictureParameterBufferH264 *p, int width, int height) 5021d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman{ 5031d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman int i; 5041d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman unsigned int width_in_mbs = (width + 15) / 16; 5051d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman unsigned int height_in_mbs = (height + 15) / 16; 5061d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 5071d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman memset(p, 0, sizeof(VAPictureParameterBufferH264)); 5081d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman p->picture_width_in_mbs_minus1 = width_in_mbs - 1; 5091d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman p->picture_height_in_mbs_minus1 = height_in_mbs - 1; 5101d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman p->num_ref_frames = 1; 5111d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman p->seq_fields.value = 145; 5121d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman p->pic_fields.value = 0x501; 5131d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman for (i = 0; i < 16; i++) { 5141d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman p->ReferenceFrames[i].flags = VA_PICTURE_H264_INVALID; 5151d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman p->ReferenceFrames[i].picture_id = 0xffffffff; 5161d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman } 5171d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman} 5181d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 5191d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartmanstatic void SetVASliceParameterBufferH264(VASliceParameterBufferH264 *p) 5201d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman{ 5211d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman int i; 5221d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman memset(p, 0, sizeof(VASliceParameterBufferH264)); 5231d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman p->slice_data_size = 0; 5241d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman p->slice_data_bit_offset = 64; 5251d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman p->slice_alpha_c0_offset_div2 = 2; 5261d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman p->slice_beta_offset_div2 = 2; 5271d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman p->chroma_weight_l0_flag = 1; 5281d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman p->chroma_weight_l0[0][0]=1; 5291d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman p->chroma_offset_l0[0][0]=0; 5301d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman p->chroma_weight_l0[0][1]=1; 5311d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman p->chroma_offset_l0[0][1]=0; 5321d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman p->luma_weight_l1_flag = 1; 5331d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman p->chroma_weight_l1_flag = 1; 5341d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman p->luma_weight_l0[0]=0x01; 5351d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman for (i = 0; i < 32; i++) { 5361d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman p->RefPicList0[i].flags = VA_PICTURE_H264_INVALID; 5371d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman p->RefPicList1[i].flags = VA_PICTURE_H264_INVALID; 5381d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman } 5391d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman p->RefPicList1[0].picture_id = 0xffffffff; 5401d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman} 5411d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 5421d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartmanstatic void SetVASliceParameterBufferH264_Intra(VASliceParameterBufferH264 *p, int first) 5431d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman{ 5441d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman int i; 5451d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman memset(p, 0, sizeof(VASliceParameterBufferH264)); 5461d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman p->slice_data_size = 0; 5471d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman p->slice_data_bit_offset = 64; 5481d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman p->slice_alpha_c0_offset_div2 = 2; 5491d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman p->slice_beta_offset_div2 = 2; 5501d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman p->slice_type = 2; 5511d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman if (first) { 5521d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman p->luma_weight_l0_flag = 1; 5531d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman p->chroma_weight_l0_flag = 1; 5541d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman p->luma_weight_l1_flag = 1; 5551d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman p->chroma_weight_l1_flag = 1; 5561d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman } else { 5571d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman p->chroma_weight_l0_flag = 1; 5581d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman p->chroma_weight_l0[0][0]=1; 5591d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman p->chroma_offset_l0[0][0]=0; 5601d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman p->chroma_weight_l0[0][1]=1; 5611d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman p->chroma_offset_l0[0][1]=0; 5621d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman p->luma_weight_l1_flag = 1; 5631d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman p->chroma_weight_l1_flag = 1; 5641d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman p->luma_weight_l0[0]=0x01; 5651d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman } 5661d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman for (i = 0; i < 32; i++) { 5671d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman p->RefPicList0[i].flags = VA_PICTURE_H264_INVALID; 5681d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman p->RefPicList1[i].flags = VA_PICTURE_H264_INVALID; 5691d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman } 5701d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman p->RefPicList1[0].picture_id = 0xffffffff; 5711d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman p->RefPicList0[0].picture_id = 0xffffffff; 5721d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman} 5731d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 5741d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartmanstatic void nv12_to_rgba(const VAImage vaImage, rfbClient *client, int ch_x, int ch_y, int ch_w, int ch_h) 5751d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman{ 5761d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman DebugLog(("%s: converting region (%d, %d)-(%d, %d) from NV12->RGBA\n", __FUNCTION__, ch_x, ch_y, ch_w, ch_h)); 5771d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 5781d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman VAStatus va_status; 5791d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman uint8_t *nv12_buf; 5801d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman va_status = vaMapBuffer(va_dpy, vaImage.buf, (void **)&nv12_buf); 5811d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman CHECK_VASTATUS(va_status, "vaMapBuffer(DecodedData)"); 5821d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 5831d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman /* adjust x, y, width, height of the affected area so 5841d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman * x, y, width and height are always even. 5851d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman */ 5861d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman if (ch_x % 2) { --ch_x; ++ch_w; } 5871d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman if (ch_y % 2) { --ch_y; ++ch_h; } 5881d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman if ((ch_x + ch_w) % 2) { ++ch_w; } 5891d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman if ((ch_y + ch_h) % 2) { ++ch_h; } 5901d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 5911d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman /* point nv12_buf and dst to upper left corner of changed area */ 5921d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman uint8_t *nv12_y = &nv12_buf[vaImage.offsets[0] + vaImage.pitches[0] * ch_y + ch_x]; 5931d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman uint8_t *nv12_uv = &nv12_buf[vaImage.offsets[1] + vaImage.pitches[1] * (ch_y / 2) + ch_x]; 5941d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman uint32_t *dst = &((uint32_t*)client->frameBuffer)[client->width * ch_y + ch_x]; 5951d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 5961d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman /* TODO: optimize R, G, B calculation. Possible ways to do this: 5971d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman * - use lookup tables 5981d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman * - convert from floating point to integer arithmetic 5991d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman * - use MMX/SSE to vectorize calculations 6001d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman * - use GPU (VA VPP, shader...) 6011d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman */ 6021d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman int src_x, src_y; 6031d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman for (src_y = 0; src_y < ch_h; src_y += 2) { 6041d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman for (src_x = 0; src_x < ch_w; src_x += 2) { 6051d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman uint8_t nv_u = nv12_uv[src_x]; 6061d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman uint8_t nv_v = nv12_uv[src_x + 1]; 6071d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman uint8_t nv_y[4] = { nv12_y[ src_x], nv12_y[ src_x + 1], 6081d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman nv12_y[vaImage.pitches[0] + src_x], nv12_y[vaImage.pitches[0] + src_x + 1] }; 6091d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 6101d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman int i; 6111d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman for (i = 0; i < 4; ++i) { 6121d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman double R = 1.164 * (nv_y[i] - 16) + 1.596 * (nv_v - 128); 6131d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman double G = 1.164 * (nv_y[i] - 16) - 0.391 * (nv_u - 128) - 0.813 * (nv_v - 128); 6141d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman double B = 1.164 * (nv_y[i] - 16) + 2.018 * (nv_u - 128); 6151d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 6161d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman /* clamp R, G, B values. For some Y, U, V combinations, 6171d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman * the results of the above calculations fall outside of 6181d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman * the range 0-255. 6191d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman */ 6201d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman if (R < 0.0) R = 0.0; 6211d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman if (G < 0.0) G = 0.0; 6221d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman if (B < 0.0) B = 0.0; 6231d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman if (R > 255.0) R = 255.0; 6241d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman if (G > 255.0) G = 255.0; 6251d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman if (B > 255.0) B = 255.0; 6261d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 6271d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman dst[client->width * (i / 2) + src_x + (i % 2)] = 0 6281d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman | ((unsigned int)(R + 0.5) << client->format.redShift) 6291d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman | ((unsigned int)(G + 0.5) << client->format.greenShift) 6301d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman | ((unsigned int)(B + 0.5) << client->format.blueShift); 6311d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman } 6321d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman } 6331d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 6341d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman nv12_y += 2 * vaImage.pitches[0]; 6351d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman nv12_uv += vaImage.pitches[1]; 6361d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman dst += 2 * client->width; 6371d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman } 6381d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 6391d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman CHECK_SURF(va_surface_id[sid]); 6401d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman va_status = vaUnmapBuffer(va_dpy, vaImage.buf); 6411d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman CHECK_VASTATUS(va_status, "vaUnmapBuffer(DecodedData)"); 6421d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman} 6431d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman 6441d80c6a8d34f59543f7df1963c22d7efa292bcb0Greg Hartman#endif /* LIBVNCSERVER_CONFIG_LIBVA */ 645