1a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat/* 2a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat * corre.c 3a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat * 4a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat * Routines to implement Compact Rise-and-Run-length Encoding (CoRRE). This 5a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat * code is based on krw's original javatel rfbserver. 6a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat */ 7a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 8a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat/* 9a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat * Copyright (C) 2002 RealVNC Ltd. 10a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat * OSXvnc Copyright (C) 2001 Dan McGuirk <mcguirk@incompleteness.net>. 11a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat * Original Xvnc code Copyright (C) 1999 AT&T Laboratories Cambridge. 12a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat * All Rights Reserved. 13a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat * 14a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat * This is free software; you can redistribute it and/or modify 15a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat * it under the terms of the GNU General Public License as published by 16a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat * the Free Software Foundation; either version 2 of the License, or 17a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat * (at your option) any later version. 18a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat * 19a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat * This software is distributed in the hope that it will be useful, 20a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat * but WITHOUT ANY WARRANTY; without even the implied warranty of 21a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 22a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat * GNU General Public License for more details. 23a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat * 24a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat * You should have received a copy of the GNU General Public License 25a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat * along with this software; if not, write to the Free Software 26a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, 27a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat * USA. 28a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat */ 29a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 30a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat#include <rfb/rfb.h> 31a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 32a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat/* 33a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat * cl->beforeEncBuf contains pixel data in the client's format. 34a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat * cl->afterEncBuf contains the RRE encoded version. If the RRE encoded version is 35a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat * larger than the raw data or if it exceeds cl->afterEncBufSize then 36a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat * raw encoding is used instead. 37a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat */ 38a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 39a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehatstatic int subrectEncode8(rfbClientPtr cl, uint8_t *data, int w, int h); 40a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehatstatic int subrectEncode16(rfbClientPtr cl, uint16_t *data, int w, int h); 41a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehatstatic int subrectEncode32(rfbClientPtr cl, uint32_t *data, int w, int h); 42a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehatstatic uint32_t getBgColour(char *data, int size, int bpp); 43a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehatstatic rfbBool rfbSendSmallRectEncodingCoRRE(rfbClientPtr cl, int x, int y, 44a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat int w, int h); 45a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 46a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 47a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat/* 48a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat * rfbSendRectEncodingCoRRE - send an arbitrary size rectangle using CoRRE 49a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat * encoding. 50a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat */ 51a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 52a430b2b5ca4f0967836f5820e8f03adc17fc0a24San MehatrfbBool 53a430b2b5ca4f0967836f5820e8f03adc17fc0a24San MehatrfbSendRectEncodingCoRRE(rfbClientPtr cl, 54a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat int x, 55a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat int y, 56a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat int w, 57a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat int h) 58a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat{ 59a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat if (h > cl->correMaxHeight) { 60a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat return (rfbSendRectEncodingCoRRE(cl, x, y, w, cl->correMaxHeight) && 61a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat rfbSendRectEncodingCoRRE(cl, x, y + cl->correMaxHeight, w, 62a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat h - cl->correMaxHeight)); 63a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat } 64a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 65a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat if (w > cl->correMaxWidth) { 66a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat return (rfbSendRectEncodingCoRRE(cl, x, y, cl->correMaxWidth, h) && 67a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat rfbSendRectEncodingCoRRE(cl, x + cl->correMaxWidth, y, 68a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat w - cl->correMaxWidth, h)); 69a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat } 70a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 71a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat rfbSendSmallRectEncodingCoRRE(cl, x, y, w, h); 72a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat return TRUE; 73a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat} 74a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 75a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 76a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 77a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat/* 78a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat * rfbSendSmallRectEncodingCoRRE - send a small (guaranteed < 256x256) 79a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat * rectangle using CoRRE encoding. 80a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat */ 81a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 82a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehatstatic rfbBool 83a430b2b5ca4f0967836f5820e8f03adc17fc0a24San MehatrfbSendSmallRectEncodingCoRRE(rfbClientPtr cl, 84a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat int x, 85a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat int y, 86a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat int w, 87a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat int h) 88a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat{ 89a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat rfbFramebufferUpdateRectHeader rect; 90a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat rfbRREHeader hdr; 91a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat int nSubrects; 92a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat int i; 93a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat char *fbptr = (cl->scaledScreen->frameBuffer + (cl->scaledScreen->paddedWidthInBytes * y) 94a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat + (x * (cl->scaledScreen->bitsPerPixel / 8))); 95a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 96a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat int maxRawSize = (cl->scaledScreen->width * cl->scaledScreen->height 97a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat * (cl->format.bitsPerPixel / 8)); 98a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 99a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat if (cl->beforeEncBufSize < maxRawSize) { 100a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat cl->beforeEncBufSize = maxRawSize; 101a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat if (cl->beforeEncBuf == NULL) 102a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat cl->beforeEncBuf = (char *)malloc(cl->beforeEncBufSize); 103a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat else 104a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat cl->beforeEncBuf = (char *)realloc(cl->beforeEncBuf, cl->beforeEncBufSize); 105a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat } 106a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 107a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat if (cl->afterEncBufSize < maxRawSize) { 108a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat cl->afterEncBufSize = maxRawSize; 109a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat if (cl->afterEncBuf == NULL) 110a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat cl->afterEncBuf = (char *)malloc(cl->afterEncBufSize); 111a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat else 112a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat cl->afterEncBuf = (char *)realloc(cl->afterEncBuf, cl->afterEncBufSize); 113a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat } 114a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 115a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat (*cl->translateFn)(cl->translateLookupTable,&(cl->screen->serverFormat), 116a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat &cl->format, fbptr, cl->beforeEncBuf, 117a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat cl->scaledScreen->paddedWidthInBytes, w, h); 118a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 119a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat switch (cl->format.bitsPerPixel) { 120a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat case 8: 121a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat nSubrects = subrectEncode8(cl, (uint8_t *)cl->beforeEncBuf, w, h); 122a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat break; 123a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat case 16: 124a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat nSubrects = subrectEncode16(cl, (uint16_t *)cl->beforeEncBuf, w, h); 125a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat break; 126a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat case 32: 127a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat nSubrects = subrectEncode32(cl, (uint32_t *)cl->beforeEncBuf, w, h); 128a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat break; 129a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat default: 130a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat rfbLog("getBgColour: bpp %d?\n",cl->format.bitsPerPixel); 131a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat return FALSE; 132a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat } 133a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 134a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat if (nSubrects < 0) { 135a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 136a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat /* RRE encoding was too large, use raw */ 137a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 138a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat return rfbSendRectEncodingRaw(cl, x, y, w, h); 139a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat } 140a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 141a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat rfbStatRecordEncodingSent(cl,rfbEncodingCoRRE, 142a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat sz_rfbFramebufferUpdateRectHeader + sz_rfbRREHeader + cl->afterEncBufLen, 143a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat sz_rfbFramebufferUpdateRectHeader + w * h * (cl->format.bitsPerPixel / 8)); 144a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 145a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat if (cl->ublen + sz_rfbFramebufferUpdateRectHeader + sz_rfbRREHeader 146a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat > UPDATE_BUF_SIZE) 147a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat { 148a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat if (!rfbSendUpdateBuf(cl)) 149a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat return FALSE; 150a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat } 151a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 152a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat rect.r.x = Swap16IfLE(x); 153a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat rect.r.y = Swap16IfLE(y); 154a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat rect.r.w = Swap16IfLE(w); 155a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat rect.r.h = Swap16IfLE(h); 156a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat rect.encoding = Swap32IfLE(rfbEncodingCoRRE); 157a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 158a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat memcpy(&cl->updateBuf[cl->ublen], (char *)&rect, 159a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat sz_rfbFramebufferUpdateRectHeader); 160a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat cl->ublen += sz_rfbFramebufferUpdateRectHeader; 161a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 162a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat hdr.nSubrects = Swap32IfLE(nSubrects); 163a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 164a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat memcpy(&cl->updateBuf[cl->ublen], (char *)&hdr, sz_rfbRREHeader); 165a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat cl->ublen += sz_rfbRREHeader; 166a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 167a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat for (i = 0; i < cl->afterEncBufLen;) { 168a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 169a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat int bytesToCopy = UPDATE_BUF_SIZE - cl->ublen; 170a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 171a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat if (i + bytesToCopy > cl->afterEncBufLen) { 172a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat bytesToCopy = cl->afterEncBufLen - i; 173a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat } 174a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 175a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat memcpy(&cl->updateBuf[cl->ublen], &cl->afterEncBuf[i], bytesToCopy); 176a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 177a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat cl->ublen += bytesToCopy; 178a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat i += bytesToCopy; 179a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 180a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat if (cl->ublen == UPDATE_BUF_SIZE) { 181a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat if (!rfbSendUpdateBuf(cl)) 182a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat return FALSE; 183a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat } 184a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat } 185a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 186a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat return TRUE; 187a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat} 188a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 189a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 190a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 191a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat/* 192a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat * subrectEncode() encodes the given multicoloured rectangle as a background 193a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat * colour overwritten by single-coloured rectangles. It returns the number 194a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat * of subrectangles in the encoded buffer, or -1 if subrect encoding won't 195a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat * fit in the buffer. It puts the encoded rectangles in cl->afterEncBuf. The 196a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat * single-colour rectangle partition is not optimal, but does find the biggest 197a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat * horizontal or vertical rectangle top-left anchored to each consecutive 198a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat * coordinate position. 199a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat * 200a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat * The coding scheme is simply [<bgcolour><subrect><subrect>...] where each 201a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat * <subrect> is [<colour><x><y><w><h>]. 202a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat */ 203a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 204a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat#define DEFINE_SUBRECT_ENCODE(bpp) \ 205a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehatstatic int \ 206a430b2b5ca4f0967836f5820e8f03adc17fc0a24San MehatsubrectEncode##bpp(rfbClientPtr client, uint##bpp##_t *data, int w, int h) { \ 207a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat uint##bpp##_t cl; \ 208a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat rfbCoRRERectangle subrect; \ 209a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat int x,y; \ 210a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat int i,j; \ 211a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat int hx=0,hy,vx=0,vy; \ 212a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat int hyflag; \ 213a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat uint##bpp##_t *seg; \ 214a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat uint##bpp##_t *line; \ 215a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat int hw,hh,vw,vh; \ 216a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat int thex,they,thew,theh; \ 217a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat int numsubs = 0; \ 218a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat int newLen; \ 219a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat uint##bpp##_t bg = (uint##bpp##_t)getBgColour((char*)data,w*h,bpp); \ 220a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat \ 221a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat *((uint##bpp##_t*)client->afterEncBuf) = bg; \ 222a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat \ 223a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat client->afterEncBufLen = (bpp/8); \ 224a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat \ 225a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat for (y=0; y<h; y++) { \ 226a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat line = data+(y*w); \ 227a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat for (x=0; x<w; x++) { \ 228a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat if (line[x] != bg) { \ 229a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat cl = line[x]; \ 230a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat hy = y-1; \ 231a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat hyflag = 1; \ 232a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat for (j=y; j<h; j++) { \ 233a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat seg = data+(j*w); \ 234a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat if (seg[x] != cl) {break;} \ 235a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat i = x; \ 236a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat while ((seg[i] == cl) && (i < w)) i += 1; \ 237a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat i -= 1; \ 238a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat if (j == y) vx = hx = i; \ 239a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat if (i < vx) vx = i; \ 240a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat if ((hyflag > 0) && (i >= hx)) {hy += 1;} else {hyflag = 0;} \ 241a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat } \ 242a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat vy = j-1; \ 243a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat \ 244a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat /* We now have two possible subrects: (x,y,hx,hy) and (x,y,vx,vy) \ 245a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat * We'll choose the bigger of the two. \ 246a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat */ \ 247a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat hw = hx-x+1; \ 248a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat hh = hy-y+1; \ 249a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat vw = vx-x+1; \ 250a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat vh = vy-y+1; \ 251a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat \ 252a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat thex = x; \ 253a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat they = y; \ 254a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat \ 255a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat if ((hw*hh) > (vw*vh)) { \ 256a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat thew = hw; \ 257a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat theh = hh; \ 258a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat } else { \ 259a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat thew = vw; \ 260a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat theh = vh; \ 261a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat } \ 262a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat \ 263a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat subrect.x = thex; \ 264a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat subrect.y = they; \ 265a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat subrect.w = thew; \ 266a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat subrect.h = theh; \ 267a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat \ 268a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat newLen = client->afterEncBufLen + (bpp/8) + sz_rfbCoRRERectangle; \ 269a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat if ((newLen > (w * h * (bpp/8))) || (newLen > client->afterEncBufSize)) \ 270a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat return -1; \ 271a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat \ 272a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat numsubs += 1; \ 273a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat *((uint##bpp##_t*)(client->afterEncBuf + client->afterEncBufLen)) = cl; \ 274a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat client->afterEncBufLen += (bpp/8); \ 275a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat memcpy(&client->afterEncBuf[client->afterEncBufLen],&subrect,sz_rfbCoRRERectangle); \ 276a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat client->afterEncBufLen += sz_rfbCoRRERectangle; \ 277a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat \ 278a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat /* \ 279a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat * Now mark the subrect as done. \ 280a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat */ \ 281a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat for (j=they; j < (they+theh); j++) { \ 282a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat for (i=thex; i < (thex+thew); i++) { \ 283a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat data[j*w+i] = bg; \ 284a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat } \ 285a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat } \ 286a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat } \ 287a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat } \ 288a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat } \ 289a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat \ 290a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat return numsubs; \ 291a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat} 292a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 293a430b2b5ca4f0967836f5820e8f03adc17fc0a24San MehatDEFINE_SUBRECT_ENCODE(8) 294a430b2b5ca4f0967836f5820e8f03adc17fc0a24San MehatDEFINE_SUBRECT_ENCODE(16) 295a430b2b5ca4f0967836f5820e8f03adc17fc0a24San MehatDEFINE_SUBRECT_ENCODE(32) 296a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 297a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 298a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat/* 299a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat * getBgColour() gets the most prevalent colour in a byte array. 300a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat */ 301a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehatstatic uint32_t 302a430b2b5ca4f0967836f5820e8f03adc17fc0a24San MehatgetBgColour(char *data, int size, int bpp) 303a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat{ 304a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 305a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat#define NUMCLRS 256 306a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 307a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat static int counts[NUMCLRS]; 308a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat int i,j,k; 309a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 310a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat int maxcount = 0; 311a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat uint8_t maxclr = 0; 312a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 313a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat if (bpp != 8) { 314a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat if (bpp == 16) { 315a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat return ((uint16_t *)data)[0]; 316a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat } else if (bpp == 32) { 317a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat return ((uint32_t *)data)[0]; 318a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat } else { 319a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat rfbLog("getBgColour: bpp %d?\n",bpp); 320a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat return 0; 321a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat } 322a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat } 323a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 324a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat for (i=0; i<NUMCLRS; i++) { 325a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat counts[i] = 0; 326a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat } 327a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 328a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat for (j=0; j<size; j++) { 329a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat k = (int)(((uint8_t *)data)[j]); 330a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat if (k >= NUMCLRS) { 331a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat rfbLog("getBgColour: unusual colour = %d\n", k); 332a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat return 0; 333a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat } 334a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat counts[k] += 1; 335a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat if (counts[k] > maxcount) { 336a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat maxcount = counts[k]; 337a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat maxclr = ((uint8_t *)data)[j]; 338a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat } 339a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat } 340a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat 341a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat return maxclr; 342a430b2b5ca4f0967836f5820e8f03adc17fc0a24San Mehat} 343