1/* 2 * Copyright (C) 2005 Johannes E. Schindelin. All Rights Reserved. 3 * 4 * This is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License as published by 6 * the Free Software Foundation; either version 2 of the License, or 7 * (at your option) any later version. 8 * 9 * This software is distributed in the hope that it will be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * You should have received a copy of the GNU General Public License 15 * along with this software; if not, write to the Free Software 16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, 17 * USA. 18 */ 19 20#ifdef LIBVNCSERVER_HAVE_LIBZ 21 22/* 23 * zrle.c - handle zrle encoding. 24 * 25 * This file shouldn't be compiled directly. It is included multiple times by 26 * rfbproto.c, each time with a different definition of the macro BPP. For 27 * each value of BPP, this file defines a function which handles an zrle 28 * encoded rectangle with BPP bits per pixel. 29 */ 30 31#ifndef REALBPP 32#define REALBPP BPP 33#endif 34 35#if !defined(UNCOMP) || UNCOMP==0 36#define HandleZRLE CONCAT2E(HandleZRLE,REALBPP) 37#define HandleZRLETile CONCAT2E(HandleZRLETile,REALBPP) 38#elif UNCOMP>0 39#define HandleZRLE CONCAT3E(HandleZRLE,REALBPP,Down) 40#define HandleZRLETile CONCAT3E(HandleZRLETile,REALBPP,Down) 41#else 42#define HandleZRLE CONCAT3E(HandleZRLE,REALBPP,Up) 43#define HandleZRLETile CONCAT3E(HandleZRLETile,REALBPP,Up) 44#endif 45#define CARDBPP CONCAT3E(uint,BPP,_t) 46#define CARDREALBPP CONCAT3E(uint,REALBPP,_t) 47 48#define ENDIAN_LITTLE 0 49#define ENDIAN_BIG 1 50#define ENDIAN_NO 2 51#define ZYWRLE_ENDIAN ENDIAN_LITTLE 52#undef END_FIX 53#if ZYWRLE_ENDIAN == ENDIAN_LITTLE 54# define END_FIX LE 55#elif ZYWRLE_ENDIAN == ENDIAN_BIG 56# define END_FIX BE 57#else 58# define END_FIX NE 59#endif 60#define __RFB_CONCAT3E(a,b,c) CONCAT3E(a,b,c) 61#define __RFB_CONCAT2E(a,b) CONCAT2E(a,b) 62#undef CPIXEL 63#if REALBPP != BPP 64#if UNCOMP == 0 65#define CPIXEL REALBPP 66#elif UNCOMP>0 67#define CPIXEL CONCAT2E(REALBPP,Down) 68#else 69#define CPIXEL CONCAT2E(REALBPP,Up) 70#endif 71#endif 72#define PIXEL_T __RFB_CONCAT3E(uint,BPP,_t) 73#if BPP!=8 74#define ZYWRLE_DECODE 1 75#include "zywrletemplate.c" 76#endif 77#undef CPIXEL 78 79static int HandleZRLETile(rfbClient* client, 80 uint8_t* buffer,size_t buffer_length, 81 int x,int y,int w,int h); 82 83static rfbBool 84HandleZRLE (rfbClient* client, int rx, int ry, int rw, int rh) 85{ 86 rfbZRLEHeader header; 87 int remaining; 88 int inflateResult; 89 int toRead; 90 int min_buffer_size = rw * rh * (REALBPP / 8) * 2; 91 92 /* First make sure we have a large enough raw buffer to hold the 93 * decompressed data. In practice, with a fixed REALBPP, fixed frame 94 * buffer size and the first update containing the entire frame 95 * buffer, this buffer allocation should only happen once, on the 96 * first update. 97 */ 98 if ( client->raw_buffer_size < min_buffer_size) { 99 100 if ( client->raw_buffer != NULL ) { 101 102 free( client->raw_buffer ); 103 104 } 105 106 client->raw_buffer_size = min_buffer_size; 107 client->raw_buffer = (char*) malloc( client->raw_buffer_size ); 108 109 } 110 111 if (!ReadFromRFBServer(client, (char *)&header, sz_rfbZRLEHeader)) 112 return FALSE; 113 114 remaining = rfbClientSwap32IfLE(header.length); 115 116 /* Need to initialize the decompressor state. */ 117 client->decompStream.next_in = ( Bytef * )client->buffer; 118 client->decompStream.avail_in = 0; 119 client->decompStream.next_out = ( Bytef * )client->raw_buffer; 120 client->decompStream.avail_out = client->raw_buffer_size; 121 client->decompStream.data_type = Z_BINARY; 122 123 /* Initialize the decompression stream structures on the first invocation. */ 124 if ( client->decompStreamInited == FALSE ) { 125 126 inflateResult = inflateInit( &client->decompStream ); 127 128 if ( inflateResult != Z_OK ) { 129 rfbClientLog( 130 "inflateInit returned error: %d, msg: %s\n", 131 inflateResult, 132 client->decompStream.msg); 133 return FALSE; 134 } 135 136 client->decompStreamInited = TRUE; 137 138 } 139 140 inflateResult = Z_OK; 141 142 /* Process buffer full of data until no more to process, or 143 * some type of inflater error, or Z_STREAM_END. 144 */ 145 while (( remaining > 0 ) && 146 ( inflateResult == Z_OK )) { 147 148 if ( remaining > RFB_BUFFER_SIZE ) { 149 toRead = RFB_BUFFER_SIZE; 150 } 151 else { 152 toRead = remaining; 153 } 154 155 /* Fill the buffer, obtaining data from the server. */ 156 if (!ReadFromRFBServer(client, client->buffer,toRead)) 157 return FALSE; 158 159 client->decompStream.next_in = ( Bytef * )client->buffer; 160 client->decompStream.avail_in = toRead; 161 162 /* Need to uncompress buffer full. */ 163 inflateResult = inflate( &client->decompStream, Z_SYNC_FLUSH ); 164 165 /* We never supply a dictionary for compression. */ 166 if ( inflateResult == Z_NEED_DICT ) { 167 rfbClientLog("zlib inflate needs a dictionary!\n"); 168 return FALSE; 169 } 170 if ( inflateResult < 0 ) { 171 rfbClientLog( 172 "zlib inflate returned error: %d, msg: %s\n", 173 inflateResult, 174 client->decompStream.msg); 175 return FALSE; 176 } 177 178 /* Result buffer allocated to be at least large enough. We should 179 * never run out of space! 180 */ 181 if (( client->decompStream.avail_in > 0 ) && 182 ( client->decompStream.avail_out <= 0 )) { 183 rfbClientLog("zlib inflate ran out of space!\n"); 184 return FALSE; 185 } 186 187 remaining -= toRead; 188 189 } /* while ( remaining > 0 ) */ 190 191 if ( inflateResult == Z_OK ) { 192 void* buf=client->raw_buffer; 193 int i,j; 194 195 remaining = client->raw_buffer_size-client->decompStream.avail_out; 196 197 for(j=0; j<rh; j+=rfbZRLETileHeight) 198 for(i=0; i<rw; i+=rfbZRLETileWidth) { 199 int subWidth=(i+rfbZRLETileWidth>rw)?rw-i:rfbZRLETileWidth; 200 int subHeight=(j+rfbZRLETileHeight>rh)?rh-j:rfbZRLETileHeight; 201 int result=HandleZRLETile(client,buf,remaining,rx+i,ry+j,subWidth,subHeight); 202 203 if(result<0) { 204 rfbClientLog("ZRLE decoding failed (%d)\n",result); 205return TRUE; 206 return FALSE; 207 } 208 209 buf+=result; 210 remaining-=result; 211 } 212 } 213 else { 214 215 rfbClientLog( 216 "zlib inflate returned error: %d, msg: %s\n", 217 inflateResult, 218 client->decompStream.msg); 219 return FALSE; 220 221 } 222 223 return TRUE; 224} 225 226#if REALBPP!=BPP && defined(UNCOMP) && UNCOMP!=0 227#if UNCOMP>0 228#define UncompressCPixel(pointer) ((*(CARDBPP*)pointer)>>UNCOMP) 229#else 230#define UncompressCPixel(pointer) ((*(CARDBPP*)pointer)<<(-(UNCOMP))) 231#endif 232#else 233#define UncompressCPixel(pointer) (*(CARDBPP*)pointer) 234#endif 235 236static int HandleZRLETile(rfbClient* client, 237 uint8_t* buffer,size_t buffer_length, 238 int x,int y,int w,int h) { 239 uint8_t* buffer_copy = buffer; 240 uint8_t* buffer_end = buffer+buffer_length; 241 uint8_t type; 242#if BPP!=8 243 uint8_t zywrle_level = (client->appData.qualityLevel & 0x80) ? 244 0 : (3 - client->appData.qualityLevel / 3); 245#endif 246 247 if(buffer_length<1) 248 return -2; 249 250 type = *buffer; 251 buffer++; 252 { 253 if( type == 0 ) /* raw */ 254#if BPP!=8 255 if( zywrle_level > 0 ){ 256 CARDBPP* pFrame = (CARDBPP*)client->frameBuffer + y*client->width+x; 257 int ret; 258 client->appData.qualityLevel |= 0x80; 259 ret = HandleZRLETile(client, buffer, buffer_end-buffer, x, y, w, h); 260 client->appData.qualityLevel &= 0x7F; 261 if( ret < 0 ){ 262 return ret; 263 } 264 ZYWRLE_SYNTHESIZE( pFrame, pFrame, w, h, client->width, zywrle_level, (int*)client->zlib_buffer ); 265 buffer += ret; 266 }else 267#endif 268 { 269#if REALBPP!=BPP 270 int i,j; 271 272 if(1+w*h*REALBPP/8>buffer_length) { 273 rfbClientLog("expected %d bytes, got only %d (%dx%d)\n",1+w*h*REALBPP/8,buffer_length,w,h); 274 return -3; 275 } 276 277 for(j=y*client->width; j<(y+h)*client->width; j+=client->width) 278 for(i=x; i<x+w; i++,buffer+=REALBPP/8) 279 ((CARDBPP*)client->frameBuffer)[j+i] = UncompressCPixel(buffer); 280#else 281 CopyRectangle(client, buffer, x, y, w, h); 282 buffer+=w*h*REALBPP/8; 283#endif 284 } 285 else if( type == 1 ) /* solid */ 286 { 287 CARDBPP color = UncompressCPixel(buffer); 288 289 if(1+REALBPP/8>buffer_length) 290 return -4; 291 292 FillRectangle(client, x, y, w, h, color); 293 294 buffer+=REALBPP/8; 295 296 } 297 else if( (type >= 2)&&(type <= 127) ) /* packed Palette */ 298 { 299 CARDBPP palette[16]; 300 int i,j,shift, 301 bpp=(type>4?(type>16?8:4):(type>2?2:1)), 302 mask=(1<<bpp)-1, 303 divider=(8/bpp); 304 305 if(1+type*REALBPP/8+((w+divider-1)/divider)*h>buffer_length) 306 return -5; 307 308 /* read palette */ 309 for(i=0; i<type; i++,buffer+=REALBPP/8) 310 palette[i] = UncompressCPixel(buffer); 311 312 /* read palettized pixels */ 313 for(j=y*client->width; j<(y+h)*client->width; j+=client->width) { 314 for(i=x,shift=8-bpp; i<x+w; i++) { 315 ((CARDBPP*)client->frameBuffer)[j+i] = palette[((*buffer)>>shift)&mask]; 316 shift-=bpp; 317 if(shift<0) { 318 shift=8-bpp; 319 buffer++; 320 } 321 } 322 if(shift<8-bpp) 323 buffer++; 324 } 325 326 } 327 /* case 17 ... 127: not used, but valid */ 328 else if( type == 128 ) /* plain RLE */ 329 { 330 int i=0,j=0; 331 while(j<h) { 332 int color,length; 333 /* read color */ 334 if(buffer+REALBPP/8+1>buffer_end) 335 return -7; 336 color = UncompressCPixel(buffer); 337 buffer+=REALBPP/8; 338 /* read run length */ 339 length=1; 340 while(*buffer==0xff) { 341 if(buffer+1>=buffer_end) 342 return -8; 343 length+=*buffer; 344 buffer++; 345 } 346 length+=*buffer; 347 buffer++; 348 while(j<h && length>0) { 349 ((CARDBPP*)client->frameBuffer)[(y+j)*client->width+x+i] = color; 350 length--; 351 i++; 352 if(i>=w) { 353 i=0; 354 j++; 355 } 356 } 357 if(length>0) 358 rfbClientLog("Warning: possible ZRLE corruption\n"); 359 } 360 361 } 362 else if( type == 129 ) /* unused */ 363 { 364 return -8; 365 } 366 else if( type >= 130 ) /* palette RLE */ 367 { 368 CARDBPP palette[128]; 369 int i,j; 370 371 if(2+(type-128)*REALBPP/8>buffer_length) 372 return -9; 373 374 /* read palette */ 375 for(i=0; i<type-128; i++,buffer+=REALBPP/8) 376 palette[i] = UncompressCPixel(buffer); 377 /* read palettized pixels */ 378 i=j=0; 379 while(j<h) { 380 int color,length; 381 /* read color */ 382 if(buffer>=buffer_end) 383 return -10; 384 color = palette[(*buffer)&0x7f]; 385 length=1; 386 if(*buffer&0x80) { 387 if(buffer+1>=buffer_end) 388 return -11; 389 buffer++; 390 /* read run length */ 391 while(*buffer==0xff) { 392 if(buffer+1>=buffer_end) 393 return -8; 394 length+=*buffer; 395 buffer++; 396 } 397 length+=*buffer; 398 } 399 buffer++; 400 while(j<h && length>0) { 401 ((CARDBPP*)client->frameBuffer)[(y+j)*client->width+x+i] = color; 402 length--; 403 i++; 404 if(i>=w) { 405 i=0; 406 j++; 407 } 408 } 409 if(length>0) 410 rfbClientLog("Warning: possible ZRLE corruption\n"); 411 } 412 } 413 } 414 415 return buffer-buffer_copy; 416} 417 418#undef CARDBPP 419#undef CARDREALBPP 420#undef HandleZRLE 421#undef HandleZRLETile 422#undef UncompressCPixel 423#undef REALBPP 424 425#endif 426 427#undef UNCOMP 428