vnc-enc-zrle.c (10521B)
1/* 2 * QEMU VNC display driver: Zlib Run-length Encoding (ZRLE) 3 * 4 * From libvncserver/libvncserver/zrle.c 5 * Copyright (C) 2002 RealVNC Ltd. All Rights Reserved. 6 * Copyright (C) 2003 Sun Microsystems, Inc. 7 * 8 * Copyright (C) 2010 Corentin Chary <corentin.chary@gmail.com> 9 * 10 * Permission is hereby granted, free of charge, to any person obtaining a copy 11 * of this software and associated documentation files (the "Software"), to deal 12 * in the Software without restriction, including without limitation the rights 13 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 14 * copies of the Software, and to permit persons to whom the Software is 15 * furnished to do so, subject to the following conditions: 16 * 17 * The above copyright notice and this permission notice shall be included in 18 * all copies or substantial portions of the Software. 19 * 20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 23 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 25 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 26 * THE SOFTWARE. 27 */ 28 29#include "qemu/osdep.h" 30#include "vnc.h" 31#include "vnc-enc-zrle.h" 32 33static const int bits_per_packed_pixel[] = { 34 0, 1, 2, 2, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4 35}; 36 37 38static void vnc_zrle_start(VncState *vs) 39{ 40 buffer_reset(&vs->zrle->zrle); 41 42 /* make the output buffer be the zlib buffer, so we can compress it later */ 43 vs->zrle->tmp = vs->output; 44 vs->output = vs->zrle->zrle; 45} 46 47static void vnc_zrle_stop(VncState *vs) 48{ 49 /* switch back to normal output/zlib buffers */ 50 vs->zrle->zrle = vs->output; 51 vs->output = vs->zrle->tmp; 52} 53 54static void *zrle_convert_fb(VncState *vs, int x, int y, int w, int h, 55 int bpp) 56{ 57 Buffer tmp; 58 59 buffer_reset(&vs->zrle->fb); 60 buffer_reserve(&vs->zrle->fb, w * h * bpp + bpp); 61 62 tmp = vs->output; 63 vs->output = vs->zrle->fb; 64 65 vnc_raw_send_framebuffer_update(vs, x, y, w, h); 66 67 vs->zrle->fb = vs->output; 68 vs->output = tmp; 69 return vs->zrle->fb.buffer; 70} 71 72static int zrle_compress_data(VncState *vs, int level) 73{ 74 z_streamp zstream = &vs->zrle->stream; 75 76 buffer_reset(&vs->zrle->zlib); 77 78 if (zstream->opaque != vs) { 79 int err; 80 81 zstream->zalloc = vnc_zlib_zalloc; 82 zstream->zfree = vnc_zlib_zfree; 83 84 err = deflateInit2(zstream, level, Z_DEFLATED, MAX_WBITS, 85 MAX_MEM_LEVEL, Z_DEFAULT_STRATEGY); 86 87 if (err != Z_OK) { 88 fprintf(stderr, "VNC: error initializing zlib\n"); 89 return -1; 90 } 91 92 zstream->opaque = vs; 93 } 94 95 /* reserve memory in output buffer */ 96 buffer_reserve(&vs->zrle->zlib, vs->zrle->zrle.offset + 64); 97 98 /* set pointers */ 99 zstream->next_in = vs->zrle->zrle.buffer; 100 zstream->avail_in = vs->zrle->zrle.offset; 101 zstream->next_out = vs->zrle->zlib.buffer; 102 zstream->avail_out = vs->zrle->zlib.capacity; 103 zstream->data_type = Z_BINARY; 104 105 /* start encoding */ 106 if (deflate(zstream, Z_SYNC_FLUSH) != Z_OK) { 107 fprintf(stderr, "VNC: error during zrle compression\n"); 108 return -1; 109 } 110 111 vs->zrle->zlib.offset = vs->zrle->zlib.capacity - zstream->avail_out; 112 return vs->zrle->zlib.offset; 113} 114 115/* Try to work out whether to use RLE and/or a palette. We do this by 116 * estimating the number of bytes which will be generated and picking the 117 * method which results in the fewest bytes. Of course this may not result 118 * in the fewest bytes after compression... */ 119static void zrle_choose_palette_rle(VncState *vs, int w, int h, 120 VncPalette *palette, int bpp_out, 121 int runs, int single_pixels, 122 int zywrle_level, 123 bool *use_rle, bool *use_palette) 124{ 125 size_t estimated_bytes; 126 size_t plain_rle_bytes; 127 128 *use_palette = *use_rle = false; 129 130 estimated_bytes = w * h * (bpp_out / 8); /* start assuming raw */ 131 132 if (bpp_out != 8) { 133 if (zywrle_level > 0 && !(zywrle_level & 0x80)) 134 estimated_bytes >>= zywrle_level; 135 } 136 137 plain_rle_bytes = ((bpp_out / 8) + 1) * (runs + single_pixels); 138 139 if (plain_rle_bytes < estimated_bytes) { 140 *use_rle = true; 141 estimated_bytes = plain_rle_bytes; 142 } 143 144 if (palette_size(palette) < 128) { 145 int palette_rle_bytes; 146 147 palette_rle_bytes = (bpp_out / 8) * palette_size(palette); 148 palette_rle_bytes += 2 * runs + single_pixels; 149 150 if (palette_rle_bytes < estimated_bytes) { 151 *use_rle = true; 152 *use_palette = true; 153 estimated_bytes = palette_rle_bytes; 154 } 155 156 if (palette_size(palette) < 17) { 157 int packed_bytes; 158 159 packed_bytes = (bpp_out / 8) * palette_size(palette); 160 packed_bytes += w * h * 161 bits_per_packed_pixel[palette_size(palette)-1] / 8; 162 163 if (packed_bytes < estimated_bytes) { 164 *use_rle = false; 165 *use_palette = true; 166 } 167 } 168 } 169} 170 171static void zrle_write_u32(VncState *vs, uint32_t value) 172{ 173 vnc_write(vs, (uint8_t *)&value, 4); 174} 175 176static void zrle_write_u24a(VncState *vs, uint32_t value) 177{ 178 vnc_write(vs, (uint8_t *)&value, 3); 179} 180 181static void zrle_write_u24b(VncState *vs, uint32_t value) 182{ 183 vnc_write(vs, ((uint8_t *)&value) + 1, 3); 184} 185 186static void zrle_write_u16(VncState *vs, uint16_t value) 187{ 188 vnc_write(vs, (uint8_t *)&value, 2); 189} 190 191static void zrle_write_u8(VncState *vs, uint8_t value) 192{ 193 vnc_write_u8(vs, value); 194} 195 196#define ENDIAN_LITTLE 0 197#define ENDIAN_BIG 1 198#define ENDIAN_NO 2 199 200#define ZRLE_BPP 8 201#define ZYWRLE_ENDIAN ENDIAN_NO 202#include "vnc-enc-zrle.c.inc" 203#undef ZRLE_BPP 204 205#define ZRLE_BPP 15 206#undef ZYWRLE_ENDIAN 207#define ZYWRLE_ENDIAN ENDIAN_LITTLE 208#include "vnc-enc-zrle.c.inc" 209 210#undef ZYWRLE_ENDIAN 211#define ZYWRLE_ENDIAN ENDIAN_BIG 212#include "vnc-enc-zrle.c.inc" 213 214#undef ZRLE_BPP 215#define ZRLE_BPP 16 216#undef ZYWRLE_ENDIAN 217#define ZYWRLE_ENDIAN ENDIAN_LITTLE 218#include "vnc-enc-zrle.c.inc" 219 220#undef ZYWRLE_ENDIAN 221#define ZYWRLE_ENDIAN ENDIAN_BIG 222#include "vnc-enc-zrle.c.inc" 223 224#undef ZRLE_BPP 225#define ZRLE_BPP 32 226#undef ZYWRLE_ENDIAN 227#define ZYWRLE_ENDIAN ENDIAN_LITTLE 228#include "vnc-enc-zrle.c.inc" 229 230#undef ZYWRLE_ENDIAN 231#define ZYWRLE_ENDIAN ENDIAN_BIG 232#include "vnc-enc-zrle.c.inc" 233 234#define ZRLE_COMPACT_PIXEL 24a 235#undef ZYWRLE_ENDIAN 236#define ZYWRLE_ENDIAN ENDIAN_LITTLE 237#include "vnc-enc-zrle.c.inc" 238 239#undef ZYWRLE_ENDIAN 240#define ZYWRLE_ENDIAN ENDIAN_BIG 241#include "vnc-enc-zrle.c.inc" 242 243#undef ZRLE_COMPACT_PIXEL 244#define ZRLE_COMPACT_PIXEL 24b 245#undef ZYWRLE_ENDIAN 246#define ZYWRLE_ENDIAN ENDIAN_LITTLE 247#include "vnc-enc-zrle.c.inc" 248 249#undef ZYWRLE_ENDIAN 250#define ZYWRLE_ENDIAN ENDIAN_BIG 251#include "vnc-enc-zrle.c.inc" 252#undef ZRLE_COMPACT_PIXEL 253#undef ZRLE_BPP 254 255static int zrle_send_framebuffer_update(VncState *vs, int x, int y, 256 int w, int h) 257{ 258 bool be = vs->client_be; 259 size_t bytes; 260 int zywrle_level; 261 262 if (vs->zrle->type == VNC_ENCODING_ZYWRLE) { 263 if (!vs->vd->lossy || vs->tight->quality == (uint8_t)-1 264 || vs->tight->quality == 9) { 265 zywrle_level = 0; 266 vs->zrle->type = VNC_ENCODING_ZRLE; 267 } else if (vs->tight->quality < 3) { 268 zywrle_level = 3; 269 } else if (vs->tight->quality < 6) { 270 zywrle_level = 2; 271 } else { 272 zywrle_level = 1; 273 } 274 } else { 275 zywrle_level = 0; 276 } 277 278 vnc_zrle_start(vs); 279 280 switch (vs->client_pf.bytes_per_pixel) { 281 case 1: 282 zrle_encode_8ne(vs, x, y, w, h, zywrle_level); 283 break; 284 285 case 2: 286 if (vs->client_pf.gmax > 0x1F) { 287 if (be) { 288 zrle_encode_16be(vs, x, y, w, h, zywrle_level); 289 } else { 290 zrle_encode_16le(vs, x, y, w, h, zywrle_level); 291 } 292 } else { 293 if (be) { 294 zrle_encode_15be(vs, x, y, w, h, zywrle_level); 295 } else { 296 zrle_encode_15le(vs, x, y, w, h, zywrle_level); 297 } 298 } 299 break; 300 301 case 4: 302 { 303 bool fits_in_ls3bytes; 304 bool fits_in_ms3bytes; 305 306 fits_in_ls3bytes = 307 ((vs->client_pf.rmax << vs->client_pf.rshift) < (1 << 24) && 308 (vs->client_pf.gmax << vs->client_pf.gshift) < (1 << 24) && 309 (vs->client_pf.bmax << vs->client_pf.bshift) < (1 << 24)); 310 311 fits_in_ms3bytes = (vs->client_pf.rshift > 7 && 312 vs->client_pf.gshift > 7 && 313 vs->client_pf.bshift > 7); 314 315 if ((fits_in_ls3bytes && !be) || (fits_in_ms3bytes && be)) { 316 if (be) { 317 zrle_encode_24abe(vs, x, y, w, h, zywrle_level); 318 } else { 319 zrle_encode_24ale(vs, x, y, w, h, zywrle_level); 320 } 321 } else if ((fits_in_ls3bytes && be) || (fits_in_ms3bytes && !be)) { 322 if (be) { 323 zrle_encode_24bbe(vs, x, y, w, h, zywrle_level); 324 } else { 325 zrle_encode_24ble(vs, x, y, w, h, zywrle_level); 326 } 327 } else { 328 if (be) { 329 zrle_encode_32be(vs, x, y, w, h, zywrle_level); 330 } else { 331 zrle_encode_32le(vs, x, y, w, h, zywrle_level); 332 } 333 } 334 } 335 break; 336 } 337 338 vnc_zrle_stop(vs); 339 bytes = zrle_compress_data(vs, Z_DEFAULT_COMPRESSION); 340 vnc_framebuffer_update(vs, x, y, w, h, vs->zrle->type); 341 vnc_write_u32(vs, bytes); 342 vnc_write(vs, vs->zrle->zlib.buffer, vs->zrle->zlib.offset); 343 return 1; 344} 345 346int vnc_zrle_send_framebuffer_update(VncState *vs, int x, int y, int w, int h) 347{ 348 vs->zrle->type = VNC_ENCODING_ZRLE; 349 return zrle_send_framebuffer_update(vs, x, y, w, h); 350} 351 352int vnc_zywrle_send_framebuffer_update(VncState *vs, int x, int y, int w, int h) 353{ 354 vs->zrle->type = VNC_ENCODING_ZYWRLE; 355 return zrle_send_framebuffer_update(vs, x, y, w, h); 356} 357 358void vnc_zrle_clear(VncState *vs) 359{ 360 if (vs->zrle->stream.opaque) { 361 deflateEnd(&vs->zrle->stream); 362 } 363 buffer_free(&vs->zrle->zrle); 364 buffer_free(&vs->zrle->fb); 365 buffer_free(&vs->zrle->zlib); 366}