1245f7b51SCorentin Chary /*
2245f7b51SCorentin Chary * QEMU VNC display driver: tight encoding
3245f7b51SCorentin Chary *
4245f7b51SCorentin Chary * From libvncserver/libvncserver/tight.c
5245f7b51SCorentin Chary * Copyright (C) 2000, 2001 Const Kaplinsky. All Rights Reserved.
6245f7b51SCorentin Chary * Copyright (C) 1999 AT&T Laboratories Cambridge. All Rights Reserved.
7245f7b51SCorentin Chary *
8245f7b51SCorentin Chary * Copyright (C) 2010 Corentin Chary <corentin.chary@gmail.com>
9245f7b51SCorentin Chary *
10245f7b51SCorentin Chary * Permission is hereby granted, free of charge, to any person obtaining a copy
11245f7b51SCorentin Chary * of this software and associated documentation files (the "Software"), to deal
12245f7b51SCorentin Chary * in the Software without restriction, including without limitation the rights
13245f7b51SCorentin Chary * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
14245f7b51SCorentin Chary * copies of the Software, and to permit persons to whom the Software is
15245f7b51SCorentin Chary * furnished to do so, subject to the following conditions:
16245f7b51SCorentin Chary *
17245f7b51SCorentin Chary * The above copyright notice and this permission notice shall be included in
18245f7b51SCorentin Chary * all copies or substantial portions of the Software.
19245f7b51SCorentin Chary *
20245f7b51SCorentin Chary * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21245f7b51SCorentin Chary * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22245f7b51SCorentin Chary * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23245f7b51SCorentin Chary * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24245f7b51SCorentin Chary * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
25245f7b51SCorentin Chary * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
26245f7b51SCorentin Chary * THE SOFTWARE.
27245f7b51SCorentin Chary */
28245f7b51SCorentin Chary
29e16f4c87SPeter Maydell #include "qemu/osdep.h"
30245f7b51SCorentin Chary
31f26e428dSRoy Tam /* This needs to be before jpeglib.h line because of conflict with
32f26e428dSRoy Tam INT32 definitions between jmorecfg.h (included by jpeglib.h) and
33f26e428dSRoy Tam Win32 basetsd.h (included by windows.h). */
34f26e428dSRoy Tam
3595f8510eSKshitij Suri #ifdef CONFIG_PNG
362fb0c09fSStefan Weil /* The following define is needed by pngconf.h. Otherwise it won't compile,
370f9668e0SMarc-André Lureau because setjmp.h was already included by osdep.h. */
382fb0c09fSStefan Weil #define PNG_SKIP_SETJMP_CHECK
39efe556adSCorentin Chary #include <png.h>
40efe556adSCorentin Chary #endif
41245f7b51SCorentin Chary #ifdef CONFIG_VNC_JPEG
42245f7b51SCorentin Chary #include <jpeglib.h>
43245f7b51SCorentin Chary #endif
44245f7b51SCorentin Chary
451de7afc9SPaolo Bonzini #include "qemu/bswap.h"
46245f7b51SCorentin Chary #include "vnc.h"
47245f7b51SCorentin Chary #include "vnc-enc-tight.h"
485136a052SCorentin Chary #include "vnc-palette.h"
49245f7b51SCorentin Chary
50245f7b51SCorentin Chary /* Compression level stuff. The following array contains various
51245f7b51SCorentin Chary encoder parameters for each of 10 compression levels (0..9).
52245f7b51SCorentin Chary Last three parameters correspond to JPEG quality levels (0..9). */
53245f7b51SCorentin Chary
54245f7b51SCorentin Chary static const struct {
55245f7b51SCorentin Chary int max_rect_size, max_rect_width;
56245f7b51SCorentin Chary int mono_min_rect_size, gradient_min_rect_size;
57245f7b51SCorentin Chary int idx_zlib_level, mono_zlib_level, raw_zlib_level, gradient_zlib_level;
58245f7b51SCorentin Chary int gradient_threshold, gradient_threshold24;
59245f7b51SCorentin Chary int idx_max_colors_divisor;
60245f7b51SCorentin Chary int jpeg_quality, jpeg_threshold, jpeg_threshold24;
61245f7b51SCorentin Chary } tight_conf[] = {
62245f7b51SCorentin Chary { 512, 32, 6, 65536, 0, 0, 0, 0, 0, 0, 4, 5, 10000, 23000 },
63245f7b51SCorentin Chary { 2048, 128, 6, 65536, 1, 1, 1, 0, 0, 0, 8, 10, 8000, 18000 },
64245f7b51SCorentin Chary { 6144, 256, 8, 65536, 3, 3, 2, 0, 0, 0, 24, 15, 6500, 15000 },
65245f7b51SCorentin Chary { 10240, 1024, 12, 65536, 5, 5, 3, 0, 0, 0, 32, 25, 5000, 12000 },
66245f7b51SCorentin Chary { 16384, 2048, 12, 65536, 6, 6, 4, 0, 0, 0, 32, 37, 4000, 10000 },
67245f7b51SCorentin Chary { 32768, 2048, 12, 4096, 7, 7, 5, 4, 150, 380, 32, 50, 3000, 8000 },
68245f7b51SCorentin Chary { 65536, 2048, 16, 4096, 7, 7, 6, 4, 170, 420, 48, 60, 2000, 5000 },
69245f7b51SCorentin Chary { 65536, 2048, 16, 4096, 8, 8, 7, 5, 180, 450, 64, 70, 1000, 2500 },
70245f7b51SCorentin Chary { 65536, 2048, 32, 8192, 9, 9, 8, 6, 190, 475, 64, 75, 500, 1200 },
71245f7b51SCorentin Chary { 65536, 2048, 32, 8192, 9, 9, 9, 6, 200, 500, 96, 80, 200, 500 }
72245f7b51SCorentin Chary };
73245f7b51SCorentin Chary
74efe556adSCorentin Chary
75efe556adSCorentin Chary static int tight_send_framebuffer_update(VncState *vs, int x, int y,
76efe556adSCorentin Chary int w, int h);
77efe556adSCorentin Chary
78ce702e93SCorentin Chary #ifdef CONFIG_VNC_JPEG
79ce702e93SCorentin Chary static const struct {
80313e1629SStefan Weil double jpeg_freq_min; /* Don't send JPEG if the freq is below */
81ce702e93SCorentin Chary double jpeg_freq_threshold; /* Always send JPEG if the freq is above */
82ce702e93SCorentin Chary int jpeg_idx; /* Allow indexed JPEG */
83ce702e93SCorentin Chary int jpeg_full; /* Allow full color JPEG */
84ce702e93SCorentin Chary } tight_jpeg_conf[] = {
858cb4a6b7SCorentin Chary { 0, 8, 1, 1 },
868cb4a6b7SCorentin Chary { 0, 8, 1, 1 },
878cb4a6b7SCorentin Chary { 0, 8, 1, 1 },
888cb4a6b7SCorentin Chary { 0, 8, 1, 1 },
898cb4a6b7SCorentin Chary { 0, 10, 1, 1 },
908cb4a6b7SCorentin Chary { 0.1, 10, 1, 1 },
918cb4a6b7SCorentin Chary { 0.2, 10, 1, 1 },
928cb4a6b7SCorentin Chary { 0.3, 12, 0, 0 },
938cb4a6b7SCorentin Chary { 0.4, 14, 0, 0 },
948cb4a6b7SCorentin Chary { 0.5, 16, 0, 0 },
95ce702e93SCorentin Chary };
96ce702e93SCorentin Chary #endif
97ce702e93SCorentin Chary
9895f8510eSKshitij Suri #ifdef CONFIG_PNG
993941bf6fSCorentin Chary static const struct {
1003941bf6fSCorentin Chary int png_zlib_level, png_filters;
1013941bf6fSCorentin Chary } tight_png_conf[] = {
1023941bf6fSCorentin Chary { 0, PNG_NO_FILTERS },
1033941bf6fSCorentin Chary { 1, PNG_NO_FILTERS },
1043941bf6fSCorentin Chary { 2, PNG_NO_FILTERS },
1053941bf6fSCorentin Chary { 3, PNG_NO_FILTERS },
1063941bf6fSCorentin Chary { 4, PNG_NO_FILTERS },
1073941bf6fSCorentin Chary { 5, PNG_ALL_FILTERS },
1083941bf6fSCorentin Chary { 6, PNG_ALL_FILTERS },
1093941bf6fSCorentin Chary { 7, PNG_ALL_FILTERS },
1103941bf6fSCorentin Chary { 8, PNG_ALL_FILTERS },
1113941bf6fSCorentin Chary { 9, PNG_ALL_FILTERS },
1123941bf6fSCorentin Chary };
1133941bf6fSCorentin Chary
114efe556adSCorentin Chary static int send_png_rect(VncState *vs, int x, int y, int w, int h,
1155136a052SCorentin Chary VncPalette *palette);
116efe556adSCorentin Chary
tight_can_send_png_rect(VncState * vs,int w,int h)117efe556adSCorentin Chary static bool tight_can_send_png_rect(VncState *vs, int w, int h)
118efe556adSCorentin Chary {
1196bf21f3dSLi Qiang if (vs->tight->type != VNC_ENCODING_TIGHT_PNG) {
120efe556adSCorentin Chary return false;
121efe556adSCorentin Chary }
122efe556adSCorentin Chary
123d39fa6d8SGerd Hoffmann if (surface_bytes_per_pixel(vs->vd->ds) == 1 ||
1249f64916dSGerd Hoffmann vs->client_pf.bytes_per_pixel == 1) {
125efe556adSCorentin Chary return false;
126efe556adSCorentin Chary }
127efe556adSCorentin Chary
128efe556adSCorentin Chary return true;
129efe556adSCorentin Chary }
130efe556adSCorentin Chary #endif
131efe556adSCorentin Chary
132245f7b51SCorentin Chary /*
133245f7b51SCorentin Chary * Code to guess if given rectangle is suitable for smooth image
134245f7b51SCorentin Chary * compression (by applying "gradient" filter or JPEG coder).
135245f7b51SCorentin Chary */
136245f7b51SCorentin Chary
137249cdb42SBlue Swirl static unsigned int
tight_detect_smooth_image24(VncState * vs,int w,int h)138245f7b51SCorentin Chary tight_detect_smooth_image24(VncState *vs, int w, int h)
139245f7b51SCorentin Chary {
140245f7b51SCorentin Chary int off;
141245f7b51SCorentin Chary int x, y, d, dx;
142249cdb42SBlue Swirl unsigned int c;
143249cdb42SBlue Swirl unsigned int stats[256];
144245f7b51SCorentin Chary int pixels = 0;
145245f7b51SCorentin Chary int pix, left[3];
146249cdb42SBlue Swirl unsigned int errors;
1476bf21f3dSLi Qiang unsigned char *buf = vs->tight->tight.buffer;
148245f7b51SCorentin Chary
149245f7b51SCorentin Chary /*
150245f7b51SCorentin Chary * If client is big-endian, color samples begin from the second
151245f7b51SCorentin Chary * byte (offset 1) of a 32-bit pixel value.
152245f7b51SCorentin Chary */
1539f64916dSGerd Hoffmann off = vs->client_be;
154245f7b51SCorentin Chary
155245f7b51SCorentin Chary memset(stats, 0, sizeof (stats));
156245f7b51SCorentin Chary
157245f7b51SCorentin Chary for (y = 0, x = 0; y < h && x < w;) {
158245f7b51SCorentin Chary for (d = 0; d < h - y && d < w - x - VNC_TIGHT_DETECT_SUBROW_WIDTH;
159245f7b51SCorentin Chary d++) {
160245f7b51SCorentin Chary for (c = 0; c < 3; c++) {
161245f7b51SCorentin Chary left[c] = buf[((y+d)*w+x+d)*4+off+c] & 0xFF;
162245f7b51SCorentin Chary }
163245f7b51SCorentin Chary for (dx = 1; dx <= VNC_TIGHT_DETECT_SUBROW_WIDTH; dx++) {
164245f7b51SCorentin Chary for (c = 0; c < 3; c++) {
165245f7b51SCorentin Chary pix = buf[((y+d)*w+x+d+dx)*4+off+c] & 0xFF;
166245f7b51SCorentin Chary stats[abs(pix - left[c])]++;
167245f7b51SCorentin Chary left[c] = pix;
168245f7b51SCorentin Chary }
169245f7b51SCorentin Chary pixels++;
170245f7b51SCorentin Chary }
171245f7b51SCorentin Chary }
172245f7b51SCorentin Chary if (w > h) {
173245f7b51SCorentin Chary x += h;
174245f7b51SCorentin Chary y = 0;
175245f7b51SCorentin Chary } else {
176245f7b51SCorentin Chary x = 0;
177245f7b51SCorentin Chary y += w;
178245f7b51SCorentin Chary }
179245f7b51SCorentin Chary }
180245f7b51SCorentin Chary
181b5299153SGonglei if (pixels == 0) {
182b5299153SGonglei return 0;
183b5299153SGonglei }
184b5299153SGonglei
185245f7b51SCorentin Chary /* 95% smooth or more ... */
186245f7b51SCorentin Chary if (stats[0] * 33 / pixels >= 95) {
187245f7b51SCorentin Chary return 0;
188245f7b51SCorentin Chary }
189245f7b51SCorentin Chary
190245f7b51SCorentin Chary errors = 0;
191245f7b51SCorentin Chary for (c = 1; c < 8; c++) {
192245f7b51SCorentin Chary errors += stats[c] * (c * c);
193245f7b51SCorentin Chary if (stats[c] == 0 || stats[c] > stats[c-1] * 2) {
194245f7b51SCorentin Chary return 0;
195245f7b51SCorentin Chary }
196245f7b51SCorentin Chary }
197245f7b51SCorentin Chary for (; c < 256; c++) {
198245f7b51SCorentin Chary errors += stats[c] * (c * c);
199245f7b51SCorentin Chary }
200245f7b51SCorentin Chary errors /= (pixels * 3 - stats[0]);
201245f7b51SCorentin Chary
202245f7b51SCorentin Chary return errors;
203245f7b51SCorentin Chary }
204245f7b51SCorentin Chary
205245f7b51SCorentin Chary #define DEFINE_DETECT_FUNCTION(bpp) \
206245f7b51SCorentin Chary \
207249cdb42SBlue Swirl static unsigned int \
208245f7b51SCorentin Chary tight_detect_smooth_image##bpp(VncState *vs, int w, int h) { \
209245f7b51SCorentin Chary bool endian; \
210245f7b51SCorentin Chary uint##bpp##_t pix; \
211245f7b51SCorentin Chary int max[3], shift[3]; \
212245f7b51SCorentin Chary int x, y, d, dx; \
213249cdb42SBlue Swirl unsigned int c; \
214249cdb42SBlue Swirl unsigned int stats[256]; \
215245f7b51SCorentin Chary int pixels = 0; \
216245f7b51SCorentin Chary int sample, sum, left[3]; \
217249cdb42SBlue Swirl unsigned int errors; \
2186bf21f3dSLi Qiang unsigned char *buf = vs->tight->tight.buffer; \
219245f7b51SCorentin Chary \
22077bfcf28SBenjamin Herrenschmidt endian = 0; /* FIXME */ \
221245f7b51SCorentin Chary \
222245f7b51SCorentin Chary \
2239f64916dSGerd Hoffmann max[0] = vs->client_pf.rmax; \
2249f64916dSGerd Hoffmann max[1] = vs->client_pf.gmax; \
2259f64916dSGerd Hoffmann max[2] = vs->client_pf.bmax; \
2269f64916dSGerd Hoffmann shift[0] = vs->client_pf.rshift; \
2279f64916dSGerd Hoffmann shift[1] = vs->client_pf.gshift; \
2289f64916dSGerd Hoffmann shift[2] = vs->client_pf.bshift; \
229245f7b51SCorentin Chary \
230245f7b51SCorentin Chary memset(stats, 0, sizeof(stats)); \
231245f7b51SCorentin Chary \
232245f7b51SCorentin Chary y = 0, x = 0; \
233245f7b51SCorentin Chary while (y < h && x < w) { \
234245f7b51SCorentin Chary for (d = 0; d < h - y && \
235245f7b51SCorentin Chary d < w - x - VNC_TIGHT_DETECT_SUBROW_WIDTH; d++) { \
236245f7b51SCorentin Chary pix = ((uint##bpp##_t *)buf)[(y+d)*w+x+d]; \
237245f7b51SCorentin Chary if (endian) { \
238ba5e7f82SIzumi Tsutsui pix = bswap##bpp(pix); \
239245f7b51SCorentin Chary } \
240245f7b51SCorentin Chary for (c = 0; c < 3; c++) { \
241245f7b51SCorentin Chary left[c] = (int)(pix >> shift[c] & max[c]); \
242245f7b51SCorentin Chary } \
243245f7b51SCorentin Chary for (dx = 1; dx <= VNC_TIGHT_DETECT_SUBROW_WIDTH; \
244245f7b51SCorentin Chary dx++) { \
245245f7b51SCorentin Chary pix = ((uint##bpp##_t *)buf)[(y+d)*w+x+d+dx]; \
246245f7b51SCorentin Chary if (endian) { \
247ba5e7f82SIzumi Tsutsui pix = bswap##bpp(pix); \
248245f7b51SCorentin Chary } \
249245f7b51SCorentin Chary sum = 0; \
250245f7b51SCorentin Chary for (c = 0; c < 3; c++) { \
251245f7b51SCorentin Chary sample = (int)(pix >> shift[c] & max[c]); \
252245f7b51SCorentin Chary sum += abs(sample - left[c]); \
253245f7b51SCorentin Chary left[c] = sample; \
254245f7b51SCorentin Chary } \
255245f7b51SCorentin Chary if (sum > 255) { \
256245f7b51SCorentin Chary sum = 255; \
257245f7b51SCorentin Chary } \
258245f7b51SCorentin Chary stats[sum]++; \
259245f7b51SCorentin Chary pixels++; \
260245f7b51SCorentin Chary } \
261245f7b51SCorentin Chary } \
262245f7b51SCorentin Chary if (w > h) { \
263245f7b51SCorentin Chary x += h; \
264245f7b51SCorentin Chary y = 0; \
265245f7b51SCorentin Chary } else { \
266245f7b51SCorentin Chary x = 0; \
267245f7b51SCorentin Chary y += w; \
268245f7b51SCorentin Chary } \
269245f7b51SCorentin Chary } \
270b5299153SGonglei if (pixels == 0) { \
271b5299153SGonglei return 0; \
272b5299153SGonglei } \
273245f7b51SCorentin Chary if ((stats[0] + stats[1]) * 100 / pixels >= 90) { \
274245f7b51SCorentin Chary return 0; \
275245f7b51SCorentin Chary } \
276245f7b51SCorentin Chary \
277245f7b51SCorentin Chary errors = 0; \
278245f7b51SCorentin Chary for (c = 1; c < 8; c++) { \
279245f7b51SCorentin Chary errors += stats[c] * (c * c); \
280245f7b51SCorentin Chary if (stats[c] == 0 || stats[c] > stats[c-1] * 2) { \
281245f7b51SCorentin Chary return 0; \
282245f7b51SCorentin Chary } \
283245f7b51SCorentin Chary } \
284245f7b51SCorentin Chary for (; c < 256; c++) { \
285245f7b51SCorentin Chary errors += stats[c] * (c * c); \
286245f7b51SCorentin Chary } \
287245f7b51SCorentin Chary errors /= (pixels - stats[0]); \
288245f7b51SCorentin Chary \
289245f7b51SCorentin Chary return errors; \
290245f7b51SCorentin Chary }
291245f7b51SCorentin Chary
292245f7b51SCorentin Chary DEFINE_DETECT_FUNCTION(16)
293245f7b51SCorentin Chary DEFINE_DETECT_FUNCTION(32)
294245f7b51SCorentin Chary
295245f7b51SCorentin Chary static int
tight_detect_smooth_image(VncState * vs,int w,int h)296245f7b51SCorentin Chary tight_detect_smooth_image(VncState *vs, int w, int h)
297245f7b51SCorentin Chary {
298249cdb42SBlue Swirl unsigned int errors;
2996bf21f3dSLi Qiang int compression = vs->tight->compression;
3006bf21f3dSLi Qiang int quality = vs->tight->quality;
301245f7b51SCorentin Chary
302245f7b51SCorentin Chary if (!vs->vd->lossy) {
303245f7b51SCorentin Chary return 0;
304245f7b51SCorentin Chary }
305245f7b51SCorentin Chary
306d39fa6d8SGerd Hoffmann if (surface_bytes_per_pixel(vs->vd->ds) == 1 ||
3079f64916dSGerd Hoffmann vs->client_pf.bytes_per_pixel == 1 ||
308245f7b51SCorentin Chary w < VNC_TIGHT_DETECT_MIN_WIDTH || h < VNC_TIGHT_DETECT_MIN_HEIGHT) {
309245f7b51SCorentin Chary return 0;
310245f7b51SCorentin Chary }
311245f7b51SCorentin Chary
3126bf21f3dSLi Qiang if (vs->tight->quality != (uint8_t)-1) {
313245f7b51SCorentin Chary if (w * h < VNC_TIGHT_JPEG_MIN_RECT_SIZE) {
314245f7b51SCorentin Chary return 0;
315245f7b51SCorentin Chary }
316245f7b51SCorentin Chary } else {
317245f7b51SCorentin Chary if (w * h < tight_conf[compression].gradient_min_rect_size) {
318245f7b51SCorentin Chary return 0;
319245f7b51SCorentin Chary }
320245f7b51SCorentin Chary }
321245f7b51SCorentin Chary
3229f64916dSGerd Hoffmann if (vs->client_pf.bytes_per_pixel == 4) {
3236bf21f3dSLi Qiang if (vs->tight->pixel24) {
324245f7b51SCorentin Chary errors = tight_detect_smooth_image24(vs, w, h);
3256bf21f3dSLi Qiang if (vs->tight->quality != (uint8_t)-1) {
326245f7b51SCorentin Chary return (errors < tight_conf[quality].jpeg_threshold24);
327245f7b51SCorentin Chary }
328245f7b51SCorentin Chary return (errors < tight_conf[compression].gradient_threshold24);
329245f7b51SCorentin Chary } else {
330245f7b51SCorentin Chary errors = tight_detect_smooth_image32(vs, w, h);
331245f7b51SCorentin Chary }
332245f7b51SCorentin Chary } else {
333245f7b51SCorentin Chary errors = tight_detect_smooth_image16(vs, w, h);
334245f7b51SCorentin Chary }
3352e7bcdb9SMarkus Armbruster if (quality != (uint8_t)-1) {
336245f7b51SCorentin Chary return (errors < tight_conf[quality].jpeg_threshold);
337245f7b51SCorentin Chary }
338245f7b51SCorentin Chary return (errors < tight_conf[compression].gradient_threshold);
339245f7b51SCorentin Chary }
340245f7b51SCorentin Chary
341245f7b51SCorentin Chary /*
342245f7b51SCorentin Chary * Code to determine how many different colors used in rectangle.
343245f7b51SCorentin Chary */
344245f7b51SCorentin Chary #define DEFINE_FILL_PALETTE_FUNCTION(bpp) \
345245f7b51SCorentin Chary \
346245f7b51SCorentin Chary static int \
347245f7b51SCorentin Chary tight_fill_palette##bpp(VncState *vs, int x, int y, \
348245f7b51SCorentin Chary int max, size_t count, \
349245f7b51SCorentin Chary uint32_t *bg, uint32_t *fg, \
350095497ffSPeter Lieven VncPalette *palette) { \
351245f7b51SCorentin Chary uint##bpp##_t *data; \
352245f7b51SCorentin Chary uint##bpp##_t c0, c1, ci; \
353245f7b51SCorentin Chary int i, n0, n1; \
354245f7b51SCorentin Chary \
3556bf21f3dSLi Qiang data = (uint##bpp##_t *)vs->tight->tight.buffer; \
356245f7b51SCorentin Chary \
357245f7b51SCorentin Chary c0 = data[0]; \
358245f7b51SCorentin Chary i = 1; \
359245f7b51SCorentin Chary while (i < count && data[i] == c0) \
360245f7b51SCorentin Chary i++; \
361245f7b51SCorentin Chary if (i >= count) { \
362245f7b51SCorentin Chary *bg = *fg = c0; \
363245f7b51SCorentin Chary return 1; \
364245f7b51SCorentin Chary } \
365245f7b51SCorentin Chary \
366245f7b51SCorentin Chary if (max < 2) { \
367245f7b51SCorentin Chary return 0; \
368245f7b51SCorentin Chary } \
369245f7b51SCorentin Chary \
370245f7b51SCorentin Chary n0 = i; \
371245f7b51SCorentin Chary c1 = data[i]; \
372245f7b51SCorentin Chary n1 = 0; \
373245f7b51SCorentin Chary for (i++; i < count; i++) { \
374245f7b51SCorentin Chary ci = data[i]; \
375245f7b51SCorentin Chary if (ci == c0) { \
376245f7b51SCorentin Chary n0++; \
377245f7b51SCorentin Chary } else if (ci == c1) { \
378245f7b51SCorentin Chary n1++; \
379245f7b51SCorentin Chary } else \
380245f7b51SCorentin Chary break; \
381245f7b51SCorentin Chary } \
382245f7b51SCorentin Chary if (i >= count) { \
383245f7b51SCorentin Chary if (n0 > n1) { \
384245f7b51SCorentin Chary *bg = (uint32_t)c0; \
385245f7b51SCorentin Chary *fg = (uint32_t)c1; \
386245f7b51SCorentin Chary } else { \
387245f7b51SCorentin Chary *bg = (uint32_t)c1; \
388245f7b51SCorentin Chary *fg = (uint32_t)c0; \
389245f7b51SCorentin Chary } \
390245f7b51SCorentin Chary return 2; \
391245f7b51SCorentin Chary } \
392245f7b51SCorentin Chary \
393245f7b51SCorentin Chary if (max == 2) { \
394245f7b51SCorentin Chary return 0; \
395245f7b51SCorentin Chary } \
396245f7b51SCorentin Chary \
397095497ffSPeter Lieven palette_init(palette, max, bpp); \
398095497ffSPeter Lieven palette_put(palette, c0); \
399095497ffSPeter Lieven palette_put(palette, c1); \
400095497ffSPeter Lieven palette_put(palette, ci); \
401245f7b51SCorentin Chary \
402245f7b51SCorentin Chary for (i++; i < count; i++) { \
403245f7b51SCorentin Chary if (data[i] == ci) { \
404245f7b51SCorentin Chary continue; \
405245f7b51SCorentin Chary } else { \
4065d8efe39SCorentin Chary ci = data[i]; \
407095497ffSPeter Lieven if (!palette_put(palette, (uint32_t)ci)) { \
408245f7b51SCorentin Chary return 0; \
409245f7b51SCorentin Chary } \
410245f7b51SCorentin Chary } \
411245f7b51SCorentin Chary } \
412245f7b51SCorentin Chary \
413095497ffSPeter Lieven return palette_size(palette); \
414245f7b51SCorentin Chary }
415245f7b51SCorentin Chary
416245f7b51SCorentin Chary DEFINE_FILL_PALETTE_FUNCTION(8)
417245f7b51SCorentin Chary DEFINE_FILL_PALETTE_FUNCTION(16)
418245f7b51SCorentin Chary DEFINE_FILL_PALETTE_FUNCTION(32)
419245f7b51SCorentin Chary
tight_fill_palette(VncState * vs,int x,int y,size_t count,uint32_t * bg,uint32_t * fg,VncPalette * palette)420245f7b51SCorentin Chary static int tight_fill_palette(VncState *vs, int x, int y,
421245f7b51SCorentin Chary size_t count, uint32_t *bg, uint32_t *fg,
422095497ffSPeter Lieven VncPalette *palette)
423245f7b51SCorentin Chary {
424245f7b51SCorentin Chary int max;
425245f7b51SCorentin Chary
4266bf21f3dSLi Qiang max = count / tight_conf[vs->tight->compression].idx_max_colors_divisor;
427245f7b51SCorentin Chary if (max < 2 &&
4286bf21f3dSLi Qiang count >= tight_conf[vs->tight->compression].mono_min_rect_size) {
429245f7b51SCorentin Chary max = 2;
430245f7b51SCorentin Chary }
431245f7b51SCorentin Chary if (max >= 256) {
432245f7b51SCorentin Chary max = 256;
433245f7b51SCorentin Chary }
434245f7b51SCorentin Chary
4359f64916dSGerd Hoffmann switch (vs->client_pf.bytes_per_pixel) {
436245f7b51SCorentin Chary case 4:
437245f7b51SCorentin Chary return tight_fill_palette32(vs, x, y, max, count, bg, fg, palette);
438245f7b51SCorentin Chary case 2:
439245f7b51SCorentin Chary return tight_fill_palette16(vs, x, y, max, count, bg, fg, palette);
440245f7b51SCorentin Chary default:
441245f7b51SCorentin Chary max = 2;
442245f7b51SCorentin Chary return tight_fill_palette8(vs, x, y, max, count, bg, fg, palette);
443245f7b51SCorentin Chary }
444245f7b51SCorentin Chary return 0;
445245f7b51SCorentin Chary }
446245f7b51SCorentin Chary
447245f7b51SCorentin Chary /*
448245f7b51SCorentin Chary * Converting truecolor samples into palette indices.
449245f7b51SCorentin Chary */
450245f7b51SCorentin Chary #define DEFINE_IDX_ENCODE_FUNCTION(bpp) \
451245f7b51SCorentin Chary \
452245f7b51SCorentin Chary static void \
453245f7b51SCorentin Chary tight_encode_indexed_rect##bpp(uint8_t *buf, int count, \
4545136a052SCorentin Chary VncPalette *palette) { \
455245f7b51SCorentin Chary uint##bpp##_t *src; \
456245f7b51SCorentin Chary uint##bpp##_t rgb; \
457245f7b51SCorentin Chary int i, rep; \
458245f7b51SCorentin Chary uint8_t idx; \
459245f7b51SCorentin Chary \
460245f7b51SCorentin Chary src = (uint##bpp##_t *) buf; \
461245f7b51SCorentin Chary \
4623f7e51bcSHerongguang (Stephen) for (i = 0; i < count; ) { \
463efe556adSCorentin Chary \
464245f7b51SCorentin Chary rgb = *src++; \
4653f7e51bcSHerongguang (Stephen) i++; \
466245f7b51SCorentin Chary rep = 0; \
467245f7b51SCorentin Chary while (i < count && *src == rgb) { \
468245f7b51SCorentin Chary rep++, src++, i++; \
469245f7b51SCorentin Chary } \
4705136a052SCorentin Chary idx = palette_idx(palette, rgb); \
471245f7b51SCorentin Chary /* \
472245f7b51SCorentin Chary * Should never happen, but don't break everything \
473245f7b51SCorentin Chary * if it does, use the first color instead \
474245f7b51SCorentin Chary */ \
4757bccf573SBlue Swirl if (idx == (uint8_t)-1) { \
476245f7b51SCorentin Chary idx = 0; \
477245f7b51SCorentin Chary } \
478245f7b51SCorentin Chary while (rep >= 0) { \
479245f7b51SCorentin Chary *buf++ = idx; \
480245f7b51SCorentin Chary rep--; \
481245f7b51SCorentin Chary } \
482245f7b51SCorentin Chary } \
483245f7b51SCorentin Chary }
484245f7b51SCorentin Chary
485245f7b51SCorentin Chary DEFINE_IDX_ENCODE_FUNCTION(16)
486245f7b51SCorentin Chary DEFINE_IDX_ENCODE_FUNCTION(32)
487245f7b51SCorentin Chary
488245f7b51SCorentin Chary #define DEFINE_MONO_ENCODE_FUNCTION(bpp) \
489245f7b51SCorentin Chary \
490245f7b51SCorentin Chary static void \
491245f7b51SCorentin Chary tight_encode_mono_rect##bpp(uint8_t *buf, int w, int h, \
492245f7b51SCorentin Chary uint##bpp##_t bg, uint##bpp##_t fg) { \
493245f7b51SCorentin Chary uint##bpp##_t *ptr; \
494245f7b51SCorentin Chary unsigned int value, mask; \
495245f7b51SCorentin Chary int aligned_width; \
496245f7b51SCorentin Chary int x, y, bg_bits; \
497245f7b51SCorentin Chary \
498245f7b51SCorentin Chary ptr = (uint##bpp##_t *) buf; \
499245f7b51SCorentin Chary aligned_width = w - w % 8; \
500245f7b51SCorentin Chary \
501245f7b51SCorentin Chary for (y = 0; y < h; y++) { \
502245f7b51SCorentin Chary for (x = 0; x < aligned_width; x += 8) { \
503245f7b51SCorentin Chary for (bg_bits = 0; bg_bits < 8; bg_bits++) { \
504245f7b51SCorentin Chary if (*ptr++ != bg) { \
505245f7b51SCorentin Chary break; \
506245f7b51SCorentin Chary } \
507245f7b51SCorentin Chary } \
508245f7b51SCorentin Chary if (bg_bits == 8) { \
509245f7b51SCorentin Chary *buf++ = 0; \
510245f7b51SCorentin Chary continue; \
511245f7b51SCorentin Chary } \
512245f7b51SCorentin Chary mask = 0x80 >> bg_bits; \
513245f7b51SCorentin Chary value = mask; \
514245f7b51SCorentin Chary for (bg_bits++; bg_bits < 8; bg_bits++) { \
515245f7b51SCorentin Chary mask >>= 1; \
516245f7b51SCorentin Chary if (*ptr++ != bg) { \
517245f7b51SCorentin Chary value |= mask; \
518245f7b51SCorentin Chary } \
519245f7b51SCorentin Chary } \
520245f7b51SCorentin Chary *buf++ = (uint8_t)value; \
521245f7b51SCorentin Chary } \
522245f7b51SCorentin Chary \
523245f7b51SCorentin Chary mask = 0x80; \
524245f7b51SCorentin Chary value = 0; \
525245f7b51SCorentin Chary if (x >= w) { \
526245f7b51SCorentin Chary continue; \
527245f7b51SCorentin Chary } \
528245f7b51SCorentin Chary \
529245f7b51SCorentin Chary for (; x < w; x++) { \
530245f7b51SCorentin Chary if (*ptr++ != bg) { \
531245f7b51SCorentin Chary value |= mask; \
532245f7b51SCorentin Chary } \
533245f7b51SCorentin Chary mask >>= 1; \
534245f7b51SCorentin Chary } \
535245f7b51SCorentin Chary *buf++ = (uint8_t)value; \
536245f7b51SCorentin Chary } \
537245f7b51SCorentin Chary }
538245f7b51SCorentin Chary
539245f7b51SCorentin Chary DEFINE_MONO_ENCODE_FUNCTION(8)
540245f7b51SCorentin Chary DEFINE_MONO_ENCODE_FUNCTION(16)
541245f7b51SCorentin Chary DEFINE_MONO_ENCODE_FUNCTION(32)
542245f7b51SCorentin Chary
543245f7b51SCorentin Chary /*
544245f7b51SCorentin Chary * ``Gradient'' filter for 24-bit color samples.
545245f7b51SCorentin Chary * Should be called only when redMax, greenMax and blueMax are 255.
546245f7b51SCorentin Chary * Color components assumed to be byte-aligned.
547245f7b51SCorentin Chary */
548245f7b51SCorentin Chary
549245f7b51SCorentin Chary static void
tight_filter_gradient24(VncState * vs,uint8_t * buf,int w,int h)550245f7b51SCorentin Chary tight_filter_gradient24(VncState *vs, uint8_t *buf, int w, int h)
551245f7b51SCorentin Chary {
552245f7b51SCorentin Chary uint32_t *buf32;
553245f7b51SCorentin Chary uint32_t pix32;
554245f7b51SCorentin Chary int shift[3];
555245f7b51SCorentin Chary int *prev;
556245f7b51SCorentin Chary int here[3], upper[3], left[3], upperleft[3];
557245f7b51SCorentin Chary int prediction;
558245f7b51SCorentin Chary int x, y, c;
559245f7b51SCorentin Chary
560245f7b51SCorentin Chary buf32 = (uint32_t *)buf;
5616bf21f3dSLi Qiang memset(vs->tight->gradient.buffer, 0, w * 3 * sizeof(int));
562245f7b51SCorentin Chary
56377bfcf28SBenjamin Herrenschmidt if (1 /* FIXME */) {
5649f64916dSGerd Hoffmann shift[0] = vs->client_pf.rshift;
5659f64916dSGerd Hoffmann shift[1] = vs->client_pf.gshift;
5669f64916dSGerd Hoffmann shift[2] = vs->client_pf.bshift;
567245f7b51SCorentin Chary } else {
5689f64916dSGerd Hoffmann shift[0] = 24 - vs->client_pf.rshift;
5699f64916dSGerd Hoffmann shift[1] = 24 - vs->client_pf.gshift;
5709f64916dSGerd Hoffmann shift[2] = 24 - vs->client_pf.bshift;
571245f7b51SCorentin Chary }
572245f7b51SCorentin Chary
573245f7b51SCorentin Chary for (y = 0; y < h; y++) {
574245f7b51SCorentin Chary for (c = 0; c < 3; c++) {
575245f7b51SCorentin Chary upper[c] = 0;
576245f7b51SCorentin Chary here[c] = 0;
577245f7b51SCorentin Chary }
5786bf21f3dSLi Qiang prev = (int *)vs->tight->gradient.buffer;
579245f7b51SCorentin Chary for (x = 0; x < w; x++) {
580245f7b51SCorentin Chary pix32 = *buf32++;
581245f7b51SCorentin Chary for (c = 0; c < 3; c++) {
582245f7b51SCorentin Chary upperleft[c] = upper[c];
583245f7b51SCorentin Chary left[c] = here[c];
584245f7b51SCorentin Chary upper[c] = *prev;
585245f7b51SCorentin Chary here[c] = (int)(pix32 >> shift[c] & 0xFF);
586245f7b51SCorentin Chary *prev++ = here[c];
587245f7b51SCorentin Chary
588245f7b51SCorentin Chary prediction = left[c] + upper[c] - upperleft[c];
589245f7b51SCorentin Chary if (prediction < 0) {
590245f7b51SCorentin Chary prediction = 0;
591245f7b51SCorentin Chary } else if (prediction > 0xFF) {
592245f7b51SCorentin Chary prediction = 0xFF;
593245f7b51SCorentin Chary }
594245f7b51SCorentin Chary *buf++ = (char)(here[c] - prediction);
595245f7b51SCorentin Chary }
596245f7b51SCorentin Chary }
597245f7b51SCorentin Chary }
598245f7b51SCorentin Chary }
599245f7b51SCorentin Chary
600245f7b51SCorentin Chary
601245f7b51SCorentin Chary /*
602245f7b51SCorentin Chary * ``Gradient'' filter for other color depths.
603245f7b51SCorentin Chary */
604245f7b51SCorentin Chary
605245f7b51SCorentin Chary #define DEFINE_GRADIENT_FILTER_FUNCTION(bpp) \
606245f7b51SCorentin Chary \
607245f7b51SCorentin Chary static void \
608245f7b51SCorentin Chary tight_filter_gradient##bpp(VncState *vs, uint##bpp##_t *buf, \
609245f7b51SCorentin Chary int w, int h) { \
610245f7b51SCorentin Chary uint##bpp##_t pix, diff; \
611245f7b51SCorentin Chary bool endian; \
612245f7b51SCorentin Chary int *prev; \
613245f7b51SCorentin Chary int max[3], shift[3]; \
614245f7b51SCorentin Chary int here[3], upper[3], left[3], upperleft[3]; \
615245f7b51SCorentin Chary int prediction; \
616245f7b51SCorentin Chary int x, y, c; \
617245f7b51SCorentin Chary \
6186bf21f3dSLi Qiang memset(vs->tight->gradient.buffer, 0, w * 3 * sizeof(int)); \
619245f7b51SCorentin Chary \
62077bfcf28SBenjamin Herrenschmidt endian = 0; /* FIXME */ \
621245f7b51SCorentin Chary \
6229f64916dSGerd Hoffmann max[0] = vs->client_pf.rmax; \
6239f64916dSGerd Hoffmann max[1] = vs->client_pf.gmax; \
6249f64916dSGerd Hoffmann max[2] = vs->client_pf.bmax; \
6259f64916dSGerd Hoffmann shift[0] = vs->client_pf.rshift; \
6269f64916dSGerd Hoffmann shift[1] = vs->client_pf.gshift; \
6279f64916dSGerd Hoffmann shift[2] = vs->client_pf.bshift; \
628245f7b51SCorentin Chary \
629245f7b51SCorentin Chary for (y = 0; y < h; y++) { \
630245f7b51SCorentin Chary for (c = 0; c < 3; c++) { \
631245f7b51SCorentin Chary upper[c] = 0; \
632245f7b51SCorentin Chary here[c] = 0; \
633245f7b51SCorentin Chary } \
6346bf21f3dSLi Qiang prev = (int *)vs->tight->gradient.buffer; \
635245f7b51SCorentin Chary for (x = 0; x < w; x++) { \
636245f7b51SCorentin Chary pix = *buf; \
637245f7b51SCorentin Chary if (endian) { \
638ba5e7f82SIzumi Tsutsui pix = bswap##bpp(pix); \
639245f7b51SCorentin Chary } \
640245f7b51SCorentin Chary diff = 0; \
641245f7b51SCorentin Chary for (c = 0; c < 3; c++) { \
642245f7b51SCorentin Chary upperleft[c] = upper[c]; \
643245f7b51SCorentin Chary left[c] = here[c]; \
644245f7b51SCorentin Chary upper[c] = *prev; \
645245f7b51SCorentin Chary here[c] = (int)(pix >> shift[c] & max[c]); \
646245f7b51SCorentin Chary *prev++ = here[c]; \
647245f7b51SCorentin Chary \
648245f7b51SCorentin Chary prediction = left[c] + upper[c] - upperleft[c]; \
649245f7b51SCorentin Chary if (prediction < 0) { \
650245f7b51SCorentin Chary prediction = 0; \
651245f7b51SCorentin Chary } else if (prediction > max[c]) { \
652245f7b51SCorentin Chary prediction = max[c]; \
653245f7b51SCorentin Chary } \
654245f7b51SCorentin Chary diff |= ((here[c] - prediction) & max[c]) \
655245f7b51SCorentin Chary << shift[c]; \
656245f7b51SCorentin Chary } \
657245f7b51SCorentin Chary if (endian) { \
658ba5e7f82SIzumi Tsutsui diff = bswap##bpp(diff); \
659245f7b51SCorentin Chary } \
660245f7b51SCorentin Chary *buf++ = diff; \
661245f7b51SCorentin Chary } \
662245f7b51SCorentin Chary } \
663245f7b51SCorentin Chary }
664245f7b51SCorentin Chary
665245f7b51SCorentin Chary DEFINE_GRADIENT_FILTER_FUNCTION(16)
666245f7b51SCorentin Chary DEFINE_GRADIENT_FILTER_FUNCTION(32)
667245f7b51SCorentin Chary
668245f7b51SCorentin Chary /*
669245f7b51SCorentin Chary * Check if a rectangle is all of the same color. If needSameColor is
670245f7b51SCorentin Chary * set to non-zero, then also check that its color equals to the
671b0cd712cSStefan Weil * *colorPtr value. The result is 1 if the test is successful, and in
672245f7b51SCorentin Chary * that case new color will be stored in *colorPtr.
673245f7b51SCorentin Chary */
674245f7b51SCorentin Chary
67594362682SGerd Hoffmann static bool
check_solid_tile32(VncState * vs,int x,int y,int w,int h,uint32_t * color,bool samecolor)67694362682SGerd Hoffmann check_solid_tile32(VncState *vs, int x, int y, int w, int h,
67794362682SGerd Hoffmann uint32_t *color, bool samecolor)
67894362682SGerd Hoffmann {
67994362682SGerd Hoffmann VncDisplay *vd = vs->vd;
68094362682SGerd Hoffmann uint32_t *fbptr;
68194362682SGerd Hoffmann uint32_t c;
68294362682SGerd Hoffmann int dx, dy;
68394362682SGerd Hoffmann
68494362682SGerd Hoffmann fbptr = vnc_server_fb_ptr(vd, x, y);
68594362682SGerd Hoffmann
68694362682SGerd Hoffmann c = *fbptr;
68794362682SGerd Hoffmann if (samecolor && (uint32_t)c != *color) {
68894362682SGerd Hoffmann return false;
689245f7b51SCorentin Chary }
690245f7b51SCorentin Chary
69194362682SGerd Hoffmann for (dy = 0; dy < h; dy++) {
69294362682SGerd Hoffmann for (dx = 0; dx < w; dx++) {
69394362682SGerd Hoffmann if (c != fbptr[dx]) {
69494362682SGerd Hoffmann return false;
69594362682SGerd Hoffmann }
69694362682SGerd Hoffmann }
69794362682SGerd Hoffmann fbptr = (uint32_t *)
69894362682SGerd Hoffmann ((uint8_t *)fbptr + vnc_server_fb_stride(vd));
69994362682SGerd Hoffmann }
70094362682SGerd Hoffmann
70194362682SGerd Hoffmann *color = (uint32_t)c;
70294362682SGerd Hoffmann return true;
70394362682SGerd Hoffmann }
704245f7b51SCorentin Chary
check_solid_tile(VncState * vs,int x,int y,int w,int h,uint32_t * color,bool samecolor)705245f7b51SCorentin Chary static bool check_solid_tile(VncState *vs, int x, int y, int w, int h,
706245f7b51SCorentin Chary uint32_t* color, bool samecolor)
707245f7b51SCorentin Chary {
708d9d2663cSAlex Bennée QEMU_BUILD_BUG_ON(VNC_SERVER_FB_BYTES != 4);
709245f7b51SCorentin Chary return check_solid_tile32(vs, x, y, w, h, color, samecolor);
710245f7b51SCorentin Chary }
711245f7b51SCorentin Chary
find_best_solid_area(VncState * vs,int x,int y,int w,int h,uint32_t color,int * w_ptr,int * h_ptr)712245f7b51SCorentin Chary static void find_best_solid_area(VncState *vs, int x, int y, int w, int h,
713245f7b51SCorentin Chary uint32_t color, int *w_ptr, int *h_ptr)
714245f7b51SCorentin Chary {
715245f7b51SCorentin Chary int dx, dy, dw, dh;
716245f7b51SCorentin Chary int w_prev;
717245f7b51SCorentin Chary int w_best = 0, h_best = 0;
718245f7b51SCorentin Chary
719245f7b51SCorentin Chary w_prev = w;
720245f7b51SCorentin Chary
721245f7b51SCorentin Chary for (dy = y; dy < y + h; dy += VNC_TIGHT_MAX_SPLIT_TILE_SIZE) {
722245f7b51SCorentin Chary
723245f7b51SCorentin Chary dh = MIN(VNC_TIGHT_MAX_SPLIT_TILE_SIZE, y + h - dy);
724245f7b51SCorentin Chary dw = MIN(VNC_TIGHT_MAX_SPLIT_TILE_SIZE, w_prev);
725245f7b51SCorentin Chary
726245f7b51SCorentin Chary if (!check_solid_tile(vs, x, dy, dw, dh, &color, true)) {
727245f7b51SCorentin Chary break;
728245f7b51SCorentin Chary }
729245f7b51SCorentin Chary
730245f7b51SCorentin Chary for (dx = x + dw; dx < x + w_prev;) {
731245f7b51SCorentin Chary dw = MIN(VNC_TIGHT_MAX_SPLIT_TILE_SIZE, x + w_prev - dx);
732245f7b51SCorentin Chary
733245f7b51SCorentin Chary if (!check_solid_tile(vs, dx, dy, dw, dh, &color, true)) {
734245f7b51SCorentin Chary break;
735245f7b51SCorentin Chary }
736245f7b51SCorentin Chary dx += dw;
737245f7b51SCorentin Chary }
738245f7b51SCorentin Chary
739245f7b51SCorentin Chary w_prev = dx - x;
740245f7b51SCorentin Chary if (w_prev * (dy + dh - y) > w_best * h_best) {
741245f7b51SCorentin Chary w_best = w_prev;
742245f7b51SCorentin Chary h_best = dy + dh - y;
743245f7b51SCorentin Chary }
744245f7b51SCorentin Chary }
745245f7b51SCorentin Chary
746245f7b51SCorentin Chary *w_ptr = w_best;
747245f7b51SCorentin Chary *h_ptr = h_best;
748245f7b51SCorentin Chary }
749245f7b51SCorentin Chary
extend_solid_area(VncState * vs,int x,int y,int w,int h,uint32_t color,int * x_ptr,int * y_ptr,int * w_ptr,int * h_ptr)750245f7b51SCorentin Chary static void extend_solid_area(VncState *vs, int x, int y, int w, int h,
751245f7b51SCorentin Chary uint32_t color, int *x_ptr, int *y_ptr,
752245f7b51SCorentin Chary int *w_ptr, int *h_ptr)
753245f7b51SCorentin Chary {
754245f7b51SCorentin Chary int cx, cy;
755245f7b51SCorentin Chary
756245f7b51SCorentin Chary /* Try to extend the area upwards. */
757245f7b51SCorentin Chary for ( cy = *y_ptr - 1;
758245f7b51SCorentin Chary cy >= y && check_solid_tile(vs, *x_ptr, cy, *w_ptr, 1, &color, true);
759245f7b51SCorentin Chary cy-- );
760245f7b51SCorentin Chary *h_ptr += *y_ptr - (cy + 1);
761245f7b51SCorentin Chary *y_ptr = cy + 1;
762245f7b51SCorentin Chary
763245f7b51SCorentin Chary /* ... downwards. */
764245f7b51SCorentin Chary for ( cy = *y_ptr + *h_ptr;
765245f7b51SCorentin Chary cy < y + h &&
766245f7b51SCorentin Chary check_solid_tile(vs, *x_ptr, cy, *w_ptr, 1, &color, true);
767245f7b51SCorentin Chary cy++ );
768245f7b51SCorentin Chary *h_ptr += cy - (*y_ptr + *h_ptr);
769245f7b51SCorentin Chary
770245f7b51SCorentin Chary /* ... to the left. */
771245f7b51SCorentin Chary for ( cx = *x_ptr - 1;
772245f7b51SCorentin Chary cx >= x && check_solid_tile(vs, cx, *y_ptr, 1, *h_ptr, &color, true);
773245f7b51SCorentin Chary cx-- );
774245f7b51SCorentin Chary *w_ptr += *x_ptr - (cx + 1);
775245f7b51SCorentin Chary *x_ptr = cx + 1;
776245f7b51SCorentin Chary
777245f7b51SCorentin Chary /* ... to the right. */
778245f7b51SCorentin Chary for ( cx = *x_ptr + *w_ptr;
779245f7b51SCorentin Chary cx < x + w &&
780245f7b51SCorentin Chary check_solid_tile(vs, cx, *y_ptr, 1, *h_ptr, &color, true);
781245f7b51SCorentin Chary cx++ );
782245f7b51SCorentin Chary *w_ptr += cx - (*x_ptr + *w_ptr);
783245f7b51SCorentin Chary }
784245f7b51SCorentin Chary
tight_init_stream(VncState * vs,int stream_id,int level,int strategy)785245f7b51SCorentin Chary static int tight_init_stream(VncState *vs, int stream_id,
786245f7b51SCorentin Chary int level, int strategy)
787245f7b51SCorentin Chary {
7886bf21f3dSLi Qiang z_streamp zstream = &vs->tight->stream[stream_id];
789245f7b51SCorentin Chary
790245f7b51SCorentin Chary if (zstream->opaque == NULL) {
791245f7b51SCorentin Chary int err;
792245f7b51SCorentin Chary
793245f7b51SCorentin Chary VNC_DEBUG("VNC: TIGHT: initializing zlib stream %d\n", stream_id);
794245f7b51SCorentin Chary VNC_DEBUG("VNC: TIGHT: opaque = %p | vs = %p\n", zstream->opaque, vs);
795245f7b51SCorentin Chary zstream->zalloc = vnc_zlib_zalloc;
796245f7b51SCorentin Chary zstream->zfree = vnc_zlib_zfree;
797245f7b51SCorentin Chary
798245f7b51SCorentin Chary err = deflateInit2(zstream, level, Z_DEFLATED, MAX_WBITS,
799245f7b51SCorentin Chary MAX_MEM_LEVEL, strategy);
800245f7b51SCorentin Chary
801245f7b51SCorentin Chary if (err != Z_OK) {
802245f7b51SCorentin Chary fprintf(stderr, "VNC: error initializing zlib\n");
803245f7b51SCorentin Chary return -1;
804245f7b51SCorentin Chary }
805245f7b51SCorentin Chary
8066bf21f3dSLi Qiang vs->tight->levels[stream_id] = level;
807245f7b51SCorentin Chary zstream->opaque = vs;
808245f7b51SCorentin Chary }
809245f7b51SCorentin Chary
8106bf21f3dSLi Qiang if (vs->tight->levels[stream_id] != level) {
811245f7b51SCorentin Chary if (deflateParams(zstream, level, strategy) != Z_OK) {
812245f7b51SCorentin Chary return -1;
813245f7b51SCorentin Chary }
8146bf21f3dSLi Qiang vs->tight->levels[stream_id] = level;
815245f7b51SCorentin Chary }
816245f7b51SCorentin Chary return 0;
817245f7b51SCorentin Chary }
818245f7b51SCorentin Chary
tight_send_compact_size(VncState * vs,size_t len)819245f7b51SCorentin Chary static void tight_send_compact_size(VncState *vs, size_t len)
820245f7b51SCorentin Chary {
821245f7b51SCorentin Chary int lpc = 0;
822245f7b51SCorentin Chary int bytes = 0;
823245f7b51SCorentin Chary char buf[3] = {0, 0, 0};
824245f7b51SCorentin Chary
825245f7b51SCorentin Chary buf[bytes++] = len & 0x7F;
826245f7b51SCorentin Chary if (len > 0x7F) {
827245f7b51SCorentin Chary buf[bytes-1] |= 0x80;
828245f7b51SCorentin Chary buf[bytes++] = (len >> 7) & 0x7F;
829245f7b51SCorentin Chary if (len > 0x3FFF) {
830245f7b51SCorentin Chary buf[bytes-1] |= 0x80;
831245f7b51SCorentin Chary buf[bytes++] = (len >> 14) & 0xFF;
832245f7b51SCorentin Chary }
833245f7b51SCorentin Chary }
834245f7b51SCorentin Chary for (lpc = 0; lpc < bytes; lpc++) {
835245f7b51SCorentin Chary vnc_write_u8(vs, buf[lpc]);
836245f7b51SCorentin Chary }
837245f7b51SCorentin Chary }
838245f7b51SCorentin Chary
tight_compress_data(VncState * vs,int stream_id,size_t bytes,int level,int strategy)839245f7b51SCorentin Chary static int tight_compress_data(VncState *vs, int stream_id, size_t bytes,
840245f7b51SCorentin Chary int level, int strategy)
841245f7b51SCorentin Chary {
8426bf21f3dSLi Qiang z_streamp zstream = &vs->tight->stream[stream_id];
843245f7b51SCorentin Chary int previous_out;
844245f7b51SCorentin Chary
845245f7b51SCorentin Chary if (bytes < VNC_TIGHT_MIN_TO_COMPRESS) {
8466bf21f3dSLi Qiang vnc_write(vs, vs->tight->tight.buffer, vs->tight->tight.offset);
847245f7b51SCorentin Chary return bytes;
848245f7b51SCorentin Chary }
849245f7b51SCorentin Chary
850245f7b51SCorentin Chary if (tight_init_stream(vs, stream_id, level, strategy)) {
851245f7b51SCorentin Chary return -1;
852245f7b51SCorentin Chary }
853245f7b51SCorentin Chary
854245f7b51SCorentin Chary /* reserve memory in output buffer */
8556bf21f3dSLi Qiang buffer_reserve(&vs->tight->zlib, bytes + 64);
856245f7b51SCorentin Chary
857245f7b51SCorentin Chary /* set pointers */
8586bf21f3dSLi Qiang zstream->next_in = vs->tight->tight.buffer;
8596bf21f3dSLi Qiang zstream->avail_in = vs->tight->tight.offset;
8606bf21f3dSLi Qiang zstream->next_out = vs->tight->zlib.buffer + vs->tight->zlib.offset;
8616bf21f3dSLi Qiang zstream->avail_out = vs->tight->zlib.capacity - vs->tight->zlib.offset;
8622caa9e9dSMichael Tokarev previous_out = zstream->avail_out;
863245f7b51SCorentin Chary zstream->data_type = Z_BINARY;
864245f7b51SCorentin Chary
865245f7b51SCorentin Chary /* start encoding */
866245f7b51SCorentin Chary if (deflate(zstream, Z_SYNC_FLUSH) != Z_OK) {
867245f7b51SCorentin Chary fprintf(stderr, "VNC: error during tight compression\n");
868245f7b51SCorentin Chary return -1;
869245f7b51SCorentin Chary }
870245f7b51SCorentin Chary
8716bf21f3dSLi Qiang vs->tight->zlib.offset = vs->tight->zlib.capacity - zstream->avail_out;
8722caa9e9dSMichael Tokarev /* ...how much data has actually been produced by deflate() */
8732caa9e9dSMichael Tokarev bytes = previous_out - zstream->avail_out;
874245f7b51SCorentin Chary
875245f7b51SCorentin Chary tight_send_compact_size(vs, bytes);
8766bf21f3dSLi Qiang vnc_write(vs, vs->tight->zlib.buffer, bytes);
877245f7b51SCorentin Chary
8786bf21f3dSLi Qiang buffer_reset(&vs->tight->zlib);
879245f7b51SCorentin Chary
880245f7b51SCorentin Chary return bytes;
881245f7b51SCorentin Chary }
882245f7b51SCorentin Chary
883245f7b51SCorentin Chary /*
884245f7b51SCorentin Chary * Subencoding implementations.
885245f7b51SCorentin Chary */
tight_pack24(VncState * vs,uint8_t * buf,size_t count,size_t * ret)886245f7b51SCorentin Chary static void tight_pack24(VncState *vs, uint8_t *buf, size_t count, size_t *ret)
887245f7b51SCorentin Chary {
88818e845f7SLi Qiang uint8_t *buf8;
889245f7b51SCorentin Chary uint32_t pix;
890245f7b51SCorentin Chary int rshift, gshift, bshift;
891245f7b51SCorentin Chary
89218e845f7SLi Qiang buf8 = buf;
893245f7b51SCorentin Chary
89477bfcf28SBenjamin Herrenschmidt if (1 /* FIXME */) {
8959f64916dSGerd Hoffmann rshift = vs->client_pf.rshift;
8969f64916dSGerd Hoffmann gshift = vs->client_pf.gshift;
8979f64916dSGerd Hoffmann bshift = vs->client_pf.bshift;
898245f7b51SCorentin Chary } else {
8999f64916dSGerd Hoffmann rshift = 24 - vs->client_pf.rshift;
9009f64916dSGerd Hoffmann gshift = 24 - vs->client_pf.gshift;
9019f64916dSGerd Hoffmann bshift = 24 - vs->client_pf.bshift;
902245f7b51SCorentin Chary }
903245f7b51SCorentin Chary
904245f7b51SCorentin Chary if (ret) {
905245f7b51SCorentin Chary *ret = count * 3;
906245f7b51SCorentin Chary }
907245f7b51SCorentin Chary
908245f7b51SCorentin Chary while (count--) {
90918e845f7SLi Qiang pix = ldl_he_p(buf8);
910245f7b51SCorentin Chary *buf++ = (char)(pix >> rshift);
911245f7b51SCorentin Chary *buf++ = (char)(pix >> gshift);
912245f7b51SCorentin Chary *buf++ = (char)(pix >> bshift);
91318e845f7SLi Qiang buf8 += 4;
914245f7b51SCorentin Chary }
915245f7b51SCorentin Chary }
916245f7b51SCorentin Chary
send_full_color_rect(VncState * vs,int x,int y,int w,int h)917efe556adSCorentin Chary static int send_full_color_rect(VncState *vs, int x, int y, int w, int h)
918245f7b51SCorentin Chary {
919245f7b51SCorentin Chary int stream = 0;
9202116eff9SJes Sorensen ssize_t bytes;
921245f7b51SCorentin Chary
92295f8510eSKshitij Suri #ifdef CONFIG_PNG
923efe556adSCorentin Chary if (tight_can_send_png_rect(vs, w, h)) {
924efe556adSCorentin Chary return send_png_rect(vs, x, y, w, h, NULL);
925efe556adSCorentin Chary }
926efe556adSCorentin Chary #endif
927efe556adSCorentin Chary
928245f7b51SCorentin Chary vnc_write_u8(vs, stream << 4); /* no flushing, no filter */
929245f7b51SCorentin Chary
9306bf21f3dSLi Qiang if (vs->tight->pixel24) {
9316bf21f3dSLi Qiang tight_pack24(vs, vs->tight->tight.buffer, w * h,
9326bf21f3dSLi Qiang &vs->tight->tight.offset);
933245f7b51SCorentin Chary bytes = 3;
934245f7b51SCorentin Chary } else {
9359f64916dSGerd Hoffmann bytes = vs->client_pf.bytes_per_pixel;
936245f7b51SCorentin Chary }
937245f7b51SCorentin Chary
938245f7b51SCorentin Chary bytes = tight_compress_data(vs, stream, w * h * bytes,
9396bf21f3dSLi Qiang tight_conf[vs->tight->compression].raw_zlib_level,
940245f7b51SCorentin Chary Z_DEFAULT_STRATEGY);
941245f7b51SCorentin Chary
942245f7b51SCorentin Chary return (bytes >= 0);
943245f7b51SCorentin Chary }
944245f7b51SCorentin Chary
send_solid_rect(VncState * vs)945245f7b51SCorentin Chary static int send_solid_rect(VncState *vs)
946245f7b51SCorentin Chary {
947245f7b51SCorentin Chary size_t bytes;
948245f7b51SCorentin Chary
949245f7b51SCorentin Chary vnc_write_u8(vs, VNC_TIGHT_FILL << 4); /* no flushing, no filter */
950245f7b51SCorentin Chary
9516bf21f3dSLi Qiang if (vs->tight->pixel24) {
9526bf21f3dSLi Qiang tight_pack24(vs, vs->tight->tight.buffer, 1, &vs->tight->tight.offset);
953245f7b51SCorentin Chary bytes = 3;
954245f7b51SCorentin Chary } else {
9559f64916dSGerd Hoffmann bytes = vs->client_pf.bytes_per_pixel;
956245f7b51SCorentin Chary }
957245f7b51SCorentin Chary
9586bf21f3dSLi Qiang vnc_write(vs, vs->tight->tight.buffer, bytes);
959245f7b51SCorentin Chary return 1;
960245f7b51SCorentin Chary }
961245f7b51SCorentin Chary
send_mono_rect(VncState * vs,int x,int y,int w,int h,uint32_t bg,uint32_t fg)962efe556adSCorentin Chary static int send_mono_rect(VncState *vs, int x, int y,
963efe556adSCorentin Chary int w, int h, uint32_t bg, uint32_t fg)
964245f7b51SCorentin Chary {
9652116eff9SJes Sorensen ssize_t bytes;
966245f7b51SCorentin Chary int stream = 1;
9676bf21f3dSLi Qiang int level = tight_conf[vs->tight->compression].mono_zlib_level;
968245f7b51SCorentin Chary
96995f8510eSKshitij Suri #ifdef CONFIG_PNG
970efe556adSCorentin Chary if (tight_can_send_png_rect(vs, w, h)) {
971efe556adSCorentin Chary int ret;
9729f64916dSGerd Hoffmann int bpp = vs->client_pf.bytes_per_pixel * 8;
9735136a052SCorentin Chary VncPalette *palette = palette_new(2, bpp);
974efe556adSCorentin Chary
9755136a052SCorentin Chary palette_put(palette, bg);
9765136a052SCorentin Chary palette_put(palette, fg);
977efe556adSCorentin Chary ret = send_png_rect(vs, x, y, w, h, palette);
9785136a052SCorentin Chary palette_destroy(palette);
979efe556adSCorentin Chary return ret;
980efe556adSCorentin Chary }
981efe556adSCorentin Chary #endif
982efe556adSCorentin Chary
983949ed4c2SPhilippe Mathieu-Daudé bytes = DIV_ROUND_UP(w, 8) * h;
984245f7b51SCorentin Chary
985245f7b51SCorentin Chary vnc_write_u8(vs, (stream | VNC_TIGHT_EXPLICIT_FILTER) << 4);
986245f7b51SCorentin Chary vnc_write_u8(vs, VNC_TIGHT_FILTER_PALETTE);
987245f7b51SCorentin Chary vnc_write_u8(vs, 1);
988245f7b51SCorentin Chary
9899f64916dSGerd Hoffmann switch (vs->client_pf.bytes_per_pixel) {
990245f7b51SCorentin Chary case 4:
991245f7b51SCorentin Chary {
992245f7b51SCorentin Chary uint32_t buf[2] = {bg, fg};
993245f7b51SCorentin Chary size_t ret = sizeof (buf);
994245f7b51SCorentin Chary
9956bf21f3dSLi Qiang if (vs->tight->pixel24) {
996245f7b51SCorentin Chary tight_pack24(vs, (unsigned char*)buf, 2, &ret);
997245f7b51SCorentin Chary }
998245f7b51SCorentin Chary vnc_write(vs, buf, ret);
999245f7b51SCorentin Chary
10006bf21f3dSLi Qiang tight_encode_mono_rect32(vs->tight->tight.buffer, w, h, bg, fg);
1001245f7b51SCorentin Chary break;
1002245f7b51SCorentin Chary }
1003245f7b51SCorentin Chary case 2:
1004245f7b51SCorentin Chary vnc_write(vs, &bg, 2);
1005245f7b51SCorentin Chary vnc_write(vs, &fg, 2);
10066bf21f3dSLi Qiang tight_encode_mono_rect16(vs->tight->tight.buffer, w, h, bg, fg);
1007245f7b51SCorentin Chary break;
1008245f7b51SCorentin Chary default:
1009245f7b51SCorentin Chary vnc_write_u8(vs, bg);
1010245f7b51SCorentin Chary vnc_write_u8(vs, fg);
10116bf21f3dSLi Qiang tight_encode_mono_rect8(vs->tight->tight.buffer, w, h, bg, fg);
1012245f7b51SCorentin Chary break;
1013245f7b51SCorentin Chary }
10146bf21f3dSLi Qiang vs->tight->tight.offset = bytes;
1015245f7b51SCorentin Chary
1016245f7b51SCorentin Chary bytes = tight_compress_data(vs, stream, bytes, level, Z_DEFAULT_STRATEGY);
1017245f7b51SCorentin Chary return (bytes >= 0);
1018245f7b51SCorentin Chary }
1019245f7b51SCorentin Chary
1020245f7b51SCorentin Chary struct palette_cb_priv {
1021245f7b51SCorentin Chary VncState *vs;
1022245f7b51SCorentin Chary uint8_t *header;
102395f8510eSKshitij Suri #ifdef CONFIG_PNG
1024efe556adSCorentin Chary png_colorp png_palette;
1025efe556adSCorentin Chary #endif
1026245f7b51SCorentin Chary };
1027245f7b51SCorentin Chary
write_palette(int idx,uint32_t color,void * opaque)10285136a052SCorentin Chary static void write_palette(int idx, uint32_t color, void *opaque)
1029245f7b51SCorentin Chary {
1030245f7b51SCorentin Chary struct palette_cb_priv *priv = opaque;
1031245f7b51SCorentin Chary VncState *vs = priv->vs;
10329f64916dSGerd Hoffmann uint32_t bytes = vs->client_pf.bytes_per_pixel;
1033245f7b51SCorentin Chary
1034245f7b51SCorentin Chary if (bytes == 4) {
1035245f7b51SCorentin Chary ((uint32_t*)priv->header)[idx] = color;
1036245f7b51SCorentin Chary } else {
1037245f7b51SCorentin Chary ((uint16_t*)priv->header)[idx] = color;
1038245f7b51SCorentin Chary }
1039245f7b51SCorentin Chary }
1040245f7b51SCorentin Chary
send_gradient_rect(VncState * vs,int x,int y,int w,int h)1041efe556adSCorentin Chary static bool send_gradient_rect(VncState *vs, int x, int y, int w, int h)
1042245f7b51SCorentin Chary {
1043245f7b51SCorentin Chary int stream = 3;
10446bf21f3dSLi Qiang int level = tight_conf[vs->tight->compression].gradient_zlib_level;
10452116eff9SJes Sorensen ssize_t bytes;
1046245f7b51SCorentin Chary
10479f64916dSGerd Hoffmann if (vs->client_pf.bytes_per_pixel == 1) {
1048efe556adSCorentin Chary return send_full_color_rect(vs, x, y, w, h);
10499f64916dSGerd Hoffmann }
1050245f7b51SCorentin Chary
1051245f7b51SCorentin Chary vnc_write_u8(vs, (stream | VNC_TIGHT_EXPLICIT_FILTER) << 4);
1052245f7b51SCorentin Chary vnc_write_u8(vs, VNC_TIGHT_FILTER_GRADIENT);
1053245f7b51SCorentin Chary
10546bf21f3dSLi Qiang buffer_reserve(&vs->tight->gradient, w * 3 * sizeof(int));
1055245f7b51SCorentin Chary
10566bf21f3dSLi Qiang if (vs->tight->pixel24) {
10576bf21f3dSLi Qiang tight_filter_gradient24(vs, vs->tight->tight.buffer, w, h);
1058245f7b51SCorentin Chary bytes = 3;
10599f64916dSGerd Hoffmann } else if (vs->client_pf.bytes_per_pixel == 4) {
10606bf21f3dSLi Qiang tight_filter_gradient32(vs, (uint32_t *)vs->tight->tight.buffer, w, h);
1061245f7b51SCorentin Chary bytes = 4;
1062245f7b51SCorentin Chary } else {
10636bf21f3dSLi Qiang tight_filter_gradient16(vs, (uint16_t *)vs->tight->tight.buffer, w, h);
1064245f7b51SCorentin Chary bytes = 2;
1065245f7b51SCorentin Chary }
1066245f7b51SCorentin Chary
10676bf21f3dSLi Qiang buffer_reset(&vs->tight->gradient);
1068245f7b51SCorentin Chary
1069245f7b51SCorentin Chary bytes = w * h * bytes;
10706bf21f3dSLi Qiang vs->tight->tight.offset = bytes;
1071245f7b51SCorentin Chary
1072245f7b51SCorentin Chary bytes = tight_compress_data(vs, stream, bytes,
1073245f7b51SCorentin Chary level, Z_FILTERED);
1074245f7b51SCorentin Chary return (bytes >= 0);
1075245f7b51SCorentin Chary }
1076245f7b51SCorentin Chary
send_palette_rect(VncState * vs,int x,int y,int w,int h,VncPalette * palette)1077efe556adSCorentin Chary static int send_palette_rect(VncState *vs, int x, int y,
10785136a052SCorentin Chary int w, int h, VncPalette *palette)
1079245f7b51SCorentin Chary {
1080245f7b51SCorentin Chary int stream = 2;
10816bf21f3dSLi Qiang int level = tight_conf[vs->tight->compression].idx_zlib_level;
1082245f7b51SCorentin Chary int colors;
10832116eff9SJes Sorensen ssize_t bytes;
1084245f7b51SCorentin Chary
108595f8510eSKshitij Suri #ifdef CONFIG_PNG
1086efe556adSCorentin Chary if (tight_can_send_png_rect(vs, w, h)) {
1087efe556adSCorentin Chary return send_png_rect(vs, x, y, w, h, palette);
1088efe556adSCorentin Chary }
1089efe556adSCorentin Chary #endif
1090efe556adSCorentin Chary
10915136a052SCorentin Chary colors = palette_size(palette);
1092245f7b51SCorentin Chary
1093245f7b51SCorentin Chary vnc_write_u8(vs, (stream | VNC_TIGHT_EXPLICIT_FILTER) << 4);
1094245f7b51SCorentin Chary vnc_write_u8(vs, VNC_TIGHT_FILTER_PALETTE);
1095245f7b51SCorentin Chary vnc_write_u8(vs, colors - 1);
1096245f7b51SCorentin Chary
10979f64916dSGerd Hoffmann switch (vs->client_pf.bytes_per_pixel) {
1098245f7b51SCorentin Chary case 4:
1099245f7b51SCorentin Chary {
1100*dd0439e1SPhilippe Mathieu-Daudé size_t old_offset, offset, palette_sz = palette_size(palette);
1101*dd0439e1SPhilippe Mathieu-Daudé g_autofree uint32_t *header = g_new(uint32_t, palette_sz);
1102245f7b51SCorentin Chary struct palette_cb_priv priv = { vs, (uint8_t *)header };
1103245f7b51SCorentin Chary
1104245f7b51SCorentin Chary old_offset = vs->output.offset;
11055136a052SCorentin Chary palette_iter(palette, write_palette, &priv);
1106*dd0439e1SPhilippe Mathieu-Daudé vnc_write(vs, header, palette_sz * sizeof(uint32_t));
1107245f7b51SCorentin Chary
11086bf21f3dSLi Qiang if (vs->tight->pixel24) {
1109245f7b51SCorentin Chary tight_pack24(vs, vs->output.buffer + old_offset, colors, &offset);
1110245f7b51SCorentin Chary vs->output.offset = old_offset + offset;
1111245f7b51SCorentin Chary }
1112245f7b51SCorentin Chary
11136bf21f3dSLi Qiang tight_encode_indexed_rect32(vs->tight->tight.buffer, w * h, palette);
1114245f7b51SCorentin Chary break;
1115245f7b51SCorentin Chary }
1116245f7b51SCorentin Chary case 2:
1117245f7b51SCorentin Chary {
1118*dd0439e1SPhilippe Mathieu-Daudé size_t palette_sz = palette_size(palette);
1119*dd0439e1SPhilippe Mathieu-Daudé g_autofree uint16_t *header = g_new(uint16_t, palette_sz);
1120245f7b51SCorentin Chary struct palette_cb_priv priv = { vs, (uint8_t *)header };
1121245f7b51SCorentin Chary
11225136a052SCorentin Chary palette_iter(palette, write_palette, &priv);
1123*dd0439e1SPhilippe Mathieu-Daudé vnc_write(vs, header, palette_sz * sizeof(uint16_t));
11246bf21f3dSLi Qiang tight_encode_indexed_rect16(vs->tight->tight.buffer, w * h, palette);
1125245f7b51SCorentin Chary break;
1126245f7b51SCorentin Chary }
1127245f7b51SCorentin Chary default:
1128245f7b51SCorentin Chary return -1; /* No palette for 8bits colors */
1129245f7b51SCorentin Chary }
1130245f7b51SCorentin Chary bytes = w * h;
11316bf21f3dSLi Qiang vs->tight->tight.offset = bytes;
1132245f7b51SCorentin Chary
1133245f7b51SCorentin Chary bytes = tight_compress_data(vs, stream, bytes,
1134245f7b51SCorentin Chary level, Z_DEFAULT_STRATEGY);
1135245f7b51SCorentin Chary return (bytes >= 0);
1136245f7b51SCorentin Chary }
1137245f7b51SCorentin Chary
1138245f7b51SCorentin Chary /*
1139efe556adSCorentin Chary * JPEG compression stuff.
1140efe556adSCorentin Chary */
1141efe556adSCorentin Chary #ifdef CONFIG_VNC_JPEG
1142efe556adSCorentin Chary /*
1143245f7b51SCorentin Chary * Destination manager implementation for JPEG library.
1144245f7b51SCorentin Chary */
1145245f7b51SCorentin Chary
1146245f7b51SCorentin Chary /* This is called once per encoding */
jpeg_init_destination(j_compress_ptr cinfo)1147245f7b51SCorentin Chary static void jpeg_init_destination(j_compress_ptr cinfo)
1148245f7b51SCorentin Chary {
1149245f7b51SCorentin Chary VncState *vs = cinfo->client_data;
11506bf21f3dSLi Qiang Buffer *buffer = &vs->tight->jpeg;
1151245f7b51SCorentin Chary
1152245f7b51SCorentin Chary cinfo->dest->next_output_byte = (JOCTET *)buffer->buffer + buffer->offset;
1153245f7b51SCorentin Chary cinfo->dest->free_in_buffer = (size_t)(buffer->capacity - buffer->offset);
1154245f7b51SCorentin Chary }
1155245f7b51SCorentin Chary
1156245f7b51SCorentin Chary /* This is called when we ran out of buffer (shouldn't happen!) */
jpeg_empty_output_buffer(j_compress_ptr cinfo)1157245f7b51SCorentin Chary static boolean jpeg_empty_output_buffer(j_compress_ptr cinfo)
1158245f7b51SCorentin Chary {
1159245f7b51SCorentin Chary VncState *vs = cinfo->client_data;
11606bf21f3dSLi Qiang Buffer *buffer = &vs->tight->jpeg;
1161245f7b51SCorentin Chary
1162245f7b51SCorentin Chary buffer->offset = buffer->capacity;
1163245f7b51SCorentin Chary buffer_reserve(buffer, 2048);
1164245f7b51SCorentin Chary jpeg_init_destination(cinfo);
1165245f7b51SCorentin Chary return TRUE;
1166245f7b51SCorentin Chary }
1167245f7b51SCorentin Chary
1168245f7b51SCorentin Chary /* This is called when we are done processing data */
jpeg_term_destination(j_compress_ptr cinfo)1169245f7b51SCorentin Chary static void jpeg_term_destination(j_compress_ptr cinfo)
1170245f7b51SCorentin Chary {
1171245f7b51SCorentin Chary VncState *vs = cinfo->client_data;
11726bf21f3dSLi Qiang Buffer *buffer = &vs->tight->jpeg;
1173245f7b51SCorentin Chary
1174245f7b51SCorentin Chary buffer->offset = buffer->capacity - cinfo->dest->free_in_buffer;
1175245f7b51SCorentin Chary }
1176245f7b51SCorentin Chary
send_jpeg_rect(VncState * vs,int x,int y,int w,int h,int quality)1177245f7b51SCorentin Chary static int send_jpeg_rect(VncState *vs, int x, int y, int w, int h, int quality)
1178245f7b51SCorentin Chary {
1179245f7b51SCorentin Chary struct jpeg_compress_struct cinfo;
1180245f7b51SCorentin Chary struct jpeg_error_mgr jerr;
1181245f7b51SCorentin Chary struct jpeg_destination_mgr manager;
118247683d66SGerd Hoffmann pixman_image_t *linebuf;
1183245f7b51SCorentin Chary JSAMPROW row[1];
1184245f7b51SCorentin Chary uint8_t *buf;
1185245f7b51SCorentin Chary int dy;
1186245f7b51SCorentin Chary
1187d39fa6d8SGerd Hoffmann if (surface_bytes_per_pixel(vs->vd->ds) == 1) {
1188efe556adSCorentin Chary return send_full_color_rect(vs, x, y, w, h);
1189d39fa6d8SGerd Hoffmann }
1190245f7b51SCorentin Chary
11916bf21f3dSLi Qiang buffer_reserve(&vs->tight->jpeg, 2048);
1192245f7b51SCorentin Chary
1193245f7b51SCorentin Chary cinfo.err = jpeg_std_error(&jerr);
1194245f7b51SCorentin Chary jpeg_create_compress(&cinfo);
1195245f7b51SCorentin Chary
1196245f7b51SCorentin Chary cinfo.client_data = vs;
1197245f7b51SCorentin Chary cinfo.image_width = w;
1198245f7b51SCorentin Chary cinfo.image_height = h;
1199245f7b51SCorentin Chary cinfo.input_components = 3;
1200245f7b51SCorentin Chary cinfo.in_color_space = JCS_RGB;
1201245f7b51SCorentin Chary
1202245f7b51SCorentin Chary jpeg_set_defaults(&cinfo);
1203245f7b51SCorentin Chary jpeg_set_quality(&cinfo, quality, true);
1204245f7b51SCorentin Chary
1205245f7b51SCorentin Chary manager.init_destination = jpeg_init_destination;
1206245f7b51SCorentin Chary manager.empty_output_buffer = jpeg_empty_output_buffer;
1207245f7b51SCorentin Chary manager.term_destination = jpeg_term_destination;
1208245f7b51SCorentin Chary cinfo.dest = &manager;
1209245f7b51SCorentin Chary
1210245f7b51SCorentin Chary jpeg_start_compress(&cinfo, true);
1211245f7b51SCorentin Chary
121247683d66SGerd Hoffmann linebuf = qemu_pixman_linebuf_create(PIXMAN_BE_r8g8b8, w);
121347683d66SGerd Hoffmann buf = (uint8_t *)pixman_image_get_data(linebuf);
1214d9c18c24SCorentin Chary row[0] = buf;
1215245f7b51SCorentin Chary for (dy = 0; dy < h; dy++) {
1216bc210eb1SGerd Hoffmann qemu_pixman_linebuf_fill(linebuf, vs->vd->server, w, x, y + dy);
1217245f7b51SCorentin Chary jpeg_write_scanlines(&cinfo, row, 1);
1218245f7b51SCorentin Chary }
121947683d66SGerd Hoffmann qemu_pixman_image_unref(linebuf);
1220245f7b51SCorentin Chary
1221245f7b51SCorentin Chary jpeg_finish_compress(&cinfo);
1222245f7b51SCorentin Chary jpeg_destroy_compress(&cinfo);
1223245f7b51SCorentin Chary
1224245f7b51SCorentin Chary vnc_write_u8(vs, VNC_TIGHT_JPEG << 4);
1225245f7b51SCorentin Chary
12266bf21f3dSLi Qiang tight_send_compact_size(vs, vs->tight->jpeg.offset);
12276bf21f3dSLi Qiang vnc_write(vs, vs->tight->jpeg.buffer, vs->tight->jpeg.offset);
12286bf21f3dSLi Qiang buffer_reset(&vs->tight->jpeg);
1229245f7b51SCorentin Chary
1230245f7b51SCorentin Chary return 1;
1231245f7b51SCorentin Chary }
1232245f7b51SCorentin Chary #endif /* CONFIG_VNC_JPEG */
1233245f7b51SCorentin Chary
1234efe556adSCorentin Chary /*
1235efe556adSCorentin Chary * PNG compression stuff.
1236efe556adSCorentin Chary */
123795f8510eSKshitij Suri #ifdef CONFIG_PNG
write_png_palette(int idx,uint32_t pix,void * opaque)12385136a052SCorentin Chary static void write_png_palette(int idx, uint32_t pix, void *opaque)
1239efe556adSCorentin Chary {
1240efe556adSCorentin Chary struct palette_cb_priv *priv = opaque;
1241efe556adSCorentin Chary VncState *vs = priv->vs;
1242efe556adSCorentin Chary png_colorp color = &priv->png_palette[idx];
1243efe556adSCorentin Chary
12446bf21f3dSLi Qiang if (vs->tight->pixel24)
1245efe556adSCorentin Chary {
12469f64916dSGerd Hoffmann color->red = (pix >> vs->client_pf.rshift) & vs->client_pf.rmax;
12479f64916dSGerd Hoffmann color->green = (pix >> vs->client_pf.gshift) & vs->client_pf.gmax;
12489f64916dSGerd Hoffmann color->blue = (pix >> vs->client_pf.bshift) & vs->client_pf.bmax;
1249efe556adSCorentin Chary }
1250efe556adSCorentin Chary else
1251efe556adSCorentin Chary {
1252efe556adSCorentin Chary int red, green, blue;
1253efe556adSCorentin Chary
12549f64916dSGerd Hoffmann red = (pix >> vs->client_pf.rshift) & vs->client_pf.rmax;
12559f64916dSGerd Hoffmann green = (pix >> vs->client_pf.gshift) & vs->client_pf.gmax;
12569f64916dSGerd Hoffmann blue = (pix >> vs->client_pf.bshift) & vs->client_pf.bmax;
12579f64916dSGerd Hoffmann color->red = ((red * 255 + vs->client_pf.rmax / 2) /
12589f64916dSGerd Hoffmann vs->client_pf.rmax);
12599f64916dSGerd Hoffmann color->green = ((green * 255 + vs->client_pf.gmax / 2) /
12609f64916dSGerd Hoffmann vs->client_pf.gmax);
12619f64916dSGerd Hoffmann color->blue = ((blue * 255 + vs->client_pf.bmax / 2) /
12629f64916dSGerd Hoffmann vs->client_pf.bmax);
1263efe556adSCorentin Chary }
1264efe556adSCorentin Chary }
1265efe556adSCorentin Chary
png_write_data(png_structp png_ptr,png_bytep data,png_size_t length)1266efe556adSCorentin Chary static void png_write_data(png_structp png_ptr, png_bytep data,
1267efe556adSCorentin Chary png_size_t length)
1268efe556adSCorentin Chary {
1269efe556adSCorentin Chary VncState *vs = png_get_io_ptr(png_ptr);
1270efe556adSCorentin Chary
12716bf21f3dSLi Qiang buffer_reserve(&vs->tight->png, vs->tight->png.offset + length);
12726bf21f3dSLi Qiang memcpy(vs->tight->png.buffer + vs->tight->png.offset, data, length);
1273efe556adSCorentin Chary
12746bf21f3dSLi Qiang vs->tight->png.offset += length;
1275efe556adSCorentin Chary }
1276efe556adSCorentin Chary
png_flush_data(png_structp png_ptr)1277efe556adSCorentin Chary static void png_flush_data(png_structp png_ptr)
1278efe556adSCorentin Chary {
1279efe556adSCorentin Chary }
1280efe556adSCorentin Chary
vnc_png_malloc(png_structp png_ptr,png_size_t size)1281efe556adSCorentin Chary static void *vnc_png_malloc(png_structp png_ptr, png_size_t size)
1282efe556adSCorentin Chary {
12837267c094SAnthony Liguori return g_malloc(size);
1284efe556adSCorentin Chary }
1285efe556adSCorentin Chary
vnc_png_free(png_structp png_ptr,png_voidp ptr)1286efe556adSCorentin Chary static void vnc_png_free(png_structp png_ptr, png_voidp ptr)
1287efe556adSCorentin Chary {
12887267c094SAnthony Liguori g_free(ptr);
1289efe556adSCorentin Chary }
1290efe556adSCorentin Chary
send_png_rect(VncState * vs,int x,int y,int w,int h,VncPalette * palette)1291efe556adSCorentin Chary static int send_png_rect(VncState *vs, int x, int y, int w, int h,
12925136a052SCorentin Chary VncPalette *palette)
1293efe556adSCorentin Chary {
1294efe556adSCorentin Chary png_byte color_type;
1295efe556adSCorentin Chary png_structp png_ptr;
1296efe556adSCorentin Chary png_infop info_ptr;
1297efe556adSCorentin Chary png_colorp png_palette = NULL;
129847683d66SGerd Hoffmann pixman_image_t *linebuf;
12996bf21f3dSLi Qiang int level = tight_png_conf[vs->tight->compression].png_zlib_level;
13006bf21f3dSLi Qiang int filters = tight_png_conf[vs->tight->compression].png_filters;
1301efe556adSCorentin Chary uint8_t *buf;
1302efe556adSCorentin Chary int dy;
1303efe556adSCorentin Chary
1304efe556adSCorentin Chary png_ptr = png_create_write_struct_2(PNG_LIBPNG_VER_STRING, NULL, NULL, NULL,
1305efe556adSCorentin Chary NULL, vnc_png_malloc, vnc_png_free);
1306efe556adSCorentin Chary
1307efe556adSCorentin Chary if (png_ptr == NULL)
1308efe556adSCorentin Chary return -1;
1309efe556adSCorentin Chary
1310efe556adSCorentin Chary info_ptr = png_create_info_struct(png_ptr);
1311efe556adSCorentin Chary
1312efe556adSCorentin Chary if (info_ptr == NULL) {
1313efe556adSCorentin Chary png_destroy_write_struct(&png_ptr, NULL);
1314efe556adSCorentin Chary return -1;
1315efe556adSCorentin Chary }
1316efe556adSCorentin Chary
1317efe556adSCorentin Chary png_set_write_fn(png_ptr, (void *) vs, png_write_data, png_flush_data);
1318efe556adSCorentin Chary png_set_compression_level(png_ptr, level);
13193941bf6fSCorentin Chary png_set_filter(png_ptr, PNG_FILTER_TYPE_DEFAULT, filters);
1320efe556adSCorentin Chary
1321efe556adSCorentin Chary if (palette) {
1322efe556adSCorentin Chary color_type = PNG_COLOR_TYPE_PALETTE;
1323efe556adSCorentin Chary } else {
1324efe556adSCorentin Chary color_type = PNG_COLOR_TYPE_RGB;
1325efe556adSCorentin Chary }
1326efe556adSCorentin Chary
1327efe556adSCorentin Chary png_set_IHDR(png_ptr, info_ptr, w, h,
1328efe556adSCorentin Chary 8, color_type, PNG_INTERLACE_NONE,
1329efe556adSCorentin Chary PNG_COMPRESSION_TYPE_DEFAULT, PNG_FILTER_TYPE_DEFAULT);
1330efe556adSCorentin Chary
1331efe556adSCorentin Chary if (color_type == PNG_COLOR_TYPE_PALETTE) {
1332efe556adSCorentin Chary struct palette_cb_priv priv;
1333efe556adSCorentin Chary
1334efe556adSCorentin Chary png_palette = png_malloc(png_ptr, sizeof(*png_palette) *
13355136a052SCorentin Chary palette_size(palette));
1336efe556adSCorentin Chary
1337efe556adSCorentin Chary priv.vs = vs;
1338efe556adSCorentin Chary priv.png_palette = png_palette;
13395136a052SCorentin Chary palette_iter(palette, write_png_palette, &priv);
1340efe556adSCorentin Chary
13415136a052SCorentin Chary png_set_PLTE(png_ptr, info_ptr, png_palette, palette_size(palette));
1342efe556adSCorentin Chary
13439f64916dSGerd Hoffmann if (vs->client_pf.bytes_per_pixel == 4) {
13446bf21f3dSLi Qiang tight_encode_indexed_rect32(vs->tight->tight.buffer, w * h,
13456bf21f3dSLi Qiang palette);
1346efe556adSCorentin Chary } else {
13476bf21f3dSLi Qiang tight_encode_indexed_rect16(vs->tight->tight.buffer, w * h,
13486bf21f3dSLi Qiang palette);
1349efe556adSCorentin Chary }
1350efe556adSCorentin Chary }
1351efe556adSCorentin Chary
1352efe556adSCorentin Chary png_write_info(png_ptr, info_ptr);
1353efe556adSCorentin Chary
13546bf21f3dSLi Qiang buffer_reserve(&vs->tight->png, 2048);
135547683d66SGerd Hoffmann linebuf = qemu_pixman_linebuf_create(PIXMAN_BE_r8g8b8, w);
135647683d66SGerd Hoffmann buf = (uint8_t *)pixman_image_get_data(linebuf);
1357efe556adSCorentin Chary for (dy = 0; dy < h; dy++)
1358efe556adSCorentin Chary {
1359efe556adSCorentin Chary if (color_type == PNG_COLOR_TYPE_PALETTE) {
13606bf21f3dSLi Qiang memcpy(buf, vs->tight->tight.buffer + (dy * w), w);
1361efe556adSCorentin Chary } else {
1362bc210eb1SGerd Hoffmann qemu_pixman_linebuf_fill(linebuf, vs->vd->server, w, x, y + dy);
1363efe556adSCorentin Chary }
1364efe556adSCorentin Chary png_write_row(png_ptr, buf);
1365efe556adSCorentin Chary }
136647683d66SGerd Hoffmann qemu_pixman_image_unref(linebuf);
1367efe556adSCorentin Chary
1368efe556adSCorentin Chary png_write_end(png_ptr, NULL);
1369efe556adSCorentin Chary
1370efe556adSCorentin Chary if (color_type == PNG_COLOR_TYPE_PALETTE) {
1371efe556adSCorentin Chary png_free(png_ptr, png_palette);
1372efe556adSCorentin Chary }
1373efe556adSCorentin Chary
1374efe556adSCorentin Chary png_destroy_write_struct(&png_ptr, &info_ptr);
1375efe556adSCorentin Chary
1376efe556adSCorentin Chary vnc_write_u8(vs, VNC_TIGHT_PNG << 4);
1377efe556adSCorentin Chary
13786bf21f3dSLi Qiang tight_send_compact_size(vs, vs->tight->png.offset);
13796bf21f3dSLi Qiang vnc_write(vs, vs->tight->png.buffer, vs->tight->png.offset);
13806bf21f3dSLi Qiang buffer_reset(&vs->tight->png);
1381efe556adSCorentin Chary return 1;
1382efe556adSCorentin Chary }
138395f8510eSKshitij Suri #endif /* CONFIG_PNG */
1384efe556adSCorentin Chary
vnc_tight_start(VncState * vs)1385245f7b51SCorentin Chary static void vnc_tight_start(VncState *vs)
1386245f7b51SCorentin Chary {
13876bf21f3dSLi Qiang buffer_reset(&vs->tight->tight);
1388245f7b51SCorentin Chary
1389245f7b51SCorentin Chary // make the output buffer be the zlib buffer, so we can compress it later
13906bf21f3dSLi Qiang vs->tight->tmp = vs->output;
13916bf21f3dSLi Qiang vs->output = vs->tight->tight;
1392245f7b51SCorentin Chary }
1393245f7b51SCorentin Chary
vnc_tight_stop(VncState * vs)1394245f7b51SCorentin Chary static void vnc_tight_stop(VncState *vs)
1395245f7b51SCorentin Chary {
1396245f7b51SCorentin Chary // switch back to normal output/zlib buffers
13976bf21f3dSLi Qiang vs->tight->tight = vs->output;
13986bf21f3dSLi Qiang vs->output = vs->tight->tmp;
1399245f7b51SCorentin Chary }
1400245f7b51SCorentin Chary
send_sub_rect_nojpeg(VncState * vs,int x,int y,int w,int h,int bg,int fg,int colors,VncPalette * palette)140103817eb8SCorentin Chary static int send_sub_rect_nojpeg(VncState *vs, int x, int y, int w, int h,
140203817eb8SCorentin Chary int bg, int fg, int colors, VncPalette *palette)
140303817eb8SCorentin Chary {
140403817eb8SCorentin Chary int ret;
140503817eb8SCorentin Chary
140603817eb8SCorentin Chary if (colors == 0) {
140703817eb8SCorentin Chary if (tight_detect_smooth_image(vs, w, h)) {
140803817eb8SCorentin Chary ret = send_gradient_rect(vs, x, y, w, h);
140903817eb8SCorentin Chary } else {
141003817eb8SCorentin Chary ret = send_full_color_rect(vs, x, y, w, h);
141103817eb8SCorentin Chary }
141203817eb8SCorentin Chary } else if (colors == 1) {
141303817eb8SCorentin Chary ret = send_solid_rect(vs);
141403817eb8SCorentin Chary } else if (colors == 2) {
141503817eb8SCorentin Chary ret = send_mono_rect(vs, x, y, w, h, bg, fg);
141603817eb8SCorentin Chary } else if (colors <= 256) {
141703817eb8SCorentin Chary ret = send_palette_rect(vs, x, y, w, h, palette);
1418d167f9bcSBlue Swirl } else {
1419d167f9bcSBlue Swirl ret = 0;
142003817eb8SCorentin Chary }
142103817eb8SCorentin Chary return ret;
142203817eb8SCorentin Chary }
142303817eb8SCorentin Chary
142403817eb8SCorentin Chary #ifdef CONFIG_VNC_JPEG
send_sub_rect_jpeg(VncState * vs,int x,int y,int w,int h,int bg,int fg,int colors,VncPalette * palette,bool force)142503817eb8SCorentin Chary static int send_sub_rect_jpeg(VncState *vs, int x, int y, int w, int h,
142603817eb8SCorentin Chary int bg, int fg, int colors,
1427ce702e93SCorentin Chary VncPalette *palette, bool force)
142803817eb8SCorentin Chary {
142903817eb8SCorentin Chary int ret;
143003817eb8SCorentin Chary
143103817eb8SCorentin Chary if (colors == 0) {
14326bf21f3dSLi Qiang if (force || (tight_jpeg_conf[vs->tight->quality].jpeg_full &&
1433ce702e93SCorentin Chary tight_detect_smooth_image(vs, w, h))) {
14346bf21f3dSLi Qiang int quality = tight_conf[vs->tight->quality].jpeg_quality;
143503817eb8SCorentin Chary
143603817eb8SCorentin Chary ret = send_jpeg_rect(vs, x, y, w, h, quality);
143703817eb8SCorentin Chary } else {
143803817eb8SCorentin Chary ret = send_full_color_rect(vs, x, y, w, h);
143903817eb8SCorentin Chary }
144003817eb8SCorentin Chary } else if (colors == 1) {
144103817eb8SCorentin Chary ret = send_solid_rect(vs);
144203817eb8SCorentin Chary } else if (colors == 2) {
144303817eb8SCorentin Chary ret = send_mono_rect(vs, x, y, w, h, bg, fg);
144403817eb8SCorentin Chary } else if (colors <= 256) {
1445ce702e93SCorentin Chary if (force || (colors > 96 &&
14466bf21f3dSLi Qiang tight_jpeg_conf[vs->tight->quality].jpeg_idx &&
1447ce702e93SCorentin Chary tight_detect_smooth_image(vs, w, h))) {
14486bf21f3dSLi Qiang int quality = tight_conf[vs->tight->quality].jpeg_quality;
144903817eb8SCorentin Chary
145003817eb8SCorentin Chary ret = send_jpeg_rect(vs, x, y, w, h, quality);
145103817eb8SCorentin Chary } else {
145203817eb8SCorentin Chary ret = send_palette_rect(vs, x, y, w, h, palette);
145303817eb8SCorentin Chary }
1454ad7ee4adSBlue Swirl } else {
1455ad7ee4adSBlue Swirl ret = 0;
145603817eb8SCorentin Chary }
145703817eb8SCorentin Chary return ret;
145803817eb8SCorentin Chary }
145903817eb8SCorentin Chary #endif
146003817eb8SCorentin Chary
146166668d19SPeter Lieven static __thread VncPalette *color_count_palette;
146266668d19SPeter Lieven static __thread Notifier vnc_tight_cleanup_notifier;
146366668d19SPeter Lieven
vnc_tight_cleanup(Notifier * n,void * value)146466668d19SPeter Lieven static void vnc_tight_cleanup(Notifier *n, void *value)
146566668d19SPeter Lieven {
146666668d19SPeter Lieven g_free(color_count_palette);
146766668d19SPeter Lieven color_count_palette = NULL;
146866668d19SPeter Lieven }
1469095497ffSPeter Lieven
send_sub_rect(VncState * vs,int x,int y,int w,int h)1470245f7b51SCorentin Chary static int send_sub_rect(VncState *vs, int x, int y, int w, int h)
1471245f7b51SCorentin Chary {
1472245f7b51SCorentin Chary uint32_t bg = 0, fg = 0;
1473245f7b51SCorentin Chary int colors;
1474245f7b51SCorentin Chary int ret = 0;
1475cf76a1ceSPeter Maydell #ifdef CONFIG_VNC_JPEG
1476ce702e93SCorentin Chary bool force_jpeg = false;
1477ce702e93SCorentin Chary bool allow_jpeg = true;
1478cf76a1ceSPeter Maydell #endif
1479245f7b51SCorentin Chary
148066668d19SPeter Lieven if (!color_count_palette) {
1481b21e2380SMarkus Armbruster color_count_palette = g_new(VncPalette, 1);
148266668d19SPeter Lieven vnc_tight_cleanup_notifier.notify = vnc_tight_cleanup;
148366668d19SPeter Lieven qemu_thread_atexit_add(&vnc_tight_cleanup_notifier);
148466668d19SPeter Lieven }
148566668d19SPeter Lieven
14866bf21f3dSLi Qiang vnc_framebuffer_update(vs, x, y, w, h, vs->tight->type);
1487245f7b51SCorentin Chary
1488245f7b51SCorentin Chary vnc_tight_start(vs);
1489245f7b51SCorentin Chary vnc_raw_send_framebuffer_update(vs, x, y, w, h);
1490245f7b51SCorentin Chary vnc_tight_stop(vs);
1491245f7b51SCorentin Chary
1492ce702e93SCorentin Chary #ifdef CONFIG_VNC_JPEG
14936bf21f3dSLi Qiang if (!vs->vd->non_adaptive && vs->tight->quality != (uint8_t)-1) {
1494ce702e93SCorentin Chary double freq = vnc_update_freq(vs, x, y, w, h);
1495ce702e93SCorentin Chary
14966bf21f3dSLi Qiang if (freq < tight_jpeg_conf[vs->tight->quality].jpeg_freq_min) {
1497ce702e93SCorentin Chary allow_jpeg = false;
1498ce702e93SCorentin Chary }
14996bf21f3dSLi Qiang if (freq >= tight_jpeg_conf[vs->tight->quality].jpeg_freq_threshold) {
1500ce702e93SCorentin Chary force_jpeg = true;
1501ce702e93SCorentin Chary vnc_sent_lossy_rect(vs, x, y, w, h);
1502ce702e93SCorentin Chary }
1503ce702e93SCorentin Chary }
1504ce702e93SCorentin Chary #endif
1505ce702e93SCorentin Chary
150666668d19SPeter Lieven colors = tight_fill_palette(vs, x, y, w * h, &bg, &fg, color_count_palette);
1507245f7b51SCorentin Chary
1508245f7b51SCorentin Chary #ifdef CONFIG_VNC_JPEG
15096bf21f3dSLi Qiang if (allow_jpeg && vs->tight->quality != (uint8_t)-1) {
151066668d19SPeter Lieven ret = send_sub_rect_jpeg(vs, x, y, w, h, bg, fg, colors,
151166668d19SPeter Lieven color_count_palette, force_jpeg);
1512245f7b51SCorentin Chary } else {
151366668d19SPeter Lieven ret = send_sub_rect_nojpeg(vs, x, y, w, h, bg, fg, colors,
151466668d19SPeter Lieven color_count_palette);
1515245f7b51SCorentin Chary }
1516245f7b51SCorentin Chary #else
151766668d19SPeter Lieven ret = send_sub_rect_nojpeg(vs, x, y, w, h, bg, fg, colors,
151866668d19SPeter Lieven color_count_palette);
1519245f7b51SCorentin Chary #endif
152003817eb8SCorentin Chary
1521245f7b51SCorentin Chary return ret;
1522245f7b51SCorentin Chary }
1523245f7b51SCorentin Chary
send_sub_rect_solid(VncState * vs,int x,int y,int w,int h)1524245f7b51SCorentin Chary static int send_sub_rect_solid(VncState *vs, int x, int y, int w, int h)
1525245f7b51SCorentin Chary {
15266bf21f3dSLi Qiang vnc_framebuffer_update(vs, x, y, w, h, vs->tight->type);
1527245f7b51SCorentin Chary
1528245f7b51SCorentin Chary vnc_tight_start(vs);
1529245f7b51SCorentin Chary vnc_raw_send_framebuffer_update(vs, x, y, w, h);
1530245f7b51SCorentin Chary vnc_tight_stop(vs);
1531245f7b51SCorentin Chary
1532245f7b51SCorentin Chary return send_solid_rect(vs);
1533245f7b51SCorentin Chary }
1534245f7b51SCorentin Chary
send_rect_simple(VncState * vs,int x,int y,int w,int h,bool split)1535ce702e93SCorentin Chary static int send_rect_simple(VncState *vs, int x, int y, int w, int h,
1536ce702e93SCorentin Chary bool split)
1537245f7b51SCorentin Chary {
1538245f7b51SCorentin Chary int max_size, max_width;
1539245f7b51SCorentin Chary int max_sub_width, max_sub_height;
1540245f7b51SCorentin Chary int dx, dy;
1541245f7b51SCorentin Chary int rw, rh;
1542245f7b51SCorentin Chary int n = 0;
1543245f7b51SCorentin Chary
15446bf21f3dSLi Qiang max_size = tight_conf[vs->tight->compression].max_rect_size;
15456bf21f3dSLi Qiang max_width = tight_conf[vs->tight->compression].max_rect_width;
1546245f7b51SCorentin Chary
1547ce702e93SCorentin Chary if (split && (w > max_width || w * h > max_size)) {
1548245f7b51SCorentin Chary max_sub_width = (w > max_width) ? max_width : w;
1549245f7b51SCorentin Chary max_sub_height = max_size / max_sub_width;
1550245f7b51SCorentin Chary
1551245f7b51SCorentin Chary for (dy = 0; dy < h; dy += max_sub_height) {
1552245f7b51SCorentin Chary for (dx = 0; dx < w; dx += max_width) {
1553245f7b51SCorentin Chary rw = MIN(max_sub_width, w - dx);
1554245f7b51SCorentin Chary rh = MIN(max_sub_height, h - dy);
1555245f7b51SCorentin Chary n += send_sub_rect(vs, x+dx, y+dy, rw, rh);
1556245f7b51SCorentin Chary }
1557245f7b51SCorentin Chary }
1558245f7b51SCorentin Chary } else {
1559245f7b51SCorentin Chary n += send_sub_rect(vs, x, y, w, h);
1560245f7b51SCorentin Chary }
1561245f7b51SCorentin Chary
1562245f7b51SCorentin Chary return n;
1563245f7b51SCorentin Chary }
1564245f7b51SCorentin Chary
find_large_solid_color_rect(VncState * vs,int x,int y,int w,int h,int max_rows)1565245f7b51SCorentin Chary static int find_large_solid_color_rect(VncState *vs, int x, int y,
1566245f7b51SCorentin Chary int w, int h, int max_rows)
1567245f7b51SCorentin Chary {
1568245f7b51SCorentin Chary int dx, dy, dw, dh;
1569245f7b51SCorentin Chary int n = 0;
1570245f7b51SCorentin Chary
1571245f7b51SCorentin Chary /* Try to find large solid-color areas and send them separately. */
1572245f7b51SCorentin Chary
1573245f7b51SCorentin Chary for (dy = y; dy < y + h; dy += VNC_TIGHT_MAX_SPLIT_TILE_SIZE) {
1574245f7b51SCorentin Chary
1575245f7b51SCorentin Chary /* If a rectangle becomes too large, send its upper part now. */
1576245f7b51SCorentin Chary
1577245f7b51SCorentin Chary if (dy - y >= max_rows) {
1578ce702e93SCorentin Chary n += send_rect_simple(vs, x, y, w, max_rows, true);
1579245f7b51SCorentin Chary y += max_rows;
1580245f7b51SCorentin Chary h -= max_rows;
1581245f7b51SCorentin Chary }
1582245f7b51SCorentin Chary
1583245f7b51SCorentin Chary dh = MIN(VNC_TIGHT_MAX_SPLIT_TILE_SIZE, (y + h - dy));
1584245f7b51SCorentin Chary
1585245f7b51SCorentin Chary for (dx = x; dx < x + w; dx += VNC_TIGHT_MAX_SPLIT_TILE_SIZE) {
1586245f7b51SCorentin Chary uint32_t color_value;
1587245f7b51SCorentin Chary int x_best, y_best, w_best, h_best;
1588245f7b51SCorentin Chary
1589245f7b51SCorentin Chary dw = MIN(VNC_TIGHT_MAX_SPLIT_TILE_SIZE, (x + w - dx));
1590245f7b51SCorentin Chary
1591245f7b51SCorentin Chary if (!check_solid_tile(vs, dx, dy, dw, dh, &color_value, false)) {
1592245f7b51SCorentin Chary continue ;
1593245f7b51SCorentin Chary }
1594245f7b51SCorentin Chary
1595245f7b51SCorentin Chary /* Get dimensions of solid-color area. */
1596245f7b51SCorentin Chary
1597245f7b51SCorentin Chary find_best_solid_area(vs, dx, dy, w - (dx - x), h - (dy - y),
1598245f7b51SCorentin Chary color_value, &w_best, &h_best);
1599245f7b51SCorentin Chary
1600245f7b51SCorentin Chary /* Make sure a solid rectangle is large enough
1601245f7b51SCorentin Chary (or the whole rectangle is of the same color). */
1602245f7b51SCorentin Chary
1603245f7b51SCorentin Chary if (w_best * h_best != w * h &&
1604245f7b51SCorentin Chary w_best * h_best < VNC_TIGHT_MIN_SOLID_SUBRECT_SIZE) {
1605245f7b51SCorentin Chary continue;
1606245f7b51SCorentin Chary }
1607245f7b51SCorentin Chary
1608245f7b51SCorentin Chary /* Try to extend solid rectangle to maximum size. */
1609245f7b51SCorentin Chary
1610245f7b51SCorentin Chary x_best = dx; y_best = dy;
1611245f7b51SCorentin Chary extend_solid_area(vs, x, y, w, h, color_value,
1612245f7b51SCorentin Chary &x_best, &y_best, &w_best, &h_best);
1613245f7b51SCorentin Chary
1614245f7b51SCorentin Chary /* Send rectangles at top and left to solid-color area. */
1615245f7b51SCorentin Chary
1616245f7b51SCorentin Chary if (y_best != y) {
1617ce702e93SCorentin Chary n += send_rect_simple(vs, x, y, w, y_best-y, true);
1618245f7b51SCorentin Chary }
1619245f7b51SCorentin Chary if (x_best != x) {
1620efe556adSCorentin Chary n += tight_send_framebuffer_update(vs, x, y_best,
1621245f7b51SCorentin Chary x_best-x, h_best);
1622245f7b51SCorentin Chary }
1623245f7b51SCorentin Chary
1624245f7b51SCorentin Chary /* Send solid-color rectangle. */
1625245f7b51SCorentin Chary n += send_sub_rect_solid(vs, x_best, y_best, w_best, h_best);
1626245f7b51SCorentin Chary
1627245f7b51SCorentin Chary /* Send remaining rectangles (at right and bottom). */
1628245f7b51SCorentin Chary
1629245f7b51SCorentin Chary if (x_best + w_best != x + w) {
1630efe556adSCorentin Chary n += tight_send_framebuffer_update(vs, x_best+w_best,
1631245f7b51SCorentin Chary y_best,
1632245f7b51SCorentin Chary w-(x_best-x)-w_best,
1633245f7b51SCorentin Chary h_best);
1634245f7b51SCorentin Chary }
1635245f7b51SCorentin Chary if (y_best + h_best != y + h) {
1636efe556adSCorentin Chary n += tight_send_framebuffer_update(vs, x, y_best+h_best,
1637245f7b51SCorentin Chary w, h-(y_best-y)-h_best);
1638245f7b51SCorentin Chary }
1639245f7b51SCorentin Chary
1640245f7b51SCorentin Chary /* Return after all recursive calls are done. */
1641245f7b51SCorentin Chary return n;
1642245f7b51SCorentin Chary }
1643245f7b51SCorentin Chary }
1644ce702e93SCorentin Chary return n + send_rect_simple(vs, x, y, w, h, true);
1645245f7b51SCorentin Chary }
1646245f7b51SCorentin Chary
tight_send_framebuffer_update(VncState * vs,int x,int y,int w,int h)1647efe556adSCorentin Chary static int tight_send_framebuffer_update(VncState *vs, int x, int y,
1648245f7b51SCorentin Chary int w, int h)
1649245f7b51SCorentin Chary {
1650245f7b51SCorentin Chary int max_rows;
1651245f7b51SCorentin Chary
16529f64916dSGerd Hoffmann if (vs->client_pf.bytes_per_pixel == 4 && vs->client_pf.rmax == 0xFF &&
16539f64916dSGerd Hoffmann vs->client_pf.bmax == 0xFF && vs->client_pf.gmax == 0xFF) {
16546bf21f3dSLi Qiang vs->tight->pixel24 = true;
1655245f7b51SCorentin Chary } else {
16566bf21f3dSLi Qiang vs->tight->pixel24 = false;
1657245f7b51SCorentin Chary }
1658245f7b51SCorentin Chary
1659cf76a1ceSPeter Maydell #ifdef CONFIG_VNC_JPEG
16606bf21f3dSLi Qiang if (vs->tight->quality != (uint8_t)-1) {
1661ce702e93SCorentin Chary double freq = vnc_update_freq(vs, x, y, w, h);
1662ce702e93SCorentin Chary
16636bf21f3dSLi Qiang if (freq > tight_jpeg_conf[vs->tight->quality].jpeg_freq_threshold) {
1664ce702e93SCorentin Chary return send_rect_simple(vs, x, y, w, h, false);
1665ce702e93SCorentin Chary }
1666ce702e93SCorentin Chary }
1667cf76a1ceSPeter Maydell #endif
1668ce702e93SCorentin Chary
1669ce702e93SCorentin Chary if (w * h < VNC_TIGHT_MIN_SPLIT_RECT_SIZE) {
1670ce702e93SCorentin Chary return send_rect_simple(vs, x, y, w, h, true);
1671ce702e93SCorentin Chary }
1672245f7b51SCorentin Chary
1673245f7b51SCorentin Chary /* Calculate maximum number of rows in one non-solid rectangle. */
1674245f7b51SCorentin Chary
16756bf21f3dSLi Qiang max_rows = tight_conf[vs->tight->compression].max_rect_size;
16766bf21f3dSLi Qiang max_rows /= MIN(tight_conf[vs->tight->compression].max_rect_width, w);
1677245f7b51SCorentin Chary
1678245f7b51SCorentin Chary return find_large_solid_color_rect(vs, x, y, w, h, max_rows);
1679245f7b51SCorentin Chary }
1680245f7b51SCorentin Chary
vnc_tight_send_framebuffer_update(VncState * vs,int x,int y,int w,int h)1681efe556adSCorentin Chary int vnc_tight_send_framebuffer_update(VncState *vs, int x, int y,
1682efe556adSCorentin Chary int w, int h)
1683efe556adSCorentin Chary {
16846bf21f3dSLi Qiang vs->tight->type = VNC_ENCODING_TIGHT;
1685efe556adSCorentin Chary return tight_send_framebuffer_update(vs, x, y, w, h);
1686efe556adSCorentin Chary }
1687efe556adSCorentin Chary
vnc_tight_png_send_framebuffer_update(VncState * vs,int x,int y,int w,int h)1688efe556adSCorentin Chary int vnc_tight_png_send_framebuffer_update(VncState *vs, int x, int y,
1689efe556adSCorentin Chary int w, int h)
1690efe556adSCorentin Chary {
16916bf21f3dSLi Qiang vs->tight->type = VNC_ENCODING_TIGHT_PNG;
1692efe556adSCorentin Chary return tight_send_framebuffer_update(vs, x, y, w, h);
1693efe556adSCorentin Chary }
1694efe556adSCorentin Chary
vnc_tight_clear(VncState * vs)1695245f7b51SCorentin Chary void vnc_tight_clear(VncState *vs)
1696245f7b51SCorentin Chary {
1697245f7b51SCorentin Chary int i;
16986bf21f3dSLi Qiang for (i = 0; i < ARRAY_SIZE(vs->tight->stream); i++) {
16996bf21f3dSLi Qiang if (vs->tight->stream[i].opaque) {
17006bf21f3dSLi Qiang deflateEnd(&vs->tight->stream[i]);
1701245f7b51SCorentin Chary }
1702245f7b51SCorentin Chary }
1703245f7b51SCorentin Chary
17046bf21f3dSLi Qiang buffer_free(&vs->tight->tight);
17056bf21f3dSLi Qiang buffer_free(&vs->tight->zlib);
17066bf21f3dSLi Qiang buffer_free(&vs->tight->gradient);
1707245f7b51SCorentin Chary #ifdef CONFIG_VNC_JPEG
17086bf21f3dSLi Qiang buffer_free(&vs->tight->jpeg);
1709245f7b51SCorentin Chary #endif
171095f8510eSKshitij Suri #ifdef CONFIG_PNG
17116bf21f3dSLi Qiang buffer_free(&vs->tight->png);
1712b5469b11SCorentin Chary #endif
1713245f7b51SCorentin Chary }
1714