1From 4fcace61801f418786c42487c6b06b693ee87666 Mon Sep 17 00:00:00 2001 2From: Romain Vimont <rom1v@videolabs.io> 3Date: Mon, 19 Sep 2022 17:17:01 +0200 4Subject: [PATCH] vnc: fix possible buffer overflow 5 6Upstream-Status: Inappropriate 7 8RPI-Distro repo forks original vlc and applies patches 9to enable raspiberry pi support. 10 11Thanks to 0xMitsurugi [1] from Synacktiv [2] for the bug report and fix. 12 13[1] https://twitter.com/0xMitsurugi 14[2] https://www.synacktiv.com/ 15 16Fixes #27335 17 18(cherry picked from commit 5eb783fd44ed6298db3e38f7765f21c42e4405f9) 19--- 20 modules/access/vnc.c | 23 ++++++++++++++++------- 21 1 file changed, 16 insertions(+), 7 deletions(-) 22 23--- a/modules/access/vnc.c 24+++ b/modules/access/vnc.c 25@@ -33,6 +33,7 @@ 26 #ifdef HAVE_CONFIG_H 27 # include "config.h" 28 #endif 29+#include <assert.h> 30 31 #include <vlc_common.h> 32 #include <vlc_plugin.h> 33@@ -115,7 +116,7 @@ 34 int i_cancel_state; 35 36 rfbClient* p_client; 37- int i_framebuffersize; 38+ size_t i_framebuffersize; 39 block_t *p_block; 40 41 float f_fps; 42@@ -143,11 +144,16 @@ 43 p_sys->es = NULL; 44 } 45 46- int i_width = p_client->width; 47- int i_height = p_client->height; 48- int i_depth = p_client->format.bitsPerPixel; 49+ assert(!(p_client->width & ~0xffff)); // fits in 16 bits 50+ uint16_t i_width = p_client->width; 51 52- switch( i_depth ) 53+ assert(!(p_client->height & ~0xffff)); // fits in 16 bits 54+ uint16_t i_height = p_client->height; 55+ 56+ uint8_t i_bits_per_pixel = p_client->format.bitsPerPixel; 57+ assert((i_bits_per_pixel & 0x7) == 0); // multiple of 8 58+ 59+ switch( i_bits_per_pixel ) 60 { 61 case 8: 62 i_chroma = VLC_CODEC_RGB8; 63@@ -180,7 +186,10 @@ 64 } 65 66 /* Set up framebuffer */ 67- p_sys->i_framebuffersize = i_width * i_height * i_depth / 8; 68+ if (mul_overflow(i_width, i_height * (i_bits_per_pixel / 8), &p_sys->i_framebuffersize)) { 69+ msg_Err(p_demux, "VNC framebuffersize overflow"); 70+ return FALSE; 71+ } 72 73 /* Reuse unsent block */ 74 if ( p_sys->p_block ) 75@@ -211,7 +220,7 @@ 76 fmt.video.i_frame_rate_base = 1000; 77 fmt.video.i_frame_rate = 1000 * p_sys->f_fps; 78 79- fmt.video.i_bits_per_pixel = i_depth; 80+ fmt.video.i_bits_per_pixel = i_bits_per_pixel; 81 fmt.video.i_rmask = p_client->format.redMax << p_client->format.redShift; 82 fmt.video.i_gmask = p_client->format.greenMax << p_client->format.greenShift; 83 fmt.video.i_bmask = p_client->format.blueMax << p_client->format.blueShift; 84