Description

Integer Overflow in function lsr_translate_coords at laser/lsr_dec.c:853

gpac version

git log
commit ea3af7c8242d1a82657dc3a518df5a5b1b5e27ed (HEAD -> master, origin/master, origin/HEAD)
Author: Romain Bouqueau <romain.bouqueau.pro@gmail.com>
Date:   Tue Jun 28 19:25:58 2022 +0200

POC

./MP4Box -bt ./poc_intof1_s.dat
laser/lsr_dec.c:853:10: runtime error: shift exponent 4294967295 is too large for 32-bit type 'u32' (aka 'unsigned int')
SUMMARY: UndefinedBehaviorSanitizer: undefined-behavior laser/lsr_dec.c:853:10 in 

poc_intof1_s.dat

function lsr_translate_coords source code

static Fixed function lsr_translate_coords(GF_LASeRCodec *lsr, u32 val, u32 nb_bits)
{
#ifdef GPAC_FIXED_POINT
        if (val >> (nb_bits-1) ) {
                s32 neg = (s32) val - (1<<nb_bits);
                if (neg < -FIX_ONE / 2)
                        return 2 * gf_divfix(INT2FIX(neg/2), lsr->res_factor);
                return gf_divfix(INT2FIX(neg), lsr->res_factor);
        } else {
                if (val > FIX_ONE / 2)
                        return 2 * gf_divfix(INT2FIX(val/2), lsr->res_factor);
                return gf_divfix(INT2FIX(val), lsr->res_factor);
        }
#else
        if (val >> (nb_bits-1) ) {    // <--- line:853
                s32 neg = (s32) val - (1<<nb_bits);
                return gf_divfix(INT2FIX(neg), lsr->res_factor);
        } else {
                return gf_divfix(INT2FIX(val), lsr->res_factor);
        }
#endif
}

GDB

gdb --args ./MP4Box -bt ./poc_intof1_s.dat

(gdb) b laser/lsr_dec.c:853
No source file named laser/lsr_dec.c.
Make breakpoint pending on future shared library load? (y or [n]) y
Breakpoint 1 (laser/lsr_dec.c:853) pending.
(gdb) r
Starting program: /home/fuzz/fuzz/gpac/gpac/bin/gcc/MP4Box -bt ./poc_intof1_s.dat
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".

Breakpoint 1, lsr_translate_coords (lsr=0x611000000540, val=0, nb_bits=0) at laser/lsr_dec.c:853
853     laser/lsr_dec.c: No such file or directory.
(gdb) l
848     in laser/lsr_dec.c
(gdb) p val
$1 = 0
(gdb) p (nb_bits-1)
$2 = 4294967295
(gdb) p nb_bits
$3 = 0
(gdb)