LAV
/
FFmpeg
1
0
Fork 0
Code

avcodec/jpegxl_parser: fix various memory issues 

The spec caps the prefix alphabet size to 32768 (i.e. 1 << 15) so we
should check for that and reject alphabets that are too large, in order
to prevent over-allocating.

Additionally, there's no need to allocate buffers that are as large as
the maximum alphabet size as these aren't stack-allocated, they're heap
allocated and thus can be variable size.

Added an overflow check as well, which fixes leaking the buffer, and
capping the alphabet size fixes two potential overruns as well.

Fixes: out of array access
Fixes: 62089/clusterfuzz-testcase-minimized-ffmpeg_DEMUXER_fuzzer-
    5437089094959104.fuzz

Found-by: continuous fuzzing process
    https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Found-by: Hardik Shah of Vehere (Dawn Treaders team)
Co-authored-by: Michael Niedermayer <michael@niedermayer.cc>
Signed-off-by: Leo Izen <leo.izen@gmail.com>
This commit is contained in:
Leo Izen 2023-10-03 11:00:35 -04:00
parent ec74553205
commit f7ac3512f5
No known key found for this signature in database
GPG Key ID: 5A71C331FD2FA19A
1 changed files with 17 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
libavcodec/jpegxl_parser.c
View File

@ -46,6 +46,8 @@
#define JXL_FLAG_USE_LF_FRAME 32
#define JXL_FLAG_SKIP_ADAPTIVE_LF_SMOOTH 128
#define MAX_PREFIX_ALPHABET_SIZE (1u << 15)
#define clog1p(x) (ff_log2(x) + !!(x))
#define unpack_signed(x) (((x) & 1 ? -(x)-1 : (x))/2)
#define div_ceil(x, y) (((x) - 1) / (y) + 1)
@ -724,16 +726,17 @@ static int read_vlc_prefix(GetBitContext *gb, JXLEntropyDecoder *dec, JXLSymbolD
if (ret < 0)
goto end;
buf = av_calloc(1, 262148); // 32768 * 8 + 4
buf = av_mallocz(dist->alphabet_size * (2 * sizeof(int8_t) + sizeof(int16_t) + sizeof(uint32_t))
+ sizeof(uint32_t));
if (!buf) {
ret = AVERROR(ENOMEM);
goto end;
}
level2_lens = (int8_t *)buf;
level2_lens_s = (int8_t *)(buf + 32768);
level2_syms = (int16_t *)(buf + 65536);
level2_codecounts = (uint32_t *)(buf + 131072);
level2_lens_s = (int8_t *)(buf + dist->alphabet_size * sizeof(int8_t));
level2_syms = (int16_t *)(buf + dist->alphabet_size * (2 * sizeof(int8_t)));
level2_codecounts = (uint32_t *)(buf + dist->alphabet_size * (2 * sizeof(int8_t) + sizeof(int16_t)));
total_code = 0;
for (int i = 0; i < dist->alphabet_size; i++) {
@ -742,6 +745,10 @@ static int read_vlc_prefix(GetBitContext *gb, JXLEntropyDecoder *dec, JXLSymbolD
int extra = 3 + get_bits(gb, 2);
if (repeat_count_prev)
extra = 4 * (repeat_count_prev - 2) - repeat_count_prev + extra;
if (i + extra > dist->alphabet_size) {
ret = AVERROR_INVALIDDATA;
goto end;
}
for (int j = 0; j < extra; j++)
level2_lens[i + j] = prev;
total_code += (32768 >> prev) * extra;
@ -772,8 +779,10 @@ static int read_vlc_prefix(GetBitContext *gb, JXLEntropyDecoder *dec, JXLSymbolD
}
}
if (total_code != 32768 && level2_codecounts[0] < dist->alphabet_size - 1)
return AVERROR_INVALIDDATA;
if (total_code != 32768 && level2_codecounts[0] < dist->alphabet_size - 1) {
ret = AVERROR_INVALIDDATA;
goto end;
}
for (int i = 1; i < dist->alphabet_size + 1; i++)
level2_codecounts[i] += level2_codecounts[i - 1];
@ -848,6 +857,8 @@ static int read_distribution_bundle(GetBitContext *gb, JXLEntropyDecoder *dec,
if (get_bits1(gb)) {
int n = get_bits(gb, 4);
dist->alphabet_size = 1 + (1 << n) + get_bitsz(gb, n);
if (dist->alphabet_size > MAX_PREFIX_ALPHABET_SIZE)
return AVERROR_INVALIDDATA;
} else {
dist->alphabet_size = 1;
}