LAV
/
FFmpeg
1
0
Fork 0
Code

avcodec/mp_cmp: reject invalid comparison function values 

Fixes tickets #10306 and #10318.

Signed-off-by: James Almer <jamrial@gmail.com>
This commit is contained in:
James Almer 2023-04-13 10:19:57 -03:00
parent 5cda6b94f4
commit 7c6e26a184
6 changed files with 26 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
libavcodec/dvenc.c
View File

@ -104,7 +104,9 @@ static av_cold int dvvideo_encode_init(AVCodecContext *avctx)
ff_fdctdsp_init(&fdsp, avctx); ff_fdctdsp_init(&fdsp, avctx);
ff_me_cmp_init(&mecc, avctx); ff_me_cmp_init(&mecc, avctx);
ff_pixblockdsp_init(&pdsp, avctx); ff_pixblockdsp_init(&pdsp, avctx);
ff_set_cmp(&mecc, mecc.ildct_cmp, avctx->ildct_cmp); ret = ff_set_cmp(&mecc, mecc.ildct_cmp, avctx->ildct_cmp);
if (ret < 0)
return AVERROR(EINVAL);
s->get_pixels = pdsp.get_pixels; s->get_pixels = pdsp.get_pixels;
s->ildct_cmp = mecc.ildct_cmp[5]; s->ildct_cmp = mecc.ildct_cmp[5];

9
libavcodec/me_cmp.c
View File

@ -473,8 +473,9 @@ static int zero_cmp(MpegEncContext *s, const uint8_t *a, const uint8_t *b,
return 0; return 0;
} }
void ff_set_cmp(MECmpContext *c, me_cmp_func *cmp, int type) int ff_set_cmp(MECmpContext *c, me_cmp_func *cmp, int type)
{ {
int ret = 0;
int i; int i;
memset(cmp, 0, sizeof(void *) * 6); memset(cmp, 0, sizeof(void *) * 6);
@ -533,9 +534,13 @@ void ff_set_cmp(MECmpContext *c, me_cmp_func *cmp, int type)
#endif #endif
default: default:
av_log(NULL, AV_LOG_ERROR, av_log(NULL, AV_LOG_ERROR,
"internal error in cmp function selection\n"); "invalid cmp function selection\n");
ret = -1;
break;
} }
} }
return ret;
} }
#define BUTTERFLY2(o1, o2, i1, i2) \ #define BUTTERFLY2(o1, o2, i1, i2) \

2
libavcodec/me_cmp.h
View File

@ -89,7 +89,7 @@ void ff_me_cmp_init_ppc(MECmpContext *c, AVCodecContext *avctx);
void ff_me_cmp_init_x86(MECmpContext *c, AVCodecContext *avctx); void ff_me_cmp_init_x86(MECmpContext *c, AVCodecContext *avctx);
void ff_me_cmp_init_mips(MECmpContext *c, AVCodecContext *avctx); void ff_me_cmp_init_mips(MECmpContext *c, AVCodecContext *avctx);
void ff_set_cmp(MECmpContext *c, me_cmp_func *cmp, int type); int ff_set_cmp(MECmpContext *c, me_cmp_func *cmp, int type);
void ff_dsputil_init_dwt(MECmpContext *c); void ff_dsputil_init_dwt(MECmpContext *c);

11
libavcodec/motion_est.c
View File

@ -309,6 +309,7 @@ int ff_init_me(MpegEncContext *s){
MotionEstContext * const c= &s->me; MotionEstContext * const c= &s->me;
int cache_size= FFMIN(ME_MAP_SIZE>>ME_MAP_SHIFT, 1<<ME_MAP_SHIFT); int cache_size= FFMIN(ME_MAP_SIZE>>ME_MAP_SHIFT, 1<<ME_MAP_SHIFT);
int dia_size= FFMAX(FFABS(s->avctx->dia_size)&255, FFABS(s->avctx->pre_dia_size)&255); int dia_size= FFMAX(FFABS(s->avctx->dia_size)&255, FFABS(s->avctx->pre_dia_size)&255);
int ret;
if(FFMIN(s->avctx->dia_size, s->avctx->pre_dia_size) < -FFMIN(ME_MAP_SIZE, MAX_SAB_SIZE)){ if(FFMIN(s->avctx->dia_size, s->avctx->pre_dia_size) < -FFMIN(ME_MAP_SIZE, MAX_SAB_SIZE)){
av_log(s->avctx, AV_LOG_ERROR, "ME_MAP size is too small for SAB diamond\n"); av_log(s->avctx, AV_LOG_ERROR, "ME_MAP size is too small for SAB diamond\n");
@ -324,10 +325,12 @@ int ff_init_me(MpegEncContext *s){
av_log(s->avctx, AV_LOG_INFO, "ME_MAP size may be a little small for the selected diamond size\n"); av_log(s->avctx, AV_LOG_INFO, "ME_MAP size may be a little small for the selected diamond size\n");
} }
ff_set_cmp(&s->mecc, s->mecc.me_pre_cmp, c->avctx->me_pre_cmp); ret = ff_set_cmp(&s->mecc, s->mecc.me_pre_cmp, c->avctx->me_pre_cmp);
ff_set_cmp(&s->mecc, s->mecc.me_cmp, c->avctx->me_cmp); ret |= ff_set_cmp(&s->mecc, s->mecc.me_cmp, c->avctx->me_cmp);
ff_set_cmp(&s->mecc, s->mecc.me_sub_cmp, c->avctx->me_sub_cmp); ret |= ff_set_cmp(&s->mecc, s->mecc.me_sub_cmp, c->avctx->me_sub_cmp);
ff_set_cmp(&s->mecc, s->mecc.mb_cmp, c->avctx->mb_cmp); ret |= ff_set_cmp(&s->mecc, s->mecc.mb_cmp, c->avctx->mb_cmp);
if (ret < 0)
return ret;
c->flags = get_flags(c, 0, c->avctx->me_cmp &FF_CMP_CHROMA); c->flags = get_flags(c, 0, c->avctx->me_cmp &FF_CMP_CHROMA);
c->sub_flags= get_flags(c, 0, c->avctx->me_sub_cmp&FF_CMP_CHROMA); c->sub_flags= get_flags(c, 0, c->avctx->me_sub_cmp&FF_CMP_CHROMA);

6
libavcodec/mpegvideo_enc.c
View File

@ -902,8 +902,10 @@ av_cold int ff_mpv_encode_init(AVCodecContext *avctx)
s->quant_precision = 5; s->quant_precision = 5;
ff_set_cmp(&s->mecc, s->mecc.ildct_cmp, avctx->ildct_cmp); ret = ff_set_cmp(&s->mecc, s->mecc.ildct_cmp, avctx->ildct_cmp);
ff_set_cmp(&s->mecc, s->mecc.frame_skip_cmp, s->frame_skip_cmp); ret |= ff_set_cmp(&s->mecc, s->mecc.frame_skip_cmp, s->frame_skip_cmp);
if (ret < 0)
return AVERROR(EINVAL);
if (CONFIG_H263_ENCODER && s->out_format == FMT_H263) { if (CONFIG_H263_ENCODER && s->out_format == FMT_H263) {
ff_h263_encode_init(s); ff_h263_encode_init(s);

6
libavcodec/snowenc.c
View File

@ -130,8 +130,10 @@ static av_cold int encode_init(AVCodecContext *avctx)
if (ret) if (ret)
return ret; return ret;
ff_set_cmp(&s->mecc, s->mecc.me_cmp, s->avctx->me_cmp); ret = ff_set_cmp(&s->mecc, s->mecc.me_cmp, s->avctx->me_cmp);
ff_set_cmp(&s->mecc, s->mecc.me_sub_cmp, s->avctx->me_sub_cmp); ret |= ff_set_cmp(&s->mecc, s->mecc.me_sub_cmp, s->avctx->me_sub_cmp);
if (ret < 0)
return AVERROR(EINVAL);
s->input_picture = av_frame_alloc(); s->input_picture = av_frame_alloc();
if (!s->input_picture) if (!s->input_picture)