diff --git a/libavcodec/dvenc.c b/libavcodec/dvenc.c index 11dd5763af..cd442b524d 100644 --- a/libavcodec/dvenc.c +++ b/libavcodec/dvenc.c @@ -104,7 +104,9 @@ static av_cold int dvvideo_encode_init(AVCodecContext *avctx) ff_fdctdsp_init(&fdsp, avctx); ff_me_cmp_init(&mecc, avctx); ff_pixblockdsp_init(&pdsp, avctx); - ff_set_cmp(&mecc, mecc.ildct_cmp, avctx->ildct_cmp); + ret = ff_set_cmp(&mecc, mecc.ildct_cmp, avctx->ildct_cmp); + if (ret < 0) + return AVERROR(EINVAL); s->get_pixels = pdsp.get_pixels; s->ildct_cmp = mecc.ildct_cmp[5]; diff --git a/libavcodec/me_cmp.c b/libavcodec/me_cmp.c index e2f9f84b05..cd05e63ffd 100644 --- a/libavcodec/me_cmp.c +++ b/libavcodec/me_cmp.c @@ -473,8 +473,9 @@ static int zero_cmp(MpegEncContext *s, const uint8_t *a, const uint8_t *b, return 0; } -void ff_set_cmp(MECmpContext *c, me_cmp_func *cmp, int type) +int ff_set_cmp(MECmpContext *c, me_cmp_func *cmp, int type) { + int ret = 0; int i; memset(cmp, 0, sizeof(void *) * 6); @@ -533,9 +534,13 @@ void ff_set_cmp(MECmpContext *c, me_cmp_func *cmp, int type) #endif default: av_log(NULL, AV_LOG_ERROR, - "internal error in cmp function selection\n"); + "invalid cmp function selection\n"); + ret = -1; + break; } } + + return ret; } #define BUTTERFLY2(o1, o2, i1, i2) \ diff --git a/libavcodec/me_cmp.h b/libavcodec/me_cmp.h index 90ea76c891..aefd32a7dc 100644 --- a/libavcodec/me_cmp.h +++ b/libavcodec/me_cmp.h @@ -89,7 +89,7 @@ void ff_me_cmp_init_ppc(MECmpContext *c, AVCodecContext *avctx); void ff_me_cmp_init_x86(MECmpContext *c, AVCodecContext *avctx); void ff_me_cmp_init_mips(MECmpContext *c, AVCodecContext *avctx); -void ff_set_cmp(MECmpContext *c, me_cmp_func *cmp, int type); +int ff_set_cmp(MECmpContext *c, me_cmp_func *cmp, int type); void ff_dsputil_init_dwt(MECmpContext *c); diff --git a/libavcodec/motion_est.c b/libavcodec/motion_est.c index d17ffe42b4..df9d1befa8 100644 --- a/libavcodec/motion_est.c +++ b/libavcodec/motion_est.c @@ -309,6 +309,7 @@ int ff_init_me(MpegEncContext *s){ MotionEstContext * const c= &s->me; int cache_size= FFMIN(ME_MAP_SIZE>>ME_MAP_SHIFT, 1<avctx->dia_size)&255, FFABS(s->avctx->pre_dia_size)&255); + int ret; if(FFMIN(s->avctx->dia_size, s->avctx->pre_dia_size) < -FFMIN(ME_MAP_SIZE, MAX_SAB_SIZE)){ av_log(s->avctx, AV_LOG_ERROR, "ME_MAP size is too small for SAB diamond\n"); @@ -324,10 +325,12 @@ int ff_init_me(MpegEncContext *s){ av_log(s->avctx, AV_LOG_INFO, "ME_MAP size may be a little small for the selected diamond size\n"); } - ff_set_cmp(&s->mecc, s->mecc.me_pre_cmp, c->avctx->me_pre_cmp); - ff_set_cmp(&s->mecc, s->mecc.me_cmp, c->avctx->me_cmp); - ff_set_cmp(&s->mecc, s->mecc.me_sub_cmp, c->avctx->me_sub_cmp); - ff_set_cmp(&s->mecc, s->mecc.mb_cmp, c->avctx->mb_cmp); + ret = ff_set_cmp(&s->mecc, s->mecc.me_pre_cmp, c->avctx->me_pre_cmp); + ret |= ff_set_cmp(&s->mecc, s->mecc.me_cmp, c->avctx->me_cmp); + ret |= ff_set_cmp(&s->mecc, s->mecc.me_sub_cmp, c->avctx->me_sub_cmp); + ret |= ff_set_cmp(&s->mecc, s->mecc.mb_cmp, c->avctx->mb_cmp); + if (ret < 0) + return ret; c->flags = get_flags(c, 0, c->avctx->me_cmp &FF_CMP_CHROMA); c->sub_flags= get_flags(c, 0, c->avctx->me_sub_cmp&FF_CMP_CHROMA); diff --git a/libavcodec/mpegvideo_enc.c b/libavcodec/mpegvideo_enc.c index 7d3c8875f2..bcd2f9ef1d 100644 --- a/libavcodec/mpegvideo_enc.c +++ b/libavcodec/mpegvideo_enc.c @@ -902,8 +902,10 @@ av_cold int ff_mpv_encode_init(AVCodecContext *avctx) s->quant_precision = 5; - ff_set_cmp(&s->mecc, s->mecc.ildct_cmp, avctx->ildct_cmp); - ff_set_cmp(&s->mecc, s->mecc.frame_skip_cmp, s->frame_skip_cmp); + ret = ff_set_cmp(&s->mecc, s->mecc.ildct_cmp, avctx->ildct_cmp); + ret |= ff_set_cmp(&s->mecc, s->mecc.frame_skip_cmp, s->frame_skip_cmp); + if (ret < 0) + return AVERROR(EINVAL); if (CONFIG_H263_ENCODER && s->out_format == FMT_H263) { ff_h263_encode_init(s); diff --git a/libavcodec/snowenc.c b/libavcodec/snowenc.c index 4cf7ff1145..31a48972a5 100644 --- a/libavcodec/snowenc.c +++ b/libavcodec/snowenc.c @@ -130,8 +130,10 @@ static av_cold int encode_init(AVCodecContext *avctx) if (ret) return ret; - ff_set_cmp(&s->mecc, s->mecc.me_cmp, s->avctx->me_cmp); - ff_set_cmp(&s->mecc, s->mecc.me_sub_cmp, s->avctx->me_sub_cmp); + ret = ff_set_cmp(&s->mecc, s->mecc.me_cmp, s->avctx->me_cmp); + ret |= ff_set_cmp(&s->mecc, s->mecc.me_sub_cmp, s->avctx->me_sub_cmp); + if (ret < 0) + return AVERROR(EINVAL); s->input_picture = av_frame_alloc(); if (!s->input_picture)