From 7ec4835386923a3f0cf9f3a257774a55f165a0e9 Mon Sep 17 00:00:00 2001 From: Thilo Borgmann Date: Sat, 2 Dec 2023 18:46:16 +0100 Subject: [PATCH] avfilter: Add fsync filter --- Changelog | 1 + MAINTAINERS | 1 + configure | 2 + doc/filters.texi | 41 ++++++ libavfilter/Makefile | 1 + libavfilter/allfilters.c | 1 + libavfilter/version.h | 2 +- libavfilter/vf_fsync.c | 298 +++++++++++++++++++++++++++++++++++++++ 8 files changed, 346 insertions(+), 1 deletion(-) create mode 100644 libavfilter/vf_fsync.c diff --git a/Changelog b/Changelog index 87a4ad886d..5b2899d05b 100644 --- a/Changelog +++ b/Changelog @@ -17,6 +17,7 @@ version : - quirc filter - lavu/eval: introduce randomi() function in expressions - VVC decoder +- fsync filter version 6.1: - libaribcaption decoder diff --git a/MAINTAINERS b/MAINTAINERS index 9c4683db19..3f7098acf8 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -343,6 +343,7 @@ Filters: vf_delogo.c Jean Delvare (CC ) vf_drawbox.c/drawgrid Andrey Utkin vf_extractplanes.c Paul B Mahol + vf_fsync.c Thilo Borgmann vf_histogram.c Paul B Mahol vf_hqx.c Clément Bœsch vf_idet.c Pascal Massimino diff --git a/configure b/configure index d15cfa4703..b86c1732c0 100755 --- a/configure +++ b/configure @@ -3768,6 +3768,7 @@ frei0r_deps_any="libdl LoadLibrary" frei0r_filter_deps="frei0r" frei0r_src_filter_deps="frei0r" fspp_filter_deps="gpl" +fsync_filter_deps="avformat" gblur_vulkan_filter_deps="vulkan spirv_compiler" hflip_vulkan_filter_deps="vulkan spirv_compiler" histeq_filter_deps="gpl" @@ -7674,6 +7675,7 @@ enabled cover_rect_filter && prepend avfilter_deps "avformat avcodec" enabled ebur128_filter && enabled swresample && prepend avfilter_deps "swresample" enabled elbg_filter && prepend avfilter_deps "avcodec" enabled find_rect_filter && prepend avfilter_deps "avformat avcodec" +enabled fsync_filter && prepend avfilter_deps "avformat" enabled mcdeint_filter && prepend avfilter_deps "avcodec" enabled movie_filter && prepend avfilter_deps "avformat avcodec" enabled pan_filter && prepend avfilter_deps "swresample" diff --git a/doc/filters.texi b/doc/filters.texi index 27efc123cf..09fa3264ea 100644 --- a/doc/filters.texi +++ b/doc/filters.texi @@ -14689,6 +14689,47 @@ option may cause flicker since the B-Frames have often larger QP. Default is @end table +@anchor{fsync} +@section fsync + +Synchronize video frames with an external mapping from a file. + +For each input PTS given in the map file it either drops or creates as many +frames as necessary to recreate the sequence of output frames given in the +map file. + +This filter is useful to recreate the output frames of a framerate conversion +by the @ref{fps} filter, recorded into a map file using the ffmpeg option +@code{-stats_mux_pre}, and do further processing to the corresponding frames +e.g. quality comparison. + +Each line of the map file must contain three items per input frame, the input +PTS (decimal), the output PTS (decimal) and the +output TIMEBASE (decimal/decimal), seperated by a space. +This file format corresponds to the output +of @code{-stats_mux_pre_fmt="@{ptsi@} @{pts@} @{tb@}"}. + +The filter assumes the map file is sorted by increasing input PTS. + +The filter accepts the following options: +@table @option + +@item file, f +The filename of the map file to be used. +@end table + +Example: +@example +# Convert a video to 25 fps and record a MAP_FILE file with the default format of this filter +ffmpeg -i INPUT -vf fps=fps=25 -stats_mux_pre MAP_FILE -stats_mux_pre_fmt "@{ptsi@} @{pts@} @{tb@}" OUTPUT + +# Sort MAP_FILE by increasing input PTS +sort -n MAP_FILE + +# Use INPUT, OUTPUT and the MAP_FILE from above to compare the corresponding frames in INPUT and OUTPUT via SSIM +ffmpeg -i INPUT -i OUTPUT -filter_complex '[0:v]fsync=file=MAP_FILE[ref];[1:v][ref]ssim' -f null - +@end example + @section gblur Apply Gaussian blur filter. diff --git a/libavfilter/Makefile b/libavfilter/Makefile index f65fb9a5a7..bba0219876 100644 --- a/libavfilter/Makefile +++ b/libavfilter/Makefile @@ -323,6 +323,7 @@ OBJS-$(CONFIG_FREEZEDETECT_FILTER) += vf_freezedetect.o OBJS-$(CONFIG_FREEZEFRAMES_FILTER) += vf_freezeframes.o OBJS-$(CONFIG_FREI0R_FILTER) += vf_frei0r.o OBJS-$(CONFIG_FSPP_FILTER) += vf_fspp.o qp_table.o +OBJS-$(CONFIG_FSYNC_FILTER) += vf_fsync.o OBJS-$(CONFIG_GBLUR_FILTER) += vf_gblur.o OBJS-$(CONFIG_GBLUR_VULKAN_FILTER) += vf_gblur_vulkan.o vulkan.o vulkan_filter.o OBJS-$(CONFIG_GEQ_FILTER) += vf_geq.o diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c index b8570dbab2..af84aa3d97 100644 --- a/libavfilter/allfilters.c +++ b/libavfilter/allfilters.c @@ -299,6 +299,7 @@ extern const AVFilter ff_vf_freezedetect; extern const AVFilter ff_vf_freezeframes; extern const AVFilter ff_vf_frei0r; extern const AVFilter ff_vf_fspp; +extern const AVFilter ff_vf_fsync; extern const AVFilter ff_vf_gblur; extern const AVFilter ff_vf_gblur_vulkan; extern const AVFilter ff_vf_geq; diff --git a/libavfilter/version.h b/libavfilter/version.h index d243558b77..12fc6c853d 100644 --- a/libavfilter/version.h +++ b/libavfilter/version.h @@ -31,7 +31,7 @@ #include "version_major.h" -#define LIBAVFILTER_VERSION_MINOR 16 +#define LIBAVFILTER_VERSION_MINOR 17 #define LIBAVFILTER_VERSION_MICRO 100 diff --git a/libavfilter/vf_fsync.c b/libavfilter/vf_fsync.c new file mode 100644 index 0000000000..58367e78f7 --- /dev/null +++ b/libavfilter/vf_fsync.c @@ -0,0 +1,298 @@ +/* + * Copyright (c) 2023 Thilo Borgmann + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/** + * @file + * Filter for syncing video frames from external source + * + * @author Thilo Borgmann + */ + +#include "libavutil/avstring.h" +#include "libavutil/error.h" +#include "libavutil/opt.h" +#include "libavformat/avio.h" +#include "video.h" +#include "filters.h" + +#define BUF_SIZE 256 + +typedef struct FsyncContext { + const AVClass *class; + AVIOContext *avio_ctx; // reading the map file + AVFrame *last_frame; // buffering the last frame for duplicating eventually + char *filename; // user-specified map file + char *buf; // line buffer for the map file + char *cur; // current position in the line buffer + char *end; // end pointer of the line buffer + int64_t ptsi; // input pts to map to [0-N] output pts + int64_t pts; // output pts + int tb_num; // output timebase num + int tb_den; // output timebase den +} FsyncContext; + +#define OFFSET(x) offsetof(FsyncContext, x) + +static const AVOption fsync_options[] = { + { "file", "set the file name to use for frame sync", OFFSET(filename), AV_OPT_TYPE_STRING, { .str = "" }, .flags= AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_FILTERING_PARAM }, + { "f", "set the file name to use for frame sync", OFFSET(filename), AV_OPT_TYPE_STRING, { .str = "" }, .flags= AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_FILTERING_PARAM }, + { NULL } +}; + +/** + * Fills the buffer from cur to end, add \0 at EOF + */ +static int buf_fill(FsyncContext *ctx) +{ + int ret; + int num = ctx->end - ctx->cur; + + ret = avio_read(ctx->avio_ctx, ctx->cur, num); + if (ret < 0) + return ret; + if (ret < num) { + *(ctx->cur + ret) = '\0'; + } + + return ret; +} + +/** + * Copies cur to end to the beginning and fills the rest + */ +static int buf_reload(FsyncContext *ctx) +{ + int i, ret; + int num = ctx->end - ctx->cur; + + for (i = 0; i < num; i++) { + ctx->buf[i] = *ctx->cur++; + } + + ctx->cur = ctx->buf + i; + ret = buf_fill(ctx); + if (ret < 0) + return ret; + ctx->cur = ctx->buf; + + return ret; +} + +/** + * Skip from cur over eol + */ +static void buf_skip_eol(FsyncContext *ctx) +{ + char *i; + for (i = ctx->cur; i < ctx->end; i++) { + if (*i != '\n')// && *i != '\r') + break; + } + ctx->cur = i; +} + +/** + * Get number of bytes from cur until eol + * + * @return >= 0 in case of success, + * -1 in case there is no line ending before end of buffer + */ +static int buf_get_line_count(FsyncContext *ctx) +{ + int ret = 0; + char *i; + for (i = ctx->cur; i < ctx->end; i++, ret++) { + if (*i == '\0' || *i == '\n') + return ret; + } + + return -1; +} + +/** + * Get number of bytes from cur to '\0' + */ +static int buf_get_zero(FsyncContext *ctx) +{ + return av_strnlen(ctx->cur, ctx->end - ctx->cur); +} + +static int activate(AVFilterContext *ctx) +{ + FsyncContext *s = ctx->priv; + AVFilterLink *inlink = ctx->inputs[0]; + AVFilterLink *outlink = ctx->outputs[0]; + + int ret, line_count; + AVFrame *frame; + + FF_FILTER_FORWARD_STATUS_BACK(outlink, inlink); + + buf_skip_eol(s); + line_count = buf_get_line_count(s); + if (line_count < 0) { + line_count = buf_reload(s); + if (line_count < 0) + return line_count; + line_count = buf_get_line_count(s); + if (line_count < 0) + return line_count; + } + + if (avio_feof(s->avio_ctx) && buf_get_zero(s) < 3) { + av_log(ctx, AV_LOG_DEBUG, "End of file. To zero = %i\n", buf_get_zero(s)); + goto end; + } + + if (s->last_frame) { + ret = av_sscanf(s->cur, "%"PRId64" %"PRId64" %d/%d", &s->ptsi, &s->pts, &s->tb_num, &s->tb_den); + if (ret != 4) { + av_log(ctx, AV_LOG_ERROR, "Unexpected format found (%i / 4).\n", ret); + ff_outlink_set_status(outlink, AVERROR_INVALIDDATA, AV_NOPTS_VALUE); + return AVERROR_INVALIDDATA; + } + + av_log(ctx, AV_LOG_DEBUG, "frame %"PRId64" ", s->last_frame->pts); + + if (s->last_frame->pts >= s->ptsi) { + av_log(ctx, AV_LOG_DEBUG, ">= %"PRId64": DUP LAST with pts = %"PRId64"\n", s->ptsi, s->pts); + + // clone frame + frame = av_frame_clone(s->last_frame); + if (!frame) { + ff_outlink_set_status(outlink, AVERROR(ENOMEM), AV_NOPTS_VALUE); + return AVERROR(ENOMEM); + } + + // set output pts and timebase + frame->pts = s->pts; + frame->time_base = av_make_q((int)s->tb_num, (int)s->tb_den); + + // advance cur to eol, skip over eol in the next call + s->cur += line_count; + + // call again + if (ff_inoutlink_check_flow(inlink, outlink)) + ff_filter_set_ready(ctx, 100); + + // filter frame + return ff_filter_frame(outlink, frame); + } else if (s->last_frame->pts < s->ptsi) { + av_log(ctx, AV_LOG_DEBUG, "< %"PRId64": DROP\n", s->ptsi); + av_frame_free(&s->last_frame); + + // call again + if (ff_inoutlink_check_flow(inlink, outlink)) + ff_filter_set_ready(ctx, 100); + + return 0; + } + } + +end: + ret = ff_inlink_consume_frame(inlink, &s->last_frame); + if (ret < 0) + return ret; + + FF_FILTER_FORWARD_STATUS(inlink, outlink); + FF_FILTER_FORWARD_WANTED(outlink, inlink); + + return FFERROR_NOT_READY; +} + +static int fsync_config_props(AVFilterLink* outlink) +{ + AVFilterContext *ctx = outlink->src; + FsyncContext *s = ctx->priv; + int ret; + + // read first line to get output timebase + ret = av_sscanf(s->cur, "%"PRId64" %"PRId64" %d/%d", &s->ptsi, &s->pts, &s->tb_num, &s->tb_den); + if (ret != 4) { + av_log(ctx, AV_LOG_ERROR, "Unexpected format found (%i of 4).\n", ret); + ff_outlink_set_status(outlink, AVERROR_INVALIDDATA, AV_NOPTS_VALUE); + return AVERROR_INVALIDDATA; + } + + outlink->frame_rate = av_make_q(1, 0); // unknown or dynamic + outlink->time_base = av_make_q(s->tb_num, s->tb_den); + + return 0; +} + +static av_cold int fsync_init(AVFilterContext *ctx) +{ + FsyncContext *s = ctx->priv; + int ret; + + av_log(ctx, AV_LOG_DEBUG, "filename: %s\n", s->filename); + + s->buf = av_malloc(BUF_SIZE + 1); + if (!s->buf) + return AVERROR(ENOMEM); + + ret = avio_open(&s->avio_ctx, s->filename, AVIO_FLAG_READ); + if (ret < 0) + return ret; + + s->cur = s->buf; + s->end = s->buf + BUF_SIZE; + s->buf[BUF_SIZE] = '\0'; + + ret = buf_fill(s); + if (ret < 0) + return ret; + + + return 0; +} + +static av_cold void fsync_uninit(AVFilterContext *ctx) +{ + FsyncContext *s = ctx->priv; + + avio_closep(&s->avio_ctx); + av_freep(&s->buf); + av_frame_free(&s->last_frame); +} + +AVFILTER_DEFINE_CLASS(fsync); + +static const AVFilterPad fsync_outputs[] = { + { + .name = "default", + .type = AVMEDIA_TYPE_VIDEO, + .config_props = fsync_config_props, + }, +}; + +const AVFilter ff_vf_fsync = { + .name = "fsync", + .description = NULL_IF_CONFIG_SMALL("Synchronize video frames from external source."), + .init = fsync_init, + .uninit = fsync_uninit, + .priv_size = sizeof(FsyncContext), + .priv_class = &fsync_class, + .activate = activate, + .formats_state = FF_FILTER_FORMATS_PASSTHROUGH, + FILTER_INPUTS(ff_video_default_filterpad), + FILTER_OUTPUTS(fsync_outputs), + .flags = AVFILTER_FLAG_METADATA_ONLY, +};