From 6364fa9e9a4b5712d817ae307d9ae07e149e2f79 Mon Sep 17 00:00:00 2001
From: Michael Niedermayer <michael@niedermayer.cc>
Date: Tue, 17 Oct 2023 19:31:37 +0200
Subject: [PATCH] avcodec/h264: Avoid using gray gap frames as references

Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
---
 libavcodec/h264_refs.c | 11 +++++++++++
 libavcodec/h264dec.c   |  1 +
 libavcodec/h264dec.h   |  1 +
 3 files changed, 13 insertions(+)

diff --git a/libavcodec/h264_refs.c b/libavcodec/h264_refs.c
index 92778e737a..9bc7b20988 100644
--- a/libavcodec/h264_refs.c
+++ b/libavcodec/h264_refs.c
@@ -410,6 +410,17 @@ int ff_h264_build_ref_list(H264Context *h, H264SliceContext *sl)
                 else
                     return -1;
             }
+            if (h->noref_gray>0 && sl->ref_list[list][index].parent->gray && h->non_gray) {
+                for (int j=0; j<sl->list_count; j++) {
+                    int list2 = (list+j)&1;
+                    if (h->default_ref[list2].parent && !h->default_ref[list2].parent->gray
+                        && !(!FIELD_PICTURE(h) && (h->default_ref[list2].reference&3) != 3)) {
+                        sl->ref_list[list][index] = h->default_ref[list2];
+                        av_log(h, AV_LOG_DEBUG, "replacement of gray gap frame\n");
+                        break;
+                    }
+                }
+            }
             av_assert0(av_buffer_get_ref_count(sl->ref_list[list][index].parent->f->buf[0]) > 0);
         }
     }
diff --git a/libavcodec/h264dec.c b/libavcodec/h264dec.c
index 7ea55db75d..b48821db24 100644
--- a/libavcodec/h264dec.c
+++ b/libavcodec/h264dec.c
@@ -1091,6 +1091,7 @@ static const AVOption h264_options[] = {
     { "nal_length_size", "nal_length_size", OFFSET(nal_length_size), AV_OPT_TYPE_INT, {.i64 = 0}, 0, 4, VDX },
     { "enable_er", "Enable error resilience on damaged frames (unsafe)", OFFSET(enable_er), AV_OPT_TYPE_BOOL, { .i64 = -1 }, -1, 1, VD },
     { "x264_build", "Assume this x264 version if no x264 version found in any SEI", OFFSET(x264_build), AV_OPT_TYPE_INT, {.i64 = -1}, -1, INT_MAX, VD },
+    { "noref_gray", "Avoid using gray gap frames as references", OFFSET(noref_gray), AV_OPT_TYPE_INT, {.i64 = 1}, -1, 1, VD },
     { NULL },
 };
 
diff --git a/libavcodec/h264dec.h b/libavcodec/h264dec.h
index 366626c056..591769ab25 100644
--- a/libavcodec/h264dec.h
+++ b/libavcodec/h264dec.h
@@ -571,6 +571,7 @@ typedef struct H264Context {
     int ref2frm[MAX_SLICES][2][64];     ///< reference to frame number lists, used in the loop filter, the first 2 are for -2,-1
 
     int non_gray;                       ///< Did we encounter a intra frame after a gray gap frame
+    int noref_gray;
 } H264Context;
 
 extern const uint16_t ff_h264_mb_sizes[4];