diff --git a/libavcodec/av1_parser.c b/libavcodec/av1_parser.c
index e57e382757..14dae92fe9 100644
--- a/libavcodec/av1_parser.c
+++ b/libavcodec/av1_parser.c
@@ -168,9 +168,6 @@ static int av1_parser_parse(AVCodecParserContext *ctx,
                   timing->num_units_in_display_tick, timing->time_scale, INT_MAX);
     }
 
-    if (avctx->framerate.num)
-        avctx->time_base = av_inv_q(av_mul_q(avctx->framerate, (AVRational){avctx->ticks_per_frame, 1}));
-
 end:
     ff_cbs_fragment_reset(td);
 
diff --git a/libavcodec/avcodec.c b/libavcodec/avcodec.c
index efa76d2740..00a5851807 100644
--- a/libavcodec/avcodec.c
+++ b/libavcodec/avcodec.c
@@ -349,11 +349,6 @@ FF_ENABLE_DEPRECATION_WARNINGS
             ret = AVERROR(EINVAL);
             goto free_and_end;
         }
-
-#if FF_API_AVCTX_TIMEBASE
-        if (avctx->framerate.num > 0 && avctx->framerate.den > 0)
-            avctx->time_base = av_inv_q(av_mul_q(avctx->framerate, (AVRational){avctx->ticks_per_frame, 1}));
-#endif
     }
     if (codec->priv_class)
         av_assert0(*(const AVClass **)avctx->priv_data == codec->priv_class);
diff --git a/libavcodec/avcodec.h b/libavcodec/avcodec.h
index cc7a8faa6a..c83e03a08f 100644
--- a/libavcodec/avcodec.h
+++ b/libavcodec/avcodec.h
@@ -552,8 +552,7 @@ typedef struct AVCodecContext {
      * (fixed_vop_rate == 0 implies that it is different from the framerate)
      *
      * - encoding: MUST be set by user.
-     * - decoding: the use of this field for decoding is deprecated.
-     *             Use framerate instead.
+     * - decoding: unused.
      */
     AVRational time_base;
 
diff --git a/libavcodec/avs2_parser.c b/libavcodec/avs2_parser.c
index 0350517493..200134f91d 100644
--- a/libavcodec/avs2_parser.c
+++ b/libavcodec/avs2_parser.c
@@ -112,9 +112,9 @@ static void parse_avs2_seq_header(AVCodecParserContext *s, const uint8_t *buf,
     s->height = height;
     s->coded_width = FFALIGN(width, 8);
     s->coded_height = FFALIGN(height, 8);
-    avctx->framerate.num = avctx->time_base.den =
+    avctx->framerate.num =
         ff_avs2_frame_rate_tab[frame_rate_code].num;
-    avctx->framerate.den = avctx->time_base.num =
+    avctx->framerate.den =
         ff_avs2_frame_rate_tab[frame_rate_code].den;
     avctx->has_b_frames = FFMAX(avctx->has_b_frames, !low_delay);
 
diff --git a/libavcodec/avs3_parser.c b/libavcodec/avs3_parser.c
index a9fd879e9d..a819b5783d 100644
--- a/libavcodec/avs3_parser.c
+++ b/libavcodec/avs3_parser.c
@@ -117,8 +117,8 @@ static void parse_avs3_nal_units(AVCodecParserContext *s, const uint8_t *buf,
             low_delay = get_bits(&gb, 1);
             avctx->has_b_frames = FFMAX(avctx->has_b_frames, !low_delay);
 
-            avctx->framerate.num = avctx->time_base.den = ff_avs3_frame_rate_tab[ratecode].num;
-            avctx->framerate.den = avctx->time_base.num = ff_avs3_frame_rate_tab[ratecode].den;
+            avctx->framerate.num = ff_avs3_frame_rate_tab[ratecode].num;
+            avctx->framerate.den = ff_avs3_frame_rate_tab[ratecode].den;
 
             s->width  = s->coded_width = avctx->width;
             s->height = s->coded_height = avctx->height;
diff --git a/libavcodec/cpia.c b/libavcodec/cpia.c
index 99362e73f0..bfd270dae2 100644
--- a/libavcodec/cpia.c
+++ b/libavcodec/cpia.c
@@ -198,14 +198,6 @@ static av_cold int cpia_decode_init(AVCodecContext *avctx)
     // output pixel format
     avctx->pix_fmt = AV_PIX_FMT_YUV420P;
 
-    /* The default timebase set by the v4l2 demuxer leads to probing which is buggy.
-     * Set some reasonable time_base to skip this.
-     */
-    if (avctx->time_base.num == 1 && avctx->time_base.den == 1000000) {
-        avctx->time_base.num = 1;
-        avctx->time_base.den = 60;
-    }
-
     s->frame = av_frame_alloc();
     if (!s->frame)
         return AVERROR(ENOMEM);
diff --git a/libavcodec/decode.c b/libavcodec/decode.c
index 34e67a649b..aba20859df 100644
--- a/libavcodec/decode.c
+++ b/libavcodec/decode.c
@@ -434,11 +434,6 @@ FF_ENABLE_DEPRECATION_WARNINGS
 #endif
         ret = pkt->size;
 
-#if FF_API_AVCTX_TIMEBASE
-    if (avctx->framerate.num > 0 && avctx->framerate.den > 0)
-        avctx->time_base = av_inv_q(av_mul_q(avctx->framerate, (AVRational){avctx->ticks_per_frame, 1}));
-#endif
-
     /* do not stop draining when actual_got_frame != 0 or ret < 0 */
     /* got_frame == 0 but actual_got_frame != 0 when frame is discarded */
     if (avci->draining && !actual_got_frame) {
diff --git a/libavcodec/h264_parser.c b/libavcodec/h264_parser.c
index 3ed23fb9ca..46134a1c48 100644
--- a/libavcodec/h264_parser.c
+++ b/libavcodec/h264_parser.c
@@ -593,6 +593,7 @@ static int h264_parse(AVCodecParserContext *s,
 {
     H264ParseContext *p = s->priv_data;
     ParseContext *pc = &p->pc;
+    AVRational time_base = { 0, 1 };
     int next;
 
     if (!p->got_first) {
@@ -624,7 +625,7 @@ static int h264_parse(AVCodecParserContext *s,
     parse_nal_units(s, avctx, buf, buf_size);
 
     if (avctx->framerate.num)
-        avctx->time_base = av_inv_q(av_mul_q(avctx->framerate, (AVRational){avctx->ticks_per_frame, 1}));
+        time_base = av_inv_q(av_mul_q(avctx->framerate, (AVRational){avctx->ticks_per_frame, 1}));
     if (p->sei.picture_timing.cpb_removal_delay >= 0) {
         s->dts_sync_point    = p->sei.buffering_period.present;
         s->dts_ref_dts_delta = p->sei.picture_timing.cpb_removal_delay;
@@ -640,9 +641,9 @@ static int h264_parse(AVCodecParserContext *s,
     }
 
     if (s->dts_sync_point >= 0) {
-        int64_t den = avctx->time_base.den * (int64_t)avctx->pkt_timebase.num;
+        int64_t den = time_base.den * (int64_t)avctx->pkt_timebase.num;
         if (den > 0) {
-            int64_t num = avctx->time_base.num * (int64_t)avctx->pkt_timebase.den;
+            int64_t num = time_base.num * (int64_t)avctx->pkt_timebase.den;
             if (s->dts != AV_NOPTS_VALUE) {
                 // got DTS from the stream, update reference timestamp
                 p->reference_dts = s->dts - av_rescale(s->dts_ref_dts_delta, num, den);
diff --git a/libavcodec/h264dec.c b/libavcodec/h264dec.c
index 9f7b3782e8..2d691731c5 100644
--- a/libavcodec/h264dec.c
+++ b/libavcodec/h264dec.c
@@ -382,12 +382,6 @@ static av_cold int h264_decode_init(AVCodecContext *avctx)
         return AVERROR_UNKNOWN;
     }
 
-    if (avctx->ticks_per_frame == 1) {
-        if(h->avctx->time_base.den < INT_MAX/2) {
-            h->avctx->time_base.den *= 2;
-        } else
-            h->avctx->time_base.num /= 2;
-    }
     avctx->ticks_per_frame = 2;
 
     if (!avctx->internal->is_copy) {
diff --git a/libavcodec/mjpegdec.c b/libavcodec/mjpegdec.c
index f33911e1a8..b2be55af4a 100644
--- a/libavcodec/mjpegdec.c
+++ b/libavcodec/mjpegdec.c
@@ -436,7 +436,7 @@ int ff_mjpeg_decode_sof(MJpegDecodeContext *s)
 
         /* test interlaced mode */
         if (s->first_picture   &&
-            (s->multiscope != 2 || s->avctx->time_base.den >= 25 * s->avctx->time_base.num) &&
+            (s->multiscope != 2 || s->avctx->pkt_timebase.den >= 25 * s->avctx->pkt_timebase.num) &&
             s->orig_height != 0 &&
             s->height < ((s->orig_height * 3) / 4)) {
             s->interlaced                    = 1;
diff --git a/libavcodec/mpeg4video_parser.c b/libavcodec/mpeg4video_parser.c
index e32a93d296..3beb5f6dae 100644
--- a/libavcodec/mpeg4video_parser.c
+++ b/libavcodec/mpeg4video_parser.c
@@ -114,11 +114,11 @@ static int mpeg4_decode_header(AVCodecParserContext *s1, AVCodecContext *avctx,
         if (ret < 0)
             return ret;
     }
-    if((s1->flags & PARSER_FLAG_USE_CODEC_TS) && s->avctx->time_base.den>0 && ret>=0){
+    if((s1->flags & PARSER_FLAG_USE_CODEC_TS) && s->avctx->framerate.num>0 && ret>=0){
         av_assert1(s1->pts == AV_NOPTS_VALUE);
         av_assert1(s1->dts == AV_NOPTS_VALUE);
 
-        s1->pts = av_rescale_q(s->time, (AVRational){1, s->avctx->time_base.den}, (AVRational){1, 1200000});
+        s1->pts = av_rescale_q(s->time, (AVRational){1, s->avctx->framerate.num}, (AVRational){1, 1200000});
     }
 
     s1->pict_type     = s->pict_type;
diff --git a/libavcodec/mpeg4videodec.c b/libavcodec/mpeg4videodec.c
index 0a100d2064..f96b6a3117 100644
--- a/libavcodec/mpeg4videodec.c
+++ b/libavcodec/mpeg4videodec.c
@@ -2648,8 +2648,6 @@ static int decode_vol_header(Mpeg4DecContext *ctx, GetBitContext *gb)
     else
         s->avctx->framerate.den = 1;
 
-    s->avctx->time_base = av_inv_q(av_mul_q(s->avctx->framerate, (AVRational){s->avctx->ticks_per_frame, 1}));
-
     ctx->t_frame = 0;
 
     if (ctx->shape != BIN_ONLY_SHAPE) {
@@ -3145,7 +3143,6 @@ static int decode_vop_header(Mpeg4DecContext *ctx, GetBitContext *gb,
                "time_increment_bits set to %d bits, based on bitstream analysis\n", ctx->time_increment_bits);
         if (s->avctx->framerate.num && 4*s->avctx->framerate.num < 1<<ctx->time_increment_bits) {
             s->avctx->framerate.num = 1<<ctx->time_increment_bits;
-            s->avctx->time_base = av_inv_q(av_mul_q(s->avctx->framerate, (AVRational){s->avctx->ticks_per_frame, 1}));
         }
     }
 
diff --git a/libavcodec/mpegvideo_parser.c b/libavcodec/mpegvideo_parser.c
index ac6efb6909..57bc1f706c 100644
--- a/libavcodec/mpegvideo_parser.c
+++ b/libavcodec/mpegvideo_parser.c
@@ -241,11 +241,6 @@ static void mpegvideo_extract_headers(AVCodecParserContext *s,
         s->coded_width  = FFALIGN(pc->width,  16);
         s->coded_height = FFALIGN(pc->height, 16);
     }
-
-#if FF_API_AVCTX_TIMEBASE
-    if (avctx->framerate.num)
-        avctx->time_base = av_inv_q(av_mul_q(avctx->framerate, (AVRational){avctx->ticks_per_frame, 1}));
-#endif
 }
 
 static int mpegvideo_parse(AVCodecParserContext *s,
diff --git a/libavcodec/vc1_parser.c b/libavcodec/vc1_parser.c
index a459a2aa7d..4167215fb1 100644
--- a/libavcodec/vc1_parser.c
+++ b/libavcodec/vc1_parser.c
@@ -112,8 +112,6 @@ static void vc1_extract_header(AVCodecParserContext *s, AVCodecContext *avctx,
 
         break;
     }
-    if (avctx->framerate.num)
-        avctx->time_base = av_inv_q(av_mul_q(avctx->framerate, (AVRational){avctx->ticks_per_frame, 1}));
     s->format = vpc->v.chromaformat == 1 ? AV_PIX_FMT_YUV420P
                                          : AV_PIX_FMT_NONE;
     if (avctx->width && avctx->height) {
diff --git a/libavcodec/version_major.h b/libavcodec/version_major.h
index 84560bcaba..573cd855ed 100644
--- a/libavcodec/version_major.h
+++ b/libavcodec/version_major.h
@@ -38,7 +38,6 @@
  */
 
 #define FF_API_INIT_PACKET         (LIBAVCODEC_VERSION_MAJOR < 60)
-#define FF_API_AVCTX_TIMEBASE    (LIBAVCODEC_VERSION_MAJOR < 60)
 #define FF_API_FLAG_TRUNCATED      (LIBAVCODEC_VERSION_MAJOR < 60)
 #define FF_API_SUB_TEXT_FORMAT     (LIBAVCODEC_VERSION_MAJOR < 60)
 #define FF_API_IDCT_NONE           (LIBAVCODEC_VERSION_MAJOR < 60)
diff --git a/libavformat/avformat.c b/libavformat/avformat.c
index 88ff46a574..fe7b068f4a 100644
--- a/libavformat/avformat.c
+++ b/libavformat/avformat.c
@@ -703,6 +703,10 @@ int avformat_transfer_internal_stream_timing_info(const AVOutputFormat *ofmt,
 {
     const AVCodecContext *const dec_ctx = cffstream(ist)->avctx;
     AVCodecContext       *const enc_ctx =  ffstream(ost)->avctx;
+    AVRational dec_ctx_tb = dec_ctx->framerate.num ? av_inv_q(av_mul_q(dec_ctx->framerate,
+                                                                       (AVRational){dec_ctx->ticks_per_frame, 1}))
+                                                   : (ist->codecpar->codec_type == AVMEDIA_TYPE_AUDIO ? (AVRational){0, 1}
+                                                                                                      : ist->time_base);
 
     enc_ctx->time_base = ist->time_base;
     /*
@@ -715,38 +719,41 @@ int avformat_transfer_internal_stream_timing_info(const AVOutputFormat *ofmt,
         if (copy_tb == AVFMT_TBCF_AUTO && ist->r_frame_rate.num
             && av_q2d(ist->r_frame_rate) >= av_q2d(ist->avg_frame_rate)
             && 0.5/av_q2d(ist->r_frame_rate) > av_q2d(ist->time_base)
-            && 0.5/av_q2d(ist->r_frame_rate) > av_q2d(dec_ctx->time_base)
-            && av_q2d(ist->time_base) < 1.0/500 && av_q2d(dec_ctx->time_base) < 1.0/500
+            && 0.5/av_q2d(ist->r_frame_rate) > av_q2d(dec_ctx_tb)
+            && av_q2d(ist->time_base) < 1.0/500 && av_q2d(dec_ctx_tb) < 1.0/500
             || copy_tb == AVFMT_TBCF_R_FRAMERATE) {
             enc_ctx->time_base.num = ist->r_frame_rate.den;
             enc_ctx->time_base.den = 2*ist->r_frame_rate.num;
             enc_ctx->ticks_per_frame = 2;
         } else
 #endif
-            if (copy_tb == AVFMT_TBCF_AUTO && av_q2d(dec_ctx->time_base)*dec_ctx->ticks_per_frame > 2*av_q2d(ist->time_base)
+            if (copy_tb == AVFMT_TBCF_AUTO && dec_ctx->framerate.num &&
+                av_q2d(av_inv_q(dec_ctx->framerate)) > 2*av_q2d(ist->time_base)
                    && av_q2d(ist->time_base) < 1.0/500
-                   || copy_tb == AVFMT_TBCF_DECODER) {
-            enc_ctx->time_base = dec_ctx->time_base;
+                   || (copy_tb == AVFMT_TBCF_DECODER &&
+                       (dec_ctx->framerate.num || ist->codecpar->codec_type == AVMEDIA_TYPE_AUDIO))) {
+            enc_ctx->time_base = dec_ctx_tb;
             enc_ctx->time_base.num *= dec_ctx->ticks_per_frame;
             enc_ctx->time_base.den *= 2;
             enc_ctx->ticks_per_frame = 2;
         }
     } else if (!(ofmt->flags & AVFMT_VARIABLE_FPS)
                && !av_match_name(ofmt->name, "mov,mp4,3gp,3g2,psp,ipod,ismv,f4v")) {
-        if (copy_tb == AVFMT_TBCF_AUTO && dec_ctx->time_base.den
-            && av_q2d(dec_ctx->time_base)*dec_ctx->ticks_per_frame > av_q2d(ist->time_base)
+        if (copy_tb == AVFMT_TBCF_AUTO && dec_ctx->framerate.num
+            && av_q2d(av_inv_q(dec_ctx->framerate)) > av_q2d(ist->time_base)
             && av_q2d(ist->time_base) < 1.0/500
-            || copy_tb == AVFMT_TBCF_DECODER) {
-            enc_ctx->time_base = dec_ctx->time_base;
+            || (copy_tb == AVFMT_TBCF_DECODER &&
+                (dec_ctx->framerate.num || ist->codecpar->codec_type == AVMEDIA_TYPE_AUDIO))) {
+            enc_ctx->time_base = dec_ctx_tb;
             enc_ctx->time_base.num *= dec_ctx->ticks_per_frame;
         }
     }
 
     if ((enc_ctx->codec_tag == AV_RL32("tmcd") || ost->codecpar->codec_tag == AV_RL32("tmcd"))
-        && dec_ctx->time_base.num < dec_ctx->time_base.den
-        && dec_ctx->time_base.num > 0
-        && 121LL*dec_ctx->time_base.num > dec_ctx->time_base.den) {
-        enc_ctx->time_base = dec_ctx->time_base;
+        && dec_ctx_tb.num < dec_ctx_tb.den
+        && dec_ctx_tb.num > 0
+        && 121LL*dec_ctx_tb.num > dec_ctx_tb.den) {
+        enc_ctx->time_base = dec_ctx_tb;
     }
 
     av_reduce(&enc_ctx->time_base.num, &enc_ctx->time_base.den,
diff --git a/libavformat/demux.c b/libavformat/demux.c
index ba2991750b..b19ab86d08 100644
--- a/libavformat/demux.c
+++ b/libavformat/demux.c
@@ -2156,10 +2156,19 @@ static int get_std_framerate(int i)
  * Old DivX and Xvid often have nonsense timebases like 1fps or 2fps.
  * MPEG-2 commonly misuses field repeat flags to store different framerates.
  * And there are "variable" fps files this needs to detect as well. */
-static int tb_unreliable(AVCodecContext *c)
+static int tb_unreliable(AVFormatContext *ic, AVStream *st)
 {
-    if (c->time_base.den >= 101LL * c->time_base.num ||
-        c->time_base.den <    5LL * c->time_base.num ||
+    FFStream *const sti = ffstream(st);
+    AVCodecContext *c = sti->avctx;
+    AVRational time_base = c->framerate.num ? av_inv_q(av_mul_q(c->framerate,
+                                                                (AVRational){c->ticks_per_frame, 1}))
+                                            /* NOHEADER check added to not break existing behavior */
+                                            : (((ic->ctx_flags & AVFMTCTX_NOHEADER) ||
+                                                st->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) ? (AVRational){0, 1}
+                                                                                                : st->time_base);
+
+    if (time_base.den >= 101LL * time_base.num ||
+        time_base.den <    5LL * time_base.num ||
         // c->codec_tag == AV_RL32("DIVX") ||
         // c->codec_tag == AV_RL32("XVID") ||
         c->codec_tag == AV_RL32("mp4v") ||
@@ -2243,11 +2252,11 @@ void ff_rfps_calculate(AVFormatContext *ic)
         // the check for tb_unreliable() is not completely correct, since this is not about handling
         // an unreliable/inexact time base, but a time base that is finer than necessary, as e.g.
         // ipmovie.c produces.
-        if (tb_unreliable(sti->avctx) && sti->info->duration_count > 15 && sti->info->duration_gcd > FFMAX(1, st->time_base.den/(500LL*st->time_base.num)) && !st->r_frame_rate.num &&
+        if (tb_unreliable(ic, st) && sti->info->duration_count > 15 && sti->info->duration_gcd > FFMAX(1, st->time_base.den/(500LL*st->time_base.num)) && !st->r_frame_rate.num &&
             sti->info->duration_gcd < INT64_MAX / st->time_base.num)
             av_reduce(&st->r_frame_rate.num, &st->r_frame_rate.den, st->time_base.den, st->time_base.num * sti->info->duration_gcd, INT_MAX);
         if (sti->info->duration_count > 1 && !st->r_frame_rate.num
-            && tb_unreliable(sti->avctx)) {
+            && tb_unreliable(ic, st)) {
             int num = 0;
             double best_error = 0.01;
             AVRational ref_rate = st->r_frame_rate.num ? st->r_frame_rate : av_inv_q(st->time_base);
@@ -2459,14 +2468,6 @@ int avformat_find_stream_info(AVFormatContext *ic, AVDictionary **options)
         FFStream *const sti = ffstream(st);
         AVCodecContext *const avctx = sti->avctx;
 
-        if (st->codecpar->codec_type == AVMEDIA_TYPE_VIDEO ||
-            st->codecpar->codec_type == AVMEDIA_TYPE_SUBTITLE) {
-/*            if (!st->time_base.num)
-                st->time_base = */
-            if (!avctx->time_base.num)
-                avctx->time_base = st->time_base;
-        }
-
         /* check if the caller has overridden the codec id */
         // only for the split stuff
         if (!sti->parser && !(ic->flags & AVFMT_FLAG_NOPARSE) && sti->request_probe <= 0) {
@@ -2544,7 +2545,7 @@ int avformat_find_stream_info(AVFormatContext *ic, AVDictionary **options)
              * the correct fps. */
             if (av_q2d(st->time_base) > 0.0005)
                 fps_analyze_framecount *= 2;
-            if (!tb_unreliable(sti->avctx))
+            if (!tb_unreliable(ic, st))
                 fps_analyze_framecount = 0;
             if (ic->fps_probe_size >= 0)
                 fps_analyze_framecount = ic->fps_probe_size;
@@ -2857,12 +2858,16 @@ int avformat_find_stream_info(AVFormatContext *ic, AVDictionary **options)
                     av_reduce(&st->avg_frame_rate.num, &st->avg_frame_rate.den,
                               best_fps, 12 * 1001, INT_MAX);
             }
-
             if (!st->r_frame_rate.num) {
-                if (    avctx->time_base.den * (int64_t) st->time_base.num
-                    <= avctx->time_base.num * (uint64_t)avctx->ticks_per_frame * st->time_base.den) {
+                AVRational time_base = avctx->framerate.num ? av_inv_q(av_mul_q(avctx->framerate,
+                                                                       (AVRational){avctx->ticks_per_frame, 1}))
+                                                            /* NOHEADER check added to not break existing behavior */
+                                                            : ((ic->ctx_flags & AVFMTCTX_NOHEADER) ? (AVRational){0, 1}
+                                                                                                   : st->time_base);
+                if (   time_base.den * (int64_t) st->time_base.num
+                    <= time_base.num * (uint64_t)avctx->ticks_per_frame * st->time_base.den) {
                     av_reduce(&st->r_frame_rate.num, &st->r_frame_rate.den,
-                              avctx->time_base.den, (int64_t)avctx->time_base.num * avctx->ticks_per_frame, INT_MAX);
+                              time_base.den, (int64_t)time_base.num * avctx->ticks_per_frame, INT_MAX);
                 } else {
                     st->r_frame_rate.num = st->time_base.den;
                     st->r_frame_rate.den = st->time_base.num;