backporting enhanced-rtmp

libavcodec/av1dec.c

@@ -678,7 +678,6 @@ static int set_context_with_sequence(AVCodecContext *avctx,
         if (ret < 0)
             return ret;
     }
-    avctx->sample_aspect_ratio = (AVRational) { 1, 1 };
 
     if (seq->timing_info.num_units_in_display_tick &&
         seq->timing_info.time_scale) {

libavformat/Makefile

@@ -214,7 +214,7 @@ OBJS-$(CONFIG_FLAC_MUXER)                += flacenc.o flacenc_header.o \
 OBJS-$(CONFIG_FLIC_DEMUXER)              += flic.o
 OBJS-$(CONFIG_FLV_DEMUXER)               += flvdec.o
 OBJS-$(CONFIG_LIVE_FLV_DEMUXER)          += flvdec.o
-OBJS-$(CONFIG_FLV_MUXER)                 += flvenc.o avc.o
+OBJS-$(CONFIG_FLV_MUXER)                 += flvenc.o avc.o hevc.o av1.o vpcc.o
 OBJS-$(CONFIG_FOURXM_DEMUXER)            += 4xm.o
 OBJS-$(CONFIG_FRAMECRC_MUXER)            += framecrcenc.o framehash.o
 OBJS-$(CONFIG_FRAMEHASH_MUXER)           += hashenc.o framehash.o

libavformat/av1.c

@@ -361,7 +361,8 @@ int ff_av1_parse_seq_header(AV1SequenceParameters *seq, const uint8_t *buf, int
     return AVERROR_INVALIDDATA;
 }
 
-int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size)
+int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size,
+                       int write_seq_header)
 {
     AVIOContext *meta_pb;
     AV1SequenceParameters seq_params;
@@ -451,7 +452,9 @@ int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size)
     flush_put_bits(&pbc);
 
     avio_write(pb, header, sizeof(header));
-    avio_write(pb, seq, seq_size);
+    if (write_seq_header) {
+        avio_write(pb, seq, seq_size);
+    }
 
     meta_size = avio_get_dyn_buf(meta_pb, &meta);
     if (meta_size)

libavformat/av1.h

@@ -94,9 +94,11 @@ int ff_av1_parse_seq_header(AV1SequenceParameters *seq, const uint8_t *buf, int
  * @param pb pointer to the AVIOContext where the av1C box shall be written
  * @param buf input data buffer
  * @param size size in bytes of the input data buffer
+ * @param write_seq_header If 1, Sequence Header OBU will be written inside the
+ *           av1C box. Otherwise, Sequence Header OBU will be omitted.
  *
  * @return >= 0 in case of success, a negative AVERROR code in case of failure
  */
-int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size);
+int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size, int write_seq_header);
 
 #endif /* AVFORMAT_AV1_H */

libavformat/dashenc.c

@@ -346,7 +346,7 @@ static int check_file_extension(const char *filename, const char *extension) {
 static void set_vp9_codec_str(AVFormatContext *s, AVCodecParameters *par,
                               AVRational *frame_rate, char *str, int size) {
     VPCC vpcc;
-    int ret = ff_isom_get_vpcc_features(s, par, frame_rate, &vpcc);
+    int ret = ff_isom_get_vpcc_features(s, par, NULL, 0, frame_rate, &vpcc);
     if (ret == 0) {
         av_strlcatf(str, size, "vp09.%02d.%02d.%02d",
                     vpcc.profile, vpcc.level, vpcc.bitdepth);

libavformat/flv.h

@@ -35,14 +35,20 @@
 
 #define FLV_VIDEO_FRAMETYPE_OFFSET   4
 
+/* Extended VideoTagHeader
+ * defined in reference link:
+ * https://github.com/veovera/enhanced-rtmp/blob/main/enhanced-rtmp-v1.pdf
+ * */
+#define FLV_IS_EX_HEADER          0x80
+
 /* bitmasks to isolate specific values */
 #define FLV_AUDIO_CHANNEL_MASK    0x01
 #define FLV_AUDIO_SAMPLESIZE_MASK 0x02
 #define FLV_AUDIO_SAMPLERATE_MASK 0x0c
 #define FLV_AUDIO_CODECID_MASK    0xf0
 
 #define FLV_VIDEO_CODECID_MASK    0x0f
-#define FLV_VIDEO_FRAMETYPE_MASK  0xf0
+#define FLV_VIDEO_FRAMETYPE_MASK  0x70
 
 #define AMF_END_OF_OBJECT         0x09
 
@@ -113,6 +119,15 @@ enum {
     FLV_CODECID_HEVC    = 12,
 };
 
+enum {
+    PacketTypeSequenceStart         = 0,
+    PacketTypeCodedFrames           = 1,
+    PacketTypeSequenceEnd           = 2,
+    PacketTypeCodedFramesX          = 3,
+    PacketTypeMetadata              = 4,
+    PacketTypeMPEG2TSSequenceStart  = 5,
+};
+
 enum {
     FLV_FRAME_KEY            = 1 << FLV_VIDEO_FRAMETYPE_OFFSET, ///< key frame (for AVC, a seekable frame)
     FLV_FRAME_INTER          = 2 << FLV_VIDEO_FRAMETYPE_OFFSET, ///< inter frame (for AVC, a non-seekable frame)

libavformat/flvdec.c

@@ -77,6 +77,8 @@ typedef struct FLVContext {
     int64_t last_ts;
     int64_t time_offset;
     int64_t time_pos;
+
+    uint8_t exheader;
 } FLVContext;
 
 /* AMF date type */
@@ -300,14 +302,18 @@ static void flv_set_audio_codec(AVFormatContext *s, AVStream *astream,
     }
 }
 
-static int flv_same_video_codec(AVCodecParameters *vpar, int flags)
+static int flv_same_video_codec(AVCodecParameters *vpar, uint32_t flv_codecid)
 {
-    int flv_codecid = flags & FLV_VIDEO_CODECID_MASK;
-
     if (!vpar->codec_id && !vpar->codec_tag)
         return 1;
 
     switch (flv_codecid) {
+    case MKBETAG('h', 'v', 'c', '1'):
+        return vpar->codec_id == AV_CODEC_ID_HEVC;
+    case MKBETAG('a', 'v', '0', '1'):
+        return vpar->codec_id == AV_CODEC_ID_AV1;
+    case MKBETAG('v', 'p', '0', '9'):
+        return vpar->codec_id == AV_CODEC_ID_VP9;
     case FLV_CODECID_H263:
         return vpar->codec_id == AV_CODEC_ID_FLV1;
     case FLV_CODECID_SCREEN:
@@ -328,12 +334,26 @@ static int flv_same_video_codec(AVCodecParameters *vpar, int flags)
 }
 
 static int flv_set_video_codec(AVFormatContext *s, AVStream *vstream,
-                               int flv_codecid, int read)
+                               uint32_t flv_codecid, int read)
 {
+    AVStream *const vstreami = vstream;
     int ret = 0;
     AVCodecParameters *par = vstream->codecpar;
     enum AVCodecID old_codec_id = vstream->codecpar->codec_id;
+
     switch (flv_codecid) {
+    case MKBETAG('h', 'v', 'c', '1'):
+        par->codec_id = AV_CODEC_ID_HEVC;
+        vstreami->need_parsing = AVSTREAM_PARSE_HEADERS;
+        break;
+    case MKBETAG('a', 'v', '0', '1'):
+        par->codec_id = AV_CODEC_ID_AV1;
+        vstreami->need_parsing = AVSTREAM_PARSE_HEADERS;
+        break;
+    case MKBETAG('v', 'p', '0', '9'):
+        par->codec_id = AV_CODEC_ID_VP9;
+        vstreami->need_parsing = AVSTREAM_PARSE_HEADERS;
+        break;
     case FLV_CODECID_H263:
         par->codec_id = AV_CODEC_ID_FLV1;
         break;
@@ -365,16 +385,13 @@ static int flv_set_video_codec(AVFormatContext *s, AVStream *vstream,
     case FLV_CODECID_H264:
         par->codec_id = AV_CODEC_ID_H264;
         vstream->need_parsing = AVSTREAM_PARSE_HEADERS;
-        ret = 3;     // not 4, reading packet type will consume one byte
         break;
     case FLV_CODECID_HEVC:
         par->codec_id = AV_CODEC_ID_HEVC;
         vstream->need_parsing = AVSTREAM_PARSE_NONE;
-        ret = 3;     // not 4, reading packet type will consume one byte
         break;
     case FLV_CODECID_MPEG4:
         par->codec_id = AV_CODEC_ID_MPEG4;
-        ret = 3;
         break;
     default:
         avpriv_request_sample(s, "Video codec (%x)", flv_codecid);
@@ -803,6 +820,7 @@ static int flv_read_header(AVFormatContext *s)
     s->start_time = 0;
     flv->sum_flv_tag_size = 0;
     flv->last_keyframe_stream_index = -1;
+    flv->exheader = 0;
 
     return 0;
 }
@@ -1033,6 +1051,7 @@ static int flv_read_packet(AVFormatContext *s, AVPacket *pkt)
     AVStream *st    = NULL;
     int last = -1;
     int orig_size;
+    uint32_t video_codec_id = 0;
 
 retry:
     /* pkt size is repeated at end. skip it */
@@ -1079,7 +1098,17 @@ static int flv_read_packet(AVFormatContext *s, AVPacket *pkt)
     } else if (type == FLV_TAG_TYPE_VIDEO) {
         stream_type = FLV_STREAM_TYPE_VIDEO;
         flags    = avio_r8(s->pb);
+        video_codec_id = flags & FLV_VIDEO_CODECID_MASK;
+        /*
+         * Reference Enhancing FLV 2023-03-v1.0.0-B.8
+         * https://github.com/veovera/enhanced-rtmp/blob/main/enhanced-rtmp-v1.pdf
+         * */
+        flv->exheader = (flags >> 7) & 1;
         size--;
+        if (flv->exheader) {
+            video_codec_id = avio_rb32(s->pb);
+            size -= 4;
+        }
         if ((flags & FLV_VIDEO_FRAMETYPE_MASK) == FLV_FRAME_VIDEO_INFO_CMD)
             goto skip;
     } else if (type == FLV_TAG_TYPE_META) {
@@ -1137,7 +1166,7 @@ static int flv_read_packet(AVFormatContext *s, AVPacket *pkt)
                 break;
         } else if (stream_type == FLV_STREAM_TYPE_VIDEO) {
             if (st->codecpar->codec_type == AVMEDIA_TYPE_VIDEO &&
-                (s->video_codec_id || flv_same_video_codec(st->codecpar, flags)))
+                (s->video_codec_id || flv_same_video_codec(st->codecpar, video_codec_id)))
                 break;
         } else if (stream_type == FLV_STREAM_TYPE_SUBTITLE) {
             if (st->codecpar->codec_type == AVMEDIA_TYPE_SUBTITLE)
@@ -1240,7 +1269,7 @@ static int flv_read_packet(AVFormatContext *s, AVPacket *pkt)
             avcodec_parameters_free(&par);
         }
     } else if (stream_type == FLV_STREAM_TYPE_VIDEO) {
-        int ret = flv_set_video_codec(s, st, flags & FLV_VIDEO_CODECID_MASK, 1);
+        int ret = flv_set_video_codec(s, st, video_codec_id, 1);
         if (ret < 0)
             return ret;
         size -= ret;
@@ -1253,16 +1282,24 @@ static int flv_read_packet(AVFormatContext *s, AVPacket *pkt)
     if (st->codecpar->codec_id == AV_CODEC_ID_AAC ||
         st->codecpar->codec_id == AV_CODEC_ID_H264 ||
         st->codecpar->codec_id == AV_CODEC_ID_MPEG4 ||
-        st->codecpar->codec_id == AV_CODEC_ID_HEVC) {
-        int type = avio_r8(s->pb);
-        size--;
+        st->codecpar->codec_id == AV_CODEC_ID_HEVC ||
+        st->codecpar->codec_id == AV_CODEC_ID_AV1 ||
+        st->codecpar->codec_id == AV_CODEC_ID_VP9) {
+        int type = 0;
+        if (flv->exheader && stream_type == FLV_STREAM_TYPE_VIDEO) {
+            type = flags & 0x0F;
+        } else {
+            type = avio_r8(s->pb);
+            size--;
+        }
 
         if (size < 0) {
             ret = AVERROR_INVALIDDATA;
             goto leave;
         }
 
-        if (st->codecpar->codec_id == AV_CODEC_ID_H264 || st->codecpar->codec_id == AV_CODEC_ID_MPEG4 || st->codecpar->codec_id == AV_CODEC_ID_HEVC) {
+        if (st->codecpar->codec_id == AV_CODEC_ID_H264 || st->codecpar->codec_id == AV_CODEC_ID_MPEG4 ||
+            (st->codecpar->codec_id == AV_CODEC_ID_HEVC && type == PacketTypeCodedFrames)) {
             // sign extension
             int32_t cts = (avio_rb24(s->pb) + 0xff800000) ^ 0xff800000;
             pts = av_sat_add64(dts, cts);
@@ -1276,9 +1313,11 @@ static int flv_read_packet(AVFormatContext *s, AVPacket *pkt)
                        "invalid timestamps %"PRId64" %"PRId64"\n", dts, pts);
                 dts = pts = AV_NOPTS_VALUE;
             }
+            size -= 3;
         }
         if (type == 0 && (!st->codecpar->extradata || st->codecpar->codec_id == AV_CODEC_ID_AAC ||
-            st->codecpar->codec_id == AV_CODEC_ID_H264 || st->codecpar->codec_id == AV_CODEC_ID_HEVC)) {
+            st->codecpar->codec_id == AV_CODEC_ID_H264 || st->codecpar->codec_id == AV_CODEC_ID_HEVC ||
+            st->codecpar->codec_id == AV_CODEC_ID_AV1 || st->codecpar->codec_id == AV_CODEC_ID_VP9)) {
             AVDictionaryEntry *t;
             if (st->codecpar->extradata) {
                 if ((ret = flv_queue_extradata(flv, s->pb, stream_type, size)) < 0)