Bugfix: HEVC SRT stream supports multiple PPS fields. v6.0.76 (#3722)

When the srs have multiple pps in hevc.the srs can't parse for this.
problem fixed this #3604

---------

Co-authored-by: chundonglinlin <[email protected]>
Co-authored-by: john <[email protected]>

Author: 3 people

trunk/doc/CHANGELOG.md

@@ -7,6 +7,7 @@ The changelog for SRS.
 <a name="v6-changes"></a>
 
 ## SRS 6.0 Changelog
+* v6.0, 2023-09-18, Merge [#3722](https://github.com/ossrs/srs/pull/3722): Bugfix: HEVC SRT stream supports multiple PPS fields. v6.0.76 (#3722)
 * v6.0, 2023-09-08, Merge [#3597](https://github.com/ossrs/srs/pull/3597): Fix RBSP stream parsing bug, should drop 0x03. v6.0.75 (#3597)
 * v6.0, 2023-09-08, Merge [#3794](https://github.com/ossrs/srs/pull/3794): Support SRS Stack token for authentication. v6.0.74 (#3794)
 * v6.0, 2023-09-07, Merge [#3795](https://github.com/ossrs/srs/pull/3795): Fix dash crash if format not supported. v6.0.73 (#3795)

trunk/src/app/srs_app_gb28181.cpp

@@ -1905,7 +1905,8 @@ srs_error_t SrsGbMuxer::write_h265_vps_sps_pps(uint32_t dts, uint32_t pts)
     }
 
     std::string sh;
-    if ((err = hevc_->mux_sequence_header(h265_vps_, h265_sps_, h265_pps_, sh)) != srs_success) {
+    std::vector<string> h265_pps = { h265_pps_ };
+    if ((err = hevc_->mux_sequence_header(h265_vps_, h265_sps_, h265_pps, sh)) != srs_success) {
         return srs_error_wrap(err, "hevc mux sequence header");
     }
 

trunk/src/app/srs_app_srt_source.cpp

@@ -538,6 +538,7 @@ srs_error_t SrsSrtFrameBuilder::on_ts_video_hevc(SrsTsMessage *msg, SrsBuffer *a
     SrsRawHEVCStream *hevc = new SrsRawHEVCStream();
     SrsAutoFree(SrsRawHEVCStream, hevc);
 
+    std::vector<std::string> hevc_pps;
     // send each frame.
     while (!avs->empty()) {
         char* frame = NULL;
@@ -587,17 +588,21 @@ srs_error_t SrsSrtFrameBuilder::on_ts_video_hevc(SrsTsMessage *msg, SrsBuffer *a
                 return srs_error_wrap(err, "demux pps");
             }
 
-            if (! pps.empty() && hevc_pps_ != pps) {
+            if (!pps.empty()) {
                 vps_sps_pps_change_ = true;
             }
 
-            hevc_pps_ = pps;
+            hevc_pps.push_back(pps);
             continue;
         }
 
         ipb_frames.push_back(make_pair(frame, frame_size));
     }
 
+    if (!hevc_pps.empty()) {
+        hevc_pps_ = hevc_pps;
+    }
+
     if ((err = check_vps_sps_pps_change(msg)) != srs_success) {
         return srs_error_wrap(err, "check vps sps pps");
     }
@@ -614,7 +619,7 @@ srs_error_t SrsSrtFrameBuilder::check_vps_sps_pps_change(SrsTsMessage* msg)
     }
 
     if (hevc_vps_.empty() || hevc_sps_.empty() || hevc_pps_.empty()) {
-        return srs_error_new(ERROR_SRT_TO_RTMP_EMPTY_SPS_PPS, "vps or sps or pps empty");
+        return err;
     }
 
     // vps/sps/pps changed, generate new video sh frame and dispatch it.
@@ -662,7 +667,7 @@ srs_error_t SrsSrtFrameBuilder::on_hevc_frame(SrsTsMessage* msg, vector<pair<cha
     srs_error_t err = srs_success;
 
     if (ipb_frames.empty()) {
-        return srs_error_new(ERROR_SRT_CONN, "empty frame");
+        return err;
     }
 
     // ts tbn to flv tbn.

trunk/src/app/srs_app_srt_source.hpp

@@ -131,7 +131,7 @@ class SrsSrtFrameBuilder : public ISrsTsHandler
     bool vps_sps_pps_change_;
     std::string hevc_vps_;
     std::string hevc_sps_;
-    std::string hevc_pps_;
+    std::vector<std::string> hevc_pps_;
 #endif
     // Record audio sepcific config had changed, if change, need to generate new audio sh frame.
     bool audio_sh_change_;

trunk/src/core/srs_core_version6.hpp

@@ -9,6 +9,6 @@
 
 #define VERSION_MAJOR       6
 #define VERSION_MINOR       0
-#define VERSION_REVISION    75
+#define VERSION_REVISION    76
 
 #endif

trunk/src/kernel/srs_kernel_ts.cpp

@@ -1952,8 +1952,13 @@ srs_error_t SrsTsPayloadPES::decode(SrsBuffer* stream, SrsTsMessage** ppmsg)
     // check when fresh, the payload_unit_start_indicator
     // should be 1 for the fresh msg.
     if (is_fresh_msg && !packet->payload_unit_start_indicator) {
-        return srs_error_new(ERROR_STREAM_CASTER_TS_PSE, "ts: PES fresh packet length=%d, us=%d, cc=%d",
+        srs_warn("ts: PES fresh packet length=%d, us=%d, cc=%d",
             msg->PES_packet_length, packet->payload_unit_start_indicator, packet->continuity_counter);
+
+        stream->skip(stream->size() - stream->pos());
+        srs_freep(msg);
+        channel->msg = NULL;
+        return err;
     }
 
     // check when not fresh and PES_packet_length>0,

trunk/src/protocol/srs_protocol_raw_avc.cpp

@@ -382,7 +382,7 @@ srs_error_t SrsRawHEVCStream::pps_demux(char *frame, int nb_frame, std::string &
     return err;
 }
 
-srs_error_t SrsRawHEVCStream::mux_sequence_header(std::string vps, std::string sps, std::string pps, std::string &hvcC)
+srs_error_t SrsRawHEVCStream::mux_sequence_header(std::string vps, std::string sps, std::vector<std::string>& pps, std::string &hvcC)
 {
     srs_error_t err = srs_success;
 
@@ -400,8 +400,13 @@ srs_error_t SrsRawHEVCStream::mux_sequence_header(std::string vps, std::string s
     //      sequenceParameterSetNALUnit
 
     // use simple mode: nalu size + nalu data
-    int nb_packet = 23 + 5 + (int)vps.length() + 5 + (int)sps.length() + 5 + (int)pps.length();
-    char *packet = new char[nb_packet];
+    int pps_size = 0;
+    for (std::vector<std::string>::iterator it = pps.begin(); it != pps.end(); it++) {
+        pps_size += 2 + it->length();
+    }
+
+    int nb_packet = 23 + 5 + (int)vps.length() + 5 + (int)sps.length() + 5 + pps_size - 2;
+    char* packet = new char[nb_packet];
     SrsAutoFreeA(char, packet);
 
     // use stream to generate the hevc packet.
@@ -495,12 +500,15 @@ srs_error_t SrsRawHEVCStream::mux_sequence_header(std::string vps, std::string s
     if (true) {
         // nal_type
         stream.write_1bytes(SrsHevcNaluType_PPS & 0x3f);
-        // numOfPictureParameterSets, always 1
-        stream.write_2bytes(0x01);
-        // pictureParameterSetLength
-        stream.write_2bytes((int16_t)pps.length());
-        // pictureParameterSetNALUnit
-        stream.write_string(pps);
+        // numOfPictureParameterSets
+        stream.write_2bytes(pps.size());
+
+        for (std::vector<std::string>::iterator it = pps.begin(); it != pps.end(); it++) {
+            //pictureParameterSetLength
+            stream.write_2bytes((int16_t)it->length());
+            //pictureParameterSetNALUnit
+            stream.write_string(*it);
+        }
     }
 
     hvcC = string(packet, nb_packet);

trunk/src/protocol/srs_protocol_raw_avc.hpp

@@ -82,7 +82,7 @@ class SrsRawHEVCStream
     // The hevc raw data to hevc packet, without flv payload header.
     // Mux the sps/pps/vps to flv sequence header packet.
     // @param sh output the sequence header.
-    virtual srs_error_t mux_sequence_header(std::string vps, std::string sps, std::string pps, std::string &sh);
+    virtual srs_error_t mux_sequence_header(std::string vps, std::string sps, std::vector<std::string>& pps, std::string& sh);
     // The hevc raw data to hevc packet, without flv payload header.
     // Mux the ibp to flv ibp packet.
     // @param ibp output the packet.

trunk/src/utest/srs_utest_avc.cpp

@@ -9,6 +9,7 @@
 #include <srs_kernel_buffer.hpp>
 #include <srs_kernel_error.hpp>
 #include <srs_core_autofree.hpp>
+#include <srs_kernel_utility.hpp>
 
 VOID TEST(SrsAVCTest, H264ParseAnnexb)
 {
@@ -599,3 +600,91 @@ VOID TEST(SrsAVCTest, AACMuxToFLV)
     }
 }
 
+#ifdef SRS_H265
+
+VOID TEST(SrsAVCTest, HevcMultiPPS)
+{
+    srs_error_t err;
+
+    vector<uint8_t> vps = {
+        0x40, 0x01, 0x0c, 0x06, 0x3f, 0x3f, 0x22, 0x20, 0x00, 0x00, 0x03, 0x00, 0x3f,
+        0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x3f, 0x00, 0x00, 0x18, 0x3f, 0x24,
+    };
+
+    vector<uint8_t> sps = {
+        0x42, 0x01, 0x06, 0x22, 0x20, 0x00, 0x00, 0x03, 0x00, 0x3f, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00,
+        0x3f, 0x00, 0x00, 0x3f, 0x01, 0x3f, 0x20, 0x02, 0x1c, 0x4d, 0x3f, 0x3f, 0x3f, 0x42, 0x3f, 0x53, 0x3f,
+        0x3f, 0x01, 0x01, 0x01, 0x04, 0x00, 0x00, 0x0f, 0x3f, 0x00, 0x01, 0x3f, 0x3f, 0x3f, 0x68, 0x3f, 0x3f,
+        0x00, 0x1f, 0x3a, 0x00, 0x0f, 0x3f, 0x04, 0x00, 0x3e, 0x74, 0x00, 0x1f, 0x3a, 0x08, 0x00, 0x7c, 0x3f,
+        0x00, 0x3e, 0x74, 0x10, 0x00, 0x3f, 0x3f, 0x00, 0x7c, 0x3f, 0x5c, 0x20, 0x10, 0x40
+    };
+
+    vector<uint8_t> pps_1 = {
+        0x44, 0x01, 0x74, 0x18, 0xc2, 0xb8, 0x33, 0x3f, 0x7d, 0x75, 0x3f, 0x42, 0x28, 0x3f, 0x6b, 0x3f, 0x3f,
+        0x11, 0x47, 0x7d, 0x72, 0x79, 0x3e, 0x4f, 0x3f, 0x3f, 0x51, 0x3f, 0x3f, 0x20, 0x3f, 0x3f, 0x3f, 0x10,
+        0x63, 0x3f, 0x0a, 0x0e, 0x3f, 0x04, 0x42, 0x3f, 0x3f, 0x3f, 0x34, 0x22, 0x3f, 0x3f, 0x3f, 0x3f, 0x7d,
+        0x7e, 0x4f, 0x3f, 0x3f, 0x7c, 0x57, 0x3f, 0x3f, 0x3f, 0x10, 0x41, 0x21, 0x14, 0x41, 0x3f, 0x3f, 0x3c,
+        0x3f, 0x5f, 0x3f, 0x3f, 0x28, 0x3f, 0x73, 0x10, 0x44, 0x49, 0x47, 0x08, 0x31, 0xc4, 0x85, 0x06, 0x46,
+        0x3f, 0x21, 0x02, 0x41, 0x54, 0x1a, 0x11, 0x45, 0x13, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x11,
+        0x3f, 0x5e, 0x3b, 0x3f, 0x30, 0x41, 0x04, 0x3f, 0x51, 0x06, 0x3f, 0x3f, 0x3f, 0x7d, 0x7e, 0x4f, 0x3f,
+        0x3f, 0x1d, 0x3f, 0x41, 0x11, 0x25, 0x1c, 0x20, 0x3f, 0x12, 0x14, 0x19, 0x1a, 0x08, 0x3f, 0x09, 0x05,
+        0x50, 0x69, 0x14, 0x4f, 0x3f, 0x4f, 0x27, 0x3f, 0x3f, 0x3f, 0x28, 0x3f, 0x73, 0x10, 0xd3, 0x94, 0x71,
+        0x3f, 0x73, 0x3f, 0x05, 0x45, 0x3f, 0x01, 0x02, 0x41, 0x54, 0x18, 0x24
+    };
+
+    vector<uint8_t> pps_2 = {
+        0x44, 0x01, 0x25, 0x06, 0x30, 0x3f, 0x0c, 0x3f, 0x4a, 0x3f, 0x3f, 0x49, 0x08, 0x3f, 0x15, 0x6b, 0x3f,
+        0x3f, 0x11, 0x4c, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x46, 0x3b, 0x3f, 0x3f, 0x22, 0x4a,
+        0x30, 0x51, 0x3f, 0x2c, 0x2c, 0x32, 0x34, 0x11, 0x08, 0x12, 0x0a, 0x3f, 0xd0, 0x8a, 0x29, 0x56, 0x3f,
+        0x3c, 0x3f, 0x5f, 0x3f, 0x3f, 0x3f, 0x1f, 0x15, 0x3f, 0x3f, 0x63, 0x04, 0x10, 0x4c, 0x45, 0x35, 0x49,
+        0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x77, 0x31, 0x04, 0x44, 0x3f, 0x60, 0x3f, 0x1c, 0x58,
+        0x58, 0x64, 0x68, 0x22, 0x10, 0x24, 0x15, 0x41, 0x3f, 0x14, 0x42, 0x55, 0x3f, 0x4f, 0x27, 0x3f, 0x3f,
+        0x3f, 0x3f, 0x47, 0x3f, 0x78, 0x3f, 0x18, 0x3f, 0x04, 0x13, 0x11, 0x4d, 0x52, 0x64, 0x3f, 0x3f, 0x3f,
+        0x7e, 0x4f, 0x3f, 0x3f, 0x1d, 0x3f, 0x41, 0x11, 0x25, 0x18, 0x28, 0x3f, 0x16, 0x16, 0x19, 0x1a, 0x08,
+        0x3f, 0x09, 0x05, 0x50, 0x69, 0x10, 0x3f, 0x6b, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x11, 0x3f, 0x5e,
+        0x3b, 0x3f, 0x30, 0x41, 0x04, 0x3f
+    };
+
+    vector<uint8_t> start_code = {0x00, 0x00, 0x00, 0x01};
+
+    string hevc_sh;
+    hevc_sh.append((const char*)start_code.data(), start_code.size());
+    hevc_sh.append((const char*)vps.data(), vps.size());
+    hevc_sh.append((const char*)start_code.data(), start_code.size());
+    hevc_sh.append((const char*)sps.data(), sps.size());
+    hevc_sh.append((const char*)start_code.data(), start_code.size());
+    hevc_sh.append((const char*)pps_1.data(), pps_1.size());
+    hevc_sh.append((const char*)start_code.data(), start_code.size());
+    hevc_sh.append((const char*)pps_2.data(), pps_2.size());
+
+    SrsBuffer stream((char*)hevc_sh.data(), hevc_sh.size());
+
+    SrsRawHEVCStream hs;
+    char* frame = NULL;
+    int frame_size = 0;
+
+    HELPER_ASSERT_SUCCESS(hs.annexb_demux(&stream, &frame, &frame_size));
+    EXPECT_TRUE(hs.is_vps(frame, frame_size));
+    EXPECT_EQ(frame_size, vps.size());
+    EXPECT_TRUE(srs_bytes_equals(frame, vps.data(), frame_size));
+
+    HELPER_ASSERT_SUCCESS(hs.annexb_demux(&stream, &frame, &frame_size));
+    EXPECT_TRUE(hs.is_sps(frame, frame_size));
+    EXPECT_EQ(frame_size, sps.size());
+    EXPECT_TRUE(srs_bytes_equals(frame, sps.data(), frame_size));
+
+    HELPER_ASSERT_SUCCESS(hs.annexb_demux(&stream, &frame, &frame_size));
+    EXPECT_TRUE(hs.is_pps(frame, frame_size));
+    EXPECT_EQ(frame_size, pps_1.size());
+    EXPECT_TRUE(srs_bytes_equals(frame, pps_1.data(), frame_size));
+
+    HELPER_ASSERT_SUCCESS(hs.annexb_demux(&stream, &frame, &frame_size));
+    EXPECT_TRUE(hs.is_pps(frame, frame_size));
+    EXPECT_EQ(frame_size, pps_2.size());
+    EXPECT_TRUE(srs_bytes_equals(frame, pps_2.data(), frame_size));
+
+    EXPECT_TRUE(stream.empty());
+}
+
+#endif
+