DVR: support mp3 audio codec. (#2593) v5.0.17

Haibo Chen · winlinvip · commit eac8996e0281 · 2021-10-16T08:16:58.000+08:00
diff --git a/trunk/doc/CHANGELOG.md b/trunk/doc/CHANGELOG.md
@@ -8,6 +8,7 @@ The changelog for SRS.
 
 ## SRS 5.0 Changelog
 
+* v5.0, 2021-10-16, DVR: support mp3 audio codec. (#2593) v5.0.17
 * v5.0, 2021-10-03, OpenWRT: Disable mprotect of ST. 5.0.16
 * v5.0, 2021-10-03, Actions: Create source tar lik srs-server-5.0.14.tar.gz
 * v5.0, 2021-10-02, ST: Support Cygwin64 and MIPS. 5.0.13
diff --git a/trunk/src/app/srs_app_dvr.cpp b/trunk/src/app/srs_app_dvr.cpp
@@ -462,7 +462,7 @@ srs_error_t SrsDvrMp4Segmenter::encode_audio(SrsSharedPtrMessage* audio, SrsForm
     SrsAudioChannels channels = format->acodec->sound_type;
     
     SrsAudioAacFrameTrait ct = format->audio->aac_packet_type;
-    if (ct == SrsAudioAacFrameTraitSequenceHeader) {
+    if (ct == SrsAudioAacFrameTraitSequenceHeader || ct == SrsAudioMp3FrameTrait) {
         enc->acodec = sound_format;
         enc->sample_rate = sound_rate;
         enc->sound_bits = sound_size;
diff --git a/trunk/src/core/srs_core_version5.hpp b/trunk/src/core/srs_core_version5.hpp
@@ -9,6 +9,6 @@
 
 #define VERSION_MAJOR       5
 #define VERSION_MINOR       0
-#define VERSION_REVISION    16
+#define VERSION_REVISION    17
 
 #endif
diff --git a/trunk/src/kernel/srs_kernel_codec.cpp b/trunk/src/kernel/srs_kernel_codec.cpp
@@ -1388,6 +1388,7 @@ srs_error_t SrsFormat::audio_mp3_demux(SrsBuffer* stream, int64_t timestamp)
     
     audio->cts = 0;
     audio->dts = timestamp;
+    audio->aac_packet_type = SrsAudioMp3FrameTrait;
     
     // @see: E.4.2 Audio Tags, video_file_format_spec_v10_1.pdf, page 76
     int8_t sound_format = stream->read_1bytes();
diff --git a/trunk/src/kernel/srs_kernel_codec.hpp b/trunk/src/kernel/srs_kernel_codec.hpp
@@ -167,6 +167,11 @@ enum SrsAudioAacFrameTrait
     SrsAudioOpusFrameTraitRaw = 2,
     SrsAudioOpusFrameTraitSamplingRate = 4,
     SrsAudioOpusFrameTraitAudioLevel = 8,
+
+    // 16/32 reserved for g711a/g711u 
+
+    // For MP3
+    SrsAudioMp3FrameTrait = 64,
 };
 
 /**
diff --git a/trunk/src/kernel/srs_kernel_mp4.cpp b/trunk/src/kernel/srs_kernel_mp4.cpp
@@ -1812,11 +1812,16 @@ SrsAudioCodecId SrsMp4TrackBox::soun_codec()
     if (box->entry_count() == 0) {
         return SrsAudioCodecIdForbidden;
     }
-    
-    SrsMp4SampleEntry* entry = box->entrie_at(0);
-    switch(entry->type) {
-        case SrsMp4BoxTypeMP4A: return SrsAudioCodecIdAAC;
-        default: return SrsAudioCodecIdForbidden;
+
+    SrsMp4EsdsBox* esds_box = mp4a()->esds();
+    switch (esds_box->es->decConfigDescr.objectTypeIndication) {
+    case SrsMp4ObjectTypeAac:
+        return SrsAudioCodecIdAAC;
+    case SrsMp4ObjectTypeMp3:
+    case SrsMp4ObjectTypeMp1a:
+        return SrsAudioCodecIdMP3;
+    default:
+        return SrsAudioCodecIdForbidden;
     }
 }
 
@@ -3429,7 +3434,7 @@ srs_error_t SrsMp4DecoderConfigDescriptor::encode_payload(SrsBuffer* buf)
     buf->write_3bytes(bufferSizeDB);
     buf->write_4bytes(maxBitrate);
     buf->write_4bytes(avgBitrate);
-    
+
     if (decSpecificInfo && (err = decSpecificInfo->encode(buf)) != srs_success) {
         return srs_error_wrap(err, "encode des specific info");
     }
@@ -3440,7 +3445,7 @@ srs_error_t SrsMp4DecoderConfigDescriptor::encode_payload(SrsBuffer* buf)
 srs_error_t SrsMp4DecoderConfigDescriptor::decode_payload(SrsBuffer* buf)
 {
     srs_error_t err = srs_success;
-    
+
     objectTypeIndication = (SrsMp4ObjectType)buf->read_1bytes();
     
     uint8_t v = buf->read_1bytes();
@@ -5488,7 +5493,7 @@ srs_error_t SrsMp4Decoder::parse_moov(SrsMp4MovieBox* moov)
     if (vide && !avcc) {
         return srs_error_new(ERROR_MP4_ILLEGAL_MOOV, "missing video sequence header");
     }
-    if (soun && !asc) {
+    if (soun && !asc && soun->soun_codec() == SrsAudioCodecIdAAC) {
         return srs_error_new(ERROR_MP4_ILLEGAL_MOOV, "missing audio sequence header");
     }
     
@@ -5910,13 +5915,15 @@ srs_error_t SrsMp4Encoder::flush()
             es->ES_ID = 0x02;
             
             SrsMp4DecoderConfigDescriptor& desc = es->decConfigDescr;
-            desc.objectTypeIndication = SrsMp4ObjectTypeAac;
+            desc.objectTypeIndication = get_audio_object_type();
             desc.streamType = SrsMp4StreamTypeAudioStream;
             srs_freep(desc.decSpecificInfo);
             
-            SrsMp4DecoderSpecificInfo* asc = new SrsMp4DecoderSpecificInfo();
-            desc.decSpecificInfo = asc;
-            asc->asc = pasc;;
+            if (SrsMp4ObjectTypeAac == desc.objectTypeIndication) {
+                SrsMp4DecoderSpecificInfo* asc = new SrsMp4DecoderSpecificInfo();
+                desc.decSpecificInfo = asc;
+                asc->asc = pasc;
+            }
         }
         
         if ((err = samples->write(moov)) != srs_success) {
@@ -6045,6 +6052,31 @@ srs_error_t SrsMp4Encoder::do_write_sample(SrsMp4Sample* ps, uint8_t* sample, ui
     return err;
 }
 
+SrsMp4ObjectType SrsMp4Encoder::get_audio_object_type()
+{
+    switch (acodec)
+    {
+    case SrsAudioCodecIdAAC:
+        return SrsMp4ObjectTypeAac;
+    case SrsAudioCodecIdMP3:
+        return (srs_flv_srates[sample_rate] > 24000) ? SrsMp4ObjectTypeMp1a : SrsMp4ObjectTypeMp3;  // 11172 - 3
+    default:
+        return SrsMp4ObjectTypeForbidden;
+    }
+}
+
+SrsMp4ObjectType SrsMp4Encoder::get_audio_object_type()
+{
+    switch (acodec) {
+    case SrsAudioCodecIdAAC:
+        return SrsMp4ObjectTypeAac;
+    case SrsAudioCodecIdMP3:
+        return (srs_flv_srates[sample_rate] > 24000) ? SrsMp4ObjectTypeMp1a : SrsMp4ObjectTypeMp3;  // 11172 - 3
+    default:
+        return SrsMp4ObjectTypeForbidden;
+    }
+}
+
 SrsMp4M2tsInitEncoder::SrsMp4M2tsInitEncoder()
 {
     writer = NULL;
diff --git a/trunk/src/kernel/srs_kernel_mp4.hpp b/trunk/src/kernel/srs_kernel_mp4.hpp
@@ -1369,6 +1369,10 @@ enum SrsMp4ObjectType
     SrsMp4ObjectTypeForbidden = 0x00,
     // Audio ISO/IEC 14496-3
     SrsMp4ObjectTypeAac = 0x40,
+    // Audio ISO/IEC 13818-3
+    SrsMp4ObjectTypeMp3 = 0x69,
+    // Audio ISO/IEC 11172-3
+    SrsMp4ObjectTypeMp1a = 0x6B,
 };
 
 // Table 6 — streamType Values
@@ -2072,6 +2076,7 @@ class SrsMp4Encoder
 private:
     virtual srs_error_t copy_sequence_header(SrsFormat* format, bool vsh, uint8_t* sample, uint32_t nb_sample);
     virtual srs_error_t do_write_sample(SrsMp4Sample* ps, uint8_t* sample, uint32_t nb_sample);
+    virtual SrsMp4ObjectType get_audio_object_type();
 };
 
 // A fMP4 encoder, to write the init.mp4 with sequence header.
diff --git a/trunk/src/utest/srs_utest_kernel.cpp b/trunk/src/utest/srs_utest_kernel.cpp
@@ -5033,6 +5033,8 @@ VOID TEST(KernelMP4Test, CoverMP4CodecSingleFrame)
             ));
         }
 
+        enc.acodec = SrsAudioCodecIdAAC;
+
         HELPER_EXPECT_SUCCESS(enc.flush());
         //mock_print_mp4(string(f.data(), f.filesize()));
     }
@@ -5147,6 +5149,8 @@ VOID TEST(KernelMP4Test, CoverMP4MultipleVideos)
             ));
         }
 
+        enc.acodec = SrsAudioCodecIdAAC;
+
         // Flush encoder.
         HELPER_EXPECT_SUCCESS(enc.flush());
         //mock_print_mp4(string(f.data(), f.filesize()));
@@ -5245,6 +5249,8 @@ VOID TEST(KernelMP4Test, CoverMP4MultipleCTTs)
             ));
         }
 
+        enc.acodec = SrsAudioCodecIdAAC;
+
         // Flush encoder.
         HELPER_EXPECT_SUCCESS(enc.flush());
         //mock_print_mp4(string(f.data(), f.filesize()));
@@ -5357,6 +5363,8 @@ VOID TEST(KernelMP4Test, CoverMP4MultipleAVs)
             ));
         }
 
+        enc.acodec = SrsAudioCodecIdAAC;
+
         // Flush encoder.
         HELPER_EXPECT_SUCCESS(enc.flush());
         //mock_print_mp4(string(f.data(), f.filesize()));
@@ -5397,6 +5405,120 @@ VOID TEST(KernelMP4Test, CoverMP4MultipleAVs)
     }
 }
 
+VOID TEST(KernelMP4Test, CoverMP4MultipleAVsWithMp3)
+{
+    srs_error_t err;
+
+    MockSrsFileWriter f;
+
+    // Encode frames.
+    // V-A V-V
+    if (true) {
+        SrsMp4Encoder enc; SrsFormat fmt;
+        HELPER_EXPECT_SUCCESS(enc.initialize(&f));
+        HELPER_EXPECT_SUCCESS(fmt.initialize());
+
+        // Sequence header, V-A
+        if (true) {
+            uint8_t raw[] = {
+                0x17, 0x00, 0x00, 0x00, 0x00, 0x01, 0x64, 0x00, 0x20, 0xff, 0xe1, 0x00, 0x19, 0x67, 0x64, 0x00, 0x20, 0xac, 0xd9, 0x40, 0xc0, 0x29, 0xb0, 0x11, 0x00, 0x00, 0x03, 0x00, 0x01, 0x00, 0x00, 0x03, 0x00, 0x32, 0x0f, 0x18, 0x31, 0x96, 0x01, 0x00, 0x05, 0x68, 0xeb, 0xec, 0xb2, 0x2c
+            };
+            HELPER_EXPECT_SUCCESS(fmt.on_video(0, (char*)raw, sizeof(raw)));
+            HELPER_EXPECT_SUCCESS(enc.write_sample(
+                &fmt, SrsMp4HandlerTypeVIDE, fmt.video->frame_type, fmt.video->avc_packet_type, 0, 0, (uint8_t*)fmt.raw, fmt.nb_raw
+            ));
+            EXPECT_EQ(768, (int)enc.width); EXPECT_EQ(320, (int)enc.height);
+        }
+
+        if (true) {
+            uint8_t raw[] = {
+                0xaf, 0x00, 0x12, 0x10
+            };
+            HELPER_EXPECT_SUCCESS(fmt.on_audio(0, (char*)raw, sizeof(raw)));
+            HELPER_EXPECT_SUCCESS(enc.write_sample(
+                &fmt, SrsMp4HandlerTypeSOUN, 0x00, fmt.audio->aac_packet_type, 0, 0, (uint8_t*)fmt.raw, fmt.nb_raw
+            ));
+        }
+
+        // Frame group #0, V-A-A-V
+        if (true) {
+            uint8_t raw[] = {
+                0x17, 0x01, 0x00, 0x00, 0x50, 0x00, 0x00, 0x00, 0x7b, 0x41, 0x9a, 0x21, 0x6c, 0x42, 0x1f, 0x00, 0x00, 0xf1, 0x68, 0x1a, 0x35, 0x84, 0xb3, 0xee, 0xe0, 0x61, 0xba, 0x4e, 0xa8, 0x52, 0x48, 0x50, 0x59, 0x75, 0x42, 0xd9, 0x96, 0x4a, 0x51, 0x38, 0x2c, 0x63, 0x5e, 0x41, 0xc9, 0x70, 0x60, 0x9d, 0x13, 0x53, 0xc2, 0xa8, 0xf5, 0x45, 0x86, 0xc5, 0x3e, 0x28, 0x1a, 0x69, 0x5f, 0x71, 0x1e, 0x51, 0x74, 0x0e, 0x31, 0x47, 0x3c, 0xd3, 0xd2, 0x10, 0x25, 0x45, 0xc5, 0xb7, 0x31, 0xec, 0x7f, 0xd8, 0x02, 0xae, 0xa4, 0x77, 0x6d, 0xcb, 0xc6, 0x1e, 0x2f, 0xa2, 0xd1, 0x12, 0x08, 0x34, 0x52, 0xea, 0xe8, 0x0b, 0x4f, 0x81, 0x21, 0x4f, 0x71, 0x3f, 0xf2, 0xad, 0x02, 0x58, 0xdf, 0x9e, 0x31, 0x86, 0x9b, 0x1b, 0x41, 0xbf, 0x2a, 0x09, 0x00, 0x43, 0x5c, 0xa1, 0x7e, 0x76, 0x59, 0xef, 0xa6, 0xfc, 0x82, 0xb2, 0x72, 0x5a
+            };
+            HELPER_EXPECT_SUCCESS(fmt.on_video(0, (char*)raw, sizeof(raw)));
+            HELPER_EXPECT_SUCCESS(enc.write_sample(
+                &fmt, SrsMp4HandlerTypeVIDE, fmt.video->frame_type, fmt.video->avc_packet_type, 0, 0, (uint8_t*)fmt.raw, fmt.nb_raw
+            ));
+        }
+
+        if (true) {
+            uint8_t raw[] = {
+                0xaf, 0x01, 0x21, 0x11, 0x45, 0x00, 0x14, 0x50, 0x01, 0x46, 0xf3, 0xf1, 0x0a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5e
+            };
+            HELPER_EXPECT_SUCCESS(fmt.on_audio(0, (char*)raw, sizeof(raw)));
+            HELPER_EXPECT_SUCCESS(enc.write_sample(
+                &fmt, SrsMp4HandlerTypeSOUN, 0x00, fmt.audio->aac_packet_type, 0, 0, (uint8_t*)fmt.raw, fmt.nb_raw
+            ));
+        }
+
+        if (true) {
+            uint8_t raw[] = {
+                0xaf, 0x01, 0x21, 0x11, 0x45, 0x00, 0x14, 0x50, 0x01, 0x46, 0xf3, 0xf1, 0x0a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5a, 0x5e
+            };
+            HELPER_EXPECT_SUCCESS(fmt.on_audio(0, (char*)raw, sizeof(raw)));
+            HELPER_EXPECT_SUCCESS(enc.write_sample(
+                &fmt, SrsMp4HandlerTypeSOUN, 0x00, fmt.audio->aac_packet_type, 20, 20, (uint8_t*)fmt.raw, fmt.nb_raw
+            ));
+        }
+
+        if (true) {
+            uint8_t raw[] = {
+                0x27, 0x01, 0x00, 0x00, 0x50, 0x00, 0x00, 0x00, 0x7b, 0x41, 0x9a, 0x21, 0x6c, 0x42, 0x1f, 0x00, 0x00, 0xf1, 0x68, 0x1a, 0x35, 0x84, 0xb3, 0xee, 0xe0, 0x61, 0xba, 0x4e, 0xa8, 0x52, 0x48, 0x50, 0x59, 0x75, 0x42, 0xd9, 0x96, 0x4a, 0x51, 0x38, 0x2c, 0x63, 0x5e, 0x41, 0xc9, 0x70, 0x60, 0x9d, 0x13, 0x53, 0xc2, 0xa8, 0xf5, 0x45, 0x86, 0xc5, 0x3e, 0x28, 0x1a, 0x69, 0x5f, 0x71, 0x1e, 0x51, 0x74, 0x0e, 0x31, 0x47, 0x3c, 0xd3, 0xd2, 0x10, 0x25, 0x45, 0xc5, 0xb7, 0x31, 0xec, 0x7f, 0xd8, 0x02, 0xae, 0xa4, 0x77, 0x6d, 0xcb, 0xc6, 0x1e, 0x2f, 0xa2, 0xd1, 0x12, 0x08, 0x34, 0x52, 0xea, 0xe8, 0x0b, 0x4f, 0x81, 0x21, 0x4f, 0x71, 0x3f, 0xf2, 0xad, 0x02, 0x58, 0xdf, 0x9e, 0x31, 0x86, 0x9b, 0x1b, 0x41, 0xbf, 0x2a, 0x09, 0x00, 0x43, 0x5c, 0xa1, 0x7e, 0x76, 0x59, 0xef, 0xa6, 0xfc, 0x82, 0xb2, 0x72, 0x5a
+            };
+            HELPER_EXPECT_SUCCESS(fmt.on_video(0, (char*)raw, sizeof(raw)));
+            HELPER_EXPECT_SUCCESS(enc.write_sample(
+                &fmt, SrsMp4HandlerTypeVIDE, fmt.video->frame_type, fmt.video->avc_packet_type, 40, 40, (uint8_t*)fmt.raw, fmt.nb_raw
+            ));
+        }
+
+        enc.acodec = SrsAudioCodecIdMP3;
+
+        // Flush encoder.
+        HELPER_EXPECT_SUCCESS(enc.flush());
+        //mock_print_mp4(string(f.data(), f.filesize()));
+    }
+
+    // Decode frames.
+    if (true) {
+        MockSrsFileReader fr((const char*)f.data(), f.filesize());
+        SrsMp4Decoder dec; HELPER_EXPECT_SUCCESS(dec.initialize(&fr));
+
+        SrsMp4HandlerType ht; uint16_t ft, ct; uint32_t dts, pts, nb_sample; uint8_t* sample = NULL;
+
+        // Sequence header.
+        HELPER_EXPECT_SUCCESS(dec.read_sample(&ht, &ft, &ct, &dts, &pts, &sample, &nb_sample));
+        EXPECT_EQ(0, (int)dts); EXPECT_EQ(41, (int)nb_sample); EXPECT_EQ(SrsMp4HandlerTypeVIDE, ht); EXPECT_EQ(SrsVideoAvcFrameTraitSequenceHeader, ct);
+        srs_freepa(sample);
+
+        // Frames order by dts asc.
+        HELPER_EXPECT_SUCCESS(dec.read_sample(&ht, &ft, &ct, &dts, &pts, &sample, &nb_sample));
+        EXPECT_EQ(0, (int)dts); EXPECT_EQ(127, (int)nb_sample); EXPECT_EQ(SrsMp4HandlerTypeVIDE, ht); EXPECT_NE(SrsAudioMp3FrameTrait, ct);
+        srs_freepa(sample);
+
+        HELPER_EXPECT_SUCCESS(dec.read_sample(&ht, &ft, &ct, &dts, &pts, &sample, &nb_sample));
+        EXPECT_EQ(0, (int)dts); EXPECT_EQ(87, (int)nb_sample); EXPECT_EQ(SrsMp4HandlerTypeSOUN, ht); EXPECT_NE(SrsVideoAvcFrameTraitSequenceHeader, ct);
+        srs_freepa(sample);
+
+        HELPER_EXPECT_SUCCESS(dec.read_sample(&ht, &ft, &ct, &dts, &pts, &sample, &nb_sample));
+        EXPECT_EQ(20, (int)dts); EXPECT_EQ(87, (int)nb_sample); EXPECT_EQ(SrsMp4HandlerTypeSOUN, ht); EXPECT_NE(SrsVideoAvcFrameTraitSequenceHeader, ct);
+        srs_freepa(sample);
+
+        HELPER_EXPECT_SUCCESS(dec.read_sample(&ht, &ft, &ct, &dts, &pts, &sample, &nb_sample));
+        EXPECT_EQ(40, (int)dts); EXPECT_EQ(40, (int)pts); EXPECT_EQ(127, (int)nb_sample); EXPECT_EQ(SrsMp4HandlerTypeVIDE, ht); EXPECT_NE(SrsAudioMp3FrameTrait, ct);
+        srs_freepa(sample);
+    }
+}
+
 VOID TEST(KernelMP4Test, CoverMP4CodecErrorNoFrames)
 {
 	srs_error_t err;
