Skip to content

拆分音视频帧

读取本地mp4文件，抽取h264帧送外部解码器，抽取音频帧使用ffmpeg的解码器解成pcm

C++
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
extern "C" {
#include "libavcodec/avcodec.h"
#include "libavformat/avformat.h"
#include "libavutil/avutil.h"
#include "libswresample/swresample.h"
}

// 没有startcode的h264需要添加
static int alloc_and_copy(AVPacket *outPacket, const uint8_t *spspps, uint32_t spsppsSize, const uint8_t *in,
                          uint32_t size)
{
    int err;
    int startCodeLen = 3;

    err = av_grow_packet(outPacket, spsppsSize + size + startCodeLen);
    if (err < 0) {
        return err;
    }
    if (spspps) {
        memcpy(outPacket->data, spspps, spsppsSize);
    }

    (outPacket->data + spsppsSize)[0] = 0;
    (outPacket->data + spsppsSize)[1] = 0;
    (outPacket->data + spsppsSize)[2] = 1;
    memcpy(outPacket->data + spsppsSize + startCodeLen, in, size);
    return 0;
}
// 解码必须sps、pps的信息
static int h264_extradata_to_annexb(const unsigned char *extraData, const int extraDataSize, AVPacket *outPacket,
                                    int padding)
{
    const unsigned char *pExtraData = nullptr;
    int len = 0;
    int spsUnitNum, ppsUnitNum;
    int unitSize, totalSize = 0;
    unsigned char startCode[] = {0, 0, 0, 1};
    unsigned char *pOut = nullptr;
    int err;

    pExtraData = extraData + 4;
    len = (*pExtraData++ & 0x3) + 1;

    // sps
    spsUnitNum = (*pExtraData++ & 0x1f);
    while (spsUnitNum--) {
        unitSize = (pExtraData[0] << 8 | pExtraData[1]);
        pExtraData += 2;
        totalSize += unitSize + sizeof(startCode);

        if (totalSize > INT_MAX - padding) {
            av_free(pOut);
            return AVERROR(EINVAL);
        }
        if (pExtraData + unitSize > extraData + extraDataSize) {
            av_free(pOut);
            return AVERROR(EINVAL);
        }
        if ((err = av_reallocp(&pOut, totalSize + padding)) < 0) {
            return err;
        }
        memcpy(pOut + totalSize - unitSize - sizeof(startCode), startCode, sizeof(startCode));
        memcpy(pOut + totalSize - unitSize, pExtraData, unitSize);

        pExtraData += unitSize;
    }

    // pps
    ppsUnitNum = (*pExtraData++ & 0x1f);
    while (ppsUnitNum--) {
        unitSize = (pExtraData[0] << 8 | pExtraData[1]);
        pExtraData += 2;
        totalSize += unitSize + sizeof(startCode);

        if (totalSize > INT_MAX - padding) {
            av_free(pOut);
            return AVERROR(EINVAL);
        }
        if (pExtraData + unitSize > extraData + extraDataSize) {
            av_free(pOut);
            return AVERROR(EINVAL);
        }
        if ((err = av_reallocp(&pOut, totalSize + padding)) < 0) {
            return err;
        }
        memcpy(pOut + totalSize - unitSize - sizeof(startCode), startCode, sizeof(startCode));
        memcpy(pOut + totalSize - unitSize, pExtraData, unitSize);

        pExtraData += unitSize;
    }
    outPacket->data = pOut;
    outPacket->size = totalSize;

    return len;
}

Version 1.0

1.0版本，基础MP4音视频读取，音频解码成pcm，视频抽出h264帧并添加sps、pps、startcode

C++
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
void WriteData()
{
    while (isRunning_) {
        AVFormatContext *inputContext = avformat_alloc_context();
        std::string inputFile = "/data/360p.mp4";
        // std::string outFile = "/data/720p.pcm";
        // FILE *fp = fopen(outFile.c_str(), "wb");
        AVPacket packet;
        AVCodec *codec = nullptr;
        AVCodecContext *codecContext = nullptr;
        // AVCodecParserContext* parseContext = nullptr;
        int video_stream = 0;
        int audio_stream = 0;
        LOGD("write file in");
        int ret = 0;
        if ((ret = avformat_open_input(&inputContext, inputFile.c_str(), nullptr, nullptr)) != 0) {
            LOGE("ffmpeg open file failed, %s", av_err2str(ret));
            break;
        }
        ret = avformat_find_stream_info(inputContext, nullptr);
        // 寻找音视频流
        video_stream = av_find_best_stream(inputContext, AVMEDIA_TYPE_VIDEO, -1, -1, nullptr, 0);
        if (video_stream < 0) {
            LOGE("find video stream error");
            avformat_close_input(&inputContext);
            break;
        }
        audio_stream = av_find_best_stream(inputContext, AVMEDIA_TYPE_AUDIO, -1, -1, nullptr, 0);
        if (audio_stream < 0) {
            LOGE("find video stream error");
            avformat_close_input(&inputContext);
            break;
        }
        av_dump_format(inputContext, audio_stream, inputFile.c_str(), 0);
        av_dump_format(inputContext, video_stream, inputFile.c_str(), 0);
        // 初始化音频解码器
        codec = avcodec_find_decoder(inputContext->streams[audio_stream]->codecpar->codec_id);
        codecContext = avcodec_alloc_context3(codec);
        avcodec_parameters_to_context(codecContext, inputContext->streams[audio_stream]->codecpar);
        if (avcodec_open2(codecContext, codec, nullptr) < 0) {
            LOGE("open codec error");
            avformat_close_input(&inputContext);
            break;
        }

        av_init_packet(&packet);
        packet.data = nullptr;
        packet.size = 0;
        struct SwrContext *convertContext = swr_alloc();
        AVSampleFormat in_sample_fmt = codecContext->sample_fmt;
        AVSampleFormat out_sample_fmt = AV_SAMPLE_FMT_S16;
        int in_sample_rate = codecContext->sample_rate;
        int out_sample_rate = 48000;
        uint64_t in_channel_layout = codecContext->channel_layout;
        uint64_t out_channel_layout = AV_CH_LAYOUT_MONO;

        convertContext = swr_alloc_set_opts(convertContext, out_channel_layout, out_sample_fmt, out_sample_rate,
                                            in_channel_layout, in_sample_fmt, in_sample_rate, 0, nullptr);
        swr_init(convertContext);
        int out_channel_nb = av_get_channel_layout_nb_channels(out_channel_layout);
        (void)out_channel_nb;
        uint8_t *buffer = (uint8_t *)av_malloc(2 * 48000);

        LOGD("write file start");
        while (av_read_frame(inputContext, &packet) >= 0) {
            if (audio_stream == packet.stream_index) {
                // 音频数据
                AVFrame *frame = av_frame_alloc();
                ret = avcodec_send_packet(codecContext, &packet);
                while (ret >= 0) {
                    ret = avcodec_receive_frame(codecContext, frame);

                    if (ret < 0) {
                        break;
                    }

                    swr_convert(convertContext, &buffer, 2 * 48000, (const uint8_t **)frame->data,
                                frame->nb_samples);
                    int totalSize =
                        av_samples_get_buffer_size(nullptr, out_channel_nb, frame->nb_samples, out_sample_fmt, 0);
                    (void)totalSize;

                    fwrite(buffer, 1, totalSize, fp);
                    ++ts_;
                    ++writeAcount_;
                    av_frame_unref(frame);
                }
            } else if (video_stream == packet.stream_index) {
                unsigned char *pData = packet.data; // frame data
                unsigned char *pEnd = nullptr;
                int dataSize = packet.size;
                int curSize = 0;
                int naluSize = 0;
                unsigned char nalHeader, nalType;
                AVPacket *outPacket = av_packet_alloc();
                AVPacket spsppsPacket;
                spsppsPacket.data = nullptr;
                spsppsPacket.size = 0;
                pEnd = pData + dataSize;
                while (curSize < dataSize) {
                    // 这个循环一般只运行一次
                    if (pEnd - pData < 4) {
                        break;
                    }
                    for (int i = 0; i < 4; ++i) {
                        naluSize <<= 8;
                        naluSize |= pData[i];
                    }
                    pData += 4;
                    if (naluSize > (pEnd - pData + 1) || naluSize <= 0) {
                        break;
                    }
                    nalHeader = *pData;
                    nalType = nalHeader & 0x1F;
                    if (nalType == 5) {
                        // key frame
                        // 处理sps、pps
                        h264_extradata_to_annexb(
                            inputContext->streams[packet.stream_index]->codecpar->extradata,
                            inputContext->streams[packet.stream_index]->codecpar->extradata_size, &spsppsPacket,
                            AV_INPUT_BUFFER_PADDING_SIZE);
                        if (alloc_and_copy(outPacket, spsppsPacket.data, spsppsPacket.size, pData, naluSize) < 0) {
                            av_packet_free(&outPacket);
                            if (spsppsPacket.data) {
                                free(spsppsPacket.data);
                                spsppsPacket.data = nullptr;
                            }
                            break;
                        }
                    } else {
                        if (alloc_and_copy(outPacket, nullptr, 0, pData, naluSize) < 0) {
                            av_packet_free(&outPacket);
                            if (spsppsPacket.data) {
                                free(spsppsPacket.data);
                                spsppsPacket.data = nullptr;
                            }
                            break;
                        }
                    }
                }
                LOGD("nalType: %u, pts: %u", nalType, outPacket->pts);
                ++ts_;
                ++writeVcount_;
                av_packet_unref(outPacket);
            }

            av_packet_unref(&packet);
            // TODO: cal sleep time
            usleep(15 * 1000);
        }
        LOGD("write file done, write audio: %u, write video: %u", writeAcount_,
                        writeVcount_);
        free(buffer);
        avcodec_close(codecContext);
        avformat_close_input(&inputContext);
        // fclose(fp);
        break;
    }
}

Version 2.0

2.0 增加循环读取文件但不重新open的功能；支持单音频、单视频

C++
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
void WriteData()
{
    while (isRunning_) {
        AVFormatContext *inputContext = avformat_alloc_context();
        std::string inputFile = "/data/360p.mp4";
        // std::string outFile = "/data/ffmpeg.txt";
        // FILE *fp = fopen(outFile.c_str(), "wb");
        AVPacket packet;
        AVCodec *codec = nullptr;
        AVCodecContext *codecContext = nullptr;
        struct SwrContext *convertContext = nullptr;
        AVSampleFormat in_sample_fmt;
        AVSampleFormat out_sample_fmt = AV_SAMPLE_FMT_S16;
        int in_sample_rate;
        int out_sample_rate = 48000;
        uint64_t in_channel_layout ;
        uint64_t out_channel_layout = AV_CH_LAYOUT_MONO;
        int out_channel_nb = 0;
        uint8_t *buffer = nullptr;
        int video_stream = -1;
        int audio_stream = -1;
        LOGD("write file in");
        int ret = 0;
        if ((ret = avformat_open_input(&inputContext, inputFile.c_str(), nullptr, nullptr)) != 0) {
            LOGE("ffmpeg open file failed, %s", av_err2str(ret));
            break;
        }
        ret = avformat_find_stream_info(inputContext, nullptr);

        video_stream = av_find_best_stream(inputContext, AVMEDIA_TYPE_VIDEO, -1, -1, nullptr, 0);
        audio_stream = av_find_best_stream(inputContext, AVMEDIA_TYPE_AUDIO, -1, -1, nullptr, 0);
        if (video_stream < 0 && audio_stream < 0) {
            LOGE("find video && audio stream error");
            avformat_close_input(&inputContext);
            break;
        }
        if (audio_stream >= 0) {
            av_dump_format(inputContext, audio_stream, inputFile.c_str(), 0);
            codec = avcodec_find_decoder(inputContext->streams[audio_stream]->codecpar->codec_id);
            codecContext = avcodec_alloc_context3(codec);
            avcodec_parameters_to_context(codecContext, inputContext->streams[audio_stream]->codecpar);
            if (avcodec_open2(codecContext, codec, nullptr) < 0) {
                LOGE("open codec error");
                avformat_close_input(&inputContext);
                break;
            }
            convertContext = swr_alloc();
            in_sample_fmt = codecContext->sample_fmt;
            in_sample_rate = codecContext->sample_rate;
            in_channel_layout = codecContext->channel_layout;
            convertContext = swr_alloc_set_opts(convertContext, out_channel_layout, out_sample_fmt, out_sample_rate,
                                                in_channel_layout, in_sample_fmt, in_sample_rate, 0, nullptr);
            swr_init(convertContext);
            out_channel_nb = av_get_channel_layout_nb_channels(out_channel_layout);
            buffer = (uint8_t *)av_malloc(2 * 48000);
        }
        if (video_stream >= 0) {
            av_dump_format(inputContext, video_stream, inputFile.c_str(), 0);
        }

        av_init_packet(&packet);
        packet.data = nullptr;
        packet.size = 0;
        bool reading = true;
        LOGD("write file start");
        while (reading) {
            int error = av_read_frame(inputContext, &packet);
            if (error == AVERROR_EOF) {
                // seek to start
                auto stream = inputContext->streams[video_stream];
                avio_seek(inputContext->pb, 0, SEEK_SET); // io context set to 0
                avformat_seek_file(inputContext, video_stream, 0, 0, stream->duration, 0);
                continue;
            } else if (error < 0) {
                break;
            }
            if (audio_stream == packet.stream_index) {
                AVFrame *frame = av_frame_alloc();
                ret = avcodec_send_packet(codecContext, &packet);
                while (ret >= 0) {
                    ret = avcodec_receive_frame(codecContext, frame);

                    if (ret < 0) {
                        break;
                    }
                    swr_convert(convertContext, &buffer, 2 * 48000, (const uint8_t **)frame->data,
                                frame->nb_samples);
                    int totalSize =
                        av_samples_get_buffer_size(nullptr, out_channel_nb, frame->nb_samples, out_sample_fmt, 0);
                    (void)totalSize;

                    fwrite(buffer, 1, totalSize, fp);
                    ++ts_;
                    ++writeAcount_;
                    av_frame_unref(frame);
                }
            } else if (video_stream == packet.stream_index) {
                unsigned char *pData = packet.data; // frame data
                unsigned char *pEnd = nullptr;
                int dataSize = packet.size;
                int curSize = 0;
                unsigned char nalHeader, nalType;
                AVPacket *outPacket = av_packet_alloc();
                pEnd = pData + dataSize;
                LOGD("packet size: %d, flag: %d", packet.size, packet.flags);
                while (curSize < dataSize) {
                    AVPacket spsppsPacket;
                    spsppsPacket.data = nullptr;
                    spsppsPacket.size = 0;
                    int naluSize = 0;
                    if (pEnd - pData < 4) {
                        break;
                    }
                    for (int i = 0; i < 4; ++i) {
                        naluSize <<= 8;
                        naluSize |= pData[i];
                    }
                    pData += 4;
                    if (naluSize > (pEnd - pData + 1) || naluSize <= 0) {
                        LOGD("nalusize error, real size: %d", pEnd - pData + 1);
                        break;
                    }
                    nalHeader = *pData;
                    nalType = nalHeader & 0x1F;
                    if (nalType == 6) {
                        pData += naluSize;
                        continue;
                    } else if (nalType == 5) {
                        // key frame
                        h264_extradata_to_annexb(
                            inputContext->streams[packet.stream_index]->codecpar->extradata,
                            inputContext->streams[packet.stream_index]->codecpar->extradata_size, &spsppsPacket,
                            AV_INPUT_BUFFER_PADDING_SIZE);
                        if (alloc_and_copy(outPacket, spsppsPacket.data, spsppsPacket.size, pData, naluSize) < 0) {
                            av_packet_free(&outPacket);
                            if (spsppsPacket.data) {
                                free(spsppsPacket.data);
                                spsppsPacket.data = nullptr;
                            }
                            break;
                        }
                    } else {
                        if (alloc_and_copy(outPacket, nullptr, 0, pData, naluSize) < 0) {
                            av_packet_free(&outPacket);
                            if (spsppsPacket.data) {
                                free(spsppsPacket.data);
                                spsppsPacket.data = nullptr;
                            }
                            break;
                        }
                    }
                    if (spsppsPacket.data) {
                        free(spsppsPacket.data);
                        spsppsPacket.data = nullptr;
                    }
                    LOGD("goto next");
                }
                LOGD("nalType: %u, pts: %u", nalType, outPacket->pts);
                ++ts_;
                ++writeVcount_;
                av_packet_unref(outPacket);
            }

            av_packet_unref(&packet);
            // TODO: cal sleep time
            usleep(15 * 1000);
        }
        LOGD("write file done, write audio: %u, write video: %u", writeAcount_,
                        writeVcount_);
        if (buffer != nullptr) {
            free(buffer);
        }
        if (codecContext != nullptr) {
            avcodec_close(codecContext);
        }
        avformat_close_input(&inputContext);
        // fclose(fp);
        // break;
    }
}

Version 3.0

3.0 根据帧率控制读取速度

C++
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
void WriteData()
{
    while (isRunning_) {
        AVFormatContext *inputContext = avformat_alloc_context();
        std::string inputFile = "/data/360p.mp4";
        // std::string outFile = "/data/ffmpeg.txt";
        // FILE *fp = fopen(outFile.c_str(), "wb");
        AVPacket packet;
        AVCodec *codec = nullptr;
        AVCodecContext *codecContext = nullptr;
        struct SwrContext *convertContext = nullptr;
        AVSampleFormat in_sample_fmt;
        AVSampleFormat out_sample_fmt = AV_SAMPLE_FMT_S16;
        int in_sample_rate;
        int out_sample_rate = 48000;
        uint64_t in_channel_layout;
        uint64_t out_channel_layout = AV_CH_LAYOUT_MONO;
        int out_channel_nb = 0;
        uint8_t *buffer = nullptr;
        int video_stream = -1;
        int audio_stream = -1;
        LOGD("write file in");
        int ret = 0;
        if ((ret = avformat_open_input(&inputContext, inputFile.c_str(), nullptr, nullptr)) != 0) {
            LOGE("ffmpeg open file failed, %s", av_err2str(ret));
            break;
        }
        ret = avformat_find_stream_info(inputContext, nullptr);

        video_stream = av_find_best_stream(inputContext, AVMEDIA_TYPE_VIDEO, -1, -1, nullptr, 0);
        audio_stream = av_find_best_stream(inputContext, AVMEDIA_TYPE_AUDIO, -1, -1, nullptr, 0);
        if (video_stream < 0 && audio_stream < 0) {
            LOGE("find video && audio stream error");
            avformat_close_input(&inputContext);
            break;
        }
        if (audio_stream >= 0) {
            av_dump_format(inputContext, audio_stream, inputFile.c_str(), 0);
            codec = avcodec_find_decoder(inputContext->streams[audio_stream]->codecpar->codec_id);
            codecContext = avcodec_alloc_context3(codec);
            avcodec_parameters_to_context(codecContext, inputContext->streams[audio_stream]->codecpar);
            if (avcodec_open2(codecContext, codec, nullptr) < 0) {
                LOGE("open codec error");
                avformat_close_input(&inputContext);
                break;
            }
            convertContext = swr_alloc();
            in_sample_fmt = codecContext->sample_fmt;
            in_sample_rate = codecContext->sample_rate;
            in_channel_layout = codecContext->channel_layout;
            convertContext = swr_alloc_set_opts(convertContext, out_channel_layout, out_sample_fmt, out_sample_rate,
                                                in_channel_layout, in_sample_fmt, in_sample_rate, 0, nullptr);
            swr_init(convertContext);
            out_channel_nb = av_get_channel_layout_nb_channels(out_channel_layout);
            buffer = (uint8_t *)av_malloc(2 * 48000);
        }
        if (video_stream >= 0) {
            av_dump_format(inputContext, video_stream, inputFile.c_str(), 0);
        }
        double vfps = 0.0;
        if (inputContext->streams[video_stream]->avg_frame_rate.den == 0 ||
            (inputContext->streams[video_stream]->avg_frame_rate.num == 0 &&
                inputContext->streams[video_stream]->avg_frame_rate.den == 1)) {
            vfps = inputContext->streams[video_stream]->r_frame_rate.num /
                    inputContext->streams[video_stream]->r_frame_rate.den;
        } else {
            vfps = inputContext->streams[video_stream]->avg_frame_rate.num /
                    inputContext->streams[video_stream]->avg_frame_rate.den;
        }
        double afps = (48000 * 2 * 16 / 8) / (1024 * 2 * 16 / 8.0); // aac
        LOGD("vfpr: %lf, afps: %lf", vfps, afps);
        av_init_packet(&packet);
        packet.data = nullptr;
        packet.size = 0;
        bool reading = true;
        int32_t keyCount = 0;
        LOGD("write file start");
        while (reading) {
            static std::chrono::time_point timeStart = std::chrono::system_clock::now();
            int error = av_read_frame(inputContext, &packet);
            if (error == AVERROR_EOF) {
                // auto stream = inputContext->streams[video_stream];
                // avio_seek(inputContext->pb, 0, SEEK_SET); // io context set to 0
                // avformat_seek_file(inputContext, video_stream, 0, 0, stream->duration, 0);
                // continue;
                reading = false; // 取消重复读取
                break;
            } else if (error < 0) {
                break;
            }
            if (audio_stream == packet.stream_index) {
                AVFrame *frame = av_frame_alloc();
                ret = avcodec_send_packet(codecContext, &packet);
                while (ret >= 0) {
                    ret = avcodec_receive_frame(codecContext, frame);

                    if (ret < 0) {
                        break;
                    }
                    swr_convert(convertContext, &buffer, 2 * 48000, (const uint8_t **)frame->data,
                                frame->nb_samples);
                    int totalSize =
                        av_samples_get_buffer_size(nullptr, out_channel_nb, frame->nb_samples, out_sample_fmt,
                        0);
                    (void)totalSize;

                    fwrite(buffer, 1, totalSize, fp);
                    // ++ts_;
                    ++writeAcount_;
                    av_frame_unref(frame);
                }
                auto timeStamp =
                    timeStart + std::chrono::microseconds(1000000 * 1000 / int(afps * 1000) * writeAcount_);
                std::this_thread::sleep_until(timeStamp);
            } else if (video_stream == packet.stream_index) {
                unsigned char *pData = packet.data; // frame data
                unsigned char *pEnd = nullptr;
                int dataSize = packet.size;
                int curSize = 0;
                unsigned char nalHeader, nalType;
                AVPacket *outPacket = av_packet_alloc();
                pEnd = pData + dataSize;
                LOGD("packet size: %d, flag: %d, pts: %lld", packet.size,
                                packet.flags, packet.pts);
                while (curSize < dataSize) {
                    AVPacket spsppsPacket;
                    spsppsPacket.data = nullptr;
                    spsppsPacket.size = 0;
                    int naluSize = 0;
                    if (pEnd - pData < 4) {
                        break;
                    }
                    for (int i = 0; i < 4; ++i) {
                        naluSize <<= 8;
                        naluSize |= pData[i];
                        // LOGD("naluSize: %d, pData[i]: %d, %02X", naluSize, pData[i],
                        //              pData[i]);
                    }
                    pData += 4;
                    curSize += 4;
                    if (naluSize > (pEnd - pData + 1) || naluSize <= 0) {
                        LOGD("nalusize error, real size: %d, naluSize: %d",
                                        pEnd - pData + 1, naluSize);
                        break;
                    }
                    nalHeader = *pData;
                    nalType = nalHeader & 0x1F;
                    LOGD("nalType: %u, flag: %d", nalType, packet.flags);
                    if (nalType == 6) {
                        pData += naluSize;
                        curSize += naluSize;
                        continue;
                    } else if (nalType == 5) {
                        // key frame
                        h264_extradata_to_annexb(
                            inputContext->streams[packet.stream_index]->codecpar->extradata,
                            inputContext->streams[packet.stream_index]->codecpar->extradata_size, &spsppsPacket,
                            AV_INPUT_BUFFER_PADDING_SIZE);
                        if (alloc_and_copy(outPacket, spsppsPacket.data, spsppsPacket.size, pData, naluSize) < 0) {
                            av_packet_free(&outPacket);
                            if (spsppsPacket.data) {
                                free(spsppsPacket.data);
                                spsppsPacket.data = nullptr;
                            }
                            break;
                        }
                    } else {
                        if (alloc_and_copy(outPacket, nullptr, 0, pData, naluSize) < 0) {
                            av_packet_free(&outPacket);
                            if (spsppsPacket.data) {
                                free(spsppsPacket.data);
                                spsppsPacket.data = nullptr;
                            }
                            break;
                        }
                    }
                    if (spsppsPacket.data) {
                        free(spsppsPacket.data);
                        spsppsPacket.data = nullptr;
                    }
                    curSize += naluSize;
                    LOGD("goto next, diffsize: %d", packet.size - outPacket->size);
                }
                // ++ts_;
                // if (outPacket->isKey) {
                //     ++keyCount;
                //     LOGD("write key frame: %d, pts: %lld", keyCount, pts);
                // }
                ++writeVcount_;
                av_packet_unref(outPacket);
                auto timeStamp =
                    timeStart + std::chrono::microseconds(1000000 * 1000 / int(vfps * 1000) * writeVcount_);
                std::this_thread::sleep_until(timeStamp); // 使用sleep_until控制时间
            }

            av_packet_unref(&packet);
        }
        LOGD("write file done, write audio: %u, write video: %u, keyCount: %d",
                        writeAcount_, writeVcount_, keyCount);
        if (buffer != nullptr) {
            free(buffer);
        }
        if (codecContext != nullptr) {
            avcodec_close(codecContext);
        }
        avformat_close_input(&inputContext);
        // fclose(fp);
        break;
    }
}