FFmpeg+SDL实时解码和渲染H264视频流

快来打我* 2024-03-17 19:48 98阅读 0赞

今天接着聊聊，如何在PC端把接收到的H264视频流实时解码并渲染出来。这次使用的语言是C++，框架有FFmpeg和SDL2。

## 解码 ##

解码部分使用FFmpeg，首先，需要初始化H264解码器：

int H264Decoder::init() {
        codec = avcodec_find_decoder(AV_CODEC_ID_H264);
        if (codec == nullptr) {
            printf("No H264 decoder found\n");
            return -1;
        }
        codecCtx = avcodec_alloc_context3(codec);
        codecCtx->flags |= AV_CODEC_FLAG_LOW_DELAY;
        if (avcodec_open2(codecCtx, codec, nullptr) < 0) {
            printf("Failed to open codec\n");
            return -2;
        }
        packet = av_packet_alloc();
        m_Frame = av_frame_alloc();
        parser = av_parser_init(AV_CODEC_ID_H264);
        return 0;
    }

然后，使用创建TCP连接到我们的Android端，读取数据包：

bool read_data(SOCKET socket, void* data, unsigned int len) {
        while (len > 0) {
            int ret = recv(socket, (char*)data, len, 0);
            if (ret <= 0) {
                return false;
            }
            len -= ret;
            data = (char*)data + ret;
        }
        return true;
    }
    
    bool read_int(SOCKET socket, ULONG* value) {
        bool ret = read_data(socket, value, 4);
        if (ret) {
            *value = ntohl(*value);
        }
        return ret;
    }
    
    int PacketReceiver::readPacket(unsigned char** data, unsigned long* size) {
        ULONG pkgSize = 0;
        bool ret = read_int(m_Socket, &pkgSize);
        if (!ret) {
            printf("Failed to read packet size\n");
            return -1;
        }
        if (m_DataLen < pkgSize) {
            if (m_Data != nullptr) {
                delete[] m_Data;
            }
            m_Data = new unsigned char[pkgSize];
            m_DataLen = pkgSize;
        }
        if (!read_data(m_Socket, m_Data, pkgSize)) {
            printf("Failed to read packet data\n");
            return -2;
        }
        *data = m_Data;
        *size = pkgSize;
        return 0;
    }

再把每个数据包传送给H264解码器解码

int H264Decoder::decode(unsigned char* data, int size, AVFrame** frame) {
        int new_pkg_ret = av_new_packet(packet, size);
        if (new_pkg_ret != 0) {
            printf("Failed to create new packet\n");
            return -1;
        }
        memcpy(packet->data, data, size);
        int ret = avcodec_send_packet(codecCtx, packet);
        if (ret < 0 && ret != AVERROR(EAGAIN)) {
            printf("Failed to parse packet\n");
            return -1;
        }
        ret = avcodec_receive_frame(codecCtx, m_Frame);
        if (ret == AVERROR(EAGAIN)) {
            *frame = nullptr;
            return 0;
        }
        if (ret != 0) {
            printf("Failed to read frame\n");
            return -1;
        }
        *frame = m_Frame;
        av_packet_unref(packet);
        return 0;
    }

解码器解码后，最终得到的是`AVFrame`对象，代表一帧画面，数据格式一般为YUV格式（跟编码端选择的像素格式有关）。

## 渲染 ##

通过使用SDL2，我们可以直接渲染YUV数据，无需手动转成RGB。

首先，我们先初始化SDL2并创建渲染窗口：

int YuvRender::init(int video_width, int video_height) {
        SDL_Init(SDL_INIT_VIDEO);
        SDL_Rect bounds;
        SDL_GetDisplayUsableBounds(0, &bounds);
        int winWidth = video_width;
        int winHeight = video_height;
        if (winWidth > bounds.w || winHeight > bounds.h) {
            float widthRatio = 1.0 * winWidth / bounds.w;
            float heightRatio = 1.0 * winHeight / bounds.h;
            float maxRatio = widthRatio > heightRatio ? widthRatio : heightRatio;
            winWidth = int(winWidth / maxRatio);
            winHeight = int(winHeight / maxRatio);
        }
        SDL_Window* window = SDL_CreateWindow(
            "NetCameraViewer",
            SDL_WINDOWPOS_UNDEFINED,
            SDL_WINDOWPOS_UNDEFINED,
            winWidth,
            winHeight,
            SDL_WINDOW_OPENGL
        );
        m_Renderer = SDL_CreateRenderer(window, -1, 0);
        m_Texture = SDL_CreateTexture(
            m_Renderer,
            SDL_PIXELFORMAT_IYUV,
            SDL_TEXTUREACCESS_STREAMING,
            video_width,
            video_height
        );
        m_VideoWidth = video_width;
        m_VideoHeight = video_height;
        m_Rect.x = 0;
        m_Rect.y = 0;
        m_Rect.w = winWidth;
        m_Rect.h = winHeight;
        return 0;
    }

每次解码出一帧画面的时候，再调用render函数渲染：

int YuvRender::render(unsigned char* data[], int pitch[]) {
        int uvHeight = m_VideoHeight / 2;
        int ySize = pitch[0] * m_VideoHeight;
        int uSize = pitch[1] * uvHeight;
        int vSize = pitch[2] * uvHeight;
        int buffSize =  ySize + uSize + vSize;
        if (m_FrameBufferSize < buffSize) {
            if (m_FrameBuffer != nullptr) {
                delete[] m_FrameBuffer;
            }
            m_FrameBuffer = new unsigned char[buffSize];
            m_FrameBufferSize = buffSize;
        }
        SDL_memcpy(m_FrameBuffer, data[0], ySize);
        SDL_memcpy(m_FrameBuffer + ySize, data[1], uSize);
        SDL_memcpy(m_FrameBuffer + ySize + uSize, data[2], vSize);
        SDL_UpdateTexture(m_Texture, NULL, m_FrameBuffer, pitch[0]);
        SDL_RenderClear(m_Renderer);
        SDL_RenderCopy(m_Renderer, m_Texture, NULL, &m_Rect);
        SDL_RenderPresent(m_Renderer);
        SDL_PollEvent(&m_Event);
        if (m_Event.type == SDL_QUIT) {
            exit(0);
        }
        return 0;
    }

## 性能 ##

在搭载AMD Ryzen 5 5600U的机器上，1800 x 1350的分辨率，解码一帧平均25ms, 渲染1~2ms，加上编码和传输延时，总体延时在70ms左右。