前言

之前實現了Android手機攝像頭資料的TCP實時傳輸，今天接著聊聊，如何在PC端把接收到的H264視訊流實時解碼並渲染出來。這次使用的語言是C++，框架有FFmpeg和SDL2。

解碼

解碼部分使用FFmpeg，首先，需要初始化H264解碼器：

int H264Decoder::init() {
    codec = avcodec_find_decoder(AV_CODEC_ID_H264);
    if (codec == nullptr) {
        printf("No H264 decoder found\n");
        return -1;
    }
    codecCtx = avcodec_alloc_context3(codec);
    codecCtx->flags |= AV_CODEC_FLAG_LOW_DELAY;
    if (avcodec_open2(codecCtx, codec, nullptr) < 0) {
        printf("Failed to open codec\n");
        return -2;
    }
    packet = av_packet_alloc();
    m_Frame = av_frame_alloc();
    parser = av_parser_init(AV_CODEC_ID_H264);
    return 0;
}

然後，使用建立TCP連線到我們的Android端，讀取封包：

bool read_data(SOCKET socket, void* data, unsigned int len) {
    while (len > 0) {
        int ret = recv(socket, (char*)data, len, 0);
        if (ret <= 0) {
            return false;
        }
        len -= ret;
        data = (char*)data + ret;
    }
    return true;
}

bool read_int(SOCKET socket, ULONG* value) {
    bool ret = read_data(socket, value, 4);
    if (ret) {
        *value = ntohl(*value);
    }
    return ret;
}

int PacketReceiver::readPacket(unsigned char** data, unsigned long* size) {
    ULONG pkgSize = 0;
    bool ret = read_int(m_Socket, &pkgSize);
    if (!ret) {
        printf("Failed to read packet size\n");
        return -1;
    }
    if (m_DataLen < pkgSize) {
        if (m_Data != nullptr) {
            delete[] m_Data;
        }
        m_Data = new unsigned char[pkgSize];
        m_DataLen = pkgSize;
    }
    if (!read_data(m_Socket, m_Data, pkgSize)) {
        printf("Failed to read packet data\n");
        return -2;
    }
    *data = m_Data;
    *size = pkgSize;
    return 0;
}

再把每個封包傳送給H264解碼器解碼

int H264Decoder::decode(unsigned char* data, int size, AVFrame** frame) {
    int new_pkg_ret = av_new_packet(packet, size);
    if (new_pkg_ret != 0) {
        printf("Failed to create new packet\n");
        return -1;
    }
    memcpy(packet->data, data, size);
    int ret = avcodec_send_packet(codecCtx, packet);
    if (ret < 0 && ret != AVERROR(EAGAIN)) {
        printf("Failed to parse packet\n");
        return -1;
    }
    ret = avcodec_receive_frame(codecCtx, m_Frame);
    if (ret == AVERROR(EAGAIN)) {
        *frame = nullptr;
        return 0;
    }
    if (ret != 0) {
        printf("Failed to read frame\n");
        return -1;
    }
    *frame = m_Frame;
    av_packet_unref(packet);
    return 0;
}

解碼器解碼後，最終得到的是AVFrame物件，代表一幀畫面，資料格式一般為YUV格式（跟編碼端選擇的畫素格式有關）。

渲染

通過使用SDL2，我們可以直接渲染YUV資料，無需手動轉成RGB。

首先，我們先初始化SDL2並建立渲染視窗：

int YuvRender::init(int video_width, int video_height) {
    SDL_Init(SDL_INIT_VIDEO);
    SDL_Rect bounds;
    SDL_GetDisplayUsableBounds(0, &bounds);
    int winWidth = video_width;
    int winHeight = video_height;
    if (winWidth > bounds.w || winHeight > bounds.h) {
        float widthRatio = 1.0 * winWidth / bounds.w;
        float heightRatio = 1.0 * winHeight / bounds.h;
        float maxRatio = widthRatio > heightRatio ? widthRatio : heightRatio;
        winWidth = int(winWidth / maxRatio);
        winHeight = int(winHeight / maxRatio);
    }
    SDL_Window* window = SDL_CreateWindow(
        "NetCameraViewer",
        SDL_WINDOWPOS_UNDEFINED,
        SDL_WINDOWPOS_UNDEFINED,
        winWidth,
        winHeight,
        SDL_WINDOW_OPENGL
    );
    m_Renderer = SDL_CreateRenderer(window, -1, 0);
    m_Texture = SDL_CreateTexture(
        m_Renderer,
        SDL_PIXELFORMAT_IYUV,
        SDL_TEXTUREACCESS_STREAMING,
        video_width,
        video_height
    );
    m_VideoWidth = video_width;
    m_VideoHeight = video_height;
    m_Rect.x = 0;
    m_Rect.y = 0;
    m_Rect.w = winWidth;
    m_Rect.h = winHeight;
    return 0;
}

每次解碼出一幀畫面的時候，再呼叫render函數渲染：

int YuvRender::render(unsigned char* data[], int pitch[]) {
    int uvHeight = m_VideoHeight / 2;
    int ySize = pitch[0] * m_VideoHeight;
    int uSize = pitch[1] * uvHeight;
    int vSize = pitch[2] * uvHeight;
    int buffSize =  ySize + uSize + vSize;
    if (m_FrameBufferSize < buffSize) {
        if (m_FrameBuffer != nullptr) {
            delete[] m_FrameBuffer;
        }
        m_FrameBuffer = new unsigned char[buffSize];
        m_FrameBufferSize = buffSize;
    }
    SDL_memcpy(m_FrameBuffer, data[0], ySize);
    SDL_memcpy(m_FrameBuffer + ySize, data[1], uSize);
    SDL_memcpy(m_FrameBuffer + ySize + uSize, data[2], vSize);
    SDL_UpdateTexture(m_Texture, NULL, m_FrameBuffer, pitch[0]);
    SDL_RenderClear(m_Renderer);
    SDL_RenderCopy(m_Renderer, m_Texture, NULL, &m_Rect);
    SDL_RenderPresent(m_Renderer);
    SDL_PollEvent(&m_Event);
    if (m_Event.type == SDL_QUIT) {
        exit(0);
    }
    return 0;
}

效能

在搭載AMD Ryzen 5 5600U的機器上，1800 x 1350的解析度，解碼一幀平均25ms, 渲染1~2ms，加上編碼和傳輸延時，總體延時在70ms左右。

完整原始碼已上傳至Github: https://github.com/kasonyang/net-camera/tree/main/viewer-app