ffmpeg如果采用硬解码器h264_cuvid解码时颜色空间AV_PIX_FMT_CUDA无法转换为BGR24

亦凉 2022-09-12 11:48 126阅读 0赞

如果我们采用ffmpeg硬解码：

AVCodec *m_codec=avcodec_find_decoder_by_name(“h264_cuvid”)；
    AVCodecContext * m_codecCtx = avcodec_alloc_context3(m_codec);
    AVCodecParameters *codecPar = inCtx.codecParams;
    avcodec_parameters_to_context(m_codecCtx, codecPar);
     m_codecCtx->bit_rate = 0;
     m_codecCtx->time_base.den = 1;
     m_codecCtx->time_base.num = 25;
      //如果在这里颜色空间设置为AV_PIX_FMT_CUDA，那么在解码receive解码后，m_frame->data指针指向
     //的数据地址还是显存，所以这时候拷贝的话就会出错，并且sws_scale也会出问题，那么在上述情况下还需要一步
     //AVFrame *frameInter = av_frame_alloc();
     // av_hwframe_transfer_data(frameInter, m_frame, 0);将数据从显存中拷贝出来，拷贝出来后
     //颜色空间就转化为NV12了
    // 如果嫌麻烦，直接将颜色空间设置为AV_PIX_FMT_NV12，自动就将数据下载到主存中了
    //这时候数据也可以用sws_scale了
      m_codecCtx->pix_fmt =  AV_PIX_FMT_NV12 /*AV_PIX_FMT_CUDA*/
                                               : m_codecCtx->pix_fmt;
    avcodec_open2(m_codecCtx, m_codec, nullptr);
    
    avcodec_send_frame(m_codecCtx, m_frame);
    while (ret >= 0) {  
    //如果前面将颜色空间设置为CuDA，那么这里m_Frame数据指向显存
       ret=avcodec_receive_frame(m_codecCtx, m_frame);
        if (ret < 0 || ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) { 
          av_strerror(ret, m_errBuf, ERR_BUF_LEN);
          LOG_ERROR("avcodec_receive_packet error: {}", m_errBuf);
          return false;
        }
        }

参考代码：  
这段代码就是，如果前面将颜色空间设置为AV\_PIX\_FMT\_CUDA，那么在这里需要m\_frame中data所指向  
的显存数据拷贝出来av\_hwframe\_transfer\_data（），拷贝出来以后就可以采用内存数据拷贝以及内存数据CPU颜色空间转换了

> 需要说明的一点是AV\_PIX\_FMT\_CUDA颜色空间其实还是NV12，只不过它的数据在GPU上，所以在GPU上进行颜色空间转换是NV12->BGR24，所以不要再问怎么AV\_PIX\_FMT\_CUDA是什么颜色空间啊，怎么找不到具体说明。

while (ret >= 0) {  // not so good
           ret = avcodec_receive_frame(m_codecCtx, m_frame);
          if (ret < 0 || ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) { 
              return false;
          }
    
          auto policy = getCapturePolicy();
          int width = m_frame->width;
          int height = m_frame->height;
          // unsigned long resolution = size_t(height) * size_t(width);
          // auto pixFmt = AVPixelFormat(m_frame->format);
    
          if (m_blackFrame.empty()) { 
              m_blackFrame = cv::Mat(height, width, CV_8UC3);
          }
          AVFrame *frameInter = av_frame_alloc();
          //这一步将解码后的数据从显存拷贝出来，并且转化为NV12
          av_hwframe_transfer_data(frameInter, m_frame, 0);
          const char *pixname = av_get_pix_fmt_name(AVPixelFormat(frameInter->format));
          cv::Mat cpuMat;
          //******************opencv颜色空间转化**********************
          // cv::Mat imgYUV;
          // imgYUV.create(int(height * 1.5), width, CV_8UC1);
          // auto p = height * width;
          // memcpy(imgYUV.data, frameInter->data[0], size_t(p));
          // memcpy(imgYUV.data + p, frameInter->data[1], size_t(p / 4));
          // memcpy(imgYUV.data + 5 * p / 4, frameInter->data[2], size_t(p / 4));
          // cv::cvtColor(imgYUV, cpuMat, cv::COLOR_YUV2BGR_I420 /*cv::COLOR_YUV2BGR_NV12*/);
          //*****************END***************************************
    
          // if (m_param.gpu < 0) { // transcode on CPU
    
          //**************ffmpgecpu转码******************
          sws_scale(m_imgcvtCtx,
                    static_cast<uint8_t const *const *>(frameInter->data),
                    frameInter->linesize,
                    0,
                    height,
                    m_bgrFrame->data,
                    m_bgrFrame->linesize);
    
          cv::Mat bgrImg(height,
                         width,
                         CV_8UC3,
                         static_cast<uchar *>(m_outbuf),
                         static_cast<size_t>(width * 3));
          cpuMat = bgrImg.clone();
          //*****************END****************************
          // }
          //这一段非常重要如果前面不将数据从显存中拷贝的时候，可以选择直接在GPU中进行颜色空间转化flow::img::Nv12ToBGR24（），这个函数就是在显存中用GPU直接转化颜色空间的，这样的好处是，少了一次从显存下载数据的步骤，减少了数据从显存拷贝到主存的操作，这个函数可以直接在github上下载，有人已经写好了，这是一个cuda核函数，如果在你的c++工程中编译的话需要专门设置，具体百度，非常简单，这里不再赘述。
          /*else { // transcode on GPU if (m_gpuImg.empty()) { cv::cuda::setDevice(m_param.gpu); // first frame m_gpuImg.create(height, width, CV_8UC3); // allocate a BGR24 gpu image m_gpuImg.step = size_t(m_bgrFrame->linesize[0]); LOG_INFO("frame height = {}, width = {}, pix format = {}, linsize0 = {}, " "linesize1 = {}, gpu image step = {}", height, width, pixFmt, m_frame->linesize[0], m_frame->linesize[1], m_bgrFrame->linesize[0]); } if (m_skipIdx == 0 || !policy.useBlackFrame) { cv::cuda::setDevice(m_param.gpu); //xassert(pixFmt == AV_PIX_FMT_NV12 || pixFmt == AV_PIX_FMT_CUDA); //xassert(!m_gpuImg.empty()); // -----------------GPU------------------- flow::img::Nv12ToBGR24(m_frame->data[0], m_frame->data[1], m_gpuImg.data, resolution, m_frame->height, m_frame->width, m_frame->linesize[0], m_frame->linesize[1]); m_gpuImg.download(cpuMat); // -------------------END-------------------- img::setCvImgFlag(BLACK_FRAME_BIT, cpuMat, false); } else { cpuMat = m_blackFrame; img::setCvImgFlag(BLACK_FRAME_BIT, cpuMat, true); } }*/
          // add tag according to the value of m_skipIdx
          auto isKeyFrame = (m_skipIdx == 0);
          img::setCvImgFlag(KEY_FRAME_BIT, cpuMat, isKeyFrame);
          CustomerMat custMat{ cpuMat, m_frame->pts};
          outFrame.push_back(custMat);
          m_skipIdx = (m_skipIdx + 1) % (policy.skip + 1);
          break;
      }

参考二：  
这段代码就是采用ffmpeg直接进行颜色空间转换，后面是采用gpu，时间比较紧张，不详细整理代码，看得懂就看，看不懂的小白就不要看了，反正也不指望你能看懂，这个主要是为了我自己笔记吧。

bool FFvideoCodec::decode(OnePktPtr &inPkt,
                              std::vector<CustomerMat> &outFrame) { 
      // CUDA pixel format can keep decoded data on GPU, otherwise decoded data
      // will download to host memory
      auto inPacket = reinterpret_cast<AVPacket *>(inPkt->data); // NOLINT
      m_codecCtx->pix_fmt = (m_param.gpu >= 0) ? AV_PIX_FMT_NV12 /*AV_PIX_FMT_CUDA*/
                                               : m_codecCtx->pix_fmt;
      auto ret = avcodec_send_packet(m_codecCtx, inPacket);
      if (ret < 0) { 
        av_strerror(ret, m_errBuf, ERR_BUF_LEN);
        LOG_ERROR("avcodec_send_packet < 0 of {} for {}, closing ...", ret,
                  m_errBuf);
        return false;
      }
    
      while (ret >= 0) {  // not so good
        ret = avcodec_receive_frame(m_codecCtx, m_frame);
        if (ret < 0 || ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) { 
          return false;
        }
    
        auto policy = getCapturePolicy();
        int width = m_frame->width;
        int height = m_frame->height;
        unsigned long resolution = size_t(height) * size_t(width);
        auto pixFmt = AVPixelFormat(m_frame->format);
    
        if (m_blackFrame.empty()) { 
          m_blackFrame = cv::Mat(height, width, CV_8UC3);
        }
        const char *pixname = av_get_pix_fmt_name(AVPixelFormat(m_frame->format));
        // LOG_INFO("pix format:{}", name);
        cv::Mat cpuMat;
    
        if (m_param.gpu < 0) {  // transcode on CPU
          auto bret = sws_scale(
              m_imgcvtCtx, static_cast<uint8_t const *const *>(m_frame->data),
              m_frame->linesize, 0, height, m_bgrFrame->data, m_bgrFrame->linesize);
    
          cv::Mat bgrImg(height, width, CV_8UC3, static_cast<uchar *>(m_outbuf),
                         static_cast<size_t>(width * 3));
    
          // cv::Mat bgrImg;
          // cv::cvtColor(nv12Mat, bgrImg, cv::COLOR_YUV2BGR_NV12);
          cpuMat = bgrImg.clone();
          img::setCvImgFlag(BLACK_FRAME_BIT, cpuMat, false);
    
          // add tag according to the value of m_skipIdx
          auto isKeyFrame = (m_skipIdx == 0);
          img::setCvImgFlag(KEY_FRAME_BIT, cpuMat, isKeyFrame);
          CustomerMat custMat{ cpuMat, m_frame->pts};
          outFrame.push_back(custMat);
          m_skipIdx = (m_skipIdx + 1) % (policy.skip + 1);
        } else {  // transcode on GPU
          if (m_gpuImg.empty()) { 
            cv::cuda::setDevice(m_param.gpu);        // first frame
            m_gpuImg.create(height, width, CV_8UC3); // allocate a BGR24 gpu image
            m_gpuImg.step = size_t(m_bgrFrame->linesize[0]);
            LOG_INFO(
                "frame height = {}, width = {}, pix format = {}, linsize0 = {}, "
                "linesize1 = {}, gpu image step = {}",
                height, width, pixFmt, m_frame->linesize[0], m_frame->linesize[1],
                m_bgrFrame->linesize[0]);
          }
    
          if (m_skipIdx == 0 || !policy.useBlackFrame) { 
            cv::cuda::setDevice(m_param.gpu);
            xassert(pixFmt == AV_PIX_FMT_NV12 || pixFmt == AV_PIX_FMT_CUDA);
            xassert(!m_gpuImg.empty());
            flow::img::Nv12ToBGR24(m_frame->data[0], m_frame->data[1],
                                   m_gpuImg.data, resolution, m_frame->height,
                                   m_frame->width, m_frame->linesize[0],
                                   m_frame->linesize[1]);
            m_gpuImg.download(cpuMat);
    
            img::setCvImgFlag(BLACK_FRAME_BIT, cpuMat, false);
          } else { 
            cpuMat = m_blackFrame;
            img::setCvImgFlag(BLACK_FRAME_BIT, cpuMat, true);
          }
    
          // add tag according to the value of m_skipIdx
          auto isKeyFrame = (m_skipIdx == 0);
          img::setCvImgFlag(KEY_FRAME_BIT, cpuMat, isKeyFrame);
          CustomerMat custMat{ cpuMat, m_frame->pts};
          outFrame.push_back(custMat);
          m_skipIdx = (m_skipIdx + 1) % (policy.skip + 1);
        }
        break;
      }
    
      av_frame_unref(m_frame);
      return (!outFrame.empty());
    }

参考：https://www.jianshu.com/p/ad05a94001b4

参考代码：

#ifndef FFVIDEOCODEC_H
    #define FFVIDEOCODEC_H
    
    #include "codec.h"
    #include <opencv2/core/cuda.hpp>
    
    namespace flow { 
    
    class FFvideoCodec : public Codec { 
    public:
      FFvideoCodec();
      ~FFvideoCodec() override;
    
    private:
      bool initEncoder() final;
      void destoryEncoder() final;
    
      bool initDecoder() final;
      void destoryDecoder() final;
    
      bool openEncoder() final;
      bool openDecoder() final;
    
      bool encode(cv::Mat &inFrame, OnePktPtr &outPkt) final;
      bool decode(OnePktPtr &inPkt, std::vector<CustomerMat> &outFrame) final;
    
    private:
      int m_streamIdx = -1; // video stream index
      AVFrame *m_bgrFrame = nullptr;
      uint8_t *m_outbuf = nullptr;
      struct SwsContext *m_imgcvtCtx = nullptr;
      cv::cuda::GpuMat m_gpuImg; // gpu BGR24 image
    
    private:
      constexpr static int ERR_BUF_LEN = 256;
      char *m_errBuf = nullptr;
      AVFrame *m_frame = nullptr;
      AVPacket *m_pkt = nullptr;
      AVCodec *m_codec = nullptr;
      AVCodecContext *m_codecCtx = nullptr;
      bool m_decodecHasOpened{ false};
      bool m_encodecHasOpened{ false};
    };
    } // namespace flow
    
    #endif // FFVIDEOCODEC_H

#include "ffvideocodec.h"
    #include "opencv.hpp"
    #include "utils/yuv2bgr.h"
    #include "xlog/xlog.h"
    extern "C" { 
    #include "libavutil/time.h"
    #include <libavutil/imgutils.h>
    #include <libavutil/opt.h>
    }
    
    namespace flow { 
    
    FFvideoCodec::FFvideoCodec() { 
    #pragma GCC diagnostic push
    #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
      av_register_all();
      avcodec_register_all();
    #pragma GCC diagnostic pop
      m_errBuf = new char[ERR_BUF_LEN];
    }
    
    FFvideoCodec::~FFvideoCodec() { 
      if (m_errBuf) { 
        delete[] m_errBuf;
        m_errBuf = nullptr;
      }
      m_decodecHasOpened = false;
      m_encodecHasOpened = false;
    }
    
    bool FFvideoCodec::initEncoder() { 
      // find encoder by name, if fail find by codecid.
      m_codec = avcodec_find_encoder_by_name(m_param.codecName.c_str());
      if (m_codec == nullptr) { 
        LOG_ERROR("avcodec_find_encoder {} failed!", m_param.codecName);
        AVCodecID codecId = m_param.codecId;
        m_codec = avcodec_find_encoder(codecId);
        if (m_codec == nullptr) { 
          LOG_ERROR("avcodec_find_encoder id {} return null.", codecId);
          return false;
        }
        LOG_INFO("avcodec_find_encoder id {} ok!", codecId);
      } else { 
        LOG_INFO("avcodec_find_encode {} ok!", m_param.codecName);
      }
    
      // allocate packet for encoded video data
      m_pkt = av_packet_alloc();
    
      // allocate frame for decoded video data
      m_frame = av_frame_alloc();
      m_frame->format = m_param.pixFmt;
      m_frame->width = m_param.width;
      m_frame->height = m_param.height;
      m_frame->pts = 1;
    
      // allocate codec context
      m_codecCtx = avcodec_alloc_context3(m_codec);
      m_codecCtx->codec_id = m_codec->id;
      m_codecCtx->codec_type = AVMEDIA_TYPE_VIDEO;
      // switch (int(m_param.pixFmt)) { 
      // case CodecParam::NV12:
      // m_codecCtx->pix_fmt = AV_PIX_FMT_NV12;
      // break;
      // case CodecParam::YUV420P:
      // m_codecCtx->pix_fmt = AV_PIX_FMT_YUV420P;
      // break;
      // case CodecParam::YUVJ420P:
      // m_codecCtx->pix_fmt = AV_PIX_FMT_YUVJ420P;
      // break;
      // default:
      // m_codecCtx->pix_fmt = AV_PIX_FMT_YUV420P;
      // break;
      // }
      m_codecCtx->pix_fmt = m_param.pixFmt;
      m_codecCtx->bit_rate = m_param.bit_rate;
      m_codecCtx->rc_max_rate = m_codecCtx->rc_max_rate;
      m_codecCtx->rc_min_rate = m_codecCtx->rc_min_rate;
      m_codecCtx->rc_buffer_size = m_codecCtx->rc_buffer_size;
      m_codecCtx->width = m_param.width;
      m_codecCtx->height = m_param.height;
      m_codecCtx->qmin = 32;
      m_codecCtx->qmax = 36;
      m_codecCtx->gop_size = m_param.gop;
      m_codecCtx->has_b_frames = 0;
      m_codecCtx->max_b_frames = m_param.bframes;
      m_codecCtx->time_base.num = 1;
      m_codecCtx->time_base.den = m_param.fps;
      m_codecCtx->framerate.num = m_param.fps;
      m_codecCtx->framerate.den = 1;
    
      // set h264 and hevc encode option
      if (m_codec->id == AV_CODEC_ID_H264 || m_codec->id == AV_CODEC_ID_HEVC) { 
        av_opt_set(m_codecCtx->priv_data, "profile", "high", 0);
        av_opt_set(m_codecCtx->priv_data, "level", "5", 0);
        av_opt_set(m_codecCtx->priv_data, "tune", "zerolatency", 0);
      }
    
      // set gpu number for h264_nvenc and hevc_nvenc
      if ((m_param.gpu >= 0) &&
          (strcmp(m_param.codecName.c_str(), "h264_nvenc") == 0 ||
           strcmp(m_param.codecName.c_str(), "hevc_nvenc") == 0)) { 
        av_opt_set(m_codecCtx->priv_data, "gpu",
                   std::to_string(m_param.gpu).c_str(), 0);
        LOG_INFO("encode using gpu: {}", m_param.gpu);
      }
      return true;
    }
    
    void FFvideoCodec::destoryEncoder() { 
      if (m_codecCtx) { 
        if (m_encodecHasOpened) { 
          m_encodecHasOpened = false;
          avcodec_close(m_codecCtx);
        }
        avcodec_free_context(&m_codecCtx);
        m_codecCtx = nullptr;
      }
      if (m_frame) { 
        av_frame_free(&m_frame);
        m_frame = nullptr;
      }
      if (m_pkt) { 
        av_packet_free(&m_pkt);
        m_pkt = nullptr;
      }
    }
    
    bool FFvideoCodec::initDecoder() { 
      m_codec = (m_param.gpu >= 0)
                    ? avcodec_find_decoder_by_name(m_param.codecName.c_str())
                    : avcodec_find_decoder(m_param.codecId);
      if (m_codec == nullptr) { 
        LOG_ERROR("avcodec_find_decoder failed, closing ...");
        return false;
      }
    
      m_codecCtx = avcodec_alloc_context3(m_codec);
    
      AVCodecParameters *codecPar = m_param.codecParams;
      int ret = avcodec_parameters_to_context(m_codecCtx, codecPar);
      if (ret < 0) { 
        av_strerror(ret, m_errBuf, ERR_BUF_LEN);
        LOG_ERROR("avcodec_parameters_to_context < 0 for {}, closing ...",
                  m_errBuf);
        destoryDecoder();
        return false;
      }
    
      m_codecCtx->bit_rate = 0;
      m_codecCtx->time_base.den = 1;
      m_codecCtx->time_base.num = 25;
      m_codecCtx->pix_fmt = (m_param.gpu >= 0) ? AV_PIX_FMT_NV12 /*AV_PIX_FMT_CUDA*/
                                               : m_codecCtx->pix_fmt;
    
      AVRational rt = m_param.avg_frame_rate;
      if (rt.den != 0) { 
        m_codecCtx->time_base.den = rt.den;
        m_codecCtx->time_base.num = rt.num;
        m_param.fps = rt.num / rt.den;
      }
    
      // set gpu number for h264_nvenc and hevc_nvenc
      if ((m_param.gpu >= 0) &&
          (strcmp(m_param.codecName.c_str(), "h264_cuvid") == 0 ||
           strcmp(m_param.codecName.c_str(), "hevc_cuvid") == 0)) { 
        av_opt_set(m_codecCtx->priv_data, "gpu",
                   std::to_string(m_param.gpu).c_str(), 0);
        LOG_INFO("decoder using gpu: {}", m_param.gpu);
      }
    
      m_param.fps = (m_param.fps >= 15 && m_param.fps <= 60) ? m_param.fps : 25;
    
      m_frame = av_frame_alloc();
      m_bgrFrame = av_frame_alloc();
    
      m_pkt = av_packet_alloc();
      av_init_packet(m_pkt);
      return true;
    }
    
    void FFvideoCodec::destoryDecoder() { 
      if (m_codecCtx) { 
        if (m_decodecHasOpened) { 
          m_decodecHasOpened = false;
          avcodec_close(m_codecCtx);
        }
        avcodec_free_context(&m_codecCtx);
        m_codecCtx = nullptr;
      }
    
      if (m_frame) { 
        av_frame_unref(m_frame);
        av_frame_free(&m_frame);
        m_frame = nullptr;
      }
      if (m_bgrFrame) { 
        av_frame_unref(m_bgrFrame);
        av_frame_free(&m_bgrFrame);
        m_bgrFrame = nullptr;
      }
      if (m_pkt) { 
        av_packet_unref(m_pkt);
        av_packet_free(&m_pkt);
        m_pkt = nullptr;
      }
    
      if (m_outbuf) { 
        av_free(m_outbuf);
        m_outbuf = nullptr;
      }
    
      if (m_imgcvtCtx) { 
        sws_freeContext(m_imgcvtCtx);
        m_imgcvtCtx = nullptr;
      }
    }
    
    bool FFvideoCodec::openEncoder() { 
      int ret = avcodec_open2(m_codecCtx, m_codec, nullptr);
      if (ret < 0) { 
        av_strerror(ret, m_errBuf, ERR_BUF_LEN);
        LOG_ERROR("avcodec_open2 with {} failed: {}", m_param.codecName, m_errBuf);
        destoryEncoder();
        return false;
      }
      m_encodecHasOpened = true;
      if (av_frame_get_buffer(m_frame, 0) < 0 ||
          av_frame_make_writable(m_frame) < 0) { 
        destoryEncoder();
        return false;
      }
    
      return true;
    }
    
    bool FFvideoCodec::openDecoder() { 
      auto openDecodeOk{ false};
      do { 
        int ret = avcodec_open2(m_codecCtx, m_codec, nullptr);
        if (ret < 0) { 
          av_strerror(ret, m_errBuf, ERR_BUF_LEN);
          LOG_ERROR("avcodec_open2 < 0 for {}, closing ...", m_errBuf);
          break;
        }
        m_decodecHasOpened = true;
        int width = m_codecCtx->width;
        int height = m_codecCtx->height;
        if (width > 3840 || height > 2160) { 
          LOG_ERROR("avcodec_open2 failed! image w:{}, h:{}", width, height);
          break;
        }
    
        // allocate buffer to store decoded frame
        AVPixelFormat av_fmt = AV_PIX_FMT_BGR24;
        int bufSize = av_image_get_buffer_size(av_fmt, width, height, 1);
        m_outbuf = static_cast<unsigned char *>(av_malloc(size_t(bufSize)));
        if (m_outbuf == nullptr) { 
          LOG_ERROR("allocate buffer failed! image w:{}, h:{}", width, height);
          break;
        }
        av_image_fill_arrays(m_bgrFrame->data, m_bgrFrame->linesize, m_outbuf,
                             av_fmt, width, height, 1);
    
        // set transcode and resize parameter
        m_imgcvtCtx = sws_getContext(
            width, height, AV_PIX_FMT_NV12 /*m_codecCtx->pix_fmt*/, width, height,
            av_fmt, SWS_BICUBIC /*SWS_FAST_BILINEAR*/, nullptr, nullptr, nullptr);
        openDecodeOk = true;
      } while (false);
      if (!openDecodeOk) { 
        if (m_imgcvtCtx) { 
          sws_freeContext(m_imgcvtCtx);
          m_imgcvtCtx = nullptr;
        }
        av_frame_unref(m_bgrFrame);
        if (m_outbuf) { 
          av_free(m_outbuf);
          m_outbuf = nullptr;
        }
        if (m_decodecHasOpened) { 
          m_decodecHasOpened = false;
          avcodec_close(m_codecCtx);
        }
      }
      return openDecodeOk;
    }
    
    bool FFvideoCodec::encode(cv::Mat &inFrame, OnePktPtr &outPkt) { 
      // Note: Cannot fill AVFrame with simply copy frame.data
      av_image_fill_arrays(m_frame->data, m_frame->linesize, inFrame.data,
                           m_codecCtx->pix_fmt, m_frame->width, m_frame->height, 1);
    
      int ret = avcodec_send_frame(m_codecCtx, m_frame);
      if (ret < 0) { 
        av_strerror(ret, m_errBuf, ERR_BUF_LEN);
        LOG_ERROR("avcodec_send_frame error: {}", m_errBuf);
        return false;
      }
    
      while (ret >= 0) {  // not so good
        ret = avcodec_receive_packet(m_codecCtx, m_pkt);
        if (ret < 0 || ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) { 
          av_strerror(ret, m_errBuf, ERR_BUF_LEN);
          LOG_ERROR("avcodec_receive_packet error: {}", m_errBuf);
          return false;
        }
    
        outPkt->len = size_t(m_pkt->size);
        if (size_t(m_pkt->size) > outPkt->max) { 
          LOG_INFO("encoded packet too large, realloc to {}.", m_pkt->size);
          outPkt->data = static_cast<unsigned char *>(
              realloc(outPkt->data, size_t(m_pkt->size))); // NOLINT
          outPkt->max = size_t(m_pkt->size);
        }
        memcpy(outPkt->data, m_pkt->data, size_t(m_pkt->size)); // NOLINT
        // pts is very important for video decode and play, if no pts it will play
        // black image on the screen.
        m_frame->pts +=
            av_rescale_q(1, m_codecCtx->time_base, m_codecCtx->time_base);
    
        av_packet_unref(m_pkt);
        return true;
      }
    
      return true;
    }
    
    bool FFvideoCodec::decode(OnePktPtr &inPkt,
                              std::vector<CustomerMat> &outFrame) { 
      // CUDA pixel format can keep decoded data on GPU, otherwise decoded data
      // will download to host memory
      auto inPacket = reinterpret_cast<AVPacket *>(inPkt->data); // NOLINT
      m_codecCtx->pix_fmt = (m_param.gpu >= 0) ? AV_PIX_FMT_NV12 /*AV_PIX_FMT_CUDA*/
                                               : m_codecCtx->pix_fmt;
      auto ret = avcodec_send_packet(m_codecCtx, inPacket);
      if (ret < 0) { 
        av_strerror(ret, m_errBuf, ERR_BUF_LEN);
        LOG_ERROR("avcodec_send_packet < 0 of {} for {}, closing ...", ret,
                  m_errBuf);
        return false;
      }
    
      while (ret >= 0) {  // not so good
        ret = avcodec_receive_frame(m_codecCtx, m_frame);
        if (ret < 0 || ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) { 
          return false;
        }
    
        auto policy = getCapturePolicy();
        int width = m_frame->width;
        int height = m_frame->height;
        unsigned long resolution = size_t(height) * size_t(width);
        auto pixFmt = AVPixelFormat(m_frame->format);
    
        if (m_blackFrame.empty()) { 
          m_blackFrame = cv::Mat(height, width, CV_8UC3);
        }
        const char *pixname = av_get_pix_fmt_name(AVPixelFormat(m_frame->format));
        // LOG_INFO("pix format:{}", name);
        cv::Mat cpuMat;
    
        if (m_param.gpu >= 0) {  // transcode on CPU
          auto bret = sws_scale(
              m_imgcvtCtx, static_cast<uint8_t const *const *>(m_frame->data),
              m_frame->linesize, 0, height, m_bgrFrame->data, m_bgrFrame->linesize);
    
          cv::Mat bgrImg(height, width, CV_8UC3, static_cast<uchar *>(m_outbuf),
                         static_cast<size_t>(width * 3));
    
          // cv::Mat bgrImg;
          // cv::cvtColor(nv12Mat, bgrImg, cv::COLOR_YUV2BGR_NV12);
          cpuMat = bgrImg.clone();
          img::setCvImgFlag(BLACK_FRAME_BIT, cpuMat, false);
    
          // add tag according to the value of m_skipIdx
          auto isKeyFrame = (m_skipIdx == 0);
          img::setCvImgFlag(KEY_FRAME_BIT, cpuMat, isKeyFrame);
          CustomerMat custMat{ cpuMat, m_frame->pts};
          outFrame.push_back(custMat);
          m_skipIdx = (m_skipIdx + 1) % (policy.skip + 1);
        } else {  // transcode on GPU
          if (m_gpuImg.empty()) { 
            cv::cuda::setDevice(m_param.gpu);        // first frame
            m_gpuImg.create(height, width, CV_8UC3); // allocate a BGR24 gpu image
            m_gpuImg.step = size_t(m_bgrFrame->linesize[0]);
            LOG_INFO(
                "frame height = {}, width = {}, pix format = {}, linsize0 = {}, "
                "linesize1 = {}, gpu image step = {}",
                height, width, pixFmt, m_frame->linesize[0], m_frame->linesize[1],
                m_bgrFrame->linesize[0]);
          }
    
          if (m_skipIdx == 0 || !policy.useBlackFrame) { 
            cv::cuda::setDevice(m_param.gpu);
            xassert(pixFmt == AV_PIX_FMT_NV12 || pixFmt == AV_PIX_FMT_CUDA);
            xassert(!m_gpuImg.empty());
            flow::img::Nv12ToBGR24(m_frame->data[0], m_frame->data[1],
                                   m_gpuImg.data, resolution, m_frame->height,
                                   m_frame->width, m_frame->linesize[0],
                                   m_frame->linesize[1]);
            m_gpuImg.download(cpuMat);
    
            img::setCvImgFlag(BLACK_FRAME_BIT, cpuMat, false);
          } else { 
            cpuMat = m_blackFrame;
            img::setCvImgFlag(BLACK_FRAME_BIT, cpuMat, true);
          }
    
          // add tag according to the value of m_skipIdx
          auto isKeyFrame = (m_skipIdx == 0);
          img::setCvImgFlag(KEY_FRAME_BIT, cpuMat, isKeyFrame);
          CustomerMat custMat{ cpuMat, m_frame->pts};
          outFrame.push_back(custMat);
          m_skipIdx = (m_skipIdx + 1) % (policy.skip + 1);
        }
        break;
      }
    
      av_frame_unref(m_frame);
      return (!outFrame.empty());
    }
    
    } // namespace flow