prograrm temp

旧城等待， 2023-01-17 13:51 235阅读 0赞

调试暂存

  • 2021.4.30:12:39temp
  • new
  • normal

2021.4.30:12:39temp

  1. /** * Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER * DEALINGS IN THE SOFTWARE. */
  3. #include <string.h>
  4. #include <string>
  5. #include <sstream>
  6. #include <iostream>
  7. #include <ostream>
  8. #include <fstream>
  9. #include "gstdsexample.h"
  10. #include <sys/time.h>
  13. #include "opencv2/core/core.hpp" 
  14. #include "opencv2/imgproc/imgproc.hpp" 
  15. #include "opencv2/calib3d/calib3d.hpp" 
  16. #include "opencv2/highgui/highgui.hpp" 
  17. using namespace cv;
  18. using namespace std;
  21. GST_DEBUG_CATEGORY_STATIC (gst_dsexample_debug);
  22. #define GST_CAT_DEFAULT gst_dsexample_debug
  23. #define USE_EGLIMAGE 1
  24. /* enable to write transformed cvmat to files */
  25. /* #define DSEXAMPLE_DEBUG */
  26. static GQuark _dsmeta_quark = 0;
  28. /* Enum to identify properties */
  29. enum
  30. { 
  31.   PROP_0,
  32.   PROP_UNIQUE_ID,
  33.   PROP_PROCESSING_WIDTH,
  34.   PROP_PROCESSING_HEIGHT,
  35.   PROP_PROCESS_FULL_FRAME,
  36.   PROP_BLUR_OBJECTS,
  37.   PROP_GPU_DEVICE_ID
  38. };
  40. #define CHECK_NVDS_MEMORY_AND_GPUID(object, surface) \ ({ int _errtype=0;\ do { \ if ((surface->memType == NVBUF_MEM_DEFAULT || surface->memType == NVBUF_MEM_CUDA_DEVICE) && \ (surface->gpuId != object->gpu_id)) { \ GST_ELEMENT_ERROR (object, RESOURCE, FAILED, \ ("Input surface gpu-id doesnt match with configured gpu-id for element," \ " please allocate input using unified memory, or use same gpu-ids"),\ ("surface-gpu-id=%d,%s-gpu-id=%d",surface->gpuId,GST_ELEMENT_NAME(object),\ object->gpu_id)); \ _errtype = 1;\ } \ } while(0); \ _errtype; \ })
  43. /* Default values for properties */
  44. #define DEFAULT_UNIQUE_ID 15
  45. #define DEFAULT_PROCESSING_WIDTH 640
  46. #define DEFAULT_PROCESSING_HEIGHT 480
  47. #define DEFAULT_PROCESS_FULL_FRAME TRUE
  48. #define DEFAULT_BLUR_OBJECTS FALSE
  49. #define DEFAULT_GPU_ID 0
  51. #define RGB_BYTES_PER_PIXEL 3
  52. #define RGBA_BYTES_PER_PIXEL 4
  53. #define Y_BYTES_PER_PIXEL 1
  54. #define UV_BYTES_PER_PIXEL 2
  56. #define MIN_INPUT_OBJECT_WIDTH 16
  57. #define MIN_INPUT_OBJECT_HEIGHT 16
  60. /*畸变矫正相关*/
  61. #define SRC_WIDTH 1920
  62. #define SRC_HEIGHT 1080
  64. Mat mapx = Mat(Size(SRC_WIDTH, SRC_HEIGHT), CV_32FC1);
  65. Mat mapy = Mat(Size(SRC_WIDTH, SRC_HEIGHT), CV_32FC1);
  66. Mat newimage ; //校正后输出图片
  68. #define CHECK_NPP_STATUS(npp_status,error_str) do { \ if ((npp_status) != NPP_SUCCESS) { \ g_print ("Error: %s in %s at line %d: NPP Error %d\n", \ error_str, __FILE__, __LINE__, npp_status); \ goto error; \ } \ } while (0)
  70. #define CHECK_CUDA_STATUS(cuda_status,error_str) do { \ if ((cuda_status) != cudaSuccess) { \ g_print ("Error: %s in %s at line %d (%s)\n", \ error_str, __FILE__, __LINE__, cudaGetErrorName(cuda_status)); \ goto error; \ } \ } while (0)
  72. /* By default NVIDIA Hardware allocated memory flows through the pipeline. We * will be processing on this type of memory only. */
  73. #define GST_CAPS_FEATURE_MEMORY_NVMM "memory:NVMM"
  74. static GstStaticPadTemplate gst_dsexample_sink_template =
  75. GST_STATIC_PAD_TEMPLATE ("sink",
  76.     GST_PAD_SINK,
  77.     GST_PAD_ALWAYS,
  78.     GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
  79.         (GST_CAPS_FEATURE_MEMORY_NVMM,
  80.             "{ NV12, RGBA, I420 }")));
  82. static GstStaticPadTemplate gst_dsexample_src_template =
  83. GST_STATIC_PAD_TEMPLATE ("src",
  84.     GST_PAD_SRC,
  85.     GST_PAD_ALWAYS,
  86.     GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
  87.         (GST_CAPS_FEATURE_MEMORY_NVMM,
  88.             "{ NV12, RGBA, I420 }")));
  90. /* Define our element type. Standard GObject/GStreamer boilerplate stuff */
  91. #define gst_dsexample_parent_class parent_class
  92. G_DEFINE_TYPE (GstDsExample, gst_dsexample, GST_TYPE_BASE_TRANSFORM);
  94. static void gst_dsexample_set_property (GObject * object, guint prop_id,
  95.     const GValue * value, GParamSpec * pspec);
  96. static void gst_dsexample_get_property (GObject * object, guint prop_id,
  97.     GValue * value, GParamSpec * pspec);
  99. static gboolean gst_dsexample_set_caps (GstBaseTransform * btrans,
  100.     GstCaps * incaps, GstCaps * outcaps);
  101. static gboolean gst_dsexample_start (GstBaseTransform * btrans);
  102. static gboolean gst_dsexample_stop (GstBaseTransform * btrans);
  104. static GstFlowReturn gst_dsexample_transform_ip (GstBaseTransform *
  105.     btrans, GstBuffer * inbuf);
  107. static void
  108. attach_metadata_full_frame (GstDsExample * dsexample, NvDsFrameMeta *frame_meta,
  109.     gdouble scale_ratio, DsExampleOutput * output, guint batch_id);
  110. static void attach_metadata_object (GstDsExample * dsexample,
  111.     NvDsObjectMeta * obj_meta, DsExampleOutput * output);
  115. void InitMat(Mat& m, float* num)
  116. { 
  117.     for (int i = 0; i<m.rows; i++)
  118.     { 
  119.         for (int j = 0; j<m.cols; j++)
  120.         { 
  121.             m.at<float>(i, j) = *(num + i * m.rows + j);
  122.         }
  123.     }
  124. }
  126. void jiaozheng_init(void)
  127. { 
  128.     int OK = 0;
  131.     Mat R = Mat::eye(3, 3, CV_32F);
  132.     //参数矩阵
  133.     float neican_data[] = {  9558.649257742036, 0, 959.3165310990756, 0, 9435.752651759443, 532.7507141910969, 0, 0, 1 };
  134.     Mat cameraMatrix(3, 3, CV_32FC1);
  135.     InitMat(neican_data,cameraMatrix);
  137.     cout << "cameraMatrix= " << endl << " " << cameraMatrix << endl << endl;
  138.     //测得的畸变系数
  139.     float jibian_data[] = {  -6.956561513881647, -68.83902522804168, -0.004834538444671919, 0.01471273691928269, -0.4916103704308509 };
  140.     Mat distCoeffs(1, 5, CV_32FC1); /* 摄像机的5个畸变系数：k1,k2,p1,p2,k3 */
  141.     InitMat( jibian_data, distCoeffs);
  142.     cout << "distCoeffs= " << endl << " " << distCoeffs << endl << endl;
  144.     /********相机矫正*******************************************************************************/
  145.     //cout << "mapx= " << endl << " " << mapx << endl << endl;
  146.     //cout << "mapy= " << endl << " " << mapy << endl << endl;
  147.     initUndistortRectifyMap(cameraMatrix, distCoeffs, R, cameraMatrix, Size(SRC_WIDTH, SRC_HEIGHT), CV_32FC1, mapx, mapy);
  148.     //cout << "mapx= " << endl << " " << mapx << endl << endl;
  149.     //cout << "mapy= " << endl << " " << mapy << endl << endl;
  151. }
  158. /******************************************************************************* Function: jibianjiaozheng Description: 该函数实现畸变矫正 Input: 原图 Output: 畸变矫正后的图片 Return: 0: Successful ohters: Failed *******************************************************************************/
  160. int jibianjiaozheng(Mat src_picture, Mat & dst_picture)
  161. { 
  162.     int OK = 0;
  163.     static unsigned int INIT_MAT_OK = 1;
  165.     if (INIT_MAT_OK)
  166.     { 
  167.         newimage = src_picture.clone(); //校正后输出图片
  168.         INIT_MAT_OK = 0;
  169.     }
  171.     remap(src_picture, newimage, mapx, mapy, INTER_LINEAR);
  172.     dst_picture= newimage+1;
  173.     return OK;
  174. }
  181. /* Install properties, set sink and src pad capabilities, override the required * functions of the base class, These are common to all instances of the * element. */
  182. static void
  183. gst_dsexample_class_init (GstDsExampleClass * klass)
  184. { 
  185.   GObjectClass *gobject_class;
  186.   GstElementClass *gstelement_class;
  187.   GstBaseTransformClass *gstbasetransform_class;
  189.   /* Indicates we want to use DS buf api */
  190.   g_setenv ("DS_NEW_BUFAPI", "1", TRUE);
  192.   gobject_class = (GObjectClass *) klass;
  193.   gstelement_class = (GstElementClass *) klass;
  194.   gstbasetransform_class = (GstBaseTransformClass *) klass;
  196.   /* Overide base class functions */
  197.   gobject_class->set_property = GST_DEBUG_FUNCPTR (gst_dsexample_set_property);
  198.   gobject_class->get_property = GST_DEBUG_FUNCPTR (gst_dsexample_get_property);
  200.   gstbasetransform_class->set_caps = GST_DEBUG_FUNCPTR (gst_dsexample_set_caps);
  201.   gstbasetransform_class->start = GST_DEBUG_FUNCPTR (gst_dsexample_start);
  202.   gstbasetransform_class->stop = GST_DEBUG_FUNCPTR (gst_dsexample_stop);
  204.   gstbasetransform_class->transform_ip =
  205.       GST_DEBUG_FUNCPTR (gst_dsexample_transform_ip);
  207.   /* Install properties */
  208.   g_object_class_install_property (gobject_class, PROP_UNIQUE_ID,
  209.       g_param_spec_uint ("unique-id",
  210.           "Unique ID",
  211.           "Unique ID for the element. Can be used to identify output of the"
  212.           " element", 0, G_MAXUINT, DEFAULT_UNIQUE_ID, (GParamFlags)
  213.           (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
  215.   g_object_class_install_property (gobject_class, PROP_PROCESSING_WIDTH,
  216.       g_param_spec_int ("processing-width",
  217.           "Processing Width",
  218.           "Width of the input buffer to algorithm",
  219.           1, G_MAXINT, DEFAULT_PROCESSING_WIDTH, (GParamFlags)
  220.           (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
  222.   g_object_class_install_property (gobject_class, PROP_PROCESSING_HEIGHT,
  223.       g_param_spec_int ("processing-height",
  224.           "Processing Height",
  225.           "Height of the input buffer to algorithm",
  226.           1, G_MAXINT, DEFAULT_PROCESSING_HEIGHT, (GParamFlags)
  227.           (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
  229.   g_object_class_install_property (gobject_class, PROP_PROCESS_FULL_FRAME,
  230.       g_param_spec_boolean ("full-frame",
  231.           "Full frame",
  232.           "Enable to process full frame or disable to process objects detected"
  233.           "by primary detector", DEFAULT_PROCESS_FULL_FRAME, (GParamFlags)
  234.           (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
  236.   g_object_class_install_property (gobject_class, PROP_BLUR_OBJECTS,
  237.       g_param_spec_boolean ("blur-objects",
  238.           "Blur Objects",
  239.           "Enable to blur the objects detected in full-frame=0 mode"
  240.           "by primary detector", DEFAULT_BLUR_OBJECTS, (GParamFlags)
  241.           (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
  243.   g_object_class_install_property (gobject_class, PROP_GPU_DEVICE_ID,
  244.       g_param_spec_uint ("gpu-id",
  245.           "Set GPU Device ID",
  246.           "Set GPU Device ID", 0,
  247.           G_MAXUINT, 0,
  248.           GParamFlags
  249.           (G_PARAM_READWRITE |
  250.               G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_READY)));
  251.   /* Set sink and src pad capabilities */
  252.   gst_element_class_add_pad_template (gstelement_class,
  253.       gst_static_pad_template_get (&gst_dsexample_src_template));
  254.   gst_element_class_add_pad_template (gstelement_class,
  255.       gst_static_pad_template_get (&gst_dsexample_sink_template));
  257.   /* Set metadata describing the element */
  258.   gst_element_class_set_details_simple (gstelement_class,
  259.       "DsExample plugin",
  260.       "DsExample Plugin",
  261.       "Process a 3rdparty example algorithm on objects / full frame",
  262.       "NVIDIA Corporation. Post on Deepstream for Tesla forum for any queries "
  263.       "@ https://devtalk.nvidia.com/default/board/209/");
  264. }
  266. static void
  267. gst_dsexample_init (GstDsExample * dsexample)
  268. { 
  269.   GstBaseTransform *btrans = GST_BASE_TRANSFORM (dsexample);
  271.   /* We will not be generating a new buffer. Just adding / updating * metadata. */
  272.   gst_base_transform_set_in_place (GST_BASE_TRANSFORM (btrans), TRUE);
  273.   /* We do not want to change the input caps. Set to passthrough. transform_ip * is still called. */
  274.   gst_base_transform_set_passthrough (GST_BASE_TRANSFORM (btrans), TRUE);
  276.   /* Initialize all property variables to default values */
  277.   dsexample->unique_id = DEFAULT_UNIQUE_ID;
  278.   dsexample->processing_width = DEFAULT_PROCESSING_WIDTH;
  279.   dsexample->processing_height = DEFAULT_PROCESSING_HEIGHT;
  280.   dsexample->process_full_frame = DEFAULT_PROCESS_FULL_FRAME;
  281.   dsexample->blur_objects = DEFAULT_BLUR_OBJECTS;
  282.   dsexample->gpu_id = DEFAULT_GPU_ID;
  283.   /* This quark is required to identify NvDsMeta when iterating through * the buffer metadatas */
  284.   if (!_dsmeta_quark)
  285.     _dsmeta_quark = g_quark_from_static_string (NVDS_META_STRING);
  286. }
  288. /* Function called when a property of the element is set. Standard boilerplate. */
  289. static void
  290. gst_dsexample_set_property (GObject * object, guint prop_id,
  291.     const GValue * value, GParamSpec * pspec)
  292. { 
  293.   GstDsExample *dsexample = GST_DSEXAMPLE (object);
  294.   switch (prop_id) { 
  295.     case PROP_UNIQUE_ID:
  296.       dsexample->unique_id = g_value_get_uint (value);
  297.       break;
  298.     case PROP_PROCESSING_WIDTH:
  299.       dsexample->processing_width = g_value_get_int (value);
  300.       break;
  301.     case PROP_PROCESSING_HEIGHT:
  302.       dsexample->processing_height = g_value_get_int (value);
  303.       break;
  304.     case PROP_PROCESS_FULL_FRAME:
  305.       dsexample->process_full_frame = g_value_get_boolean (value);
  306.       break;
  307.     case PROP_BLUR_OBJECTS:
  308.       dsexample->blur_objects = g_value_get_boolean (value);
  309.       break;
  310.     case PROP_GPU_DEVICE_ID:
  311.       dsexample->gpu_id = g_value_get_uint (value);
  312.       break;
  313.     default:
  314.       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
  315.       break;
  316.   }
  317. }
  319. /* Function called when a property of the element is requested. Standard * boilerplate. */
  320. static void
  321. gst_dsexample_get_property (GObject * object, guint prop_id,
  322.     GValue * value, GParamSpec * pspec)
  323. { 
  324.   GstDsExample *dsexample = GST_DSEXAMPLE (object);
  326.   switch (prop_id) { 
  327.     case PROP_UNIQUE_ID:
  328.       g_value_set_uint (value, dsexample->unique_id);
  329.       break;
  330.     case PROP_PROCESSING_WIDTH:
  331.       g_value_set_int (value, dsexample->processing_width);
  332.       break;
  333.     case PROP_PROCESSING_HEIGHT:
  334.       g_value_set_int (value, dsexample->processing_height);
  335.       break;
  336.     case PROP_PROCESS_FULL_FRAME:
  337.       g_value_set_boolean (value, dsexample->process_full_frame);
  338.       break;
  339.     case PROP_BLUR_OBJECTS:
  340.       g_value_set_boolean (value, dsexample->blur_objects);
  341.       break;
  342.     case PROP_GPU_DEVICE_ID:
  343.       g_value_set_uint (value, dsexample->gpu_id);
  344.       break;
  345.     default:
  346.       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
  347.       break;
  348.   }
  349. }
  351. /** * Initialize all resources and start the output thread */
  352. static gboolean
  353. gst_dsexample_start (GstBaseTransform * btrans)
  354. { 
  355.   GstDsExample *dsexample = GST_DSEXAMPLE (btrans);
  356.   NvBufSurfaceCreateParams create_params;
  357.   DsExampleInitParams init_params =
  358.       {  dsexample->processing_width, dsexample->processing_height,
  359.     dsexample->process_full_frame
  360.   };
  362.   GstQuery *queryparams = NULL;
  363.   guint batch_size = 1;
  365.   /* Algorithm specific initializations and resource allocation. */
  366.   dsexample->dsexamplelib_ctx = DsExampleCtxInit (&init_params);
  368.   GST_DEBUG_OBJECT (dsexample, "ctx lib %p \n", dsexample->dsexamplelib_ctx);
  370.   CHECK_CUDA_STATUS (cudaSetDevice (dsexample->gpu_id),
  371.       "Unable to set cuda device");
  373.   dsexample->batch_size = 1;
  374.   queryparams = gst_nvquery_batch_size_new ();
  375.   if (gst_pad_peer_query (GST_BASE_TRANSFORM_SINK_PAD (btrans), queryparams)
  376.       || gst_pad_peer_query (GST_BASE_TRANSFORM_SRC_PAD (btrans), queryparams)) { 
  377.     if (gst_nvquery_batch_size_parse (queryparams, &batch_size)) { 
  378.       dsexample->batch_size = batch_size;
  379.     }
  380.   }
  381.   GST_DEBUG_OBJECT (dsexample, "Setting batch-size %d \n",
  382.       dsexample->batch_size);
  383.   gst_query_unref (queryparams);
  385.   if (dsexample->process_full_frame && dsexample->blur_objects) { 
  386.     GST_ERROR ("Error: does not support blurring while processing full frame");
  387.     goto error;
  388.   }
  390.   CHECK_CUDA_STATUS (cudaStreamCreate (&dsexample->cuda_stream),
  391.       "Could not create cuda stream");
  393.   if (dsexample->inter_buf)
  394.     NvBufSurfaceDestroy (dsexample->inter_buf);
  395.   dsexample->inter_buf = NULL;
  397.   /* An intermediate buffer for NV12/RGBA to BGR conversion will be * required. Can be skipped if custom algorithm can work directly on NV12/RGBA. */
  398.   create_params.gpuId  = dsexample->gpu_id;
  399.   create_params.width  = dsexample->processing_width;
  400.   create_params.height = dsexample->processing_height;
  401.   create_params.size = 0;
  402.   create_params.colorFormat = NVBUF_COLOR_FORMAT_RGBA;
  403.   create_params.layout = NVBUF_LAYOUT_PITCH;
  404. #ifdef __aarch64__
  405.   create_params.memType = NVBUF_MEM_DEFAULT;
  406. #else
  407.   create_params.memType = NVBUF_MEM_CUDA_UNIFIED;
  408. #endif
  410.   if (NvBufSurfaceCreate (&dsexample->inter_buf, 1,
  411.           &create_params) != 0) { 
  412.     GST_ERROR ("Error: Could not allocate internal buffer for dsexample");
  413.     goto error;
  414.   }
  416.   /* Create host memory for storing converted/scaled interleaved RGB data */
  417.   CHECK_CUDA_STATUS (cudaMallocHost (&dsexample->host_rgb_buf,
  418.           dsexample->processing_width * dsexample->processing_height *
  419.           RGB_BYTES_PER_PIXEL), "Could not allocate cuda host buffer");
  421.   GST_DEBUG_OBJECT (dsexample, "allocated cuda buffer %p \n",
  422.       dsexample->host_rgb_buf);
  424.   /* CV Mat containing interleaved RGB data. This call does not allocate memory. * It uses host_rgb_buf as data. */
  425.   dsexample->cvmat =
  426.       new cv::Mat (dsexample->processing_height, dsexample->processing_width,
  427.       CV_8UC3, dsexample->host_rgb_buf,
  428.       dsexample->processing_width * RGB_BYTES_PER_PIXEL);
  430.   if (!dsexample->cvmat)
  431.     goto error;
  433.   GST_DEBUG_OBJECT (dsexample, "created CV Mat\n");
  435.   return TRUE;
  436. error:
  437.   if (dsexample->host_rgb_buf) { 
  438.     cudaFreeHost (dsexample->host_rgb_buf);
  439.     dsexample->host_rgb_buf = NULL;
  440.   }
  442.   if (dsexample->cuda_stream) { 
  443.     cudaStreamDestroy (dsexample->cuda_stream);
  444.     dsexample->cuda_stream = NULL;
  445.   }
  446.   if (dsexample->dsexamplelib_ctx)
  447.     DsExampleCtxDeinit (dsexample->dsexamplelib_ctx);
  448.   return FALSE;
  449. }
  451. /** * Stop the output thread and free up all the resources */
  452. static gboolean
  453. gst_dsexample_stop (GstBaseTransform * btrans)
  454. { 
  455.   GstDsExample *dsexample = GST_DSEXAMPLE (btrans);
  457.   if (dsexample->inter_buf)
  458.     NvBufSurfaceDestroy(dsexample->inter_buf);
  459.   dsexample->inter_buf = NULL;
  461.   if (dsexample->cuda_stream)
  462.     cudaStreamDestroy (dsexample->cuda_stream);
  463.   dsexample->cuda_stream = NULL;
  465.   delete dsexample->cvmat;
  466.   dsexample->cvmat = NULL;
  468.   if (dsexample->host_rgb_buf) { 
  469.     cudaFreeHost (dsexample->host_rgb_buf);
  470.     dsexample->host_rgb_buf = NULL;
  471.   }
  473.   GST_DEBUG_OBJECT (dsexample, "deleted CV Mat \n");
  475.   /* Deinit the algorithm library */
  476.   DsExampleCtxDeinit (dsexample->dsexamplelib_ctx);
  477.   dsexample->dsexamplelib_ctx = NULL;
  479.   GST_DEBUG_OBJECT (dsexample, "ctx lib released \n");
  481.   return TRUE;
  482. }
  484. /** * Called when source / sink pad capabilities have been negotiated. */
  485. static gboolean
  486. gst_dsexample_set_caps (GstBaseTransform * btrans, GstCaps * incaps,
  487.     GstCaps * outcaps)
  488. { 
  489.   GstDsExample *dsexample = GST_DSEXAMPLE (btrans);
  490.   /* Save the input video information, since this will be required later. */
  491.   gst_video_info_from_caps (&dsexample->video_info, incaps);
  493.   if (dsexample->blur_objects && !dsexample->process_full_frame) { 
  494.     /* requires RGBA format for blurring the objects in opencv */
  495.      if (dsexample->video_info.finfo->format != GST_VIDEO_FORMAT_RGBA) { 
  496.       GST_ELEMENT_ERROR (dsexample, STREAM, FAILED,
  497.           ("input format should be RGBA when using blur-objects property"), (NULL));
  498.       goto error;
  499.       }
  500.   }
  502.   return TRUE;
  504. error:
  505.   return FALSE;
  506. }
  508. /** * Scale the entire frame to the processing resolution maintaining aspect ratio. * Or crop and scale objects to the processing resolution maintaining the aspect * ratio. Remove the padding required by hardware and convert from RGBA to RGB * using openCV. These steps can be skipped if the algorithm can work with * padded data and/or can work with RGBA. */
  509. static GstFlowReturn
  510. get_converted_mat (GstDsExample * dsexample, NvBufSurface *input_buf, gint idx,
  511.     NvOSD_RectParams * crop_rect_params, gdouble & ratio, gint input_width,
  512.     gint input_height)
  513. { 
  514.   NvBufSurfTransform_Error err;
  515.   NvBufSurfTransformConfigParams transform_config_params;
  516.   NvBufSurfTransformParams transform_params;
  517.   NvBufSurfTransformRect src_rect;
  518.   NvBufSurfTransformRect dst_rect;
  519.   NvBufSurface ip_surf;
  520.   cv::Mat in_mat;
  521.   ip_surf = *input_buf;
  523.   ip_surf.numFilled = ip_surf.batchSize = 1;
  524.   ip_surf.surfaceList = &(input_buf->surfaceList[idx]);
  526.   gint src_left = GST_ROUND_UP_2((unsigned int)crop_rect_params->left);
  527.   gint src_top = GST_ROUND_UP_2((unsigned int)crop_rect_params->top);
  528.   gint src_width = GST_ROUND_DOWN_2((unsigned int)crop_rect_params->width);
  529.   gint src_height = GST_ROUND_DOWN_2((unsigned int)crop_rect_params->height);
  531.   /* Maintain aspect ratio */
  532.   double hdest = dsexample->processing_width * src_height / (double) src_width;
  533.   double wdest = dsexample->processing_height * src_width / (double) src_height;
  534.   guint dest_width, dest_height;
  536.   if (hdest <= dsexample->processing_height) { 
  537.     dest_width = dsexample->processing_width;
  538.     dest_height = hdest;
  539.   } else { 
  540.     dest_width = wdest;
  541.     dest_height = dsexample->processing_height;
  542.   }
  544.   /* Configure transform session parameters for the transformation */
  545.   transform_config_params.compute_mode = NvBufSurfTransformCompute_Default;
  546.   transform_config_params.gpu_id = dsexample->gpu_id;
  547.   transform_config_params.cuda_stream = dsexample->cuda_stream;
  549.   /* Set the transform session parameters for the conversions executed in this * thread. */
  550.   err = NvBufSurfTransformSetSessionParams (&transform_config_params);
  551.   if (err != NvBufSurfTransformError_Success) { 
  552.     GST_ELEMENT_ERROR (dsexample, STREAM, FAILED,
  553.         ("NvBufSurfTransformSetSessionParams failed with error %d", err), (NULL));
  554.     goto error;
  555.   }
  557.   /* Calculate scaling ratio while maintaining aspect ratio */
  558.   ratio = MIN (1.0 * dest_width/ src_width, 1.0 * dest_height / src_height);
  560.   if ((crop_rect_params->width == 0) || (crop_rect_params->height == 0)) { 
  561.     GST_ELEMENT_ERROR (dsexample, STREAM, FAILED,
  562.         ("%s:crop_rect_params dimensions are zero",__func__), (NULL));
  563.     goto error;
  564.   }
  566. #ifdef __aarch64__
  567.   if (ratio <= 1.0 / 16 || ratio >= 16.0) { 
  568.     /* Currently cannot scale by ratio > 16 or < 1/16 for Jetson */
  569.     goto error;
  570.   }
  571. #endif
  572.   /* Set the transform ROIs for source and destination */
  573.   src_rect = { (guint)src_top, (guint)src_left, (guint)src_width, (guint)src_height};
  574.   dst_rect = { 0, 0, (guint)dest_width, (guint)dest_height};
  576.   /* Set the transform parameters */
  577.   transform_params.src_rect = &src_rect;
  578.   transform_params.dst_rect = &dst_rect;
  579.   transform_params.transform_flag =
  580.     NVBUFSURF_TRANSFORM_FILTER | NVBUFSURF_TRANSFORM_CROP_SRC |
  581.       NVBUFSURF_TRANSFORM_CROP_DST;
  582.   transform_params.transform_filter = NvBufSurfTransformInter_Default;
  584.   /* Memset the memory */
  585.   NvBufSurfaceMemSet (dsexample->inter_buf, 0, 0, 0);
  587.   GST_DEBUG_OBJECT (dsexample, "Scaling and converting input buffer\n");
  589.   /* Transformation scaling+format conversion if any. */
  590.   err = NvBufSurfTransform (&ip_surf, dsexample->inter_buf, &transform_params);
  591.   if (err != NvBufSurfTransformError_Success) { 
  592.     GST_ELEMENT_ERROR (dsexample, STREAM, FAILED,
  593.         ("NvBufSurfTransform failed with error %d while converting buffer", err),
  594.         (NULL));
  595.     goto error;
  596.   }
  597.   /* Map the buffer so that it can be accessed by CPU */
  598.   if (NvBufSurfaceMap (dsexample->inter_buf, 0, 0, NVBUF_MAP_READ) != 0){ 
  599.     goto error;
  600.   }
  602.   /* Cache the mapped data for CPU access */
  603.   NvBufSurfaceSyncForCpu (dsexample->inter_buf, 0, 0);
  605.   /* Use openCV to remove padding and convert RGBA to BGR. Can be skipped if * algorithm can handle padded RGBA data. */
  606.   in_mat =
  607.       cv::Mat (dsexample->processing_height, dsexample->processing_width,
  608.       CV_8UC4, dsexample->inter_buf->surfaceList[0].mappedAddr.addr[0],
  609.       dsexample->inter_buf->surfaceList[0].pitch);
  611. #if (CV_MAJOR_VERSION >= 4)
  612.   cv::cvtColor (in_mat, *dsexample->cvmat, cv::COLOR_RGBA2BGR);
  613. #else
  614.   cv::cvtColor (in_mat, *dsexample->cvmat, CV_RGBA2BGR);
  615. #endif
  617.   if (NvBufSurfaceUnMap (dsexample->inter_buf, 0, 0)){ 
  618.     goto error;
  619.   }
  621. #ifdef __aarch64__
  622.   /* To use the converted buffer in CUDA, create an EGLImage and then use * CUDA-EGL interop APIs */
  623.   if (USE_EGLIMAGE) { 
  624.     if (NvBufSurfaceMapEglImage (dsexample->inter_buf, 0) !=0 ) { 
  625.       goto error;
  626.     }
  628.     /* dsexample->inter_buf->surfaceList[0].mappedAddr.eglImage * Use interop APIs cuGraphicsEGLRegisterImage and * cuGraphicsResourceGetMappedEglFrame to access the buffer in CUDA */
  630.     /* Destroy the EGLImage */
  631.     NvBufSurfaceUnMapEglImage (dsexample->inter_buf, 0);
  632.   }
  633. #endif
  635.   /* We will first convert only the Region of Interest (the entire frame or the * object bounding box) to RGB and then scale the converted RGB frame to * processing resolution. */
  636.   return GST_FLOW_OK;
  638. error:
  639.   return GST_FLOW_ERROR;
  640. }
  642. #if 0
  643. /* * Blur the detected objects when processing in object mode (full-frame=0) */
  644. static GstFlowReturn
  645. blur_objects (GstDsExample * dsexample, gint idx,
  646.     NvOSD_RectParams * crop_rect_params, cv::Mat in_mat)
  647. { 
  648.   cv::Rect crop_rect;
  649.   if ((crop_rect_params->width == 0) || (crop_rect_params->height == 0)) { 
  650.     GST_ELEMENT_ERROR (dsexample, STREAM, FAILED,
  651.         ("%s:crop_rect_params dimensions are zero",__func__), (NULL));
  652.     return GST_FLOW_ERROR;
  653.   }
  655. /* rectangle for cropped objects */
  656.   crop_rect = cv::Rect (crop_rect_params->left, crop_rect_params->top,
  657.   crop_rect_params->width, crop_rect_params->height);
  659. /* apply gaussian blur to the detected objects */
  660.   GaussianBlur(in_mat(crop_rect), in_mat(crop_rect), cv::Size(15,15), 4);
  661.   return GST_FLOW_OK;
  662. }
  664. #endif
  665. /* * Blur the detected objects when processing in object mode (full-frame=0) */
  666. static GstFlowRetur blur_objects (GstDsExample * dsexample, gint idx,
  667.     NvOSD_RectParams * crop_rect_params, cv::Mat in_mat)
  668. { 
  669.     int hrst=1;
  670.     cv::Rect crop_rect;
  671.     static unsigned int jiaozheng_init_flag = 0;
  672.     if ((crop_rect_params->width == 0) || (crop_rect_params->height == 0)) 
  673.     { 
  674.         GST_ELEMENT_ERROR (dsexample, STREAM, FAILED,
  675.         ("%s:crop_rect_params dimensions are zero",__func__), (NULL));
  676.         return GST_FLOW_ERROR;
  677.     }
  678.     steady_clock::time_point t1 = steady_clock::now();
  679.     if (!jiaozheng_init_flag)
  680.     { 
  681.         printf("初始化\n");
  682.         jiaozheng_init();
  683.         jiaozheng_init_flag = 1;
  684.     }
  685.     hrst = jibianjiaozheng(in_mat, in_mat);
  686.     if (hrst != 0)
  687.     { 
  688.         printf("畸变矫正失败\n");
  689.     }
  690.     else
  691.     { 
  692.         //printf("畸变矫正成功\n");
  693.     }
  694.     /*time cost calc code */
  695.     steady_clock::time_point t2 = steady_clock::now();
  696.     duration<double> time_used = duration_cast<duration<double>>(t2 - t1);
  697.     cout<" time cost= "<< (time_used.count() * 1000) <<" ms."<<endl;
  699.     return GST_FLOW_OK;
  700. }
  704. /** * Called when element recieves an input buffer from upstream element. */
  705. static GstFlowReturn
  706. gst_dsexample_transform_ip (GstBaseTransform * btrans, GstBuffer * inbuf)
  707. { 
  708.   GstDsExample *dsexample = GST_DSEXAMPLE (btrans);
  709.   GstMapInfo in_map_info;
  710.   GstFlowReturn flow_ret = GST_FLOW_ERROR;
  711.   gdouble scale_ratio = 1.0;
  712.   DsExampleOutput *output;
  714.   NvBufSurface *surface = NULL;
  715.   NvDsBatchMeta *batch_meta = NULL;
  716.   NvDsFrameMeta *frame_meta = NULL;
  717.   NvDsMetaList * l_frame = NULL;
  718.   guint i = 0;
  720.   dsexample->frame_num++;
  721.   CHECK_CUDA_STATUS (cudaSetDevice (dsexample->gpu_id),
  722.       "Unable to set cuda device");
  724.   memset (&in_map_info, 0, sizeof (in_map_info));
  725.   if (!gst_buffer_map (inbuf, &in_map_info, GST_MAP_READ)) { 
  726.     g_print ("Error: Failed to map gst buffer\n");
  727.     goto error;
  728.   }
  730.   nvds_set_input_system_timestamp (inbuf, GST_ELEMENT_NAME (dsexample));
  731.   surface = (NvBufSurface *) in_map_info.data;
  732.   GST_DEBUG_OBJECT (dsexample,
  733.       "Processing Frame %" G_GUINT64_FORMAT " Surface %p\n",
  734.       dsexample->frame_num, surface);
  736.   if (CHECK_NVDS_MEMORY_AND_GPUID (dsexample, surface))
  737.     goto error;
  739.   batch_meta = gst_buffer_get_nvds_batch_meta (inbuf);
  740.   if (batch_meta == nullptr) { 
  741.     GST_ELEMENT_ERROR (dsexample, STREAM, FAILED,
  742.         ("NvDsBatchMeta not found for input buffer."), (NULL));
  743.     return GST_FLOW_ERROR;
  744.   }
  746.   if (dsexample->process_full_frame) { 
  747.     for (l_frame = batch_meta->frame_meta_list; l_frame != NULL;
  748.       l_frame = l_frame->next)
  749.     { 
  750.       frame_meta = (NvDsFrameMeta *) (l_frame->data);
  751.       NvOSD_RectParams rect_params;
  753.       /* Scale the entire frame to processing resolution */
  754.       rect_params.left = 0;
  755.       rect_params.top = 0;
  756.       rect_params.width = dsexample->video_info.width;
  757.       rect_params.height = dsexample->video_info.height;
  759.       /* Scale and convert the frame */
  760.       if (get_converted_mat (dsexample, surface, i, &rect_params,
  761.             scale_ratio, dsexample->video_info.width,
  762.             dsexample->video_info.height) != GST_FLOW_OK) { 
  763.         goto error;
  764.       }
  766.       /* Process to get the output */
  767.       output =
  768.           DsExampleProcess (dsexample->dsexamplelib_ctx,
  769.           dsexample->cvmat->data);
  770.       /* Attach the metadata for the full frame */
  771.       attach_metadata_full_frame (dsexample, frame_meta, scale_ratio, output, i);
  772.       i++;
  773.       free (output);
  774.     }    
  777.   } else { 
  778.     /* Using object crops as input to the algorithm. The objects are detected by * the primary detector */
  779.     NvDsMetaList * l_obj = NULL;
  780.     NvDsObjectMeta *obj_meta = NULL;
  782. #ifndef __aarch64__
  783.     if (dsexample->blur_objects) { 
  784.       if (surface->memType != NVBUF_MEM_CUDA_UNIFIED){ 
  785.         GST_ELEMENT_ERROR (dsexample, STREAM, FAILED,
  786.             ("%s:need NVBUF_MEM_CUDA_UNIFIED memory for opencv blurring",__func__), (NULL));
  787.         return GST_FLOW_ERROR;
  788.       }
  789.     }
  790. #endif
  792.     for (l_frame = batch_meta->frame_meta_list; l_frame != NULL;
  793.       l_frame = l_frame->next)
  794.     { 
  795.       frame_meta = (NvDsFrameMeta *) (l_frame->data);
  796.       cv::Mat in_mat;
  798.       if (dsexample->blur_objects) { 
  799.         /* Map the buffer so that it can be accessed by CPU */
  800.         if (surface->surfaceList[frame_meta->batch_id].mappedAddr.addr[0] == NULL){ 
  801.           if (NvBufSurfaceMap (surface, frame_meta->batch_id, 0, NVBUF_MAP_READ_WRITE) != 0){ 
  802.             GST_ELEMENT_ERROR (dsexample, STREAM, FAILED,
  803.                 ("%s:buffer map to be accessed by CPU failed", __func__), (NULL));
  804.             return GST_FLOW_ERROR;
  805.           }
  806.         }
  808.         /* Cache the mapped data for CPU access */
  809.         NvBufSurfaceSyncForCpu (surface, frame_meta->batch_id, 0);
  811.         in_mat =
  812.             cv::Mat (surface->surfaceList[frame_meta->batch_id].planeParams.height[0],
  813.             surface->surfaceList[frame_meta->batch_id].planeParams.width[0], CV_8UC4,
  814.             surface->surfaceList[frame_meta->batch_id].mappedAddr.addr[0],
  815.             surface->surfaceList[frame_meta->batch_id].planeParams.pitch[0]);
  816.       }
  818.       for (l_obj = frame_meta->obj_meta_list; l_obj != NULL;
  819.           l_obj = l_obj->next)
  820.       { 
  821.         obj_meta = (NvDsObjectMeta *) (l_obj->data);
  823.         if (dsexample->blur_objects) { 
  824.           /* gaussian blur the detected objects using opencv */
  825.           if (blur_objects (dsexample, frame_meta->batch_id,
  826.             &obj_meta->rect_params, in_mat) != GST_FLOW_OK) { 
  827.           /* Error in blurring, skip processing on object. */
  828.             GST_ELEMENT_ERROR (dsexample, STREAM, FAILED,
  829.             ("blurring the object failed"), (NULL));
  830.             if (NvBufSurfaceUnMap (surface, frame_meta->batch_id, 0)){ 
  831.               GST_ELEMENT_ERROR (dsexample, STREAM, FAILED,
  832.                 ("%s:buffer unmap to be accessed by CPU failed", __func__), (NULL));
  833.             }
  834.             return GST_FLOW_ERROR;
  835.           }
  836.           continue;
  837.         }
  839.         /* Should not process on objects smaller than MIN_INPUT_OBJECT_WIDTH x MIN_INPUT_OBJECT_HEIGHT * since it will cause hardware scaling issues. */
  840.         if (obj_meta->rect_params.width < MIN_INPUT_OBJECT_WIDTH ||
  841.             obj_meta->rect_params.height < MIN_INPUT_OBJECT_HEIGHT)
  842.           continue;
  844.         /* Crop and scale the object */
  845.         if (get_converted_mat (dsexample,
  846.               surface, frame_meta->batch_id, &obj_meta->rect_params,
  847.               scale_ratio, dsexample->video_info.width,
  848.               dsexample->video_info.height) != GST_FLOW_OK) { 
  849.           /* Error in conversion, skip processing on object. */
  850.           continue;
  851.         }
  853.         /* Process the object crop to obtain label */
  854.         output = DsExampleProcess (dsexample->dsexamplelib_ctx,
  855.             dsexample->cvmat->data);
  857.         /* Attach labels for the object */
  858.         attach_metadata_object (dsexample, obj_meta, output);
  860.         free (output);
  861.       }
  863.       if (dsexample->blur_objects) { 
  864.       /* Cache the mapped data for device access */
  865.         NvBufSurfaceSyncForDevice (surface, frame_meta->batch_id, 0);
  867. #ifdef DSEXAMPLE_DEBUG
  868.         /* Use openCV to remove padding and convert RGBA to BGR. Can be skipped if * algorithm can handle padded RGBA data. */
  869. #if (CV_MAJOR_VERSION >= 4)
  870.         cv::cvtColor (in_mat, *dsexample->cvmat, cv::COLOR_RGBA2BGR);
  871. #else
  872.         cv::cvtColor (in_mat, *dsexample->cvmat, CV_RGBA2BGR);
  873. #endif
  874.         /* used to dump the converted mat to files for debug */
  875.         static guint cnt = 0;
  876.         cv::imwrite("out_" + std::to_string (cnt) + ".jpeg", *dsexample->cvmat);
  877.         cnt++;
  878. #endif
  879.       }
  880.     }
  881.   }
  882.   flow_ret = GST_FLOW_OK;
  884. error:
  886.   nvds_set_output_system_timestamp (inbuf, GST_ELEMENT_NAME (dsexample));
  887.   gst_buffer_unmap (inbuf, &in_map_info);
  888.   return flow_ret;
  889. }
  891. /** * Attach metadata for the full frame. We will be adding a new metadata. */
  892. static void
  893. attach_metadata_full_frame (GstDsExample * dsexample, NvDsFrameMeta *frame_meta,
  894.     gdouble scale_ratio, DsExampleOutput * output, guint batch_id)
  895. { 
  896.   NvDsBatchMeta *batch_meta = frame_meta->base_meta.batch_meta;
  897.   NvDsObjectMeta *object_meta = NULL;
  898.   static gchar font_name[] = "Serif";
  899.   GST_DEBUG_OBJECT (dsexample, "Attaching metadata %d\n", output->numObjects);
  901.   for (gint i = 0; i < output->numObjects; i++) { 
  902.     DsExampleObject *obj = &output->object[i];
  903.     object_meta = nvds_acquire_obj_meta_from_pool(batch_meta);
  904.     NvOSD_RectParams & rect_params = object_meta->rect_params;
  905.     NvOSD_TextParams & text_params = object_meta->text_params;
  907.     /* Assign bounding box coordinates */
  908.     rect_params.left = obj->left;
  909.     rect_params.top = obj->top;
  910.     rect_params.width = obj->width;
  911.     rect_params.height = obj->height;
  913.     /* Semi-transparent yellow background */
  914.     rect_params.has_bg_color = 0;
  915.     rect_params.bg_color = (NvOSD_ColorParams) { 
  916.     1, 1, 0, 0.4};
  917.     /* Red border of width 6 */
  918.     rect_params.border_width = 3;
  919.     rect_params.border_color = (NvOSD_ColorParams) { 
  920.     1, 0, 0, 1};
  922.     /* Scale the bounding boxes proportionally based on how the object/frame was * scaled during input */
  923.     rect_params.left /= scale_ratio;
  924.     rect_params.top /= scale_ratio;
  925.     rect_params.width /= scale_ratio;
  926.     rect_params.height /= scale_ratio;
  927.     GST_DEBUG_OBJECT (dsexample, "Attaching rect%d of batch%u"
  928.         " left->%f top->%f width->%f"
  929.         " height->%f label->%s\n", i, batch_id, rect_params.left,
  930.         rect_params.top, rect_params.width, rect_params.height, obj->label);
  932.     object_meta->object_id = UNTRACKED_OBJECT_ID;
  933.     g_strlcpy (object_meta->obj_label, obj->label, MAX_LABEL_SIZE);
  934.     /* display_text required heap allocated memory */
  935.     text_params.display_text = g_strdup (obj->label);
  936.     /* Display text above the left top corner of the object */
  937.     text_params.x_offset = rect_params.left;
  938.     text_params.y_offset = rect_params.top - 10;
  939.     /* Set black background for the text */
  940.     text_params.set_bg_clr = 1;
  941.     text_params.text_bg_clr = (NvOSD_ColorParams) { 
  942.     0, 0, 0, 1};
  943.     /* Font face, size and color */
  944.     text_params.font_params.font_name = font_name;
  945.     text_params.font_params.font_size = 11;
  946.     text_params.font_params.font_color = (NvOSD_ColorParams) { 
  947.     1, 1, 1, 1};
  949.     nvds_add_obj_meta_to_frame(frame_meta, object_meta, NULL);
  950.     frame_meta->bInferDone = TRUE;
  951.   }
  952. }
  954. /** * Only update string label in an existing object metadata. No bounding boxes. * We assume only one label per object is generated */
  955. static void
  956. attach_metadata_object (GstDsExample * dsexample, NvDsObjectMeta * obj_meta,
  957.     DsExampleOutput * output)
  958. { 
  959.   if (output->numObjects == 0)
  960.     return;
  961.   NvDsBatchMeta *batch_meta = obj_meta->base_meta.batch_meta;
  963.   NvDsClassifierMeta *classifier_meta =
  964.     nvds_acquire_classifier_meta_from_pool (batch_meta);
  966.   classifier_meta->unique_component_id = dsexample->unique_id;
  968.   NvDsLabelInfo *label_info =
  969.     nvds_acquire_label_info_meta_from_pool (batch_meta);
  970.   g_strlcpy (label_info->result_label, output->object[0].label, MAX_LABEL_SIZE);
  971.   nvds_add_label_info_meta_to_classifier(classifier_meta, label_info);
  972.   nvds_add_classifier_meta_to_object (obj_meta, classifier_meta);
  974.   nvds_acquire_meta_lock (batch_meta);
  975.   NvOSD_TextParams & text_params = obj_meta->text_params;
  976.   NvOSD_RectParams & rect_params = obj_meta->rect_params;
  978.   /* Below code to display the result */
  979.   /* Set black background for the text * display_text required heap allocated memory */
  980.   if (text_params.display_text) { 
  981.     gchar *conc_string = g_strconcat (text_params.display_text, " ",
  982.         output->object[0].label, NULL);
  983.     g_free (text_params.display_text);
  984.     text_params.display_text = conc_string;
  985.   } else { 
  986.     /* Display text above the left top corner of the object */
  987.     text_params.x_offset = rect_params.left;
  988.     text_params.y_offset = rect_params.top - 10;
  989.     text_params.display_text = g_strdup (output->object[0].label);
  990.     /* Font face, size and color */
  991.     text_params.font_params.font_name = (char *)"Serif";
  992.     text_params.font_params.font_size = 11;
  993.     text_params.font_params.font_color = (NvOSD_ColorParams) { 
  994.     1, 1, 1, 1};
  995.     /* Set black background for the text */
  996.     text_params.set_bg_clr = 1;
  997.     text_params.text_bg_clr = (NvOSD_ColorParams) { 
  998.     0, 0, 0, 1};
  999.   }
  1000.   nvds_release_meta_lock (batch_meta);
  1001. }
  1003. /** * Boiler plate for registering a plugin and an element. */
  1004. static gboolean
  1005. dsexample_plugin_init (GstPlugin * plugin)
  1006. { 
  1007.   GST_DEBUG_CATEGORY_INIT (gst_dsexample_debug, "dsexample", 0,
  1008.       "dsexample plugin");
  1010.   return gst_element_register (plugin, "dsexample", GST_RANK_PRIMARY,
  1011.       GST_TYPE_DSEXAMPLE);
  1012. }
  1014. GST_PLUGIN_DEFINE (GST_VERSION_MAJOR,
  1015.     GST_VERSION_MINOR,
  1016.     nvdsgst_dsexample,
  1017.     DESCRIPTION, dsexample_plugin_init, DS_VERSION, LICENSE, BINARY_PACKAGE, URL)

new

  1. /** * Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER * DEALINGS IN THE SOFTWARE. */
  3. #include <string.h>
  4. #include <string>
  5. #include <sstream>
  6. #include <iostream>
  7. #include <ostream>
  8. #include <fstream>
  9. #include "gstdsexample.h"
  10. #include <sys/time.h>
  13. #include "opencv2/core/core.hpp" 
  14. #include "opencv2/imgproc/imgproc.hpp" 
  15. #include "opencv2/calib3d/calib3d.hpp" 
  16. #include "opencv2/highgui/highgui.hpp" 
  17. using namespace cv;
  18. using namespace std;
  21. GST_DEBUG_CATEGORY_STATIC (gst_dsexample_debug);
  22. #define GST_CAT_DEFAULT gst_dsexample_debug
  23. #define USE_EGLIMAGE 1
  24. /* enable to write transformed cvmat to files */
  25. /* #define DSEXAMPLE_DEBUG */
  26. static GQuark _dsmeta_quark = 0;
  28. /* Enum to identify properties */
  29. enum
  30. { 
  31.   PROP_0,
  32.   PROP_UNIQUE_ID,
  33.   PROP_PROCESSING_WIDTH,
  34.   PROP_PROCESSING_HEIGHT,
  35.   PROP_PROCESS_FULL_FRAME,
  36.   PROP_BLUR_OBJECTS,
  37.   PROP_GPU_DEVICE_ID
  38. };
  40. #define CHECK_NVDS_MEMORY_AND_GPUID(object, surface) \ ({ int _errtype=0;\ do { \ if ((surface->memType == NVBUF_MEM_DEFAULT || surface->memType == NVBUF_MEM_CUDA_DEVICE) && \ (surface->gpuId != object->gpu_id)) { \ GST_ELEMENT_ERROR (object, RESOURCE, FAILED, \ ("Input surface gpu-id doesnt match with configured gpu-id for element," \ " please allocate input using unified memory, or use same gpu-ids"),\ ("surface-gpu-id=%d,%s-gpu-id=%d",surface->gpuId,GST_ELEMENT_NAME(object),\ object->gpu_id)); \ _errtype = 1;\ } \ } while(0); \ _errtype; \ })
  43. /* Default values for properties */
  44. #define DEFAULT_UNIQUE_ID 15
  45. #define DEFAULT_PROCESSING_WIDTH 640
  46. #define DEFAULT_PROCESSING_HEIGHT 480
  47. #define DEFAULT_PROCESS_FULL_FRAME TRUE
  48. #define DEFAULT_BLUR_OBJECTS FALSE
  49. #define DEFAULT_GPU_ID 0
  51. #define RGB_BYTES_PER_PIXEL 3
  52. #define RGBA_BYTES_PER_PIXEL 4
  53. #define Y_BYTES_PER_PIXEL 1
  54. #define UV_BYTES_PER_PIXEL 2
  56. #define MIN_INPUT_OBJECT_WIDTH 16
  57. #define MIN_INPUT_OBJECT_HEIGHT 16
  59. #define CHECK_NPP_STATUS(npp_status,error_str) do { \ if ((npp_status) != NPP_SUCCESS) { \ g_print ("Error: %s in %s at line %d: NPP Error %d\n", \ error_str, __FILE__, __LINE__, npp_status); \ goto error; \ } \ } while (0)
  61. #define CHECK_CUDA_STATUS(cuda_status,error_str) do { \ if ((cuda_status) != cudaSuccess) { \ g_print ("Error: %s in %s at line %d (%s)\n", \ error_str, __FILE__, __LINE__, cudaGetErrorName(cuda_status)); \ goto error; \ } \ } while (0)
  63. /* By default NVIDIA Hardware allocated memory flows through the pipeline. We * will be processing on this type of memory only. */
  64. #define GST_CAPS_FEATURE_MEMORY_NVMM "memory:NVMM"
  65. static GstStaticPadTemplate gst_dsexample_sink_template =
  66. GST_STATIC_PAD_TEMPLATE ("sink",
  67.     GST_PAD_SINK,
  68.     GST_PAD_ALWAYS,
  69.     GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
  70.         (GST_CAPS_FEATURE_MEMORY_NVMM,
  71.             "{ NV12, RGBA, I420 }")));
  73. static GstStaticPadTemplate gst_dsexample_src_template =
  74. GST_STATIC_PAD_TEMPLATE ("src",
  75.     GST_PAD_SRC,
  76.     GST_PAD_ALWAYS,
  77.     GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
  78.         (GST_CAPS_FEATURE_MEMORY_NVMM,
  79.             "{ NV12, RGBA, I420 }")));
  81. /* Define our element type. Standard GObject/GStreamer boilerplate stuff */
  82. #define gst_dsexample_parent_class parent_class
  83. G_DEFINE_TYPE (GstDsExample, gst_dsexample, GST_TYPE_BASE_TRANSFORM);
  85. static void gst_dsexample_set_property (GObject * object, guint prop_id,
  86.     const GValue * value, GParamSpec * pspec);
  87. static void gst_dsexample_get_property (GObject * object, guint prop_id,
  88.     GValue * value, GParamSpec * pspec);
  90. static gboolean gst_dsexample_set_caps (GstBaseTransform * btrans,
  91.     GstCaps * incaps, GstCaps * outcaps);
  92. static gboolean gst_dsexample_start (GstBaseTransform * btrans);
  93. static gboolean gst_dsexample_stop (GstBaseTransform * btrans);
  95. static GstFlowReturn gst_dsexample_transform_ip (GstBaseTransform *
  96.     btrans, GstBuffer * inbuf);
  98. static void
  99. attach_metadata_full_frame (GstDsExample * dsexample, NvDsFrameMeta *frame_meta,
  100.     gdouble scale_ratio, DsExampleOutput * output, guint batch_id);
  101. static void attach_metadata_object (GstDsExample * dsexample,
  102.     NvDsObjectMeta * obj_meta, DsExampleOutput * output);
  108. void InitMat(Mat& m, float* num)
  109. { 
  110.     for (int i = 0; i<m.rows; i++)
  111.     for (int j = 0; j<m.cols; j++)
  112.         m.at<float>(i, j) = *(num + i * m.rows + j);
  113. }
  115. /******************************************************************************* Function: jibianjiaozheng Description: 该函数实现畸变矫正 Input: 原图 Output: 畸变矫正后的图片 Return: 0: Successful ohters: Failed *******************************************************************************/
  116. int jibianjiaozheng(Mat src_picture, Mat & dst_picture)
  117. { 
  118.     int OK = 0;
  119.     int i = 1000;
  120.     int n = 1;
  121.     Mat edges;
  122.     Mat R = Mat::eye(3, 3, CV_32F);
  124.     if (src_picture.data == NULL)
  125.     { 
  126.         printf("param error,input Image is empty\n");
  127.         return -1;
  128.     }
  130.     Size image_size;  /* 图像的尺寸 */
  131.                       //获取图像大小
  132.     image_size.width = 1920;
  133.     image_size.height = 1080;
  135.     //cameraMatrix为 "相机内参数矩阵：" << endl;
  136.     Mat mapx = Mat(image_size, CV_32FC1);
  137.     Mat mapy = Mat(image_size, CV_32FC1);
  139.     //参数矩阵
  140.     float neican_data[] = {  9558.649257742036, 0, 959.3165310990756, 0, 9435.752651759443, 532.7507141910969, 0, 0, 1 };
  141.     Mat cameraMatrix(3, 3, CV_32FC1);
  142.     InitMat(cameraMatrix, neican_data);
  144.     //cout << "cameraMatrix= " << endl << " " << cameraMatrix << endl << endl;
  145.     //测得的畸变系数
  146.     float jibian_data[] = {  -6.956561513881647, -68.83902522804168, -0.004834538444671919, 0.01471273691928269, -0.4916103704308509 };
  147.     Mat distCoeffs(1, 5, CV_32FC1); /* 摄像机的5个畸变系数：k1,k2,p1,p2,k3 */
  148.     InitMat(distCoeffs, jibian_data);
  149.     //cout << "distCoeffs= " << endl << " " << distCoeffs << endl << endl;
  152.     i = 0;
  153.     //namedWindow("【原始图】", 0);//参数为零，则可以自由拖动
  154.     //imshow("【原始图】", src_picture);
  155.     /********相机矫正*******************************************************************************/
  157.     initUndistortRectifyMap(cameraMatrix, distCoeffs, R, cameraMatrix, image_size, CV_32FC1, mapx, mapy);
  160.     Mat imageSource = src_picture; //读取畸变图片
  161.     Mat newimage = imageSource.clone(); //校正后输出图片
  163.     remap(imageSource, newimage, mapx, mapy, INTER_LINEAR);
  165.     //namedWindow("畸变校正后的图片", 0);//参数为零，则可以自由拖动
  166.     //imshow("畸变校正后的图片", newimage);
  169.     dst_picture= newimage+1;
  170.     return OK;
  171. }
  177. /* Install properties, set sink and src pad capabilities, override the required * functions of the base class, These are common to all instances of the * element. */
  178. static void
  179. gst_dsexample_class_init (GstDsExampleClass * klass)
  180. { 
  181.   GObjectClass *gobject_class;
  182.   GstElementClass *gstelement_class;
  183.   GstBaseTransformClass *gstbasetransform_class;
  185.   /* Indicates we want to use DS buf api */
  186.   g_setenv ("DS_NEW_BUFAPI", "1", TRUE);
  188.   gobject_class = (GObjectClass *) klass;
  189.   gstelement_class = (GstElementClass *) klass;
  190.   gstbasetransform_class = (GstBaseTransformClass *) klass;
  192.   /* Overide base class functions */
  193.   gobject_class->set_property = GST_DEBUG_FUNCPTR (gst_dsexample_set_property);
  194.   gobject_class->get_property = GST_DEBUG_FUNCPTR (gst_dsexample_get_property);
  196.   gstbasetransform_class->set_caps = GST_DEBUG_FUNCPTR (gst_dsexample_set_caps);
  197.   gstbasetransform_class->start = GST_DEBUG_FUNCPTR (gst_dsexample_start);
  198.   gstbasetransform_class->stop = GST_DEBUG_FUNCPTR (gst_dsexample_stop);
  200.   gstbasetransform_class->transform_ip =
  201.       GST_DEBUG_FUNCPTR (gst_dsexample_transform_ip);
  203.   /* Install properties */
  204.   g_object_class_install_property (gobject_class, PROP_UNIQUE_ID,
  205.       g_param_spec_uint ("unique-id",
  206.           "Unique ID",
  207.           "Unique ID for the element. Can be used to identify output of the"
  208.           " element", 0, G_MAXUINT, DEFAULT_UNIQUE_ID, (GParamFlags)
  209.           (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
  211.   g_object_class_install_property (gobject_class, PROP_PROCESSING_WIDTH,
  212.       g_param_spec_int ("processing-width",
  213.           "Processing Width",
  214.           "Width of the input buffer to algorithm",
  215.           1, G_MAXINT, DEFAULT_PROCESSING_WIDTH, (GParamFlags)
  216.           (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
  218.   g_object_class_install_property (gobject_class, PROP_PROCESSING_HEIGHT,
  219.       g_param_spec_int ("processing-height",
  220.           "Processing Height",
  221.           "Height of the input buffer to algorithm",
  222.           1, G_MAXINT, DEFAULT_PROCESSING_HEIGHT, (GParamFlags)
  223.           (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
  225.   g_object_class_install_property (gobject_class, PROP_PROCESS_FULL_FRAME,
  226.       g_param_spec_boolean ("full-frame",
  227.           "Full frame",
  228.           "Enable to process full frame or disable to process objects detected"
  229.           "by primary detector", DEFAULT_PROCESS_FULL_FRAME, (GParamFlags)
  230.           (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
  232.   g_object_class_install_property (gobject_class, PROP_BLUR_OBJECTS,
  233.       g_param_spec_boolean ("blur-objects",
  234.           "Blur Objects",
  235.           "Enable to blur the objects detected in full-frame=0 mode"
  236.           "by primary detector", DEFAULT_BLUR_OBJECTS, (GParamFlags)
  237.           (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
  239.   g_object_class_install_property (gobject_class, PROP_GPU_DEVICE_ID,
  240.       g_param_spec_uint ("gpu-id",
  241.           "Set GPU Device ID",
  242.           "Set GPU Device ID", 0,
  243.           G_MAXUINT, 0,
  244.           GParamFlags
  245.           (G_PARAM_READWRITE |
  246.               G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_READY)));
  247.   /* Set sink and src pad capabilities */
  248.   gst_element_class_add_pad_template (gstelement_class,
  249.       gst_static_pad_template_get (&gst_dsexample_src_template));
  250.   gst_element_class_add_pad_template (gstelement_class,
  251.       gst_static_pad_template_get (&gst_dsexample_sink_template));
  253.   /* Set metadata describing the element */
  254.   gst_element_class_set_details_simple (gstelement_class,
  255.       "DsExample plugin",
  256.       "DsExample Plugin",
  257.       "Process a 3rdparty example algorithm on objects / full frame",
  258.       "NVIDIA Corporation. Post on Deepstream for Tesla forum for any queries "
  259.       "@ https://devtalk.nvidia.com/default/board/209/");
  260. }
  262. static void
  263. gst_dsexample_init (GstDsExample * dsexample)
  264. { 
  265.   GstBaseTransform *btrans = GST_BASE_TRANSFORM (dsexample);
  267.   /* We will not be generating a new buffer. Just adding / updating * metadata. */
  268.   gst_base_transform_set_in_place (GST_BASE_TRANSFORM (btrans), TRUE);
  269.   /* We do not want to change the input caps. Set to passthrough. transform_ip * is still called. */
  270.   gst_base_transform_set_passthrough (GST_BASE_TRANSFORM (btrans), TRUE);
  272.   /* Initialize all property variables to default values */
  273.   dsexample->unique_id = DEFAULT_UNIQUE_ID;
  274.   dsexample->processing_width = DEFAULT_PROCESSING_WIDTH;
  275.   dsexample->processing_height = DEFAULT_PROCESSING_HEIGHT;
  276.   dsexample->process_full_frame = DEFAULT_PROCESS_FULL_FRAME;
  277.   dsexample->blur_objects = DEFAULT_BLUR_OBJECTS;
  278.   dsexample->gpu_id = DEFAULT_GPU_ID;
  279.   /* This quark is required to identify NvDsMeta when iterating through * the buffer metadatas */
  280.   if (!_dsmeta_quark)
  281.     _dsmeta_quark = g_quark_from_static_string (NVDS_META_STRING);
  282. }
  284. /* Function called when a property of the element is set. Standard boilerplate. */
  285. static void
  286. gst_dsexample_set_property (GObject * object, guint prop_id,
  287.     const GValue * value, GParamSpec * pspec)
  288. { 
  289.   GstDsExample *dsexample = GST_DSEXAMPLE (object);
  290.   switch (prop_id) { 
  291.     case PROP_UNIQUE_ID:
  292.       dsexample->unique_id = g_value_get_uint (value);
  293.       break;
  294.     case PROP_PROCESSING_WIDTH:
  295.       dsexample->processing_width = g_value_get_int (value);
  296.       break;
  297.     case PROP_PROCESSING_HEIGHT:
  298.       dsexample->processing_height = g_value_get_int (value);
  299.       break;
  300.     case PROP_PROCESS_FULL_FRAME:
  301.       dsexample->process_full_frame = g_value_get_boolean (value);
  302.       break;
  303.     case PROP_BLUR_OBJECTS:
  304.       dsexample->blur_objects = g_value_get_boolean (value);
  305.       break;
  306.     case PROP_GPU_DEVICE_ID:
  307.       dsexample->gpu_id = g_value_get_uint (value);
  308.       break;
  309.     default:
  310.       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
  311.       break;
  312.   }
  313. }
  315. /* Function called when a property of the element is requested. Standard * boilerplate. */
  316. static void
  317. gst_dsexample_get_property (GObject * object, guint prop_id,
  318.     GValue * value, GParamSpec * pspec)
  319. { 
  320.   GstDsExample *dsexample = GST_DSEXAMPLE (object);
  322.   switch (prop_id) { 
  323.     case PROP_UNIQUE_ID:
  324.       g_value_set_uint (value, dsexample->unique_id);
  325.       break;
  326.     case PROP_PROCESSING_WIDTH:
  327.       g_value_set_int (value, dsexample->processing_width);
  328.       break;
  329.     case PROP_PROCESSING_HEIGHT:
  330.       g_value_set_int (value, dsexample->processing_height);
  331.       break;
  332.     case PROP_PROCESS_FULL_FRAME:
  333.       g_value_set_boolean (value, dsexample->process_full_frame);
  334.       break;
  335.     case PROP_BLUR_OBJECTS:
  336.       g_value_set_boolean (value, dsexample->blur_objects);
  337.       break;
  338.     case PROP_GPU_DEVICE_ID:
  339.       g_value_set_uint (value, dsexample->gpu_id);
  340.       break;
  341.     default:
  342.       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
  343.       break;
  344.   }
  345. }
  347. /** * Initialize all resources and start the output thread */
  348. static gboolean
  349. gst_dsexample_start (GstBaseTransform * btrans)
  350. { 
  351.   GstDsExample *dsexample = GST_DSEXAMPLE (btrans);
  352.   NvBufSurfaceCreateParams create_params;
  353.   DsExampleInitParams init_params =
  354.       {  dsexample->processing_width, dsexample->processing_height,
  355.     dsexample->process_full_frame
  356.   };
  358.   GstQuery *queryparams = NULL;
  359.   guint batch_size = 1;
  361.   /* Algorithm specific initializations and resource allocation. */
  362.   dsexample->dsexamplelib_ctx = DsExampleCtxInit (&init_params);
  364.   GST_DEBUG_OBJECT (dsexample, "ctx lib %p \n", dsexample->dsexamplelib_ctx);
  366.   CHECK_CUDA_STATUS (cudaSetDevice (dsexample->gpu_id),
  367.       "Unable to set cuda device");
  369.   dsexample->batch_size = 1;
  370.   queryparams = gst_nvquery_batch_size_new ();
  371.   if (gst_pad_peer_query (GST_BASE_TRANSFORM_SINK_PAD (btrans), queryparams)
  372.       || gst_pad_peer_query (GST_BASE_TRANSFORM_SRC_PAD (btrans), queryparams)) { 
  373.     if (gst_nvquery_batch_size_parse (queryparams, &batch_size)) { 
  374.       dsexample->batch_size = batch_size;
  375.     }
  376.   }
  377.   GST_DEBUG_OBJECT (dsexample, "Setting batch-size %d \n",
  378.       dsexample->batch_size);
  379.   gst_query_unref (queryparams);
  381.   if (dsexample->process_full_frame && dsexample->blur_objects) { 
  382.     GST_ERROR ("Error: does not support blurring while processing full frame");
  383.     goto error;
  384.   }
  386.   CHECK_CUDA_STATUS (cudaStreamCreate (&dsexample->cuda_stream),
  387.       "Could not create cuda stream");
  389.   if (dsexample->inter_buf)
  390.     NvBufSurfaceDestroy (dsexample->inter_buf);
  391.   dsexample->inter_buf = NULL;
  393.   /* An intermediate buffer for NV12/RGBA to BGR conversion will be * required. Can be skipped if custom algorithm can work directly on NV12/RGBA. */
  394.   create_params.gpuId  = dsexample->gpu_id;
  395.   create_params.width  = dsexample->processing_width;
  396.   create_params.height = dsexample->processing_height;
  397.   create_params.size = 0;
  398.   create_params.colorFormat = NVBUF_COLOR_FORMAT_RGBA;
  399.   create_params.layout = NVBUF_LAYOUT_PITCH;
  400. #ifdef __aarch64__
  401.   create_params.memType = NVBUF_MEM_DEFAULT;
  402. #else
  403.   create_params.memType = NVBUF_MEM_CUDA_UNIFIED;
  404. #endif
  406.   if (NvBufSurfaceCreate (&dsexample->inter_buf, 1,
  407.           &create_params) != 0) { 
  408.     GST_ERROR ("Error: Could not allocate internal buffer for dsexample");
  409.     goto error;
  410.   }
  412.   /* Create host memory for storing converted/scaled interleaved RGB data */
  413.   CHECK_CUDA_STATUS (cudaMallocHost (&dsexample->host_rgb_buf,
  414.           dsexample->processing_width * dsexample->processing_height *
  415.           RGB_BYTES_PER_PIXEL), "Could not allocate cuda host buffer");
  417.   GST_DEBUG_OBJECT (dsexample, "allocated cuda buffer %p \n",
  418.       dsexample->host_rgb_buf);
  420.   /* CV Mat containing interleaved RGB data. This call does not allocate memory. * It uses host_rgb_buf as data. */
  421.   dsexample->cvmat =
  422.       new cv::Mat (dsexample->processing_height, dsexample->processing_width,
  423.       CV_8UC3, dsexample->host_rgb_buf,
  424.       dsexample->processing_width * RGB_BYTES_PER_PIXEL);
  426.   if (!dsexample->cvmat)
  427.     goto error;
  429.   GST_DEBUG_OBJECT (dsexample, "created CV Mat\n");
  431.   return TRUE;
  432. error:
  433.   if (dsexample->host_rgb_buf) { 
  434.     cudaFreeHost (dsexample->host_rgb_buf);
  435.     dsexample->host_rgb_buf = NULL;
  436.   }
  438.   if (dsexample->cuda_stream) { 
  439.     cudaStreamDestroy (dsexample->cuda_stream);
  440.     dsexample->cuda_stream = NULL;
  441.   }
  442.   if (dsexample->dsexamplelib_ctx)
  443.     DsExampleCtxDeinit (dsexample->dsexamplelib_ctx);
  444.   return FALSE;
  445. }
  447. /** * Stop the output thread and free up all the resources */
  448. static gboolean
  449. gst_dsexample_stop (GstBaseTransform * btrans)
  450. { 
  451.   GstDsExample *dsexample = GST_DSEXAMPLE (btrans);
  453.   if (dsexample->inter_buf)
  454.     NvBufSurfaceDestroy(dsexample->inter_buf);
  455.   dsexample->inter_buf = NULL;
  457.   if (dsexample->cuda_stream)
  458.     cudaStreamDestroy (dsexample->cuda_stream);
  459.   dsexample->cuda_stream = NULL;
  461.   delete dsexample->cvmat;
  462.   dsexample->cvmat = NULL;
  464.   if (dsexample->host_rgb_buf) { 
  465.     cudaFreeHost (dsexample->host_rgb_buf);
  466.     dsexample->host_rgb_buf = NULL;
  467.   }
  469.   GST_DEBUG_OBJECT (dsexample, "deleted CV Mat \n");
  471.   /* Deinit the algorithm library */
  472.   DsExampleCtxDeinit (dsexample->dsexamplelib_ctx);
  473.   dsexample->dsexamplelib_ctx = NULL;
  475.   GST_DEBUG_OBJECT (dsexample, "ctx lib released \n");
  477.   return TRUE;
  478. }
  480. /** * Called when source / sink pad capabilities have been negotiated. */
  481. static gboolean
  482. gst_dsexample_set_caps (GstBaseTransform * btrans, GstCaps * incaps,
  483.     GstCaps * outcaps)
  484. { 
  485.   GstDsExample *dsexample = GST_DSEXAMPLE (btrans);
  486.   /* Save the input video information, since this will be required later. */
  487.   gst_video_info_from_caps (&dsexample->video_info, incaps);
  489.   if (dsexample->blur_objects && !dsexample->process_full_frame) { 
  490.     /* requires RGBA format for blurring the objects in opencv */
  491.      if (dsexample->video_info.finfo->format != GST_VIDEO_FORMAT_RGBA) { 
  492.       GST_ELEMENT_ERROR (dsexample, STREAM, FAILED,
  493.           ("input format should be RGBA when using blur-objects property"), (NULL));
  494.       goto error;
  495.       }
  496.   }
  498.   return TRUE;
  500. error:
  501.   return FALSE;
  502. }
  504. /** * Scale the entire frame to the processing resolution maintaining aspect ratio. * Or crop and scale objects to the processing resolution maintaining the aspect * ratio. Remove the padding required by hardware and convert from RGBA to RGB * using openCV. These steps can be skipped if the algorithm can work with * padded data and/or can work with RGBA. */
  505. static GstFlowReturn
  506. get_converted_mat (GstDsExample * dsexample, NvBufSurface *input_buf, gint idx,
  507.     NvOSD_RectParams * crop_rect_params, gdouble & ratio, gint input_width,
  508.     gint input_height)
  509. { 
  510.   NvBufSurfTransform_Error err;
  511.   NvBufSurfTransformConfigParams transform_config_params;
  512.   NvBufSurfTransformParams transform_params;
  513.   NvBufSurfTransformRect src_rect;
  514.   NvBufSurfTransformRect dst_rect;
  515.   NvBufSurface ip_surf;
  516.   cv::Mat in_mat;
  517.   ip_surf = *input_buf;
  519.   ip_surf.numFilled = ip_surf.batchSize = 1;
  520.   ip_surf.surfaceList = &(input_buf->surfaceList[idx]);
  522.   gint src_left = GST_ROUND_UP_2((unsigned int)crop_rect_params->left);
  523.   gint src_top = GST_ROUND_UP_2((unsigned int)crop_rect_params->top);
  524.   gint src_width = GST_ROUND_DOWN_2((unsigned int)crop_rect_params->width);
  525.   gint src_height = GST_ROUND_DOWN_2((unsigned int)crop_rect_params->height);
  527.   /* Maintain aspect ratio */
  528.   double hdest = dsexample->processing_width * src_height / (double) src_width;
  529.   double wdest = dsexample->processing_height * src_width / (double) src_height;
  530.   guint dest_width, dest_height;
  532.   if (hdest <= dsexample->processing_height) { 
  533.     dest_width = dsexample->processing_width;
  534.     dest_height = hdest;
  535.   } else { 
  536.     dest_width = wdest;
  537.     dest_height = dsexample->processing_height;
  538.   }
  540.   /* Configure transform session parameters for the transformation */
  541.   transform_config_params.compute_mode = NvBufSurfTransformCompute_Default;
  542.   transform_config_params.gpu_id = dsexample->gpu_id;
  543.   transform_config_params.cuda_stream = dsexample->cuda_stream;
  545.   /* Set the transform session parameters for the conversions executed in this * thread. */
  546.   err = NvBufSurfTransformSetSessionParams (&transform_config_params);
  547.   if (err != NvBufSurfTransformError_Success) { 
  548.     GST_ELEMENT_ERROR (dsexample, STREAM, FAILED,
  549.         ("NvBufSurfTransformSetSessionParams failed with error %d", err), (NULL));
  550.     goto error;
  551.   }
  553.   /* Calculate scaling ratio while maintaining aspect ratio */
  554.   ratio = MIN (1.0 * dest_width/ src_width, 1.0 * dest_height / src_height);
  556.   if ((crop_rect_params->width == 0) || (crop_rect_params->height == 0)) { 
  557.     GST_ELEMENT_ERROR (dsexample, STREAM, FAILED,
  558.         ("%s:crop_rect_params dimensions are zero",__func__), (NULL));
  559.     goto error;
  560.   }
  562. #ifdef __aarch64__
  563.   if (ratio <= 1.0 / 16 || ratio >= 16.0) { 
  564.     /* Currently cannot scale by ratio > 16 or < 1/16 for Jetson */
  565.     goto error;
  566.   }
  567. #endif
  568.   /* Set the transform ROIs for source and destination */
  569.   src_rect = { (guint)src_top, (guint)src_left, (guint)src_width, (guint)src_height};
  570.   dst_rect = { 0, 0, (guint)dest_width, (guint)dest_height};
  572.   /* Set the transform parameters */
  573.   transform_params.src_rect = &src_rect;
  574.   transform_params.dst_rect = &dst_rect;
  575.   transform_params.transform_flag =
  576.     NVBUFSURF_TRANSFORM_FILTER | NVBUFSURF_TRANSFORM_CROP_SRC |
  577.       NVBUFSURF_TRANSFORM_CROP_DST;
  578.   transform_params.transform_filter = NvBufSurfTransformInter_Default;
  580.   /* Memset the memory */
  581.   NvBufSurfaceMemSet (dsexample->inter_buf, 0, 0, 0);
  583.   GST_DEBUG_OBJECT (dsexample, "Scaling and converting input buffer\n");
  585.   /* Transformation scaling+format conversion if any. */
  586.   err = NvBufSurfTransform (&ip_surf, dsexample->inter_buf, &transform_params);
  587.   if (err != NvBufSurfTransformError_Success) { 
  588.     GST_ELEMENT_ERROR (dsexample, STREAM, FAILED,
  589.         ("NvBufSurfTransform failed with error %d while converting buffer", err),
  590.         (NULL));
  591.     goto error;
  592.   }
  593.   /* Map the buffer so that it can be accessed by CPU */
  594.   if (NvBufSurfaceMap (dsexample->inter_buf, 0, 0, NVBUF_MAP_READ) != 0){ 
  595.     goto error;
  596.   }
  598.   /* Cache the mapped data for CPU access */
  599.   NvBufSurfaceSyncForCpu (dsexample->inter_buf, 0, 0);
  601.   /* Use openCV to remove padding and convert RGBA to BGR. Can be skipped if * algorithm can handle padded RGBA data. */
  602.   in_mat =
  603.       cv::Mat (dsexample->processing_height, dsexample->processing_width,
  604.       CV_8UC4, dsexample->inter_buf->surfaceList[0].mappedAddr.addr[0],
  605.       dsexample->inter_buf->surfaceList[0].pitch);
  607. #if (CV_MAJOR_VERSION >= 4)
  608.   cv::cvtColor (in_mat, *dsexample->cvmat, cv::COLOR_RGBA2BGR);
  609. #else
  610.   cv::cvtColor (in_mat, *dsexample->cvmat, CV_RGBA2BGR);
  611. #endif
  613.   if (NvBufSurfaceUnMap (dsexample->inter_buf, 0, 0)){ 
  614.     goto error;
  615.   }
  617. #ifdef __aarch64__
  618.   /* To use the converted buffer in CUDA, create an EGLImage and then use * CUDA-EGL interop APIs */
  619.   if (USE_EGLIMAGE) { 
  620.     if (NvBufSurfaceMapEglImage (dsexample->inter_buf, 0) !=0 ) { 
  621.       goto error;
  622.     }
  624.     /* dsexample->inter_buf->surfaceList[0].mappedAddr.eglImage * Use interop APIs cuGraphicsEGLRegisterImage and * cuGraphicsResourceGetMappedEglFrame to access the buffer in CUDA */
  626.     /* Destroy the EGLImage */
  627.     NvBufSurfaceUnMapEglImage (dsexample->inter_buf, 0);
  628.   }
  629. #endif
  631.   /* We will first convert only the Region of Interest (the entire frame or the * object bounding box) to RGB and then scale the converted RGB frame to * processing resolution. */
  632.   return GST_FLOW_OK;
  634. error:
  635.   return GST_FLOW_ERROR;
  636. }
  638. /* * Blur the detected objects when processing in object mode (full-frame=0) */
  639. static GstFlowReturn
  640. blur_objects (GstDsExample * dsexample, gint idx,
  641.     NvOSD_RectParams * crop_rect_params, cv::Mat in_mat)
  642. { 
  643.   int hrst=1;
  644.   cv::Rect crop_rect;
  646.   if ((crop_rect_params->width == 0) || (crop_rect_params->height == 0)) { 
  647.     GST_ELEMENT_ERROR (dsexample, STREAM, FAILED,
  648.         ("%s:crop_rect_params dimensions are zero",__func__), (NULL));
  649.     return GST_FLOW_ERROR;
  650.   }
  652. /* rectangle for cropped objects */
  653.   crop_rect = cv::Rect (crop_rect_params->left, crop_rect_params->top,
  654.   crop_rect_params->width, crop_rect_params->height);
  656. /* apply gaussian blur to the detected objects */
  657.   //GaussianBlur(in_mat(crop_rect), in_mat(crop_rect), cv::Size(15,15), 4);
  658.   hrst = jibianjiaozheng(in_mat, in_mat);
  659.   if (hrst != 0)
  660.   { 
  661.     printf("畸变矫正失败\n");
  662.   }
  663.   else
  664.   { 
  665.     //printf("畸变矫正成功\n");
  666.   }
  667.   return GST_FLOW_OK;
  668. }
  670. /** * Called when element recieves an input buffer from upstream element. */
  671. static GstFlowReturn
  672. gst_dsexample_transform_ip (GstBaseTransform * btrans, GstBuffer * inbuf)
  673. { 
  674.   GstDsExample *dsexample = GST_DSEXAMPLE (btrans);
  675.   GstMapInfo in_map_info;
  676.   GstFlowReturn flow_ret = GST_FLOW_ERROR;
  677.   gdouble scale_ratio = 1.0;
  678.   DsExampleOutput *output;
  680.   NvBufSurface *surface = NULL;
  681.   NvDsBatchMeta *batch_meta = NULL;
  682.   NvDsFrameMeta *frame_meta = NULL;
  683.   NvDsMetaList * l_frame = NULL;
  684.   guint i = 0;
  686.   dsexample->frame_num++;
  687.   CHECK_CUDA_STATUS (cudaSetDevice (dsexample->gpu_id),
  688.       "Unable to set cuda device");
  690.   memset (&in_map_info, 0, sizeof (in_map_info));
  691.   if (!gst_buffer_map (inbuf, &in_map_info, GST_MAP_READ)) { 
  692.     g_print ("Error: Failed to map gst buffer\n");
  693.     goto error;
  694.   }
  696.   nvds_set_input_system_timestamp (inbuf, GST_ELEMENT_NAME (dsexample));
  697.   surface = (NvBufSurface *) in_map_info.data;
  698.   GST_DEBUG_OBJECT (dsexample,
  699.       "Processing Frame %" G_GUINT64_FORMAT " Surface %p\n",
  700.       dsexample->frame_num, surface);
  702.   if (CHECK_NVDS_MEMORY_AND_GPUID (dsexample, surface))
  703.     goto error;
  705.   batch_meta = gst_buffer_get_nvds_batch_meta (inbuf);
  706.   if (batch_meta == nullptr) { 
  707.     GST_ELEMENT_ERROR (dsexample, STREAM, FAILED,
  708.         ("NvDsBatchMeta not found for input buffer."), (NULL));
  709.     return GST_FLOW_ERROR;
  710.   }
  712.   if (dsexample->process_full_frame) { 
  713.     for (l_frame = batch_meta->frame_meta_list; l_frame != NULL;
  714.       l_frame = l_frame->next)
  715.     { 
  716.       frame_meta = (NvDsFrameMeta *) (l_frame->data);
  717.       NvOSD_RectParams rect_params;
  719.       /* Scale the entire frame to processing resolution */
  720.       rect_params.left = 0;
  721.       rect_params.top = 0;
  722.       rect_params.width = dsexample->video_info.width;
  723.       rect_params.height = dsexample->video_info.height;
  725.       /* Scale and convert the frame */
  726.       if (get_converted_mat (dsexample, surface, i, &rect_params,
  727.             scale_ratio, dsexample->video_info.width,
  728.             dsexample->video_info.height) != GST_FLOW_OK) { 
  729.         goto error;
  730.       }
  732.       /* Process to get the output */
  733.       output =
  734.           DsExampleProcess (dsexample->dsexamplelib_ctx,
  735.           dsexample->cvmat->data);
  736.       /* Attach the metadata for the full frame */
  737.       attach_metadata_full_frame (dsexample, frame_meta, scale_ratio, output, i);
  738.       i++;
  739.       free (output);
  740.     }    
  743.   } else { 
  744.     /* Using object crops as input to the algorithm. The objects are detected by * the primary detector */
  745.     NvDsMetaList * l_obj = NULL;
  746.     NvDsObjectMeta *obj_meta = NULL;
  748. #ifndef __aarch64__
  749.     if (dsexample->blur_objects) { 
  750.       if (surface->memType != NVBUF_MEM_CUDA_UNIFIED){ 
  751.         GST_ELEMENT_ERROR (dsexample, STREAM, FAILED,
  752.             ("%s:need NVBUF_MEM_CUDA_UNIFIED memory for opencv blurring",__func__), (NULL));
  753.         return GST_FLOW_ERROR;
  754.       }
  755.     }
  756. #endif
  758.     for (l_frame = batch_meta->frame_meta_list; l_frame != NULL;
  759.       l_frame = l_frame->next)
  760.     { 
  761.       frame_meta = (NvDsFrameMeta *) (l_frame->data);
  762.       cv::Mat in_mat;
  764.       if (dsexample->blur_objects) { 
  765.         /* Map the buffer so that it can be accessed by CPU */
  766.         if (surface->surfaceList[frame_meta->batch_id].mappedAddr.addr[0] == NULL){ 
  767.           if (NvBufSurfaceMap (surface, frame_meta->batch_id, 0, NVBUF_MAP_READ_WRITE) != 0){ 
  768.             GST_ELEMENT_ERROR (dsexample, STREAM, FAILED,
  769.                 ("%s:buffer map to be accessed by CPU failed", __func__), (NULL));
  770.             return GST_FLOW_ERROR;
  771.           }
  772.         }
  774.         /* Cache the mapped data for CPU access */
  775.         NvBufSurfaceSyncForCpu (surface, frame_meta->batch_id, 0);
  777.         in_mat =
  778.             cv::Mat (surface->surfaceList[frame_meta->batch_id].planeParams.height[0],
  779.             surface->surfaceList[frame_meta->batch_id].planeParams.width[0], CV_8UC4,
  780.             surface->surfaceList[frame_meta->batch_id].mappedAddr.addr[0],
  781.             surface->surfaceList[frame_meta->batch_id].planeParams.pitch[0]);
  782.       }
  784.       for (l_obj = frame_meta->obj_meta_list; l_obj != NULL;
  785.           l_obj = l_obj->next)
  786.       { 
  787.         obj_meta = (NvDsObjectMeta *) (l_obj->data);
  789.         if (dsexample->blur_objects) { 
  790.           /* gaussian blur the detected objects using opencv */
  791.           if (blur_objects (dsexample, frame_meta->batch_id,
  792.             &obj_meta->rect_params, in_mat) != GST_FLOW_OK) { 
  793.           /* Error in blurring, skip processing on object. */
  794.             GST_ELEMENT_ERROR (dsexample, STREAM, FAILED,
  795.             ("blurring the object failed"), (NULL));
  796.             if (NvBufSurfaceUnMap (surface, frame_meta->batch_id, 0)){ 
  797.               GST_ELEMENT_ERROR (dsexample, STREAM, FAILED,
  798.                 ("%s:buffer unmap to be accessed by CPU failed", __func__), (NULL));
  799.             }
  800.             return GST_FLOW_ERROR;
  801.           }
  802.           continue;
  803.         }
  805.         /* Should not process on objects smaller than MIN_INPUT_OBJECT_WIDTH x MIN_INPUT_OBJECT_HEIGHT * since it will cause hardware scaling issues. */
  806.         if (obj_meta->rect_params.width < MIN_INPUT_OBJECT_WIDTH ||
  807.             obj_meta->rect_params.height < MIN_INPUT_OBJECT_HEIGHT)
  808.           continue;
  810.         /* Crop and scale the object */
  811.         if (get_converted_mat (dsexample,
  812.               surface, frame_meta->batch_id, &obj_meta->rect_params,
  813.               scale_ratio, dsexample->video_info.width,
  814.               dsexample->video_info.height) != GST_FLOW_OK) { 
  815.           /* Error in conversion, skip processing on object. */
  816.           continue;
  817.         }
  819.         /* Process the object crop to obtain label */
  820.         output = DsExampleProcess (dsexample->dsexamplelib_ctx,
  821.             dsexample->cvmat->data);
  823.         /* Attach labels for the object */
  824.         attach_metadata_object (dsexample, obj_meta, output);
  826.         free (output);
  827.       }
  829.       if (dsexample->blur_objects) { 
  830.       /* Cache the mapped data for device access */
  831.         NvBufSurfaceSyncForDevice (surface, frame_meta->batch_id, 0);
  833. #ifdef DSEXAMPLE_DEBUG
  834.         /* Use openCV to remove padding and convert RGBA to BGR. Can be skipped if * algorithm can handle padded RGBA data. */
  835. #if (CV_MAJOR_VERSION >= 4)
  836.         cv::cvtColor (in_mat, *dsexample->cvmat, cv::COLOR_RGBA2BGR);
  837. #else
  838.         cv::cvtColor (in_mat, *dsexample->cvmat, CV_RGBA2BGR);
  839. #endif
  840.         /* used to dump the converted mat to files for debug */
  841.         static guint cnt = 0;
  842.         cv::imwrite("out_" + std::to_string (cnt) + ".jpeg", *dsexample->cvmat);
  843.         cnt++;
  844. #endif
  845.       }
  846.     }
  847.   }
  848.   flow_ret = GST_FLOW_OK;
  850. error:
  852.   nvds_set_output_system_timestamp (inbuf, GST_ELEMENT_NAME (dsexample));
  853.   gst_buffer_unmap (inbuf, &in_map_info);
  854.   return flow_ret;
  855. }
  857. /** * Attach metadata for the full frame. We will be adding a new metadata. */
  858. static void
  859. attach_metadata_full_frame (GstDsExample * dsexample, NvDsFrameMeta *frame_meta,
  860.     gdouble scale_ratio, DsExampleOutput * output, guint batch_id)
  861. { 
  862.   NvDsBatchMeta *batch_meta = frame_meta->base_meta.batch_meta;
  863.   NvDsObjectMeta *object_meta = NULL;
  864.   static gchar font_name[] = "Serif";
  865.   GST_DEBUG_OBJECT (dsexample, "Attaching metadata %d\n", output->numObjects);
  867.   for (gint i = 0; i < output->numObjects; i++) { 
  868.     DsExampleObject *obj = &output->object[i];
  869.     object_meta = nvds_acquire_obj_meta_from_pool(batch_meta);
  870.     NvOSD_RectParams & rect_params = object_meta->rect_params;
  871.     NvOSD_TextParams & text_params = object_meta->text_params;
  873.     /* Assign bounding box coordinates */
  874.     rect_params.left = obj->left;
  875.     rect_params.top = obj->top;
  876.     rect_params.width = obj->width;
  877.     rect_params.height = obj->height;
  879.     /* Semi-transparent yellow background */
  880.     rect_params.has_bg_color = 0;
  881.     rect_params.bg_color = (NvOSD_ColorParams) { 
  882.     1, 1, 0, 0.4};
  883.     /* Red border of width 6 */
  884.     rect_params.border_width = 3;
  885.     rect_params.border_color = (NvOSD_ColorParams) { 
  886.     1, 0, 0, 1};
  888.     /* Scale the bounding boxes proportionally based on how the object/frame was * scaled during input */
  889.     rect_params.left /= scale_ratio;
  890.     rect_params.top /= scale_ratio;
  891.     rect_params.width /= scale_ratio;
  892.     rect_params.height /= scale_ratio;
  893.     GST_DEBUG_OBJECT (dsexample, "Attaching rect%d of batch%u"
  894.         " left->%f top->%f width->%f"
  895.         " height->%f label->%s\n", i, batch_id, rect_params.left,
  896.         rect_params.top, rect_params.width, rect_params.height, obj->label);
  898.     object_meta->object_id = UNTRACKED_OBJECT_ID;
  899.     g_strlcpy (object_meta->obj_label, obj->label, MAX_LABEL_SIZE);
  900.     /* display_text required heap allocated memory */
  901.     text_params.display_text = g_strdup (obj->label);
  902.     /* Display text above the left top corner of the object */
  903.     text_params.x_offset = rect_params.left;
  904.     text_params.y_offset = rect_params.top - 10;
  905.     /* Set black background for the text */
  906.     text_params.set_bg_clr = 1;
  907.     text_params.text_bg_clr = (NvOSD_ColorParams) { 
  908.     0, 0, 0, 1};
  909.     /* Font face, size and color */
  910.     text_params.font_params.font_name = font_name;
  911.     text_params.font_params.font_size = 11;
  912.     text_params.font_params.font_color = (NvOSD_ColorParams) { 
  913.     1, 1, 1, 1};
  915.     nvds_add_obj_meta_to_frame(frame_meta, object_meta, NULL);
  916.     frame_meta->bInferDone = TRUE;
  917.   }
  918. }
  920. /** * Only update string label in an existing object metadata. No bounding boxes. * We assume only one label per object is generated */
  921. static void
  922. attach_metadata_object (GstDsExample * dsexample, NvDsObjectMeta * obj_meta,
  923.     DsExampleOutput * output)
  924. { 
  925.   if (output->numObjects == 0)
  926.     return;
  927.   NvDsBatchMeta *batch_meta = obj_meta->base_meta.batch_meta;
  929.   NvDsClassifierMeta *classifier_meta =
  930.     nvds_acquire_classifier_meta_from_pool (batch_meta);
  932.   classifier_meta->unique_component_id = dsexample->unique_id;
  934.   NvDsLabelInfo *label_info =
  935.     nvds_acquire_label_info_meta_from_pool (batch_meta);
  936.   g_strlcpy (label_info->result_label, output->object[0].label, MAX_LABEL_SIZE);
  937.   nvds_add_label_info_meta_to_classifier(classifier_meta, label_info);
  938.   nvds_add_classifier_meta_to_object (obj_meta, classifier_meta);
  940.   nvds_acquire_meta_lock (batch_meta);
  941.   NvOSD_TextParams & text_params = obj_meta->text_params;
  942.   NvOSD_RectParams & rect_params = obj_meta->rect_params;
  944.   /* Below code to display the result */
  945.   /* Set black background for the text * display_text required heap allocated memory */
  946.   if (text_params.display_text) { 
  947.     gchar *conc_string = g_strconcat (text_params.display_text, " ",
  948.         output->object[0].label, NULL);
  949.     g_free (text_params.display_text);
  950.     text_params.display_text = conc_string;
  951.   } else { 
  952.     /* Display text above the left top corner of the object */
  953.     text_params.x_offset = rect_params.left;
  954.     text_params.y_offset = rect_params.top - 10;
  955.     text_params.display_text = g_strdup (output->object[0].label);
  956.     /* Font face, size and color */
  957.     text_params.font_params.font_name = (char *)"Serif";
  958.     text_params.font_params.font_size = 11;
  959.     text_params.font_params.font_color = (NvOSD_ColorParams) { 
  960.     1, 1, 1, 1};
  961.     /* Set black background for the text */
  962.     text_params.set_bg_clr = 1;
  963.     text_params.text_bg_clr = (NvOSD_ColorParams) { 
  964.     0, 0, 0, 1};
  965.   }
  966.   nvds_release_meta_lock (batch_meta);
  967. }
  969. /** * Boiler plate for registering a plugin and an element. */
  970. static gboolean
  971. dsexample_plugin_init (GstPlugin * plugin)
  972. { 
  973.   GST_DEBUG_CATEGORY_INIT (gst_dsexample_debug, "dsexample", 0,
  974.       "dsexample plugin");
  976.   return gst_element_register (plugin, "dsexample", GST_RANK_PRIMARY,
  977.       GST_TYPE_DSEXAMPLE);
  978. }
  980. GST_PLUGIN_DEFINE (GST_VERSION_MAJOR,
  981.     GST_VERSION_MINOR,
  982.     nvdsgst_dsexample,
  983.     DESCRIPTION, dsexample_plugin_init, DS_VERSION, LICENSE, BINARY_PACKAGE, URL)

normal

  1. /** * Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER * DEALINGS IN THE SOFTWARE. */
  3. #include <string.h>
  4. #include <string>
  5. #include <sstream>
  6. #include <iostream>
  7. #include <ostream>
  8. #include <fstream>
  9. #include "gstdsexample.h"
  10. #include <sys/time.h>
  11. GST_DEBUG_CATEGORY_STATIC (gst_dsexample_debug);
  12. #define GST_CAT_DEFAULT gst_dsexample_debug
  13. #define USE_EGLIMAGE 1
  14. /* enable to write transformed cvmat to files */
  15. /* #define DSEXAMPLE_DEBUG */
  16. static GQuark _dsmeta_quark = 0;
  18. /* Enum to identify properties */
  19. enum
  20. { 
  21.   PROP_0,
  22.   PROP_UNIQUE_ID,
  23.   PROP_PROCESSING_WIDTH,
  24.   PROP_PROCESSING_HEIGHT,
  25.   PROP_PROCESS_FULL_FRAME,
  26.   PROP_BLUR_OBJECTS,
  27.   PROP_GPU_DEVICE_ID
  28. };
  30. #define CHECK_NVDS_MEMORY_AND_GPUID(object, surface) \ ({ int _errtype=0;\ do { \ if ((surface->memType == NVBUF_MEM_DEFAULT || surface->memType == NVBUF_MEM_CUDA_DEVICE) && \ (surface->gpuId != object->gpu_id)) { \ GST_ELEMENT_ERROR (object, RESOURCE, FAILED, \ ("Input surface gpu-id doesnt match with configured gpu-id for element," \ " please allocate input using unified memory, or use same gpu-ids"),\ ("surface-gpu-id=%d,%s-gpu-id=%d",surface->gpuId,GST_ELEMENT_NAME(object),\ object->gpu_id)); \ _errtype = 1;\ } \ } while(0); \ _errtype; \ })
  33. /* Default values for properties */
  34. #define DEFAULT_UNIQUE_ID 15
  35. #define DEFAULT_PROCESSING_WIDTH 640
  36. #define DEFAULT_PROCESSING_HEIGHT 480
  37. #define DEFAULT_PROCESS_FULL_FRAME TRUE
  38. #define DEFAULT_BLUR_OBJECTS FALSE
  39. #define DEFAULT_GPU_ID 0
  41. #define RGB_BYTES_PER_PIXEL 3
  42. #define RGBA_BYTES_PER_PIXEL 4
  43. #define Y_BYTES_PER_PIXEL 1
  44. #define UV_BYTES_PER_PIXEL 2
  46. #define MIN_INPUT_OBJECT_WIDTH 16
  47. #define MIN_INPUT_OBJECT_HEIGHT 16
  49. #define CHECK_NPP_STATUS(npp_status,error_str) do { \ if ((npp_status) != NPP_SUCCESS) { \ g_print ("Error: %s in %s at line %d: NPP Error %d\n", \ error_str, __FILE__, __LINE__, npp_status); \ goto error; \ } \ } while (0)
  51. #define CHECK_CUDA_STATUS(cuda_status,error_str) do { \ if ((cuda_status) != cudaSuccess) { \ g_print ("Error: %s in %s at line %d (%s)\n", \ error_str, __FILE__, __LINE__, cudaGetErrorName(cuda_status)); \ goto error; \ } \ } while (0)
  53. /* By default NVIDIA Hardware allocated memory flows through the pipeline. We * will be processing on this type of memory only. */
  54. #define GST_CAPS_FEATURE_MEMORY_NVMM "memory:NVMM"
  55. static GstStaticPadTemplate gst_dsexample_sink_template =
  56. GST_STATIC_PAD_TEMPLATE ("sink",
  57.     GST_PAD_SINK,
  58.     GST_PAD_ALWAYS,
  59.     GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
  60.         (GST_CAPS_FEATURE_MEMORY_NVMM,
  61.             "{ NV12, RGBA, I420 }")));
  63. static GstStaticPadTemplate gst_dsexample_src_template =
  64. GST_STATIC_PAD_TEMPLATE ("src",
  65.     GST_PAD_SRC,
  66.     GST_PAD_ALWAYS,
  67.     GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
  68.         (GST_CAPS_FEATURE_MEMORY_NVMM,
  69.             "{ NV12, RGBA, I420 }")));
  71. /* Define our element type. Standard GObject/GStreamer boilerplate stuff */
  72. #define gst_dsexample_parent_class parent_class
  73. G_DEFINE_TYPE (GstDsExample, gst_dsexample, GST_TYPE_BASE_TRANSFORM);
  75. static void gst_dsexample_set_property (GObject * object, guint prop_id,
  76.     const GValue * value, GParamSpec * pspec);
  77. static void gst_dsexample_get_property (GObject * object, guint prop_id,
  78.     GValue * value, GParamSpec * pspec);
  80. static gboolean gst_dsexample_set_caps (GstBaseTransform * btrans,
  81.     GstCaps * incaps, GstCaps * outcaps);
  82. static gboolean gst_dsexample_start (GstBaseTransform * btrans);
  83. static gboolean gst_dsexample_stop (GstBaseTransform * btrans);
  85. static GstFlowReturn gst_dsexample_transform_ip (GstBaseTransform *
  86.     btrans, GstBuffer * inbuf);
  88. static void
  89. attach_metadata_full_frame (GstDsExample * dsexample, NvDsFrameMeta *frame_meta,
  90.     gdouble scale_ratio, DsExampleOutput * output, guint batch_id);
  91. static void attach_metadata_object (GstDsExample * dsexample,
  92.     NvDsObjectMeta * obj_meta, DsExampleOutput * output);
  94. /* Install properties, set sink and src pad capabilities, override the required * functions of the base class, These are common to all instances of the * element. */
  95. static void
  96. gst_dsexample_class_init (GstDsExampleClass * klass)
  97. { 
  98.   GObjectClass *gobject_class;
  99.   GstElementClass *gstelement_class;
  100.   GstBaseTransformClass *gstbasetransform_class;
  102.   /* Indicates we want to use DS buf api */
  103.   g_setenv ("DS_NEW_BUFAPI", "1", TRUE);
  105.   gobject_class = (GObjectClass *) klass;
  106.   gstelement_class = (GstElementClass *) klass;
  107.   gstbasetransform_class = (GstBaseTransformClass *) klass;
  109.   /* Overide base class functions */
  110.   gobject_class->set_property = GST_DEBUG_FUNCPTR (gst_dsexample_set_property);
  111.   gobject_class->get_property = GST_DEBUG_FUNCPTR (gst_dsexample_get_property);
  113.   gstbasetransform_class->set_caps = GST_DEBUG_FUNCPTR (gst_dsexample_set_caps);
  114.   gstbasetransform_class->start = GST_DEBUG_FUNCPTR (gst_dsexample_start);
  115.   gstbasetransform_class->stop = GST_DEBUG_FUNCPTR (gst_dsexample_stop);
  117.   gstbasetransform_class->transform_ip =
  118.       GST_DEBUG_FUNCPTR (gst_dsexample_transform_ip);
  120.   /* Install properties */
  121.   g_object_class_install_property (gobject_class, PROP_UNIQUE_ID,
  122.       g_param_spec_uint ("unique-id",
  123.           "Unique ID",
  124.           "Unique ID for the element. Can be used to identify output of the"
  125.           " element", 0, G_MAXUINT, DEFAULT_UNIQUE_ID, (GParamFlags)
  126.           (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
  128.   g_object_class_install_property (gobject_class, PROP_PROCESSING_WIDTH,
  129.       g_param_spec_int ("processing-width",
  130.           "Processing Width",
  131.           "Width of the input buffer to algorithm",
  132.           1, G_MAXINT, DEFAULT_PROCESSING_WIDTH, (GParamFlags)
  133.           (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
  135.   g_object_class_install_property (gobject_class, PROP_PROCESSING_HEIGHT,
  136.       g_param_spec_int ("processing-height",
  137.           "Processing Height",
  138.           "Height of the input buffer to algorithm",
  139.           1, G_MAXINT, DEFAULT_PROCESSING_HEIGHT, (GParamFlags)
  140.           (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
  142.   g_object_class_install_property (gobject_class, PROP_PROCESS_FULL_FRAME,
  143.       g_param_spec_boolean ("full-frame",
  144.           "Full frame",
  145.           "Enable to process full frame or disable to process objects detected"
  146.           "by primary detector", DEFAULT_PROCESS_FULL_FRAME, (GParamFlags)
  147.           (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
  149.   g_object_class_install_property (gobject_class, PROP_BLUR_OBJECTS,
  150.       g_param_spec_boolean ("blur-objects",
  151.           "Blur Objects",
  152.           "Enable to blur the objects detected in full-frame=0 mode"
  153.           "by primary detector", DEFAULT_BLUR_OBJECTS, (GParamFlags)
  154.           (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
  156.   g_object_class_install_property (gobject_class, PROP_GPU_DEVICE_ID,
  157.       g_param_spec_uint ("gpu-id",
  158.           "Set GPU Device ID",
  159.           "Set GPU Device ID", 0,
  160.           G_MAXUINT, 0,
  161.           GParamFlags
  162.           (G_PARAM_READWRITE |
  163.               G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_READY)));
  164.   /* Set sink and src pad capabilities */
  165.   gst_element_class_add_pad_template (gstelement_class,
  166.       gst_static_pad_template_get (&gst_dsexample_src_template));
  167.   gst_element_class_add_pad_template (gstelement_class,
  168.       gst_static_pad_template_get (&gst_dsexample_sink_template));
  170.   /* Set metadata describing the element */
  171.   gst_element_class_set_details_simple (gstelement_class,
  172.       "DsExample plugin",
  173.       "DsExample Plugin",
  174.       "Process a 3rdparty example algorithm on objects / full frame",
  175.       "NVIDIA Corporation. Post on Deepstream for Tesla forum for any queries "
  176.       "@ https://devtalk.nvidia.com/default/board/209/");
  177. }
  179. static void
  180. gst_dsexample_init (GstDsExample * dsexample)
  181. { 
  182.   GstBaseTransform *btrans = GST_BASE_TRANSFORM (dsexample);
  184.   /* We will not be generating a new buffer. Just adding / updating * metadata. */
  185.   gst_base_transform_set_in_place (GST_BASE_TRANSFORM (btrans), TRUE);
  186.   /* We do not want to change the input caps. Set to passthrough. transform_ip * is still called. */
  187.   gst_base_transform_set_passthrough (GST_BASE_TRANSFORM (btrans), TRUE);
  189.   /* Initialize all property variables to default values */
  190.   dsexample->unique_id = DEFAULT_UNIQUE_ID;
  191.   dsexample->processing_width = DEFAULT_PROCESSING_WIDTH;
  192.   dsexample->processing_height = DEFAULT_PROCESSING_HEIGHT;
  193.   dsexample->process_full_frame = DEFAULT_PROCESS_FULL_FRAME;
  194.   dsexample->blur_objects = DEFAULT_BLUR_OBJECTS;
  195.   dsexample->gpu_id = DEFAULT_GPU_ID;
  196.   /* This quark is required to identify NvDsMeta when iterating through * the buffer metadatas */
  197.   if (!_dsmeta_quark)
  198.     _dsmeta_quark = g_quark_from_static_string (NVDS_META_STRING);
  199. }
  201. /* Function called when a property of the element is set. Standard boilerplate. */
  202. static void
  203. gst_dsexample_set_property (GObject * object, guint prop_id,
  204.     const GValue * value, GParamSpec * pspec)
  205. { 
  206.   GstDsExample *dsexample = GST_DSEXAMPLE (object);
  207.   switch (prop_id) { 
  208.     case PROP_UNIQUE_ID:
  209.       dsexample->unique_id = g_value_get_uint (value);
  210.       break;
  211.     case PROP_PROCESSING_WIDTH:
  212.       dsexample->processing_width = g_value_get_int (value);
  213.       break;
  214.     case PROP_PROCESSING_HEIGHT:
  215.       dsexample->processing_height = g_value_get_int (value);
  216.       break;
  217.     case PROP_PROCESS_FULL_FRAME:
  218.       dsexample->process_full_frame = g_value_get_boolean (value);
  219.       break;
  220.     case PROP_BLUR_OBJECTS:
  221.       dsexample->blur_objects = g_value_get_boolean (value);
  222.       break;
  223.     case PROP_GPU_DEVICE_ID:
  224.       dsexample->gpu_id = g_value_get_uint (value);
  225.       break;
  226.     default:
  227.       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
  228.       break;
  229.   }
  230. }
  232. /* Function called when a property of the element is requested. Standard * boilerplate. */
  233. static void
  234. gst_dsexample_get_property (GObject * object, guint prop_id,
  235.     GValue * value, GParamSpec * pspec)
  236. { 
  237.   GstDsExample *dsexample = GST_DSEXAMPLE (object);
  239.   switch (prop_id) { 
  240.     case PROP_UNIQUE_ID:
  241.       g_value_set_uint (value, dsexample->unique_id);
  242.       break;
  243.     case PROP_PROCESSING_WIDTH:
  244.       g_value_set_int (value, dsexample->processing_width);
  245.       break;
  246.     case PROP_PROCESSING_HEIGHT:
  247.       g_value_set_int (value, dsexample->processing_height);
  248.       break;
  249.     case PROP_PROCESS_FULL_FRAME:
  250.       g_value_set_boolean (value, dsexample->process_full_frame);
  251.       break;
  252.     case PROP_BLUR_OBJECTS:
  253.       g_value_set_boolean (value, dsexample->blur_objects);
  254.       break;
  255.     case PROP_GPU_DEVICE_ID:
  256.       g_value_set_uint (value, dsexample->gpu_id);
  257.       break;
  258.     default:
  259.       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
  260.       break;
  261.   }
  262. }
  264. /** * Initialize all resources and start the output thread */
  265. static gboolean
  266. gst_dsexample_start (GstBaseTransform * btrans)
  267. { 
  268.   GstDsExample *dsexample = GST_DSEXAMPLE (btrans);
  269.   NvBufSurfaceCreateParams create_params;
  270.   DsExampleInitParams init_params =
  271.       {  dsexample->processing_width, dsexample->processing_height,
  272.     dsexample->process_full_frame
  273.   };
  275.   GstQuery *queryparams = NULL;
  276.   guint batch_size = 1;
  278.   /* Algorithm specific initializations and resource allocation. */
  279.   dsexample->dsexamplelib_ctx = DsExampleCtxInit (&init_params);
  281.   GST_DEBUG_OBJECT (dsexample, "ctx lib %p \n", dsexample->dsexamplelib_ctx);
  283.   CHECK_CUDA_STATUS (cudaSetDevice (dsexample->gpu_id),
  284.       "Unable to set cuda device");
  286.   dsexample->batch_size = 1;
  287.   queryparams = gst_nvquery_batch_size_new ();
  288.   if (gst_pad_peer_query (GST_BASE_TRANSFORM_SINK_PAD (btrans), queryparams)
  289.       || gst_pad_peer_query (GST_BASE_TRANSFORM_SRC_PAD (btrans), queryparams)) { 
  290.     if (gst_nvquery_batch_size_parse (queryparams, &batch_size)) { 
  291.       dsexample->batch_size = batch_size;
  292.     }
  293.   }
  294.   GST_DEBUG_OBJECT (dsexample, "Setting batch-size %d \n",
  295.       dsexample->batch_size);
  296.   gst_query_unref (queryparams);
  298.   if (dsexample->process_full_frame && dsexample->blur_objects) { 
  299.     GST_ERROR ("Error: does not support blurring while processing full frame");
  300.     goto error;
  301.   }
  303.   CHECK_CUDA_STATUS (cudaStreamCreate (&dsexample->cuda_stream),
  304.       "Could not create cuda stream");
  306.   if (dsexample->inter_buf)
  307.     NvBufSurfaceDestroy (dsexample->inter_buf);
  308.   dsexample->inter_buf = NULL;
  310.   /* An intermediate buffer for NV12/RGBA to BGR conversion will be * required. Can be skipped if custom algorithm can work directly on NV12/RGBA. */
  311.   create_params.gpuId  = dsexample->gpu_id;
  312.   create_params.width  = dsexample->processing_width;
  313.   create_params.height = dsexample->processing_height;
  314.   create_params.size = 0;
  315.   create_params.colorFormat = NVBUF_COLOR_FORMAT_RGBA;
  316.   create_params.layout = NVBUF_LAYOUT_PITCH;
  317. #ifdef __aarch64__
  318.   create_params.memType = NVBUF_MEM_DEFAULT;
  319. #else
  320.   create_params.memType = NVBUF_MEM_CUDA_UNIFIED;
  321. #endif
  323.   if (NvBufSurfaceCreate (&dsexample->inter_buf, 1,
  324.           &create_params) != 0) { 
  325.     GST_ERROR ("Error: Could not allocate internal buffer for dsexample");
  326.     goto error;
  327.   }
  329.   /* Create host memory for storing converted/scaled interleaved RGB data */
  330.   CHECK_CUDA_STATUS (cudaMallocHost (&dsexample->host_rgb_buf,
  331.           dsexample->processing_width * dsexample->processing_height *
  332.           RGB_BYTES_PER_PIXEL), "Could not allocate cuda host buffer");
  334.   GST_DEBUG_OBJECT (dsexample, "allocated cuda buffer %p \n",
  335.       dsexample->host_rgb_buf);
  337.   /* CV Mat containing interleaved RGB data. This call does not allocate memory. * It uses host_rgb_buf as data. */
  338.   dsexample->cvmat =
  339.       new cv::Mat (dsexample->processing_height, dsexample->processing_width,
  340.       CV_8UC3, dsexample->host_rgb_buf,
  341.       dsexample->processing_width * RGB_BYTES_PER_PIXEL);
  343.   if (!dsexample->cvmat)
  344.     goto error;
  346.   GST_DEBUG_OBJECT (dsexample, "created CV Mat\n");
  348.   return TRUE;
  349. error:
  350.   if (dsexample->host_rgb_buf) { 
  351.     cudaFreeHost (dsexample->host_rgb_buf);
  352.     dsexample->host_rgb_buf = NULL;
  353.   }
  355.   if (dsexample->cuda_stream) { 
  356.     cudaStreamDestroy (dsexample->cuda_stream);
  357.     dsexample->cuda_stream = NULL;
  358.   }
  359.   if (dsexample->dsexamplelib_ctx)
  360.     DsExampleCtxDeinit (dsexample->dsexamplelib_ctx);
  361.   return FALSE;
  362. }
  364. /** * Stop the output thread and free up all the resources */
  365. static gboolean
  366. gst_dsexample_stop (GstBaseTransform * btrans)
  367. { 
  368.   GstDsExample *dsexample = GST_DSEXAMPLE (btrans);
  370.   if (dsexample->inter_buf)
  371.     NvBufSurfaceDestroy(dsexample->inter_buf);
  372.   dsexample->inter_buf = NULL;
  374.   if (dsexample->cuda_stream)
  375.     cudaStreamDestroy (dsexample->cuda_stream);
  376.   dsexample->cuda_stream = NULL;
  378.   delete dsexample->cvmat;
  379.   dsexample->cvmat = NULL;
  381.   if (dsexample->host_rgb_buf) { 
  382.     cudaFreeHost (dsexample->host_rgb_buf);
  383.     dsexample->host_rgb_buf = NULL;
  384.   }
  386.   GST_DEBUG_OBJECT (dsexample, "deleted CV Mat \n");
  388.   /* Deinit the algorithm library */
  389.   DsExampleCtxDeinit (dsexample->dsexamplelib_ctx);
  390.   dsexample->dsexamplelib_ctx = NULL;
  392.   GST_DEBUG_OBJECT (dsexample, "ctx lib released \n");
  394.   return TRUE;
  395. }
  397. /** * Called when source / sink pad capabilities have been negotiated. */
  398. static gboolean
  399. gst_dsexample_set_caps (GstBaseTransform * btrans, GstCaps * incaps,
  400.     GstCaps * outcaps)
  401. { 
  402.   GstDsExample *dsexample = GST_DSEXAMPLE (btrans);
  403.   /* Save the input video information, since this will be required later. */
  404.   gst_video_info_from_caps (&dsexample->video_info, incaps);
  406.   if (dsexample->blur_objects && !dsexample->process_full_frame) { 
  407.     /* requires RGBA format for blurring the objects in opencv */
  408.      if (dsexample->video_info.finfo->format != GST_VIDEO_FORMAT_RGBA) { 
  409.       GST_ELEMENT_ERROR (dsexample, STREAM, FAILED,
  410.           ("input format should be RGBA when using blur-objects property"), (NULL));
  411.       goto error;
  412.       }
  413.   }
  415.   return TRUE;
  417. error:
  418.   return FALSE;
  419. }
  421. /** * Scale the entire frame to the processing resolution maintaining aspect ratio. * Or crop and scale objects to the processing resolution maintaining the aspect * ratio. Remove the padding required by hardware and convert from RGBA to RGB * using openCV. These steps can be skipped if the algorithm can work with * padded data and/or can work with RGBA. */
  422. static GstFlowReturn
  423. get_converted_mat (GstDsExample * dsexample, NvBufSurface *input_buf, gint idx,
  424.     NvOSD_RectParams * crop_rect_params, gdouble & ratio, gint input_width,
  425.     gint input_height)
  426. { 
  427.   NvBufSurfTransform_Error err;
  428.   NvBufSurfTransformConfigParams transform_config_params;
  429.   NvBufSurfTransformParams transform_params;
  430.   NvBufSurfTransformRect src_rect;
  431.   NvBufSurfTransformRect dst_rect;
  432.   NvBufSurface ip_surf;
  433.   cv::Mat in_mat;
  434.   ip_surf = *input_buf;
  436.   ip_surf.numFilled = ip_surf.batchSize = 1;
  437.   ip_surf.surfaceList = &(input_buf->surfaceList[idx]);
  439.   gint src_left = GST_ROUND_UP_2((unsigned int)crop_rect_params->left);
  440.   gint src_top = GST_ROUND_UP_2((unsigned int)crop_rect_params->top);
  441.   gint src_width = GST_ROUND_DOWN_2((unsigned int)crop_rect_params->width);
  442.   gint src_height = GST_ROUND_DOWN_2((unsigned int)crop_rect_params->height);
  444.   /* Maintain aspect ratio */
  445.   double hdest = dsexample->processing_width * src_height / (double) src_width;
  446.   double wdest = dsexample->processing_height * src_width / (double) src_height;
  447.   guint dest_width, dest_height;
  449.   if (hdest <= dsexample->processing_height) { 
  450.     dest_width = dsexample->processing_width;
  451.     dest_height = hdest;
  452.   } else { 
  453.     dest_width = wdest;
  454.     dest_height = dsexample->processing_height;
  455.   }
  457.   /* Configure transform session parameters for the transformation */
  458.   transform_config_params.compute_mode = NvBufSurfTransformCompute_Default;
  459.   transform_config_params.gpu_id = dsexample->gpu_id;
  460.   transform_config_params.cuda_stream = dsexample->cuda_stream;
  462.   /* Set the transform session parameters for the conversions executed in this * thread. */
  463.   err = NvBufSurfTransformSetSessionParams (&transform_config_params);
  464.   if (err != NvBufSurfTransformError_Success) { 
  465.     GST_ELEMENT_ERROR (dsexample, STREAM, FAILED,
  466.         ("NvBufSurfTransformSetSessionParams failed with error %d", err), (NULL));
  467.     goto error;
  468.   }
  470.   /* Calculate scaling ratio while maintaining aspect ratio */
  471.   ratio = MIN (1.0 * dest_width/ src_width, 1.0 * dest_height / src_height);
  473.   if ((crop_rect_params->width == 0) || (crop_rect_params->height == 0)) { 
  474.     GST_ELEMENT_ERROR (dsexample, STREAM, FAILED,
  475.         ("%s:crop_rect_params dimensions are zero",__func__), (NULL));
  476.     goto error;
  477.   }
  479. #ifdef __aarch64__
  480.   if (ratio <= 1.0 / 16 || ratio >= 16.0) { 
  481.     /* Currently cannot scale by ratio > 16 or < 1/16 for Jetson */
  482.     goto error;
  483.   }
  484. #endif
  485.   /* Set the transform ROIs for source and destination */
  486.   src_rect = { (guint)src_top, (guint)src_left, (guint)src_width, (guint)src_height};
  487.   dst_rect = { 0, 0, (guint)dest_width, (guint)dest_height};
  489.   /* Set the transform parameters */
  490.   transform_params.src_rect = &src_rect;
  491.   transform_params.dst_rect = &dst_rect;
  492.   transform_params.transform_flag =
  493.     NVBUFSURF_TRANSFORM_FILTER | NVBUFSURF_TRANSFORM_CROP_SRC |
  494.       NVBUFSURF_TRANSFORM_CROP_DST;
  495.   transform_params.transform_filter = NvBufSurfTransformInter_Default;
  497.   /* Memset the memory */
  498.   NvBufSurfaceMemSet (dsexample->inter_buf, 0, 0, 0);
  500.   GST_DEBUG_OBJECT (dsexample, "Scaling and converting input buffer\n");
  502.   /* Transformation scaling+format conversion if any. */
  503.   err = NvBufSurfTransform (&ip_surf, dsexample->inter_buf, &transform_params);
  504.   if (err != NvBufSurfTransformError_Success) { 
  505.     GST_ELEMENT_ERROR (dsexample, STREAM, FAILED,
  506.         ("NvBufSurfTransform failed with error %d while converting buffer", err),
  507.         (NULL));
  508.     goto error;
  509.   }
  510.   /* Map the buffer so that it can be accessed by CPU */
  511.   if (NvBufSurfaceMap (dsexample->inter_buf, 0, 0, NVBUF_MAP_READ) != 0){ 
  512.     goto error;
  513.   }
  515.   /* Cache the mapped data for CPU access */
  516.   NvBufSurfaceSyncForCpu (dsexample->inter_buf, 0, 0);
  518.   /* Use openCV to remove padding and convert RGBA to BGR. Can be skipped if * algorithm can handle padded RGBA data. */
  519.   in_mat =
  520.       cv::Mat (dsexample->processing_height, dsexample->processing_width,
  521.       CV_8UC4, dsexample->inter_buf->surfaceList[0].mappedAddr.addr[0],
  522.       dsexample->inter_buf->surfaceList[0].pitch);
  524. #if (CV_MAJOR_VERSION >= 4)
  525.   cv::cvtColor (in_mat, *dsexample->cvmat, cv::COLOR_RGBA2BGR);
  526. #else
  527.   cv::cvtColor (in_mat, *dsexample->cvmat, CV_RGBA2BGR);
  528. #endif
  530.   if (NvBufSurfaceUnMap (dsexample->inter_buf, 0, 0)){ 
  531.     goto error;
  532.   }
  534. #ifdef __aarch64__
  535.   /* To use the converted buffer in CUDA, create an EGLImage and then use * CUDA-EGL interop APIs */
  536.   if (USE_EGLIMAGE) { 
  537.     if (NvBufSurfaceMapEglImage (dsexample->inter_buf, 0) !=0 ) { 
  538.       goto error;
  539.     }
  541.     /* dsexample->inter_buf->surfaceList[0].mappedAddr.eglImage * Use interop APIs cuGraphicsEGLRegisterImage and * cuGraphicsResourceGetMappedEglFrame to access the buffer in CUDA */
  543.     /* Destroy the EGLImage */
  544.     NvBufSurfaceUnMapEglImage (dsexample->inter_buf, 0);
  545.   }
  546. #endif
  548.   /* We will first convert only the Region of Interest (the entire frame or the * object bounding box) to RGB and then scale the converted RGB frame to * processing resolution. */
  549.   return GST_FLOW_OK;
  551. error:
  552.   return GST_FLOW_ERROR;
  553. }
  555. /* * Blur the detected objects when processing in object mode (full-frame=0) */
  556. static GstFlowReturn
  557. blur_objects (GstDsExample * dsexample, gint idx,
  558.     NvOSD_RectParams * crop_rect_params, cv::Mat in_mat)
  559. { 
  560.   cv::Rect crop_rect;
  562.   if ((crop_rect_params->width == 0) || (crop_rect_params->height == 0)) { 
  563.     GST_ELEMENT_ERROR (dsexample, STREAM, FAILED,
  564.         ("%s:crop_rect_params dimensions are zero",__func__), (NULL));
  565.     return GST_FLOW_ERROR;
  566.   }
  568. /* rectangle for cropped objects */
  569.   crop_rect = cv::Rect (crop_rect_params->left, crop_rect_params->top,
  570.   crop_rect_params->width, crop_rect_params->height);
  572. /* apply gaussian blur to the detected objects */
  573.   //GaussianBlur(in_mat(crop_rect), in_mat(crop_rect), cv::Size(15,15), 4);
  575.   return GST_FLOW_OK;
  576. }
  578. /** * Called when element recieves an input buffer from upstream element. */
  579. static GstFlowReturn
  580. gst_dsexample_transform_ip (GstBaseTransform * btrans, GstBuffer * inbuf)
  581. { 
  582.   GstDsExample *dsexample = GST_DSEXAMPLE (btrans);
  583.   GstMapInfo in_map_info;
  584.   GstFlowReturn flow_ret = GST_FLOW_ERROR;
  585.   gdouble scale_ratio = 1.0;
  586.   DsExampleOutput *output;
  588.   NvBufSurface *surface = NULL;
  589.   NvDsBatchMeta *batch_meta = NULL;
  590.   NvDsFrameMeta *frame_meta = NULL;
  591.   NvDsMetaList * l_frame = NULL;
  592.   guint i = 0;
  594.   dsexample->frame_num++;
  595.   CHECK_CUDA_STATUS (cudaSetDevice (dsexample->gpu_id),
  596.       "Unable to set cuda device");
  598.   memset (&in_map_info, 0, sizeof (in_map_info));
  599.   if (!gst_buffer_map (inbuf, &in_map_info, GST_MAP_READ)) { 
  600.     g_print ("Error: Failed to map gst buffer\n");
  601.     goto error;
  602.   }
  604.   nvds_set_input_system_timestamp (inbuf, GST_ELEMENT_NAME (dsexample));
  605.   surface = (NvBufSurface *) in_map_info.data;
  606.   GST_DEBUG_OBJECT (dsexample,
  607.       "Processing Frame %" G_GUINT64_FORMAT " Surface %p\n",
  608.       dsexample->frame_num, surface);
  610.   if (CHECK_NVDS_MEMORY_AND_GPUID (dsexample, surface))
  611.     goto error;
  613.   batch_meta = gst_buffer_get_nvds_batch_meta (inbuf);
  614.   if (batch_meta == nullptr) { 
  615.     GST_ELEMENT_ERROR (dsexample, STREAM, FAILED,
  616.         ("NvDsBatchMeta not found for input buffer."), (NULL));
  617.     return GST_FLOW_ERROR;
  618.   }
  620.   if (dsexample->process_full_frame) { 
  621.     for (l_frame = batch_meta->frame_meta_list; l_frame != NULL;
  622.       l_frame = l_frame->next)
  623.     { 
  624.       frame_meta = (NvDsFrameMeta *) (l_frame->data);
  625.       NvOSD_RectParams rect_params;
  627.       /* Scale the entire frame to processing resolution */
  628.       rect_params.left = 0;
  629.       rect_params.top = 0;
  630.       rect_params.width = dsexample->video_info.width;
  631.       rect_params.height = dsexample->video_info.height;
  633.       /* Scale and convert the frame */
  634.       if (get_converted_mat (dsexample, surface, i, &rect_params,
  635.             scale_ratio, dsexample->video_info.width,
  636.             dsexample->video_info.height) != GST_FLOW_OK) { 
  637.         goto error;
  638.       }
  640.       /* Process to get the output */
  641.       output =
  642.           DsExampleProcess (dsexample->dsexamplelib_ctx,
  643.           dsexample->cvmat->data);
  644.       /* Attach the metadata for the full frame */
  645.       attach_metadata_full_frame (dsexample, frame_meta, scale_ratio, output, i);
  646.       i++;
  647.       free (output);
  648.     }
  650.   } else { 
  651.     /* Using object crops as input to the algorithm. The objects are detected by * the primary detector */
  652.     NvDsMetaList * l_obj = NULL;
  653.     NvDsObjectMeta *obj_meta = NULL;
  655. #ifndef __aarch64__
  656.     if (dsexample->blur_objects) { 
  657.       if (surface->memType != NVBUF_MEM_CUDA_UNIFIED){ 
  658.         GST_ELEMENT_ERROR (dsexample, STREAM, FAILED,
  659.             ("%s:need NVBUF_MEM_CUDA_UNIFIED memory for opencv blurring",__func__), (NULL));
  660.         return GST_FLOW_ERROR;
  661.       }
  662.     }
  663. #endif
  665.     for (l_frame = batch_meta->frame_meta_list; l_frame != NULL;
  666.       l_frame = l_frame->next)
  667.     { 
  668.       frame_meta = (NvDsFrameMeta *) (l_frame->data);
  669.       cv::Mat in_mat;
  671.       if (dsexample->blur_objects) { 
  672.         /* Map the buffer so that it can be accessed by CPU */
  673.         if (surface->surfaceList[frame_meta->batch_id].mappedAddr.addr[0] == NULL){ 
  674.           if (NvBufSurfaceMap (surface, frame_meta->batch_id, 0, NVBUF_MAP_READ_WRITE) != 0){ 
  675.             GST_ELEMENT_ERROR (dsexample, STREAM, FAILED,
  676.                 ("%s:buffer map to be accessed by CPU failed", __func__), (NULL));
  677.             return GST_FLOW_ERROR;
  678.           }
  679.         }
  681.         /* Cache the mapped data for CPU access */
  682.         NvBufSurfaceSyncForCpu (surface, frame_meta->batch_id, 0);
  684.         in_mat =
  685.             cv::Mat (surface->surfaceList[frame_meta->batch_id].planeParams.height[0],
  686.             surface->surfaceList[frame_meta->batch_id].planeParams.width[0], CV_8UC4,
  687.             surface->surfaceList[frame_meta->batch_id].mappedAddr.addr[0],
  688.             surface->surfaceList[frame_meta->batch_id].planeParams.pitch[0]);
  689.       }
  691.       for (l_obj = frame_meta->obj_meta_list; l_obj != NULL;
  692.           l_obj = l_obj->next)
  693.       { 
  694.         obj_meta = (NvDsObjectMeta *) (l_obj->data);
  696.         if (dsexample->blur_objects) { 
  697.           /* gaussian blur the detected objects using opencv */
  698.           if (blur_objects (dsexample, frame_meta->batch_id,
  699.             &obj_meta->rect_params, in_mat) != GST_FLOW_OK) { 
  700.           /* Error in blurring, skip processing on object. */
  701.             GST_ELEMENT_ERROR (dsexample, STREAM, FAILED,
  702.             ("blurring the object failed"), (NULL));
  703.             if (NvBufSurfaceUnMap (surface, frame_meta->batch_id, 0)){ 
  704.               GST_ELEMENT_ERROR (dsexample, STREAM, FAILED,
  705.                 ("%s:buffer unmap to be accessed by CPU failed", __func__), (NULL));
  706.             }
  707.             return GST_FLOW_ERROR;
  708.           }
  709.           continue;
  710.         }
  712.         /* Should not process on objects smaller than MIN_INPUT_OBJECT_WIDTH x MIN_INPUT_OBJECT_HEIGHT * since it will cause hardware scaling issues. */
  713.         if (obj_meta->rect_params.width < MIN_INPUT_OBJECT_WIDTH ||
  714.             obj_meta->rect_params.height < MIN_INPUT_OBJECT_HEIGHT)
  715.           continue;
  717.         /* Crop and scale the object */
  718.         if (get_converted_mat (dsexample,
  719.               surface, frame_meta->batch_id, &obj_meta->rect_params,
  720.               scale_ratio, dsexample->video_info.width,
  721.               dsexample->video_info.height) != GST_FLOW_OK) { 
  722.           /* Error in conversion, skip processing on object. */
  723.           continue;
  724.         }
  726.         /* Process the object crop to obtain label */
  727.         output = DsExampleProcess (dsexample->dsexamplelib_ctx,
  728.             dsexample->cvmat->data);
  730.         /* Attach labels for the object */
  731.         attach_metadata_object (dsexample, obj_meta, output);
  733.         free (output);
  734.       }
  736.       if (dsexample->blur_objects) { 
  737.       /* Cache the mapped data for device access */
  738.         NvBufSurfaceSyncForDevice (surface, frame_meta->batch_id, 0);
  740. #ifdef DSEXAMPLE_DEBUG
  741.         /* Use openCV to remove padding and convert RGBA to BGR. Can be skipped if * algorithm can handle padded RGBA data. */
  742. #if (CV_MAJOR_VERSION >= 4)
  743.         cv::cvtColor (in_mat, *dsexample->cvmat, cv::COLOR_RGBA2BGR);
  744. #else
  745.         cv::cvtColor (in_mat, *dsexample->cvmat, CV_RGBA2BGR);
  746. #endif
  747.         /* used to dump the converted mat to files for debug */
  748.         static guint cnt = 0;
  749.         cv::imwrite("out_" + std::to_string (cnt) + ".jpeg", *dsexample->cvmat);
  750.         cnt++;
  751. #endif
  752.       }
  753.     }
  754.   }
  755.   flow_ret = GST_FLOW_OK;
  757. error:
  759.   nvds_set_output_system_timestamp (inbuf, GST_ELEMENT_NAME (dsexample));
  760.   gst_buffer_unmap (inbuf, &in_map_info);
  761.   return flow_ret;
  762. }
  764. /** * Attach metadata for the full frame. We will be adding a new metadata. */
  765. static void
  766. attach_metadata_full_frame (GstDsExample * dsexample, NvDsFrameMeta *frame_meta,
  767.     gdouble scale_ratio, DsExampleOutput * output, guint batch_id)
  768. { 
  769.   NvDsBatchMeta *batch_meta = frame_meta->base_meta.batch_meta;
  770.   NvDsObjectMeta *object_meta = NULL;
  771.   static gchar font_name[] = "Serif";
  772.   GST_DEBUG_OBJECT (dsexample, "Attaching metadata %d\n", output->numObjects);
  774.   for (gint i = 0; i < output->numObjects; i++) { 
  775.     DsExampleObject *obj = &output->object[i];
  776.     object_meta = nvds_acquire_obj_meta_from_pool(batch_meta);
  777.     NvOSD_RectParams & rect_params = object_meta->rect_params;
  778.     NvOSD_TextParams & text_params = object_meta->text_params;
  780.     /* Assign bounding box coordinates */
  781.     rect_params.left = obj->left;
  782.     rect_params.top = obj->top;
  783.     rect_params.width = obj->width;
  784.     rect_params.height = obj->height;
  786.     /* Semi-transparent yellow background */
  787.     rect_params.has_bg_color = 0;
  788.     rect_params.bg_color = (NvOSD_ColorParams) { 
  789.     1, 1, 0, 0.4};
  790.     /* Red border of width 6 */
  791.     rect_params.border_width = 3;
  792.     rect_params.border_color = (NvOSD_ColorParams) { 
  793.     1, 0, 0, 1};
  795.     /* Scale the bounding boxes proportionally based on how the object/frame was * scaled during input */
  796.     rect_params.left /= scale_ratio;
  797.     rect_params.top /= scale_ratio;
  798.     rect_params.width /= scale_ratio;
  799.     rect_params.height /= scale_ratio;
  800.     GST_DEBUG_OBJECT (dsexample, "Attaching rect%d of batch%u"
  801.         " left->%f top->%f width->%f"
  802.         " height->%f label->%s\n", i, batch_id, rect_params.left,
  803.         rect_params.top, rect_params.width, rect_params.height, obj->label);
  805.     object_meta->object_id = UNTRACKED_OBJECT_ID;
  806.     g_strlcpy (object_meta->obj_label, obj->label, MAX_LABEL_SIZE);
  807.     /* display_text required heap allocated memory */
  808.     text_params.display_text = g_strdup (obj->label);
  809.     /* Display text above the left top corner of the object */
  810.     text_params.x_offset = rect_params.left;
  811.     text_params.y_offset = rect_params.top - 10;
  812.     /* Set black background for the text */
  813.     text_params.set_bg_clr = 1;
  814.     text_params.text_bg_clr = (NvOSD_ColorParams) { 
  815.     0, 0, 0, 1};
  816.     /* Font face, size and color */
  817.     text_params.font_params.font_name = font_name;
  818.     text_params.font_params.font_size = 11;
  819.     text_params.font_params.font_color = (NvOSD_ColorParams) { 
  820.     1, 1, 1, 1};
  822.     nvds_add_obj_meta_to_frame(frame_meta, object_meta, NULL);
  823.     frame_meta->bInferDone = TRUE;
  824.   }
  825. }
  827. /** * Only update string label in an existing object metadata. No bounding boxes. * We assume only one label per object is generated */
  828. static void
  829. attach_metadata_object (GstDsExample * dsexample, NvDsObjectMeta * obj_meta,
  830.     DsExampleOutput * output)
  831. { 
  832.   if (output->numObjects == 0)
  833.     return;
  834.   NvDsBatchMeta *batch_meta = obj_meta->base_meta.batch_meta;
  836.   NvDsClassifierMeta *classifier_meta =
  837.     nvds_acquire_classifier_meta_from_pool (batch_meta);
  839.   classifier_meta->unique_component_id = dsexample->unique_id;
  841.   NvDsLabelInfo *label_info =
  842.     nvds_acquire_label_info_meta_from_pool (batch_meta);
  843.   g_strlcpy (label_info->result_label, output->object[0].label, MAX_LABEL_SIZE);
  844.   nvds_add_label_info_meta_to_classifier(classifier_meta, label_info);
  845.   nvds_add_classifier_meta_to_object (obj_meta, classifier_meta);
  847.   nvds_acquire_meta_lock (batch_meta);
  848.   NvOSD_TextParams & text_params = obj_meta->text_params;
  849.   NvOSD_RectParams & rect_params = obj_meta->rect_params;
  851.   /* Below code to display the result */
  852.   /* Set black background for the text * display_text required heap allocated memory */
  853.   if (text_params.display_text) { 
  854.     gchar *conc_string = g_strconcat (text_params.display_text, " ",
  855.         output->object[0].label, NULL);
  856.     g_free (text_params.display_text);
  857.     text_params.display_text = conc_string;
  858.   } else { 
  859.     /* Display text above the left top corner of the object */
  860.     text_params.x_offset = rect_params.left;
  861.     text_params.y_offset = rect_params.top - 10;
  862.     text_params.display_text = g_strdup (output->object[0].label);
  863.     /* Font face, size and color */
  864.     text_params.font_params.font_name = (char *)"Serif";
  865.     text_params.font_params.font_size = 11;
  866.     text_params.font_params.font_color = (NvOSD_ColorParams) { 
  867.     1, 1, 1, 1};
  868.     /* Set black background for the text */
  869.     text_params.set_bg_clr = 1;
  870.     text_params.text_bg_clr = (NvOSD_ColorParams) { 
  871.     0, 0, 0, 1};
  872.   }
  873.   nvds_release_meta_lock (batch_meta);
  874. }
  876. /** * Boiler plate for registering a plugin and an element. */
  877. static gboolean
  878. dsexample_plugin_init (GstPlugin * plugin)
  879. { 
  880.   GST_DEBUG_CATEGORY_INIT (gst_dsexample_debug, "dsexample", 0,
  881.       "dsexample plugin");
  883.   return gst_element_register (plugin, "dsexample", GST_RANK_PRIMARY,
  884.       GST_TYPE_DSEXAMPLE);
  885. }
  887. GST_PLUGIN_DEFINE (GST_VERSION_MAJOR,
  888.     GST_VERSION_MINOR,
  889.     nvdsgst_dsexample,
  890.     DESCRIPTION, dsexample_plugin_init, DS_VERSION, LICENSE, BINARY_PACKAGE, URL)

发表评论

表情：
评论列表 （有 0 条评论，235人围观）

还没有评论，来说两句吧...

相关阅读

    相关 temp ssr

    由服务端请求首屏数据，而不是客户端请求首屏数据，这是“快”的一个主要原因。服务端在内网进行请求，数据响应速度快。客户端在不同网络环境进行数据请求，且外网http请求开销大，导致

    柔光的暖阳◎柔光的暖阳◎/ 0 赞/ 255 阅读