diff --git a/CHANGELOG b/CHANGELOG index 48619ae..fd387df 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,16 +1,15 @@ -Changelog (v0.8.1.0) +Changelog (v0.8.2.0) -------------------- - - NEW! Eye Contact feature: an AI algorithm to help users keep their gaze engaged in video communication. The feature jointly estimates a user’s gaze direction and redirects it to frontal in video sequences. - - NEW! Face Expression Estimation (Beta) feature estimates facial expression coefficients From the video or the provided facial landmarks. ExpressionApp is added to demonstrate the new Face Expressions feature. - - NEW! Default face model for the Face 3D mesh and tracking feature, face_model2.nvf, now ships with the SDK. The old SFM based face_model0.nvf is no longer required. - - 3D Body Pose Estimation: - - NEW! Added the support for Multi Person Tracking. This feature is supported by the Windows SDK only. - - FocalLength is now a NvAR_Parameter_Input. Users can now change FocalLength at every NvAR_Run() without having to call NvAR_Load(). - - The reference pose returned by the feature has been updated - - Facial landmark estimation - - NEW! There are now 2 modalities for facial landmark tracking: {0,1} -> {performance, quality}. Make sure to choose the preferred mode for your application. The default for face mesh fitting and expression estimation are 1, and the others are 0. - - Head Pose output from the NvAR_Feature_LandmarkDetection feature is now in the OpenGL convention. Changed from X-back(towards the camera), Y-right, Z-down to X-right, Y-up, Z-back(towards the camera). - - The sample apps now show the headpose in the OpenGL convention. The color coding of the axes is Red - X , Green - Y, Blue - Z - - NvCVImage_Transfer() now sets alpha to 255 or 1.0f when doing RGB -> RGBA. NvCVImage_CompositeRect() has a premultiplied alpha mode added - - Migrated to TensorRT 8.4.2.2 - - Migrated to CUDA 11.6u1 + - Face Expression Estimation + - 6DOF head pose now available. + - Enable by setting NvAR_Parameter_Config(PoseMode) to 1. Default is 0. + - Get the pose translation using NvAR_Parameter_Output(PoseTranslation). + - Camera intrinsics can be set using NvAR_Parameter_Input(CameraIntrinsicParams). + - Expression estimation model updated. The new model is more accurate, and enables subtle expressions such as asymmetric brows, inner/outer brow separation, asymmetric smile and frown, jaw left/right, squint shapes, crossing eye gaze, cheek puff, and more. + - Enable cheek puff shapes by setting NvAR_Parameter_Config(EnableCheekPuff) to 1. + - New face model for visualization recommended for FaceExpressions feature. The model has updated blendshapes, and face area partitioning. + - Enable using --render_model=face_model3.nvf in ExpressionApp. + - Eye Contact + - Performance improvements + - CUDA graphs functionality to enable CUDA optimization now available for Eye contact feature. + - Use the parameter NvAR_Parameter_Config(UseCudaGraph) to enable/disable CUDA graphs. Default is OFF diff --git a/nvar/include/nvAR_defs.h b/nvar/include/nvAR_defs.h index 5d8b169..80bcd89 100644 --- a/nvar/include/nvAR_defs.h +++ b/nvar/include/nvAR_defs.h @@ -117,20 +117,19 @@ typedef const char* NvAR_FeatureID; #define NvAR_Feature_BodyPoseEstimation "BodyPoseEstimation" // #define NvAR_Feature_GazeRedirection "GazeRedirection" // #define NvAR_Feature_FaceExpressions "FaceExpressions" // -#define NvAR_Feature_LivePortrait "LivePortrait" // -#define NvAR_Feature_FrameSelection "FrameSelection" // !FrameSelection! #define NvAR_Parameter_Input(Name) "NvAR_Parameter_Input_" #Name #define NvAR_Parameter_Output(Name) "NvAR_Parameter_Output_" #Name #define NvAR_Parameter_Config(Name) "NvAR_Parameter_Config_" #Name #define NvAR_Parameter_InOut(Name) "NvAR_Parameter_InOut_" #Name -#define NVAR_TEMPORAL_FILTER_FACE_BOX (1 << 0) // 0x001 -#define NVAR_TEMPORAL_FILTER_FACIAL_LANDMARKS (1 << 1) // 0x002 -#define NVAR_TEMPORAL_FILTER_FACE_ROTATIONAL_POSE (1 << 2) // 0x004 -#define NVAR_TEMPORAL_FILTER_FACIAL_EXPRESSIONS (1 << 4) // 0x010 -#define NVAR_TEMPORAL_FILTER_FACIAL_GAZE (1 << 5) // 0x020 -#define NVAR_TEMPORAL_FILTER_ENHANCE_EXPRESSIONS (1 << 8) // 0x100 +#define NVAR_TEMPORAL_FILTER_FACE_BOX (1U << 0) // 0x001 +#define NVAR_TEMPORAL_FILTER_FACIAL_LANDMARKS (1U << 1) // 0x002 +#define NVAR_TEMPORAL_FILTER_FACE_ROTATIONAL_POSE (1U << 2) // 0x004 +#define NVAR_TEMPORAL_FILTER_FACIAL_EXPRESSIONS (1U << 4) // 0x010 +#define NVAR_TEMPORAL_FILTER_FACIAL_GAZE (1U << 5) // 0x020 +#define NVAR_TEMPORAL_FILTER_ENHANCE_EXPRESSIONS (1U << 8) // 0x100 + /* Parameters supported by each NvAR_FeatureID @@ -178,6 +177,7 @@ NvAR_Parameter_Config(CUDAStream) - OPTIONAL // NvAR_Parameter_Config(Temporal) - OPTIONAL // NvAR_Parameter_Config(GazeMode) - OPTIONAL // NvAR_Parameter_Config(ModelName) - OPTIONAL // +NvAR_Parameter_Config(Mode) - OPTIONAL // NvAR_Parameter_Config(GPU) - OPTIONAL // NvAR_Parameter_Config(VertexCount) - QUERY // NvAR_Parameter_Config(TriangleCount) - QUERY // @@ -286,45 +286,20 @@ NvAR_Parameter_Config(ExpressionCount) - QUERY // Input: // NvAR_Parameter_Input(Image) // NvAR_Parameter_Input(Landmarks) - OPTIONAL // +NvAR_Parameter_Input(PoseMode) - OPTIONAL // +NvAR_Parameter_Input(CameraIntrinsicParams) - OPTIONAL // // Output: // NvAR_Parameter_Output(ExpressionCoefficients) // NvAR_Parameter_Output(Landmarks) - OPTIONAL // NvAR_Parameter_Output(LandmarksConfidence) - OPTIONAL // NvAR_Parameter_Output(Pose) - OPTIONAL // +NvAR_Parameter_Output(PoseTranslation) - OPTIONAL // NvAR_Parameter_Output(BoundingBoxes) - OPTIONAL // NvAR_Parameter_Output(BoundingBoxesConfidence) - OPTIONAL // -*******NvAR_Feature_LivePortrait******* // -Config: // -NvAR_Parameter_Config(FeatureDescription) // -NvAR_Parameter_Config(CUDAStream) // -NvAR_Parameter_Config(ModelDir) // -NvAR_Parameter_Config(Temporal) // -NvAR_Parameter_Config(Mode) // - // -Input: // -NvAR_Parameter_Input(SourceImage) // -NvAR_Parameter_Input(DriveImage) // -NvAR_Parameter_Input(BackgroundImage) // -Output: // -NvAR_Parameter_Output(GeneratedImage) // -*******NvAR_Feature_FrameSelection******* // !FrameSelection! -Config: // !FrameSelection! -NvAR_Parameter_Config(FeatureDescription) // !FrameSelection! -NvAR_Parameter_Config(CUDAStream) // !FrameSelection! -NvAR_Parameter_Config(ModelDir) // !FrameSelection! -NvAR_Parameter_Config(Mode) // !FrameSelection! - // !FrameSelection! -Input: // !FrameSelection! -NvAR_Parameter_Input(Image) // !FrameSelection! - // !FrameSelection! -Output: // !FrameSelection! -NvAR_Parameter_Output(FrameSelected) // !FrameSelection! */ - - #endif // NvAR_DEFS_H diff --git a/nvar/include/nvCVImage.h b/nvar/include/nvCVImage.h index 543df6e..3906119 100644 --- a/nvar/include/nvCVImage.h +++ b/nvar/include/nvCVImage.h @@ -538,6 +538,7 @@ NvCV_Status NvCV_API NvCVImage_TransferToYUV( //! Composite one source image over another using the given matte. //! This accommodates all RGB and RGBA formats, with u8 and f32 components. +//! If the bg has alpha, then the dst alpha is updated for use in subsequent composition. //! \param[in] fg the foreground source image. //! \param[in] bg the background source image. //! \param[in] mat the matte Yu8 (or Au8) image, indicating where the src should come through. @@ -547,7 +548,6 @@ NvCV_Status NvCV_API NvCVImage_TransferToYUV( //! \return NVCV_ERR_PIXELFORMAT if the pixel format is not accommodated. //! \return NVCV_ERR_MISMATCH if either the fg & bg & dst formats do not match, or if fg & bg & dst & mat are not //! in the same address space (CPU or GPU). -//! \bug Though RGBA destinations are accommodated, the A channel is not updated at all. #if RTX_CAMERA_IMAGE == 0 NvCV_Status NvCV_API NvCVImage_Composite(const NvCVImage *fg, const NvCVImage *bg, const NvCVImage *mat, NvCVImage *dst, struct CUstream_st *stream); @@ -556,8 +556,11 @@ NvCV_Status NvCV_API NvCVImage_Composite(const NvCVImage *fg, const NvCVImage *b #endif // RTX_CAMERA_IMAGE == 1 -//! Composite one source image over another using the given matte. +//! Composite one source image rectangular region over another using the given matte. //! This accommodates all RGB and RGBA formats, with u8 and f32 components. +//! If the bg has alpha, then the dst alpha is updated for use in subsequent composition. +//! If the background is not opaque, it is recommended that all images be premultiplied by alpha, +//! and mode 1 composition be used, to yield the most meaningful composite matte. //! \param[in] fg the foreground source image. //! \param[in] fgOrg the upper-left corner of the fg image to be composited (NULL implies (0,0)). //! \param[in] bg the background source image. @@ -570,14 +573,11 @@ NvCV_Status NvCV_API NvCVImage_Composite(const NvCVImage *fg, const NvCVImage *b //! \param[in] dstOrg the upper-left corner of the dst image to be updated (NULL implies (0,0)). //! \param[in] stream the CUDA stream on which the composition is to be performed. //! \note If a smaller region of a matte is desired, a window can be created using -//! NvCVImage_InitView() for chunky or NvCVImage_Init() for planar pixels. +//! NvCVImage_InitView() for chunky pixels, as illustrated below in NvCVImage_CompositeRectA(). //! \return NVCV_SUCCESS if the operation was successful. //! \return NVCV_ERR_PIXELFORMAT if the pixel format is not accommodated. //! \return NVCV_ERR_MISMATCH if either the fg & bg & dst formats do not match, or if fg & bg & dst & mat are not //! in the same address space (CPU or GPU). -//! \bug Though RGBA destinations are accommodated, the A channel is not updated at all. -//! \todo Accommodate premultiplied alpha, either as a flag in NvCVImage or as a different mode. -//! \todo If the destination has an A channel, update it as per Adobe and Pixar. NvCV_Status NvCV_API NvCVImage_CompositeRect( const NvCVImage *fg, const NvCVPoint2i *fgOrg, const NvCVImage *bg, const NvCVPoint2i *bgOrg, @@ -586,6 +586,41 @@ NvCV_Status NvCV_API NvCVImage_CompositeRect( struct CUstream_st *stream); +//! Composite one RGBA or BGRA source image rectangular region over another RGB, BGR, RGBA or BGRA region. +//! This accommodates all RGB and RGBA formats, with u8 and f32 components. +//! If the bg has alpha, then the dst alpha is updated for use in subsequent composition. +//! If the background is not opaque, it is recommended that all images be premultiplied by alpha, +//! and mode 1 composition be used, to yield the most meaningful composite matte. +//! \param[in] fg the foreground RGBA or BGRA source image. +//! \param[in] fgRect a sub-rectangle of the fg image (NULL implies the whole image). +//! \param[in] bg the background source image. +//! \param[in] bgOrg the upper-left corner of the bg image to be composited (NULL implies (0,0)). +//! \param[in] mode the composition mode: 0 (straight alpha over) or 1 (premultiplied alpha over). +//! \param[out] dst the destination image. This can be the same as fg or bg. +//! \param[in] dstOrg the upper-left corner of the dst image to be updated (NULL implies (0,0)). +//! \param[in] stream the CUDA stream on which the composition is to be performed. +//! \return NVCV_SUCCESS if the operation was successful. +//! \return NVCV_ERR_PIXELFORMAT if the pixel format is not accommodated. +//! \return NVCV_ERR_MISMATCH if either the fg & bg & dst formats do not match, or if fg & bg & dst & mat are not +//! in the same address space (CPU or GPU). +//! \bug fgRect will only work for chunky images, not planar. +#ifdef __cplusplus +inline NvCV_Status NvCVImage_CompositeRectA( + const NvCVImage *fg, const NvCVRect2i *fgRect, + const NvCVImage *bg, const NvCVPoint2i *bgOrg, + unsigned mode, + NvCVImage *dst, const NvCVPoint2i *dstOrg, + struct CUstream_st *stream +) { + if (fgRect) { + NvCVImage fgView(const_cast(fg), fgRect->x, fgRect->y, fgRect->width, fgRect->height); + return NvCVImage_CompositeRect(&fgView, nullptr, bg, bgOrg, &fgView, mode, dst, dstOrg, stream); + } + return NvCVImage_CompositeRect(fg, nullptr, bg, bgOrg, fg, mode, dst, dstOrg, stream); +} +#endif // __cplusplus + + //! Composite a source image over a constant color field using the given matte. //! \param[in] src the source image. //! \param[in] mat the matte image, indicating where the src should come through. @@ -598,7 +633,6 @@ NvCV_Status NvCV_API NvCVImage_CompositeRect( //! \return NVCV_ERR_MISMATCH if fg & mat & dst & bgColor are not in the same address space (CPU or GPU). //! \note The bgColor must remain valid until complete; this is an important consideration especially if //! the buffers are on the GPU and NvCVImage_CompositeOverConstant() runs asynchronously. -//! \bug Though RGBA destinations are accommodated, the A channel is not updated at all. NvCV_Status NvCV_API NvCVImage_CompositeOverConstant( #if RTX_CAMERA_IMAGE == 0 const NvCVImage *src, const NvCVImage *mat, const void *bgColor, NvCVImage *dst, struct CUstream_st *stream diff --git a/nvar/include/nvCVStatus.h b/nvar/include/nvCVStatus.h index 4b6dfa6..f560edc 100644 --- a/nvar/include/nvCVStatus.h +++ b/nvar/include/nvCVStatus.h @@ -80,6 +80,10 @@ typedef enum NvCV_Status { NVCV_ERR_TOOSMALL = -33, //!< A supplied parameter or buffer is not large enough. NVCV_ERR_TOOBIG = -34, //!< A supplied parameter is too big. NVCV_ERR_WRONGSIZE = -35, //!< A supplied parameter is not the expected size. + NVCV_ERR_OBJECTNOTFOUND = -36, //!< The specified object was not found. + NVCV_ERR_SINGULAR = -37, //!< A mathematical singularity has been encountered. + NVCV_ERR_NOTHINGRENDERED = -38, //!< Nothing was rendered in the specified region. + NVCV_ERR_CONVERGENCE = -39, //!< An iteration did not converge satisfactorily. NVCV_ERR_OPENGL = -98, //!< An OpenGL error has occurred. NVCV_ERR_DIRECT3D = -99, //!< A Direct3D error has occurred. diff --git a/samples/BodyTrack/BodyEngine.cpp b/samples/BodyTrack/BodyEngine.cpp index c94e536..e34824c 100644 --- a/samples/BodyTrack/BodyEngine.cpp +++ b/samples/BodyTrack/BodyEngine.cpp @@ -222,32 +222,32 @@ BodyEngine::Err BodyEngine::initKeyPointDetectionIOParams(NvCVImage* inBuf) { BAIL_IF_CVERR(nvErr, err, BodyEngine::Err::errParameter); nvErr = NvAR_SetF32Array(keyPointDetectHandle, NvAR_Parameter_Output(KeyPointsConfidence), - keypoints_confidence.data(), sizeof(float)); + keypoints_confidence.data(), sizeof(float)); BAIL_IF_CVERR(nvErr, err, BodyEngine::Err::errParameter); #if NV_MULTI_OBJECT_TRACKER if (bEnablePeopleTracking) { - output_tracking_bbox_size = maxTargetsTracked; - output_tracking_bbox_data.assign(output_tracking_bbox_size, { 0.f, 0.f, 0.f, 0.f, 0 }); - output_tracking_bboxes.boxes = output_tracking_bbox_data.data(); - output_tracking_bboxes.max_boxes = (uint8_t)output_tracking_bbox_size; - output_tracking_bboxes.num_boxes = 0; - nvErr = - NvAR_SetObject(keyPointDetectHandle, NvAR_Parameter_Output(TrackingBoundingBoxes), &output_tracking_bboxes, sizeof(NvAR_TrackingBBoxes)); - BAIL_IF_CVERR(nvErr, err, BodyEngine::Err::errParameter); + output_tracking_bbox_size = maxTargetsTracked; + output_tracking_bbox_data.assign(output_tracking_bbox_size, { 0.f, 0.f, 0.f, 0.f, 0 }); + output_tracking_bboxes.boxes = output_tracking_bbox_data.data(); + output_tracking_bboxes.max_boxes = (uint8_t)output_tracking_bbox_size; + output_tracking_bboxes.num_boxes = 0; + nvErr = + NvAR_SetObject(keyPointDetectHandle, NvAR_Parameter_Output(TrackingBoundingBoxes), &output_tracking_bboxes, sizeof(NvAR_TrackingBBoxes)); + BAIL_IF_CVERR(nvErr, err, BodyEngine::Err::errParameter); } else { - output_bbox_data.assign(25, { 0.f, 0.f, 0.f, 0.f }); - output_bbox_conf_data.assign(25, 0.f); - output_bboxes.boxes = output_bbox_data.data(); - output_bboxes.max_boxes = (uint8_t)output_bbox_data.size(); - output_bboxes.num_boxes = 0; - nvErr = NvAR_SetObject(keyPointDetectHandle, NvAR_Parameter_Output(BoundingBoxes), &output_bboxes, sizeof(NvAR_BBoxes)); - BAIL_IF_CVERR(nvErr, err, BodyEngine::Err::errParameter); + output_bbox_data.assign(25, { 0.f, 0.f, 0.f, 0.f }); + output_bbox_conf_data.assign(25, 0.f); + output_bboxes.boxes = output_bbox_data.data(); + output_bboxes.max_boxes = (uint8_t)output_bbox_data.size(); + output_bboxes.num_boxes = 0; + nvErr = NvAR_SetObject(keyPointDetectHandle, NvAR_Parameter_Output(BoundingBoxes), &output_bboxes, sizeof(NvAR_BBoxes)); + BAIL_IF_CVERR(nvErr, err, BodyEngine::Err::errParameter); - nvErr = NvAR_SetF32Array(keyPointDetectHandle, NvAR_Parameter_Output(BoundingBoxesConfidence), - output_bbox_conf_data.data(), output_bboxes.max_boxes); - BAIL_IF_CVERR(nvErr, err, BodyEngine::Err::errParameter); + nvErr = NvAR_SetF32Array(keyPointDetectHandle, NvAR_Parameter_Output(BoundingBoxesConfidence), + output_bbox_conf_data.data(), output_bboxes.max_boxes); + BAIL_IF_CVERR(nvErr, err, BodyEngine::Err::errParameter); } #else output_bbox_data.assign(25, { 0.f, 0.f, 0.f, 0.f }); @@ -407,7 +407,7 @@ NvAR_TrackingBBoxes* BodyEngine::getTrackingBBoxes() { return &output_tracking_b float average_confidence = 0.0f; float* keypoints_confidence_all = getKeyPointsConfidence(); for (int i = 0; i < output_bboxes.num_boxes; i++) { - for (unsigned int j = 0; j < numKeyPoints; j++) { + for (unsigned int j = 0; j < numKeyPoints; j++) { average_confidence += keypoints_confidence_all[i * numKeyPoints + j]; } } @@ -483,39 +483,39 @@ unsigned BodyEngine::acquireBodyBoxAndKeyPoints(cv::Mat& src, NvAR_Point2f* refM } #if NV_MULTI_OBJECT_TRACKER unsigned BodyEngine::acquireBodyBoxAndKeyPoints(cv::Mat& src, NvAR_Point2f* refMarks, NvAR_Point3f* refKeyPoints3D, - NvAR_Quaternion* refJointAngles, NvAR_TrackingBBoxes* refBodyBoxes, int /*variant*/) { - unsigned n = 0; - NvCVImage fxSrcChunkyCPU; - (void)NVWrapperForCVMat(&src, &fxSrcChunkyCPU); - NvCV_Status cvErr = NvCVImage_Transfer(&fxSrcChunkyCPU, &inputImageBuffer, 1.0f, stream, &tmpImage); + NvAR_Quaternion* refJointAngles, NvAR_TrackingBBoxes* refBodyBoxes, int /*variant*/) { + unsigned n = 0; + NvCVImage fxSrcChunkyCPU; + (void)NVWrapperForCVMat(&src, &fxSrcChunkyCPU); + NvCV_Status cvErr = NvCVImage_Transfer(&fxSrcChunkyCPU, &inputImageBuffer, 1.0f, stream, &tmpImage); - if (NVCV_SUCCESS != cvErr) { - return n; - } + if (NVCV_SUCCESS != cvErr) { + return n; + } #ifdef DEBUG_PERF_RUNTIME - auto start = std::chrono::high_resolution_clock::now(); + auto start = std::chrono::high_resolution_clock::now(); #endif - if (findKeyPoints() != NVCV_SUCCESS) return 0; - memcpy(refBodyBoxes, getTrackingBBoxes(), sizeof(NvAR_TrackingBBoxes)); - n = 1; + if (findKeyPoints() != NVCV_SUCCESS) return 0; + memcpy(refBodyBoxes, getTrackingBBoxes(), sizeof(NvAR_TrackingBBoxes)); + n = 1; #ifdef DEBUG_PERF_RUNTIME - auto start2 = std::chrono::high_resolution_clock::now(); + auto start2 = std::chrono::high_resolution_clock::now(); #endif - memcpy(refMarks, getKeyPoints(), sizeof(NvAR_Point2f) * numKeyPoints * batchSize); - memcpy(refKeyPoints3D, getKeyPoints3D(), sizeof(NvAR_Point3f) * numKeyPoints * batchSize); - memcpy(refJointAngles, getJointAngles(), sizeof(NvAR_Quaternion) * numKeyPoints * batchSize); + memcpy(refMarks, getKeyPoints(), sizeof(NvAR_Point2f) * numKeyPoints * batchSize); + memcpy(refKeyPoints3D, getKeyPoints3D(), sizeof(NvAR_Point3f) * numKeyPoints * batchSize); + memcpy(refJointAngles, getJointAngles(), sizeof(NvAR_Quaternion) * numKeyPoints * batchSize); #ifdef DEBUG_PERF_RUNTIME - auto end = std::chrono::high_resolution_clock::now(); - auto duration3 = std::chrono::duration_cast(start2 - start); - std::cout << "[bodypose] run findKeyPoints(): " << duration3.count() << " microseconds" << std::endl; - auto duration2 = std::chrono::duration_cast(end - start2); - std::cout << "[bodypose] keypoint copy time: " << duration2.count() << " microseconds" << std::endl; - auto duration = std::chrono::duration_cast(end - start); - std::cout << "[bodypose] end-to-end time: " << duration.count() << " microseconds" << std::endl; + auto end = std::chrono::high_resolution_clock::now(); + auto duration3 = std::chrono::duration_cast(start2 - start); + std::cout << "[bodypose] run findKeyPoints(): " << duration3.count() << " microseconds" << std::endl; + auto duration2 = std::chrono::duration_cast(end - start2); + std::cout << "[bodypose] keypoint copy time: " << duration2.count() << " microseconds" << std::endl; + auto duration = std::chrono::duration_cast(end - start); + std::cout << "[bodypose] end-to-end time: " << duration.count() << " microseconds" << std::endl; #endif - return n; + return n; } #endif void BodyEngine::setBodyStabilization(bool _bStabilizeBody) { bStabilizeBody = _bStabilizeBody; } @@ -534,10 +534,10 @@ BodyEngine::Err BodyEngine::setFocalLength(float _bFocalLength) { void BodyEngine::useCudaGraph(bool _bUseCudaGraph) { bUseCudaGraph = _bUseCudaGraph; } #if NV_MULTI_OBJECT_TRACKER void BodyEngine::enablePeopleTracking(bool _bEnablePeopleTracking, unsigned int _shadowTrackingAge, unsigned int _probationAge, unsigned int _maxTargetsTracked) { - bEnablePeopleTracking = _bEnablePeopleTracking; - shadowTrackingAge = _shadowTrackingAge; + bEnablePeopleTracking = _bEnablePeopleTracking; + shadowTrackingAge = _shadowTrackingAge; probationAge = _probationAge; - maxTargetsTracked = _maxTargetsTracked; + maxTargetsTracked = _maxTargetsTracked; } #endif void BodyEngine::setAppMode(BodyEngine::mode _mode) { appMode = _mode; } diff --git a/samples/BodyTrack/BodyEngine.h b/samples/BodyTrack/BodyEngine.h index a1669f0..fb3dc63 100644 --- a/samples/BodyTrack/BodyEngine.h +++ b/samples/BodyTrack/BodyEngine.h @@ -154,7 +154,7 @@ class BodyEngine { NvAR_Quaternion* refJointAngles, NvAR_BBoxes* refBodyBoxes, int variant = 0); #if NV_MULTI_OBJECT_TRACKER unsigned acquireBodyBoxAndKeyPoints(cv::Mat& src, NvAR_Point2f* refMarks, NvAR_Point3f* refKeyPoints3D, - NvAR_Quaternion* refJointAngles, NvAR_TrackingBBoxes* refBodyBoxes, int variant = 0); + NvAR_Quaternion* refJointAngles, NvAR_TrackingBBoxes* refBodyBoxes, int variant = 0); #endif void setBodyStabilization(bool); void setMode(int); @@ -206,10 +206,10 @@ class BodyEngine { bStabilizeBody = true; bUseCudaGraph = true; #if NV_MULTI_OBJECT_TRACKER - bEnablePeopleTracking = false; - shadowTrackingAge = 90; + bEnablePeopleTracking = false; + shadowTrackingAge = 90; probationAge = 10; - maxTargetsTracked = 30; + maxTargetsTracked = 30; #endif bFocalLength = FOCAL_LENGTH_DEFAULT; confidenceThreshold = 0.f; diff --git a/samples/BodyTrack/BodyTrack.cpp b/samples/BodyTrack/BodyTrack.cpp index b3d3eed..2966e61 100644 --- a/samples/BodyTrack/BodyTrack.cpp +++ b/samples/BodyTrack/BodyTrack.cpp @@ -108,10 +108,10 @@ static void Usage() { " --mode[=0|1] Model Mode. 0: High Quality, 1: High Performance\n" " --app_mode[=(0|1)] App mode. 0: Body detection, 1: Keypoint detection " #if NV_MULTI_OBJECT_TRACKER - " --enable_people_tracking[=(0|1)] Enables people tracking " - " --shadow_tracking_age Shadow Tracking Age after which tracking information of a person is removed. Measured in frames" + " --enable_people_tracking[=(0|1)] Enables people tracking " + " --shadow_tracking_age Shadow Tracking Age after which tracking information of a person is removed. Measured in frames" " --probation_age Length of probationary period. Measured in frames" - " --max_targets_tracked Maximum number of targets to be tracked " + " --max_targets_tracked Maximum number of targets to be tracked " #endif "(Default).\n" " --benchmarks[=] run benchmarks\n"); @@ -213,10 +213,10 @@ static int ParseMyArgs(int argc, char **argv) { GetFlagArgVal("camindex", arg, &FLAG_camindex) || GetFlagArgVal("use_cuda_graph", arg, &FLAG_useCudaGraph) || #if NV_MULTI_OBJECT_TRACKER - GetFlagArgVal("enable_people_tracking", arg, &FLAG_enablePeopleTracking) || - GetFlagArgVal("shadow_tracking_age", arg, &FLAG_shadowTrackingAge) || + GetFlagArgVal("enable_people_tracking", arg, &FLAG_enablePeopleTracking) || + GetFlagArgVal("shadow_tracking_age", arg, &FLAG_shadowTrackingAge) || GetFlagArgVal("probation_age", arg, &FLAG_probationAge) || - GetFlagArgVal("max_targets_tracked", arg, &FLAG_maxTargetsTracked) || + GetFlagArgVal("max_targets_tracked", arg, &FLAG_maxTargetsTracked) || #endif GetFlagArgVal("temporal", arg, &FLAG_temporal))) { continue; @@ -389,7 +389,7 @@ void DoApp::processKey(int key) { if (FLAG_enablePeopleTracking) body_ar_engine.createFeatures(FLAG_modelPath.c_str()); else #endif - body_ar_engine.createFeatures(FLAG_modelPath.c_str(), 1); + body_ar_engine.createFeatures(FLAG_modelPath.c_str(), 1); body_ar_engine.initFeatureIOParams(); break; case '1': @@ -471,48 +471,48 @@ void DoApp::DrawBBoxes(const cv::Mat &src, NvAR_Rect *output_bbox) { } void DoApp::DrawBBoxes(const cv::Mat &src, NvAR_BBoxes *output_bbox) { - cv::Mat frm; - if (FLAG_offlineMode) - frm = src.clone(); - else - frm = src; + cv::Mat frm; + if (FLAG_offlineMode) + frm = src.clone(); + else + frm = src; - if (output_bbox) { - for (int i = 0; i < output_bbox->num_boxes; i++) { - auto color = cv::Scalar(255, 255, 255); - cv::rectangle(frm, cv::Point(lround(output_bbox->boxes[i].x), lround(output_bbox->boxes[i].y)), - cv::Point(lround(output_bbox->boxes[i].x + output_bbox->boxes[i].width), lround(output_bbox->boxes[i].y + output_bbox->boxes[i].height)), - cv::Scalar(255, 0, 0), 2); - } - - } - - if (FLAG_offlineMode) bodyDetectOutputVideo.write(frm); + if (output_bbox) { + for (int i = 0; i < output_bbox->num_boxes; i++) { + auto color = cv::Scalar(255, 255, 255); + cv::rectangle(frm, cv::Point(lround(output_bbox->boxes[i].x), lround(output_bbox->boxes[i].y)), + cv::Point(lround(output_bbox->boxes[i].x + output_bbox->boxes[i].width), lround(output_bbox->boxes[i].y + output_bbox->boxes[i].height)), + cv::Scalar(255, 0, 0), 2); + } + + } + + if (FLAG_offlineMode) bodyDetectOutputVideo.write(frm); } #if NV_MULTI_OBJECT_TRACKER void DoApp::DrawBBoxes(const cv::Mat &src, NvAR_TrackingBBoxes *output_bbox) { - cv::Mat frm; - if (FLAG_offlineMode) - frm = src.clone(); - else - frm = src; + cv::Mat frm; + if (FLAG_offlineMode) + frm = src.clone(); + else + frm = src; - if (output_bbox) { - for (int i = 0; i < output_bbox->num_boxes; i++) { - if (colorCodes.size() <= output_bbox->boxes[i].tracking_id) - colorCodes.push_back(cv::Scalar(rand() & 0xFF, rand() & 0xFF, rand() & 0xFF)); - auto color = colorCodes[output_bbox->boxes[i].tracking_id]; - std::string text = "ID: " + std::to_string(output_bbox->boxes[i].tracking_id); - - cv::rectangle(frm, cv::Point(lround(output_bbox->boxes[i].bbox.x), lround(output_bbox->boxes[i].bbox.y)), - cv::Point(lround(output_bbox->boxes[i].bbox.x + output_bbox->boxes[i].bbox.width), lround(output_bbox->boxes[i].bbox.y + output_bbox->boxes[i].bbox.height)), - color, 2); - cv::putText(frm, text, cv::Point(lround(output_bbox->boxes[i].bbox.x), lround(output_bbox->boxes[i].bbox.y) - 10), cv::FONT_HERSHEY_SIMPLEX, 0.9, color, 2); - } + if (output_bbox) { + for (int i = 0; i < output_bbox->num_boxes; i++) { + if (colorCodes.size() <= output_bbox->boxes[i].tracking_id) + colorCodes.push_back(cv::Scalar(rand() & 0xFF, rand() & 0xFF, rand() & 0xFF)); + auto color = colorCodes[output_bbox->boxes[i].tracking_id]; + std::string text = "ID: " + std::to_string(output_bbox->boxes[i].tracking_id); + + cv::rectangle(frm, cv::Point(lround(output_bbox->boxes[i].bbox.x), lround(output_bbox->boxes[i].bbox.y)), + cv::Point(lround(output_bbox->boxes[i].bbox.x + output_bbox->boxes[i].bbox.width), lround(output_bbox->boxes[i].bbox.y + output_bbox->boxes[i].bbox.height)), + color, 2); + cv::putText(frm, text, cv::Point(lround(output_bbox->boxes[i].bbox.x), lround(output_bbox->boxes[i].bbox.y) - 10), cv::FONT_HERSHEY_SIMPLEX, 0.9, color, 2); + } - } + } - if (FLAG_offlineMode) bodyDetectOutputVideo.write(frm); + if (FLAG_offlineMode) bodyDetectOutputVideo.write(frm); } #endif void DoApp::writeVideoAndEstResults(const cv::Mat &frm, NvAR_BBoxes output_bboxes, NvAR_Point2f* keypoints) { @@ -561,50 +561,50 @@ void DoApp::writeVideoAndEstResults(const cv::Mat &frm, NvAR_BBoxes output_bboxe } #if NV_MULTI_OBJECT_TRACKER void DoApp::writeVideoAndEstResults(const cv::Mat &frm, NvAR_TrackingBBoxes output_bboxes, NvAR_Point2f* keypoints) { - if (captureVideo) { - if (!capturedVideo.isOpened()) { - const std::string currentCalendarTime = getCalendarTime(); - const std::string capturedOutputFileName = currentCalendarTime + ".mp4"; - getFPS(); - if (frameTime) { - float fps = (float)(1.0 / frameTime); - capturedVideo.open(capturedOutputFileName, StringToFourcc(FLAG_captureCodec), fps, - cv::Size(frm.cols, frm.rows)); - if (!capturedVideo.isOpened()) { - std::cout << "Error: Could not open video: \"" << capturedOutputFileName << "\"\n"; - return; - } - if (FLAG_verbose) { - std::cout << "Capturing video started" << std::endl; - } - } - else { // If frameTime is 0.f, returns without writing the frame to the Video - return; - } - const std::string outputsFileName = currentCalendarTime + ".txt"; - bodyEngineVideoOutputFile.open(outputsFileName, std::ios_base::out); - if (!bodyEngineVideoOutputFile.is_open()) { - std::cout << "Error: Could not open file: \"" << outputsFileName << "\"\n"; - return; - } - std::string keyPointDetectionMode = (keypoints == NULL) ? "Off" : "On"; - bodyEngineVideoOutputFile << "// BodyDetectOn, KeyPointDetect" << keyPointDetectionMode << "\n "; - bodyEngineVideoOutputFile - << "// kNumPeople, (bbox_x, bbox_y, bbox_w, bbox_h){ kNumPeople}, kNumLMs, [lm_x, lm_y]{kNumLMs}\n"; - } - // Write each frame to the Video - capturedVideo << frm; - writeEstResults(bodyEngineVideoOutputFile, output_bboxes, keypoints); - } - else { - if (capturedVideo.isOpened()) { - if (FLAG_verbose) { - std::cout << "Capturing video ended" << std::endl; - } - capturedVideo.release(); - if (bodyEngineVideoOutputFile.is_open()) bodyEngineVideoOutputFile.close(); - } - } + if (captureVideo) { + if (!capturedVideo.isOpened()) { + const std::string currentCalendarTime = getCalendarTime(); + const std::string capturedOutputFileName = currentCalendarTime + ".mp4"; + getFPS(); + if (frameTime) { + float fps = (float)(1.0 / frameTime); + capturedVideo.open(capturedOutputFileName, StringToFourcc(FLAG_captureCodec), fps, + cv::Size(frm.cols, frm.rows)); + if (!capturedVideo.isOpened()) { + std::cout << "Error: Could not open video: \"" << capturedOutputFileName << "\"\n"; + return; + } + if (FLAG_verbose) { + std::cout << "Capturing video started" << std::endl; + } + } + else { // If frameTime is 0.f, returns without writing the frame to the Video + return; + } + const std::string outputsFileName = currentCalendarTime + ".txt"; + bodyEngineVideoOutputFile.open(outputsFileName, std::ios_base::out); + if (!bodyEngineVideoOutputFile.is_open()) { + std::cout << "Error: Could not open file: \"" << outputsFileName << "\"\n"; + return; + } + std::string keyPointDetectionMode = (keypoints == NULL) ? "Off" : "On"; + bodyEngineVideoOutputFile << "// BodyDetectOn, KeyPointDetect" << keyPointDetectionMode << "\n "; + bodyEngineVideoOutputFile + << "// kNumPeople, (bbox_x, bbox_y, bbox_w, bbox_h){ kNumPeople}, kNumLMs, [lm_x, lm_y]{kNumLMs}\n"; + } + // Write each frame to the Video + capturedVideo << frm; + writeEstResults(bodyEngineVideoOutputFile, output_bboxes, keypoints); + } + else { + if (capturedVideo.isOpened()) { + if (FLAG_verbose) { + std::cout << "Capturing video ended" << std::endl; + } + capturedVideo.release(); + if (bodyEngineVideoOutputFile.is_open()) bodyEngineVideoOutputFile.close(); + } + } } #endif void DoApp::writeEstResults(std::ofstream &outputFile, NvAR_BBoxes output_bboxes, NvAR_Point2f* keypoints) { @@ -651,49 +651,49 @@ void DoApp::writeEstResults(std::ofstream &outputFile, NvAR_BBoxes output_bboxes } #if NV_MULTI_OBJECT_TRACKER void DoApp::writeEstResults(std::ofstream &outputFile, NvAR_TrackingBBoxes output_bboxes, NvAR_Point2f* keypoints) { - /** - * Output File Format : - * BodyDetectOn, KeyPointDetectOn - * kNumPeople, (bbox_x, bbox_y, bbox_w, bbox_h){ kNumPeople}, kNumKPs, [j_x, j_y]{kNumKPs} - */ + /** + * Output File Format : + * BodyDetectOn, KeyPointDetectOn + * kNumPeople, (bbox_x, bbox_y, bbox_w, bbox_h){ kNumPeople}, kNumKPs, [j_x, j_y]{kNumKPs} + */ - int bodyDetectOn = (body_ar_engine.appMode == BodyEngine::mode::bodyDetection || - body_ar_engine.appMode == BodyEngine::mode::keyPointDetection) - ? 1 - : 0; - int keyPointDetectOn = (body_ar_engine.appMode == BodyEngine::mode::keyPointDetection) - ? 1 - : 0; - outputFile << bodyDetectOn << "," << keyPointDetectOn << "\n"; + int bodyDetectOn = (body_ar_engine.appMode == BodyEngine::mode::bodyDetection || + body_ar_engine.appMode == BodyEngine::mode::keyPointDetection) + ? 1 + : 0; + int keyPointDetectOn = (body_ar_engine.appMode == BodyEngine::mode::keyPointDetection) + ? 1 + : 0; + outputFile << bodyDetectOn << "," << keyPointDetectOn << "\n"; - if (bodyDetectOn && output_bboxes.num_boxes) { - // Append number of bodies detected in the current frame - outputFile << unsigned(output_bboxes.num_boxes) << ","; - // write outputbboxes to outputFile - for (size_t i = 0; i < output_bboxes.num_boxes; i++) { - int x1 = (int)output_bboxes.boxes[i].bbox.x, y1 = (int)output_bboxes.boxes[i].bbox.y, - width = (int)output_bboxes.boxes[i].bbox.width, height = (int)output_bboxes.boxes[i].bbox.height; - unsigned int tracking_id = output_bboxes.boxes[i].tracking_id; - outputFile << x1 << "," << y1 << "," << width << "," << height << "," << tracking_id << ","; - } - } - else { - outputFile << "0,"; - } - if (keyPointDetectOn && output_bboxes.num_boxes) { - int numKeyPoints = body_ar_engine.getNumKeyPoints(); - // Append number of keypoints - outputFile << numKeyPoints << ","; - // Append 2 * number of keypoint values - NvAR_Point2f *pt, *endPt; - for (endPt = (pt = (NvAR_Point2f *)keypoints) + numKeyPoints; pt < endPt; ++pt) - outputFile << pt->x << "," << pt->y << ","; - } - else { - outputFile << "0,"; - } + if (bodyDetectOn && output_bboxes.num_boxes) { + // Append number of bodies detected in the current frame + outputFile << unsigned(output_bboxes.num_boxes) << ","; + // write outputbboxes to outputFile + for (size_t i = 0; i < output_bboxes.num_boxes; i++) { + int x1 = (int)output_bboxes.boxes[i].bbox.x, y1 = (int)output_bboxes.boxes[i].bbox.y, + width = (int)output_bboxes.boxes[i].bbox.width, height = (int)output_bboxes.boxes[i].bbox.height; + unsigned int tracking_id = output_bboxes.boxes[i].tracking_id; + outputFile << x1 << "," << y1 << "," << width << "," << height << "," << tracking_id << ","; + } + } + else { + outputFile << "0,"; + } + if (keyPointDetectOn && output_bboxes.num_boxes) { + int numKeyPoints = body_ar_engine.getNumKeyPoints(); + // Append number of keypoints + outputFile << numKeyPoints << ","; + // Append 2 * number of keypoint values + NvAR_Point2f *pt, *endPt; + for (endPt = (pt = (NvAR_Point2f *)keypoints) + numKeyPoints; pt < endPt; ++pt) + outputFile << pt->x << "," << pt->y << ","; + } + else { + outputFile << "0,"; + } - outputFile << "\n"; + outputFile << "\n"; } #endif void DoApp::writeFrameAndEstResults(const cv::Mat &frm, NvAR_BBoxes output_bboxes, NvAR_Point2f* keypoints) { @@ -722,28 +722,28 @@ void DoApp::writeFrameAndEstResults(const cv::Mat &frm, NvAR_BBoxes output_bboxe } #if NV_MULTI_OBJECT_TRACKER void DoApp::writeFrameAndEstResults(const cv::Mat &frm, NvAR_TrackingBBoxes output_bboxes, NvAR_Point2f* keypoints) { - if (captureFrame) { - const std::string currentCalendarTime = getCalendarTime(); - const std::string capturedFrame = currentCalendarTime + ".png"; - cv::imwrite(capturedFrame, frm); - if (FLAG_verbose) { - std::cout << "Captured the frame" << std::endl; - } - // Write Body Engine Outputs - const std::string outputFilename = currentCalendarTime + ".txt"; - std::ofstream outputFile; - outputFile.open(outputFilename, std::ios_base::out); - if (!outputFile.is_open()) { - std::cout << "Error: Could not open file: \"" << outputFilename << "\"\n"; - return; - } - std::string keyPointDetectionMode = (keypoints == NULL) ? "Off" : "On"; - outputFile << "// BodyDetectOn, KeyPointDetect" << keyPointDetectionMode << "\n"; - outputFile << "// kNumPeople, (bbox_x, bbox_y, bbox_w, bbox_h){ kNumPeople}, kNumLMs, [lm_x, lm_y]{kNumLMs}\n"; - writeEstResults(outputFile, output_bboxes, keypoints); - if (outputFile.is_open()) outputFile.close(); - captureFrame = false; - } + if (captureFrame) { + const std::string currentCalendarTime = getCalendarTime(); + const std::string capturedFrame = currentCalendarTime + ".png"; + cv::imwrite(capturedFrame, frm); + if (FLAG_verbose) { + std::cout << "Captured the frame" << std::endl; + } + // Write Body Engine Outputs + const std::string outputFilename = currentCalendarTime + ".txt"; + std::ofstream outputFile; + outputFile.open(outputFilename, std::ios_base::out); + if (!outputFile.is_open()) { + std::cout << "Error: Could not open file: \"" << outputFilename << "\"\n"; + return; + } + std::string keyPointDetectionMode = (keypoints == NULL) ? "Off" : "On"; + outputFile << "// BodyDetectOn, KeyPointDetect" << keyPointDetectionMode << "\n"; + outputFile << "// kNumPeople, (bbox_x, bbox_y, bbox_w, bbox_h){ kNumPeople}, kNumLMs, [lm_x, lm_y]{kNumLMs}\n"; + writeEstResults(outputFile, output_bboxes, keypoints); + if (outputFile.is_open()) outputFile.close(); + captureFrame = false; + } } #endif void DoApp::DrawKeyPointLine(const cv::Mat& src, NvAR_Point2f* keypoints, int point1, int point2, int color) { @@ -799,184 +799,184 @@ void DoApp::DrawKeyPointsAndEdges(const cv::Mat& src, NvAR_Point2f* keypoints, i for (int i = 0; i < body_ar_engine.output_tracking_bboxes.num_boxes; i++) { - keypoints = keypointsBatch8 + (i * 34); + keypoints = keypointsBatch8 + (i * 34); - for (endPt = (pt = (NvAR_Point2f*)keypoints) + numKeyPoints; pt < endPt; ++pt) - cv::circle(frm, cv::Point(lround(pt->x), lround(pt->y)), 4, cv::Scalar(180, 180, 180), -1); + for (endPt = (pt = (NvAR_Point2f*)keypoints) + numKeyPoints; pt < endPt; ++pt) + cv::circle(frm, cv::Point(lround(pt->x), lround(pt->y)), 4, cv::Scalar(180, 180, 180), -1); - if (output_bbox) { - while (colorCodes.size()<= output_bbox->boxes[i].tracking_id) - colorCodes.push_back(cv::Scalar(rand() & 0xFF, rand() & 0xFF, rand() & 0xFF)); - auto color = colorCodes[output_bbox->boxes[i].tracking_id]; - std::string text = "ID: "+std::to_string(output_bbox->boxes[i].tracking_id); - cv::rectangle(frm, cv::Point(lround(output_bbox->boxes[i].bbox.x), lround(output_bbox->boxes[i].bbox.y)), - cv::Point(lround(output_bbox->boxes[i].bbox.x + output_bbox->boxes[i].bbox.width), lround(output_bbox->boxes[i].bbox.y + output_bbox->boxes[i].bbox.height)), - color, 2); - cv::putText(frm, text, cv::Point(lround(output_bbox->boxes[i].bbox.x), lround(output_bbox->boxes[i].bbox.y) - 10), cv::FONT_HERSHEY_SIMPLEX, 0.5, color, 2); - } - + if (output_bbox) { + while (colorCodes.size()<= output_bbox->boxes[i].tracking_id) + colorCodes.push_back(cv::Scalar(rand() & 0xFF, rand() & 0xFF, rand() & 0xFF)); + auto color = colorCodes[output_bbox->boxes[i].tracking_id]; + std::string text = "ID: "+std::to_string(output_bbox->boxes[i].tracking_id); + cv::rectangle(frm, cv::Point(lround(output_bbox->boxes[i].bbox.x), lround(output_bbox->boxes[i].bbox.y)), + cv::Point(lround(output_bbox->boxes[i].bbox.x + output_bbox->boxes[i].bbox.width), lround(output_bbox->boxes[i].bbox.y + output_bbox->boxes[i].bbox.height)), + color, 2); + cv::putText(frm, text, cv::Point(lround(output_bbox->boxes[i].bbox.x), lround(output_bbox->boxes[i].bbox.y) - 10), cv::FONT_HERSHEY_SIMPLEX, 0.5, color, 2); + } + - // center body - DrawKeyPointLine(frm, keypoints, pelvis, torso, kColorGreen); - DrawKeyPointLine(frm, keypoints, torso, neck, kColorGreen); - DrawKeyPointLine(frm, keypoints, neck, pelvis, kColorGreen); + // center body + DrawKeyPointLine(frm, keypoints, pelvis, torso, kColorGreen); + DrawKeyPointLine(frm, keypoints, torso, neck, kColorGreen); + DrawKeyPointLine(frm, keypoints, neck, pelvis, kColorGreen); - // right side - DrawKeyPointLine(frm, keypoints, right_ankle, right_knee, kColorRed); - DrawKeyPointLine(frm, keypoints, right_knee, right_hip, kColorRed); - DrawKeyPointLine(frm, keypoints, right_hip, pelvis, kColorRed); - DrawKeyPointLine(frm, keypoints, right_hip, right_shoulder, kColorRed); - DrawKeyPointLine(frm, keypoints, right_shoulder, right_elbow, kColorRed); - DrawKeyPointLine(frm, keypoints, right_elbow, right_wrist, kColorRed); - DrawKeyPointLine(frm, keypoints, right_shoulder, neck, kColorRed); + // right side + DrawKeyPointLine(frm, keypoints, right_ankle, right_knee, kColorRed); + DrawKeyPointLine(frm, keypoints, right_knee, right_hip, kColorRed); + DrawKeyPointLine(frm, keypoints, right_hip, pelvis, kColorRed); + DrawKeyPointLine(frm, keypoints, right_hip, right_shoulder, kColorRed); + DrawKeyPointLine(frm, keypoints, right_shoulder, right_elbow, kColorRed); + DrawKeyPointLine(frm, keypoints, right_elbow, right_wrist, kColorRed); + DrawKeyPointLine(frm, keypoints, right_shoulder, neck, kColorRed); - // right side hand and feet - DrawKeyPointLine(frm, keypoints, right_wrist, right_pinky_knuckle, kColorRed); - DrawKeyPointLine(frm, keypoints, right_wrist, right_middle_tip, kColorRed); - DrawKeyPointLine(frm, keypoints, right_wrist, right_index_knuckle, kColorRed); - DrawKeyPointLine(frm, keypoints, right_wrist, right_thumb_tip, kColorRed); - DrawKeyPointLine(frm, keypoints, right_ankle, right_heel, kColorRed); - DrawKeyPointLine(frm, keypoints, right_ankle, right_big_toe, kColorRed); - DrawKeyPointLine(frm, keypoints, right_big_toe, right_small_toe, kColorRed); + // right side hand and feet + DrawKeyPointLine(frm, keypoints, right_wrist, right_pinky_knuckle, kColorRed); + DrawKeyPointLine(frm, keypoints, right_wrist, right_middle_tip, kColorRed); + DrawKeyPointLine(frm, keypoints, right_wrist, right_index_knuckle, kColorRed); + DrawKeyPointLine(frm, keypoints, right_wrist, right_thumb_tip, kColorRed); + DrawKeyPointLine(frm, keypoints, right_ankle, right_heel, kColorRed); + DrawKeyPointLine(frm, keypoints, right_ankle, right_big_toe, kColorRed); + DrawKeyPointLine(frm, keypoints, right_big_toe, right_small_toe, kColorRed); - //left side - DrawKeyPointLine(frm, keypoints, left_ankle, left_knee, kColorBlue); - DrawKeyPointLine(frm, keypoints, left_knee, left_hip, kColorBlue); - DrawKeyPointLine(frm, keypoints, left_hip, pelvis, kColorBlue); - DrawKeyPointLine(frm, keypoints, left_hip, left_shoulder, kColorBlue); - DrawKeyPointLine(frm, keypoints, left_shoulder, left_elbow, kColorBlue); - DrawKeyPointLine(frm, keypoints, left_elbow, left_wrist, kColorBlue); - DrawKeyPointLine(frm, keypoints, left_shoulder, neck, kColorBlue); + //left side + DrawKeyPointLine(frm, keypoints, left_ankle, left_knee, kColorBlue); + DrawKeyPointLine(frm, keypoints, left_knee, left_hip, kColorBlue); + DrawKeyPointLine(frm, keypoints, left_hip, pelvis, kColorBlue); + DrawKeyPointLine(frm, keypoints, left_hip, left_shoulder, kColorBlue); + DrawKeyPointLine(frm, keypoints, left_shoulder, left_elbow, kColorBlue); + DrawKeyPointLine(frm, keypoints, left_elbow, left_wrist, kColorBlue); + DrawKeyPointLine(frm, keypoints, left_shoulder, neck, kColorBlue); - // left side hand and feet - DrawKeyPointLine(frm, keypoints, left_wrist, left_pinky_knuckle, kColorBlue); - DrawKeyPointLine(frm, keypoints, left_wrist, left_middle_tip, kColorBlue); - DrawKeyPointLine(frm, keypoints, left_wrist, left_index_knuckle, kColorBlue); - DrawKeyPointLine(frm, keypoints, left_wrist, left_thumb_tip, kColorBlue); - DrawKeyPointLine(frm, keypoints, left_ankle, left_heel, kColorBlue); - DrawKeyPointLine(frm, keypoints, left_ankle, left_big_toe, kColorBlue); - DrawKeyPointLine(frm, keypoints, left_big_toe, left_small_toe, kColorBlue); + // left side hand and feet + DrawKeyPointLine(frm, keypoints, left_wrist, left_pinky_knuckle, kColorBlue); + DrawKeyPointLine(frm, keypoints, left_wrist, left_middle_tip, kColorBlue); + DrawKeyPointLine(frm, keypoints, left_wrist, left_index_knuckle, kColorBlue); + DrawKeyPointLine(frm, keypoints, left_wrist, left_thumb_tip, kColorBlue); + DrawKeyPointLine(frm, keypoints, left_ankle, left_heel, kColorBlue); + DrawKeyPointLine(frm, keypoints, left_ankle, left_big_toe, kColorBlue); + DrawKeyPointLine(frm, keypoints, left_big_toe, left_small_toe, kColorBlue); - // head - DrawKeyPointLine(frm, keypoints, neck, nose, kColorGreen); - DrawKeyPointLine(frm, keypoints, nose, right_eye, kColorGreen); - DrawKeyPointLine(frm, keypoints, right_eye, right_ear, kColorGreen); - DrawKeyPointLine(frm, keypoints, nose, left_eye, kColorGreen); - DrawKeyPointLine(frm, keypoints, left_eye, left_ear, kColorGreen); + // head + DrawKeyPointLine(frm, keypoints, neck, nose, kColorGreen); + DrawKeyPointLine(frm, keypoints, nose, right_eye, kColorGreen); + DrawKeyPointLine(frm, keypoints, right_eye, right_ear, kColorGreen); + DrawKeyPointLine(frm, keypoints, nose, left_eye, kColorGreen); + DrawKeyPointLine(frm, keypoints, left_eye, left_ear, kColorGreen); } if (FLAG_offlineMode) keyPointsOutputVideo.write(frm); } #endif void DoApp::DrawKeyPointsAndEdges(const cv::Mat& src, NvAR_Point2f* keypoints, int numKeyPoints, NvAR_BBoxes* output_bbox) { - cv::Mat frm; - if (FLAG_offlineMode) - frm = src.clone(); - else - frm = src; - NvAR_Point2f *pt, *endPt; - NvAR_Point2f* keypointsBatch8 = keypoints; + cv::Mat frm; + if (FLAG_offlineMode) + frm = src.clone(); + else + frm = src; + NvAR_Point2f *pt, *endPt; + NvAR_Point2f* keypointsBatch8 = keypoints; - int pelvis = 0; - int left_hip = 1; - int right_hip = 2; - int torso = 3; - int left_knee = 4; - int right_knee = 5; - int neck = 6; - int left_ankle = 7; - int right_ankle = 8; - int left_big_toe = 9; - int right_big_toe = 10; - int left_small_toe = 11; - int right_small_toe = 12; - int left_heel = 13; - int right_heel = 14; - int nose = 15; - int left_eye = 16; - int right_eye = 17; - int left_ear = 18; - int right_ear = 19; - int left_shoulder = 20; - int right_shoulder = 21; - int left_elbow = 22; - int right_elbow = 23; - int left_wrist = 24; - int right_wrist = 25; - int left_pinky_knuckle = 26; - int right_pinky_knuckle = 27; - int left_middle_tip = 28; - int right_middle_tip = 29; - int left_index_knuckle = 30; - int right_index_knuckle = 31; - int left_thumb_tip = 32; - int right_thumb_tip = 33; + int pelvis = 0; + int left_hip = 1; + int right_hip = 2; + int torso = 3; + int left_knee = 4; + int right_knee = 5; + int neck = 6; + int left_ankle = 7; + int right_ankle = 8; + int left_big_toe = 9; + int right_big_toe = 10; + int left_small_toe = 11; + int right_small_toe = 12; + int left_heel = 13; + int right_heel = 14; + int nose = 15; + int left_eye = 16; + int right_eye = 17; + int left_ear = 18; + int right_ear = 19; + int left_shoulder = 20; + int right_shoulder = 21; + int left_elbow = 22; + int right_elbow = 23; + int left_wrist = 24; + int right_wrist = 25; + int left_pinky_knuckle = 26; + int right_pinky_knuckle = 27; + int left_middle_tip = 28; + int right_middle_tip = 29; + int left_index_knuckle = 30; + int right_index_knuckle = 31; + int left_thumb_tip = 32; + int right_thumb_tip = 33; - for (int i = 0; i < body_ar_engine.output_bboxes.num_boxes; i++) { + for (int i = 0; i < body_ar_engine.output_bboxes.num_boxes; i++) { - keypoints = keypointsBatch8 + (i * 34); + keypoints = keypointsBatch8 + (i * 34); - for (endPt = (pt = (NvAR_Point2f*)keypoints) + numKeyPoints; pt < endPt; ++pt) - cv::circle(frm, cv::Point(lround(pt->x), lround(pt->y)), 4, cv::Scalar(180, 180, 180), -1); + for (endPt = (pt = (NvAR_Point2f*)keypoints) + numKeyPoints; pt < endPt; ++pt) + cv::circle(frm, cv::Point(lround(pt->x), lround(pt->y)), 4, cv::Scalar(180, 180, 180), -1); - if (output_bbox) { - cv::rectangle(frm, cv::Point(lround(output_bbox->boxes[i].x), lround(output_bbox->boxes[i].y)), - cv::Point(lround(output_bbox->boxes[i].x + output_bbox->boxes[i].width), lround(output_bbox->boxes[i].y + output_bbox->boxes[i].height)), - cv::Scalar(255, 0, 0), 2); - } + if (output_bbox) { + cv::rectangle(frm, cv::Point(lround(output_bbox->boxes[i].x), lround(output_bbox->boxes[i].y)), + cv::Point(lround(output_bbox->boxes[i].x + output_bbox->boxes[i].width), lround(output_bbox->boxes[i].y + output_bbox->boxes[i].height)), + cv::Scalar(255, 0, 0), 2); + } - // center body - DrawKeyPointLine(frm, keypoints, pelvis, torso, kColorGreen); - DrawKeyPointLine(frm, keypoints, torso, neck, kColorGreen); - DrawKeyPointLine(frm, keypoints, neck, pelvis, kColorGreen); + // center body + DrawKeyPointLine(frm, keypoints, pelvis, torso, kColorGreen); + DrawKeyPointLine(frm, keypoints, torso, neck, kColorGreen); + DrawKeyPointLine(frm, keypoints, neck, pelvis, kColorGreen); - // right side - DrawKeyPointLine(frm, keypoints, right_ankle, right_knee, kColorRed); - DrawKeyPointLine(frm, keypoints, right_knee, right_hip, kColorRed); - DrawKeyPointLine(frm, keypoints, right_hip, pelvis, kColorRed); - DrawKeyPointLine(frm, keypoints, right_hip, right_shoulder, kColorRed); - DrawKeyPointLine(frm, keypoints, right_shoulder, right_elbow, kColorRed); - DrawKeyPointLine(frm, keypoints, right_elbow, right_wrist, kColorRed); - DrawKeyPointLine(frm, keypoints, right_shoulder, neck, kColorRed); + // right side + DrawKeyPointLine(frm, keypoints, right_ankle, right_knee, kColorRed); + DrawKeyPointLine(frm, keypoints, right_knee, right_hip, kColorRed); + DrawKeyPointLine(frm, keypoints, right_hip, pelvis, kColorRed); + DrawKeyPointLine(frm, keypoints, right_hip, right_shoulder, kColorRed); + DrawKeyPointLine(frm, keypoints, right_shoulder, right_elbow, kColorRed); + DrawKeyPointLine(frm, keypoints, right_elbow, right_wrist, kColorRed); + DrawKeyPointLine(frm, keypoints, right_shoulder, neck, kColorRed); - // right side hand and feet - DrawKeyPointLine(frm, keypoints, right_wrist, right_pinky_knuckle, kColorRed); - DrawKeyPointLine(frm, keypoints, right_wrist, right_middle_tip, kColorRed); - DrawKeyPointLine(frm, keypoints, right_wrist, right_index_knuckle, kColorRed); - DrawKeyPointLine(frm, keypoints, right_wrist, right_thumb_tip, kColorRed); - DrawKeyPointLine(frm, keypoints, right_ankle, right_heel, kColorRed); - DrawKeyPointLine(frm, keypoints, right_ankle, right_big_toe, kColorRed); - DrawKeyPointLine(frm, keypoints, right_big_toe, right_small_toe, kColorRed); + // right side hand and feet + DrawKeyPointLine(frm, keypoints, right_wrist, right_pinky_knuckle, kColorRed); + DrawKeyPointLine(frm, keypoints, right_wrist, right_middle_tip, kColorRed); + DrawKeyPointLine(frm, keypoints, right_wrist, right_index_knuckle, kColorRed); + DrawKeyPointLine(frm, keypoints, right_wrist, right_thumb_tip, kColorRed); + DrawKeyPointLine(frm, keypoints, right_ankle, right_heel, kColorRed); + DrawKeyPointLine(frm, keypoints, right_ankle, right_big_toe, kColorRed); + DrawKeyPointLine(frm, keypoints, right_big_toe, right_small_toe, kColorRed); - //left side - DrawKeyPointLine(frm, keypoints, left_ankle, left_knee, kColorBlue); - DrawKeyPointLine(frm, keypoints, left_knee, left_hip, kColorBlue); - DrawKeyPointLine(frm, keypoints, left_hip, pelvis, kColorBlue); - DrawKeyPointLine(frm, keypoints, left_hip, left_shoulder, kColorBlue); - DrawKeyPointLine(frm, keypoints, left_shoulder, left_elbow, kColorBlue); - DrawKeyPointLine(frm, keypoints, left_elbow, left_wrist, kColorBlue); - DrawKeyPointLine(frm, keypoints, left_shoulder, neck, kColorBlue); + //left side + DrawKeyPointLine(frm, keypoints, left_ankle, left_knee, kColorBlue); + DrawKeyPointLine(frm, keypoints, left_knee, left_hip, kColorBlue); + DrawKeyPointLine(frm, keypoints, left_hip, pelvis, kColorBlue); + DrawKeyPointLine(frm, keypoints, left_hip, left_shoulder, kColorBlue); + DrawKeyPointLine(frm, keypoints, left_shoulder, left_elbow, kColorBlue); + DrawKeyPointLine(frm, keypoints, left_elbow, left_wrist, kColorBlue); + DrawKeyPointLine(frm, keypoints, left_shoulder, neck, kColorBlue); - // left side hand and feet - DrawKeyPointLine(frm, keypoints, left_wrist, left_pinky_knuckle, kColorBlue); - DrawKeyPointLine(frm, keypoints, left_wrist, left_middle_tip, kColorBlue); - DrawKeyPointLine(frm, keypoints, left_wrist, left_index_knuckle, kColorBlue); - DrawKeyPointLine(frm, keypoints, left_wrist, left_thumb_tip, kColorBlue); - DrawKeyPointLine(frm, keypoints, left_ankle, left_heel, kColorBlue); - DrawKeyPointLine(frm, keypoints, left_ankle, left_big_toe, kColorBlue); - DrawKeyPointLine(frm, keypoints, left_big_toe, left_small_toe, kColorBlue); + // left side hand and feet + DrawKeyPointLine(frm, keypoints, left_wrist, left_pinky_knuckle, kColorBlue); + DrawKeyPointLine(frm, keypoints, left_wrist, left_middle_tip, kColorBlue); + DrawKeyPointLine(frm, keypoints, left_wrist, left_index_knuckle, kColorBlue); + DrawKeyPointLine(frm, keypoints, left_wrist, left_thumb_tip, kColorBlue); + DrawKeyPointLine(frm, keypoints, left_ankle, left_heel, kColorBlue); + DrawKeyPointLine(frm, keypoints, left_ankle, left_big_toe, kColorBlue); + DrawKeyPointLine(frm, keypoints, left_big_toe, left_small_toe, kColorBlue); - // head - DrawKeyPointLine(frm, keypoints, neck, nose, kColorGreen); - DrawKeyPointLine(frm, keypoints, nose, right_eye, kColorGreen); - DrawKeyPointLine(frm, keypoints, right_eye, right_ear, kColorGreen); - DrawKeyPointLine(frm, keypoints, nose, left_eye, kColorGreen); - DrawKeyPointLine(frm, keypoints, left_eye, left_ear, kColorGreen); + // head + DrawKeyPointLine(frm, keypoints, neck, nose, kColorGreen); + DrawKeyPointLine(frm, keypoints, nose, right_eye, kColorGreen); + DrawKeyPointLine(frm, keypoints, right_eye, right_ear, kColorGreen); + DrawKeyPointLine(frm, keypoints, nose, left_eye, kColorGreen); + DrawKeyPointLine(frm, keypoints, left_eye, left_ear, kColorGreen); - } + } - if (FLAG_offlineMode) keyPointsOutputVideo.write(frm); + if (FLAG_offlineMode) keyPointsOutputVideo.write(frm); } DoApp::Err DoApp::acquireFrame() { @@ -1016,14 +1016,14 @@ DoApp::Err DoApp::acquireBodyBox() { } if (FLAG_captureOutputs) { #if NV_MULTI_OBJECT_TRACKER - if (FLAG_enablePeopleTracking) { - writeFrameAndEstResults(frame, body_ar_engine.output_tracking_bboxes); - writeVideoAndEstResults(frame, body_ar_engine.output_tracking_bboxes); - } - else { - writeFrameAndEstResults(frame, body_ar_engine.output_bboxes); - writeVideoAndEstResults(frame, body_ar_engine.output_bboxes); - } + if (FLAG_enablePeopleTracking) { + writeFrameAndEstResults(frame, body_ar_engine.output_tracking_bboxes); + writeVideoAndEstResults(frame, body_ar_engine.output_tracking_bboxes); + } + else { + writeFrameAndEstResults(frame, body_ar_engine.output_bboxes); + writeVideoAndEstResults(frame, body_ar_engine.output_bboxes); + } #else writeFrameAndEstResults(frame, body_ar_engine.output_bboxes); writeVideoAndEstResults(frame, body_ar_engine.output_bboxes); @@ -1061,13 +1061,13 @@ DoApp::Err DoApp::acquireBodyBoxAndKeyPoints() { // get keypoints in original image resolution coordinate space #if NV_MULTI_OBJECT_TRACKER if (FLAG_enablePeopleTracking) - n = body_ar_engine.acquireBodyBoxAndKeyPoints(frame, keypoints2D.data(), keypoints3D.data(), - jointAngles.data(), &output_tracking_bbox, 0); + n = body_ar_engine.acquireBodyBoxAndKeyPoints(frame, keypoints2D.data(), keypoints3D.data(), + jointAngles.data(), &output_tracking_bbox, 0); else #endif - n = body_ar_engine.acquireBodyBoxAndKeyPoints(frame, keypoints2D.data(), keypoints3D.data(), - jointAngles.data(), &output_bbox, 0); + n = body_ar_engine.acquireBodyBoxAndKeyPoints(frame, keypoints2D.data(), keypoints3D.data(), + jointAngles.data(), &output_bbox, 0); #ifdef DEBUG_PERF_RUNTIME @@ -1091,14 +1091,14 @@ DoApp::Err DoApp::acquireBodyBoxAndKeyPoints() { } if (FLAG_captureOutputs) { #if NV_MULTI_OBJECT_TRACKER - if (FLAG_enablePeopleTracking) { - writeFrameAndEstResults(frame, body_ar_engine.output_tracking_bboxes, keypoints2D.data()); - writeVideoAndEstResults(frame, body_ar_engine.output_tracking_bboxes, keypoints2D.data()); - } - else { - writeFrameAndEstResults(frame, body_ar_engine.output_bboxes, keypoints2D.data()); - writeVideoAndEstResults(frame, body_ar_engine.output_bboxes, keypoints2D.data()); - } + if (FLAG_enablePeopleTracking) { + writeFrameAndEstResults(frame, body_ar_engine.output_tracking_bboxes, keypoints2D.data()); + writeVideoAndEstResults(frame, body_ar_engine.output_tracking_bboxes, keypoints2D.data()); + } + else { + writeFrameAndEstResults(frame, body_ar_engine.output_bboxes, keypoints2D.data()); + writeVideoAndEstResults(frame, body_ar_engine.output_bboxes, keypoints2D.data()); + } #else writeFrameAndEstResults(frame, body_ar_engine.output_bboxes, keypoints2D.data()); writeVideoAndEstResults(frame, body_ar_engine.output_bboxes, keypoints2D.data()); @@ -1111,14 +1111,14 @@ DoApp::Err DoApp::acquireBodyBoxAndKeyPoints() { if (drawVisualization) { #if NV_MULTI_OBJECT_TRACKER - if (FLAG_enablePeopleTracking) DrawKeyPointsAndEdges(frame, keypoints2D.data(), numKeyPoints, &output_tracking_bbox); - else + if (FLAG_enablePeopleTracking) DrawKeyPointsAndEdges(frame, keypoints2D.data(), numKeyPoints, &output_tracking_bbox); + else #endif DrawKeyPointsAndEdges(frame, keypoints2D.data(), numKeyPoints, &output_bbox); if (FLAG_offlineMode) { #if NV_MULTI_OBJECT_TRACKER - if (FLAG_enablePeopleTracking) DrawBBoxes(frame, &output_tracking_bbox); - else + if (FLAG_enablePeopleTracking) DrawBBoxes(frame, &output_tracking_bbox); + else #endif DrawBBoxes(frame, &output_bbox); } @@ -1296,7 +1296,7 @@ DoApp::Err DoApp::run() { } cv::imshow(windowTitle, frame); } - + if (!FLAG_offlineMode) { diff --git a/samples/BodyTrack/BodyTrack.exe b/samples/BodyTrack/BodyTrack.exe index cf24b32..0021362 100644 Binary files a/samples/BodyTrack/BodyTrack.exe and b/samples/BodyTrack/BodyTrack.exe differ diff --git a/samples/ExpressionApp/BackEndOpenGL/OpenGLMeshRenderer.cpp b/samples/ExpressionApp/BackEndOpenGL/OpenGLMeshRenderer.cpp index eae139e..b719ce2 100644 --- a/samples/ExpressionApp/BackEndOpenGL/OpenGLMeshRenderer.cpp +++ b/samples/ExpressionApp/BackEndOpenGL/OpenGLMeshRenderer.cpp @@ -40,6 +40,7 @@ #include "glm/glm.hpp" #include "glm/gtc/matrix_transform.hpp" #include "glm/gtc/quaternion.hpp" +#include #include "GLMaterial.h" #include "GLMesh.h" #include "GLShaders.h" @@ -305,17 +306,21 @@ public: } void setViewOfBound(const GLMesh::BoundingSphere& bsph, const glm::vec3& lookAt, const glm::vec3& up, float vfov, - float fracFill, float yDir = 1.f) { + float fracFill, float yDir = 1.f, float z_near = 0.0f, float z_far = 0.0f) { float r = bsph.radius() / fracFill, aspect = (float)m_width / (float)m_height, signZ = -yDir, dist; - if (vfov > 0) { // Perspective + if (vfov > 0) { // Perspective dist = r * .5f / tanf(vfov * .5f); - m_P = glm::perspective(vfov, aspect, (dist - r) * 0.2f, (dist + r) * 2.0f); - } - else { // Orthographic + if (z_near == 0.0f && z_far == 0.0f) { + // If z_near and z_far are both 0, use default values + z_near = (dist - r) * 0.2f; + z_far = (dist + r) * 2.0f; + } + m_P = glm::perspective(vfov, aspect, z_near, z_far); + } else { // Orthographic float w = r, h = r; if (aspect < 1.f) h /= aspect; // Wide @@ -329,7 +334,7 @@ public: } void setViewOfBound(const GLMesh::BoundingBox& bbox, const glm::vec3& lookAt, const glm::vec3& up, float vfov, - float fracFill, float yDir = 1.f, float yOff = 0.f) { + float fracFill, float yDir = 1.f, float yOff = 0.f, float z_near = 0.0f, float z_far = 0.0f) { glm::vec3 boxSize = bbox.max() - bbox.min(); glm::vec3 boxCenter = bbox.center(); float borderFrac = ((1.f - fracFill) / fracFill), @@ -341,11 +346,15 @@ public: signZ = -yDir, dist; - if (vfov > 0) { // Perspective + if (vfov > 0) { // Perspective dist = r * .5f / tanf(vfov * .5f); - m_P = glm::perspective(vfov, aspectWind, dist - r, dist + r); - } - else { // Orthographic + if (z_near == 0.0f && z_far == 0.0f) { + // If z_near and z_far are both 0, use default values + z_near = dist - r; + z_far = dist + r; + } + m_P = glm::perspective(vfov, aspectWind, z_near, z_far); + } else { // Orthographic if (aspectGeom > aspectWind) dy *= aspectGeom / aspectWind; else dx *= aspectWind / aspectGeom; dist = r * 2.f; @@ -447,11 +456,11 @@ private: static NvCV_Status info(const char **str); static NvCV_Status read(MeshRenderer *han, const char *modelFile); static NvCV_Status init(MeshRenderer *han, unsigned width, unsigned height, const char *windowName); - static NvCV_Status setCamera(MeshRenderer *han, - const float locPt[3], const float lookVec[3], const float upVec[3], float vfov); + static NvCV_Status setCamera(MeshRenderer *han, const float locPt[3], const float lookVec[3], const float upVec[3], + float vfov, float near_z = 0.0f, float far_z = 0.0f); static NvCV_Status render(MeshRenderer *han, const float exprs[53], const float qrot[4], const float tran[3], NvCVImage *result); - NvCV_Status setFOV(float radians); + NvCV_Status setFOV(float radians, float near_z = 0.0, float far_z = 0.0f); }; NvCV_Status OpenGLMeshRenderer_InitDispatch(MeshRenderer::Dispatch *dispatch) { @@ -575,7 +584,7 @@ NvCV_Status OpenGLMeshRenderer::init(MeshRenderer *han, return NVCV_SUCCESS; } -NvCV_Status OpenGLMeshRenderer::setFOV(float fov) { +NvCV_Status OpenGLMeshRenderer::setFOV(float fov, float near_z, float far_z) { if (0 == _mesh.numVertices()) return NVCV_ERR_MODEL; GLMesh::BoundingBox bbox; @@ -583,18 +592,26 @@ NvCV_Status OpenGLMeshRenderer::setFOV(float fov) { _ctrRot = bbox.center(); _ctrRot.y = bbox.min().y; // Assume that assets are designed with Y-up. float vShift = (_mesh.numVertices() > 10000) ? 0.15f : 0.0f; // Heuristic to determine whether there is a neck - _ctx.setViewOfBound(bbox, glm::vec3(0.f, 0.f, -1.f), glm::vec3(0.f, +1.f, 0.f), fov, .9f, +1, vShift); + _ctx.setViewOfBound(bbox, glm::vec3(0.f, 0.f, -1.f), glm::vec3(0.f, +1.f, 0.f), fov, .7f, +1, vShift, near_z, far_z); return NVCV_SUCCESS; } -NvCV_Status OpenGLMeshRenderer::setCamera(MeshRenderer *han, - const float locPt[3], const float lookVec[3], const float upVec[3], float vfov) { - if (locPt || lookVec || upVec) {} // We don't accommodate these yet - return static_cast(han)->setFOV(vfov); +NvCV_Status OpenGLMeshRenderer::setCamera(MeshRenderer *han, const float locPt[3], const float lookVec[3], + const float upVec[3], float vfov, float near_z, float far_z) { + if (locPt || lookVec || upVec) { + if (!locPt || !lookVec || !upVec || vfov <= 0.0f || (near_z == 0.0f && far_z == 0.0f)) return NVCV_ERR_PARAMETER; + auto &ctx = static_cast(han)->_ctx; + const float aspect = static_cast(ctx.m_width) / static_cast(ctx.m_height); + ctx.m_P = glm::perspective(vfov, aspect, near_z, far_z); + ctx.setViewMatrix(glm::lookAt(glm::make_vec3(locPt), glm::make_vec3(lookVec), glm::make_vec3(upVec))); + return NVCV_SUCCESS; + } + // Default to camera based on bounding box if not enough input arguments are provided + return static_cast(han)->setFOV(vfov, near_z, far_z); } NvCV_Status OpenGLMeshRenderer::render(MeshRenderer *han, - const float exprs[53], const float qrot[4], const float* /*tran*/, NvCVImage *result) { + const float exprs[53], const float qrot[4], const float* trans, NvCVImage *result) { OpenGLMeshRenderer *ren = static_cast(han); NvCV_Status nvErr; glm::mat4x4 M; @@ -606,15 +623,14 @@ NvCV_Status OpenGLMeshRenderer::render(MeshRenderer *han, if (qrot) { q.x = qrot[0]; q.y = qrot[1]; q.z = qrot[2]; q.w = qrot[3]; } else { q.x = 0.0f; q.y = 0.0f; q.z = 0.0f; q.w = 1.0f; } - #ifndef TRANSLATE_POSE + M = glm::mat4_cast(q); + if (trans) { + M = glm::translate(glm::mat4x4(1.f), *((const glm::vec3 *)(trans))) * M; + } else { M = glm::translate(glm::mat4x4(1.f), -ren->_ctrRot); M = glm::mat4_cast(q) * M; M = glm::translate(M, ren->_ctrRot); - #else // TRANSLATE_POSE - M = glm::mat4_cast(q); - if (tran) - M = glm::translate(M, *((const glm::vec3*)(trans))); - #endif // TRANSLATE_POSE + } nvErr = DeformModel(ren->_sfma.fm, nullptr, exprs, &ren->_mesh); glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); diff --git a/samples/ExpressionApp/ExpressionApp.cpp b/samples/ExpressionApp/ExpressionApp.cpp index 367ffde..6100cbe 100644 --- a/samples/ExpressionApp/ExpressionApp.cpp +++ b/samples/ExpressionApp/ExpressionApp.cpp @@ -84,6 +84,7 @@ #define DEFAULT_CODEC "avc1" #define DEFAULT_FACE_MODEL "face_model2.nvf" #define DEFAULT_RENDER_MODEL "face_model2.nvf" +#define NUM_CAMERA_INTRINSIC_PARAMS 3 /******************************************************************************** * Command-line arguments @@ -113,6 +114,8 @@ int //| NVAR_TEMPORAL_FILTER_ENHANCE_EXPRESSIONS, FLAG_viewMode = 0xF, // VIEW_MESH | VIEW_IMAGE | VIEW_PLOT | VIEW_LM FLAG_exprMode = 2, // 1=mesh, 2=MLP + FLAG_poseMode = 0, + FLAG_cheekPuff = 0, FLAG_gaze = 0; double FLAG_fov = 0.0; // Orthographic by default @@ -130,6 +133,8 @@ static void Usage() { " --codec= FOURCC code for the desired codec (default avc1)\n" " --debug[=(true|false)] report debugging info (default false)\n" " --expr_mode= SDK feature used for generation expressions: 1=Face3DReconstruction, 2=FaceExpressions (default 2)\n" + " --pose_mode= Pose mode used for the FaceExpressions feature only: 0:3DOF, 1:6DOF (default 0)\n" + " --cheekpuff[=(true|false)] (Experimental) Enable cheek puff blendshapes (default=false)\n" " --face_model= specify the face model to be used for fitting (default " DEFAULT_FACE_MODEL ")\n" " --filter= 1: face box, 2: landmarks, 4: pose, 16: expressions, 32: gaze,\n" " 256: eye and mouth closure\n" @@ -266,6 +271,8 @@ static int ParseMyArgs(int argc, char **argv) { GetFlagArgVal("codec", arg, &FLAG_codec) || GetFlagArgVal("debug", arg, &FLAG_debug) || GetFlagArgVal("expr_mode", arg, &FLAG_exprMode) || + GetFlagArgVal("pose_mode", arg, &FLAG_poseMode) || + GetFlagArgVal("cheekpuff", arg, &FLAG_cheekPuff) || GetFlagArgVal("face_model", arg, &FLAG_fitModel) || GetFlagArgVal("filter", arg, &FLAG_filter) || GetFlagArgVal("gaze", arg, &FLAG_gaze) || @@ -375,12 +382,14 @@ public: NvCV_Status calibrateExpressionWeights(); NvCV_Status unCalibrateExpressionWeights(); NvCV_Status normalizeExpressionsWeights(); + NvCV_Status updateCamera(); NvCV_Status toggleFaceBoxFiltering(); NvCV_Status toggleLandmarkFiltering(); NvCV_Status togglePoseFiltering(); NvCV_Status toggleExpressionFiltering(); NvCV_Status toggleGazeFiltering(); NvCV_Status toggleClosureEnhancement(); + NvCV_Status togglePoseMode(); NvCV_Status overlayLandmarks(const float landmarks[126 * 2], unsigned screenHeight, NvCVImage *im); void getFPS(); void drawFPS(cv::Mat& img); @@ -405,6 +414,7 @@ public: NvAR_RenderingParams _renderParams; NvCVImage _srcImg, _compImg, _srcGpu, _renderImg; // wrapper, alloced, alloced, alloced Pose _pose; + float _cameraIntrinsicParams[NUM_CAMERA_INTRINSIC_PARAMS]; std::string _inFile, _outFile; std::vector _outputBboxData; NvAR_BBoxes _outputBboxes; @@ -416,12 +426,15 @@ public: unsigned _videoWidth, _videoHeight, _miniWidth, _miniHeight, _renderWidth, _renderHeight, _plotWidth, _plotHeight, _compWidth, _compHeight, _eigenCount, _exprCount, _landmarkCount, _viewMode, _exprMode, _filtering; + unsigned _poseMode = 0; + bool _enableCheekPuff; MeshRendererBroker _broker; MeshRenderer *_renderer = nullptr; static const char _windowTitle[], *_exprAbbr[][4], *_sfmExprAbbr[][4]; MyTimer _timer; bool _showFPS; bool _performCalibration; + bool _cameraNeedsUpdate; double _frameTime; float _globalExpressionParam; #ifdef _ENABLE_UI @@ -735,6 +748,8 @@ NvCV_Status App::initMLPExpressions() { BAIL_IF_ERR(err = NvAR_SetString(_featureHan, NvAR_Parameter_Config(ModelDir), FLAG_modelDir.c_str())); BAIL_IF_ERR(err = NvAR_SetCudaStream(_featureHan, NvAR_Parameter_Config(CUDAStream), _stream)); BAIL_IF_ERR(err = NvAR_SetU32(_featureHan, NvAR_Parameter_Config(Temporal), _filtering)); + BAIL_IF_ERR(err = NvAR_SetU32(_featureHan, NvAR_Parameter_Config(PoseMode), _poseMode)); + BAIL_IF_ERR(err = NvAR_SetU32(_featureHan, NvAR_Parameter_Config(EnableCheekPuff), _enableCheekPuff)); BAIL_IF_ERR(err = NvAR_Load(_featureHan)); _outputBboxData.assign(25, { 0.f, 0.f, 0.f, 0.f }); _outputBboxes.boxes = _outputBboxData.data(); @@ -752,7 +767,15 @@ NvCV_Status App::initMLPExpressions() { _expressionExponent.resize(_exprCount, 1.0f); BAIL_IF_ERR(err = NvAR_SetF32Array(_featureHan, NvAR_Parameter_Output(ExpressionCoefficients), _expressions.data(), _exprCount)); BAIL_IF_ERR(err = NvAR_SetObject(_featureHan, NvAR_Parameter_Input(Image), &_srcGpu, sizeof(NvCVImage))); - BAIL_IF_ERR(err = NvAR_SetObject(_featureHan, NvAR_Parameter_Output(Pose), &_pose, sizeof(NvAR_Quaternion))); + // Heuristic for focal length if it is not known + _cameraIntrinsicParams[0] = static_cast(_srcGpu.height); // focal length + _cameraIntrinsicParams[1] = static_cast(_srcGpu.width) / 2.0f; // cx + _cameraIntrinsicParams[2] = static_cast(_srcGpu.height) / 2.0f; // cy + BAIL_IF_ERR(err = NvAR_SetF32Array(_featureHan, NvAR_Parameter_Input(CameraIntrinsicParams), _cameraIntrinsicParams, + NUM_CAMERA_INTRINSIC_PARAMS)); + BAIL_IF_ERR(err = NvAR_SetObject(_featureHan, NvAR_Parameter_Output(Pose), &_pose.rotation, sizeof(NvAR_Quaternion))); + BAIL_IF_ERR(err = NvAR_SetObject(_featureHan, NvAR_Parameter_Output(PoseTranslation), &_pose.translation, sizeof(NvAR_Vector3f))); + _exprMode = EXPR_MODE_MLP; bail: @@ -792,6 +815,22 @@ NvCV_Status App::normalizeExpressionsWeights() { return NVCV_SUCCESS; } +NvCV_Status App::updateCamera() { + NvCV_Status err; + if (_poseMode == 1) { + const NvAR_Vector3f cam_loc = {0, 0, 0}; + const NvAR_Vector3f cam_dir = {0, 0, -1}; + const NvAR_Vector3f cam_up = {0, 1, 0}; + const float focal_length = _cameraIntrinsicParams[0]; + const float v_fov = focal_length == 0.0f ? 0.0f : 2.0f * atan(_videoHeight / (2 * focal_length)); + err = _renderer->setCamera(&cam_loc.vec[0], &cam_dir.vec[0], &cam_up.vec[0], v_fov, 0.01f, 1000.0f); + } else { + err = _renderer->setCamera(nullptr, nullptr, nullptr, 0.0f); + } + _cameraNeedsUpdate = false; + return err; +} + NvCV_Status App::init() { NvCV_Status err; std::string path; @@ -816,9 +855,15 @@ NvCV_Status App::init() { _performCalibration = false; _frameTime = 0.f; _exprMode = 0; + _poseMode = FLAG_poseMode; + _enableCheekPuff = FLAG_cheekPuff; _featureHan = nullptr; _filtering = FLAG_filter; _globalExpressionParam = 1.0f; + _cameraIntrinsicParams[0] = 0.0f; + _cameraIntrinsicParams[1] = 0.0f; + _cameraIntrinsicParams[2] = 0.0f; + _cameraNeedsUpdate = true; err = _broker.getMeshRendererList(rendererList); if (NVCV_SUCCESS != err) { @@ -855,16 +900,13 @@ NvCV_Status App::init() { printf("renderer init: %s\n", NvCV_GetErrorStringFromCode(err)); return err; } - err = _renderer->setCamera(nullptr, nullptr, nullptr, (float)(FLAG_fov * D_RADIANS_PER_DEGREE)); - if (NVCV_SUCCESS != err) { - printf("renderer setCamera: %s\n", NvCV_GetErrorStringFromCode(err)); - return err; - } BAIL_IF_ERR(err = NvCVImage_Alloc(&_srcGpu, _videoWidth, _videoHeight, NVCV_BGR, NVCV_U8, NVCV_CHUNKY, NVCV_GPU, 1)); + BAIL_IF_ERR(err = NvCVImage_Alloc(&_srcImg, _videoWidth, _videoHeight, NVCV_BGR, NVCV_U8, NVCV_CHUNKY, NVCV_CPU_PINNED, 0)); BAIL_IF_ERR(err = NvCVImage_Alloc(&_compImg, _compWidth, _compHeight, NVCV_BGR, NVCV_U8, NVCV_CHUNKY, NVCV_CPU, 0)); BAIL_IF_ERR(err = NvCVImage_Alloc(&_renderImg, _renderWidth, _renderHeight, NVCV_RGBA, NVCV_U8, NVCV_CHUNKY, NVCV_CPU, 0)); CVWrapperForNvCVImage(&_compImg, &_ocvDstImg); + CVWrapperForNvCVImage(&_srcImg, &_ocvSrcImg); resizeDst(); if (!FLAG_outFile.empty() || !FLAG_show) { @@ -912,7 +954,6 @@ NvCV_Status App::overlayLandmarks(const float landmarks[126 * 2], unsigned scree return NVCV_SUCCESS; } - NvCV_Status App::run() { NvCV_Status err = NVCV_SUCCESS; NvCVImage tmpImg, view; @@ -925,7 +966,6 @@ NvCV_Status App::run() { --frameCount; // Account for the wasted frame continue; // Read the first frame again } - NVWrapperForCVMat(&_ocvSrcImg, &_srcImg); // We probably don't need to do this every frame #ifdef _ENABLE_UI unsigned int exprMode = 0; @@ -986,6 +1026,13 @@ NvCV_Status App::run() { BAIL_IF_ERR(err = NvCVImage_Transfer(&_srcImg, &_srcGpu, 1.f, _stream, nullptr)); BAIL_IF_ERR(err = NvAR_Run(_featureHan)); unsigned isFaceDetected = (_outputBboxes.num_boxes > 0) ? 0xFF : 0; + if (_cameraNeedsUpdate) { + err = updateCamera(); + if (NVCV_SUCCESS != err) { + printf("renderer setCamera: %s\n", NvCV_GetErrorStringFromCode(err)); + return err; + } + } if (_performCalibration) { calibrateExpressionWeights(); } @@ -1003,7 +1050,8 @@ NvCV_Status App::run() { } if (_viewMode & VIEW_MESH) { if (isFaceDetected) { - BAIL_IF_ERR(err = _renderer->render(_expressions.data(), &_pose.rotation.x, nullptr, &_renderImg)); // GL _renderImg is upside down + float* head_translation = _poseMode == 1 ? &_pose.translation.vec[0] : nullptr; + BAIL_IF_ERR(err = _renderer->render(_expressions.data(), &_pose.rotation.x, head_translation, &_renderImg)); // GL _renderImg is upside down NvCVImage_InitView(&view, &_compImg, _renderX, _renderY, _renderWidth, _renderHeight); NvCVImage_FlipY(&view, &view); // Since OpenGL renderImg is upside-down, we copy it to a flipped dst NvCVImage_Transfer(&_renderImg, &view, 1.0f, _stream, nullptr); // VFlip RGBA --> BGR @@ -1047,6 +1095,7 @@ NvCV_Status App::run() { case 'E': case CTL('E'): toggleExpressionFiltering(); break; case 'G': case CTL('G'): toggleGazeFiltering(); break; case 'C': case CTL('C'): toggleClosureEnhancement(); break; + case 'M': case CTL('M'): togglePoseMode(); break; default: if (key < 0) continue; // No key break; // Non-mapped key @@ -1208,6 +1257,13 @@ NvCV_Status App::toggleClosureEnhancement() { return err; } +NvCV_Status App::togglePoseMode() { + _poseMode = !_poseMode; + NvCV_Status err = NvAR_SetU32(_featureHan, NvAR_Parameter_Config(PoseMode), _poseMode); + if (NVCV_SUCCESS == err) err = NvAR_Load(_featureHan); + _cameraNeedsUpdate = true; + return err; +} /******************************************************************************** * main diff --git a/samples/ExpressionApp/ExpressionApp.exe b/samples/ExpressionApp/ExpressionApp.exe index 9ab42f4..1815c00 100644 Binary files a/samples/ExpressionApp/ExpressionApp.exe and b/samples/ExpressionApp/ExpressionApp.exe differ diff --git a/samples/ExpressionApp/MeshRenderer.cpp b/samples/ExpressionApp/MeshRenderer.cpp index 82f8194..d4a3f84 100644 --- a/samples/ExpressionApp/MeshRenderer.cpp +++ b/samples/ExpressionApp/MeshRenderer.cpp @@ -105,8 +105,8 @@ NvCV_Status MeshRenderer::init(unsigned width, unsigned height, const char *wind } -NvCV_Status MeshRenderer::setCamera(const float locPt[3], const float lookVec[3], const float upVec[3], float vfov) { - return m_dispatch.setCamera(this, locPt, lookVec, upVec, vfov); +NvCV_Status MeshRenderer::setCamera(const float locPt[3], const float lookVec[3], const float upVec[3], float vfov, float z_near, float z_far) { + return m_dispatch.setCamera(this, locPt, lookVec, upVec, vfov, z_near, z_far); } diff --git a/samples/ExpressionApp/MeshRenderer.h b/samples/ExpressionApp/MeshRenderer.h index 2f3e308..cc60958 100644 --- a/samples/ExpressionApp/MeshRenderer.h +++ b/samples/ExpressionApp/MeshRenderer.h @@ -40,8 +40,8 @@ public: void (*destroy)(MeshRenderer *han); NvCV_Status (*read)(MeshRenderer *han, const char *modelFile); NvCV_Status (*init)(MeshRenderer *han, unsigned width, unsigned height, const char *windowName); - NvCV_Status (*setCamera)(MeshRenderer *han, - const float locPt[3], const float lookVec[3], const float upVec[3], float vfov); + NvCV_Status (*setCamera)(MeshRenderer *han, const float locPt[3], const float lookVec[3], const float upVec[3], + float vfov, float near_z, float far_z); NvCV_Status (*render)(MeshRenderer *han, const float exprs[53], const float qrot[4], const float trans[3], NvCVImage *result); Dispatch(); @@ -83,8 +83,12 @@ public: /// @param[in] upVec the 3D vector pointing up. This does not need to be normalized. /// NULL implies the default up direction, derived from the rest pose of the model. /// @param[in] vfov the vertical field of view of the camera. Zero implies an orthographic camera. + /// @param[in] near_z Near z clipping plane. Distance along camera's negative z-axis. If both near_z and far_z are + /// 0.0, the z clipping planes are computed based on the mesh bounding box. + /// @param[in] far_z Far z clipping plane. Distance along camera's negative z-axis. If both near_z and far_z are + /// 0.0, the z clipping planes are computed based on the mesh bounding box. /// @return NVCV_SUCCESS if the camera was initialized successfully. - NvCV_Status setCamera(const float locPt[3], const float lookVec[3], const float upVec[3], float vfov); + NvCV_Status setCamera(const float locPt[3], const float lookVec[3], const float upVec[3], float vfov, float near_z = 0.0f, float far_z = 0.0f); /// Render the mesh as deformed by the expression signals. /// @param[in] exprs the expression signals (53 of them). diff --git a/samples/FaceTrack/FaceEngine.cpp b/samples/FaceTrack/FaceEngine.cpp index 876c024..a1f1be4 100644 --- a/samples/FaceTrack/FaceEngine.cpp +++ b/samples/FaceTrack/FaceEngine.cpp @@ -439,10 +439,10 @@ void FaceEngine::releaseFaceFittingIOParams() { } } -unsigned FaceEngine::findFaceBoxes() { +NvCV_Status FaceEngine::findFaceBoxes(unsigned &num_boxes) { NvCV_Status nvErr = NvAR_Run(faceDetectHandle); - if (NVCV_SUCCESS != nvErr) return 0; - return (unsigned)output_bboxes.num_boxes; + num_boxes = (unsigned)output_bboxes.num_boxes; + return nvErr; } NvAR_Rect* FaceEngine::getLargestBox() { @@ -587,16 +587,18 @@ void FaceEngine::DrawPose(const cv::Mat& src, const NvAR_Quaternion* pose) const cv::line(src, p0, pz, CV_RGB(0, 0, 255), thickness); } -NvCV_Status FaceEngine::findLandmarks() { +FaceEngine::Err FaceEngine::findLandmarks() { NvCV_Status nvErr; nvErr = NvAR_Run(landmarkDetectHandle); if (NVCV_SUCCESS != nvErr) { - return nvErr; + return FaceEngine::Err::errRun; } - if (getAverageLandmarksConfidence() < confidenceThreshold) { - return NVCV_ERR_GENERAL; + float avg_confidence = getAverageLandmarksConfidence(); + + if (avg_confidence < confidenceThreshold) { + return FaceEngine::Err::errNoFaceDetected; } else { average_poses(getPose(), batchSize); NvAR_Point2f *pt, *endPt; @@ -611,7 +613,7 @@ NvCV_Status FaceEngine::findLandmarks() { pt->y /= batchSize; } } - return NVCV_SUCCESS; + return FaceEngine::Err::errNone; } NvAR_Point2f* FaceEngine::getLandmarks() { return facial_landmarks.data(); } @@ -644,12 +646,16 @@ float FaceEngine::getAverageLandmarksConfidence() { NvAR_Quaternion* FaceEngine::getPose() { return facial_pose.data(); } -unsigned FaceEngine::findLargestFaceBox(NvAR_Rect& faceBox, int variant) { - unsigned n; +FaceEngine::Err FaceEngine::findLargestFaceBox(NvAR_Rect& faceBox, int variant) { NvAR_Rect* pFaceBox; + unsigned num_boxes; - n = findFaceBoxes(); - if (n >= 1) { + NvCV_Status nv_err = findFaceBoxes(num_boxes); + if (NVCV_SUCCESS != nv_err) { + return FaceEngine::Err::errRun; + } + + if (num_boxes >= 1) { pFaceBox = getLargestBox(); if (nullptr == pFaceBox) { faceBox.x = faceBox.y = faceBox.width = faceBox.height = 0.0f; @@ -658,40 +664,41 @@ unsigned FaceEngine::findLargestFaceBox(NvAR_Rect& faceBox, int variant) { } enlargeAndSquarifyImageBox(.2f, faceBox, variant); } - return n; + else return FaceEngine::Err::errNoFaceDetected; + + return FaceEngine::Err::errNone; } -unsigned FaceEngine::acquireFaceBox(cv::Mat& src, NvAR_Rect& faceBox, int variant) { - unsigned n = 0; +FaceEngine::Err FaceEngine::acquireFaceBox(cv::Mat& src, NvAR_Rect& faceBox, int variant) { NvCVImage fxSrcChunkyCPU; (void)NVWrapperForCVMat(&src, &fxSrcChunkyCPU); NvCV_Status cvErr = NvCVImage_Transfer(&fxSrcChunkyCPU, &inputImageBuffer, 1.0f, stream, &tmpImage); if (NVCV_SUCCESS != cvErr) { - return n; + return FaceEngine::Err::errRun; } - - n = findLargestFaceBox(faceBox, variant); - return n; + FaceEngine::Err nv_err = findLargestFaceBox(faceBox, variant); + + return nv_err; } -unsigned FaceEngine::acquireFaceBoxAndLandmarks(cv::Mat& src, NvAR_Point2f* refMarks, NvAR_Rect& faceBox, int /*variant*/) { - unsigned n = 0; +FaceEngine::Err FaceEngine::acquireFaceBoxAndLandmarks(cv::Mat& src, NvAR_Point2f* refMarks, NvAR_Rect& faceBox, int /*variant*/) { NvCVImage fxSrcChunkyCPU; (void)NVWrapperForCVMat(&src, &fxSrcChunkyCPU); NvCV_Status cvErr = NvCVImage_Transfer(&fxSrcChunkyCPU, &inputImageBuffer, 1.0f, stream, &tmpImage); if (NVCV_SUCCESS != cvErr) { - return n; + return FaceEngine::Err::errRun; } - - if (findLandmarks() != NVCV_SUCCESS) return 0; - faceBox = output_bboxes.boxes[0]; - n = 1; - memcpy(refMarks, getLandmarks(), sizeof(NvAR_Point2f) * numLandmarks); - - return n; + FaceEngine::Err nv_err = findLandmarks(); + + if(nv_err == FaceEngine::Err::errNone){ + faceBox = output_bboxes.boxes[0]; + memcpy(refMarks, getLandmarks(), sizeof(NvAR_Point2f) * numLandmarks); + } + + return nv_err; } void FaceEngine::setFaceStabilization(bool _bStabilizeFace) { bStabilizeFace = _bStabilizeFace; } diff --git a/samples/FaceTrack/FaceEngine.h b/samples/FaceTrack/FaceEngine.h index 27932b9..33b5873 100644 --- a/samples/FaceTrack/FaceEngine.h +++ b/samples/FaceTrack/FaceEngine.h @@ -107,7 +107,7 @@ typedef struct LandmarksProperties { class FaceEngine { public: - enum Err { errNone, errGeneral, errRun, errInitialization, errRead, errEffect, errParameter }; + enum Err { errNone, errGeneral, errRun, errInitialization, errRead, errEffect, errParameter, errNoFaceDetected }; int input_image_width, input_image_height, input_image_pitch; const LandmarksProperties LANDMARKS_INFO[2] = { { 68, 15.0f }, // number of landmark points, confidence threshold value @@ -144,9 +144,9 @@ class FaceEngine { void releaseLandmarkDetectionIOParams(); void releaseFaceFittingIOParams(); - unsigned findFaceBoxes(); + NvCV_Status findFaceBoxes(unsigned &num_boxes); NvAR_Rect* getLargestBox(); - NvCV_Status findLandmarks(); + FaceEngine::Err findLandmarks(); NvAR_BBoxes* getBoundingBoxes(); NvAR_Point2f* getLandmarks(); NvAR_Quaternion* getPose(); @@ -154,9 +154,9 @@ class FaceEngine { float getAverageLandmarksConfidence(); void enlargeAndSquarifyImageBox(float enlarge, NvAR_Rect& box, int FLAG_variant); static void jiggleBox(std::mt19937& ran, float minMag, float maxMag, const NvAR_Rect& cleanBox, NvAR_Rect& noisyBox); - unsigned findLargestFaceBox(NvAR_Rect& faceBox, int variant = 0); - unsigned acquireFaceBox(cv::Mat& src, NvAR_Rect& faceBox, int variant = 0); - unsigned acquireFaceBoxAndLandmarks(cv::Mat& src, NvAR_Point2f* refMarks, NvAR_Rect& faceBox, int variant = 0); + FaceEngine::Err findLargestFaceBox(NvAR_Rect& faceBox, int variant = 0); + FaceEngine::Err acquireFaceBox(cv::Mat& src, NvAR_Rect& faceBox,int variant = 0); + FaceEngine::Err acquireFaceBoxAndLandmarks(cv::Mat& src, NvAR_Point2f* refMarks, NvAR_Rect& faceBox, int variant = 0); Err fitFaceModel(cv::Mat& frame); NvAR_FaceMesh* getFaceMesh(); NvAR_RenderingParams* getRenderingParams(); diff --git a/samples/FaceTrack/FaceTrack.cpp b/samples/FaceTrack/FaceTrack.cpp index c78c87e..57b76c0 100644 --- a/samples/FaceTrack/FaceTrack.cpp +++ b/samples/FaceTrack/FaceTrack.cpp @@ -598,8 +598,10 @@ void DoApp::DrawLandmarkPoints(const cv::Mat &src, NvAR_Point2f *facial_landmark else frm = src; NvAR_Point2f *pt, *endPt; + // Draw larger keypoints if input frame is greater than 720p resolution + int circle_radius = (frm.rows <= 720) ? 1 : 2; for (endPt = (pt = (NvAR_Point2f *)facial_landmarks) + numLandmarks; pt < endPt; ++pt) - cv::circle(frm, cv::Point(lround(pt->x), lround(pt->y)), 1, cv::Scalar(0, 0, 255), -1); + cv::circle(frm, cv::Point(lround(pt->x), lround(pt->y)), circle_radius, cv::Scalar(0, 0, 255), -1); NvAR_Quaternion *pose = face_ar_engine.getPose(); if (pose) face_ar_engine.DrawPose(frm, pose); @@ -664,34 +666,36 @@ DoApp::Err DoApp::acquireFaceBox() { NvAR_Rect output_bbox; // get landmarks in original image resolution coordinate space - unsigned n = face_ar_engine.acquireFaceBox(frame, output_bbox, 0); + nvErr = face_ar_engine.acquireFaceBox(frame, output_bbox, 0); - if (n && FLAG_verbose) { + if (nvErr==FaceEngine::Err::errNone) { + if (FLAG_verbose) { printf("FaceBox: [\n"); printf("%7.1f%7.1f%7.1f%7.1f\n", output_bbox.x, output_bbox.y, output_bbox.x + output_bbox.width, output_bbox.y + output_bbox.height); printf("]\n"); - } - if (FLAG_captureOutputs) { - writeFrameAndEstResults(frame, face_ar_engine.output_bboxes); - writeVideoAndEstResults(frame, face_ar_engine.output_bboxes); - } - + } + + if (FLAG_captureOutputs) { + writeFrameAndEstResults(frame, face_ar_engine.output_bboxes); + writeVideoAndEstResults(frame, face_ar_engine.output_bboxes); + } + #ifdef VISUALIZE - if (n > 0) { // At least one face was found if (drawVisualization) { DrawBBoxes(frame, &output_bbox); // This will write a frame if in offlineMode } +#endif // VISUALIZE } else { // No faces found +#ifdef VISUALIZE if (FLAG_offlineMode) { faceDetectOutputVideo.write(frame); // This will write a frame if in offlineMode } - err = errNoFace; +#endif // !VISUALIZE + if (nvErr==FaceEngine::Err::errNoFaceDetected) err = errNoFace; + else {err = errGeneral;} } -#else // !VISUALIZE - if (0 == n) err = errNoFace; -#endif // VISUALIZE frameIndex++; return err; @@ -704,38 +708,51 @@ DoApp::Err DoApp::acquireFaceBoxAndLandmarks() { std::vector facial_landmarks(numLandmarks); // get landmarks in original image resolution coordinate space - unsigned n = face_ar_engine.acquireFaceBoxAndLandmarks(frame, facial_landmarks.data(), output_bbox, 0); + nvErr = face_ar_engine.acquireFaceBoxAndLandmarks(frame, facial_landmarks.data(), output_bbox, 0); - if (n && FLAG_verbose && face_ar_engine.appMode != FaceEngine::mode::faceDetection) { - printf("Landmarks: [\n"); - for (const auto &pt : facial_landmarks) { - printf("%7.1f%7.1f\n", pt.x, pt.y); + if (nvErr == FaceEngine::Err::errNone) + { + if (FLAG_verbose && face_ar_engine.appMode != FaceEngine::mode::faceDetection) + { + printf("Landmarks: [\n"); + for (const auto &pt : facial_landmarks) + { + printf("%7.1f%7.1f\n", pt.x, pt.y); + } + printf("]\n"); + } + if (FLAG_captureOutputs) + { + writeFrameAndEstResults(frame, face_ar_engine.output_bboxes, facial_landmarks.data()); + writeVideoAndEstResults(frame, face_ar_engine.output_bboxes, facial_landmarks.data()); } - printf("]\n"); - } - if (FLAG_captureOutputs) { - writeFrameAndEstResults(frame, face_ar_engine.output_bboxes, facial_landmarks.data()); - writeVideoAndEstResults(frame, face_ar_engine.output_bboxes, facial_landmarks.data()); - } - #ifdef VISUALIZE - if (n > 0) { // At least one face found - if (drawVisualization) { + if (drawVisualization) + { DrawLandmarkPoints(frame, facial_landmarks.data(), numLandmarks); // Writes frame in offline mode - if (FLAG_offlineMode) { - DrawBBoxes(frame, &output_bbox); // Writes frame in offline mode + if (FLAG_offlineMode) + { + DrawBBoxes(frame, &output_bbox); // Writes frame in offline mode } } +#endif // VISUALIZE } - else { // No faces found - if (FLAG_offlineMode) { - faceDetectOutputVideo.write(frame); // These two wrtite frames if a face was not detected + else + { // No faces found +#ifdef VISUALIZE + if (FLAG_offlineMode) + { + faceDetectOutputVideo.write(frame); // These two write frames if a face was not detected landMarkOutputVideo.write(frame); } +#endif // !VISUALIZE + if (nvErr == FaceEngine::Err::errNoFaceDetected) + err = errNoFace; + else + { + err = errGeneral; + } } -#else // !VISUALIZE - if (0 == n) err = errNoFace; -#endif // VISUALIZE frameIndex++; return err; diff --git a/samples/FaceTrack/FaceTrack.exe b/samples/FaceTrack/FaceTrack.exe index a8e7877..4b26e40 100644 Binary files a/samples/FaceTrack/FaceTrack.exe and b/samples/FaceTrack/FaceTrack.exe differ diff --git a/samples/GazeRedirect/GazeEngine.cpp b/samples/GazeRedirect/GazeEngine.cpp index bbbffe7..9928d82 100644 --- a/samples/GazeRedirect/GazeEngine.cpp +++ b/samples/GazeRedirect/GazeEngine.cpp @@ -91,6 +91,8 @@ GazeEngine::Err GazeEngine::createGazeRedirectionFeature(const char* modelPath, BAIL_IF_NVERR(nvErr, err, GazeEngine::Err::errParameter); nvErr = NvAR_SetCudaStream(gazeRedirectHandle, NvAR_Parameter_Config(CUDAStream), stream); BAIL_IF_NVERR(nvErr, err, GazeEngine::Err::errParameter); + nvErr = NvAR_SetU32(gazeRedirectHandle, NvAR_Parameter_Config(UseCudaGraph), bUseCudaGraph); + BAIL_IF_NVERR(nvErr, err, GazeEngine::Err::errParameter); nvErr = NvAR_SetU32(gazeRedirectHandle, NvAR_Parameter_Config(EyeSizeSensitivity), eyeSizeSensitivity); BAIL_IF_NVERR(nvErr, err, GazeEngine::Err::errParameter); nvErr = NvAR_Load(gazeRedirectHandle); @@ -437,6 +439,11 @@ void GazeEngine::setGazeRedirect(bool _bGazeRedirect) { bGazeRedirect = _bGazeRedirect; } +void GazeEngine::setUseCudaGraph(bool _bUseCudaGraph) { + // If this variable is set, gaze redirection occurs in addition to estimation. + bUseCudaGraph = _bUseCudaGraph; +} + GazeEngine::Err GazeEngine::setNumLandmarks(int n) { GazeEngine::Err err = errNone; for (auto const& info : LANDMARKS_INFO) { diff --git a/samples/GazeRedirect/GazeEngine.h b/samples/GazeRedirect/GazeEngine.h index 6e1cda8..a9614f5 100644 --- a/samples/GazeRedirect/GazeEngine.h +++ b/samples/GazeRedirect/GazeEngine.h @@ -150,6 +150,7 @@ class GazeEngine { void setFaceStabilization(bool); Err setNumLandmarks(int); void setGazeRedirect(bool _bGazeRedirect); + void setUseCudaGraph(bool _bUseCudaGraph); void setEyeSizeSensitivity(unsigned); int getNumLandmarks() { return numLandmarks; } int getNumGazeOutputLandmarks() { return num_output_landmarks; } @@ -184,12 +185,14 @@ class GazeEngine { bool bStabilizeFace; bool bUseOTAU; bool bGazeRedirect; + bool bUseCudaGraph; char *fdOTAModelPath, *ldOTAModelPath; GazeEngine() { batchSize = 1; bStabilizeFace = true; bGazeRedirect = true; + bUseCudaGraph = true; numLandmarks = LANDMARKS_INFO[0].numPoints; num_output_landmarks = 12; confidenceThreshold = LANDMARKS_INFO[0].confidence_threshold; diff --git a/samples/GazeRedirect/GazeRedirect.cpp b/samples/GazeRedirect/GazeRedirect.cpp index 7db6058..89b47e7 100644 --- a/samples/GazeRedirect/GazeRedirect.cpp +++ b/samples/GazeRedirect/GazeRedirect.cpp @@ -75,7 +75,8 @@ bool FLAG_debug = false, FLAG_verbose = false, FLAG_temporal = true, FLAG_captureOutputs = false, FLAG_drawVisualization = true, FLAG_offlineMode = false, FLAG_isNumLandmarks126 = false, - FLAG_splitScreenView = true, FLAG_displayLandmarks = false, FLAG_gazeRedirect=true; + FLAG_splitScreenView = true, FLAG_displayLandmarks = false, FLAG_gazeRedirect = true, FLAG_useCudaGraph = false; +; std::string FLAG_outDir, FLAG_inFile, FLAG_outFile, FLAG_modelPath, FLAG_landmarks, FLAG_captureCodec = "avc1", FLAG_camRes = "480"; unsigned FLAG_camID = 0; @@ -106,6 +107,7 @@ static void Usage() { " --draw_visualization draw the landmarks, display gaze estimation and head rotation, default to true\n" " --redirect_gaze redirection of the eyes in addition to estimating gaze, default to true\n" "(Default)." + " --use_cuda_graph use cuda graph to optimize computations " " --benchmarks[=] run benchmarks\n"); } @@ -211,7 +213,8 @@ static int ParseMyArgs(int argc, char **argv) { GetFlagArgVal("split_screen_view", arg, &FLAG_splitScreenView) || GetFlagArgVal("temporal", arg, &FLAG_temporal) || GetFlagArgVal("draw_visualization", arg, &FLAG_drawVisualization) || - GetFlagArgVal("redirect_gaze", arg, &FLAG_gazeRedirect))) { + GetFlagArgVal("redirect_gaze", arg, &FLAG_gazeRedirect) || + GetFlagArgVal("use_cuda_graph", arg, &FLAG_useCudaGraph))) { continue; } else if (GetFlagArgVal("help", arg, &help)) { Usage(); @@ -312,7 +315,8 @@ class DoApp { ~DoApp(); void stop(); - Err initGazeEngine(const char *modelPath = nullptr, bool isLandmarks126 = false, bool gazeRedirect=true, unsigned eyeSizeSensitivity=3); + Err initGazeEngine(const char *modelPath = nullptr, bool isLandmarks126 = false, bool gazeRedirect = true, + unsigned eyeSizeSensitivity = 3, bool useCudaGraph = false); Err initCamera(const char *camRes = nullptr, unsigned int camID = 0); Err initOfflineMode(const char *inputFilename = nullptr, const char *outputFilename = nullptr); Err acquireFrame(); @@ -383,12 +387,14 @@ void DoApp::processKey(int key) { } } -DoApp::Err DoApp::initGazeEngine(const char *modelPath, bool isNumLandmarks126, bool gazeRedirect, unsigned eyeSizeSensitivity) { +DoApp::Err DoApp::initGazeEngine(const char *modelPath, bool isNumLandmarks126, bool gazeRedirect, + unsigned eyeSizeSensitivity, bool useCudaGraph) { if (!cap.isOpened()) return errVideo; int numLandmarkPoints = isNumLandmarks126 ? 126 : 68; gaze_ar_engine.setNumLandmarks(numLandmarkPoints); gaze_ar_engine.setGazeRedirect(gazeRedirect); + gaze_ar_engine.setUseCudaGraph(useCudaGraph); gaze_ar_engine.setEyeSizeSensitivity(eyeSizeSensitivity); nvErr = gaze_ar_engine.createGazeRedirectionFeature(modelPath); @@ -613,6 +619,15 @@ DoApp::Err DoApp::initCamera(const char *camRes, unsigned int camID) { inputWidth = (int)cap.get(CV_CAP_PROP_FRAME_WIDTH); inputHeight = (int)cap.get(CV_CAP_PROP_FRAME_HEIGHT); + // openCV API(CAP_PROP_FRAME_WIDTH) to get camera resolution is not always reliable with some cameras + cap >> frame; + if (frame.empty()) return errCamera; + if (inputWidth != frame.cols || inputHeight != frame.rows) { + std::cout << "!!! warning: openCV API(CAP_PROP_FRAME_WIDTH/CV_CAP_PROP_FRAME_HEIGHT) to get camera resolution is not trustable. Using the resolution from the actual frame" << std::endl; + inputWidth = frame.cols; + inputHeight = frame.rows; + } + gaze_ar_engine.setInputImageWidth(inputWidth); gaze_ar_engine.setInputImageHeight(inputHeight); } @@ -844,7 +859,7 @@ int main(int argc, char **argv) { } BAIL_IF_ERR(doErr); - doErr = app.initGazeEngine(FLAG_modelPath.c_str(), FLAG_isNumLandmarks126, FLAG_gazeRedirect, FLAG_eyeSizeSensitivity); + doErr = app.initGazeEngine(FLAG_modelPath.c_str(), FLAG_isNumLandmarks126, FLAG_gazeRedirect, FLAG_eyeSizeSensitivity, FLAG_useCudaGraph); BAIL_IF_ERR(doErr); doErr = app.run(); diff --git a/samples/GazeRedirect/GazeRedirect.exe b/samples/GazeRedirect/GazeRedirect.exe index 57fb30b..3c94949 100644 Binary files a/samples/GazeRedirect/GazeRedirect.exe and b/samples/GazeRedirect/GazeRedirect.exe differ diff --git a/samples/external/Imgui/imgui-docking/examples/example_glfw_vulkan/CMakeLists.txt b/samples/external/Imgui/imgui-docking/examples/example_glfw_vulkan/CMakeLists.txt index 05eab3b..e026b62 100644 --- a/samples/external/Imgui/imgui-docking/examples/example_glfw_vulkan/CMakeLists.txt +++ b/samples/external/Imgui/imgui-docking/examples/example_glfw_vulkan/CMakeLists.txt @@ -7,7 +7,7 @@ cmake_minimum_required(VERSION 2.8) project(imgui_example_glfw_vulkan C CXX) if(NOT CMAKE_BUILD_TYPE) - set(CMAKE_BUILD_TYPE Debug CACHE STRING "" FORCE) + set(CMAKE_BUILD_TYPE Release CACHE STRING "" FORCE) endif() set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DVK_PROTOTYPES") diff --git a/version.h b/version.h index 7400d35..14cd8f1 100644 --- a/version.h +++ b/version.h @@ -23,12 +23,12 @@ #define NVIDIA_AR_SDK_VERSION_MAJOR 0 #define NVIDIA_AR_SDK_VERSION_MINOR 8 -#define NVIDIA_AR_SDK_VERSION_RELEASE 1 +#define NVIDIA_AR_SDK_VERSION_RELEASE 2 #define NVIDIA_AR_SDK_VERSION_BUILD 0 -#define NVIDIA_AR_SDK_VERSION 0,8,1,0 +#define NVIDIA_AR_SDK_VERSION 0,8,2,0 #define NVIDIA_AR_SDK_VERSION_MAJOR_MINOR 0,8 -#define NVIDIA_AR_SDK_VERSION_STRING "0.8.1.0" -#define NVIDIA_AR_SDK_VERSION_STRING_SHORT "0.8.1" +#define NVIDIA_AR_SDK_VERSION_STRING "0.8.2.0" +#define NVIDIA_AR_SDK_VERSION_STRING_SHORT "0.8.2" #define NVIDIA_AR_SDK_VERSION_STRING_MAJOR_MINOR "0.8"