demonkenny/MAXINE-AR-SDK
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v0.8.2.0 Release

Browse Source 
v0.8.2.0 Release
This commit is contained in:
Joy D'Souza
2023-01-06 10:03:38 -08:00
parent cf68600c4f
commit ca10ac3b39
23 changed files with 690 additions and 553 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
CHANGELOG
View File

@@ -1,16 +1,15 @@
Changelog (v0.8.1.0)
Changelog (v0.8.2.0)
--------------------
- NEW! Eye Contact feature: an AI algorithm to help users keep their gaze engaged in video communication. The feature jointly estimates a user’s gaze direction and redirects it to frontal in video sequences.
- NEW! Face Expression Estimation (Beta) feature estimates facial expression coefficients From the video or the provided facial landmarks. ExpressionApp is added to demonstrate the new Face Expressions feature.
- NEW! Default face model for the Face 3D mesh and tracking feature, face_model2.nvf, now ships with the SDK. The old SFM based face_model0.nvf is no longer required.
- 3D Body Pose Estimation:
- NEW! Added the support for Multi Person Tracking. This feature is supported by the Windows SDK only.
- FocalLength is now a NvAR_Parameter_Input. Users can now change FocalLength at every NvAR_Run() without having to call NvAR_Load().
- The reference pose returned by the feature has been updated
- Facial landmark estimation
- NEW! There are now 2 modalities for facial landmark tracking: {0,1} -> {performance, quality}. Make sure to choose the preferred mode for your application. The default for face mesh fitting and expression estimation are 1, and the others are 0.
- Head Pose output from the NvAR_Feature_LandmarkDetection feature is now in the OpenGL convention. Changed from X-back(towards the camera), Y-right, Z-down to X-right, Y-up, Z-back(towards the camera).
- The sample apps now show the headpose in the OpenGL convention. The color coding of the axes is Red - X , Green - Y, Blue - Z
- NvCVImage_Transfer() now sets alpha to 255 or 1.0f when doing RGB -> RGBA. NvCVImage_CompositeRect() has a premultiplied alpha mode added
- Migrated to TensorRT 8.4.2.2
- Migrated to CUDA 11.6u1
- Face Expression Estimation
- 6DOF head pose now available.
- Enable by setting NvAR_Parameter_Config(PoseMode) to 1. Default is 0.
- Get the pose translation using NvAR_Parameter_Output(PoseTranslation).
- Camera intrinsics can be set using NvAR_Parameter_Input(CameraIntrinsicParams).
- Expression estimation model updated. The new model is more accurate, and enables subtle expressions such as asymmetric brows, inner/outer brow separation, asymmetric smile and frown, jaw left/right, squint shapes, crossing eye gaze, cheek puff, and more.
- Enable cheek puff shapes by setting NvAR_Parameter_Config(EnableCheekPuff) to 1.
- New face model for visualization recommended for FaceExpressions feature. The model has updated blendshapes, and face area partitioning.
- Enable using --render_model=face_model3.nvf in ExpressionApp.
- Eye Contact
- Performance improvements
- CUDA graphs functionality to enable CUDA optimization now available for Eye contact feature.
- Use the parameter NvAR_Parameter_Config(UseCudaGraph) to enable/disable CUDA graphs. Default is OFF

47
nvar/include/nvAR_defs.h
View File

@@ -117,20 +117,19 @@ typedef const char* NvAR_FeatureID;
#define NvAR_Feature_BodyPoseEstimation "BodyPoseEstimation" //
#define NvAR_Feature_GazeRedirection "GazeRedirection" //
#define NvAR_Feature_FaceExpressions "FaceExpressions" //
#define NvAR_Feature_LivePortrait "LivePortrait" //
#define NvAR_Feature_FrameSelection "FrameSelection" // !FrameSelection!
#define NvAR_Parameter_Input(Name) "NvAR_Parameter_Input_" #Name
#define NvAR_Parameter_Output(Name) "NvAR_Parameter_Output_" #Name
#define NvAR_Parameter_Config(Name) "NvAR_Parameter_Config_" #Name
#define NvAR_Parameter_InOut(Name) "NvAR_Parameter_InOut_" #Name
#define NVAR_TEMPORAL_FILTER_FACE_BOX (1 << 0) // 0x001
#define NVAR_TEMPORAL_FILTER_FACIAL_LANDMARKS (1 << 1) // 0x002
#define NVAR_TEMPORAL_FILTER_FACE_ROTATIONAL_POSE (1 << 2) // 0x004
#define NVAR_TEMPORAL_FILTER_FACIAL_EXPRESSIONS (1 << 4) // 0x010
#define NVAR_TEMPORAL_FILTER_FACIAL_GAZE (1 << 5) // 0x020
#define NVAR_TEMPORAL_FILTER_ENHANCE_EXPRESSIONS (1 << 8) // 0x100
#define NVAR_TEMPORAL_FILTER_FACE_BOX (1U << 0) // 0x001
#define NVAR_TEMPORAL_FILTER_FACIAL_LANDMARKS (1U << 1) // 0x002
#define NVAR_TEMPORAL_FILTER_FACE_ROTATIONAL_POSE (1U << 2) // 0x004
#define NVAR_TEMPORAL_FILTER_FACIAL_EXPRESSIONS (1U << 4) // 0x010
#define NVAR_TEMPORAL_FILTER_FACIAL_GAZE (1U << 5) // 0x020
#define NVAR_TEMPORAL_FILTER_ENHANCE_EXPRESSIONS (1U << 8) // 0x100
/*
Parameters supported by each NvAR_FeatureID
@@ -178,6 +177,7 @@ NvAR_Parameter_Config(CUDAStream) - OPTIONAL //
NvAR_Parameter_Config(Temporal) - OPTIONAL //
NvAR_Parameter_Config(GazeMode) - OPTIONAL //
NvAR_Parameter_Config(ModelName) - OPTIONAL //
NvAR_Parameter_Config(Mode) - OPTIONAL //
NvAR_Parameter_Config(GPU) - OPTIONAL //
NvAR_Parameter_Config(VertexCount) - QUERY //
NvAR_Parameter_Config(TriangleCount) - QUERY //
@@ -286,45 +286,20 @@ NvAR_Parameter_Config(ExpressionCount) - QUERY //
Input: //
NvAR_Parameter_Input(Image) //
NvAR_Parameter_Input(Landmarks) - OPTIONAL //
NvAR_Parameter_Input(PoseMode) - OPTIONAL //
NvAR_Parameter_Input(CameraIntrinsicParams) - OPTIONAL //
//
Output: //
NvAR_Parameter_Output(ExpressionCoefficients) //
NvAR_Parameter_Output(Landmarks) - OPTIONAL //
NvAR_Parameter_Output(LandmarksConfidence) - OPTIONAL //
NvAR_Parameter_Output(Pose) - OPTIONAL //
NvAR_Parameter_Output(PoseTranslation) - OPTIONAL //
NvAR_Parameter_Output(BoundingBoxes) - OPTIONAL //
NvAR_Parameter_Output(BoundingBoxesConfidence) - OPTIONAL //
*******NvAR_Feature_LivePortrait******* //
Config: //
NvAR_Parameter_Config(FeatureDescription) //
NvAR_Parameter_Config(CUDAStream) //
NvAR_Parameter_Config(ModelDir) //
NvAR_Parameter_Config(Temporal) //
NvAR_Parameter_Config(Mode) //
//
Input: //
NvAR_Parameter_Input(SourceImage) //
NvAR_Parameter_Input(DriveImage) //
NvAR_Parameter_Input(BackgroundImage) //
Output: //
NvAR_Parameter_Output(GeneratedImage) //
*******NvAR_Feature_FrameSelection******* // !FrameSelection!
Config: // !FrameSelection!
NvAR_Parameter_Config(FeatureDescription) // !FrameSelection!
NvAR_Parameter_Config(CUDAStream) // !FrameSelection!
NvAR_Parameter_Config(ModelDir) // !FrameSelection!
NvAR_Parameter_Config(Mode) // !FrameSelection!
// !FrameSelection!
Input: // !FrameSelection!
NvAR_Parameter_Input(Image) // !FrameSelection!
// !FrameSelection!
Output: // !FrameSelection!
NvAR_Parameter_Output(FrameSelected) // !FrameSelection!
*/
#endif // NvAR_DEFS_H

48
nvar/include/nvCVImage.h
View File

@@ -538,6 +538,7 @@ NvCV_Status NvCV_API NvCVImage_TransferToYUV(
//! Composite one source image over another using the given matte.
//! This accommodates all RGB and RGBA formats, with u8 and f32 components.
//! If the bg has alpha, then the dst alpha is updated for use in subsequent composition.
//! \param[in] fg the foreground source image.
//! \param[in] bg the background source image.
//! \param[in] mat the matte Yu8 (or Au8) image, indicating where the src should come through.
@@ -547,7 +548,6 @@ NvCV_Status NvCV_API NvCVImage_TransferToYUV(
//! \return NVCV_ERR_PIXELFORMAT if the pixel format is not accommodated.
//! \return NVCV_ERR_MISMATCH if either the fg & bg & dst formats do not match, or if fg & bg & dst & mat are not
//! in the same address space (CPU or GPU).
//! \bug Though RGBA destinations are accommodated, the A channel is not updated at all.
#if RTX_CAMERA_IMAGE == 0
NvCV_Status NvCV_API NvCVImage_Composite(const NvCVImage *fg, const NvCVImage *bg, const NvCVImage *mat, NvCVImage *dst,
struct CUstream_st *stream);
@@ -556,8 +556,11 @@ NvCV_Status NvCV_API NvCVImage_Composite(const NvCVImage *fg, const NvCVImage *b
#endif // RTX_CAMERA_IMAGE == 1
//! Composite one source image over another using the given matte.
//! Composite one source image rectangular region over another using the given matte.
//! This accommodates all RGB and RGBA formats, with u8 and f32 components.
//! If the bg has alpha, then the dst alpha is updated for use in subsequent composition.
//! If the background is not opaque, it is recommended that all images be premultiplied by alpha,
//! and mode 1 composition be used, to yield the most meaningful composite matte.
//! \param[in] fg the foreground source image.
//! \param[in] fgOrg the upper-left corner of the fg image to be composited (NULL implies (0,0)).
//! \param[in] bg the background source image.
@@ -570,14 +573,11 @@ NvCV_Status NvCV_API NvCVImage_Composite(const NvCVImage *fg, const NvCVImage *b
//! \param[in] dstOrg the upper-left corner of the dst image to be updated (NULL implies (0,0)).
//! \param[in] stream the CUDA stream on which the composition is to be performed.
//! \note If a smaller region of a matte is desired, a window can be created using
//! NvCVImage_InitView() for chunky or NvCVImage_Init() for planar pixels.
//! NvCVImage_InitView() for chunky pixels, as illustrated below in NvCVImage_CompositeRectA().
//! \return NVCV_SUCCESS if the operation was successful.
//! \return NVCV_ERR_PIXELFORMAT if the pixel format is not accommodated.
//! \return NVCV_ERR_MISMATCH if either the fg & bg & dst formats do not match, or if fg & bg & dst & mat are not
//! in the same address space (CPU or GPU).
//! \bug Though RGBA destinations are accommodated, the A channel is not updated at all.
//! \todo Accommodate premultiplied alpha, either as a flag in NvCVImage or as a different mode.
//! \todo If the destination has an A channel, update it as per Adobe and Pixar.
NvCV_Status NvCV_API NvCVImage_CompositeRect(
const NvCVImage *fg, const NvCVPoint2i *fgOrg,
const NvCVImage *bg, const NvCVPoint2i *bgOrg,
@@ -586,6 +586,41 @@ NvCV_Status NvCV_API NvCVImage_CompositeRect(
struct CUstream_st *stream);
//! Composite one RGBA or BGRA source image rectangular region over another RGB, BGR, RGBA or BGRA region.
//! This accommodates all RGB and RGBA formats, with u8 and f32 components.
//! If the bg has alpha, then the dst alpha is updated for use in subsequent composition.
//! If the background is not opaque, it is recommended that all images be premultiplied by alpha,
//! and mode 1 composition be used, to yield the most meaningful composite matte.
//! \param[in] fg the foreground RGBA or BGRA source image.
//! \param[in] fgRect a sub-rectangle of the fg image (NULL implies the whole image).
//! \param[in] bg the background source image.
//! \param[in] bgOrg the upper-left corner of the bg image to be composited (NULL implies (0,0)).
//! \param[in] mode the composition mode: 0 (straight alpha over) or 1 (premultiplied alpha over).
//! \param[out] dst the destination image. This can be the same as fg or bg.
//! \param[in] dstOrg the upper-left corner of the dst image to be updated (NULL implies (0,0)).
//! \param[in] stream the CUDA stream on which the composition is to be performed.
//! \return NVCV_SUCCESS if the operation was successful.
//! \return NVCV_ERR_PIXELFORMAT if the pixel format is not accommodated.
//! \return NVCV_ERR_MISMATCH if either the fg & bg & dst formats do not match, or if fg & bg & dst & mat are not
//! in the same address space (CPU or GPU).
//! \bug fgRect will only work for chunky images, not planar.
#ifdef __cplusplus
inline NvCV_Status NvCVImage_CompositeRectA(
const NvCVImage *fg, const NvCVRect2i *fgRect,
const NvCVImage *bg, const NvCVPoint2i *bgOrg,
unsigned mode,
NvCVImage *dst, const NvCVPoint2i *dstOrg,
struct CUstream_st *stream
) {
if (fgRect) {
NvCVImage fgView(const_cast<NvCVImage*>(fg), fgRect->x, fgRect->y, fgRect->width, fgRect->height);
return NvCVImage_CompositeRect(&fgView, nullptr, bg, bgOrg, &fgView, mode, dst, dstOrg, stream);
}
return NvCVImage_CompositeRect(fg, nullptr, bg, bgOrg, fg, mode, dst, dstOrg, stream);
}
#endif // __cplusplus
//! Composite a source image over a constant color field using the given matte.
//! \param[in] src the source image.
//! \param[in] mat the matte image, indicating where the src should come through.
@@ -598,7 +633,6 @@ NvCV_Status NvCV_API NvCVImage_CompositeRect(
//! \return NVCV_ERR_MISMATCH if fg & mat & dst & bgColor are not in the same address space (CPU or GPU).
//! \note The bgColor must remain valid until complete; this is an important consideration especially if
//! the buffers are on the GPU and NvCVImage_CompositeOverConstant() runs asynchronously.
//! \bug Though RGBA destinations are accommodated, the A channel is not updated at all.
NvCV_Status NvCV_API NvCVImage_CompositeOverConstant(
#if RTX_CAMERA_IMAGE == 0
const NvCVImage *src, const NvCVImage *mat, const void *bgColor, NvCVImage *dst, struct CUstream_st *stream

4
nvar/include/nvCVStatus.h
View File

@@ -80,6 +80,10 @@ typedef enum NvCV_Status {
NVCV_ERR_TOOSMALL = -33, //!< A supplied parameter or buffer is not large enough.
NVCV_ERR_TOOBIG = -34, //!< A supplied parameter is too big.
NVCV_ERR_WRONGSIZE = -35, //!< A supplied parameter is not the expected size.
NVCV_ERR_OBJECTNOTFOUND = -36, //!< The specified object was not found.
NVCV_ERR_SINGULAR = -37, //!< A mathematical singularity has been encountered.
NVCV_ERR_NOTHINGRENDERED = -38, //!< Nothing was rendered in the specified region.
NVCV_ERR_CONVERGENCE = -39, //!< An iteration did not converge satisfactorily.
NVCV_ERR_OPENGL = -98, //!< An OpenGL error has occurred.
NVCV_ERR_DIRECT3D = -99, //!< A Direct3D error has occurred.

94
samples/BodyTrack/BodyEngine.cpp
View File

@@ -222,32 +222,32 @@ BodyEngine::Err BodyEngine::initKeyPointDetectionIOParams(NvCVImage* inBuf) {
BAIL_IF_CVERR(nvErr, err, BodyEngine::Err::errParameter);
nvErr = NvAR_SetF32Array(keyPointDetectHandle, NvAR_Parameter_Output(KeyPointsConfidence),
keypoints_confidence.data(), sizeof(float));
keypoints_confidence.data(), sizeof(float));
BAIL_IF_CVERR(nvErr, err, BodyEngine::Err::errParameter);
#if NV_MULTI_OBJECT_TRACKER
if (bEnablePeopleTracking) {
output_tracking_bbox_size = maxTargetsTracked;
output_tracking_bbox_data.assign(output_tracking_bbox_size, { 0.f, 0.f, 0.f, 0.f, 0 });
output_tracking_bboxes.boxes = output_tracking_bbox_data.data();
output_tracking_bboxes.max_boxes = (uint8_t)output_tracking_bbox_size;
output_tracking_bboxes.num_boxes = 0;
nvErr =
NvAR_SetObject(keyPointDetectHandle, NvAR_Parameter_Output(TrackingBoundingBoxes), &output_tracking_bboxes, sizeof(NvAR_TrackingBBoxes));
BAIL_IF_CVERR(nvErr, err, BodyEngine::Err::errParameter);
output_tracking_bbox_size = maxTargetsTracked;
output_tracking_bbox_data.assign(output_tracking_bbox_size, { 0.f, 0.f, 0.f, 0.f, 0 });
output_tracking_bboxes.boxes = output_tracking_bbox_data.data();
output_tracking_bboxes.max_boxes = (uint8_t)output_tracking_bbox_size;
output_tracking_bboxes.num_boxes = 0;
nvErr =
NvAR_SetObject(keyPointDetectHandle, NvAR_Parameter_Output(TrackingBoundingBoxes), &output_tracking_bboxes, sizeof(NvAR_TrackingBBoxes));
BAIL_IF_CVERR(nvErr, err, BodyEngine::Err::errParameter);
}
else {
output_bbox_data.assign(25, { 0.f, 0.f, 0.f, 0.f });
output_bbox_conf_data.assign(25, 0.f);
output_bboxes.boxes = output_bbox_data.data();
output_bboxes.max_boxes = (uint8_t)output_bbox_data.size();
output_bboxes.num_boxes = 0;
nvErr = NvAR_SetObject(keyPointDetectHandle, NvAR_Parameter_Output(BoundingBoxes), &output_bboxes, sizeof(NvAR_BBoxes));
BAIL_IF_CVERR(nvErr, err, BodyEngine::Err::errParameter);
output_bbox_data.assign(25, { 0.f, 0.f, 0.f, 0.f });
output_bbox_conf_data.assign(25, 0.f);
output_bboxes.boxes = output_bbox_data.data();
output_bboxes.max_boxes = (uint8_t)output_bbox_data.size();
output_bboxes.num_boxes = 0;
nvErr = NvAR_SetObject(keyPointDetectHandle, NvAR_Parameter_Output(BoundingBoxes), &output_bboxes, sizeof(NvAR_BBoxes));
BAIL_IF_CVERR(nvErr, err, BodyEngine::Err::errParameter);
nvErr = NvAR_SetF32Array(keyPointDetectHandle, NvAR_Parameter_Output(BoundingBoxesConfidence),
output_bbox_conf_data.data(), output_bboxes.max_boxes);
BAIL_IF_CVERR(nvErr, err, BodyEngine::Err::errParameter);
nvErr = NvAR_SetF32Array(keyPointDetectHandle, NvAR_Parameter_Output(BoundingBoxesConfidence),
output_bbox_conf_data.data(), output_bboxes.max_boxes);
BAIL_IF_CVERR(nvErr, err, BodyEngine::Err::errParameter);
}
#else
output_bbox_data.assign(25, { 0.f, 0.f, 0.f, 0.f });
@@ -407,7 +407,7 @@ NvAR_TrackingBBoxes* BodyEngine::getTrackingBBoxes() { return &output_tracking_b
float average_confidence = 0.0f;
float* keypoints_confidence_all = getKeyPointsConfidence();
for (int i = 0; i < output_bboxes.num_boxes; i++) {
for (unsigned int j = 0; j < numKeyPoints; j++) {
for (unsigned int j = 0; j < numKeyPoints; j++) {
average_confidence += keypoints_confidence_all[i * numKeyPoints + j];
}
}
@@ -483,39 +483,39 @@ unsigned BodyEngine::acquireBodyBoxAndKeyPoints(cv::Mat& src, NvAR_Point2f* refM
}
#if NV_MULTI_OBJECT_TRACKER
unsigned BodyEngine::acquireBodyBoxAndKeyPoints(cv::Mat& src, NvAR_Point2f* refMarks, NvAR_Point3f* refKeyPoints3D,
NvAR_Quaternion* refJointAngles, NvAR_TrackingBBoxes* refBodyBoxes, int /*variant*/) {
unsigned n = 0;
NvCVImage fxSrcChunkyCPU;
(void)NVWrapperForCVMat(&src, &fxSrcChunkyCPU);
NvCV_Status cvErr = NvCVImage_Transfer(&fxSrcChunkyCPU, &inputImageBuffer, 1.0f, stream, &tmpImage);
NvAR_Quaternion* refJointAngles, NvAR_TrackingBBoxes* refBodyBoxes, int /*variant*/) {
unsigned n = 0;
NvCVImage fxSrcChunkyCPU;
(void)NVWrapperForCVMat(&src, &fxSrcChunkyCPU);
NvCV_Status cvErr = NvCVImage_Transfer(&fxSrcChunkyCPU, &inputImageBuffer, 1.0f, stream, &tmpImage);
if (NVCV_SUCCESS != cvErr) {
return n;
}
if (NVCV_SUCCESS != cvErr) {
return n;
}
#ifdef DEBUG_PERF_RUNTIME
auto start = std::chrono::high_resolution_clock::now();
auto start = std::chrono::high_resolution_clock::now();
#endif
if (findKeyPoints() != NVCV_SUCCESS) return 0;
memcpy(refBodyBoxes, getTrackingBBoxes(), sizeof(NvAR_TrackingBBoxes));
n = 1;
if (findKeyPoints() != NVCV_SUCCESS) return 0;
memcpy(refBodyBoxes, getTrackingBBoxes(), sizeof(NvAR_TrackingBBoxes));
n = 1;
#ifdef DEBUG_PERF_RUNTIME
auto start2 = std::chrono::high_resolution_clock::now();
auto start2 = std::chrono::high_resolution_clock::now();
#endif
memcpy(refMarks, getKeyPoints(), sizeof(NvAR_Point2f) * numKeyPoints * batchSize);
memcpy(refKeyPoints3D, getKeyPoints3D(), sizeof(NvAR_Point3f) * numKeyPoints * batchSize);
memcpy(refJointAngles, getJointAngles(), sizeof(NvAR_Quaternion) * numKeyPoints * batchSize);
memcpy(refMarks, getKeyPoints(), sizeof(NvAR_Point2f) * numKeyPoints * batchSize);
memcpy(refKeyPoints3D, getKeyPoints3D(), sizeof(NvAR_Point3f) * numKeyPoints * batchSize);
memcpy(refJointAngles, getJointAngles(), sizeof(NvAR_Quaternion) * numKeyPoints * batchSize);
#ifdef DEBUG_PERF_RUNTIME
auto end = std::chrono::high_resolution_clock::now();
auto duration3 = std::chrono::duration_cast<std::chrono::microseconds>(start2 - start);
std::cout << "[bodypose] run findKeyPoints(): " << duration3.count() << " microseconds" << std::endl;
auto duration2 = std::chrono::duration_cast<std::chrono::microseconds>(end - start2);
std::cout << "[bodypose] keypoint copy time: " << duration2.count() << " microseconds" << std::endl;
auto duration = std::chrono::duration_cast<std::chrono::microseconds>(end - start);
std::cout << "[bodypose] end-to-end time: " << duration.count() << " microseconds" << std::endl;
auto end = std::chrono::high_resolution_clock::now();
auto duration3 = std::chrono::duration_cast<std::chrono::microseconds>(start2 - start);
std::cout << "[bodypose] run findKeyPoints(): " << duration3.count() << " microseconds" << std::endl;
auto duration2 = std::chrono::duration_cast<std::chrono::microseconds>(end - start2);
std::cout << "[bodypose] keypoint copy time: " << duration2.count() << " microseconds" << std::endl;
auto duration = std::chrono::duration_cast<std::chrono::microseconds>(end - start);
std::cout << "[bodypose] end-to-end time: " << duration.count() << " microseconds" << std::endl;
#endif
return n;
return n;
}
#endif
void BodyEngine::setBodyStabilization(bool _bStabilizeBody) { bStabilizeBody = _bStabilizeBody; }
@@ -534,10 +534,10 @@ BodyEngine::Err BodyEngine::setFocalLength(float _bFocalLength) {
void BodyEngine::useCudaGraph(bool _bUseCudaGraph) { bUseCudaGraph = _bUseCudaGraph; }
#if NV_MULTI_OBJECT_TRACKER
void BodyEngine::enablePeopleTracking(bool _bEnablePeopleTracking, unsigned int _shadowTrackingAge, unsigned int _probationAge, unsigned int _maxTargetsTracked) {
bEnablePeopleTracking = _bEnablePeopleTracking;
shadowTrackingAge = _shadowTrackingAge;
bEnablePeopleTracking = _bEnablePeopleTracking;
shadowTrackingAge = _shadowTrackingAge;
probationAge = _probationAge;
maxTargetsTracked = _maxTargetsTracked;
maxTargetsTracked = _maxTargetsTracked;
}
#endif
void BodyEngine::setAppMode(BodyEngine::mode _mode) { appMode = _mode; }

8
samples/BodyTrack/BodyEngine.h
View File

@@ -154,7 +154,7 @@ class BodyEngine {
NvAR_Quaternion* refJointAngles, NvAR_BBoxes* refBodyBoxes, int variant = 0);
#if NV_MULTI_OBJECT_TRACKER
unsigned acquireBodyBoxAndKeyPoints(cv::Mat& src, NvAR_Point2f* refMarks, NvAR_Point3f* refKeyPoints3D,
NvAR_Quaternion* refJointAngles, NvAR_TrackingBBoxes* refBodyBoxes, int variant = 0);
NvAR_Quaternion* refJointAngles, NvAR_TrackingBBoxes* refBodyBoxes, int variant = 0);
#endif
void setBodyStabilization(bool);
void setMode(int);
@@ -206,10 +206,10 @@ class BodyEngine {
bStabilizeBody = true;
bUseCudaGraph = true;
#if NV_MULTI_OBJECT_TRACKER
bEnablePeopleTracking = false;
shadowTrackingAge = 90;
bEnablePeopleTracking = false;
shadowTrackingAge = 90;
probationAge = 10;
maxTargetsTracked = 30;
maxTargetsTracked = 30;
#endif
bFocalLength = FOCAL_LENGTH_DEFAULT;
confidenceThreshold = 0.f;

646
samples/BodyTrack/BodyTrack.cpp
View File

@@ -108,10 +108,10 @@ static void Usage() {
" --mode[=0|1] Model Mode. 0: High Quality, 1: High Performance\n"
" --app_mode[=(0|1)] App mode. 0: Body detection, 1: Keypoint detection "
#if NV_MULTI_OBJECT_TRACKER
" --enable_people_tracking[=(0|1)] Enables people tracking "
" --shadow_tracking_age Shadow Tracking Age after which tracking information of a person is removed. Measured in frames"
" --enable_people_tracking[=(0|1)] Enables people tracking "
" --shadow_tracking_age Shadow Tracking Age after which tracking information of a person is removed. Measured in frames"
" --probation_age Length of probationary period. Measured in frames"
" --max_targets_tracked Maximum number of targets to be tracked "
" --max_targets_tracked Maximum number of targets to be tracked "
#endif
"(Default).\n"
" --benchmarks[=<pattern>] run benchmarks\n");
@@ -213,10 +213,10 @@ static int ParseMyArgs(int argc, char **argv) {
GetFlagArgVal("camindex", arg, &FLAG_camindex) ||
GetFlagArgVal("use_cuda_graph", arg, &FLAG_useCudaGraph) ||
#if NV_MULTI_OBJECT_TRACKER
GetFlagArgVal("enable_people_tracking", arg, &FLAG_enablePeopleTracking) ||
GetFlagArgVal("shadow_tracking_age", arg, &FLAG_shadowTrackingAge) ||
GetFlagArgVal("enable_people_tracking", arg, &FLAG_enablePeopleTracking) ||
GetFlagArgVal("shadow_tracking_age", arg, &FLAG_shadowTrackingAge) ||
GetFlagArgVal("probation_age", arg, &FLAG_probationAge) ||
GetFlagArgVal("max_targets_tracked", arg, &FLAG_maxTargetsTracked) ||
GetFlagArgVal("max_targets_tracked", arg, &FLAG_maxTargetsTracked) ||
#endif
GetFlagArgVal("temporal", arg, &FLAG_temporal))) {
continue;
@@ -389,7 +389,7 @@ void DoApp::processKey(int key) {
if (FLAG_enablePeopleTracking) body_ar_engine.createFeatures(FLAG_modelPath.c_str());
else
#endif
body_ar_engine.createFeatures(FLAG_modelPath.c_str(), 1);
body_ar_engine.createFeatures(FLAG_modelPath.c_str(), 1);
body_ar_engine.initFeatureIOParams();
break;
case '1':
@@ -471,48 +471,48 @@ void DoApp::DrawBBoxes(const cv::Mat &src, NvAR_Rect *output_bbox) {
}
void DoApp::DrawBBoxes(const cv::Mat &src, NvAR_BBoxes *output_bbox) {
cv::Mat frm;
if (FLAG_offlineMode)
frm = src.clone();
else
frm = src;
cv::Mat frm;
if (FLAG_offlineMode)
frm = src.clone();
else
frm = src;
if (output_bbox) {
for (int i = 0; i < output_bbox->num_boxes; i++) {
auto color = cv::Scalar(255, 255, 255);
cv::rectangle(frm, cv::Point(lround(output_bbox->boxes[i].x), lround(output_bbox->boxes[i].y)),
cv::Point(lround(output_bbox->boxes[i].x + output_bbox->boxes[i].width), lround(output_bbox->boxes[i].y + output_bbox->boxes[i].height)),
cv::Scalar(255, 0, 0), 2);
}
}
if (FLAG_offlineMode) bodyDetectOutputVideo.write(frm);
if (output_bbox) {
for (int i = 0; i < output_bbox->num_boxes; i++) {
auto color = cv::Scalar(255, 255, 255);
cv::rectangle(frm, cv::Point(lround(output_bbox->boxes[i].x), lround(output_bbox->boxes[i].y)),
cv::Point(lround(output_bbox->boxes[i].x + output_bbox->boxes[i].width), lround(output_bbox->boxes[i].y + output_bbox->boxes[i].height)),
cv::Scalar(255, 0, 0), 2);
}
}
if (FLAG_offlineMode) bodyDetectOutputVideo.write(frm);
}
#if NV_MULTI_OBJECT_TRACKER
void DoApp::DrawBBoxes(const cv::Mat &src, NvAR_TrackingBBoxes *output_bbox) {
cv::Mat frm;
if (FLAG_offlineMode)
frm = src.clone();
else
frm = src;
cv::Mat frm;
if (FLAG_offlineMode)
frm = src.clone();
else
frm = src;
if (output_bbox) {
for (int i = 0; i < output_bbox->num_boxes; i++) {
if (colorCodes.size() <= output_bbox->boxes[i].tracking_id)
colorCodes.push_back(cv::Scalar(rand() & 0xFF, rand() & 0xFF, rand() & 0xFF));
auto color = colorCodes[output_bbox->boxes[i].tracking_id];
std::string text = "ID: " + std::to_string(output_bbox->boxes[i].tracking_id);
cv::rectangle(frm, cv::Point(lround(output_bbox->boxes[i].bbox.x), lround(output_bbox->boxes[i].bbox.y)),
cv::Point(lround(output_bbox->boxes[i].bbox.x + output_bbox->boxes[i].bbox.width), lround(output_bbox->boxes[i].bbox.y + output_bbox->boxes[i].bbox.height)),
color, 2);
cv::putText(frm, text, cv::Point(lround(output_bbox->boxes[i].bbox.x), lround(output_bbox->boxes[i].bbox.y) - 10), cv::FONT_HERSHEY_SIMPLEX, 0.9, color, 2);
}
if (output_bbox) {
for (int i = 0; i < output_bbox->num_boxes; i++) {
if (colorCodes.size() <= output_bbox->boxes[i].tracking_id)
colorCodes.push_back(cv::Scalar(rand() & 0xFF, rand() & 0xFF, rand() & 0xFF));
auto color = colorCodes[output_bbox->boxes[i].tracking_id];
std::string text = "ID: " + std::to_string(output_bbox->boxes[i].tracking_id);
cv::rectangle(frm, cv::Point(lround(output_bbox->boxes[i].bbox.x), lround(output_bbox->boxes[i].bbox.y)),
cv::Point(lround(output_bbox->boxes[i].bbox.x + output_bbox->boxes[i].bbox.width), lround(output_bbox->boxes[i].bbox.y + output_bbox->boxes[i].bbox.height)),
color, 2);
cv::putText(frm, text, cv::Point(lround(output_bbox->boxes[i].bbox.x), lround(output_bbox->boxes[i].bbox.y) - 10), cv::FONT_HERSHEY_SIMPLEX, 0.9, color, 2);
}
}
}
if (FLAG_offlineMode) bodyDetectOutputVideo.write(frm);
if (FLAG_offlineMode) bodyDetectOutputVideo.write(frm);
}
#endif
void DoApp::writeVideoAndEstResults(const cv::Mat &frm, NvAR_BBoxes output_bboxes, NvAR_Point2f* keypoints) {
@@ -561,50 +561,50 @@ void DoApp::writeVideoAndEstResults(const cv::Mat &frm, NvAR_BBoxes output_bboxe
}
#if NV_MULTI_OBJECT_TRACKER
void DoApp::writeVideoAndEstResults(const cv::Mat &frm, NvAR_TrackingBBoxes output_bboxes, NvAR_Point2f* keypoints) {
if (captureVideo) {
if (!capturedVideo.isOpened()) {
const std::string currentCalendarTime = getCalendarTime();
const std::string capturedOutputFileName = currentCalendarTime + ".mp4";
getFPS();
if (frameTime) {
float fps = (float)(1.0 / frameTime);
capturedVideo.open(capturedOutputFileName, StringToFourcc(FLAG_captureCodec), fps,
cv::Size(frm.cols, frm.rows));
if (!capturedVideo.isOpened()) {
std::cout << "Error: Could not open video: \"" << capturedOutputFileName << "\"\n";
return;
}
if (FLAG_verbose) {
std::cout << "Capturing video started" << std::endl;
}
}
else { // If frameTime is 0.f, returns without writing the frame to the Video
return;
}
const std::string outputsFileName = currentCalendarTime + ".txt";
bodyEngineVideoOutputFile.open(outputsFileName, std::ios_base::out);
if (!bodyEngineVideoOutputFile.is_open()) {
std::cout << "Error: Could not open file: \"" << outputsFileName << "\"\n";
return;
}
std::string keyPointDetectionMode = (keypoints == NULL) ? "Off" : "On";
bodyEngineVideoOutputFile << "// BodyDetectOn, KeyPointDetect" << keyPointDetectionMode << "\n ";
bodyEngineVideoOutputFile
<< "// kNumPeople, (bbox_x, bbox_y, bbox_w, bbox_h){ kNumPeople}, kNumLMs, [lm_x, lm_y]{kNumLMs}\n";
}
// Write each frame to the Video
capturedVideo << frm;
writeEstResults(bodyEngineVideoOutputFile, output_bboxes, keypoints);
}
else {
if (capturedVideo.isOpened()) {
if (FLAG_verbose) {
std::cout << "Capturing video ended" << std::endl;
}
capturedVideo.release();
if (bodyEngineVideoOutputFile.is_open()) bodyEngineVideoOutputFile.close();
}
}
if (captureVideo) {
if (!capturedVideo.isOpened()) {
const std::string currentCalendarTime = getCalendarTime();
const std::string capturedOutputFileName = currentCalendarTime + ".mp4";
getFPS();
if (frameTime) {
float fps = (float)(1.0 / frameTime);
capturedVideo.open(capturedOutputFileName, StringToFourcc(FLAG_captureCodec), fps,
cv::Size(frm.cols, frm.rows));
if (!capturedVideo.isOpened()) {
std::cout << "Error: Could not open video: \"" << capturedOutputFileName << "\"\n";
return;
}
if (FLAG_verbose) {
std::cout << "Capturing video started" << std::endl;
}
}
else { // If frameTime is 0.f, returns without writing the frame to the Video
return;
}
const std::string outputsFileName = currentCalendarTime + ".txt";
bodyEngineVideoOutputFile.open(outputsFileName, std::ios_base::out);
if (!bodyEngineVideoOutputFile.is_open()) {
std::cout << "Error: Could not open file: \"" << outputsFileName << "\"\n";
return;
}
std::string keyPointDetectionMode = (keypoints == NULL) ? "Off" : "On";
bodyEngineVideoOutputFile << "// BodyDetectOn, KeyPointDetect" << keyPointDetectionMode << "\n ";
bodyEngineVideoOutputFile
<< "// kNumPeople, (bbox_x, bbox_y, bbox_w, bbox_h){ kNumPeople}, kNumLMs, [lm_x, lm_y]{kNumLMs}\n";
}
// Write each frame to the Video
capturedVideo << frm;
writeEstResults(bodyEngineVideoOutputFile, output_bboxes, keypoints);
}
else {
if (capturedVideo.isOpened()) {
if (FLAG_verbose) {
std::cout << "Capturing video ended" << std::endl;
}
capturedVideo.release();
if (bodyEngineVideoOutputFile.is_open()) bodyEngineVideoOutputFile.close();
}
}
}
#endif
void DoApp::writeEstResults(std::ofstream &outputFile, NvAR_BBoxes output_bboxes, NvAR_Point2f* keypoints) {
@@ -651,49 +651,49 @@ void DoApp::writeEstResults(std::ofstream &outputFile, NvAR_BBoxes output_bboxes
}
#if NV_MULTI_OBJECT_TRACKER
void DoApp::writeEstResults(std::ofstream &outputFile, NvAR_TrackingBBoxes output_bboxes, NvAR_Point2f* keypoints) {
/**
* Output File Format :
* BodyDetectOn, KeyPointDetectOn
* kNumPeople, (bbox_x, bbox_y, bbox_w, bbox_h){ kNumPeople}, kNumKPs, [j_x, j_y]{kNumKPs}
*/
/**
* Output File Format :
* BodyDetectOn, KeyPointDetectOn
* kNumPeople, (bbox_x, bbox_y, bbox_w, bbox_h){ kNumPeople}, kNumKPs, [j_x, j_y]{kNumKPs}
*/
int bodyDetectOn = (body_ar_engine.appMode == BodyEngine::mode::bodyDetection ||
body_ar_engine.appMode == BodyEngine::mode::keyPointDetection)
? 1
: 0;
int keyPointDetectOn = (body_ar_engine.appMode == BodyEngine::mode::keyPointDetection)
? 1
: 0;
outputFile << bodyDetectOn << "," << keyPointDetectOn << "\n";
int bodyDetectOn = (body_ar_engine.appMode == BodyEngine::mode::bodyDetection ||
body_ar_engine.appMode == BodyEngine::mode::keyPointDetection)
? 1
: 0;
int keyPointDetectOn = (body_ar_engine.appMode == BodyEngine::mode::keyPointDetection)
? 1
: 0;
outputFile << bodyDetectOn << "," << keyPointDetectOn << "\n";
if (bodyDetectOn && output_bboxes.num_boxes) {
// Append number of bodies detected in the current frame
outputFile << unsigned(output_bboxes.num_boxes) << ",";
// write outputbboxes to outputFile
for (size_t i = 0; i < output_bboxes.num_boxes; i++) {
int x1 = (int)output_bboxes.boxes[i].bbox.x, y1 = (int)output_bboxes.boxes[i].bbox.y,
width = (int)output_bboxes.boxes[i].bbox.width, height = (int)output_bboxes.boxes[i].bbox.height;
unsigned int tracking_id = output_bboxes.boxes[i].tracking_id;
outputFile << x1 << "," << y1 << "," << width << "," << height << "," << tracking_id << ",";
}
}
else {
outputFile << "0,";
}
if (keyPointDetectOn && output_bboxes.num_boxes) {
int numKeyPoints = body_ar_engine.getNumKeyPoints();
// Append number of keypoints
outputFile << numKeyPoints << ",";
// Append 2 * number of keypoint values
NvAR_Point2f *pt, *endPt;
for (endPt = (pt = (NvAR_Point2f *)keypoints) + numKeyPoints; pt < endPt; ++pt)
outputFile << pt->x << "," << pt->y << ",";
}
else {
outputFile << "0,";
}
if (bodyDetectOn && output_bboxes.num_boxes) {
// Append number of bodies detected in the current frame
outputFile << unsigned(output_bboxes.num_boxes) << ",";
// write outputbboxes to outputFile
for (size_t i = 0; i < output_bboxes.num_boxes; i++) {
int x1 = (int)output_bboxes.boxes[i].bbox.x, y1 = (int)output_bboxes.boxes[i].bbox.y,
width = (int)output_bboxes.boxes[i].bbox.width, height = (int)output_bboxes.boxes[i].bbox.height;
unsigned int tracking_id = output_bboxes.boxes[i].tracking_id;
outputFile << x1 << "," << y1 << "," << width << "," << height << "," << tracking_id << ",";
}
}
else {
outputFile << "0,";
}
if (keyPointDetectOn && output_bboxes.num_boxes) {
int numKeyPoints = body_ar_engine.getNumKeyPoints();
// Append number of keypoints
outputFile << numKeyPoints << ",";
// Append 2 * number of keypoint values
NvAR_Point2f *pt, *endPt;
for (endPt = (pt = (NvAR_Point2f *)keypoints) + numKeyPoints; pt < endPt; ++pt)
outputFile << pt->x << "," << pt->y << ",";
}
else {
outputFile << "0,";
}
outputFile << "\n";
outputFile << "\n";
}
#endif
void DoApp::writeFrameAndEstResults(const cv::Mat &frm, NvAR_BBoxes output_bboxes, NvAR_Point2f* keypoints) {
@@ -722,28 +722,28 @@ void DoApp::writeFrameAndEstResults(const cv::Mat &frm, NvAR_BBoxes output_bboxe
}
#if NV_MULTI_OBJECT_TRACKER
void DoApp::writeFrameAndEstResults(const cv::Mat &frm, NvAR_TrackingBBoxes output_bboxes, NvAR_Point2f* keypoints) {
if (captureFrame) {
const std::string currentCalendarTime = getCalendarTime();
const std::string capturedFrame = currentCalendarTime + ".png";
cv::imwrite(capturedFrame, frm);
if (FLAG_verbose) {
std::cout << "Captured the frame" << std::endl;
}
// Write Body Engine Outputs
const std::string outputFilename = currentCalendarTime + ".txt";
std::ofstream outputFile;
outputFile.open(outputFilename, std::ios_base::out);
if (!outputFile.is_open()) {
std::cout << "Error: Could not open file: \"" << outputFilename << "\"\n";
return;
}
std::string keyPointDetectionMode = (keypoints == NULL) ? "Off" : "On";
outputFile << "// BodyDetectOn, KeyPointDetect" << keyPointDetectionMode << "\n";
outputFile << "// kNumPeople, (bbox_x, bbox_y, bbox_w, bbox_h){ kNumPeople}, kNumLMs, [lm_x, lm_y]{kNumLMs}\n";
writeEstResults(outputFile, output_bboxes, keypoints);
if (outputFile.is_open()) outputFile.close();
captureFrame = false;
}
if (captureFrame) {
const std::string currentCalendarTime = getCalendarTime();
const std::string capturedFrame = currentCalendarTime + ".png";
cv::imwrite(capturedFrame, frm);
if (FLAG_verbose) {
std::cout << "Captured the frame" << std::endl;
}
// Write Body Engine Outputs
const std::string outputFilename = currentCalendarTime + ".txt";
std::ofstream outputFile;
outputFile.open(outputFilename, std::ios_base::out);
if (!outputFile.is_open()) {
std::cout << "Error: Could not open file: \"" << outputFilename << "\"\n";
return;
}
std::string keyPointDetectionMode = (keypoints == NULL) ? "Off" : "On";
outputFile << "// BodyDetectOn, KeyPointDetect" << keyPointDetectionMode << "\n";
outputFile << "// kNumPeople, (bbox_x, bbox_y, bbox_w, bbox_h){ kNumPeople}, kNumLMs, [lm_x, lm_y]{kNumLMs}\n";
writeEstResults(outputFile, output_bboxes, keypoints);
if (outputFile.is_open()) outputFile.close();
captureFrame = false;
}
}
#endif
void DoApp::DrawKeyPointLine(const cv::Mat& src, NvAR_Point2f* keypoints, int point1, int point2, int color) {
@@ -799,184 +799,184 @@ void DoApp::DrawKeyPointsAndEdges(const cv::Mat& src, NvAR_Point2f* keypoints, i
for (int i = 0; i < body_ar_engine.output_tracking_bboxes.num_boxes; i++) {
keypoints = keypointsBatch8 + (i * 34);
keypoints = keypointsBatch8 + (i * 34);
for (endPt = (pt = (NvAR_Point2f*)keypoints) + numKeyPoints; pt < endPt; ++pt)
cv::circle(frm, cv::Point(lround(pt->x), lround(pt->y)), 4, cv::Scalar(180, 180, 180), -1);
for (endPt = (pt = (NvAR_Point2f*)keypoints) + numKeyPoints; pt < endPt; ++pt)
cv::circle(frm, cv::Point(lround(pt->x), lround(pt->y)), 4, cv::Scalar(180, 180, 180), -1);
if (output_bbox) {
while (colorCodes.size()<= output_bbox->boxes[i].tracking_id)
colorCodes.push_back(cv::Scalar(rand() & 0xFF, rand() & 0xFF, rand() & 0xFF));
auto color = colorCodes[output_bbox->boxes[i].tracking_id];
std::string text = "ID: "+std::to_string(output_bbox->boxes[i].tracking_id);
cv::rectangle(frm, cv::Point(lround(output_bbox->boxes[i].bbox.x), lround(output_bbox->boxes[i].bbox.y)),
cv::Point(lround(output_bbox->boxes[i].bbox.x + output_bbox->boxes[i].bbox.width), lround(output_bbox->boxes[i].bbox.y + output_bbox->boxes[i].bbox.height)),
color, 2);
cv::putText(frm, text, cv::Point(lround(output_bbox->boxes[i].bbox.x), lround(output_bbox->boxes[i].bbox.y) - 10), cv::FONT_HERSHEY_SIMPLEX, 0.5, color, 2);
}
if (output_bbox) {
while (colorCodes.size()<= output_bbox->boxes[i].tracking_id)
colorCodes.push_back(cv::Scalar(rand() & 0xFF, rand() & 0xFF, rand() & 0xFF));
auto color = colorCodes[output_bbox->boxes[i].tracking_id];
std::string text = "ID: "+std::to_string(output_bbox->boxes[i].tracking_id);
cv::rectangle(frm, cv::Point(lround(output_bbox->boxes[i].bbox.x), lround(output_bbox->boxes[i].bbox.y)),
cv::Point(lround(output_bbox->boxes[i].bbox.x + output_bbox->boxes[i].bbox.width), lround(output_bbox->boxes[i].bbox.y + output_bbox->boxes[i].bbox.height)),
color, 2);
cv::putText(frm, text, cv::Point(lround(output_bbox->boxes[i].bbox.x), lround(output_bbox->boxes[i].bbox.y) - 10), cv::FONT_HERSHEY_SIMPLEX, 0.5, color, 2);
}
// center body
DrawKeyPointLine(frm, keypoints, pelvis, torso, kColorGreen);
DrawKeyPointLine(frm, keypoints, torso, neck, kColorGreen);
DrawKeyPointLine(frm, keypoints, neck, pelvis, kColorGreen);
// center body
DrawKeyPointLine(frm, keypoints, pelvis, torso, kColorGreen);
DrawKeyPointLine(frm, keypoints, torso, neck, kColorGreen);
DrawKeyPointLine(frm, keypoints, neck, pelvis, kColorGreen);
// right side
DrawKeyPointLine(frm, keypoints, right_ankle, right_knee, kColorRed);
DrawKeyPointLine(frm, keypoints, right_knee, right_hip, kColorRed);
DrawKeyPointLine(frm, keypoints, right_hip, pelvis, kColorRed);
DrawKeyPointLine(frm, keypoints, right_hip, right_shoulder, kColorRed);
DrawKeyPointLine(frm, keypoints, right_shoulder, right_elbow, kColorRed);
DrawKeyPointLine(frm, keypoints, right_elbow, right_wrist, kColorRed);
DrawKeyPointLine(frm, keypoints, right_shoulder, neck, kColorRed);
// right side
DrawKeyPointLine(frm, keypoints, right_ankle, right_knee, kColorRed);
DrawKeyPointLine(frm, keypoints, right_knee, right_hip, kColorRed);
DrawKeyPointLine(frm, keypoints, right_hip, pelvis, kColorRed);
DrawKeyPointLine(frm, keypoints, right_hip, right_shoulder, kColorRed);
DrawKeyPointLine(frm, keypoints, right_shoulder, right_elbow, kColorRed);
DrawKeyPointLine(frm, keypoints, right_elbow, right_wrist, kColorRed);
DrawKeyPointLine(frm, keypoints, right_shoulder, neck, kColorRed);
// right side hand and feet
DrawKeyPointLine(frm, keypoints, right_wrist, right_pinky_knuckle, kColorRed);
DrawKeyPointLine(frm, keypoints, right_wrist, right_middle_tip, kColorRed);
DrawKeyPointLine(frm, keypoints, right_wrist, right_index_knuckle, kColorRed);
DrawKeyPointLine(frm, keypoints, right_wrist, right_thumb_tip, kColorRed);
DrawKeyPointLine(frm, keypoints, right_ankle, right_heel, kColorRed);
DrawKeyPointLine(frm, keypoints, right_ankle, right_big_toe, kColorRed);
DrawKeyPointLine(frm, keypoints, right_big_toe, right_small_toe, kColorRed);
// right side hand and feet
DrawKeyPointLine(frm, keypoints, right_wrist, right_pinky_knuckle, kColorRed);
DrawKeyPointLine(frm, keypoints, right_wrist, right_middle_tip, kColorRed);
DrawKeyPointLine(frm, keypoints, right_wrist, right_index_knuckle, kColorRed);
DrawKeyPointLine(frm, keypoints, right_wrist, right_thumb_tip, kColorRed);
DrawKeyPointLine(frm, keypoints, right_ankle, right_heel, kColorRed);
DrawKeyPointLine(frm, keypoints, right_ankle, right_big_toe, kColorRed);
DrawKeyPointLine(frm, keypoints, right_big_toe, right_small_toe, kColorRed);
//left side
DrawKeyPointLine(frm, keypoints, left_ankle, left_knee, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_knee, left_hip, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_hip, pelvis, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_hip, left_shoulder, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_shoulder, left_elbow, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_elbow, left_wrist, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_shoulder, neck, kColorBlue);
//left side
DrawKeyPointLine(frm, keypoints, left_ankle, left_knee, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_knee, left_hip, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_hip, pelvis, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_hip, left_shoulder, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_shoulder, left_elbow, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_elbow, left_wrist, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_shoulder, neck, kColorBlue);
// left side hand and feet
DrawKeyPointLine(frm, keypoints, left_wrist, left_pinky_knuckle, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_wrist, left_middle_tip, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_wrist, left_index_knuckle, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_wrist, left_thumb_tip, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_ankle, left_heel, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_ankle, left_big_toe, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_big_toe, left_small_toe, kColorBlue);
// left side hand and feet
DrawKeyPointLine(frm, keypoints, left_wrist, left_pinky_knuckle, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_wrist, left_middle_tip, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_wrist, left_index_knuckle, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_wrist, left_thumb_tip, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_ankle, left_heel, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_ankle, left_big_toe, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_big_toe, left_small_toe, kColorBlue);
// head
DrawKeyPointLine(frm, keypoints, neck, nose, kColorGreen);
DrawKeyPointLine(frm, keypoints, nose, right_eye, kColorGreen);
DrawKeyPointLine(frm, keypoints, right_eye, right_ear, kColorGreen);
DrawKeyPointLine(frm, keypoints, nose, left_eye, kColorGreen);
DrawKeyPointLine(frm, keypoints, left_eye, left_ear, kColorGreen);
// head
DrawKeyPointLine(frm, keypoints, neck, nose, kColorGreen);
DrawKeyPointLine(frm, keypoints, nose, right_eye, kColorGreen);
DrawKeyPointLine(frm, keypoints, right_eye, right_ear, kColorGreen);
DrawKeyPointLine(frm, keypoints, nose, left_eye, kColorGreen);
DrawKeyPointLine(frm, keypoints, left_eye, left_ear, kColorGreen);
}
if (FLAG_offlineMode) keyPointsOutputVideo.write(frm);
}
#endif
void DoApp::DrawKeyPointsAndEdges(const cv::Mat& src, NvAR_Point2f* keypoints, int numKeyPoints, NvAR_BBoxes* output_bbox) {
cv::Mat frm;
if (FLAG_offlineMode)
frm = src.clone();
else
frm = src;
NvAR_Point2f *pt, *endPt;
NvAR_Point2f* keypointsBatch8 = keypoints;
cv::Mat frm;
if (FLAG_offlineMode)
frm = src.clone();
else
frm = src;
NvAR_Point2f *pt, *endPt;
NvAR_Point2f* keypointsBatch8 = keypoints;
int pelvis = 0;
int left_hip = 1;
int right_hip = 2;
int torso = 3;
int left_knee = 4;
int right_knee = 5;
int neck = 6;
int left_ankle = 7;
int right_ankle = 8;
int left_big_toe = 9;
int right_big_toe = 10;
int left_small_toe = 11;
int right_small_toe = 12;
int left_heel = 13;
int right_heel = 14;
int nose = 15;
int left_eye = 16;
int right_eye = 17;
int left_ear = 18;
int right_ear = 19;
int left_shoulder = 20;
int right_shoulder = 21;
int left_elbow = 22;
int right_elbow = 23;
int left_wrist = 24;
int right_wrist = 25;
int left_pinky_knuckle = 26;
int right_pinky_knuckle = 27;
int left_middle_tip = 28;
int right_middle_tip = 29;
int left_index_knuckle = 30;
int right_index_knuckle = 31;
int left_thumb_tip = 32;
int right_thumb_tip = 33;
int pelvis = 0;
int left_hip = 1;
int right_hip = 2;
int torso = 3;
int left_knee = 4;
int right_knee = 5;
int neck = 6;
int left_ankle = 7;
int right_ankle = 8;
int left_big_toe = 9;
int right_big_toe = 10;
int left_small_toe = 11;
int right_small_toe = 12;
int left_heel = 13;
int right_heel = 14;
int nose = 15;
int left_eye = 16;
int right_eye = 17;
int left_ear = 18;
int right_ear = 19;
int left_shoulder = 20;
int right_shoulder = 21;
int left_elbow = 22;
int right_elbow = 23;
int left_wrist = 24;
int right_wrist = 25;
int left_pinky_knuckle = 26;
int right_pinky_knuckle = 27;
int left_middle_tip = 28;
int right_middle_tip = 29;
int left_index_knuckle = 30;
int right_index_knuckle = 31;
int left_thumb_tip = 32;
int right_thumb_tip = 33;
for (int i = 0; i < body_ar_engine.output_bboxes.num_boxes; i++) {
for (int i = 0; i < body_ar_engine.output_bboxes.num_boxes; i++) {
keypoints = keypointsBatch8 + (i * 34);
keypoints = keypointsBatch8 + (i * 34);
for (endPt = (pt = (NvAR_Point2f*)keypoints) + numKeyPoints; pt < endPt; ++pt)
cv::circle(frm, cv::Point(lround(pt->x), lround(pt->y)), 4, cv::Scalar(180, 180, 180), -1);
for (endPt = (pt = (NvAR_Point2f*)keypoints) + numKeyPoints; pt < endPt; ++pt)
cv::circle(frm, cv::Point(lround(pt->x), lround(pt->y)), 4, cv::Scalar(180, 180, 180), -1);
if (output_bbox) {
cv::rectangle(frm, cv::Point(lround(output_bbox->boxes[i].x), lround(output_bbox->boxes[i].y)),
cv::Point(lround(output_bbox->boxes[i].x + output_bbox->boxes[i].width), lround(output_bbox->boxes[i].y + output_bbox->boxes[i].height)),
cv::Scalar(255, 0, 0), 2);
}
if (output_bbox) {
cv::rectangle(frm, cv::Point(lround(output_bbox->boxes[i].x), lround(output_bbox->boxes[i].y)),
cv::Point(lround(output_bbox->boxes[i].x + output_bbox->boxes[i].width), lround(output_bbox->boxes[i].y + output_bbox->boxes[i].height)),
cv::Scalar(255, 0, 0), 2);
}
// center body
DrawKeyPointLine(frm, keypoints, pelvis, torso, kColorGreen);
DrawKeyPointLine(frm, keypoints, torso, neck, kColorGreen);
DrawKeyPointLine(frm, keypoints, neck, pelvis, kColorGreen);
// center body
DrawKeyPointLine(frm, keypoints, pelvis, torso, kColorGreen);
DrawKeyPointLine(frm, keypoints, torso, neck, kColorGreen);
DrawKeyPointLine(frm, keypoints, neck, pelvis, kColorGreen);
// right side
DrawKeyPointLine(frm, keypoints, right_ankle, right_knee, kColorRed);
DrawKeyPointLine(frm, keypoints, right_knee, right_hip, kColorRed);
DrawKeyPointLine(frm, keypoints, right_hip, pelvis, kColorRed);
DrawKeyPointLine(frm, keypoints, right_hip, right_shoulder, kColorRed);
DrawKeyPointLine(frm, keypoints, right_shoulder, right_elbow, kColorRed);
DrawKeyPointLine(frm, keypoints, right_elbow, right_wrist, kColorRed);
DrawKeyPointLine(frm, keypoints, right_shoulder, neck, kColorRed);
// right side
DrawKeyPointLine(frm, keypoints, right_ankle, right_knee, kColorRed);
DrawKeyPointLine(frm, keypoints, right_knee, right_hip, kColorRed);
DrawKeyPointLine(frm, keypoints, right_hip, pelvis, kColorRed);
DrawKeyPointLine(frm, keypoints, right_hip, right_shoulder, kColorRed);
DrawKeyPointLine(frm, keypoints, right_shoulder, right_elbow, kColorRed);
DrawKeyPointLine(frm, keypoints, right_elbow, right_wrist, kColorRed);
DrawKeyPointLine(frm, keypoints, right_shoulder, neck, kColorRed);
// right side hand and feet
DrawKeyPointLine(frm, keypoints, right_wrist, right_pinky_knuckle, kColorRed);
DrawKeyPointLine(frm, keypoints, right_wrist, right_middle_tip, kColorRed);
DrawKeyPointLine(frm, keypoints, right_wrist, right_index_knuckle, kColorRed);
DrawKeyPointLine(frm, keypoints, right_wrist, right_thumb_tip, kColorRed);
DrawKeyPointLine(frm, keypoints, right_ankle, right_heel, kColorRed);
DrawKeyPointLine(frm, keypoints, right_ankle, right_big_toe, kColorRed);
DrawKeyPointLine(frm, keypoints, right_big_toe, right_small_toe, kColorRed);
// right side hand and feet
DrawKeyPointLine(frm, keypoints, right_wrist, right_pinky_knuckle, kColorRed);
DrawKeyPointLine(frm, keypoints, right_wrist, right_middle_tip, kColorRed);
DrawKeyPointLine(frm, keypoints, right_wrist, right_index_knuckle, kColorRed);
DrawKeyPointLine(frm, keypoints, right_wrist, right_thumb_tip, kColorRed);
DrawKeyPointLine(frm, keypoints, right_ankle, right_heel, kColorRed);
DrawKeyPointLine(frm, keypoints, right_ankle, right_big_toe, kColorRed);
DrawKeyPointLine(frm, keypoints, right_big_toe, right_small_toe, kColorRed);
//left side
DrawKeyPointLine(frm, keypoints, left_ankle, left_knee, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_knee, left_hip, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_hip, pelvis, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_hip, left_shoulder, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_shoulder, left_elbow, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_elbow, left_wrist, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_shoulder, neck, kColorBlue);
//left side
DrawKeyPointLine(frm, keypoints, left_ankle, left_knee, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_knee, left_hip, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_hip, pelvis, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_hip, left_shoulder, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_shoulder, left_elbow, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_elbow, left_wrist, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_shoulder, neck, kColorBlue);
// left side hand and feet
DrawKeyPointLine(frm, keypoints, left_wrist, left_pinky_knuckle, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_wrist, left_middle_tip, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_wrist, left_index_knuckle, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_wrist, left_thumb_tip, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_ankle, left_heel, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_ankle, left_big_toe, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_big_toe, left_small_toe, kColorBlue);
// left side hand and feet
DrawKeyPointLine(frm, keypoints, left_wrist, left_pinky_knuckle, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_wrist, left_middle_tip, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_wrist, left_index_knuckle, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_wrist, left_thumb_tip, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_ankle, left_heel, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_ankle, left_big_toe, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_big_toe, left_small_toe, kColorBlue);
// head
DrawKeyPointLine(frm, keypoints, neck, nose, kColorGreen);
DrawKeyPointLine(frm, keypoints, nose, right_eye, kColorGreen);
DrawKeyPointLine(frm, keypoints, right_eye, right_ear, kColorGreen);
DrawKeyPointLine(frm, keypoints, nose, left_eye, kColorGreen);
DrawKeyPointLine(frm, keypoints, left_eye, left_ear, kColorGreen);
// head
DrawKeyPointLine(frm, keypoints, neck, nose, kColorGreen);
DrawKeyPointLine(frm, keypoints, nose, right_eye, kColorGreen);
DrawKeyPointLine(frm, keypoints, right_eye, right_ear, kColorGreen);
DrawKeyPointLine(frm, keypoints, nose, left_eye, kColorGreen);
DrawKeyPointLine(frm, keypoints, left_eye, left_ear, kColorGreen);
}
}
if (FLAG_offlineMode) keyPointsOutputVideo.write(frm);
if (FLAG_offlineMode) keyPointsOutputVideo.write(frm);
}
DoApp::Err DoApp::acquireFrame() {
@@ -1016,14 +1016,14 @@ DoApp::Err DoApp::acquireBodyBox() {
}
if (FLAG_captureOutputs) {
#if NV_MULTI_OBJECT_TRACKER
if (FLAG_enablePeopleTracking) {
writeFrameAndEstResults(frame, body_ar_engine.output_tracking_bboxes);
writeVideoAndEstResults(frame, body_ar_engine.output_tracking_bboxes);
}
else {
writeFrameAndEstResults(frame, body_ar_engine.output_bboxes);
writeVideoAndEstResults(frame, body_ar_engine.output_bboxes);
}
if (FLAG_enablePeopleTracking) {
writeFrameAndEstResults(frame, body_ar_engine.output_tracking_bboxes);
writeVideoAndEstResults(frame, body_ar_engine.output_tracking_bboxes);
}
else {
writeFrameAndEstResults(frame, body_ar_engine.output_bboxes);
writeVideoAndEstResults(frame, body_ar_engine.output_bboxes);
}
#else
writeFrameAndEstResults(frame, body_ar_engine.output_bboxes);
writeVideoAndEstResults(frame, body_ar_engine.output_bboxes);
@@ -1061,13 +1061,13 @@ DoApp::Err DoApp::acquireBodyBoxAndKeyPoints() {
// get keypoints in original image resolution coordinate space
#if NV_MULTI_OBJECT_TRACKER
if (FLAG_enablePeopleTracking)
n = body_ar_engine.acquireBodyBoxAndKeyPoints(frame, keypoints2D.data(), keypoints3D.data(),
jointAngles.data(), &output_tracking_bbox, 0);
n = body_ar_engine.acquireBodyBoxAndKeyPoints(frame, keypoints2D.data(), keypoints3D.data(),
jointAngles.data(), &output_tracking_bbox, 0);
else
#endif
n = body_ar_engine.acquireBodyBoxAndKeyPoints(frame, keypoints2D.data(), keypoints3D.data(),
jointAngles.data(), &output_bbox, 0);
n = body_ar_engine.acquireBodyBoxAndKeyPoints(frame, keypoints2D.data(), keypoints3D.data(),
jointAngles.data(), &output_bbox, 0);
#ifdef DEBUG_PERF_RUNTIME
@@ -1091,14 +1091,14 @@ DoApp::Err DoApp::acquireBodyBoxAndKeyPoints() {
}
if (FLAG_captureOutputs) {
#if NV_MULTI_OBJECT_TRACKER
if (FLAG_enablePeopleTracking) {
writeFrameAndEstResults(frame, body_ar_engine.output_tracking_bboxes, keypoints2D.data());
writeVideoAndEstResults(frame, body_ar_engine.output_tracking_bboxes, keypoints2D.data());
}
else {
writeFrameAndEstResults(frame, body_ar_engine.output_bboxes, keypoints2D.data());
writeVideoAndEstResults(frame, body_ar_engine.output_bboxes, keypoints2D.data());
}
if (FLAG_enablePeopleTracking) {
writeFrameAndEstResults(frame, body_ar_engine.output_tracking_bboxes, keypoints2D.data());
writeVideoAndEstResults(frame, body_ar_engine.output_tracking_bboxes, keypoints2D.data());
}
else {
writeFrameAndEstResults(frame, body_ar_engine.output_bboxes, keypoints2D.data());
writeVideoAndEstResults(frame, body_ar_engine.output_bboxes, keypoints2D.data());
}
#else
writeFrameAndEstResults(frame, body_ar_engine.output_bboxes, keypoints2D.data());
writeVideoAndEstResults(frame, body_ar_engine.output_bboxes, keypoints2D.data());
@@ -1111,14 +1111,14 @@ DoApp::Err DoApp::acquireBodyBoxAndKeyPoints() {
if (drawVisualization) {
#if NV_MULTI_OBJECT_TRACKER
if (FLAG_enablePeopleTracking) DrawKeyPointsAndEdges(frame, keypoints2D.data(), numKeyPoints, &output_tracking_bbox);
else
if (FLAG_enablePeopleTracking) DrawKeyPointsAndEdges(frame, keypoints2D.data(), numKeyPoints, &output_tracking_bbox);
else
#endif
DrawKeyPointsAndEdges(frame, keypoints2D.data(), numKeyPoints, &output_bbox);
if (FLAG_offlineMode) {
#if NV_MULTI_OBJECT_TRACKER
if (FLAG_enablePeopleTracking) DrawBBoxes(frame, &output_tracking_bbox);
else
if (FLAG_enablePeopleTracking) DrawBBoxes(frame, &output_tracking_bbox);
else
#endif
DrawBBoxes(frame, &output_bbox);
}
@@ -1296,7 +1296,7 @@ DoApp::Err DoApp::run() {
}
cv::imshow(windowTitle, frame);
}
if (!FLAG_offlineMode) {

BIN
samples/BodyTrack/BodyTrack.exe
View File

Binary file not shown.

68
samples/ExpressionApp/BackEndOpenGL/OpenGLMeshRenderer.cpp
View File

@@ -40,6 +40,7 @@
#include "glm/glm.hpp"
#include "glm/gtc/matrix_transform.hpp"
#include "glm/gtc/quaternion.hpp"
#include <glm/gtc/type_ptr.hpp>
#include "GLMaterial.h"
#include "GLMesh.h"
#include "GLShaders.h"
@@ -305,17 +306,21 @@ public:
}
void setViewOfBound(const GLMesh::BoundingSphere& bsph, const glm::vec3& lookAt, const glm::vec3& up, float vfov,
float fracFill, float yDir = 1.f) {
float fracFill, float yDir = 1.f, float z_near = 0.0f, float z_far = 0.0f) {
float r = bsph.radius() / fracFill,
aspect = (float)m_width / (float)m_height,
signZ = -yDir,
dist;
if (vfov > 0) { // Perspective
if (vfov > 0) { // Perspective
dist = r * .5f / tanf(vfov * .5f);
m_P = glm::perspective(vfov, aspect, (dist - r) * 0.2f, (dist + r) * 2.0f);
}
else { // Orthographic
if (z_near == 0.0f && z_far == 0.0f) {
// If z_near and z_far are both 0, use default values
z_near = (dist - r) * 0.2f;
z_far = (dist + r) * 2.0f;
}
m_P = glm::perspective(vfov, aspect, z_near, z_far);
} else { // Orthographic
float w = r,
h = r;
if (aspect < 1.f) h /= aspect; // Wide
@@ -329,7 +334,7 @@ public:
}
void setViewOfBound(const GLMesh::BoundingBox& bbox, const glm::vec3& lookAt, const glm::vec3& up, float vfov,
float fracFill, float yDir = 1.f, float yOff = 0.f) {
float fracFill, float yDir = 1.f, float yOff = 0.f, float z_near = 0.0f, float z_far = 0.0f) {
glm::vec3 boxSize = bbox.max() - bbox.min();
glm::vec3 boxCenter = bbox.center();
float borderFrac = ((1.f - fracFill) / fracFill),
@@ -341,11 +346,15 @@ public:
signZ = -yDir,
dist;
if (vfov > 0) { // Perspective
if (vfov > 0) { // Perspective
dist = r * .5f / tanf(vfov * .5f);
m_P = glm::perspective(vfov, aspectWind, dist - r, dist + r);
}
else { // Orthographic
if (z_near == 0.0f && z_far == 0.0f) {
// If z_near and z_far are both 0, use default values
z_near = dist - r;
z_far = dist + r;
}
m_P = glm::perspective(vfov, aspectWind, z_near, z_far);
} else { // Orthographic
if (aspectGeom > aspectWind) dy *= aspectGeom / aspectWind;
else dx *= aspectWind / aspectGeom;
dist = r * 2.f;
@@ -447,11 +456,11 @@ private:
static NvCV_Status info(const char **str);
static NvCV_Status read(MeshRenderer *han, const char *modelFile);
static NvCV_Status init(MeshRenderer *han, unsigned width, unsigned height, const char *windowName);
static NvCV_Status setCamera(MeshRenderer *han,
const float locPt[3], const float lookVec[3], const float upVec[3], float vfov);
static NvCV_Status setCamera(MeshRenderer *han, const float locPt[3], const float lookVec[3], const float upVec[3],
float vfov, float near_z = 0.0f, float far_z = 0.0f);
static NvCV_Status render(MeshRenderer *han,
const float exprs[53], const float qrot[4], const float tran[3], NvCVImage *result);
NvCV_Status setFOV(float radians);
NvCV_Status setFOV(float radians, float near_z = 0.0, float far_z = 0.0f);
};
NvCV_Status OpenGLMeshRenderer_InitDispatch(MeshRenderer::Dispatch *dispatch) {
@@ -575,7 +584,7 @@ NvCV_Status OpenGLMeshRenderer::init(MeshRenderer *han,
return NVCV_SUCCESS;
}
NvCV_Status OpenGLMeshRenderer::setFOV(float fov) {
NvCV_Status OpenGLMeshRenderer::setFOV(float fov, float near_z, float far_z) {
if (0 == _mesh.numVertices())
return NVCV_ERR_MODEL;
GLMesh::BoundingBox bbox;
@@ -583,18 +592,26 @@ NvCV_Status OpenGLMeshRenderer::setFOV(float fov) {
_ctrRot = bbox.center();
_ctrRot.y = bbox.min().y; // Assume that assets are designed with Y-up.
float vShift = (_mesh.numVertices() > 10000) ? 0.15f : 0.0f; // Heuristic to determine whether there is a neck
_ctx.setViewOfBound(bbox, glm::vec3(0.f, 0.f, -1.f), glm::vec3(0.f, +1.f, 0.f), fov, .9f, +1, vShift);
_ctx.setViewOfBound(bbox, glm::vec3(0.f, 0.f, -1.f), glm::vec3(0.f, +1.f, 0.f), fov, .7f, +1, vShift, near_z, far_z);
return NVCV_SUCCESS;
}
NvCV_Status OpenGLMeshRenderer::setCamera(MeshRenderer *han,
const float locPt[3], const float lookVec[3], const float upVec[3], float vfov) {
if (locPt || lookVec || upVec) {} // We don't accommodate these yet
return static_cast<OpenGLMeshRenderer*>(han)->setFOV(vfov);
NvCV_Status OpenGLMeshRenderer::setCamera(MeshRenderer *han, const float locPt[3], const float lookVec[3],
const float upVec[3], float vfov, float near_z, float far_z) {
if (locPt || lookVec || upVec) {
if (!locPt || !lookVec || !upVec || vfov <= 0.0f || (near_z == 0.0f && far_z == 0.0f)) return NVCV_ERR_PARAMETER;
auto &ctx = static_cast<OpenGLMeshRenderer *>(han)->_ctx;
const float aspect = static_cast<float>(ctx.m_width) / static_cast<float>(ctx.m_height);
ctx.m_P = glm::perspective(vfov, aspect, near_z, far_z);
ctx.setViewMatrix(glm::lookAt(glm::make_vec3(locPt), glm::make_vec3(lookVec), glm::make_vec3(upVec)));
return NVCV_SUCCESS;
}
// Default to camera based on bounding box if not enough input arguments are provided
return static_cast<OpenGLMeshRenderer *>(han)->setFOV(vfov, near_z, far_z);
}
NvCV_Status OpenGLMeshRenderer::render(MeshRenderer *han,
const float exprs[53], const float qrot[4], const float* /*tran*/, NvCVImage *result) {
const float exprs[53], const float qrot[4], const float* trans, NvCVImage *result) {
OpenGLMeshRenderer *ren = static_cast<OpenGLMeshRenderer*>(han);
NvCV_Status nvErr;
glm::mat4x4 M;
@@ -606,15 +623,14 @@ NvCV_Status OpenGLMeshRenderer::render(MeshRenderer *han,
if (qrot) { q.x = qrot[0]; q.y = qrot[1]; q.z = qrot[2]; q.w = qrot[3]; }
else { q.x = 0.0f; q.y = 0.0f; q.z = 0.0f; q.w = 1.0f; }
#ifndef TRANSLATE_POSE
M = glm::mat4_cast(q);
if (trans) {
M = glm::translate(glm::mat4x4(1.f), *((const glm::vec3 *)(trans))) * M;
} else {
M = glm::translate(glm::mat4x4(1.f), -ren->_ctrRot);
M = glm::mat4_cast(q) * M;
M = glm::translate(M, ren->_ctrRot);
#else // TRANSLATE_POSE
M = glm::mat4_cast(q);
if (tran)
M = glm::translate(M, *((const glm::vec3*)(trans)));
#endif // TRANSLATE_POSE
}
nvErr = DeformModel(ren->_sfma.fm, nullptr, exprs, &ren->_mesh);
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);

74
samples/ExpressionApp/ExpressionApp.cpp
View File

@@ -84,6 +84,7 @@
#define DEFAULT_CODEC "avc1"
#define DEFAULT_FACE_MODEL "face_model2.nvf"
#define DEFAULT_RENDER_MODEL "face_model2.nvf"
#define NUM_CAMERA_INTRINSIC_PARAMS 3
/********************************************************************************
* Command-line arguments
@@ -113,6 +114,8 @@ int
//| NVAR_TEMPORAL_FILTER_ENHANCE_EXPRESSIONS,
FLAG_viewMode = 0xF, // VIEW_MESH | VIEW_IMAGE | VIEW_PLOT | VIEW_LM
FLAG_exprMode = 2, // 1=mesh, 2=MLP
FLAG_poseMode = 0,
FLAG_cheekPuff = 0,
FLAG_gaze = 0;
double
FLAG_fov = 0.0; // Orthographic by default
@@ -130,6 +133,8 @@ static void Usage() {
" --codec=<fourcc> FOURCC code for the desired codec (default avc1)\n"
" --debug[=(true|false)] report debugging info (default false)\n"
" --expr_mode=<number> SDK feature used for generation expressions: 1=Face3DReconstruction, 2=FaceExpressions (default 2)\n"
" --pose_mode=<number> Pose mode used for the FaceExpressions feature only: 0:3DOF, 1:6DOF (default 0)\n"
" --cheekpuff[=(true|false)] (Experimental) Enable cheek puff blendshapes (default=false)\n"
" --face_model=<file> specify the face model to be used for fitting (default " DEFAULT_FACE_MODEL ")\n"
" --filter=<bitfield> 1: face box, 2: landmarks, 4: pose, 16: expressions, 32: gaze,\n"
" 256: eye and mouth closure\n"
@@ -266,6 +271,8 @@ static int ParseMyArgs(int argc, char **argv) {
GetFlagArgVal("codec", arg, &FLAG_codec) ||
GetFlagArgVal("debug", arg, &FLAG_debug) ||
GetFlagArgVal("expr_mode", arg, &FLAG_exprMode) ||
GetFlagArgVal("pose_mode", arg, &FLAG_poseMode) ||
GetFlagArgVal("cheekpuff", arg, &FLAG_cheekPuff) ||
GetFlagArgVal("face_model", arg, &FLAG_fitModel) ||
GetFlagArgVal("filter", arg, &FLAG_filter) ||
GetFlagArgVal("gaze", arg, &FLAG_gaze) ||
@@ -375,12 +382,14 @@ public:
NvCV_Status calibrateExpressionWeights();
NvCV_Status unCalibrateExpressionWeights();
NvCV_Status normalizeExpressionsWeights();
NvCV_Status updateCamera();
NvCV_Status toggleFaceBoxFiltering();
NvCV_Status toggleLandmarkFiltering();
NvCV_Status togglePoseFiltering();
NvCV_Status toggleExpressionFiltering();
NvCV_Status toggleGazeFiltering();
NvCV_Status toggleClosureEnhancement();
NvCV_Status togglePoseMode();
NvCV_Status overlayLandmarks(const float landmarks[126 * 2], unsigned screenHeight, NvCVImage *im);
void getFPS();
void drawFPS(cv::Mat& img);
@@ -405,6 +414,7 @@ public:
NvAR_RenderingParams _renderParams;
NvCVImage _srcImg, _compImg, _srcGpu, _renderImg; // wrapper, alloced, alloced, alloced
Pose _pose;
float _cameraIntrinsicParams[NUM_CAMERA_INTRINSIC_PARAMS];
std::string _inFile, _outFile;
std::vector<NvAR_Rect> _outputBboxData;
NvAR_BBoxes _outputBboxes;
@@ -416,12 +426,15 @@ public:
unsigned _videoWidth, _videoHeight, _miniWidth, _miniHeight, _renderWidth, _renderHeight,
_plotWidth, _plotHeight, _compWidth, _compHeight, _eigenCount, _exprCount, _landmarkCount,
_viewMode, _exprMode, _filtering;
unsigned _poseMode = 0;
bool _enableCheekPuff;
MeshRendererBroker _broker;
MeshRenderer *_renderer = nullptr;
static const char _windowTitle[], *_exprAbbr[][4], *_sfmExprAbbr[][4];
MyTimer _timer;
bool _showFPS;
bool _performCalibration;
bool _cameraNeedsUpdate;
double _frameTime;
float _globalExpressionParam;
#ifdef _ENABLE_UI
@@ -735,6 +748,8 @@ NvCV_Status App::initMLPExpressions() {
BAIL_IF_ERR(err = NvAR_SetString(_featureHan, NvAR_Parameter_Config(ModelDir), FLAG_modelDir.c_str()));
BAIL_IF_ERR(err = NvAR_SetCudaStream(_featureHan, NvAR_Parameter_Config(CUDAStream), _stream));
BAIL_IF_ERR(err = NvAR_SetU32(_featureHan, NvAR_Parameter_Config(Temporal), _filtering));
BAIL_IF_ERR(err = NvAR_SetU32(_featureHan, NvAR_Parameter_Config(PoseMode), _poseMode));
BAIL_IF_ERR(err = NvAR_SetU32(_featureHan, NvAR_Parameter_Config(EnableCheekPuff), _enableCheekPuff));
BAIL_IF_ERR(err = NvAR_Load(_featureHan));
_outputBboxData.assign(25, { 0.f, 0.f, 0.f, 0.f });
_outputBboxes.boxes = _outputBboxData.data();
@@ -752,7 +767,15 @@ NvCV_Status App::initMLPExpressions() {
_expressionExponent.resize(_exprCount, 1.0f);
BAIL_IF_ERR(err = NvAR_SetF32Array(_featureHan, NvAR_Parameter_Output(ExpressionCoefficients), _expressions.data(), _exprCount));
BAIL_IF_ERR(err = NvAR_SetObject(_featureHan, NvAR_Parameter_Input(Image), &_srcGpu, sizeof(NvCVImage)));
BAIL_IF_ERR(err = NvAR_SetObject(_featureHan, NvAR_Parameter_Output(Pose), &_pose, sizeof(NvAR_Quaternion)));
// Heuristic for focal length if it is not known
_cameraIntrinsicParams[0] = static_cast<float>(_srcGpu.height); // focal length
_cameraIntrinsicParams[1] = static_cast<float>(_srcGpu.width) / 2.0f; // cx
_cameraIntrinsicParams[2] = static_cast<float>(_srcGpu.height) / 2.0f; // cy
BAIL_IF_ERR(err = NvAR_SetF32Array(_featureHan, NvAR_Parameter_Input(CameraIntrinsicParams), _cameraIntrinsicParams,
NUM_CAMERA_INTRINSIC_PARAMS));
BAIL_IF_ERR(err = NvAR_SetObject(_featureHan, NvAR_Parameter_Output(Pose), &_pose.rotation, sizeof(NvAR_Quaternion)));
BAIL_IF_ERR(err = NvAR_SetObject(_featureHan, NvAR_Parameter_Output(PoseTranslation), &_pose.translation, sizeof(NvAR_Vector3f)));
_exprMode = EXPR_MODE_MLP;
bail:
@@ -792,6 +815,22 @@ NvCV_Status App::normalizeExpressionsWeights() {
return NVCV_SUCCESS;
}
NvCV_Status App::updateCamera() {
NvCV_Status err;
if (_poseMode == 1) {
const NvAR_Vector3f cam_loc = {0, 0, 0};
const NvAR_Vector3f cam_dir = {0, 0, -1};
const NvAR_Vector3f cam_up = {0, 1, 0};
const float focal_length = _cameraIntrinsicParams[0];
const float v_fov = focal_length == 0.0f ? 0.0f : 2.0f * atan(_videoHeight / (2 * focal_length));
err = _renderer->setCamera(&cam_loc.vec[0], &cam_dir.vec[0], &cam_up.vec[0], v_fov, 0.01f, 1000.0f);
} else {
err = _renderer->setCamera(nullptr, nullptr, nullptr, 0.0f);
}
_cameraNeedsUpdate = false;
return err;
}
NvCV_Status App::init() {
NvCV_Status err;
std::string path;
@@ -816,9 +855,15 @@ NvCV_Status App::init() {
_performCalibration = false;
_frameTime = 0.f;
_exprMode = 0;
_poseMode = FLAG_poseMode;
_enableCheekPuff = FLAG_cheekPuff;
_featureHan = nullptr;
_filtering = FLAG_filter;
_globalExpressionParam = 1.0f;
_cameraIntrinsicParams[0] = 0.0f;
_cameraIntrinsicParams[1] = 0.0f;
_cameraIntrinsicParams[2] = 0.0f;
_cameraNeedsUpdate = true;
err = _broker.getMeshRendererList(rendererList);
if (NVCV_SUCCESS != err) {
@@ -855,16 +900,13 @@ NvCV_Status App::init() {
printf("renderer init: %s\n", NvCV_GetErrorStringFromCode(err));
return err;
}
err = _renderer->setCamera(nullptr, nullptr, nullptr, (float)(FLAG_fov * D_RADIANS_PER_DEGREE));
if (NVCV_SUCCESS != err) {
printf("renderer setCamera: %s\n", NvCV_GetErrorStringFromCode(err));
return err;
}
BAIL_IF_ERR(err = NvCVImage_Alloc(&_srcGpu, _videoWidth, _videoHeight, NVCV_BGR, NVCV_U8, NVCV_CHUNKY, NVCV_GPU, 1));
BAIL_IF_ERR(err = NvCVImage_Alloc(&_srcImg, _videoWidth, _videoHeight, NVCV_BGR, NVCV_U8, NVCV_CHUNKY, NVCV_CPU_PINNED, 0));
BAIL_IF_ERR(err = NvCVImage_Alloc(&_compImg, _compWidth, _compHeight, NVCV_BGR, NVCV_U8, NVCV_CHUNKY, NVCV_CPU, 0));
BAIL_IF_ERR(err = NvCVImage_Alloc(&_renderImg, _renderWidth, _renderHeight, NVCV_RGBA, NVCV_U8, NVCV_CHUNKY, NVCV_CPU, 0));
CVWrapperForNvCVImage(&_compImg, &_ocvDstImg);
CVWrapperForNvCVImage(&_srcImg, &_ocvSrcImg);
resizeDst();
if (!FLAG_outFile.empty() || !FLAG_show) {
@@ -912,7 +954,6 @@ NvCV_Status App::overlayLandmarks(const float landmarks[126 * 2], unsigned scree
return NVCV_SUCCESS;
}
NvCV_Status App::run() {
NvCV_Status err = NVCV_SUCCESS;
NvCVImage tmpImg, view;
@@ -925,7 +966,6 @@ NvCV_Status App::run() {
--frameCount; // Account for the wasted frame
continue; // Read the first frame again
}
NVWrapperForCVMat(&_ocvSrcImg, &_srcImg); // We probably don't need to do this every frame
#ifdef _ENABLE_UI
unsigned int exprMode = 0;
@@ -986,6 +1026,13 @@ NvCV_Status App::run() {
BAIL_IF_ERR(err = NvCVImage_Transfer(&_srcImg, &_srcGpu, 1.f, _stream, nullptr));
BAIL_IF_ERR(err = NvAR_Run(_featureHan));
unsigned isFaceDetected = (_outputBboxes.num_boxes > 0) ? 0xFF : 0;
if (_cameraNeedsUpdate) {
err = updateCamera();
if (NVCV_SUCCESS != err) {
printf("renderer setCamera: %s\n", NvCV_GetErrorStringFromCode(err));
return err;
}
}
if (_performCalibration) {
calibrateExpressionWeights();
}
@@ -1003,7 +1050,8 @@ NvCV_Status App::run() {
}
if (_viewMode & VIEW_MESH) {
if (isFaceDetected) {
BAIL_IF_ERR(err = _renderer->render(_expressions.data(), &_pose.rotation.x, nullptr, &_renderImg)); // GL _renderImg is upside down
float* head_translation = _poseMode == 1 ? &_pose.translation.vec[0] : nullptr;
BAIL_IF_ERR(err = _renderer->render(_expressions.data(), &_pose.rotation.x, head_translation, &_renderImg)); // GL _renderImg is upside down
NvCVImage_InitView(&view, &_compImg, _renderX, _renderY, _renderWidth, _renderHeight);
NvCVImage_FlipY(&view, &view); // Since OpenGL renderImg is upside-down, we copy it to a flipped dst
NvCVImage_Transfer(&_renderImg, &view, 1.0f, _stream, nullptr); // VFlip RGBA --> BGR
@@ -1047,6 +1095,7 @@ NvCV_Status App::run() {
case 'E': case CTL('E'): toggleExpressionFiltering(); break;
case 'G': case CTL('G'): toggleGazeFiltering(); break;
case 'C': case CTL('C'): toggleClosureEnhancement(); break;
case 'M': case CTL('M'): togglePoseMode(); break;
default:
if (key < 0) continue; // No key
break; // Non-mapped key
@@ -1208,6 +1257,13 @@ NvCV_Status App::toggleClosureEnhancement() {
return err;
}
NvCV_Status App::togglePoseMode() {
_poseMode = !_poseMode;
NvCV_Status err = NvAR_SetU32(_featureHan, NvAR_Parameter_Config(PoseMode), _poseMode);
if (NVCV_SUCCESS == err) err = NvAR_Load(_featureHan);
_cameraNeedsUpdate = true;
return err;
}
/********************************************************************************
* main

BIN
samples/ExpressionApp/ExpressionApp.exe
View File

Binary file not shown.

4
samples/ExpressionApp/MeshRenderer.cpp
View File

@@ -105,8 +105,8 @@ NvCV_Status MeshRenderer::init(unsigned width, unsigned height, const char *wind
}
NvCV_Status MeshRenderer::setCamera(const float locPt[3], const float lookVec[3], const float upVec[3], float vfov) {
return m_dispatch.setCamera(this, locPt, lookVec, upVec, vfov);
NvCV_Status MeshRenderer::setCamera(const float locPt[3], const float lookVec[3], const float upVec[3], float vfov, float z_near, float z_far) {
return m_dispatch.setCamera(this, locPt, lookVec, upVec, vfov, z_near, z_far);
}

10
samples/ExpressionApp/MeshRenderer.h
View File

@@ -40,8 +40,8 @@ public:
void (*destroy)(MeshRenderer *han);
NvCV_Status (*read)(MeshRenderer *han, const char *modelFile);
NvCV_Status (*init)(MeshRenderer *han, unsigned width, unsigned height, const char *windowName);
NvCV_Status (*setCamera)(MeshRenderer *han,
const float locPt[3], const float lookVec[3], const float upVec[3], float vfov);
NvCV_Status (*setCamera)(MeshRenderer *han, const float locPt[3], const float lookVec[3], const float upVec[3],
float vfov, float near_z, float far_z);
NvCV_Status (*render)(MeshRenderer *han,
const float exprs[53], const float qrot[4], const float trans[3], NvCVImage *result);
Dispatch();
@@ -83,8 +83,12 @@ public:
/// @param[in] upVec the 3D vector pointing up. This does not need to be normalized.
/// NULL implies the default up direction, derived from the rest pose of the model.
/// @param[in] vfov the vertical field of view of the camera. Zero implies an orthographic camera.
/// @param[in] near_z Near z clipping plane. Distance along camera's negative z-axis. If both near_z and far_z are
/// 0.0, the z clipping planes are computed based on the mesh bounding box.
/// @param[in] far_z Far z clipping plane. Distance along camera's negative z-axis. If both near_z and far_z are
/// 0.0, the z clipping planes are computed based on the mesh bounding box.
/// @return NVCV_SUCCESS if the camera was initialized successfully.
NvCV_Status setCamera(const float locPt[3], const float lookVec[3], const float upVec[3], float vfov);
NvCV_Status setCamera(const float locPt[3], const float lookVec[3], const float upVec[3], float vfov, float near_z = 0.0f, float far_z = 0.0f);
/// Render the mesh as deformed by the expression signals.
/// @param[in] exprs the expression signals (53 of them).

65
samples/FaceTrack/FaceEngine.cpp
View File

@@ -439,10 +439,10 @@ void FaceEngine::releaseFaceFittingIOParams() {
}
}
unsigned FaceEngine::findFaceBoxes() {
NvCV_Status FaceEngine::findFaceBoxes(unsigned &num_boxes) {
NvCV_Status nvErr = NvAR_Run(faceDetectHandle);
if (NVCV_SUCCESS != nvErr) return 0;
return (unsigned)output_bboxes.num_boxes;
num_boxes = (unsigned)output_bboxes.num_boxes;
return nvErr;
}
NvAR_Rect* FaceEngine::getLargestBox() {
@@ -587,16 +587,18 @@ void FaceEngine::DrawPose(const cv::Mat& src, const NvAR_Quaternion* pose) const
cv::line(src, p0, pz, CV_RGB(0, 0, 255), thickness);
}
NvCV_Status FaceEngine::findLandmarks() {
FaceEngine::Err FaceEngine::findLandmarks() {
NvCV_Status nvErr;
nvErr = NvAR_Run(landmarkDetectHandle);
if (NVCV_SUCCESS != nvErr) {
return nvErr;
return FaceEngine::Err::errRun;
}
if (getAverageLandmarksConfidence() < confidenceThreshold) {
return NVCV_ERR_GENERAL;
float avg_confidence = getAverageLandmarksConfidence();
if (avg_confidence < confidenceThreshold) {
return FaceEngine::Err::errNoFaceDetected;
} else {
average_poses(getPose(), batchSize);
NvAR_Point2f *pt, *endPt;
@@ -611,7 +613,7 @@ NvCV_Status FaceEngine::findLandmarks() {
pt->y /= batchSize;
}
}
return NVCV_SUCCESS;
return FaceEngine::Err::errNone;
}
NvAR_Point2f* FaceEngine::getLandmarks() { return facial_landmarks.data(); }
@@ -644,12 +646,16 @@ float FaceEngine::getAverageLandmarksConfidence() {
NvAR_Quaternion* FaceEngine::getPose() { return facial_pose.data(); }
unsigned FaceEngine::findLargestFaceBox(NvAR_Rect& faceBox, int variant) {
unsigned n;
FaceEngine::Err FaceEngine::findLargestFaceBox(NvAR_Rect& faceBox, int variant) {
NvAR_Rect* pFaceBox;
unsigned num_boxes;
n = findFaceBoxes();
if (n >= 1) {
NvCV_Status nv_err = findFaceBoxes(num_boxes);
if (NVCV_SUCCESS != nv_err) {
return FaceEngine::Err::errRun;
}
if (num_boxes >= 1) {
pFaceBox = getLargestBox();
if (nullptr == pFaceBox) {
faceBox.x = faceBox.y = faceBox.width = faceBox.height = 0.0f;
@@ -658,40 +664,41 @@ unsigned FaceEngine::findLargestFaceBox(NvAR_Rect& faceBox, int variant) {
}
enlargeAndSquarifyImageBox(.2f, faceBox, variant);
}
return n;
else return FaceEngine::Err::errNoFaceDetected;
return FaceEngine::Err::errNone;
}
unsigned FaceEngine::acquireFaceBox(cv::Mat& src, NvAR_Rect& faceBox, int variant) {
unsigned n = 0;
FaceEngine::Err FaceEngine::acquireFaceBox(cv::Mat& src, NvAR_Rect& faceBox, int variant) {
NvCVImage fxSrcChunkyCPU;
(void)NVWrapperForCVMat(&src, &fxSrcChunkyCPU);
NvCV_Status cvErr = NvCVImage_Transfer(&fxSrcChunkyCPU, &inputImageBuffer, 1.0f, stream, &tmpImage);
if (NVCV_SUCCESS != cvErr) {
return n;
return FaceEngine::Err::errRun;
}
n = findLargestFaceBox(faceBox, variant);
return n;
FaceEngine::Err nv_err = findLargestFaceBox(faceBox, variant);
return nv_err;
}
unsigned FaceEngine::acquireFaceBoxAndLandmarks(cv::Mat& src, NvAR_Point2f* refMarks, NvAR_Rect& faceBox, int /*variant*/) {
unsigned n = 0;
FaceEngine::Err FaceEngine::acquireFaceBoxAndLandmarks(cv::Mat& src, NvAR_Point2f* refMarks, NvAR_Rect& faceBox, int /*variant*/) {
NvCVImage fxSrcChunkyCPU;
(void)NVWrapperForCVMat(&src, &fxSrcChunkyCPU);
NvCV_Status cvErr = NvCVImage_Transfer(&fxSrcChunkyCPU, &inputImageBuffer, 1.0f, stream, &tmpImage);
if (NVCV_SUCCESS != cvErr) {
return n;
return FaceEngine::Err::errRun;
}
if (findLandmarks() != NVCV_SUCCESS) return 0;
faceBox = output_bboxes.boxes[0];
n = 1;
memcpy(refMarks, getLandmarks(), sizeof(NvAR_Point2f) * numLandmarks);
return n;
FaceEngine::Err nv_err = findLandmarks();
if(nv_err == FaceEngine::Err::errNone){
faceBox = output_bboxes.boxes[0];
memcpy(refMarks, getLandmarks(), sizeof(NvAR_Point2f) * numLandmarks);
}
return nv_err;
}
void FaceEngine::setFaceStabilization(bool _bStabilizeFace) { bStabilizeFace = _bStabilizeFace; }

12
samples/FaceTrack/FaceEngine.h
View File

@@ -107,7 +107,7 @@ typedef struct LandmarksProperties {
class FaceEngine {
public:
enum Err { errNone, errGeneral, errRun, errInitialization, errRead, errEffect, errParameter };
enum Err { errNone, errGeneral, errRun, errInitialization, errRead, errEffect, errParameter, errNoFaceDetected };
int input_image_width, input_image_height, input_image_pitch;
const LandmarksProperties LANDMARKS_INFO[2] = {
{ 68, 15.0f }, // number of landmark points, confidence threshold value
@@ -144,9 +144,9 @@ class FaceEngine {
void releaseLandmarkDetectionIOParams();
void releaseFaceFittingIOParams();
unsigned findFaceBoxes();
NvCV_Status findFaceBoxes(unsigned &num_boxes);
NvAR_Rect* getLargestBox();
NvCV_Status findLandmarks();
FaceEngine::Err findLandmarks();
NvAR_BBoxes* getBoundingBoxes();
NvAR_Point2f* getLandmarks();
NvAR_Quaternion* getPose();
@@ -154,9 +154,9 @@ class FaceEngine {
float getAverageLandmarksConfidence();
void enlargeAndSquarifyImageBox(float enlarge, NvAR_Rect& box, int FLAG_variant);
static void jiggleBox(std::mt19937& ran, float minMag, float maxMag, const NvAR_Rect& cleanBox, NvAR_Rect& noisyBox);
unsigned findLargestFaceBox(NvAR_Rect& faceBox, int variant = 0);
unsigned acquireFaceBox(cv::Mat& src, NvAR_Rect& faceBox, int variant = 0);
unsigned acquireFaceBoxAndLandmarks(cv::Mat& src, NvAR_Point2f* refMarks, NvAR_Rect& faceBox, int variant = 0);
FaceEngine::Err findLargestFaceBox(NvAR_Rect& faceBox, int variant = 0);
FaceEngine::Err acquireFaceBox(cv::Mat& src, NvAR_Rect& faceBox,int variant = 0);
FaceEngine::Err acquireFaceBoxAndLandmarks(cv::Mat& src, NvAR_Point2f* refMarks, NvAR_Rect& faceBox, int variant = 0);
Err fitFaceModel(cv::Mat& frame);
NvAR_FaceMesh* getFaceMesh();
NvAR_RenderingParams* getRenderingParams();

89
samples/FaceTrack/FaceTrack.cpp
View File

@@ -598,8 +598,10 @@ void DoApp::DrawLandmarkPoints(const cv::Mat &src, NvAR_Point2f *facial_landmark
else
frm = src;
NvAR_Point2f *pt, *endPt;
// Draw larger keypoints if input frame is greater than 720p resolution
int circle_radius = (frm.rows <= 720) ? 1 : 2;
for (endPt = (pt = (NvAR_Point2f *)facial_landmarks) + numLandmarks; pt < endPt; ++pt)
cv::circle(frm, cv::Point(lround(pt->x), lround(pt->y)), 1, cv::Scalar(0, 0, 255), -1);
cv::circle(frm, cv::Point(lround(pt->x), lround(pt->y)), circle_radius, cv::Scalar(0, 0, 255), -1);
NvAR_Quaternion *pose = face_ar_engine.getPose();
if (pose)
face_ar_engine.DrawPose(frm, pose);
@@ -664,34 +666,36 @@ DoApp::Err DoApp::acquireFaceBox() {
NvAR_Rect output_bbox;
// get landmarks in original image resolution coordinate space
unsigned n = face_ar_engine.acquireFaceBox(frame, output_bbox, 0);
nvErr = face_ar_engine.acquireFaceBox(frame, output_bbox, 0);
if (n && FLAG_verbose) {
if (nvErr==FaceEngine::Err::errNone) {
if (FLAG_verbose) {
printf("FaceBox: [\n");
printf("%7.1f%7.1f%7.1f%7.1f\n", output_bbox.x, output_bbox.y, output_bbox.x + output_bbox.width,
output_bbox.y + output_bbox.height);
printf("]\n");
}
if (FLAG_captureOutputs) {
writeFrameAndEstResults(frame, face_ar_engine.output_bboxes);
writeVideoAndEstResults(frame, face_ar_engine.output_bboxes);
}
}
if (FLAG_captureOutputs) {
writeFrameAndEstResults(frame, face_ar_engine.output_bboxes);
writeVideoAndEstResults(frame, face_ar_engine.output_bboxes);
}
#ifdef VISUALIZE
if (n > 0) { // At least one face was found
if (drawVisualization) {
DrawBBoxes(frame, &output_bbox); // This will write a frame if in offlineMode
}
#endif // VISUALIZE
}
else { // No faces found
#ifdef VISUALIZE
if (FLAG_offlineMode) {
faceDetectOutputVideo.write(frame); // This will write a frame if in offlineMode
}
err = errNoFace;
#endif // !VISUALIZE
if (nvErr==FaceEngine::Err::errNoFaceDetected) err = errNoFace;
else {err = errGeneral;}
}
#else // !VISUALIZE
if (0 == n) err = errNoFace;
#endif // VISUALIZE
frameIndex++;
return err;
@@ -704,38 +708,51 @@ DoApp::Err DoApp::acquireFaceBoxAndLandmarks() {
std::vector<NvAR_Point2f> facial_landmarks(numLandmarks);
// get landmarks in original image resolution coordinate space
unsigned n = face_ar_engine.acquireFaceBoxAndLandmarks(frame, facial_landmarks.data(), output_bbox, 0);
nvErr = face_ar_engine.acquireFaceBoxAndLandmarks(frame, facial_landmarks.data(), output_bbox, 0);
if (n && FLAG_verbose && face_ar_engine.appMode != FaceEngine::mode::faceDetection) {
printf("Landmarks: [\n");
for (const auto &pt : facial_landmarks) {
printf("%7.1f%7.1f\n", pt.x, pt.y);
if (nvErr == FaceEngine::Err::errNone)
{
if (FLAG_verbose && face_ar_engine.appMode != FaceEngine::mode::faceDetection)
{
printf("Landmarks: [\n");
for (const auto &pt : facial_landmarks)
{
printf("%7.1f%7.1f\n", pt.x, pt.y);
}
printf("]\n");
}
if (FLAG_captureOutputs)
{
writeFrameAndEstResults(frame, face_ar_engine.output_bboxes, facial_landmarks.data());
writeVideoAndEstResults(frame, face_ar_engine.output_bboxes, facial_landmarks.data());
}
printf("]\n");
}
if (FLAG_captureOutputs) {
writeFrameAndEstResults(frame, face_ar_engine.output_bboxes, facial_landmarks.data());
writeVideoAndEstResults(frame, face_ar_engine.output_bboxes, facial_landmarks.data());
}
#ifdef VISUALIZE
if (n > 0) { // At least one face found
if (drawVisualization) {
if (drawVisualization)
{
DrawLandmarkPoints(frame, facial_landmarks.data(), numLandmarks); // Writes frame in offline mode
if (FLAG_offlineMode) {
DrawBBoxes(frame, &output_bbox); // Writes frame in offline mode
if (FLAG_offlineMode)
{
DrawBBoxes(frame, &output_bbox); // Writes frame in offline mode
}
}
#endif // VISUALIZE
}
else { // No faces found
if (FLAG_offlineMode) {
faceDetectOutputVideo.write(frame); // These two wrtite frames if a face was not detected
else
{ // No faces found
#ifdef VISUALIZE
if (FLAG_offlineMode)
{
faceDetectOutputVideo.write(frame); // These two write frames if a face was not detected
landMarkOutputVideo.write(frame);
}
#endif // !VISUALIZE
if (nvErr == FaceEngine::Err::errNoFaceDetected)
err = errNoFace;
else
{
err = errGeneral;
}
}
#else // !VISUALIZE
if (0 == n) err = errNoFace;
#endif // VISUALIZE
frameIndex++;
return err;

BIN
samples/FaceTrack/FaceTrack.exe
View File

Binary file not shown.

7
samples/GazeRedirect/GazeEngine.cpp
View File

@@ -91,6 +91,8 @@ GazeEngine::Err GazeEngine::createGazeRedirectionFeature(const char* modelPath,
BAIL_IF_NVERR(nvErr, err, GazeEngine::Err::errParameter);
nvErr = NvAR_SetCudaStream(gazeRedirectHandle, NvAR_Parameter_Config(CUDAStream), stream);
BAIL_IF_NVERR(nvErr, err, GazeEngine::Err::errParameter);
nvErr = NvAR_SetU32(gazeRedirectHandle, NvAR_Parameter_Config(UseCudaGraph), bUseCudaGraph);
BAIL_IF_NVERR(nvErr, err, GazeEngine::Err::errParameter);
nvErr = NvAR_SetU32(gazeRedirectHandle, NvAR_Parameter_Config(EyeSizeSensitivity), eyeSizeSensitivity);
BAIL_IF_NVERR(nvErr, err, GazeEngine::Err::errParameter);
nvErr = NvAR_Load(gazeRedirectHandle);
@@ -437,6 +439,11 @@ void GazeEngine::setGazeRedirect(bool _bGazeRedirect) {
bGazeRedirect = _bGazeRedirect;
}
void GazeEngine::setUseCudaGraph(bool _bUseCudaGraph) {
// If this variable is set, gaze redirection occurs in addition to estimation.
bUseCudaGraph = _bUseCudaGraph;
}
GazeEngine::Err GazeEngine::setNumLandmarks(int n) {
GazeEngine::Err err = errNone;
for (auto const& info : LANDMARKS_INFO) {

3
samples/GazeRedirect/GazeEngine.h
View File

@@ -150,6 +150,7 @@ class GazeEngine {
void setFaceStabilization(bool);
Err setNumLandmarks(int);
void setGazeRedirect(bool _bGazeRedirect);
void setUseCudaGraph(bool _bUseCudaGraph);
void setEyeSizeSensitivity(unsigned);
int getNumLandmarks() { return numLandmarks; }
int getNumGazeOutputLandmarks() { return num_output_landmarks; }
@@ -184,12 +185,14 @@ class GazeEngine {
bool bStabilizeFace;
bool bUseOTAU;
bool bGazeRedirect;
bool bUseCudaGraph;
char *fdOTAModelPath, *ldOTAModelPath;
GazeEngine() {
batchSize = 1;
bStabilizeFace = true;
bGazeRedirect = true;
bUseCudaGraph = true;
numLandmarks = LANDMARKS_INFO[0].numPoints;
num_output_landmarks = 12;
confidenceThreshold = LANDMARKS_INFO[0].confidence_threshold;

25
samples/GazeRedirect/GazeRedirect.cpp
View File

@@ -75,7 +75,8 @@
bool FLAG_debug = false, FLAG_verbose = false, FLAG_temporal = true, FLAG_captureOutputs = false,
FLAG_drawVisualization = true, FLAG_offlineMode = false, FLAG_isNumLandmarks126 = false,
FLAG_splitScreenView = true, FLAG_displayLandmarks = false, FLAG_gazeRedirect=true;
FLAG_splitScreenView = true, FLAG_displayLandmarks = false, FLAG_gazeRedirect = true, FLAG_useCudaGraph = false;
;
std::string FLAG_outDir, FLAG_inFile, FLAG_outFile, FLAG_modelPath, FLAG_landmarks, FLAG_captureCodec = "avc1",
FLAG_camRes = "480";
unsigned FLAG_camID = 0;
@@ -106,6 +107,7 @@ static void Usage() {
" --draw_visualization draw the landmarks, display gaze estimation and head rotation, default to true\n"
" --redirect_gaze redirection of the eyes in addition to estimating gaze, default to true\n"
"(Default)."
" --use_cuda_graph use cuda graph to optimize computations "
" --benchmarks[=<pattern>] run benchmarks\n");
}
@@ -211,7 +213,8 @@ static int ParseMyArgs(int argc, char **argv) {
GetFlagArgVal("split_screen_view", arg, &FLAG_splitScreenView) ||
GetFlagArgVal("temporal", arg, &FLAG_temporal) ||
GetFlagArgVal("draw_visualization", arg, &FLAG_drawVisualization) ||
GetFlagArgVal("redirect_gaze", arg, &FLAG_gazeRedirect))) {
GetFlagArgVal("redirect_gaze", arg, &FLAG_gazeRedirect) ||
GetFlagArgVal("use_cuda_graph", arg, &FLAG_useCudaGraph))) {
continue;
} else if (GetFlagArgVal("help", arg, &help)) {
Usage();
@@ -312,7 +315,8 @@ class DoApp {
~DoApp();
void stop();
Err initGazeEngine(const char *modelPath = nullptr, bool isLandmarks126 = false, bool gazeRedirect=true, unsigned eyeSizeSensitivity=3);
Err initGazeEngine(const char *modelPath = nullptr, bool isLandmarks126 = false, bool gazeRedirect = true,
unsigned eyeSizeSensitivity = 3, bool useCudaGraph = false);
Err initCamera(const char *camRes = nullptr, unsigned int camID = 0);
Err initOfflineMode(const char *inputFilename = nullptr, const char *outputFilename = nullptr);
Err acquireFrame();
@@ -383,12 +387,14 @@ void DoApp::processKey(int key) {
}
}
DoApp::Err DoApp::initGazeEngine(const char *modelPath, bool isNumLandmarks126, bool gazeRedirect, unsigned eyeSizeSensitivity) {
DoApp::Err DoApp::initGazeEngine(const char *modelPath, bool isNumLandmarks126, bool gazeRedirect,
unsigned eyeSizeSensitivity, bool useCudaGraph) {
if (!cap.isOpened()) return errVideo;
int numLandmarkPoints = isNumLandmarks126 ? 126 : 68;
gaze_ar_engine.setNumLandmarks(numLandmarkPoints);
gaze_ar_engine.setGazeRedirect(gazeRedirect);
gaze_ar_engine.setUseCudaGraph(useCudaGraph);
gaze_ar_engine.setEyeSizeSensitivity(eyeSizeSensitivity);
nvErr = gaze_ar_engine.createGazeRedirectionFeature(modelPath);
@@ -613,6 +619,15 @@ DoApp::Err DoApp::initCamera(const char *camRes, unsigned int camID) {
inputWidth = (int)cap.get(CV_CAP_PROP_FRAME_WIDTH);
inputHeight = (int)cap.get(CV_CAP_PROP_FRAME_HEIGHT);
// openCV API(CAP_PROP_FRAME_WIDTH) to get camera resolution is not always reliable with some cameras
cap >> frame;
if (frame.empty()) return errCamera;
if (inputWidth != frame.cols || inputHeight != frame.rows) {
std::cout << "!!! warning: openCV API(CAP_PROP_FRAME_WIDTH/CV_CAP_PROP_FRAME_HEIGHT) to get camera resolution is not trustable. Using the resolution from the actual frame" << std::endl;
inputWidth = frame.cols;
inputHeight = frame.rows;
}
gaze_ar_engine.setInputImageWidth(inputWidth);
gaze_ar_engine.setInputImageHeight(inputHeight);
}
@@ -844,7 +859,7 @@ int main(int argc, char **argv) {
}
BAIL_IF_ERR(doErr);
doErr = app.initGazeEngine(FLAG_modelPath.c_str(), FLAG_isNumLandmarks126, FLAG_gazeRedirect, FLAG_eyeSizeSensitivity);
doErr = app.initGazeEngine(FLAG_modelPath.c_str(), FLAG_isNumLandmarks126, FLAG_gazeRedirect, FLAG_eyeSizeSensitivity, FLAG_useCudaGraph);
BAIL_IF_ERR(doErr);
doErr = app.run();

BIN
samples/GazeRedirect/GazeRedirect.exe
View File

Binary file not shown.

2
samples/external/Imgui/imgui-docking/examples/example_glfw_vulkan/CMakeLists.txt vendored
View File

@@ -7,7 +7,7 @@ cmake_minimum_required(VERSION 2.8)
project(imgui_example_glfw_vulkan C CXX)
if(NOT CMAKE_BUILD_TYPE)
set(CMAKE_BUILD_TYPE Debug CACHE STRING "" FORCE)
set(CMAKE_BUILD_TYPE Release CACHE STRING "" FORCE)
endif()
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DVK_PROTOTYPES")

8
version.h
View File

@@ -23,12 +23,12 @@
#define NVIDIA_AR_SDK_VERSION_MAJOR 0
#define NVIDIA_AR_SDK_VERSION_MINOR 8
#define NVIDIA_AR_SDK_VERSION_RELEASE 1
#define NVIDIA_AR_SDK_VERSION_RELEASE 2
#define NVIDIA_AR_SDK_VERSION_BUILD 0
#define NVIDIA_AR_SDK_VERSION 0,8,1,0
#define NVIDIA_AR_SDK_VERSION 0,8,2,0
#define NVIDIA_AR_SDK_VERSION_MAJOR_MINOR 0,8
#define NVIDIA_AR_SDK_VERSION_STRING "0.8.1.0"
#define NVIDIA_AR_SDK_VERSION_STRING_SHORT "0.8.1"
#define NVIDIA_AR_SDK_VERSION_STRING "0.8.2.0"
#define NVIDIA_AR_SDK_VERSION_STRING_SHORT "0.8.2"
#define NVIDIA_AR_SDK_VERSION_STRING_MAJOR_MINOR "0.8"