demonkenny/MAXINE-AR-SDK
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v0.8.1.0 Release

Browse Source 
v0.8.1.0 Release
This commit is contained in:
jdsouza90
2022-09-20 09:59:34 -07:00
parent 3bd2be62a4
commit cf68600c4f
691 changed files with 181019 additions and 22147 deletions
Download Patch File Download Diff File Expand all files Collapse all files

167
samples/BodyTrack/BodyEngine.cpp
View File

@@ -110,12 +110,21 @@ BodyEngine::Err BodyEngine::createKeyPointDetectionFeature(const char* modelPath
nvErr = NvAR_SetU32(keyPointDetectHandle, NvAR_Parameter_Config(Temporal), bStabilizeBody);
BAIL_IF_CVERR(nvErr, err, BodyEngine::Err::errParameter);
nvErr = NvAR_SetF32(keyPointDetectHandle, NvAR_Parameter_Config(FocalLength), bFocalLength);
BAIL_IF_CVERR(nvErr, err, BodyEngine::Err::errParameter);
nvErr = NvAR_SetF32(keyPointDetectHandle, NvAR_Parameter_Config(UseCudaGraph), bUseCudaGraph);
BAIL_IF_CVERR(nvErr, err, BodyEngine::Err::errParameter);
#if NV_MULTI_OBJECT_TRACKER
nvErr = NvAR_SetF32(keyPointDetectHandle, NvAR_Parameter_Config(TrackPeople), bEnablePeopleTracking);
BAIL_IF_CVERR(nvErr, err, BodyEngine::Err::errParameter);
nvErr = NvAR_SetU32(keyPointDetectHandle, NvAR_Parameter_Config(ShadowTrackingAge), shadowTrackingAge);
BAIL_IF_CVERR(nvErr, err, BodyEngine::Err::errParameter);
nvErr = NvAR_SetU32(keyPointDetectHandle, NvAR_Parameter_Config(ProbationAge), probationAge);
BAIL_IF_CVERR(nvErr, err, BodyEngine::Err::errParameter);
nvErr = NvAR_SetU32(keyPointDetectHandle, NvAR_Parameter_Config(MaxTargetsTracked), maxTargetsTracked);
BAIL_IF_CVERR(nvErr, err, BodyEngine::Err::errParameter);
#endif
nvErr = NvAR_Load(keyPointDetectHandle);
BAIL_IF_CVERR(nvErr, err, BodyEngine::Err::errInitialization);
@@ -176,18 +185,23 @@ BodyEngine::Err BodyEngine::initKeyPointDetectionIOParams(NvCVImage* inBuf) {
NvCV_Status nvErr = NVCV_SUCCESS;
BodyEngine::Err err = BodyEngine::Err::errNone;
uint output_bbox_size;
#if NV_MULTI_OBJECT_TRACKER
uint output_tracking_bbox_size;
#endif
nvErr = NvAR_SetObject(keyPointDetectHandle, NvAR_Parameter_Input(Image), inBuf, sizeof(NvCVImage));
BAIL_IF_CVERR(nvErr, err, BodyEngine::Err::errParameter);
nvErr = NvAR_SetF32(keyPointDetectHandle, NvAR_Parameter_Input(FocalLength), bFocalLength);
BAIL_IF_CVERR(nvErr, err, BodyEngine::Err::errParameter);
nvErr = NvAR_GetU32(keyPointDetectHandle, NvAR_Parameter_Config(NumKeyPoints), &numKeyPoints);
BAIL_IF_CVERR(nvErr, err, BodyEngine::Err::errParameter);
keypoints.assign(batchSize * numKeyPoints, {0.f, 0.f});
keypoints3D.assign(batchSize * numKeyPoints, {0.f, 0.f, 0.f});
jointAngles.assign(batchSize * numKeyPoints, {0.f, 0.f, 0.f, 1.f});
keypoints.assign(batchSize * numKeyPoints, { 0.f, 0.f });
keypoints3D.assign(batchSize * numKeyPoints, { 0.f, 0.f, 0.f });
jointAngles.assign(batchSize * numKeyPoints, { 0.f, 0.f, 0.f, 1.f });
keypoints_confidence.assign(batchSize * numKeyPoints, 0.f);
referencePose.assign(numKeyPoints, {0.f, 0.f, 0.f});
referencePose.assign(numKeyPoints, { 0.f, 0.f, 0.f });
const void* pReferencePose;
nvErr = NvAR_GetObject(keyPointDetectHandle, NvAR_Parameter_Config(ReferencePose), &pReferencePose,
@@ -208,19 +222,47 @@ BodyEngine::Err BodyEngine::initKeyPointDetectionIOParams(NvCVImage* inBuf) {
BAIL_IF_CVERR(nvErr, err, BodyEngine::Err::errParameter);
nvErr = NvAR_SetF32Array(keyPointDetectHandle, NvAR_Parameter_Output(KeyPointsConfidence),
keypoints_confidence.data(), batchSize * numKeyPoints);
keypoints_confidence.data(), sizeof(float));
BAIL_IF_CVERR(nvErr, err, BodyEngine::Err::errParameter);
output_bbox_size = batchSize;
if (!bStabilizeBody) output_bbox_size = 25;
output_bbox_data.assign(output_bbox_size, {0.f, 0.f, 0.f, 0.f});
#if NV_MULTI_OBJECT_TRACKER
if (bEnablePeopleTracking) {
output_tracking_bbox_size = maxTargetsTracked;
output_tracking_bbox_data.assign(output_tracking_bbox_size, { 0.f, 0.f, 0.f, 0.f, 0 });
output_tracking_bboxes.boxes = output_tracking_bbox_data.data();
output_tracking_bboxes.max_boxes = (uint8_t)output_tracking_bbox_size;
output_tracking_bboxes.num_boxes = 0;
nvErr =
NvAR_SetObject(keyPointDetectHandle, NvAR_Parameter_Output(TrackingBoundingBoxes), &output_tracking_bboxes, sizeof(NvAR_TrackingBBoxes));
BAIL_IF_CVERR(nvErr, err, BodyEngine::Err::errParameter);
}
else {
output_bbox_data.assign(25, { 0.f, 0.f, 0.f, 0.f });
output_bbox_conf_data.assign(25, 0.f);
output_bboxes.boxes = output_bbox_data.data();
output_bboxes.max_boxes = (uint8_t)output_bbox_data.size();
output_bboxes.num_boxes = 0;
nvErr = NvAR_SetObject(keyPointDetectHandle, NvAR_Parameter_Output(BoundingBoxes), &output_bboxes, sizeof(NvAR_BBoxes));
BAIL_IF_CVERR(nvErr, err, BodyEngine::Err::errParameter);
nvErr = NvAR_SetF32Array(keyPointDetectHandle, NvAR_Parameter_Output(BoundingBoxesConfidence),
output_bbox_conf_data.data(), output_bboxes.max_boxes);
BAIL_IF_CVERR(nvErr, err, BodyEngine::Err::errParameter);
}
#else
output_bbox_data.assign(25, { 0.f, 0.f, 0.f, 0.f });
output_bbox_conf_data.assign(25, 0.f);
output_bboxes.boxes = output_bbox_data.data();
output_bboxes.max_boxes = (uint8_t)output_bbox_size;
output_bboxes.num_boxes = (uint8_t)output_bbox_size;
nvErr =
NvAR_SetObject(keyPointDetectHandle, NvAR_Parameter_Output(BoundingBoxes), &output_bboxes, sizeof(NvAR_BBoxes));
output_bboxes.max_boxes = (uint8_t)output_bbox_data.size();
output_bboxes.num_boxes = 0;
nvErr = NvAR_SetObject(keyPointDetectHandle, NvAR_Parameter_Output(BoundingBoxes), &output_bboxes, sizeof(NvAR_BBoxes));
BAIL_IF_CVERR(nvErr, err, BodyEngine::Err::errParameter);
nvErr = NvAR_SetF32Array(keyPointDetectHandle, NvAR_Parameter_Output(BoundingBoxesConfidence),
output_bbox_conf_data.data(), output_bboxes.max_boxes);
BAIL_IF_CVERR(nvErr, err, BodyEngine::Err::errParameter);
#endif
bail:
return err;
}
@@ -262,6 +304,9 @@ void BodyEngine::releaseBodyDetectionIOParams() {
void BodyEngine::releaseKeyPointDetectionIOParams() {
NvCVImage_Dealloc(&inputImageBuffer);
if (!output_bbox_data.empty()) output_bbox_data.clear();
#if NV_MULTI_OBJECT_TRACKER
if (!output_tracking_bbox_data.empty()) output_tracking_bbox_data.clear();
#endif
if (!keypoints.empty()) keypoints.clear();
if (!keypoints3D.empty()) keypoints3D.clear();
if (!jointAngles.empty()) jointAngles.clear();
@@ -338,21 +383,6 @@ void BodyEngine::enlargeAndSquarifyImageBox(float enlarge, NvAR_Rect& box, int F
std::cout << "[bodypose] > NvAR_Run(keyPointDetectHandle): " << duration.count() << " microseconds" << std::endl;
#endif
if (getAverageKeyPointsConfidence() < confidenceThreshold) {
return NVCV_ERR_GENERAL;
} else {
NvAR_Point2f *pt, *endPt;
int i = 0;
for (endPt = (pt = getKeyPoints()) + numKeyPoints; pt != endPt; ++pt, i += 2) {
for (int j = 1; j < batchSize; j++) {
pt->x += pt[j * numKeyPoints].x;
pt->y += pt[j * numKeyPoints].y;
}
// average batch of inferences to generate final result keypoints
pt->x /= batchSize;
pt->y /= batchSize;
}
}
#ifdef DEBUG_PERF_RUNTIME
end = std::chrono::high_resolution_clock::now();
duration = std::chrono::duration_cast<std::chrono::microseconds>(end - start);
@@ -367,17 +397,21 @@ NvAR_Point3f* BodyEngine::getKeyPoints3D() { return keypoints3D.data(); }
NvAR_Quaternion* BodyEngine::getJointAngles() { return jointAngles.data(); }
NvAR_BBoxes* BodyEngine::getBBoxes(){ return &output_bboxes; }
#if NV_MULTI_OBJECT_TRACKER
NvAR_TrackingBBoxes* BodyEngine::getTrackingBBoxes() { return &output_tracking_bboxes; }
#endif
float* BodyEngine::getKeyPointsConfidence() { return keypoints_confidence.data(); }
float BodyEngine::getAverageKeyPointsConfidence() {
float average_confidence = 0.0f;
float* keypoints_confidence_all = getKeyPointsConfidence();
for (int i = 0; i < batchSize; i++) {
for (unsigned int j = 0; j < numKeyPoints; j++) {
for (int i = 0; i < output_bboxes.num_boxes; i++) {
for (unsigned int j = 0; j < numKeyPoints; j++) {
average_confidence += keypoints_confidence_all[i * numKeyPoints + j];
}
}
average_confidence /= batchSize * numKeyPoints;
average_confidence /= output_bboxes.num_boxes * numKeyPoints;
return average_confidence;
}
@@ -413,7 +447,7 @@ unsigned BodyEngine::acquireBodyBox(cv::Mat& src, NvAR_Rect& bodyBox, int varian
}
unsigned BodyEngine::acquireBodyBoxAndKeyPoints(cv::Mat& src, NvAR_Point2f* refMarks, NvAR_Point3f* refKeyPoints3D,
NvAR_Quaternion* refJointAngles, NvAR_Rect& bodyBox, int /*variant*/) {
NvAR_Quaternion* refJointAngles, NvAR_BBoxes* refBodyBoxes, int /*variant*/) {
unsigned n = 0;
NvCVImage fxSrcChunkyCPU;
(void)NVWrapperForCVMat(&src, &fxSrcChunkyCPU);
@@ -426,14 +460,15 @@ unsigned BodyEngine::acquireBodyBoxAndKeyPoints(cv::Mat& src, NvAR_Point2f* refM
auto start = std::chrono::high_resolution_clock::now();
#endif
if (findKeyPoints() != NVCV_SUCCESS) return 0;
bodyBox = output_bboxes.boxes[0];
memcpy(refBodyBoxes, getBBoxes(), sizeof(NvAR_BBoxes) );
n = 1;
#ifdef DEBUG_PERF_RUNTIME
auto start2 = std::chrono::high_resolution_clock::now();
#endif
memcpy(refMarks, getKeyPoints(), sizeof(NvAR_Point2f) * numKeyPoints);
memcpy(refKeyPoints3D, getKeyPoints3D(), sizeof(NvAR_Point3f) * numKeyPoints);
memcpy(refJointAngles, getJointAngles(), sizeof(NvAR_Quaternion) * numKeyPoints);
memcpy(refMarks, getKeyPoints(), sizeof(NvAR_Point2f) * numKeyPoints * batchSize);
memcpy(refKeyPoints3D, getKeyPoints3D(), sizeof(NvAR_Point3f) * numKeyPoints * batchSize);
memcpy(refJointAngles, getJointAngles(), sizeof(NvAR_Quaternion) * numKeyPoints * batchSize);
#ifdef DEBUG_PERF_RUNTIME
auto end = std::chrono::high_resolution_clock::now();
@@ -446,13 +481,63 @@ unsigned BodyEngine::acquireBodyBoxAndKeyPoints(cv::Mat& src, NvAR_Point2f* refM
#endif
return n;
}
#if NV_MULTI_OBJECT_TRACKER
unsigned BodyEngine::acquireBodyBoxAndKeyPoints(cv::Mat& src, NvAR_Point2f* refMarks, NvAR_Point3f* refKeyPoints3D,
NvAR_Quaternion* refJointAngles, NvAR_TrackingBBoxes* refBodyBoxes, int /*variant*/) {
unsigned n = 0;
NvCVImage fxSrcChunkyCPU;
(void)NVWrapperForCVMat(&src, &fxSrcChunkyCPU);
NvCV_Status cvErr = NvCVImage_Transfer(&fxSrcChunkyCPU, &inputImageBuffer, 1.0f, stream, &tmpImage);
if (NVCV_SUCCESS != cvErr) {
return n;
}
#ifdef DEBUG_PERF_RUNTIME
auto start = std::chrono::high_resolution_clock::now();
#endif
if (findKeyPoints() != NVCV_SUCCESS) return 0;
memcpy(refBodyBoxes, getTrackingBBoxes(), sizeof(NvAR_TrackingBBoxes));
n = 1;
#ifdef DEBUG_PERF_RUNTIME
auto start2 = std::chrono::high_resolution_clock::now();
#endif
memcpy(refMarks, getKeyPoints(), sizeof(NvAR_Point2f) * numKeyPoints * batchSize);
memcpy(refKeyPoints3D, getKeyPoints3D(), sizeof(NvAR_Point3f) * numKeyPoints * batchSize);
memcpy(refJointAngles, getJointAngles(), sizeof(NvAR_Quaternion) * numKeyPoints * batchSize);
#ifdef DEBUG_PERF_RUNTIME
auto end = std::chrono::high_resolution_clock::now();
auto duration3 = std::chrono::duration_cast<std::chrono::microseconds>(start2 - start);
std::cout << "[bodypose] run findKeyPoints(): " << duration3.count() << " microseconds" << std::endl;
auto duration2 = std::chrono::duration_cast<std::chrono::microseconds>(end - start2);
std::cout << "[bodypose] keypoint copy time: " << duration2.count() << " microseconds" << std::endl;
auto duration = std::chrono::duration_cast<std::chrono::microseconds>(end - start);
std::cout << "[bodypose] end-to-end time: " << duration.count() << " microseconds" << std::endl;
#endif
return n;
}
#endif
void BodyEngine::setBodyStabilization(bool _bStabilizeBody) { bStabilizeBody = _bStabilizeBody; }
void BodyEngine::setMode(int _mode) { nvARMode = _mode; }
void BodyEngine::setFocalLength(float _bFocalLength) { bFocalLength = _bFocalLength; }
BodyEngine::Err BodyEngine::setFocalLength(float _bFocalLength) {
bFocalLength = _bFocalLength;
NvCV_Status nvErr = NvAR_SetF32(keyPointDetectHandle, NvAR_Parameter_Input(FocalLength), bFocalLength);
BodyEngine::Err err = BodyEngine::Err::errNone;
if (nvErr != NVCV_SUCCESS) err = BodyEngine::Err::errParameter;
return err;
}
void BodyEngine::useCudaGraph(bool _bUseCudaGraph) { bUseCudaGraph = _bUseCudaGraph; }
#if NV_MULTI_OBJECT_TRACKER
void BodyEngine::enablePeopleTracking(bool _bEnablePeopleTracking, unsigned int _shadowTrackingAge, unsigned int _probationAge, unsigned int _maxTargetsTracked) {
bEnablePeopleTracking = _bEnablePeopleTracking;
shadowTrackingAge = _shadowTrackingAge;
probationAge = _probationAge;
maxTargetsTracked = _maxTargetsTracked;
}
#endif
void BodyEngine::setAppMode(BodyEngine::mode _mode) { appMode = _mode; }

33
samples/BodyTrack/BodyEngine.h
View File

@@ -141,17 +141,28 @@ class BodyEngine {
NvAR_Point2f* getKeyPoints();
NvAR_Point3f* getKeyPoints3D();
NvAR_Quaternion* getJointAngles();
NvAR_BBoxes* getBBoxes();
#if NV_MULTI_OBJECT_TRACKER
NvAR_TrackingBBoxes* getTrackingBBoxes();
#endif
float* getKeyPointsConfidence();
float getAverageKeyPointsConfidence();
void enlargeAndSquarifyImageBox(float enlarge, NvAR_Rect& box, int FLAG_variant);
unsigned findLargestBodyBox(NvAR_Rect& bodyBox, int variant = 0);
unsigned acquireBodyBox(cv::Mat& src, NvAR_Rect& bodyBox, int variant = 0);
unsigned acquireBodyBoxAndKeyPoints(cv::Mat& src, NvAR_Point2f* refMarks, NvAR_Point3f* refKeyPoints3D,
NvAR_Quaternion* refJointAngles, NvAR_Rect& bodyBox, int variant = 0);
NvAR_Quaternion* refJointAngles, NvAR_BBoxes* refBodyBoxes, int variant = 0);
#if NV_MULTI_OBJECT_TRACKER
unsigned acquireBodyBoxAndKeyPoints(cv::Mat& src, NvAR_Point2f* refMarks, NvAR_Point3f* refKeyPoints3D,
NvAR_Quaternion* refJointAngles, NvAR_TrackingBBoxes* refBodyBoxes, int variant = 0);
#endif
void setBodyStabilization(bool);
void setMode(int);
void setFocalLength(float);
BodyEngine::Err setFocalLength(float);
void useCudaGraph(bool); // Using cuda graph improves model latency
#if NV_MULTI_OBJECT_TRACKER
void enablePeopleTracking(bool _bEnablePeopleTracking, unsigned int _shadowTrackingAge = 90, unsigned int _probationAge = 10, unsigned int _maxTargetsTracked = 30);
#endif
int getNumKeyPoints() { return numKeyPoints; }
std::vector<NvAR_Point3f> getReferencePose() { return referencePose; }
@@ -165,6 +176,11 @@ class BodyEngine {
std::vector<NvAR_Rect> output_bbox_data;
std::vector<float> output_bbox_conf_data;
NvAR_BBoxes output_bboxes{};
#if NV_MULTI_OBJECT_TRACKER
NvAR_TrackingBBoxes output_tracking_bboxes{};
std::vector<NvAR_TrackingBBox> output_tracking_bbox_data;
#endif
int batchSize;
int nvARMode;
std::mt19937 ran;
@@ -178,12 +194,23 @@ class BodyEngine {
char *bdOTAModelPath, *ldOTAModelPath;
float bFocalLength;
bool bUseCudaGraph;
#if NV_MULTI_OBJECT_TRACKER
bool bEnablePeopleTracking;
unsigned int shadowTrackingAge;
unsigned int probationAge;
unsigned int maxTargetsTracked;
#endif
BodyEngine() {
batchSize = 1;
nvARMode = 1;
bStabilizeBody = true;
bUseCudaGraph = true;
#if NV_MULTI_OBJECT_TRACKER
bEnablePeopleTracking = false;
shadowTrackingAge = 90;
probationAge = 10;
maxTargetsTracked = 30;
#endif
bFocalLength = FOCAL_LENGTH_DEFAULT;
confidenceThreshold = 0.f;
appMode = keyPointDetection;

532
samples/BodyTrack/BodyTrack.cpp
View File

@@ -37,6 +37,13 @@
#include "nvAR_defs.h"
#include "opencv2/opencv.hpp"
#if CV_MAJOR_VERSION >= 4
#define CV_CAP_PROP_FRAME_WIDTH cv::CAP_PROP_FRAME_WIDTH
#define CV_CAP_PROP_FRAME_HEIGHT cv::CAP_PROP_FRAME_HEIGHT
#define CV_CAP_PROP_FPS cv::CAP_PROP_FPS
#define CV_CAP_PROP_FRAME_COUNT cv::CAP_PROP_FRAME_COUNT
#endif
#ifndef M_PI
#define M_PI 3.1415926535897932385
#endif /* M_PI */
@@ -62,6 +69,8 @@
#define DEBUG_RUNTIME
#define PEOPLE_TRACKING_BATCH_SIZE 8
/********************************************************************************
* Command-line arguments
********************************************************************************/
@@ -71,7 +80,10 @@ bool FLAG_debug = false, FLAG_verbose = false, FLAG_temporal = true, FLAG_captur
std::string FLAG_outDir, FLAG_inFile, FLAG_outFile, FLAG_modelPath, FLAG_captureCodec = "avc1",
FLAG_camRes, FLAG_bodyModel;
unsigned int FLAG_appMode = 1, FLAG_mode = 1, FLAG_camindex=0;
#if NV_MULTI_OBJECT_TRACKER
bool FLAG_enablePeopleTracking = false;
unsigned int FLAG_shadowTrackingAge = 90, FLAG_probationAge = 10, FLAG_maxTargetsTracked = 30;
#endif
/********************************************************************************
* Usage
********************************************************************************/
@@ -95,6 +107,12 @@ static void Usage() {
" --model_path=<path> specify the directory containing the TRT models\n"
" --mode[=0|1] Model Mode. 0: High Quality, 1: High Performance\n"
" --app_mode[=(0|1)] App mode. 0: Body detection, 1: Keypoint detection "
#if NV_MULTI_OBJECT_TRACKER
" --enable_people_tracking[=(0|1)] Enables people tracking "
" --shadow_tracking_age Shadow Tracking Age after which tracking information of a person is removed. Measured in frames"
" --probation_age Length of probationary period. Measured in frames"
" --max_targets_tracked Maximum number of targets to be tracked "
#endif
"(Default).\n"
" --benchmarks[=<pattern>] run benchmarks\n");
}
@@ -194,6 +212,12 @@ static int ParseMyArgs(int argc, char **argv) {
GetFlagArgVal("mode", arg, &FLAG_mode) ||
GetFlagArgVal("camindex", arg, &FLAG_camindex) ||
GetFlagArgVal("use_cuda_graph", arg, &FLAG_useCudaGraph) ||
#if NV_MULTI_OBJECT_TRACKER
GetFlagArgVal("enable_people_tracking", arg, &FLAG_enablePeopleTracking) ||
GetFlagArgVal("shadow_tracking_age", arg, &FLAG_shadowTrackingAge) ||
GetFlagArgVal("probation_age", arg, &FLAG_probationAge) ||
GetFlagArgVal("max_targets_tracked", arg, &FLAG_maxTargetsTracked) ||
#endif
GetFlagArgVal("temporal", arg, &FLAG_temporal))) {
continue;
} else if (GetFlagArgVal("help", arg, &help)) {
@@ -231,10 +255,11 @@ enum {
#if 1
class MyTimer {
public:
void start() { t0 = std::chrono::high_resolution_clock::now(); } /**< Start the timer. */
void pause() { dt = std::chrono::high_resolution_clock::now() - t0; } /**< Pause the timer. */
void resume() { t0 = std::chrono::high_resolution_clock::now() - dt; } /**< Resume the timer. */
void stop() { pause(); } /**< Stop the timer. */
MyTimer() { dt = dt.zero(); } /**< Clear the duration to 0. */
void start() { t0 = std::chrono::high_resolution_clock::now(); } /**< Start the timer. */
void pause() { dt = std::chrono::high_resolution_clock::now() - t0; } /**< Pause the timer. */
void resume() { t0 = std::chrono::high_resolution_clock::now() - dt; } /**< Resume the timer. */
void stop() { pause(); } /**< Stop the timer. */
double elapsedTimeFloat() const {
return std::chrono::duration<double>(dt).count();
} /**< Report the elapsed time as a float. */
@@ -310,14 +335,23 @@ class DoApp {
Err run();
void drawFPS(cv::Mat &img);
void DrawBBoxes(const cv::Mat &src, NvAR_Rect *output_bbox);
//TODO: Look into ways of simplifying the app for these functions.
void DrawBBoxes(const cv::Mat &src, NvAR_BBoxes *output_bbox);
void DrawBBoxes(const cv::Mat &src, NvAR_TrackingBBoxes *output_bbox);
void DrawKeyPointLine(const cv::Mat& src, NvAR_Point2f* keypoints, int point1, int point2, int color);
void DrawKeyPointsAndEdges(const cv::Mat &src, NvAR_Point2f *keypoints, int numKeyPoints, NvAR_Rect* output_bbox);
void DrawKeyPointsAndEdges(const cv::Mat &src, NvAR_Point2f *keypoints, int numKeyPoints, NvAR_BBoxes* output_bbox);
void drawKalmanStatus(cv::Mat &img);
void drawVideoCaptureStatus(cv::Mat &img);
void processKey(int key);
void writeVideoAndEstResults(const cv::Mat &frame, NvAR_BBoxes output_bboxes, NvAR_Point2f *keypoints = NULL);
void writeFrameAndEstResults(const cv::Mat &frame, NvAR_BBoxes output_bboxes, NvAR_Point2f *keypoints = NULL);
void writeEstResults(std::ofstream &outputFile, NvAR_BBoxes output_bboxes, NvAR_Point2f *keypoints = NULL);
#if NV_MULTI_OBJECT_TRACKER
void DrawKeyPointsAndEdges(const cv::Mat &src, NvAR_Point2f *keypoints, int numKeyPoints, NvAR_TrackingBBoxes* output_bbox);
void writeVideoAndEstResults(const cv::Mat &frame, NvAR_TrackingBBoxes output_bboxes, NvAR_Point2f *keypoints = NULL);
void writeFrameAndEstResults(const cv::Mat &frame, NvAR_TrackingBBoxes output_bboxes, NvAR_Point2f *keypoints = NULL);
void writeEstResults(std::ofstream &outputFile, NvAR_TrackingBBoxes output_bboxes, NvAR_Point2f *keypoints = NULL);
#endif
void getFPS();
static const char *errorStringFromCode(Err code);
@@ -337,6 +371,10 @@ class DoApp {
float expr[6];
bool drawVisualization, showFPS, captureVideo, captureFrame;
float scaleOffsetXY[4];
#if NV_MULTI_OBJECT_TRACKER
std::vector<cv::Scalar> colorCodes = { cv::Scalar(255,255,255) };
const unsigned int peopleTrackingBatchSize = 8; // Batch Size has to be 8 when people tracking is enabled
#endif
};
DoApp *gApp = nullptr;
@@ -347,13 +385,17 @@ void DoApp::processKey(int key) {
case '2':
body_ar_engine.destroyFeatures();
body_ar_engine.setAppMode(BodyEngine::mode::keyPointDetection);
body_ar_engine.createFeatures(FLAG_modelPath.c_str());
#if NV_MULTI_OBJECT_TRACKER
if (FLAG_enablePeopleTracking) body_ar_engine.createFeatures(FLAG_modelPath.c_str());
else
#endif
body_ar_engine.createFeatures(FLAG_modelPath.c_str(), 1);
body_ar_engine.initFeatureIOParams();
break;
case '1':
body_ar_engine.destroyFeatures();
body_ar_engine.setAppMode(BodyEngine::mode::bodyDetection);
body_ar_engine.createFeatures(FLAG_modelPath.c_str());
body_ar_engine.createFeatures(FLAG_modelPath.c_str(), 1);
body_ar_engine.initFeatureIOParams();
break;
case 'C':
@@ -381,8 +423,11 @@ DoApp::Err DoApp::initBodyEngine(const char *modelPath) {
if (!cap.isOpened()) return errVideo;
int numKeyPoints = body_ar_engine.getNumKeyPoints();
nvErr = body_ar_engine.createFeatures(modelPath);
#if NV_MULTI_OBJECT_TRACKER
if (FLAG_enablePeopleTracking) nvErr = body_ar_engine.createFeatures(modelPath, peopleTrackingBatchSize);
else
#endif
nvErr = body_ar_engine.createFeatures(modelPath, 1);
#ifdef DEBUG
detector->setOutputLocation(outputDir);
@@ -425,6 +470,51 @@ void DoApp::DrawBBoxes(const cv::Mat &src, NvAR_Rect *output_bbox) {
if (FLAG_offlineMode) bodyDetectOutputVideo.write(frm);
}
void DoApp::DrawBBoxes(const cv::Mat &src, NvAR_BBoxes *output_bbox) {
cv::Mat frm;
if (FLAG_offlineMode)
frm = src.clone();
else
frm = src;
if (output_bbox) {
for (int i = 0; i < output_bbox->num_boxes; i++) {
auto color = cv::Scalar(255, 255, 255);
cv::rectangle(frm, cv::Point(lround(output_bbox->boxes[i].x), lround(output_bbox->boxes[i].y)),
cv::Point(lround(output_bbox->boxes[i].x + output_bbox->boxes[i].width), lround(output_bbox->boxes[i].y + output_bbox->boxes[i].height)),
cv::Scalar(255, 0, 0), 2);
}
}
if (FLAG_offlineMode) bodyDetectOutputVideo.write(frm);
}
#if NV_MULTI_OBJECT_TRACKER
void DoApp::DrawBBoxes(const cv::Mat &src, NvAR_TrackingBBoxes *output_bbox) {
cv::Mat frm;
if (FLAG_offlineMode)
frm = src.clone();
else
frm = src;
if (output_bbox) {
for (int i = 0; i < output_bbox->num_boxes; i++) {
if (colorCodes.size() <= output_bbox->boxes[i].tracking_id)
colorCodes.push_back(cv::Scalar(rand() & 0xFF, rand() & 0xFF, rand() & 0xFF));
auto color = colorCodes[output_bbox->boxes[i].tracking_id];
std::string text = "ID: " + std::to_string(output_bbox->boxes[i].tracking_id);
cv::rectangle(frm, cv::Point(lround(output_bbox->boxes[i].bbox.x), lround(output_bbox->boxes[i].bbox.y)),
cv::Point(lround(output_bbox->boxes[i].bbox.x + output_bbox->boxes[i].bbox.width), lround(output_bbox->boxes[i].bbox.y + output_bbox->boxes[i].bbox.height)),
color, 2);
cv::putText(frm, text, cv::Point(lround(output_bbox->boxes[i].bbox.x), lround(output_bbox->boxes[i].bbox.y) - 10), cv::FONT_HERSHEY_SIMPLEX, 0.9, color, 2);
}
}
if (FLAG_offlineMode) bodyDetectOutputVideo.write(frm);
}
#endif
void DoApp::writeVideoAndEstResults(const cv::Mat &frm, NvAR_BBoxes output_bboxes, NvAR_Point2f* keypoints) {
if (captureVideo) {
if (!capturedVideo.isOpened()) {
@@ -469,7 +559,54 @@ void DoApp::writeVideoAndEstResults(const cv::Mat &frm, NvAR_BBoxes output_bboxe
}
}
}
#if NV_MULTI_OBJECT_TRACKER
void DoApp::writeVideoAndEstResults(const cv::Mat &frm, NvAR_TrackingBBoxes output_bboxes, NvAR_Point2f* keypoints) {
if (captureVideo) {
if (!capturedVideo.isOpened()) {
const std::string currentCalendarTime = getCalendarTime();
const std::string capturedOutputFileName = currentCalendarTime + ".mp4";
getFPS();
if (frameTime) {
float fps = (float)(1.0 / frameTime);
capturedVideo.open(capturedOutputFileName, StringToFourcc(FLAG_captureCodec), fps,
cv::Size(frm.cols, frm.rows));
if (!capturedVideo.isOpened()) {
std::cout << "Error: Could not open video: \"" << capturedOutputFileName << "\"\n";
return;
}
if (FLAG_verbose) {
std::cout << "Capturing video started" << std::endl;
}
}
else { // If frameTime is 0.f, returns without writing the frame to the Video
return;
}
const std::string outputsFileName = currentCalendarTime + ".txt";
bodyEngineVideoOutputFile.open(outputsFileName, std::ios_base::out);
if (!bodyEngineVideoOutputFile.is_open()) {
std::cout << "Error: Could not open file: \"" << outputsFileName << "\"\n";
return;
}
std::string keyPointDetectionMode = (keypoints == NULL) ? "Off" : "On";
bodyEngineVideoOutputFile << "// BodyDetectOn, KeyPointDetect" << keyPointDetectionMode << "\n ";
bodyEngineVideoOutputFile
<< "// kNumPeople, (bbox_x, bbox_y, bbox_w, bbox_h){ kNumPeople}, kNumLMs, [lm_x, lm_y]{kNumLMs}\n";
}
// Write each frame to the Video
capturedVideo << frm;
writeEstResults(bodyEngineVideoOutputFile, output_bboxes, keypoints);
}
else {
if (capturedVideo.isOpened()) {
if (FLAG_verbose) {
std::cout << "Capturing video ended" << std::endl;
}
capturedVideo.release();
if (bodyEngineVideoOutputFile.is_open()) bodyEngineVideoOutputFile.close();
}
}
}
#endif
void DoApp::writeEstResults(std::ofstream &outputFile, NvAR_BBoxes output_bboxes, NvAR_Point2f* keypoints) {
/**
* Output File Format :
@@ -512,7 +649,53 @@ void DoApp::writeEstResults(std::ofstream &outputFile, NvAR_BBoxes output_bboxes
outputFile << "\n";
}
#if NV_MULTI_OBJECT_TRACKER
void DoApp::writeEstResults(std::ofstream &outputFile, NvAR_TrackingBBoxes output_bboxes, NvAR_Point2f* keypoints) {
/**
* Output File Format :
* BodyDetectOn, KeyPointDetectOn
* kNumPeople, (bbox_x, bbox_y, bbox_w, bbox_h){ kNumPeople}, kNumKPs, [j_x, j_y]{kNumKPs}
*/
int bodyDetectOn = (body_ar_engine.appMode == BodyEngine::mode::bodyDetection ||
body_ar_engine.appMode == BodyEngine::mode::keyPointDetection)
? 1
: 0;
int keyPointDetectOn = (body_ar_engine.appMode == BodyEngine::mode::keyPointDetection)
? 1
: 0;
outputFile << bodyDetectOn << "," << keyPointDetectOn << "\n";
if (bodyDetectOn && output_bboxes.num_boxes) {
// Append number of bodies detected in the current frame
outputFile << unsigned(output_bboxes.num_boxes) << ",";
// write outputbboxes to outputFile
for (size_t i = 0; i < output_bboxes.num_boxes; i++) {
int x1 = (int)output_bboxes.boxes[i].bbox.x, y1 = (int)output_bboxes.boxes[i].bbox.y,
width = (int)output_bboxes.boxes[i].bbox.width, height = (int)output_bboxes.boxes[i].bbox.height;
unsigned int tracking_id = output_bboxes.boxes[i].tracking_id;
outputFile << x1 << "," << y1 << "," << width << "," << height << "," << tracking_id << ",";
}
}
else {
outputFile << "0,";
}
if (keyPointDetectOn && output_bboxes.num_boxes) {
int numKeyPoints = body_ar_engine.getNumKeyPoints();
// Append number of keypoints
outputFile << numKeyPoints << ",";
// Append 2 * number of keypoint values
NvAR_Point2f *pt, *endPt;
for (endPt = (pt = (NvAR_Point2f *)keypoints) + numKeyPoints; pt < endPt; ++pt)
outputFile << pt->x << "," << pt->y << ",";
}
else {
outputFile << "0,";
}
outputFile << "\n";
}
#endif
void DoApp::writeFrameAndEstResults(const cv::Mat &frm, NvAR_BBoxes output_bboxes, NvAR_Point2f* keypoints) {
if (captureFrame) {
const std::string currentCalendarTime = getCalendarTime();
@@ -537,28 +720,47 @@ void DoApp::writeFrameAndEstResults(const cv::Mat &frm, NvAR_BBoxes output_bboxe
captureFrame = false;
}
}
#if NV_MULTI_OBJECT_TRACKER
void DoApp::writeFrameAndEstResults(const cv::Mat &frm, NvAR_TrackingBBoxes output_bboxes, NvAR_Point2f* keypoints) {
if (captureFrame) {
const std::string currentCalendarTime = getCalendarTime();
const std::string capturedFrame = currentCalendarTime + ".png";
cv::imwrite(capturedFrame, frm);
if (FLAG_verbose) {
std::cout << "Captured the frame" << std::endl;
}
// Write Body Engine Outputs
const std::string outputFilename = currentCalendarTime + ".txt";
std::ofstream outputFile;
outputFile.open(outputFilename, std::ios_base::out);
if (!outputFile.is_open()) {
std::cout << "Error: Could not open file: \"" << outputFilename << "\"\n";
return;
}
std::string keyPointDetectionMode = (keypoints == NULL) ? "Off" : "On";
outputFile << "// BodyDetectOn, KeyPointDetect" << keyPointDetectionMode << "\n";
outputFile << "// kNumPeople, (bbox_x, bbox_y, bbox_w, bbox_h){ kNumPeople}, kNumLMs, [lm_x, lm_y]{kNumLMs}\n";
writeEstResults(outputFile, output_bboxes, keypoints);
if (outputFile.is_open()) outputFile.close();
captureFrame = false;
}
}
#endif
void DoApp::DrawKeyPointLine(const cv::Mat& src, NvAR_Point2f* keypoints, int point1, int point2, int color) {
NvAR_Point2f point1_pos = *(keypoints + point1);
NvAR_Point2f point2_pos = *(keypoints + point2);
cv::line(src, cv::Point((int)point1_pos.x, (int)point1_pos.y), cv::Point((int)point2_pos.x, (int)point2_pos.y), cv_colors[color], 2);
}
void DoApp::DrawKeyPointsAndEdges(const cv::Mat& src, NvAR_Point2f* keypoints, int numKeyPoints, NvAR_Rect* output_bbox) {
#if NV_MULTI_OBJECT_TRACKER
void DoApp::DrawKeyPointsAndEdges(const cv::Mat& src, NvAR_Point2f* keypoints, int numKeyPoints, NvAR_TrackingBBoxes* output_bbox) {
cv::Mat frm;
if (FLAG_offlineMode)
frm = src.clone();
else
frm = src;
NvAR_Point2f *pt, *endPt;
for (endPt = (pt = (NvAR_Point2f *)keypoints) + numKeyPoints; pt < endPt; ++pt)
cv::circle(frm, cv::Point(lround(pt->x), lround(pt->y)), 4, cv::Scalar(180, 180, 180), -1);
if (output_bbox)
cv::rectangle(frm, cv::Point(lround(output_bbox->x), lround(output_bbox->y)),
cv::Point(lround(output_bbox->x + output_bbox->width), lround(output_bbox->y + output_bbox->height)),
cv::Scalar(255, 0, 0), 2);
NvAR_Point2f* keypointsBatch8 = keypoints;
int pelvis = 0;
int left_hip = 1;
@@ -595,56 +797,187 @@ void DoApp::DrawKeyPointsAndEdges(const cv::Mat& src, NvAR_Point2f* keypoints, i
int left_thumb_tip = 32;
int right_thumb_tip = 33;
// center body
DrawKeyPointLine(frm, keypoints, pelvis, torso, kColorGreen);
DrawKeyPointLine(frm, keypoints, torso, neck, kColorGreen);
DrawKeyPointLine(frm, keypoints, neck, pelvis, kColorGreen);
for (int i = 0; i < body_ar_engine.output_tracking_bboxes.num_boxes; i++) {
// right side
DrawKeyPointLine(frm, keypoints, right_ankle, right_knee, kColorRed);
DrawKeyPointLine(frm, keypoints, right_knee, right_hip, kColorRed);
DrawKeyPointLine(frm, keypoints, right_hip, pelvis, kColorRed);
DrawKeyPointLine(frm, keypoints, right_hip, right_shoulder, kColorRed);
DrawKeyPointLine(frm, keypoints, right_shoulder, right_elbow, kColorRed);
DrawKeyPointLine(frm, keypoints, right_elbow, right_wrist, kColorRed);
DrawKeyPointLine(frm, keypoints, right_shoulder, neck, kColorRed);
keypoints = keypointsBatch8 + (i * 34);
// right side hand and feet
DrawKeyPointLine(frm, keypoints, right_wrist, right_pinky_knuckle, kColorRed);
DrawKeyPointLine(frm, keypoints, right_wrist, right_middle_tip, kColorRed);
DrawKeyPointLine(frm, keypoints, right_wrist, right_index_knuckle, kColorRed);
DrawKeyPointLine(frm, keypoints, right_wrist, right_thumb_tip, kColorRed);
DrawKeyPointLine(frm, keypoints, right_ankle, right_heel, kColorRed);
DrawKeyPointLine(frm, keypoints, right_ankle, right_big_toe, kColorRed);
DrawKeyPointLine(frm, keypoints, right_big_toe, right_small_toe, kColorRed);
for (endPt = (pt = (NvAR_Point2f*)keypoints) + numKeyPoints; pt < endPt; ++pt)
cv::circle(frm, cv::Point(lround(pt->x), lround(pt->y)), 4, cv::Scalar(180, 180, 180), -1);
//left side
DrawKeyPointLine(frm, keypoints, left_ankle, left_knee, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_knee, left_hip, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_hip, pelvis, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_hip, left_shoulder, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_shoulder, left_elbow, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_elbow, left_wrist, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_shoulder, neck, kColorBlue);
if (output_bbox) {
while (colorCodes.size()<= output_bbox->boxes[i].tracking_id)
colorCodes.push_back(cv::Scalar(rand() & 0xFF, rand() & 0xFF, rand() & 0xFF));
auto color = colorCodes[output_bbox->boxes[i].tracking_id];
std::string text = "ID: "+std::to_string(output_bbox->boxes[i].tracking_id);
cv::rectangle(frm, cv::Point(lround(output_bbox->boxes[i].bbox.x), lround(output_bbox->boxes[i].bbox.y)),
cv::Point(lround(output_bbox->boxes[i].bbox.x + output_bbox->boxes[i].bbox.width), lround(output_bbox->boxes[i].bbox.y + output_bbox->boxes[i].bbox.height)),
color, 2);
cv::putText(frm, text, cv::Point(lround(output_bbox->boxes[i].bbox.x), lround(output_bbox->boxes[i].bbox.y) - 10), cv::FONT_HERSHEY_SIMPLEX, 0.5, color, 2);
}
// left side hand and feet
DrawKeyPointLine(frm, keypoints, left_wrist, left_pinky_knuckle, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_wrist, left_middle_tip, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_wrist, left_index_knuckle, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_wrist, left_thumb_tip, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_ankle, left_heel, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_ankle, left_big_toe, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_big_toe, left_small_toe, kColorBlue);
// center body
DrawKeyPointLine(frm, keypoints, pelvis, torso, kColorGreen);
DrawKeyPointLine(frm, keypoints, torso, neck, kColorGreen);
DrawKeyPointLine(frm, keypoints, neck, pelvis, kColorGreen);
// head
DrawKeyPointLine(frm, keypoints, neck, nose, kColorGreen);
DrawKeyPointLine(frm, keypoints, nose, right_eye, kColorGreen);
DrawKeyPointLine(frm, keypoints, right_eye, right_ear, kColorGreen);
DrawKeyPointLine(frm, keypoints, nose, left_eye, kColorGreen);
DrawKeyPointLine(frm, keypoints, left_eye, left_ear, kColorGreen);
// right side
DrawKeyPointLine(frm, keypoints, right_ankle, right_knee, kColorRed);
DrawKeyPointLine(frm, keypoints, right_knee, right_hip, kColorRed);
DrawKeyPointLine(frm, keypoints, right_hip, pelvis, kColorRed);
DrawKeyPointLine(frm, keypoints, right_hip, right_shoulder, kColorRed);
DrawKeyPointLine(frm, keypoints, right_shoulder, right_elbow, kColorRed);
DrawKeyPointLine(frm, keypoints, right_elbow, right_wrist, kColorRed);
DrawKeyPointLine(frm, keypoints, right_shoulder, neck, kColorRed);
// right side hand and feet
DrawKeyPointLine(frm, keypoints, right_wrist, right_pinky_knuckle, kColorRed);
DrawKeyPointLine(frm, keypoints, right_wrist, right_middle_tip, kColorRed);
DrawKeyPointLine(frm, keypoints, right_wrist, right_index_knuckle, kColorRed);
DrawKeyPointLine(frm, keypoints, right_wrist, right_thumb_tip, kColorRed);
DrawKeyPointLine(frm, keypoints, right_ankle, right_heel, kColorRed);
DrawKeyPointLine(frm, keypoints, right_ankle, right_big_toe, kColorRed);
DrawKeyPointLine(frm, keypoints, right_big_toe, right_small_toe, kColorRed);
//left side
DrawKeyPointLine(frm, keypoints, left_ankle, left_knee, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_knee, left_hip, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_hip, pelvis, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_hip, left_shoulder, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_shoulder, left_elbow, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_elbow, left_wrist, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_shoulder, neck, kColorBlue);
// left side hand and feet
DrawKeyPointLine(frm, keypoints, left_wrist, left_pinky_knuckle, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_wrist, left_middle_tip, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_wrist, left_index_knuckle, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_wrist, left_thumb_tip, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_ankle, left_heel, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_ankle, left_big_toe, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_big_toe, left_small_toe, kColorBlue);
// head
DrawKeyPointLine(frm, keypoints, neck, nose, kColorGreen);
DrawKeyPointLine(frm, keypoints, nose, right_eye, kColorGreen);
DrawKeyPointLine(frm, keypoints, right_eye, right_ear, kColorGreen);
DrawKeyPointLine(frm, keypoints, nose, left_eye, kColorGreen);
DrawKeyPointLine(frm, keypoints, left_eye, left_ear, kColorGreen);
}
if (FLAG_offlineMode) keyPointsOutputVideo.write(frm);
}
#endif
void DoApp::DrawKeyPointsAndEdges(const cv::Mat& src, NvAR_Point2f* keypoints, int numKeyPoints, NvAR_BBoxes* output_bbox) {
cv::Mat frm;
if (FLAG_offlineMode)
frm = src.clone();
else
frm = src;
NvAR_Point2f *pt, *endPt;
NvAR_Point2f* keypointsBatch8 = keypoints;
int pelvis = 0;
int left_hip = 1;
int right_hip = 2;
int torso = 3;
int left_knee = 4;
int right_knee = 5;
int neck = 6;
int left_ankle = 7;
int right_ankle = 8;
int left_big_toe = 9;
int right_big_toe = 10;
int left_small_toe = 11;
int right_small_toe = 12;
int left_heel = 13;
int right_heel = 14;
int nose = 15;
int left_eye = 16;
int right_eye = 17;
int left_ear = 18;
int right_ear = 19;
int left_shoulder = 20;
int right_shoulder = 21;
int left_elbow = 22;
int right_elbow = 23;
int left_wrist = 24;
int right_wrist = 25;
int left_pinky_knuckle = 26;
int right_pinky_knuckle = 27;
int left_middle_tip = 28;
int right_middle_tip = 29;
int left_index_knuckle = 30;
int right_index_knuckle = 31;
int left_thumb_tip = 32;
int right_thumb_tip = 33;
for (int i = 0; i < body_ar_engine.output_bboxes.num_boxes; i++) {
keypoints = keypointsBatch8 + (i * 34);
for (endPt = (pt = (NvAR_Point2f*)keypoints) + numKeyPoints; pt < endPt; ++pt)
cv::circle(frm, cv::Point(lround(pt->x), lround(pt->y)), 4, cv::Scalar(180, 180, 180), -1);
if (output_bbox) {
cv::rectangle(frm, cv::Point(lround(output_bbox->boxes[i].x), lround(output_bbox->boxes[i].y)),
cv::Point(lround(output_bbox->boxes[i].x + output_bbox->boxes[i].width), lround(output_bbox->boxes[i].y + output_bbox->boxes[i].height)),
cv::Scalar(255, 0, 0), 2);
}
// center body
DrawKeyPointLine(frm, keypoints, pelvis, torso, kColorGreen);
DrawKeyPointLine(frm, keypoints, torso, neck, kColorGreen);
DrawKeyPointLine(frm, keypoints, neck, pelvis, kColorGreen);
// right side
DrawKeyPointLine(frm, keypoints, right_ankle, right_knee, kColorRed);
DrawKeyPointLine(frm, keypoints, right_knee, right_hip, kColorRed);
DrawKeyPointLine(frm, keypoints, right_hip, pelvis, kColorRed);
DrawKeyPointLine(frm, keypoints, right_hip, right_shoulder, kColorRed);
DrawKeyPointLine(frm, keypoints, right_shoulder, right_elbow, kColorRed);
DrawKeyPointLine(frm, keypoints, right_elbow, right_wrist, kColorRed);
DrawKeyPointLine(frm, keypoints, right_shoulder, neck, kColorRed);
// right side hand and feet
DrawKeyPointLine(frm, keypoints, right_wrist, right_pinky_knuckle, kColorRed);
DrawKeyPointLine(frm, keypoints, right_wrist, right_middle_tip, kColorRed);
DrawKeyPointLine(frm, keypoints, right_wrist, right_index_knuckle, kColorRed);
DrawKeyPointLine(frm, keypoints, right_wrist, right_thumb_tip, kColorRed);
DrawKeyPointLine(frm, keypoints, right_ankle, right_heel, kColorRed);
DrawKeyPointLine(frm, keypoints, right_ankle, right_big_toe, kColorRed);
DrawKeyPointLine(frm, keypoints, right_big_toe, right_small_toe, kColorRed);
//left side
DrawKeyPointLine(frm, keypoints, left_ankle, left_knee, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_knee, left_hip, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_hip, pelvis, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_hip, left_shoulder, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_shoulder, left_elbow, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_elbow, left_wrist, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_shoulder, neck, kColorBlue);
// left side hand and feet
DrawKeyPointLine(frm, keypoints, left_wrist, left_pinky_knuckle, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_wrist, left_middle_tip, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_wrist, left_index_knuckle, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_wrist, left_thumb_tip, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_ankle, left_heel, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_ankle, left_big_toe, kColorBlue);
DrawKeyPointLine(frm, keypoints, left_big_toe, left_small_toe, kColorBlue);
// head
DrawKeyPointLine(frm, keypoints, neck, nose, kColorGreen);
DrawKeyPointLine(frm, keypoints, nose, right_eye, kColorGreen);
DrawKeyPointLine(frm, keypoints, right_eye, right_ear, kColorGreen);
DrawKeyPointLine(frm, keypoints, nose, left_eye, kColorGreen);
DrawKeyPointLine(frm, keypoints, left_eye, left_ear, kColorGreen);
}
if (FLAG_offlineMode) keyPointsOutputVideo.write(frm);
}
DoApp::Err DoApp::acquireFrame() {
Err err = errNone;
@@ -682,8 +1015,20 @@ DoApp::Err DoApp::acquireBodyBox() {
printf("]\n");
}
if (FLAG_captureOutputs) {
writeFrameAndEstResults(frame, body_ar_engine.output_bboxes);
writeVideoAndEstResults(frame, body_ar_engine.output_bboxes);
#if NV_MULTI_OBJECT_TRACKER
if (FLAG_enablePeopleTracking) {
writeFrameAndEstResults(frame, body_ar_engine.output_tracking_bboxes);
writeVideoAndEstResults(frame, body_ar_engine.output_tracking_bboxes);
}
else {
writeFrameAndEstResults(frame, body_ar_engine.output_bboxes);
writeVideoAndEstResults(frame, body_ar_engine.output_bboxes);
}
#else
writeFrameAndEstResults(frame, body_ar_engine.output_bboxes);
writeVideoAndEstResults(frame, body_ar_engine.output_bboxes);
#endif
}
if (0 == n) return errNoBody;
@@ -701,18 +1046,29 @@ DoApp::Err DoApp::acquireBodyBox() {
DoApp::Err DoApp::acquireBodyBoxAndKeyPoints() {
Err err = errNone;
int numKeyPoints = body_ar_engine.getNumKeyPoints();
NvAR_Rect output_bbox;
std::vector<NvAR_Point2f> keypoints2D(numKeyPoints);
std::vector<NvAR_Point3f> keypoints3D(numKeyPoints);
std::vector<NvAR_Quaternion> jointAngles(numKeyPoints);
NvAR_BBoxes output_bbox;
NvAR_TrackingBBoxes output_tracking_bbox;
std::vector<NvAR_Point2f> keypoints2D(numKeyPoints * 8);
std::vector<NvAR_Point3f> keypoints3D(numKeyPoints * 8);
std::vector<NvAR_Quaternion> jointAngles(numKeyPoints * 8);
#ifdef DEBUG_PERF_RUNTIME
auto start = std::chrono::high_resolution_clock::now();
#endif
unsigned n;
// get keypoints in original image resolution coordinate space
unsigned n = body_ar_engine.acquireBodyBoxAndKeyPoints(frame, keypoints2D.data(), keypoints3D.data(),
jointAngles.data(), output_bbox, 0);
#if NV_MULTI_OBJECT_TRACKER
if (FLAG_enablePeopleTracking)
n = body_ar_engine.acquireBodyBoxAndKeyPoints(frame, keypoints2D.data(), keypoints3D.data(),
jointAngles.data(), &output_tracking_bbox, 0);
else
#endif
n = body_ar_engine.acquireBodyBoxAndKeyPoints(frame, keypoints2D.data(), keypoints3D.data(),
jointAngles.data(), &output_bbox, 0);
#ifdef DEBUG_PERF_RUNTIME
auto end = std::chrono::high_resolution_clock::now();
@@ -734,17 +1090,37 @@ DoApp::Err DoApp::acquireBodyBoxAndKeyPoints() {
printf("]\n");
}
if (FLAG_captureOutputs) {
writeFrameAndEstResults(frame, body_ar_engine.output_bboxes, keypoints2D.data());
writeVideoAndEstResults(frame, body_ar_engine.output_bboxes, keypoints2D.data());
#if NV_MULTI_OBJECT_TRACKER
if (FLAG_enablePeopleTracking) {
writeFrameAndEstResults(frame, body_ar_engine.output_tracking_bboxes, keypoints2D.data());
writeVideoAndEstResults(frame, body_ar_engine.output_tracking_bboxes, keypoints2D.data());
}
else {
writeFrameAndEstResults(frame, body_ar_engine.output_bboxes, keypoints2D.data());
writeVideoAndEstResults(frame, body_ar_engine.output_bboxes, keypoints2D.data());
}
#else
writeFrameAndEstResults(frame, body_ar_engine.output_bboxes, keypoints2D.data());
writeVideoAndEstResults(frame, body_ar_engine.output_bboxes, keypoints2D.data());
#endif
}
if (0 == n) return errNoBody;
#ifdef VISUALIZE
if (drawVisualization) {
DrawKeyPointsAndEdges(frame, keypoints2D.data(), numKeyPoints, &output_bbox);
#if NV_MULTI_OBJECT_TRACKER
if (FLAG_enablePeopleTracking) DrawKeyPointsAndEdges(frame, keypoints2D.data(), numKeyPoints, &output_tracking_bbox);
else
#endif
DrawKeyPointsAndEdges(frame, keypoints2D.data(), numKeyPoints, &output_bbox);
if (FLAG_offlineMode) {
DrawBBoxes(frame, &output_bbox);
#if NV_MULTI_OBJECT_TRACKER
if (FLAG_enablePeopleTracking) DrawBBoxes(frame, &output_tracking_bbox);
else
#endif
DrawBBoxes(frame, &output_bbox);
}
}
#endif // VISUALIZE
@@ -920,6 +1296,8 @@ DoApp::Err DoApp::run() {
}
cv::imshow(windowTitle, frame);
}
if (!FLAG_offlineMode) {
int n = cv::waitKey(1);
@@ -991,7 +1369,9 @@ int main(int argc, char **argv) {
if (FLAG_useCudaGraph) printf("Enable capturing cuda graph = %d\n", FLAG_useCudaGraph);
app.body_ar_engine.useCudaGraph(FLAG_useCudaGraph);
#if NV_MULTI_OBJECT_TRACKER
app.body_ar_engine.enablePeopleTracking(FLAG_enablePeopleTracking, FLAG_shadowTrackingAge, FLAG_probationAge, FLAG_maxTargetsTracked);
#endif
doErr = DoApp::errBodyModelInit;
if (FLAG_modelPath.empty()) {
printf("WARNING: Model path not specified. Please set --model_path=/path/to/trt/and/body/models, "

BIN
samples/BodyTrack/BodyTrack.exe
View File

Binary file not shown.

35
samples/BodyTrack/CMakeLists.txt
View File

@@ -4,12 +4,14 @@ set(SOURCE_FILES BodyEngine.cpp
)
set(HEADER_FILES BodyEngine.h)
set(SOURCE_FILES ${SOURCE_FILES}
../../nvar/src/nvARProxy.cpp
../../nvar/src/nvCVImageProxy.cpp)
if(MSVC)
set(SOURCE_FILES ${SOURCE_FILES}
../../nvar/src/nvARProxy.cpp
../../nvar/src/nvCVImageProxy.cpp)
set(HEADER_FILES ${HEADER_FILES}
../utils/RenderingUtils.h)
set(HEADER_FILES ${HEADER_FILES}
../utils/RenderingUtils.h)
endif(MSVC)
# Set Visual Studio source filters
source_group("Source Files" FILES ${SOURCE_FILES})
@@ -21,6 +23,7 @@ target_include_directories(BodyTrack PUBLIC
${SDK_INCLUDES_PATH}
)
if(MSVC)
target_link_libraries(BodyTrack PUBLIC
opencv346
utils_sample
@@ -29,11 +32,21 @@ target_link_libraries(BodyTrack PUBLIC
set(ARSDK_PATH_STR ${CMAKE_CURRENT_SOURCE_DIR}/../../bin)
set(OPENCV_PATH_STR ${CMAKE_CURRENT_SOURCE_DIR}/../external/opencv/bin)
set(PATH_STR "PATH=%PATH%" ${OPENCV_PATH_STR})
set(CMD_ARG_STR "")
set_target_properties(BodyTrack PROPERTIES
FOLDER SampleApps
VS_DEBUGGER_ENVIRONMENT "${PATH_STR}"
VS_DEBUGGER_COMMAND_ARGUMENTS "${CMD_ARG_STR}"
set(CMD_ARG_STR "--model_path=\"${CMAKE_CURRENT_SOURCE_DIR}/../../bin/models\"")
set_target_properties(BodyTrack PROPERTIES
FOLDER SampleApps
VS_DEBUGGER_ENVIRONMENT "${PATH_STR}"
VS_DEBUGGER_COMMAND_ARGUMENTS "${CMD_ARG_STR}"
)
elseif(UNIX)
find_package(PNG REQUIRED)
find_package(JPEG REQUIRED)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread")
target_link_libraries(BodyTrack PUBLIC
nvARPose
NVCVImage
OpenCV
utils_sample
)
endif(MSVC)