demonkenny/MAXINE-AR-SDK
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update to latest version 09/17

Browse Source 
Updating AR SDK to latest version
This commit is contained in:
jdsouza90
2020-09-17 17:17:55 -07:00
parent 5781e23b9d
commit bfd21190df
19 changed files with 604 additions and 251 deletions
Download Patch File Download Diff File Expand all files Collapse all files

195
samples/FaceTrack/FaceTrack.cpp
View File

@@ -63,10 +63,10 @@
********************************************************************************/
bool FLAG_debug = false, FLAG_verbose = false, FLAG_temporal = true, FLAG_captureOutputs = false,
FLAG_offlineMode = false;
FLAG_offlineMode = false, FLAG_isNumLandmarks126 = false;
std::string FLAG_outDir, FLAG_inFile, FLAG_outFile, FLAG_modelPath, FLAG_landmarks, FLAG_proxyWireframe,
FLAG_captureCodec = "avc1", FLAG_camRes;
unsigned int FLAG_batch = 1;
FLAG_captureCodec = "avc1", FLAG_camRes, FLAG_faceModel;
unsigned int FLAG_batch = 1, FLAG_appMode = 2;
/********************************************************************************
* Usage
@@ -88,8 +88,12 @@ static void Usage() {
" --out_file=<file> specify the output file\n"
" --out=<file> specify the output file\n"
" --model_path=<path> specify the directory containing the TRT models\n"
" --landmarks_126[=(true|false)] set the number of facial landmark points to 126, otherwise default to 68\n"
" --face_model=<file> specify the name of the face model\n"
" --wireframe_mesh=<path> specify the path to a proxy wireframe mesh\n"
" --batch=<uint> 1 - 8, used for batch inferencing in landmark detector "
" --batch=<uint> 1 - 8, used for batch inferencing in landmark detector\n"
" --app_mode[=(0|1|2)] App mode. 0: Face detection, 1: Landmark detection, 2: Face fitting "
"(Default)."
" --benchmarks[=<pattern>] run benchmarks\n");
}
@@ -181,11 +185,13 @@ static int ParseMyArgs(int argc, char **argv) {
GetFlagArgVal("in", arg, &FLAG_inFile) || GetFlagArgVal("in_file", arg, &FLAG_inFile) ||
GetFlagArgVal("out", arg, &FLAG_outFile) || GetFlagArgVal("out_file", arg, &FLAG_outFile) ||
GetFlagArgVal("offline_mode", arg, &FLAG_offlineMode) ||
GetFlagArgVal("landmarks_126", arg, &FLAG_isNumLandmarks126) ||
GetFlagArgVal("capture_outputs", arg, &FLAG_captureOutputs) ||
GetFlagArgVal("cam_res", arg, &FLAG_camRes) || GetFlagArgVal("codec", arg, &FLAG_captureCodec) ||
GetFlagArgVal("landmarks", arg, &FLAG_landmarks) || GetFlagArgVal("model_path", arg, &FLAG_modelPath) ||
GetFlagArgVal("wireframe_mesh", arg, &FLAG_proxyWireframe) ||
GetFlagArgVal("temporal", arg, &FLAG_temporal))) {
GetFlagArgVal("face_model", arg, &FLAG_faceModel) ||
GetFlagArgVal("app_mode", arg, &FLAG_appMode) || GetFlagArgVal("temporal", arg, &FLAG_temporal))) {
continue;
} else if (GetFlagArgVal("help", arg, &help)) {
Usage();
@@ -251,7 +257,13 @@ std::string getCalendarTime() {
class DoApp {
public:
enum Err {
errNone,
errNone = FaceEngine::Err::errNone,
errGeneral = FaceEngine::Err::errGeneral,
errRun = FaceEngine::Err::errRun,
errInitialization = FaceEngine::Err::errInitialization,
errRead = FaceEngine::Err::errRead,
errEffect = FaceEngine::Err::errEffect,
errParameter = FaceEngine::Err::errParameter,
errUnimplemented,
errMissing,
errVideo,
@@ -268,15 +280,15 @@ class DoApp {
errSDK,
errCuda,
errCancel,
errInitFaceEngine
errCamera
};
Err doAppErr(FaceEngine::Err status) { return (Err)status; }
FaceEngine face_ar_engine;
DoApp();
~DoApp();
void stop();
Err initFaceEngine(const char *modelPath = nullptr);
Err initFaceEngine(const char *modelPath = nullptr, bool isLandmarks126 = false);
Err initCamera(const char *camRes = nullptr);
Err initOfflineMode(const char *inputFilename = nullptr, const char *outputFilename = nullptr);
Err acquireFrame();
@@ -287,7 +299,7 @@ class DoApp {
void showFaceFitErrorMessage();
void drawFPS(cv::Mat &img);
void DrawBBoxes(const cv::Mat &src, NvAR_Rect *output_bbox);
void DrawLandmarkPoints(const cv::Mat &src, NvAR_Point2f *facial_landmarks);
void DrawLandmarkPoints(const cv::Mat &src, NvAR_Point2f *facial_landmarks, int numLandmarks);
void DrawFaceMesh(const cv::Mat &src, NvAR_FaceMesh *face_mesh);
void drawKalmanStatus(cv::Mat &img);
void drawVideoCaptureStatus(cv::Mat &img);
@@ -320,7 +332,7 @@ class DoApp {
};
DoApp *gApp = nullptr;
const char DoApp::windowTitle[] = "WINDOW";
const char DoApp::windowTitle[] = "FaceTrack App";
void DoApp::processKey(int key) {
switch (key) {
@@ -368,21 +380,23 @@ void DoApp::processKey(int key) {
}
}
DoApp::Err DoApp::initFaceEngine(const char *modelPath) {
DoApp::Err DoApp::initFaceEngine(const char *modelPath, bool isNumLandmarks126) {
Err err = errNone;
if (!cap.isOpened()) return errVideo;
int numLandmarkPoints = isNumLandmarks126 ? 126 : 68;
face_ar_engine.setNumLandmarks(numLandmarkPoints);
nvErr = face_ar_engine.createFeatures(modelPath);
if (nvErr != FaceEngine::Err::errNone) {
if (nvErr == FaceEngine::Err::errInitialization && face_ar_engine.appMode == FaceEngine::mode::faceMeshGeneration) {
showFaceFitErrorMessage();
printf("WARNING: face fitting has failed, trying to initialize Landmark Detection\n");
face_ar_engine.destroyFeatures();
face_ar_engine.setAppMode(FaceEngine::mode::landmarkDetection);
nvErr = face_ar_engine.createFeatures(modelPath);
}
if (nvErr != FaceEngine::Err::errNone)
err = errInitFaceEngine;
}
#ifdef DEBUG
@@ -396,7 +410,7 @@ DoApp::Err DoApp::initFaceEngine(const char *modelPath) {
frameIndex = 0;
return err;
return doAppErr(nvErr);
}
void DoApp::stop() {
@@ -428,29 +442,29 @@ void DoApp::showFaceFitErrorMessage() {
}
void DoApp::DrawBBoxes(const cv::Mat &src, NvAR_Rect *output_bbox) {
cv::Mat frame;
cv::Mat frm;
if (FLAG_offlineMode)
frame = src.clone();
frm = src.clone();
else
frame = src;
frm = src;
if (output_bbox)
cv::rectangle(frame, cv::Point((int)output_bbox->x, (int)output_bbox->y),
cv::Point((int)output_bbox->x + output_bbox->width, (int)output_bbox->y + output_bbox->height),
cv::rectangle(frm, cv::Point(lround(output_bbox->x), lround(output_bbox->y)),
cv::Point(lround(output_bbox->x + output_bbox->width), lround(output_bbox->y + output_bbox->height)),
cv::Scalar(255, 0, 0), 2);
if (FLAG_offlineMode) faceDetectOutputVideo.write(frame);
if (FLAG_offlineMode) faceDetectOutputVideo.write(frm);
}
void DoApp::writeVideoAndEstResults(const cv::Mat &frame, NvAR_BBoxes output_bboxes, NvAR_Point2f *landmarks) {
void DoApp::writeVideoAndEstResults(const cv::Mat &frm, NvAR_BBoxes output_bboxes, NvAR_Point2f *landmarks) {
if (captureVideo) {
if (!capturedVideo.isOpened()) {
const std::string currentCalendarTime = getCalendarTime();
const std::string capturedOutputFileName = currentCalendarTime + ".mp4";
getFPS();
if (frameTime) {
float fps = 1. / frameTime;
float fps = (float)(1.0 / frameTime);
capturedVideo.open(capturedOutputFileName, StringToFourcc(FLAG_captureCodec), fps,
cv::Size(frame.cols, frame.rows));
cv::Size(frm.cols, frm.rows));
if (!capturedVideo.isOpened()) {
std::cout << "Error: Could not open video: \"" << capturedOutputFileName << "\"\n";
return;
@@ -473,7 +487,7 @@ void DoApp::writeVideoAndEstResults(const cv::Mat &frame, NvAR_BBoxes output_bbo
<< "// kNumFaces, (bbox_x, bbox_y, bbox_w, bbox_h){ kNumFaces}, kNumLMs, [lm_x, lm_y]{kNumLMs}\n";
}
// Write each frame to the Video
capturedVideo << frame;
capturedVideo << frm;
writeEstResults(faceEngineVideoOutputFile, output_bboxes, landmarks);
} else {
if (capturedVideo.isOpened()) {
@@ -517,11 +531,12 @@ void DoApp::writeEstResults(std::ofstream &outputFile, NvAR_BBoxes output_bboxes
outputFile << "0,";
}
if (landmarkDetectOn && output_bboxes.num_boxes) {
int numLandmarks = face_ar_engine.getNumLandmarks();
// Append number of landmarks
outputFile << FaceEngine::NUM_LANDMARKS << ",";
// Append NUM_LANDMARKS * 2 points
outputFile << numLandmarks << ",";
// Append 2 * number of landmarks values
NvAR_Point2f *pt, *endPt;
for (endPt = (pt = (NvAR_Point2f *)landmarks) + FaceEngine::NUM_LANDMARKS; pt < endPt; ++pt)
for (endPt = (pt = (NvAR_Point2f *)landmarks) + numLandmarks; pt < endPt; ++pt)
outputFile << pt->x << "," << pt->y << ",";
} else {
outputFile << "0,";
@@ -530,11 +545,11 @@ void DoApp::writeEstResults(std::ofstream &outputFile, NvAR_BBoxes output_bboxes
outputFile << "\n";
}
void DoApp::writeFrameAndEstResults(const cv::Mat &frame, NvAR_BBoxes output_bboxes, NvAR_Point2f *landmarks) {
void DoApp::writeFrameAndEstResults(const cv::Mat &frm, NvAR_BBoxes output_bboxes, NvAR_Point2f *landmarks) {
if (captureFrame) {
const std::string currentCalendarTime = getCalendarTime();
const std::string capturedFrame = currentCalendarTime + ".png";
cv::imwrite(capturedFrame, frame);
cv::imwrite(capturedFrame, frm);
if (FLAG_verbose) {
std::cout << "Captured the frame" << std::endl;
}
@@ -555,19 +570,19 @@ void DoApp::writeFrameAndEstResults(const cv::Mat &frame, NvAR_BBoxes output_bbo
}
}
void DoApp::DrawLandmarkPoints(const cv::Mat &src, NvAR_Point2f *facial_landmarks) {
cv::Mat frame;
void DoApp::DrawLandmarkPoints(const cv::Mat &src, NvAR_Point2f *facial_landmarks, int numLandmarks) {
cv::Mat frm;
if (FLAG_offlineMode)
frame = src.clone();
frm = src.clone();
else
frame = src;
frm = src;
NvAR_Point2f *pt, *endPt;
for (endPt = (pt = (NvAR_Point2f *)facial_landmarks) + FaceEngine::NUM_LANDMARKS; pt < endPt; ++pt)
cv::circle(frame, cv::Point(lround(pt->x), lround(pt->y)), 1, cv::Scalar(0, 0, 255), -1);
for (endPt = (pt = (NvAR_Point2f *)facial_landmarks) + numLandmarks; pt < endPt; ++pt)
cv::circle(frm, cv::Point(lround(pt->x), lround(pt->y)), 1, cv::Scalar(0, 0, 255), -1);
NvAR_Quaternion *pose = face_ar_engine.getPose();
if (pose)
face_ar_engine.DrawPose(frame, pose);
if (FLAG_offlineMode) landMarkOutputVideo.write(frame);
face_ar_engine.DrawPose(frm, pose);
if (FLAG_offlineMode) landMarkOutputVideo.write(frm);
}
void DoApp::DrawFaceMesh(const cv::Mat &src, NvAR_FaceMesh *face_mesh) {
@@ -608,12 +623,14 @@ DoApp::Err DoApp::acquireFrame() {
// frames we try to read are empty. So we try to re-initialize the camera with the same resolution settings. If the
// resolution has changed, you will need to destroy and create the features again with the new camera resolution (not
// done here) as well as reallocate memory accordingly with FaceEngine::initFeatureIOParams()
cap >> frame; // get a new frame from camera
cap >> frame; // get a new frame from camera into the class variable frame.
if (frame.empty()) {
// if in Offline mode, this means end of video,so we return
if (FLAG_offlineMode) return errVideo;
// try Init one more time if reading frames from camera
initCamera(FLAG_camRes.c_str());
err = initCamera(FLAG_camRes.c_str());
if (err != errNone)
return err;
cap >> frame;
if (frame.empty()) return errVideo;
}
@@ -653,30 +670,30 @@ DoApp::Err DoApp::acquireFaceBox() {
DoApp::Err DoApp::acquireFaceBoxAndLandmarks() {
Err err = errNone;
int numLandmarks = face_ar_engine.getNumLandmarks();
NvAR_Rect output_bbox;
NvAR_Point2f facial_landmarks[FaceEngine::NUM_LANDMARKS];
std::vector<NvAR_Point2f> facial_landmarks(numLandmarks);
// get landmarks in original image resolution coordinate space
unsigned n = face_ar_engine.acquireFaceBoxAndLandmarks(frame, facial_landmarks, output_bbox, 0);
unsigned n = face_ar_engine.acquireFaceBoxAndLandmarks(frame, facial_landmarks.data(), output_bbox, 0);
if (n && FLAG_verbose && face_ar_engine.appMode != FaceEngine::mode::faceDetection) {
printf("Landmarks: [\n");
NvAR_Point2f *pt, *endPt;
for (endPt = (pt = (NvAR_Point2f *)facial_landmarks) + FaceEngine::NUM_LANDMARKS; pt < endPt; ++pt)
printf("%7.1f%7.1f\n", pt->x, pt->y);
for (const auto &pt : facial_landmarks) {
printf("%7.1f%7.1f\n", pt.x, pt.y);
}
printf("]\n");
}
if (FLAG_captureOutputs) {
writeFrameAndEstResults(frame, face_ar_engine.output_bboxes, facial_landmarks);
writeVideoAndEstResults(frame, face_ar_engine.output_bboxes, facial_landmarks);
writeFrameAndEstResults(frame, face_ar_engine.output_bboxes, facial_landmarks.data());
writeVideoAndEstResults(frame, face_ar_engine.output_bboxes, facial_landmarks.data());
}
if (0 == n) return errNoFace;
#ifdef VISUALIZE
if (drawVisualization) {
DrawLandmarkPoints(frame, facial_landmarks);
DrawLandmarkPoints(frame, facial_landmarks.data(), numLandmarks);
if (FLAG_offlineMode) {
DrawBBoxes(frame, &output_bbox);
}
@@ -707,24 +724,25 @@ DoApp::Err DoApp::initCamera(const char *camRes) {
if (inputWidth) cap.set(CV_CAP_PROP_FRAME_WIDTH, inputWidth);
if (inputHeight) cap.set(CV_CAP_PROP_FRAME_HEIGHT, inputHeight);
inputWidth = cap.get(CV_CAP_PROP_FRAME_WIDTH);
inputHeight = cap.get(CV_CAP_PROP_FRAME_HEIGHT);
inputWidth = (int)cap.get(CV_CAP_PROP_FRAME_WIDTH);
inputHeight = (int)cap.get(CV_CAP_PROP_FRAME_HEIGHT);
face_ar_engine.setInputImageWidth(inputWidth);
face_ar_engine.setInputImageHeight(inputHeight);
}
} else
return errVideo;
return errCamera;
return errNone;
}
DoApp::Err DoApp::initOfflineMode(const char *inputFilename, const char *outputFilename) {
if (cap.open(inputFilename)) {
inputWidth = cap.get(CV_CAP_PROP_FRAME_WIDTH);
inputHeight = cap.get(CV_CAP_PROP_FRAME_HEIGHT);
inputWidth = (int)cap.get(CV_CAP_PROP_FRAME_WIDTH);
inputHeight = (int)cap.get(CV_CAP_PROP_FRAME_HEIGHT);
face_ar_engine.setInputImageWidth(inputWidth);
face_ar_engine.setInputImageHeight(inputHeight);
} else {
return Err::errNotFound;
printf("ERROR: Unable to open the input video file \"%s\" \n", inputFilename);
return Err::errVideo;
}
std::string fdOutputVideoName, fldOutputVideoName, ffOutputVideoName;
@@ -740,14 +758,20 @@ DoApp::Err DoApp::initOfflineMode(const char *inputFilename, const char *outputF
ffOutputVideoName = outputFilePrefix + "_faceModel.mp4";
if (!faceDetectOutputVideo.open(fdOutputVideoName, StringToFourcc(FLAG_captureCodec), cap.get(CV_CAP_PROP_FPS),
cv::Size(inputWidth, inputHeight)))
return Err::errSDK;
cv::Size(inputWidth, inputHeight))) {
printf("ERROR: Unable to open the output video file \"%s\" \n", fdOutputVideoName.c_str());
return Err::errGeneral;
}
if (!landMarkOutputVideo.open(fldOutputVideoName, StringToFourcc(FLAG_captureCodec), cap.get(CV_CAP_PROP_FPS),
cv::Size(inputWidth, inputHeight)))
return Err::errSDK;
cv::Size(inputWidth, inputHeight))) {
printf("ERROR: Unable to open the output video file \"%s\" \n", fldOutputVideoName.c_str());
return Err::errGeneral;
}
if (!faceFittingOutputVideo.open(ffOutputVideoName, StringToFourcc(FLAG_captureCodec), cap.get(CV_CAP_PROP_FPS),
cv::Size(inputWidth, inputHeight)))
return Err::errSDK;
cv::Size(inputWidth, inputHeight))) {
printf("ERROR: Unable to open the output video file \"%s\" \n", ffOutputVideoName.c_str());
return Err::errGeneral;
}
return Err::errNone;
}
@@ -766,7 +790,7 @@ DoApp::Err DoApp::fitFaceModel() {
if (drawVisualization) {
DrawFaceMesh(frame, face_ar_engine.getFaceMesh());
if (FLAG_offlineMode) {
DrawLandmarkPoints(frame, face_ar_engine.getLandmarks());
DrawLandmarkPoints(frame, face_ar_engine.getLandmarks(), face_ar_engine.getNumLandmarks());
DrawBBoxes(frame, face_ar_engine.getLargestBox());
}
}
@@ -835,11 +859,20 @@ void DoApp::drawVideoCaptureStatus(cv::Mat &img) {
DoApp::Err DoApp::run() {
DoApp::Err doErr = errNone;
face_ar_engine.initFeatureIOParams();
FaceEngine::Err err = face_ar_engine.initFeatureIOParams();
if (err != FaceEngine::Err::errNone ) {
return doAppErr(err);
}
while (1) {
doErr = acquireFrame();
if (doErr != DoApp::errNone) return doErr;
if (frame.empty() && FLAG_offlineMode) {
// We have reached the end of the video
// so return without any error.
return DoApp::errNone;
} else if (doErr != DoApp::errNone) {
return doErr;
}
if (face_ar_engine.appMode == FaceEngine::mode::faceDetection) {
doErr = acquireFaceBox();
} else if (face_ar_engine.appMode == FaceEngine::mode::landmarkDetection) {
@@ -898,6 +931,12 @@ const char *DoApp::errorStringFromCode(DoApp::Err code) {
};
static const LUTEntry lut[] = {
{errNone, "no error"},
{errGeneral, "an error has occured"},
{errRun, "an error has occured while the feature is running"},
{errInitialization, "Initializing Face Engine failed"},
{errRead, "an error has occured while reading a file"},
{errEffect, "an error has occured while creating a feature"},
{errParameter, "an error has occured while setting a parameter for a feature"},
{errUnimplemented, "the feature is unimplemented"},
{errMissing, "missing input parameter"},
{errVideo, "no video source has been found"},
@@ -914,7 +953,7 @@ const char *DoApp::errorStringFromCode(DoApp::Err code) {
{errSDK, "an SDK error has occurred"},
{errCuda, "a CUDA error has occurred"},
{errCancel, "the user cancelled"},
{errInitFaceEngine, "an error occurred while initializing the Face Engine"},
{errCamera, "unable to connect to the camera"},
};
for (const LUTEntry *p = lut; p < &lut[sizeof(lut) / sizeof(lut[0])]; ++p)
if (p->code == code) return p->str;
@@ -930,19 +969,23 @@ const char *DoApp::errorStringFromCode(DoApp::Err code) {
int main(int argc, char **argv) {
DoApp app;
DoApp::Err doErr;
NvCV_Status nvErr;
// Parse the arguments
if (0 != ParseMyArgs(argc, argv)) return -100;
app.face_ar_engine.setAppMode(FaceEngine::mode(FLAG_appMode));
if (FLAG_verbose) printf("Enable temporal optimizations in detecting face and landmarks = %d\n", FLAG_temporal);
app.face_ar_engine.setFaceStabilization(FLAG_temporal);
doErr = DoApp::errFaceModelInit;
if (FLAG_modelPath.empty()) {
printf("WARNING: Model path not specified. Please set --model_path=/path/to/trt/and/face/models\n"
"SDK will attempt to load the models from NVAR_MODEL_DIR environment variable");
printf("WARNING: Model path not specified. Please set --model_path=/path/to/trt/and/face/models, "
"SDK will attempt to load the models from NVAR_MODEL_DIR environment variable, "
"please restart your application after the SDK Installation. \n");
}
if (!FLAG_faceModel.empty())
app.face_ar_engine.setFaceModel(FLAG_faceModel.c_str());
if (FLAG_offlineMode) {
if (FLAG_inFile.empty()) {
@@ -950,21 +993,23 @@ int main(int argc, char **argv) {
printf("ERROR: %s, please specify input file using --in_file or --in \n", app.errorStringFromCode(doErr));
goto bail;
}
app.initOfflineMode(FLAG_inFile.c_str(), FLAG_outFile.c_str());
doErr = app.initOfflineMode(FLAG_inFile.c_str(), FLAG_outFile.c_str());
} else {
app.initCamera(FLAG_camRes.c_str());
}
doErr = app.initFaceEngine(FLAG_modelPath.c_str());
if (DoApp::errNone != doErr) {
printf("ERROR: %s\n", app.errorStringFromCode(doErr));
goto bail;
doErr = app.initCamera(FLAG_camRes.c_str());
}
BAIL_IF_ERR(doErr);
doErr = app.initFaceEngine(FLAG_modelPath.c_str(), FLAG_isNumLandmarks126);
BAIL_IF_ERR(doErr);
if (!FLAG_proxyWireframe.empty()) app.setProxyWireframe(FLAG_proxyWireframe.c_str());
doErr = app.run();
BAIL_IF_ERR(doErr);
bail:
if(doErr)
printf("ERROR: %s\n", app.errorStringFromCode(doErr));
app.stop();
return (int)doErr;
}