diff --git a/NVIDIA AR SDK Programming Guide.pdf b/NVIDIA AR SDK Programming Guide.pdf new file mode 100644 index 0000000..f155ff3 Binary files /dev/null and b/NVIDIA AR SDK Programming Guide.pdf differ diff --git a/README.MD b/README.MD new file mode 100644 index 0000000..bccfa5d --- /dev/null +++ b/README.MD @@ -0,0 +1,16 @@ +**Nvidia AR SDK: API Source Code and Sample Applications** + +NVIDIA AR SDK enables real-time modeling and tracking of human faces from video. The SDK is powered by NVIDIA graphics processing units (GPUs) with Tensor Cores, and as a result, the algorithm throughput is greatly accelerated, and latency is reduced. + +NVIDIA AR SDK has the following features: + +- **Face detection and tracking**, which detects, localizes, and tracks human faces in images or videos by using bounding boxes. +- **Facial landmark detection and tracking**, which which predicts and tracks the pixel locations of human facial landmark points and head poses in images or videos.The detected facial landmarks follow the _Multi-PIE 68 point mark-ups_ information in [Facial point annotations](https://ibug.doc.ic.ac.uk/resources/facial-point-annotations/). +- **Face 3D mesh and tracking**, which reconstructs and tracks a 3D human face and its head pose from the provided facial landmarks **.** NVIDIA AR SDK provides a sample application that demonstrates the features listed above n real time by using a webcam or offline videos. + +NVIDIA AR SDK is distributed in the following parts: + +- This open source repository that includes the SDK API and proxy linking source code (_add link to the "nvar" folder in this repo_), sample applications and their dependency libraries (_add link to the "samples" folder in this repo)_. +- An Installer hosted on Nvidia Dev Zone (_link to be added_) that installs the SDK DLLs, the models, and the SDK dependency libraries. + +Please refer to SDK programming guide (_add link to programming guide in this repo_) for configuring the SDK, integrating the SDK, compiling and running the sample application. diff --git a/docs/NVIDIA AR SDK Programming Guide.pdf b/docs/NVIDIA AR SDK Programming Guide.pdf deleted file mode 100644 index 2a07b52..0000000 Binary files a/docs/NVIDIA AR SDK Programming Guide.pdf and /dev/null differ diff --git a/nvar/include/nvAR.h b/nvar/include/nvAR.h index f7413cf..34a3f3f 100644 --- a/nvar/include/nvAR.h +++ b/nvar/include/nvAR.h @@ -38,13 +38,6 @@ typedef struct CUstream_st *CUstream; typedef struct nvAR_Feature nvAR_Feature; typedef struct nvAR_Feature *NvAR_FeatureHandle; -//! Get the SDK version -//! \param[in,out] version Pointer to an unsigned int set to -//! (major << 24) | (minor << 16) | (build << 8) | 0 -//! \return NVCV_SUCCESS if the version was set -//! \return NVCV_ERR_PARAMETER if version was NULL -NvCV_Status NvAR_API NvAR_GetVersion(unsigned int *version); - //! Create a new feature instantiation. //! \param[in] InFeatureID The selector code for the desired feature. //! \param[out] handle Handle to the feature instance. @@ -97,7 +90,7 @@ NvCV_Status NvAR_API NvAR_GetObject(NvAR_FeatureHandle handle, const char *name, unsigned long typeSize); NvCV_Status NvAR_API NvAR_GetString(NvAR_FeatureHandle handle, const char *name, const char **str); NvCV_Status NvAR_API NvAR_GetCudaStream(NvAR_FeatureHandle handle, const char *name, const CUstream *stream); -NvCV_Status NvAR_API NvAR_GetF32Array(NvAR_FeatureHandle handle, const char *name, const float **vals, int* /*count*/); +NvCV_Status NvAR_API NvAR_GetF32Array(NvAR_FeatureHandle handle, const char *name, const float **vals, int */*count*/); #ifdef __cplusplus } diff --git a/nvar/include/nvAR_defs.h b/nvar/include/nvAR_defs.h index 04ea3d5..85951a0 100644 --- a/nvar/include/nvAR_defs.h +++ b/nvar/include/nvAR_defs.h @@ -136,18 +136,12 @@ NvAR_Parameter_Output(Pose) - OPTIONAL NvAR_Parameter_Output(LandmarksConfidence) - OPTIONAL *******NvAR_Feature_Face3DReconstruction******* -Config: +Config NvAR_Parameter_Config(FeatureDescription) NvAR_Parameter_Config(ModelDir) NvAR_Parameter_Config(Landmarks_Size) NvAR_Parameter_Config(CUDAStream) -OPTIONAL NvAR_Parameter_Config(Temporal) - OPTIONAL -NvAR_Parameter_Config(ModelName) - OPTIONAL -NvAR_Parameter_Config(GPU) - OPTIONAL -NvAR_Parameter_Config(VertexCount) - QUERY -NvAR_Parameter_Config(TriangleCount) - QUERY -NvAR_Parameter_Config(ExpressionCount) - QUERY -NvAR_Parameter_Config(ShapeEigenValueCount) - QUERY Input: NvAR_Parameter_Input(Width) @@ -163,8 +157,6 @@ NvAR_Parameter_Output(BoundingBoxesConfidence) - OPTIONAL NvAR_Parameter_Output(Landmarks) - OPTIONAL NvAR_Parameter_Output(Pose) - OPTIONAL NvAR_Parameter_Output(LandmarksConfidence) - OPTIONAL -NvAR_Parameter_Output(ExpressionCoefficients) - OPTIONAL -NvAR_Parameter_Output(ShapeEigenValues) - OPTIONAL */ #endif // NvAR_DEFS_H diff --git a/nvar/include/nvCVImage.h b/nvar/include/nvCVImage.h index 7f07360..f63b71c 100644 --- a/nvar/include/nvCVImage.h +++ b/nvar/include/nvCVImage.h @@ -63,7 +63,7 @@ typedef enum NvCVImage_ComponentType { } NvCVImage_ComponentType; -//! Value for the planar field or layout argument. Two values are currently accommodated for RGB: +//! Value for the planar field or isPlanar argument. Two values are currently accommodated for RGB: //! Interleaved or chunky storage locates all components of a pixel adjacent in memory, //! e.g. RGBRGBRGB... (denoted [RGB]). //! Planar storage locates the same component of all pixels adjacent in memory, @@ -104,11 +104,12 @@ typedef enum NvCVImage_ComponentType { #define NVCV_CHROMA_MPEG2 NVCV_CHROMA_COSITED #define NVCV_CHROMA_MPEG1 NVCV_CHROMA_INTSTITIAL -//! This is the value for the gpuMem field or the memSpace argument. +//! This is the value for the gpuMem field or the onGPU argument. Two values are currently accommodated: +//! CPU indicates standard CPU memory. +//! GPU indicates CUDA buffers. #define NVCV_CPU 0 //!< The buffer is stored in CPU memory. #define NVCV_GPU 1 //!< The buffer is stored in CUDA memory. #define NVCV_CUDA 1 //!< The buffer is stored in CUDA memory. -#define NVCV_CPU_PINNED 2 //!< The buffer is stored in pinned CPU memory. //! Image descriptor. typedef struct @@ -124,8 +125,8 @@ NvCVImage { unsigned char pixelBytes; //!< The number of bytes in a chunky pixel. unsigned char componentBytes; //!< The number of bytes in each pixel component. unsigned char numComponents; //!< The number of components in each pixel. - unsigned char planar; //!< NVCV_CHUNKY, NVCV_PLANAR, NVCV_UYVY, .... - unsigned char gpuMem; //!< NVCV_CPU, NVCV_CPU_PINNED, NVCV_CUDA, NVCV_GPU + unsigned char planar; //!< 0=chunky, 1=planar, 2=semi-planar (NV12, NV21). + unsigned char gpuMem; //!< 0=cpu mem, 1=cuda mem, unsigned char colorspace; //!< an OR of colorspace, range and chroma phase. unsigned char reserved[2]; //!< For structure padding and future expansion. Set to 0. void *pixels; //!< Pointer to pixel(0,0) in the image. @@ -144,14 +145,14 @@ NvCVImage { //! \param[in] height the number of pixels vertically. //! \param[in] format the format of the pixels. //! \param[in] type the type of each pixel component. - //! \param[in] layout One of { NVCV_CHUNKY, NVCV_PLANAR } or one of the YUV layouts. - //! \param[in] memSpace One of { NVCV_CPU, NVCV_CPU_PINNED, NVCV_GPU, NVCV_CUDA } + //! \param[in] isPlanar One of { NVCV_CHUNKY, NVCV_PLANAR }. + //! \param[in] onGPU One of { NVCV_CPU, NVCV_GPU } //! \param[in] alignment row byte alignment. Choose 0 or a power of 2. //! 1: yields no gap whatsoever between scanlines; //! 0: default alignment: 4 on CPU, and cudaMallocPitch's choice on GPU. //! Other common values are 16 or 32 for cache line size. inline NvCVImage(unsigned width, unsigned height, NvCVImage_PixelFormat format, NvCVImage_ComponentType type, - unsigned layout = NVCV_CHUNKY, unsigned memSpace = NVCV_CPU, unsigned alignment = 0); + unsigned isPlanar = 0, unsigned onGPU = 0, unsigned alignment = 0); //! Subimage constructor. //! \param[in] fullImg the full image, from which this subImage view is to be created. @@ -205,12 +206,12 @@ NvCVImage { //! \param[in] pixels a pointer to the pixel buffer. //! \param[in] format the format of the pixels. //! \param[in] type the type of the components of the pixels. -//! \param[in] layout One of { NVCV_CHUNKY, NVCV_PLANAR } or one of the YUV layouts. -//! \param[in] memSpace Location of the buffer: one of { NVCV_CPU, NVCV_CPU_PINNED, NVCV_GPU, NVCV_CUDA } +//! \param[in] isPlanar One of { NVCV_CHUNKY, NVCV_PLANAR }. +//! \param[in] onGPU Location of the buffer: one of { NVCV_CPU, NVCV_GPU } //! \return NVCV_SUCCESS if successful //! \return NVCV_ERR_PIXELFORMAT if the pixel format is not yet accommodated. NvCV_Status NvCV_API NvCVImage_Init(NvCVImage *im, unsigned width, unsigned height, int pitch, void *pixels, - NvCVImage_PixelFormat format, NvCVImage_ComponentType type, unsigned layout, unsigned memSpace); + NvCVImage_PixelFormat format, NvCVImage_ComponentType type, unsigned isPlanar, unsigned onGPU); //! Initialize a view into a subset of an existing image. @@ -233,8 +234,8 @@ void NvCV_API NvCVImage_InitView(NvCVImage *subImg, NvCVImage *fullImg, int x, i //! \param[in] height the desired height of the image, in pixels. //! \param[in] format the format of the pixels. //! \param[in] type the type of the components of the pixels. -//! \param[in] layout One of { NVCV_CHUNKY, NVCV_PLANAR } or one of the YUV layouts. -//! \param[in] memSpace Location of the buffer: one of { NVCV_CPU, NVCV_CPU_PINNED, NVCV_GPU, NVCV_CUDA } +//! \param[in] isPlanar One of { NVCV_CHUNKY, NVCV_PLANAR }. +//! \param[in] onGPU Location of the buffer: one of { NVCV_CPU, NVCV_GPU } //! \param[in] alignment row byte alignment. Choose 0 or a power of 2. //! 1: yields no gap whatsoever between scanlines; //! 0: default alignment: 4 on CPU, and cudaMallocPitch's choice on GPU. @@ -243,7 +244,7 @@ void NvCV_API NvCVImage_InitView(NvCVImage *subImg, NvCVImage *fullImg, int x, i //! \return NVCV_ERR_PIXELFORMAT if the pixel format is not accommodated. //! \return NVCV_ERR_MEMORY if there is not enough memory to allocate the buffer. NvCV_Status NvCV_API NvCVImage_Alloc(NvCVImage *im, unsigned width, unsigned height, NvCVImage_PixelFormat format, - NvCVImage_ComponentType type, unsigned layout, unsigned memSpace, unsigned alignment); + NvCVImage_ComponentType type, unsigned isPlanar, unsigned onGPU, unsigned alignment); //! Reallocate memory for, and initialize an image. This assumes that the image is valid. @@ -254,8 +255,8 @@ NvCV_Status NvCV_API NvCVImage_Alloc(NvCVImage *im, unsigned width, unsigned hei //! \param[in] height the desired height of the image, in pixels. //! \param[in] format the format of the pixels. //! \param[in] type the type of the components of the pixels. -//! \param[in] layout One of { NVCV_CHUNKY, NVCV_PLANAR } or one of the YUV layouts. -//! \param[in] memSpace Location of the buffer: one of { NVCV_CPU, NVCV_CPU_PINNED, NVCV_GPU, NVCV_CUDA } +//! \param[in] isPlanar One of { NVCV_CHUNKY, NVCV_PLANAR }. +//! \param[in] onGPU Location of the buffer: one of { NVCV_CPU, NVCV_GPU } //! \param[in] alignment row byte alignment. Choose 0 or a power of 2. //! 1: yields no gap whatsoever between scanlines; //! 0: default alignment: 4 on CPU, and cudaMallocPitch's choice on GPU. @@ -264,7 +265,7 @@ NvCV_Status NvCV_API NvCVImage_Alloc(NvCVImage *im, unsigned width, unsigned hei //! \return NVCV_ERR_PIXELFORMAT if the pixel format is not accommodated. //! \return NVCV_ERR_MEMORY if there is not enough memory to allocate the buffer. NvCV_Status NvCV_API NvCVImage_Realloc(NvCVImage *im, unsigned width, unsigned height, NvCVImage_PixelFormat format, - NvCVImage_ComponentType type, unsigned layout, unsigned memSpace, unsigned alignment); + NvCVImage_ComponentType type, unsigned isPlanar, unsigned onGPU, unsigned alignment); //! Deallocate the image buffer from the image. The image is not deallocated. @@ -277,8 +278,8 @@ void NvCV_API NvCVImage_Dealloc(NvCVImage *im); //! \param[in] height the desired height of the image, in pixels. //! \param[in] format the format of the pixels. //! \param[in] type the type of the components of the pixels. -//! \param[in] layout One of { NVCV_CHUNKY, NVCV_PLANAR } or one of the YUV layouts. -//! \param[in] memSpace Location of the buffer: one of { NVCV_CPU, NVCV_CPU_PINNED, NVCV_GPU, NVCV_CUDA } +//! \param[in] isPlanar One of { NVCV_CHUNKY, NVCV_PLANAR }. +//! \param[in] onGPU Location of the buffer: one of { NVCV_CPU, NVCV_GPU } //! \param[in] alignment row byte alignment. Choose 0 or a power of 2. //! 1: yields no gap whatsoever between scanlines; //! 0: default alignment: 4 on CPU, and cudaMallocPitch's choice on GPU. @@ -288,7 +289,7 @@ void NvCV_API NvCVImage_Dealloc(NvCVImage *im); //! \return NVCV_ERR_PIXELFORMAT if the pixel format is not accommodated. //! \return NVCV_ERR_MEMORY if there is not enough memory to allocate the buffer. NvCV_Status NvCV_API NvCVImage_Create(unsigned width, unsigned height, NvCVImage_PixelFormat format, - NvCVImage_ComponentType type, unsigned layout, unsigned memSpace, unsigned alignment, NvCVImage **out); + NvCVImage_ComponentType type, unsigned isPlanar, unsigned onGPU, unsigned alignment, NvCVImage **out); //! Deallocate the image allocated with NvCVImage_Create() (C-style destructor). @@ -307,38 +308,22 @@ void NvCV_API NvCVImage_ComponentOffsets(NvCVImage_PixelFormat format, int *rOff //! Transfer one image to another, with a limited set of conversions. -//! //! If any of the images resides on the GPU, it may run asynchronously, //! so cudaStreamSynchronize() should be called if it is necessary to run synchronously. -//! The following table indicates the currently-implemented conversions: -//! +------------------+-------------+-------------+-------------+-------------+ -//! | | u8 --> u8 | u8 --> f32 | f32 --> u8 | f32 --> f32 | -//! +------------------+-------------+-------------+-------------+-------------+ -//! | Y -- > Y | X | | X | X | -//! | Y -- > A | X | | X | X | -//! | Y -- > RGB | X | X | X | X | -//! | Y -- > RGBA | X | X | X | X | -//! | A -- > Y | X | | X | X | -//! | A -- > A | X | | X | X | -//! | A -- > RGB | X | X | X | X | -//! | A -- > RGBA | X | | | | -//! | RGB -- > Y | X | X | | | -//! | RGB -- > A | X | X | | | -//! | RGB -- > RGB | X | X | X | X | -//! | RGB -- > RGBA | X | X | X | X | -//! | RGBA -- > Y | X | X | | | -//! | RGBA -- > A | | X | | | -//! | RGBA -- > RGB | X | X | X | X | -//! | RGBA -- > RGBA | X | | | | -//! | YUV420 -- > RGB | X | | | | -//! | YUV422 -- > RGB | X | | | | -//! +------------------+-------------+-------------+-------------+-------------+ -//! where -//! * Either source or destination can be CHUNKY or PLANAR. -//! * Either source or destination can reside on the CPU or the GPU. -//! * The RGB components are in any order (i.e. RGB or BGR; RGBA or BGRA). -//! * YUV requires that the colorspace field be set manually prior to Transfer. -//! * Additionally, when the src and dst formats are the same, all formats are accommodated on CPU and GPU, +//! Conversions are between +//! - RGBu8 --> RGBu8, where the RGB components are in any order; +//! - RGBu8 <--> RGBf32, where the RGB components are in any order (e.g. BGR, RGB); +//! - RGBu8 --> GRAYf32, where the RGB components are in any order; +//! - RGBf32 --> RGBAu8, setting A=255, where the RGB and RGBA components are in any order; +//! - RGBAu8 --> RGBu8, by removing the alpha component; +//! - GRAYf32 --> GRAYu8; +//! - GRAYu8 --> GRAYu8, where the RGB components are in any order (also works with ALPHAu8); +//! - ALPHAu8 --> RGBAu8, (insertion) without touching the RGB components. +//! - GRAYu8 --> RGBu8, by replicating gray into all RGB components. +//! - YUVu8 --> RGBu8, though colorspace field needs to be set manually prior to calling. +//! - chunky <--> planar; +//! - CPU <--> GPU; +//! Additionally, when the src and dst formats are the same, all formats are accommodated on CPU and GPU, //! and this can be used as a replacement for cudaMemcpy2DAsync() (which it utilizes). //! //! When there is some kind of conversion AND the src and dst reside on different processors (CPU, GPU), @@ -369,15 +354,14 @@ NvCV_Status NvCV_API NvCVImage_Transfer( //! Composite one BGRu8 source image over another using the given matte. -//! \param[in] fg the foreground source BGRu8 (or RGBu8) image. -//! \param[in] bg the background source BGRu8 (or RGBu8) image. +//! \param[in] src the source BGRu8 (or RGBu8) image. //! \param[in] mat the matte Yu8 (or Au8) image, indicating where the src should come through. -//! \param[out] dst the destination BGRu8 (or RGBu8) image. This can be the same as fg or bg. +//! \param[out] dst the destination BGRu8 (or RGBu8) image. //! \return NVCV_SUCCESS if the operation was successful. //! \return NVCV_ERR_PIXELFORMAT if the pixel format is not accommodated. -//! \bug This is only implemented for 3-component u8 fg, bg and dst, and 1-component u8 mat, +//! \bug This is only implemented for 3-component u8 src and dst, and 1-component mat, //! where all images are resident on the CPU. -NvCV_Status NvCV_API NvCVImage_Composite(const NvCVImage *fg, const NvCVImage *bg, const NvCVImage *mat, NvCVImage *dst); +NvCV_Status NvCV_API NvCVImage_Composite(const NvCVImage *src, const NvCVImage *mat, NvCVImage *dst); //! Composite a BGRu8 source image over a constant color field using the given matte. @@ -439,9 +423,9 @@ NvCVImage::NvCVImage() { ********************************************************************************/ NvCVImage::NvCVImage(unsigned width, unsigned height, NvCVImage_PixelFormat format, NvCVImage_ComponentType type, - unsigned layout, unsigned memSpace, unsigned alignment) { + unsigned isPlanar, unsigned onGPU, unsigned alignment) { pixels = nullptr; - (void)NvCVImage_Alloc(this, width, height, format, type, layout, memSpace, alignment); + (void)NvCVImage_Alloc(this, width, height, format, type, isPlanar, onGPU, alignment); } /******************************************************************************** diff --git a/nvar/include/nvCVStatus.h b/nvar/include/nvCVStatus.h index 17997dc..5c32654 100644 --- a/nvar/include/nvCVStatus.h +++ b/nvar/include/nvCVStatus.h @@ -61,9 +61,6 @@ typedef enum NvCV_Status { NVCV_ERR_FEATURENOTFOUND = -14, //!< The requested feature was not found NVCV_ERR_MISSINGINPUT = -15, //!< A required parameter was not set NVCV_ERR_RESOLUTION = -16, //!< The specified image resolution is not supported. - NVCV_ERR_UNSUPPORTEDGPU = -17, //!< The GPU is not supported - NVCV_ERR_WRONGGPU = -18, //!< The current GPU is not the one selected. - NVCV_ERR_UNSUPPORTEDDRIVER = -19, //!< The currently installed graphics driver is not supported NVCV_ERR_CUDA_MEMORY = -20, //!< There is not enough CUDA memory for the requested operation. NVCV_ERR_CUDA_VALUE = -21, //!< A CUDA parameter is not within the acceptable range. diff --git a/nvar/src/nvARProxy.cpp b/nvar/src/nvARProxy.cpp index 44f1c02..8368ca9 100644 --- a/nvar/src/nvARProxy.cpp +++ b/nvar/src/nvARProxy.cpp @@ -62,7 +62,7 @@ inline int nvFreeLibrary(HINSTANCE handle) { HINSTANCE getNvARLib() { - TCHAR path[MAX_PATH], fullPath[MAX_PATH]; + TCHAR path[MAX_PATH], fullPath[2*MAX_PATH]; // There can be multiple apps on the system, // some might include the SDK in the app package and @@ -80,13 +80,6 @@ HINSTANCE getNvARLib() { return NvArLib; } -NvCV_Status NvAR_API NvAR_GetVersion(unsigned int* version) { - static const auto funcPtr = (decltype(NvAR_GetVersion)*)nvGetProcAddress(getNvARLib(), "NvAR_GetVersion"); - - if (nullptr == funcPtr) return NVCV_ERR_LIBRARY; - return funcPtr(version); -} - NvCV_Status NvAR_API NvCVImage_Init(NvCVImage* im, unsigned width, unsigned height, int pitch, void* pixels, NvCVImage_PixelFormat format, NvCVImage_ComponentType type, unsigned isPlanar, unsigned onGPU) { @@ -157,11 +150,11 @@ NvCV_Status NvAR_API NvCVImage_Transfer(const NvCVImage* src, NvCVImage* dst, fl return funcPtr(src, dst, scale, stream, tmp); } -NvCV_Status NvAR_API NvCVImage_Composite(const NvCVImage* fg, const NvCVImage* bg, const NvCVImage* mat, NvCVImage* dst) { +NvCV_Status NvAR_API NvCVImage_Composite(const NvCVImage* src, const NvCVImage* mat, NvCVImage* dst) { static const auto funcPtr = (decltype(NvCVImage_Composite)*)nvGetProcAddress(getNvARLib(), "NvCVImage_Composite"); if (nullptr == funcPtr) return NVCV_ERR_LIBRARY; - return funcPtr(fg, bg, mat, dst); + return funcPtr(src, mat, dst); } NvCV_Status NvAR_API NvCVImage_CompositeOverConstant(const NvCVImage* src, const NvCVImage* mat, @@ -314,7 +307,7 @@ NvCV_Status NvAR_API NvAR_GetCudaStream(NvAR_FeatureHandle handle, const char* n } NvCV_Status NvAR_API NvAR_GetF32Array(NvAR_FeatureHandle handle, const char* name, const float** vals, int* count) { - static const auto funcPtr = (decltype(NvAR_GetF32Array)*)nvGetProcAddress(getNvARLib(), "NvAR_GetF32Array"); + static const auto funcPtr = (decltype(NvAR_GetF32Array)*)nvGetProcAddress(getNvARLib(), "NvAR_GetCNvAR_GetF32ArrayudaStream"); if (nullptr == funcPtr) return NVCV_ERR_LIBRARY; return funcPtr(handle, name, vals, count); diff --git a/samples/FaceTrack/CMakeLists.txt b/samples/FaceTrack/CMakeLists.txt index 8b3eca2..49f8858 100644 --- a/samples/FaceTrack/CMakeLists.txt +++ b/samples/FaceTrack/CMakeLists.txt @@ -1,10 +1,4 @@ -set(SOURCE_FILES FaceEngine.cpp -FaceTrack.cpp -../utils/RenderingUtils.cpp -../../nvar/src/nvARProxy.cpp -../utils/FeatureVertexName.cpp -../utils/FeatureVertexName.h -) +set(SOURCE_FILES FaceEngine.cpp FaceTrack.cpp ../utils/RenderingUtils.cpp ../../nvar/src/nvARProxy.cpp) set(HEADER_FILES FaceEngine.h) # Set Visual Studio source filters @@ -22,14 +16,10 @@ target_link_libraries(FaceTrack PUBLIC GLM ) -set(ARSDK_PATH_STR ${CMAKE_CURRENT_SOURCE_DIR}/../../bin) set(OPENCV_PATH_STR ${CMAKE_CURRENT_SOURCE_DIR}/../external/opencv/bin) -set(PATH_STR "PATH=%PATH%" ${ARSDK_PATH_STR} ${OPENCV_PATH_STR}) -set(CMD_ARG_STR "--model_path=\"${CMAKE_CURRENT_SOURCE_DIR}/../../bin/models\"") +set(PATH_STR "PATH=%PATH%" ${OPENCV_PATH_STR}) set_target_properties(FaceTrack PROPERTIES FOLDER SampleApps VS_DEBUGGER_ENVIRONMENT "${PATH_STR}" - VS_DEBUGGER_COMMAND_ARGUMENTS "${CMD_ARG_STR}" ) - diff --git a/samples/FaceTrack/FaceEngine.cpp b/samples/FaceTrack/FaceEngine.cpp index 83dc3e4..ddab3d9 100644 --- a/samples/FaceTrack/FaceEngine.cpp +++ b/samples/FaceTrack/FaceEngine.cpp @@ -23,10 +23,31 @@ #include "FaceEngine.h" #include "RenderingUtils.h" +const char* NvCV_StatusStringFromCode(NvCV_Status code) { + struct TabEntry { + NvCV_Status code; + const char* str; + }; + static const TabEntry lut[] = { + {NVCV_SUCCESS, "no error"}, + {NVCV_ERR_GENERAL, "unspecified failure"}, + {NVCV_ERR_FEATURENOTFOUND, "Feature not found"}, + {NVCV_ERR_PARAMETER, "invalid parameter"}, + {NVCV_ERR_MEMORY, "provided buffer too small"}, + {NVCV_ERR_INITIALIZATION, "not initialized"}, + {NVCV_ERR_MISSINGINPUT, "missing input"}, + {NVCV_ERR_INITIALIZATION, "unable to initialize feature"}, + {NVCV_ERR_CUDA_MEMORY, "out of GPU memory"}, + {NVCV_ERR_SELECTOR, "unsupported parameter"}, + }; + for (const TabEntry* p = lut; p != &lut[sizeof(lut) / sizeof(lut[0])]; ++p) + if (p->code == code) return p->str; + return "UNKNOWN ERROR"; +} bool CheckResult(NvCV_Status nvErr, unsigned line) { if (NVCV_SUCCESS == nvErr) return true; - std::cout << "ERROR: " << NvCV_GetErrorStringFromCode(nvErr) << ", line " << line << std::endl; + std::cout << "ERROR: " << NvCV_StatusStringFromCode(nvErr) << ", line " << line << std::endl; return false; } @@ -46,7 +67,7 @@ FaceEngine::Err FaceEngine::fitFaceModel(cv::Mat& frame) { nvErr = NvAR_Run(faceFitHandle); BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errRun); - if (getAverageLandmarksConfidence() < confidenceThreshold) return FaceEngine::Err::errRun; + if (getAverageLandmarksConfidence() < LANDMARK_CONF_THRESH) return FaceEngine::Err::errRun; bail: return err; @@ -60,44 +81,30 @@ FaceEngine::Err FaceEngine::createFeatures(const char* modelPath, unsigned int _ FaceEngine::Err err = FaceEngine::Err::errNone; NvCV_Status cuErr = NvAR_CudaStreamCreate(&stream); - if (NVCV_SUCCESS != cuErr) { - printf("Cannot create a cuda stream: %s\n", NvCV_GetErrorStringFromCode(cuErr)); - return errInitialization; - } - if (appMode == faceDetection) { + if (appMode == faceDetection) err = createFaceDetectionFeature(modelPath, stream); - if (err != Err::errNone) { - printf("ERROR: An error has occured while initializing Face Detection\n"); - } - } else if (appMode == landmarkDetection) { + else if (appMode == landmarkDetection) err = createLandmarkDetectionFeature(modelPath, _batchSize, stream); - if (err != Err::errNone) { - printf("ERROR: An error has occured while initializing Landmark Detection\n"); - } - } else if (appMode == faceMeshGeneration) { + else if (appMode == faceMeshGeneration) err = createFaceFittingFeature(modelPath, stream); - if (err != Err::errNone) { - printf("ERROR: An error has occured while initializing Face Fitting\n"); - } - } return err; } -FaceEngine::Err FaceEngine::createFaceDetectionFeature(const char* modelPath, CUstream str) { +FaceEngine::Err FaceEngine::createFaceDetectionFeature(const char* modelPath, CUstream stream) { FaceEngine::Err err = FaceEngine::Err::errNone; NvCV_Status nvErr; nvErr = NvAR_Create(NvAR_Feature_FaceDetection, &faceDetectHandle); - BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errEffect); + BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errInitialization); nvErr = NvAR_SetString(faceDetectHandle, NvAR_Parameter_Config(ModelDir), modelPath); - BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errParameter); + BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errInitialization); - nvErr = NvAR_SetCudaStream(faceDetectHandle, NvAR_Parameter_Config(CUDAStream), str); - BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errParameter); + nvErr = NvAR_SetCudaStream(faceDetectHandle, NvAR_Parameter_Config(CUDAStream), stream); + BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errInitialization); nvErr = NvAR_SetU32(faceDetectHandle, NvAR_Parameter_Config(Temporal), bStabilizeFace); - BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errParameter); + BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errInitialization); nvErr = NvAR_Load(faceDetectHandle); BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errInitialization); @@ -107,31 +114,25 @@ bail: } FaceEngine::Err FaceEngine::createLandmarkDetectionFeature(const char* modelPath, unsigned int _batchSize, - CUstream str) { + CUstream stream) { FaceEngine::Err err = FaceEngine::Err::errNone; NvCV_Status nvErr; batchSize = _batchSize; nvErr = NvAR_Create(NvAR_Feature_LandmarkDetection, &landmarkDetectHandle); - BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errEffect); + BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errInitialization); nvErr = NvAR_SetString(landmarkDetectHandle, NvAR_Parameter_Config(ModelDir), modelPath); - BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errParameter); + BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errInitialization); - nvErr = NvAR_SetCudaStream(landmarkDetectHandle, NvAR_Parameter_Config(CUDAStream), str); - BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errParameter); + nvErr = NvAR_SetCudaStream(landmarkDetectHandle, NvAR_Parameter_Config(CUDAStream), stream); + BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errInitialization); nvErr = NvAR_SetU32(landmarkDetectHandle, NvAR_Parameter_Config(BatchSize), batchSize); - BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errParameter); + BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errInitialization); nvErr = NvAR_SetU32(landmarkDetectHandle, NvAR_Parameter_Config(Temporal), bStabilizeFace); - BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errParameter); - - nvErr = NvAR_SetU32(landmarkDetectHandle, NvAR_Parameter_Config(Landmarks_Size), numLandmarks); - BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errParameter); - - nvErr = NvAR_SetU32(landmarkDetectHandle, NvAR_Parameter_Config(LandmarksConfidence_Size), numLandmarks); - BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errParameter); + BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errInitialization); nvErr = NvAR_Load(landmarkDetectHandle); BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errInitialization); @@ -140,26 +141,18 @@ bail: return err; } -FaceEngine::Err FaceEngine::createFaceFittingFeature(const char* modelPath, CUstream str) { +FaceEngine::Err FaceEngine::createFaceFittingFeature(const char* modelPath, CUstream stream) { FaceEngine::Err err = FaceEngine::Err::errNone; NvCV_Status nvErr; nvErr = NvAR_Create(NvAR_Feature_Face3DReconstruction, &faceFitHandle); - BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errEffect); + BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errInitialization); nvErr = NvAR_SetString(faceFitHandle, NvAR_Parameter_Config(ModelDir), modelPath); - BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errParameter); + BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errInitialization); - nvErr = NvAR_SetCudaStream(faceFitHandle, NvAR_Parameter_Config(CUDAStream), str); - BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errParameter); - - nvErr = NvAR_SetU32(faceFitHandle, NvAR_Parameter_Config(Landmarks_Size), numLandmarks); // TODO: Check if nonzero?? - BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errParameter); - - if (!face_model.empty()) { - nvErr = NvAR_SetString(faceFitHandle, NvAR_Parameter_Config(ModelName), face_model.c_str()); - BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errParameter); - } + nvErr = NvAR_SetCudaStream(faceFitHandle, NvAR_Parameter_Config(CUDAStream), stream); + BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errInitialization); nvErr = NvAR_Load(faceFitHandle); BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errInitialization); @@ -176,64 +169,55 @@ FaceEngine::Err FaceEngine::initFeatureIOParams() { BAIL_IF_CVERR(cvErr, err, FaceEngine::Err::errInitialization); - if (appMode == faceDetection) { + if (appMode == faceDetection) err = initFaceDetectionIOParams(&inputImageBuffer); - if (err != Err::errNone) { - printf("ERROR: An error has occured while setting input, output parmeters for Face Detection\n"); - } - } else if (appMode == landmarkDetection) { + else if (appMode == landmarkDetection) err = initLandmarkDetectionIOParams(&inputImageBuffer); - if (err != Err::errNone) { - printf("ERROR: An error has occured while setting input, output parmeters for Landmark Detection\n"); - } - } else if (appMode == faceMeshGeneration) { + else if (appMode == faceMeshGeneration) err = initFaceFittingIOParams(&inputImageBuffer); - if (err != Err::errNone) { - printf("ERROR: An error has occured while setting input, output parmeters for Face Fitting\n"); - } - } + return err; bail: return err; } -FaceEngine::Err FaceEngine::initFaceDetectionIOParams(NvCVImage* inBuf) { +FaceEngine::Err FaceEngine::initFaceDetectionIOParams(NvCVImage* _inputImageBuffer) { NvCV_Status nvErr = NVCV_SUCCESS; FaceEngine::Err err = FaceEngine::Err::errNone; - nvErr = NvAR_SetObject(faceDetectHandle, NvAR_Parameter_Input(Image), inBuf, sizeof(NvCVImage)); - BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errParameter); + nvErr = NvAR_SetObject(faceDetectHandle, NvAR_Parameter_Input(Image), &inputImageBuffer, sizeof(NvCVImage)); + BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errInitialization); output_bbox_data.assign(25, {0.f, 0.f, 0.f, 0.f}); output_bbox_conf_data.assign(25, 0.f); output_bboxes.boxes = output_bbox_data.data(); - output_bboxes.max_boxes = (uint8_t)output_bbox_data.size(); + output_bboxes.max_boxes = output_bbox_data.size(); output_bboxes.num_boxes = 0; nvErr = NvAR_SetObject(faceDetectHandle, NvAR_Parameter_Output(BoundingBoxes), &output_bboxes, sizeof(NvAR_BBoxes)); - BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errParameter); + BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errInitialization); nvErr = NvAR_SetF32Array(faceDetectHandle, NvAR_Parameter_Output(BoundingBoxesConfidence), output_bbox_conf_data.data(), output_bboxes.max_boxes); - BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errParameter); + BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errInitialization); bail: return err; } -FaceEngine::Err FaceEngine::initLandmarkDetectionIOParams(NvCVImage* inBuf) { +FaceEngine::Err FaceEngine::initLandmarkDetectionIOParams(NvCVImage* _inputImageBuffer) { NvCV_Status nvErr = NVCV_SUCCESS; FaceEngine::Err err = FaceEngine::Err::errNone; - nvErr = NvAR_SetObject(landmarkDetectHandle, NvAR_Parameter_Input(Image), inBuf, sizeof(NvCVImage)); - BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errParameter); + nvErr = NvAR_SetObject(landmarkDetectHandle, NvAR_Parameter_Input(Image), &inputImageBuffer, sizeof(NvCVImage)); + BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errInitialization); unsigned int OUTPUT_SIZE_KPTS, OUTPUT_SIZE_KPTS_CONF; nvErr = NvAR_GetU32(landmarkDetectHandle, NvAR_Parameter_Config(Landmarks_Size), &OUTPUT_SIZE_KPTS); - BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errParameter); + BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errInitialization); nvErr = NvAR_GetU32(landmarkDetectHandle, NvAR_Parameter_Config(LandmarksConfidence_Size), &OUTPUT_SIZE_KPTS_CONF); - BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errParameter); + BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errInitialization); facial_landmarks.assign(batchSize * OUTPUT_SIZE_KPTS, {0.f, 0.f}); facial_pose.assign(batchSize, {0.f, 0.f, 0.f, 0.f}); @@ -241,78 +225,74 @@ FaceEngine::Err FaceEngine::initLandmarkDetectionIOParams(NvCVImage* inBuf) { nvErr = NvAR_SetObject(landmarkDetectHandle, NvAR_Parameter_Output(Landmarks), facial_landmarks.data(), sizeof(NvAR_Point2f)); - BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errParameter); + BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errInitialization); nvErr = NvAR_SetObject(landmarkDetectHandle, NvAR_Parameter_Output(Pose), facial_pose.data(), sizeof(NvAR_Quaternion)); - BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errParameter); + BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errInitialization); nvErr = NvAR_SetF32Array(landmarkDetectHandle, NvAR_Parameter_Output(LandmarksConfidence), facial_landmarks_confidence.data(), batchSize * OUTPUT_SIZE_KPTS); - BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errParameter); + BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errInitialization); uint output_bbox_size = batchSize; if (!bStabilizeFace) output_bbox_size = 25; output_bbox_data.assign(output_bbox_size, {0.f, 0.f, 0.f, 0.f}); output_bboxes.boxes = output_bbox_data.data(); - output_bboxes.max_boxes = (uint8_t)output_bbox_size; - output_bboxes.num_boxes = (uint8_t)output_bbox_size; + output_bboxes.max_boxes = output_bbox_size; + output_bboxes.num_boxes = output_bbox_size; nvErr = NvAR_SetObject(landmarkDetectHandle, NvAR_Parameter_Output(BoundingBoxes), &output_bboxes, sizeof(NvAR_BBoxes)); - BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errParameter); + BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errInitialization); bail: return err; } -FaceEngine::Err FaceEngine::initFaceFittingIOParams(NvCVImage* inBuf) { +FaceEngine::Err FaceEngine::initFaceFittingIOParams(NvCVImage* inputImageBuffer) { NvCV_Status nvErr = NVCV_SUCCESS; FaceEngine::Err err = FaceEngine::Err::errNone; face_mesh = new NvAR_FaceMesh(); - face_mesh->vertices = nullptr; //new NvAR_Vector3f[FACE_MODEL_NUM_VERTICES]; - face_mesh->tvi = nullptr; // new NvAR_Vector3u16[FACE_MODEL_NUM_INDICES]; + face_mesh->vertices = new NvAR_Vector3f[FACE_MODEL_NUM_VERTICES]; + face_mesh->tvi = new NvAR_Vector3u16[FACE_MODEL_NUM_INDICES]; rendering_params = new NvAR_RenderingParams(); - nvErr = NvAR_SetObject(faceFitHandle, NvAR_Parameter_Input(Image), inBuf, sizeof(NvCVImage)); - BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errParameter); + nvErr = NvAR_SetObject(faceFitHandle, NvAR_Parameter_Input(Image), inputImageBuffer, sizeof(NvCVImage)); + BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errInitialization); nvErr = NvAR_SetS32(faceFitHandle, NvAR_Parameter_Input(Width), input_image_width); - BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errParameter); + BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errInitialization); nvErr = NvAR_SetS32(faceFitHandle, NvAR_Parameter_Input(Height), input_image_height); - BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errParameter); + BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errInitialization); unsigned int OUTPUT_SIZE_KPTS; nvErr = NvAR_GetU32(faceFitHandle, NvAR_Parameter_Config(Landmarks_Size), &OUTPUT_SIZE_KPTS); - BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errParameter); + BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errInitialization); facial_landmarks.assign(batchSize * OUTPUT_SIZE_KPTS, {0.f, 0.f}); nvErr = NvAR_SetObject(faceFitHandle, NvAR_Parameter_Output(Landmarks), facial_landmarks.data(), sizeof(NvAR_Point2f)); - BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errParameter); + BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errInitialization); facial_landmarks_confidence.assign(batchSize * OUTPUT_SIZE_KPTS, 0.f); nvErr = NvAR_SetF32Array(faceFitHandle, NvAR_Parameter_Output(LandmarksConfidence), facial_landmarks_confidence.data(), batchSize * OUTPUT_SIZE_KPTS); - BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errParameter); - - facial_pose.assign(batchSize, {0.f, 0.f, 0.f, 0.f}); - nvErr = NvAR_SetObject(faceFitHandle, NvAR_Parameter_Output(Pose), facial_pose.data(), sizeof(NvAR_Quaternion)); - BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errParameter); + BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errInitialization); output_bbox_data.assign(batchSize, {0.f, 0.f, 0.f, 0.f}); output_bboxes.boxes = output_bbox_data.data(); - output_bboxes.max_boxes = (uint8_t)batchSize; - output_bboxes.num_boxes = (uint8_t)batchSize; + output_bboxes.max_boxes = batchSize; + output_bboxes.num_boxes = batchSize; nvErr = NvAR_SetObject(faceFitHandle, NvAR_Parameter_Output(BoundingBoxes), &output_bboxes, sizeof(NvAR_BBoxes)); nvErr = NvAR_SetObject(faceFitHandle, NvAR_Parameter_Output(FaceMesh), face_mesh, sizeof(NvAR_FaceMesh)); - BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errParameter); + BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errInitialization); nvErr = NvAR_SetObject(faceFitHandle, NvAR_Parameter_Output(RenderingParams), rendering_params, sizeof(NvAR_RenderingParams)); - BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errParameter); + BAIL_IF_CVERR(nvErr, err, FaceEngine::Err::errInitialization); bail: return err; @@ -372,7 +352,6 @@ void FaceEngine::releaseFaceFittingIOParams() { if (!output_bbox_data.empty()) output_bbox_data.clear(); if (!facial_landmarks.empty()) facial_landmarks.clear(); if (!facial_landmarks_confidence.empty()) facial_landmarks_confidence.clear(); - if (!facial_pose.empty()) facial_pose.clear(); NvCVImage_Dealloc(&inputImageBuffer); if (rendering_params) { delete rendering_params; @@ -482,7 +461,6 @@ void FaceEngine::jiggleBox(std::mt19937& ran, float minMag, float maxMag, const #endif // None of these makes a significant difference } -#ifdef UNUSED /** Intersect a rectangle with an image. * @param[in] srcRect the source rectangle. * @param[in] src the source image. @@ -515,7 +493,6 @@ static bool IntersectRectWithImage(const cv::Rect& srcRect, const cv::Mat& src, clipRect.height = rect[1].y - rect[0].y; return result; } -#endif // UNUSED void FaceEngine::DrawPose(const cv::Mat& src, const NvAR_Quaternion* pose) { float R[3][3]; @@ -528,19 +505,17 @@ void FaceEngine::DrawPose(const cv::Mat& src, const NvAR_Quaternion* pose) { float x3 = radius * R[1][2] * -1.f; float y3 = radius * R[2][2] * -1.f; - int nose_tip = 0; - const char* kNoseTipName = "nose-tip"; - nose_tip = FindLandmarkIndexFromName(numLandmarks, kNoseTipName); - + // 30th point is the tip of the nose + const int nose_tip = 30; int width = src.cols; int height = src.rows; NvAR_Point2f cxy = *(facial_landmarks.data() + nose_tip); - float cx1 = (float)std::min(std::max(0, int(cxy.x + x1)), width - 1); - float cy1 = (float)std::min(std::max(0, int(cxy.y + y1)), height - 1); - float cx2 = (float)std::min(std::max(0, int(cxy.x + x2)), width - 1); - float cy2 = (float)std::min(std::max(0, int(cxy.y + y2)), height - 1); - float cx3 = (float)std::min(std::max(0, int(cxy.x + x3)), width - 1); - float cy3 = (float)std::min(std::max(0, int(cxy.y + y3)), height - 1); + float cx1 = std::min(std::max(0, int(cxy.x + x1)), width - 1); + float cy1 = std::min(std::max(0, int(cxy.y + y1)), height - 1); + float cx2 = std::min(std::max(0, int(cxy.x + x2)), width - 1); + float cy2 = std::min(std::max(0, int(cxy.y + y2)), height - 1); + float cx3 = std::min(std::max(0, int(cxy.x + x3)), width - 1); + float cy3 = std::min(std::max(0, int(cxy.y + y3)), height - 1); cv::line(src, cv::Point((int)cxy.x, (int)cxy.y), cv::Point((int)cx1, (int)cy1), cv::Scalar(0, 0, 255), 2); cv::line(src, cv::Point((int)cxy.x, (int)cxy.y), cv::Point((int)cx2, (int)cy2), cv::Scalar(0, 255, 0), 2); @@ -555,16 +530,16 @@ NvCV_Status FaceEngine::findLandmarks() { return nvErr; } - if (getAverageLandmarksConfidence() < confidenceThreshold) { + if (getAverageLandmarksConfidence() < LANDMARK_CONF_THRESH) { return NVCV_ERR_GENERAL; } else { average_poses(getPose(), batchSize); NvAR_Point2f *pt, *endPt; int i = 0; - for (endPt = (pt = getLandmarks()) + numLandmarks; pt != endPt; ++pt, i += 2) { + for (endPt = (pt = getLandmarks()) + NUM_LANDMARKS; pt != endPt; ++pt, i += 2) { for (int j = 1; j < batchSize; j++) { - pt->x += pt[j * numLandmarks].x; - pt->y += pt[j * numLandmarks].y; + pt->x += pt[j * NUM_LANDMARKS].x; + pt->y += pt[j * NUM_LANDMARKS].y; } // average batch of inferences to generate final result landmark points pt->x /= batchSize; @@ -582,11 +557,11 @@ float FaceEngine::getAverageLandmarksConfidence() { float average_confidence = 0.0f; float* keypoints_landmarks_confidence = getLandmarksConfidence(); for (int i = 0; i < batchSize; i++) { - for (int j = 0; j < numLandmarks; j++) { - average_confidence += keypoints_landmarks_confidence[i * numLandmarks + j]; + for (int j = 0; j < NUM_LANDMARKS; j++) { + average_confidence += keypoints_landmarks_confidence[i * NUM_LANDMARKS + j]; } } - average_confidence /= batchSize * numLandmarks; + average_confidence /= batchSize * NUM_LANDMARKS; return average_confidence; } @@ -623,7 +598,7 @@ unsigned FaceEngine::acquireFaceBox(cv::Mat& src, NvAR_Rect& faceBox, int varian return n; } -unsigned FaceEngine::acquireFaceBoxAndLandmarks(cv::Mat& src, NvAR_Point2f* refMarks, NvAR_Rect& faceBox, int /*variant*/) { +unsigned FaceEngine::acquireFaceBoxAndLandmarks(cv::Mat& src, NvAR_Point2f* refMarks, NvAR_Rect& faceBox, int variant) { unsigned n = 0; NvCVImage fxSrcChunkyCPU; (void)NVWrapperForCVMat(&src, &fxSrcChunkyCPU); @@ -636,24 +611,11 @@ unsigned FaceEngine::acquireFaceBoxAndLandmarks(cv::Mat& src, NvAR_Point2f* refM if (findLandmarks() != NVCV_SUCCESS) return 0; faceBox = output_bboxes.boxes[0]; n = 1; - memcpy(refMarks, getLandmarks(), sizeof(NvAR_Point2f) * numLandmarks); + memcpy(refMarks, getLandmarks(), sizeof(NvAR_Point2f) * FaceEngine::NUM_LANDMARKS); return n; } void FaceEngine::setFaceStabilization(bool _bStabilizeFace) { bStabilizeFace = _bStabilizeFace; } -FaceEngine::Err FaceEngine::setNumLandmarks(int n) { - FaceEngine::Err err = errNone; - for (auto const& info : LANDMARKS_INFO) { - if (n == info.numPoints) { - numLandmarks = info.numPoints; - confidenceThreshold = info.confidence_threshold; - return err; - } - } - err = errGeneral; - return err; -} - void FaceEngine::setAppMode(FaceEngine::mode _mode) { appMode = _mode; } diff --git a/samples/FaceTrack/FaceEngine.h b/samples/FaceTrack/FaceEngine.h index 2cbb055..743e016 100644 --- a/samples/FaceTrack/FaceEngine.h +++ b/samples/FaceTrack/FaceEngine.h @@ -27,7 +27,6 @@ #include "nvAR.h" #include "nvCVOpenCV.h" #include "opencv2/opencv.hpp" -#include "FeatureVertexName.h" #define FITFACE_PRIVATE class KalmanFilter1D { @@ -79,14 +78,9 @@ class KalmanFilter1D { } }; -bool CheckResult(NvCV_Status nvErr, unsigned line); +const char* NvCV_StatusStringFromCode(NvCV_Status code); -#define BAIL_IF_ERR(err) \ -do { \ - if (0!=err) { \ - goto bail; \ - } \ - } while (0) +bool CheckResult(NvCV_Status nvErr, unsigned line); #define BAIL_IF_CVERR(nvErr, err, code) \ do { \ @@ -96,30 +90,23 @@ do { \ } \ } while (0) -typedef struct LandmarksProperties { - int numPoints; - float confidence_threshold; -}LandmarksProperties; - /******************************************************************************** * FaceEngine ********************************************************************************/ class FaceEngine { public: - enum Err { errNone, errGeneral, errRun, errInitialization, errRead, errEffect, errParameter }; + enum Err { errNone, errGeneral, errRun, errInitialization, errRead }; int input_image_width, input_image_height, input_image_pitch; - const LandmarksProperties LANDMARKS_INFO[2] = { - { 68, 10.0f }, // number of landmark points, confidence threshold value - { 126, 5.0f} - }; + static const int NUM_LANDMARKS = 68; // TODO: get this instead from the SDK. + static const int FACE_MODEL_NUM_VERTICES = 3448, FACE_MODEL_NUM_INDICES = 6736; + static const long LANDMARK_CONF_THRESH = 10.f; void setInputImageWidth(int width) { input_image_width = width; } void setInputImageHeight(int height) { input_image_height = height; } int getInputImageWidth() { return input_image_width; } int getInputImageHeight() { return input_image_height; } int getInputImagePitch() { return input_image_pitch = input_image_width * 3 * sizeof(unsigned char); } - void setFaceModel(const char *faceModel) { face_model = faceModel; } Err createFeatures(const char* modelPath, unsigned int _batchSize = 1); Err createFaceDetectionFeature(const char* modelPath, CUstream stream); @@ -155,8 +142,6 @@ class FaceEngine { NvAR_FaceMesh* getFaceMesh(); NvAR_RenderingParams* getRenderingParams(); void setFaceStabilization(bool); - Err setNumLandmarks(int); - int getNumLandmarks() { return numLandmarks; } void DrawPose(const cv::Mat& src, const NvAR_Quaternion* pose); NvCVImage inputImageBuffer{}, tmpImage{}; @@ -172,17 +157,13 @@ class FaceEngine { NvAR_BBoxes output_bboxes{}; int batchSize; std::mt19937 ran; - int numLandmarks; - float confidenceThreshold; - std::string face_model; bool bStabilizeFace; + NvAR_Point2f prevLandmark[NUM_LANDMARKS] = {0}; FaceEngine() { batchSize = 1; bStabilizeFace = true; - numLandmarks = LANDMARKS_INFO[0].numPoints; - confidenceThreshold = LANDMARKS_INFO[0].confidence_threshold; appMode = faceMeshGeneration; input_image_width = 640; input_image_height = 480; diff --git a/samples/FaceTrack/FaceTrack.cpp b/samples/FaceTrack/FaceTrack.cpp index cbe6cf2..2ba4a46 100644 --- a/samples/FaceTrack/FaceTrack.cpp +++ b/samples/FaceTrack/FaceTrack.cpp @@ -63,10 +63,10 @@ ********************************************************************************/ bool FLAG_debug = false, FLAG_verbose = false, FLAG_temporal = true, FLAG_captureOutputs = false, - FLAG_offlineMode = false, FLAG_isNumLandmarks126 = false; + FLAG_offlineMode = false; std::string FLAG_outDir, FLAG_inFile, FLAG_outFile, FLAG_modelPath, FLAG_landmarks, FLAG_proxyWireframe, - FLAG_captureCodec = "avc1", FLAG_camRes, FLAG_faceModel; -unsigned int FLAG_batch = 1, FLAG_appMode = 2; + FLAG_captureCodec = "avc1", FLAG_camRes; +unsigned int FLAG_batch = 1; /******************************************************************************** * Usage @@ -88,12 +88,8 @@ static void Usage() { " --out_file= specify the output file\n" " --out= specify the output file\n" " --model_path= specify the directory containing the TRT models\n" - " --landmarks_126[=(true|false)] set the number of facial landmark points to 126, otherwise default to 68\n" - " --face_model= specify the name of the face model\n" " --wireframe_mesh= specify the path to a proxy wireframe mesh\n" - " --batch= 1 - 8, used for batch inferencing in landmark detector\n" - " --app_mode[=(0|1|2)] App mode. 0: Face detection, 1: Landmark detection, 2: Face fitting " - "(Default)." + " --batch= 1 - 8, used for batch inferencing in landmark detector " " --benchmarks[=] run benchmarks\n"); } @@ -185,13 +181,11 @@ static int ParseMyArgs(int argc, char **argv) { GetFlagArgVal("in", arg, &FLAG_inFile) || GetFlagArgVal("in_file", arg, &FLAG_inFile) || GetFlagArgVal("out", arg, &FLAG_outFile) || GetFlagArgVal("out_file", arg, &FLAG_outFile) || GetFlagArgVal("offline_mode", arg, &FLAG_offlineMode) || - GetFlagArgVal("landmarks_126", arg, &FLAG_isNumLandmarks126) || GetFlagArgVal("capture_outputs", arg, &FLAG_captureOutputs) || GetFlagArgVal("cam_res", arg, &FLAG_camRes) || GetFlagArgVal("codec", arg, &FLAG_captureCodec) || GetFlagArgVal("landmarks", arg, &FLAG_landmarks) || GetFlagArgVal("model_path", arg, &FLAG_modelPath) || GetFlagArgVal("wireframe_mesh", arg, &FLAG_proxyWireframe) || - GetFlagArgVal("face_model", arg, &FLAG_faceModel) || - GetFlagArgVal("app_mode", arg, &FLAG_appMode) || GetFlagArgVal("temporal", arg, &FLAG_temporal))) { + GetFlagArgVal("temporal", arg, &FLAG_temporal))) { continue; } else if (GetFlagArgVal("help", arg, &help)) { Usage(); @@ -257,13 +251,7 @@ std::string getCalendarTime() { class DoApp { public: enum Err { - errNone = FaceEngine::Err::errNone, - errGeneral = FaceEngine::Err::errGeneral, - errRun = FaceEngine::Err::errRun, - errInitialization = FaceEngine::Err::errInitialization, - errRead = FaceEngine::Err::errRead, - errEffect = FaceEngine::Err::errEffect, - errParameter = FaceEngine::Err::errParameter, + errNone, errUnimplemented, errMissing, errVideo, @@ -280,15 +268,15 @@ class DoApp { errSDK, errCuda, errCancel, - errCamera + errInitFaceEngine }; - Err doAppErr(FaceEngine::Err status) { return (Err)status; } + FaceEngine face_ar_engine; DoApp(); ~DoApp(); void stop(); - Err initFaceEngine(const char *modelPath = nullptr, bool isLandmarks126 = false); + Err initFaceEngine(const char *modelPath = nullptr); Err initCamera(const char *camRes = nullptr); Err initOfflineMode(const char *inputFilename = nullptr, const char *outputFilename = nullptr); Err acquireFrame(); @@ -299,7 +287,7 @@ class DoApp { void showFaceFitErrorMessage(); void drawFPS(cv::Mat &img); void DrawBBoxes(const cv::Mat &src, NvAR_Rect *output_bbox); - void DrawLandmarkPoints(const cv::Mat &src, NvAR_Point2f *facial_landmarks, int numLandmarks); + void DrawLandmarkPoints(const cv::Mat &src, NvAR_Point2f *facial_landmarks); void DrawFaceMesh(const cv::Mat &src, NvAR_FaceMesh *face_mesh); void drawKalmanStatus(cv::Mat &img); void drawVideoCaptureStatus(cv::Mat &img); @@ -332,7 +320,7 @@ class DoApp { }; DoApp *gApp = nullptr; -const char DoApp::windowTitle[] = "FaceTrack App"; +const char DoApp::windowTitle[] = "WINDOW"; void DoApp::processKey(int key) { switch (key) { @@ -380,23 +368,21 @@ void DoApp::processKey(int key) { } } -DoApp::Err DoApp::initFaceEngine(const char *modelPath, bool isNumLandmarks126) { +DoApp::Err DoApp::initFaceEngine(const char *modelPath) { Err err = errNone; if (!cap.isOpened()) return errVideo; - int numLandmarkPoints = isNumLandmarks126 ? 126 : 68; - face_ar_engine.setNumLandmarks(numLandmarkPoints); - nvErr = face_ar_engine.createFeatures(modelPath); if (nvErr != FaceEngine::Err::errNone) { if (nvErr == FaceEngine::Err::errInitialization && face_ar_engine.appMode == FaceEngine::mode::faceMeshGeneration) { showFaceFitErrorMessage(); - printf("WARNING: face fitting has failed, trying to initialize Landmark Detection\n"); face_ar_engine.destroyFeatures(); face_ar_engine.setAppMode(FaceEngine::mode::landmarkDetection); nvErr = face_ar_engine.createFeatures(modelPath); } + if (nvErr != FaceEngine::Err::errNone) + err = errInitFaceEngine; } #ifdef DEBUG @@ -410,7 +396,7 @@ DoApp::Err DoApp::initFaceEngine(const char *modelPath, bool isNumLandmarks126) frameIndex = 0; - return doAppErr(nvErr); + return err; } void DoApp::stop() { @@ -442,29 +428,29 @@ void DoApp::showFaceFitErrorMessage() { } void DoApp::DrawBBoxes(const cv::Mat &src, NvAR_Rect *output_bbox) { - cv::Mat frm; + cv::Mat frame; if (FLAG_offlineMode) - frm = src.clone(); + frame = src.clone(); else - frm = src; + frame = src; if (output_bbox) - cv::rectangle(frm, cv::Point(lround(output_bbox->x), lround(output_bbox->y)), - cv::Point(lround(output_bbox->x + output_bbox->width), lround(output_bbox->y + output_bbox->height)), + cv::rectangle(frame, cv::Point((int)output_bbox->x, (int)output_bbox->y), + cv::Point((int)output_bbox->x + output_bbox->width, (int)output_bbox->y + output_bbox->height), cv::Scalar(255, 0, 0), 2); - if (FLAG_offlineMode) faceDetectOutputVideo.write(frm); + if (FLAG_offlineMode) faceDetectOutputVideo.write(frame); } -void DoApp::writeVideoAndEstResults(const cv::Mat &frm, NvAR_BBoxes output_bboxes, NvAR_Point2f *landmarks) { +void DoApp::writeVideoAndEstResults(const cv::Mat &frame, NvAR_BBoxes output_bboxes, NvAR_Point2f *landmarks) { if (captureVideo) { if (!capturedVideo.isOpened()) { const std::string currentCalendarTime = getCalendarTime(); const std::string capturedOutputFileName = currentCalendarTime + ".mp4"; getFPS(); if (frameTime) { - float fps = (float)(1.0 / frameTime); + float fps = 1. / frameTime; capturedVideo.open(capturedOutputFileName, StringToFourcc(FLAG_captureCodec), fps, - cv::Size(frm.cols, frm.rows)); + cv::Size(frame.cols, frame.rows)); if (!capturedVideo.isOpened()) { std::cout << "Error: Could not open video: \"" << capturedOutputFileName << "\"\n"; return; @@ -487,7 +473,7 @@ void DoApp::writeVideoAndEstResults(const cv::Mat &frm, NvAR_BBoxes output_bboxe << "// kNumFaces, (bbox_x, bbox_y, bbox_w, bbox_h){ kNumFaces}, kNumLMs, [lm_x, lm_y]{kNumLMs}\n"; } // Write each frame to the Video - capturedVideo << frm; + capturedVideo << frame; writeEstResults(faceEngineVideoOutputFile, output_bboxes, landmarks); } else { if (capturedVideo.isOpened()) { @@ -531,12 +517,11 @@ void DoApp::writeEstResults(std::ofstream &outputFile, NvAR_BBoxes output_bboxes outputFile << "0,"; } if (landmarkDetectOn && output_bboxes.num_boxes) { - int numLandmarks = face_ar_engine.getNumLandmarks(); // Append number of landmarks - outputFile << numLandmarks << ","; - // Append 2 * number of landmarks values + outputFile << FaceEngine::NUM_LANDMARKS << ","; + // Append NUM_LANDMARKS * 2 points NvAR_Point2f *pt, *endPt; - for (endPt = (pt = (NvAR_Point2f *)landmarks) + numLandmarks; pt < endPt; ++pt) + for (endPt = (pt = (NvAR_Point2f *)landmarks) + FaceEngine::NUM_LANDMARKS; pt < endPt; ++pt) outputFile << pt->x << "," << pt->y << ","; } else { outputFile << "0,"; @@ -545,11 +530,11 @@ void DoApp::writeEstResults(std::ofstream &outputFile, NvAR_BBoxes output_bboxes outputFile << "\n"; } -void DoApp::writeFrameAndEstResults(const cv::Mat &frm, NvAR_BBoxes output_bboxes, NvAR_Point2f *landmarks) { +void DoApp::writeFrameAndEstResults(const cv::Mat &frame, NvAR_BBoxes output_bboxes, NvAR_Point2f *landmarks) { if (captureFrame) { const std::string currentCalendarTime = getCalendarTime(); const std::string capturedFrame = currentCalendarTime + ".png"; - cv::imwrite(capturedFrame, frm); + cv::imwrite(capturedFrame, frame); if (FLAG_verbose) { std::cout << "Captured the frame" << std::endl; } @@ -570,19 +555,19 @@ void DoApp::writeFrameAndEstResults(const cv::Mat &frm, NvAR_BBoxes output_bboxe } } -void DoApp::DrawLandmarkPoints(const cv::Mat &src, NvAR_Point2f *facial_landmarks, int numLandmarks) { - cv::Mat frm; +void DoApp::DrawLandmarkPoints(const cv::Mat &src, NvAR_Point2f *facial_landmarks) { + cv::Mat frame; if (FLAG_offlineMode) - frm = src.clone(); + frame = src.clone(); else - frm = src; + frame = src; NvAR_Point2f *pt, *endPt; - for (endPt = (pt = (NvAR_Point2f *)facial_landmarks) + numLandmarks; pt < endPt; ++pt) - cv::circle(frm, cv::Point(lround(pt->x), lround(pt->y)), 1, cv::Scalar(0, 0, 255), -1); + for (endPt = (pt = (NvAR_Point2f *)facial_landmarks) + FaceEngine::NUM_LANDMARKS; pt < endPt; ++pt) + cv::circle(frame, cv::Point(lround(pt->x), lround(pt->y)), 1, cv::Scalar(0, 0, 255), -1); NvAR_Quaternion *pose = face_ar_engine.getPose(); if (pose) - face_ar_engine.DrawPose(frm, pose); - if (FLAG_offlineMode) landMarkOutputVideo.write(frm); + face_ar_engine.DrawPose(frame, pose); + if (FLAG_offlineMode) landMarkOutputVideo.write(frame); } void DoApp::DrawFaceMesh(const cv::Mat &src, NvAR_FaceMesh *face_mesh) { @@ -623,14 +608,12 @@ DoApp::Err DoApp::acquireFrame() { // frames we try to read are empty. So we try to re-initialize the camera with the same resolution settings. If the // resolution has changed, you will need to destroy and create the features again with the new camera resolution (not // done here) as well as reallocate memory accordingly with FaceEngine::initFeatureIOParams() - cap >> frame; // get a new frame from camera into the class variable frame. + cap >> frame; // get a new frame from camera if (frame.empty()) { // if in Offline mode, this means end of video,so we return if (FLAG_offlineMode) return errVideo; // try Init one more time if reading frames from camera - err = initCamera(FLAG_camRes.c_str()); - if (err != errNone) - return err; + initCamera(FLAG_camRes.c_str()); cap >> frame; if (frame.empty()) return errVideo; } @@ -670,30 +653,30 @@ DoApp::Err DoApp::acquireFaceBox() { DoApp::Err DoApp::acquireFaceBoxAndLandmarks() { Err err = errNone; - int numLandmarks = face_ar_engine.getNumLandmarks(); + NvAR_Rect output_bbox; - std::vector facial_landmarks(numLandmarks); + NvAR_Point2f facial_landmarks[FaceEngine::NUM_LANDMARKS]; // get landmarks in original image resolution coordinate space - unsigned n = face_ar_engine.acquireFaceBoxAndLandmarks(frame, facial_landmarks.data(), output_bbox, 0); + unsigned n = face_ar_engine.acquireFaceBoxAndLandmarks(frame, facial_landmarks, output_bbox, 0); if (n && FLAG_verbose && face_ar_engine.appMode != FaceEngine::mode::faceDetection) { printf("Landmarks: [\n"); - for (const auto &pt : facial_landmarks) { - printf("%7.1f%7.1f\n", pt.x, pt.y); - } + NvAR_Point2f *pt, *endPt; + for (endPt = (pt = (NvAR_Point2f *)facial_landmarks) + FaceEngine::NUM_LANDMARKS; pt < endPt; ++pt) + printf("%7.1f%7.1f\n", pt->x, pt->y); printf("]\n"); } if (FLAG_captureOutputs) { - writeFrameAndEstResults(frame, face_ar_engine.output_bboxes, facial_landmarks.data()); - writeVideoAndEstResults(frame, face_ar_engine.output_bboxes, facial_landmarks.data()); + writeFrameAndEstResults(frame, face_ar_engine.output_bboxes, facial_landmarks); + writeVideoAndEstResults(frame, face_ar_engine.output_bboxes, facial_landmarks); } if (0 == n) return errNoFace; #ifdef VISUALIZE if (drawVisualization) { - DrawLandmarkPoints(frame, facial_landmarks.data(), numLandmarks); + DrawLandmarkPoints(frame, facial_landmarks); if (FLAG_offlineMode) { DrawBBoxes(frame, &output_bbox); } @@ -724,25 +707,24 @@ DoApp::Err DoApp::initCamera(const char *camRes) { if (inputWidth) cap.set(CV_CAP_PROP_FRAME_WIDTH, inputWidth); if (inputHeight) cap.set(CV_CAP_PROP_FRAME_HEIGHT, inputHeight); - inputWidth = (int)cap.get(CV_CAP_PROP_FRAME_WIDTH); - inputHeight = (int)cap.get(CV_CAP_PROP_FRAME_HEIGHT); + inputWidth = cap.get(CV_CAP_PROP_FRAME_WIDTH); + inputHeight = cap.get(CV_CAP_PROP_FRAME_HEIGHT); face_ar_engine.setInputImageWidth(inputWidth); face_ar_engine.setInputImageHeight(inputHeight); } } else - return errCamera; + return errVideo; return errNone; } DoApp::Err DoApp::initOfflineMode(const char *inputFilename, const char *outputFilename) { if (cap.open(inputFilename)) { - inputWidth = (int)cap.get(CV_CAP_PROP_FRAME_WIDTH); - inputHeight = (int)cap.get(CV_CAP_PROP_FRAME_HEIGHT); + inputWidth = cap.get(CV_CAP_PROP_FRAME_WIDTH); + inputHeight = cap.get(CV_CAP_PROP_FRAME_HEIGHT); face_ar_engine.setInputImageWidth(inputWidth); face_ar_engine.setInputImageHeight(inputHeight); } else { - printf("ERROR: Unable to open the input video file \"%s\" \n", inputFilename); - return Err::errVideo; + return Err::errNotFound; } std::string fdOutputVideoName, fldOutputVideoName, ffOutputVideoName; @@ -758,20 +740,14 @@ DoApp::Err DoApp::initOfflineMode(const char *inputFilename, const char *outputF ffOutputVideoName = outputFilePrefix + "_faceModel.mp4"; if (!faceDetectOutputVideo.open(fdOutputVideoName, StringToFourcc(FLAG_captureCodec), cap.get(CV_CAP_PROP_FPS), - cv::Size(inputWidth, inputHeight))) { - printf("ERROR: Unable to open the output video file \"%s\" \n", fdOutputVideoName.c_str()); - return Err::errGeneral; - } + cv::Size(inputWidth, inputHeight))) + return Err::errSDK; if (!landMarkOutputVideo.open(fldOutputVideoName, StringToFourcc(FLAG_captureCodec), cap.get(CV_CAP_PROP_FPS), - cv::Size(inputWidth, inputHeight))) { - printf("ERROR: Unable to open the output video file \"%s\" \n", fldOutputVideoName.c_str()); - return Err::errGeneral; - } + cv::Size(inputWidth, inputHeight))) + return Err::errSDK; if (!faceFittingOutputVideo.open(ffOutputVideoName, StringToFourcc(FLAG_captureCodec), cap.get(CV_CAP_PROP_FPS), - cv::Size(inputWidth, inputHeight))) { - printf("ERROR: Unable to open the output video file \"%s\" \n", ffOutputVideoName.c_str()); - return Err::errGeneral; - } + cv::Size(inputWidth, inputHeight))) + return Err::errSDK; return Err::errNone; } @@ -790,7 +766,7 @@ DoApp::Err DoApp::fitFaceModel() { if (drawVisualization) { DrawFaceMesh(frame, face_ar_engine.getFaceMesh()); if (FLAG_offlineMode) { - DrawLandmarkPoints(frame, face_ar_engine.getLandmarks(), face_ar_engine.getNumLandmarks()); + DrawLandmarkPoints(frame, face_ar_engine.getLandmarks()); DrawBBoxes(frame, face_ar_engine.getLargestBox()); } } @@ -859,20 +835,11 @@ void DoApp::drawVideoCaptureStatus(cv::Mat &img) { DoApp::Err DoApp::run() { DoApp::Err doErr = errNone; - FaceEngine::Err err = face_ar_engine.initFeatureIOParams(); - if (err != FaceEngine::Err::errNone ) { - return doAppErr(err); - } + face_ar_engine.initFeatureIOParams(); while (1) { doErr = acquireFrame(); - if (frame.empty() && FLAG_offlineMode) { - // We have reached the end of the video - // so return without any error. - return DoApp::errNone; - } else if (doErr != DoApp::errNone) { - return doErr; - } + if (doErr != DoApp::errNone) return doErr; if (face_ar_engine.appMode == FaceEngine::mode::faceDetection) { doErr = acquireFaceBox(); } else if (face_ar_engine.appMode == FaceEngine::mode::landmarkDetection) { @@ -931,12 +898,6 @@ const char *DoApp::errorStringFromCode(DoApp::Err code) { }; static const LUTEntry lut[] = { {errNone, "no error"}, - {errGeneral, "an error has occured"}, - {errRun, "an error has occured while the feature is running"}, - {errInitialization, "Initializing Face Engine failed"}, - {errRead, "an error has occured while reading a file"}, - {errEffect, "an error has occured while creating a feature"}, - {errParameter, "an error has occured while setting a parameter for a feature"}, {errUnimplemented, "the feature is unimplemented"}, {errMissing, "missing input parameter"}, {errVideo, "no video source has been found"}, @@ -953,7 +914,7 @@ const char *DoApp::errorStringFromCode(DoApp::Err code) { {errSDK, "an SDK error has occurred"}, {errCuda, "a CUDA error has occurred"}, {errCancel, "the user cancelled"}, - {errCamera, "unable to connect to the camera"}, + {errInitFaceEngine, "an error occurred while initializing the Face Engine"}, }; for (const LUTEntry *p = lut; p < &lut[sizeof(lut) / sizeof(lut[0])]; ++p) if (p->code == code) return p->str; @@ -969,23 +930,19 @@ const char *DoApp::errorStringFromCode(DoApp::Err code) { int main(int argc, char **argv) { DoApp app; DoApp::Err doErr; + NvCV_Status nvErr; // Parse the arguments if (0 != ParseMyArgs(argc, argv)) return -100; - app.face_ar_engine.setAppMode(FaceEngine::mode(FLAG_appMode)); - if (FLAG_verbose) printf("Enable temporal optimizations in detecting face and landmarks = %d\n", FLAG_temporal); app.face_ar_engine.setFaceStabilization(FLAG_temporal); doErr = DoApp::errFaceModelInit; if (FLAG_modelPath.empty()) { - printf("WARNING: Model path not specified. Please set --model_path=/path/to/trt/and/face/models, " - "SDK will attempt to load the models from NVAR_MODEL_DIR environment variable, " - "please restart your application after the SDK Installation. \n"); + printf("WARNING: Model path not specified. Please set --model_path=/path/to/trt/and/face/models\n" + "SDK will attempt to load the models from NVAR_MODEL_DIR environment variable"); } - if (!FLAG_faceModel.empty()) - app.face_ar_engine.setFaceModel(FLAG_faceModel.c_str()); if (FLAG_offlineMode) { if (FLAG_inFile.empty()) { @@ -993,23 +950,21 @@ int main(int argc, char **argv) { printf("ERROR: %s, please specify input file using --in_file or --in \n", app.errorStringFromCode(doErr)); goto bail; } - doErr = app.initOfflineMode(FLAG_inFile.c_str(), FLAG_outFile.c_str()); + app.initOfflineMode(FLAG_inFile.c_str(), FLAG_outFile.c_str()); } else { - doErr = app.initCamera(FLAG_camRes.c_str()); + app.initCamera(FLAG_camRes.c_str()); + } + doErr = app.initFaceEngine(FLAG_modelPath.c_str()); + if (DoApp::errNone != doErr) { + printf("ERROR: %s\n", app.errorStringFromCode(doErr)); + goto bail; } - BAIL_IF_ERR(doErr); - - doErr = app.initFaceEngine(FLAG_modelPath.c_str(), FLAG_isNumLandmarks126); - BAIL_IF_ERR(doErr); if (!FLAG_proxyWireframe.empty()) app.setProxyWireframe(FLAG_proxyWireframe.c_str()); doErr = app.run(); - BAIL_IF_ERR(doErr); bail: - if(doErr) - printf("ERROR: %s\n", app.errorStringFromCode(doErr)); app.stop(); return (int)doErr; } diff --git a/samples/FaceTrack/FaceTrack.exe b/samples/FaceTrack/FaceTrack.exe index 1714066..e149aed 100644 Binary files a/samples/FaceTrack/FaceTrack.exe and b/samples/FaceTrack/FaceTrack.exe differ diff --git a/samples/FaceTrack/Readme.txt b/samples/FaceTrack/Readme.txt index e1cc66d..4dc0475 100644 --- a/samples/FaceTrack/Readme.txt +++ b/samples/FaceTrack/Readme.txt @@ -47,5 +47,4 @@ Either the forward slash (/) or back slash (\) can be used as a separator betwee The ConvertSurreyFaceModel.exe file is distributed in the https://github.com/nvidia/BROADCAST-AR-SDK repo. 3) The sample application provided with NVIDIA AR SDK requires that the model file be named face_model0.nvf. -Place the face_model0.nvf file in the model folder. By default the models folder is your_sdk_install_path/models -where all models (including *.trtpkg files) are installed, for example: C:\Program Files\NVIDIA Corporation\NVIDIA AR SDK\models. +Place the face_model0.nvf file in the /bin/models folder. diff --git a/samples/utils/FeatureVertexName.cpp b/samples/utils/FeatureVertexName.cpp deleted file mode 100644 index c811647..0000000 --- a/samples/utils/FeatureVertexName.cpp +++ /dev/null @@ -1,187 +0,0 @@ -/*############################################################################### -# -# Copyright(c) 2019 NVIDIA CORPORATION.All Rights Reserved. -# -# NVIDIA CORPORATION and its licensors retain all intellectual property -# and proprietary rights in and to this software, related documentation -# and any modifications thereto.Any use, reproduction, disclosure or -# distribution of this software and related documentation without an express -# license agreement from NVIDIA CORPORATION is strictly prohibited. -# -###############################################################################*/ - -#include "FeatureVertexName.h" -#include -#include - - -// TODO: We should read this from a file -const LandmarkEOSMap LandmarkMapEOS[] = { - { 33, "chin bottom" }, - { 225, "right eyebrow outer-corner" }, - { 229, "right eyebrow between middle and outer corner" }, - { 233, "right eyebrow middle, vertical middle" }, - { 2086, "right eyebrow between middle and inner corner" }, - { 157, "right eyebrow inner-corner" }, - { 590, "left eyebrow inner-corner" }, - { 2091, "left eyebrow between inner corner and middle" }, - { 666, "left eyebrow middle" }, - { 662, "left eyebrow between middle and outer corner" }, - { 658, "left eyebrow outer-corner" }, - { 2842, "bridge of the nose (parallel to upper eye lids)" }, - { 379, "middle of the nose, a bit below the lower eye lids" }, - { 272, "above nose-tip (1cm or so)" }, - { 114, "nose-tip" }, - { 100, "right nostril, below nose, nose-lip junction" }, - { 2794, "nose-lip junction" }, - { 270, "nose-lip junction" }, - { 2797, "nose-lip junction" }, - { 537, "left nostril, below nose, nose-lip junction" }, - { 177, "right eye outer-corner" }, - { 172, "right eye pupil top right (from subject's perspective)" }, - { 191, "right eye pupil top left" }, - { 181, "right eye inner-corner" }, - { 173, "right eye pupil bottom left" }, - { 174, "right eye pupil bottom right" }, - { 614, "left eye inner-corner" }, - { 624, "left eye pupil top right" }, - { 605, "left eye pupil top left" }, - { 610, "left eye outer-corner" }, - { 607, "left eye pupil bottom left" }, - { 606, "left eye pupil bottom right" }, - { 398, "right mouth corner" }, - { 315, "upper lip right top outer" }, - { 413, "upper lip middle top right" }, - { 329, "upper lip middle top" }, - { 825, "upper lip middle top left" }, - { 736, "upper lip left top outer" }, - { 812, "left mouth corner" }, - { 841, "lower lip left bottom outer" }, - { 693, "lower lip middle bottom left" }, - { 411, "lower lip middle bottom" }, - { 264, "lower lip middle bottom right" }, - { 431, "lower lip right bottom outer" }, - { 416, "upper lip right bottom outer" }, - { 423, "upper lip middle bottom" }, - { 828, "upper lip left bottom outer" }, - { 817, "lower lip left top outer" }, - { 442, "lower lip middle top" }, - { 404, "lower lip right top outer" }, - { 0xFFFF, nullptr } -}; - -static const LandmarksMap LandmarkMap[] = { - { 0, 0, "right contour point 1" }, - { 1, 2, "right contour point 2" }, - { 2, 4, "right contour point 3" }, - { 3, 6, "right contour point 4" }, - { 4, 8, "right contour point 5" }, - { 5, 10, "right contour point 6" }, - { 6, 12, "right contour point 7" }, - { 7, 14, "right contour point 8" }, - { 8, 16, "chin bottom" }, - { 9, 18, "left contour point 1" }, - { 10, 20, "left contour point 2" }, - { 11, 22, "left contour point 3" }, - { 12, 24, "left contour point 4" }, - { 13, 26, "left contour point 5" }, - { 14, 28, "left contour point 6" }, - { 15, 30, "left contour point 7" }, - { 16, 32, "left contour point 8" }, - { 17, 33, "right eyebrow outer-corner" }, - { 18, 34, "right eyebrow between middle and outer corner" }, - { 19, 35, "right eyebrow middle, vertical middle" }, - { 20, 36, "right eyebrow between middle and inner corner" }, - { 21, 37, "right eyebrow inner-corner" }, - { 22, 42, "left eyebrow inner-corner" }, - { 23, 43, "left eyebrow between inner corner and middle" }, - { 24, 44, "left eyebrow middle" }, - { 25, 45, "left eyebrow between middle and outer corner" }, - { 26, 46, "left eyebrow outer-corner" }, - { 27, 51, "bridge of the nose (parallel to upper eye lids)" }, - { 28, 52, "middle of the nose, a bit below the lower eye lids" }, - { 29, 53, "above nose-tip (1cm or so)" }, - { 30, 54, "nose-tip" }, - { 31, 57, "right nostril, below nose, nose-lip junction" }, - { 32, 58, "nose-lip junction" }, - { 33, 59, "nose-lip junction" }, - { 34, 60, "nose-lip junction" }, - { 35, 61, "left nostril, below nose, nose-lip junction" }, - { 36, 64, "right eye outer-corner" }, - { 37, 65, "right eye pupil top right (from subject's perspective)" }, - { 38, 67, "right eye pupil top left" }, - { 39, 68, "right eye inner-corner" }, - { 40, 69, "right eye pupil bottom left" }, - { 41, 71, "right eye pupil bottom right" }, - { 42, 81, "left eye inner-corner" }, - { 43, 82, "left eye pupil top right" }, - { 44, 84, "left eye pupil top left" }, - { 45, 85, "left eye outer-corner" }, - { 46, 86, "left eye pupil bottom left" }, - { 47, 88, "left eye pupil bottom right" }, - { 48, 98, "right mouth corner" }, - { 49, 99, "upper lip right top outer" }, - { 50, 100, "upper lip middle top right" }, - { 51, 101, "upper lip middle top" }, - { 52, 102, "upper lip middle top left" }, - { 53, 103, "upper lip left top outer" }, - { 54, 104, "left mouth corner" }, - { 55, 105, "lower lip left bottom outer" }, - { 56, 106, "lower lip middle bottom left" }, - { 57, 107, "lower lip middle bottom" }, - { 58, 108, "lower lip middle bottom right" }, - { 59, 109, "lower lip right bottom outer" }, - { 60, 110, "right inner mouth corner "}, - { 61, 111, "upper lip right bottom outer" }, - { 62, 112, "upper lip middle bottom" }, - { 63, 113, "upper lip left bottom outer" }, - { 64, 114, "left inner mouth corner"}, - { 65, 115, "lower lip left top outer" }, - { 66, 116, "lower lip middle top" }, - { 67, 117, "lower lip right top outer" }, - { 0xFFFF, 0xFFFF, nullptr } -}; - - -unsigned short FindEOSLandmarkIndexFromName(const char* name) -{ - if (!name) - return 0xFFFF; - switch (name[0]) { - case '#': // 1-based index ... - return (unsigned short)(strtol(name + 1, nullptr, 10) - 1); // ... gets converted into a 0-based index - case '@': // 0-based index - return (unsigned short)strtol(name + 1, nullptr, 10); - default: - break; - } - const LandmarkEOSMap* lmList = LandmarkMapEOS; - for (; lmList->name != nullptr; ++lmList) - if (!strcmp(name, lmList->name)) - break; - return lmList->index; -} - -unsigned short FindLandmarkIndexFromName(const unsigned int numLandmarks, const char* name) { - if (!name) - return 0xFFFF; - switch (name[0]) { - case '#': // 1-based index ... - return (unsigned short)(strtol(name + 1, nullptr, 10) - 1); // ... gets converted into a 0-based index - case '@': // 0-based index - return (unsigned short)strtol(name + 1, nullptr, 10); - default: - break; - } - const LandmarksMap* lmList = LandmarkMap; - for (; lmList->name != nullptr; ++lmList) - if (!strcmp(name, lmList->name)) - break; - if (numLandmarks == 68) { - return lmList->index_68; - } else if (numLandmarks == 126) { - return lmList->index_126; - } else { - return 0xFFFF; - } -} diff --git a/samples/utils/FeatureVertexName.h b/samples/utils/FeatureVertexName.h deleted file mode 100644 index d61acc3..0000000 --- a/samples/utils/FeatureVertexName.h +++ /dev/null @@ -1,23 +0,0 @@ -/*############################################################################### -# -# Copyright(c) 2019 NVIDIA CORPORATION.All Rights Reserved. -# -# NVIDIA CORPORATION and its licensors retain all intellectual property -# and proprietary rights in and to this software, related documentation -# and any modifications thereto.Any use, reproduction, disclosure or -# distribution of this software and related documentation without an express -# license agreement from NVIDIA CORPORATION is strictly prohibited. -# -###############################################################################*/ - -#ifndef __FEATURE_VERTEX_NAME__ -#define __FEATURE_VERTEX_NAME__ - - -struct LandmarkEOSMap { unsigned short index; const char *name; }; -struct LandmarksMap { unsigned short index_68; unsigned short index_126; const char* name; }; - -unsigned short FindEOSLandmarkIndexFromName(const char *name); -unsigned short FindLandmarkIndexFromName(const unsigned int numLandmarks, const char* name); - -#endif /* __FEATURE_VERTEX_NAME__ */ diff --git a/samples/utils/nvCVOpenCV.h b/samples/utils/nvCVOpenCV.h index cdfbe1a..188012f 100644 --- a/samples/utils/nvCVOpenCV.h +++ b/samples/utils/nvCVOpenCV.h @@ -21,9 +21,6 @@ # ###############################################################################*/ -#ifndef __NVCVOPENCV_H__ -#define __NVCVOPENCV_H__ - #include "nvCVImage.h" #include "opencv2/opencv.hpp" @@ -76,5 +73,3 @@ inline void NVWrapperForCVMat(const cv::Mat *cvIm, NvCVImage *nvcvIm) { nvcvIm->reserved[0] = 0; nvcvIm->reserved[1] = 0; } - -#endif // __NVCVOPENCV_H__ \ No newline at end of file diff --git a/tools/ConvertSurreyFaceModel.exe b/tools/ConvertSurreyFaceModel.exe index 205b48a..0ae6758 100644 Binary files a/tools/ConvertSurreyFaceModel.exe and b/tools/ConvertSurreyFaceModel.exe differ diff --git a/version.h b/version.h index 75f9976..2a24909 100644 --- a/version.h +++ b/version.h @@ -22,12 +22,12 @@ ###############################################################################*/ #define NVIDIA_AR_SDK_VERSION_MAJOR 0 -#define NVIDIA_AR_SDK_VERSION_MINOR 6 -#define NVIDIA_AR_SDK_VERSION_RELEASE 1 +#define NVIDIA_AR_SDK_VERSION_MINOR 5 +#define NVIDIA_AR_SDK_VERSION_RELEASE 0 -#define NVIDIA_AR_SDK_VERSION 0,6,1,0 -#define NVIDIA_AR_SDK_VERSION_MAJOR_MINOR 0,6 -#define NVIDIA_AR_SDK_VERSION_STRING "0.6.1.0" -#define NVIDIA_AR_SDK_VERSION_STRING_SHORT "0.6.1" -#define NVIDIA_AR_SDK_VERSION_STRING_MAJOR_MINOR "0.6" +#define NVIDIA_AR_SDK_VERSION 0,5,0,0 +#define NVIDIA_AR_SDK_VERSION_MAJOR_MINOR 0,5 +#define NVIDIA_AR_SDK_VERSION_STRING "0.5.0.0" +#define NVIDIA_AR_SDK_VERSION_STRING_SHORT "0.5.0" +#define NVIDIA_AR_SDK_VERSION_STRING_MAJOR_MINOR "0.5"