diff --git a/CMakeLists.txt b/CMakeLists.txt index 5e60ef4..4309db4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.12.0) +cmake_minimum_required(VERSION 3.10) # Set path where samples will be installed set(CMAKE_INSTALL_PREFIX ${CMAKE_SOURCE_DIR} CACHE PATH "Path to where the samples will be installed") @@ -9,7 +9,7 @@ project(NvVideoEffects_SDK CXX) set(CMAKE_CONFIGURATION_TYPES "Release") # Require C++11 and disable non-standard extensions -set(CMAKE_CXX_STANDARD 14) +set(CMAKE_CXX_STANDARD 11) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) @@ -19,12 +19,47 @@ add_definitions(-DNOMINMAX -DWIN32_LEAN_AND_MEAN) set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}) set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}) +set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake" ${CMAKE_MODULE_PATH}) -set(SDK_INCLUDES_PATH ${CMAKE_CURRENT_SOURCE_DIR}/nvvfx/include) -# Add target for NVVideoEffects -add_library(NVVideoEffects INTERFACE) -target_include_directories(NVVideoEffects INTERFACE ${SDK_INCLUDES_PATH}) +if(MSVC) + + set(SDK_INCLUDES_PATH ${CMAKE_CURRENT_SOURCE_DIR}/nvvfx/include) + # Add target for NVVideoEffects + add_library(NVVideoEffects INTERFACE) + target_include_directories(NVVideoEffects INTERFACE ${SDK_INCLUDES_PATH}) + +else() + # Add target for NVVideoEffects + add_library(NVVideoEffects INTERFACE) + + # found in different locations depending on type of package + find_path(VideoFX_INCLUDES + NAMES nvVideoEffects.h + PATHS + /usr/local/VideoFX/include + /usr/include/x86_64-linux-gnu + /usr/include + REQUIRED + ) + + target_include_directories(NVVideoEffects INTERFACE ${VideoFX_INCLUDES}) + set(SDK_INCLUDES_PATH ${VideoFX_INCLUDES}) + + find_library(VideoFX_LIB + NAMES libVideoFX.so + PATHS + /usr/local/VideoFX/lib + /usr/lib/x86_64-linux-gnu + /usr/lib64 + /usr/lib + REQUIRED + NO_DEFAULT_PATH) + + target_link_libraries(NVVideoEffects INTERFACE "${VideoFX_LIB}") + + message(STATUS "VideoFX_LIB: ${VideoFX_LIB}") + message(STATUS "SDK_INCLUDES_PATH: ${SDK_INCLUDES_PATH}") + +endif() -set(ENABLE_SAMPLES TRUE) add_subdirectory(samples) - diff --git a/LICENSE b/LICENSE index 9102747..8e4b6ed 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2020 NVIDIA Corporation +Copyright (c) 2021 NVIDIA Corporation Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/README.MD b/README.MD index 5475b6f..253e009 100644 --- a/README.MD +++ b/README.MD @@ -1,47 +1,56 @@ # README -## NVIDIA VideoEffects SDK: API Source Code and Sample Applications +## NVIDIA MAXINE VideoEffects SDK: API Source Code and Sample Applications -NVIDIA VideoEffects SDK is an SDK for enhancing and applying filters to videos at real-time. The SDK is powered by NVIDIA graphics processing units (GPUs) with Tensor Cores, and as a result, the algorithm throughput is greatly accelerated, and latency is reduced. +NVIDIA MAXINE VideoEffects SDK is an SDK for enhancing and applying filters to videos at real-time. The SDK is powered by NVIDIA graphics processing units (GPUs) with Tensor Cores, and as a result, the algorithm throughput is greatly accelerated, and latency is reduced. -NVIDIA VideoEffects SDK has the following AI features: +The SDK has the following AI features: +- **AI Green Screen**, which segments and masks the background areas in a video or image. +- **Background Blur**, which uses the segmentation mask from the AI Green Screen filter or other sources, and produces a blur effect over the background of a video or iamge. - **Encoder Artifact Reduction**, which reduces the blocky and noisy artifacts from an encoded video while preserving the details of the original video. - **Super Resolution**, which upscales a video while also reducing the blocky and noisy artifacts. It can enhance the details and sharpen the output while simultaneously preserving the content. This is suitable for upscaling lossy content. - **Upscale**, which is a very fast and light-weight method for upscaling an input video. It also provides a sharpening parameter to sharpen the resulting output. This feature can be optionally pipelined with the encoder artifact reduction feature to enhance the scale while reducing the video artifacts. +- **Webcam Denoising**, which removes noise from a webcam video while preserving the texture details.

-NVIDIA Super Resolution +NVIDIA Super Resolution

-NVIDIA VideoEffects SDK provides two sample applications that demonstrate the features listed above in real time by using offline videos. -- **VideoEffects App**, which is a sample app that can invoke each feature individually. +

+NVIDIA Webcam Denoising +

+ +The SDK provides several sample applications that demonstrate the features listed above in real time by using offline videos. +- **AI Green Screen App**, which is a sample app that demonstrates the background segmentation feature. +- **VideoEffects App**, which is a sample app that can invoke each of Encoder Artifact Reduction, Super Resolution or Upscale features individually. - **UpscalePipeline App**, which is a sample app that pipelines the Encoder Artifact Reduction feature with the Upscale feature. +- **DenoiseEffect App**, which is a sample app that demonstrates the webcam denoising feature. -All features in the VideoEffects SDK support 720p and 1080p as input resolutions. These are the scaling factors supported by the Super Resolution feature: -- **720p inputs** can be scaled by a factor of 1.5x or 2x. -- **1080p inputs** can be scaled by a factor of 4/3x (~1.33x) or 2x. +The input and output resolutions supported by the features of the SDK are listed below. +- The Super Resolution and Encoder Artifact Reduction features support between 90p to 1080p as input resolutions. + - Super Resolution supports the following scaling factors: 4/3x (~1.33x), 1.5x, 2x, 3x and 4x. + - The maximum output resolution for the Super Resolution feature is 2160p. +- The Upscale feature supports any input resolution, and the following scaling factors: 4/3x (~1.33x), 1.5x, 2x, 3x and 4x. +- The Webcam Denoising feature supports between 80p to 1080p as input resolutions. +- The AI Green Screen and Background Blur features require that an input image/video be at least 288 pixels high. -Additionally, the Upscale feature supports any input resolution, and the following scaling factors: -- 4/3x (~1.33x), 1.5x, 2x or 3x +NVIDIA MAXINE VideoEffects SDK is distributed in the following parts: -NVIDIA VideoEffects SDK is distributed in the following parts: +- This open source repository that includes the [SDK API and proxy linking source code](https://github.com/NVIDIA/MAXINE-VFX-SDK/tree/master/nvvfx), and [sample applications and their dependency libraries](https://github.com/NVIDIA/MAXINE-VFX-SDK/tree/master/samples). +- An installer hosted on [NVIDIA Maxine developer page](https://www.nvidia.com/broadcast-sdk-resources) that installs the SDK DLLs, the models, and the SDK dependency libraries. -- This open source repository that includes the [SDK API and proxy linking source code](https://github.com/NVIDIA/BROADCAST-VFX-SDK/tree/master/nvvfx), and [sample applications and their dependency libraries](https://github.com/NVIDIA/BROADCAST-VFX-SDK/tree/master/samples). -- An installer hosted on [RTX broadcast engine developer page](https://developer.nvidia.com/rtx-broadcast-engine) that installs the SDK DLLs, the models, and the SDK dependency libraries. - -Please refer to [SDK programming guide](https://github.com/NVIDIA/BROADCAST-VFX-SDK/blob/master/docs/NVIDIA%20Video%20Effects%20SDK%20Programming%20Guide.pdf) for configuring and integrating the SDK, compiling and running the sample applications. +Please refer to [SDK programming guide](https://github.com/NVIDIA/MAXINE-VFX-SDK/blob/master/docs/NVIDIA%20Video%20Effects%20SDK%20Programming%20Guide.pdf) for configuring and integrating the SDK, compiling and running the sample applications. Please visit the [NVIDIA MAXINE Video Effects SDK](https://developer.nvidia.com/maxine-getting-started) webpage for more information about the SDK. ## System requirements -The SDK is supported on NVIDIA GPUs that are based on the NVIDIA® Turing™ architecture. Although the SDK can run on Turing™ GPUs without Tensor Cores, it is optimized for much higher performance on GPUs with Tensor Cores. +The SDK is supported on NVIDIA GPUs that are based on the NVIDIA® Turing™ or Ampere™ architecture and have Tensor Cores. * Windows OS supported: 64-bit Windows 10 * Microsoft Visual Studio: 2017 (MSVC15.0) or later * CMake: v3.12 or later * NVIDIA Graphics Driver for Windows: 455.57 or later -## NVIDIA Branding Guidelines -If you integrate an NVIDIA Broadcast Engine SDK within your product, please follow the required branding guidelines that are available [here]( -https://nvidia.frontify.com/d/uAobRitG8H8B) +## NVIDIA MAXINE Branding Guidelines +If you integrate an NVIDIA MAXINE SDK within your product, please follow the required branding guidelines that are available [here](https://www.nvidia.com/maxine-sdk-guidelines/) ## Compiling the sample apps @@ -49,7 +58,7 @@ https://nvidia.frontify.com/d/uAobRitG8H8B) The open source repository includes the source code to build the sample applications, and a proxy file NVVideoEffectsProxy.cpp to enable compilation without explicitly linking against the SDK DLL. -**Note: To download the models and runtime dependencies required by the features, you need to run the [SDK Installer](https://developer.nvidia.com/rtx-broadcast-engine).** +**Note: To download the models and runtime dependencies required by the features, you need to run the [SDK Installer](https://www.nvidia.com/broadcast-sdk-resources).** 1. In the root folder of the downloaded source code, start the CMake GUI and specify the source folder and a build folder for the binary files. * For the source folder, ensure that the path ends in OSS. diff --git a/docs/NVIDIA Video Effects SDK Programming Guide.pdf b/docs/NVIDIA Video Effects SDK Programming Guide.pdf index d196dd0..c779dc4 100644 Binary files a/docs/NVIDIA Video Effects SDK Programming Guide.pdf and b/docs/NVIDIA Video Effects SDK Programming Guide.pdf differ diff --git a/nvvfx/include/nvCVImage.h b/nvvfx/include/nvCVImage.h index 7f07360..c5ed9ed 100644 --- a/nvvfx/include/nvCVImage.h +++ b/nvvfx/include/nvCVImage.h @@ -1,6 +1,6 @@ /*############################################################################### # -# Copyright 2020 NVIDIA Corporation +# Copyright 2020-2021 NVIDIA Corporation # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -30,6 +30,12 @@ extern "C" { #endif // ___cplusplus + +#ifndef RTX_CAMERA_IMAGE // Compile with -DRTX_CAMERA_IMAGE=0 to get more functionality and bug fixes. + #define RTX_CAMERA_IMAGE 0 // Set to 1 for RTXCamera, which needs an old version, that avoids new functionality +#endif // RTX_CAMERA_IMAGE + + struct CUstream_st; // typedef struct CUstream_st *CUstream; //! The format of pixels in an image. @@ -42,8 +48,16 @@ typedef enum NvCVImage_PixelFormat { NVCV_BGR = 5, //!< { Red, Green, Blue } NVCV_RGBA = 6, //!< { Red, Green, Blue, Alpha } NVCV_BGRA = 7, //!< { Red, Green, Blue, Alpha } +#if RTX_CAMERA_IMAGE NVCV_YUV420 = 8, //!< Luminance and subsampled Chrominance { Y, Cb, Cr } NVCV_YUV422 = 9, //!< Luminance and subsampled Chrominance { Y, Cb, Cr } +#else // !RTX_CAMERA_IMAGE + NVCV_ARGB = 8, //!< { Red, Green, Blue, Alpha } + NVCV_ABGR = 9, //!< { Red, Green, Blue, Alpha } + NVCV_YUV420 = 10, //!< Luminance and subsampled Chrominance { Y, Cb, Cr } + NVCV_YUV422 = 11, //!< Luminance and subsampled Chrominance { Y, Cb, Cr } +#endif // !RTX_CAMERA_IMAGE + NVCV_YUV444 = 12, //!< Luminance and full bandwidth Chrominance { Y, Cb, Cr } } NvCVImage_PixelFormat; @@ -80,35 +94,51 @@ typedef enum NvCVImage_ComponentType { #define NVCV_VYUY 4 //!< [VYUY] Chunky 4:2:2 #define NVCV_YUYV 6 //!< [YUYV] Chunky 4:2:2 #define NVCV_YVYU 8 //!< [YVYU] Chunky 4:2:2 -#define NVCV_YUV 3 //!< [Y][U][V] Planar 4:2:2 or 4:2:0 -#define NVCV_YVU 5 //!< [Y][V][U] Planar 4:2:2 or 4:2:0 +#define NVCV_CYUV 10 //!< [YUV] Chunky 4:4:4 +#define NVCV_CYVU 12 //!< [YVU] Chunky 4:4:4 +#define NVCV_YUV 3 //!< [Y][U][V] Planar 4:2:2 or 4:2:0 or 4:4:4 +#define NVCV_YVU 5 //!< [Y][V][U] Planar 4:2:2 or 4:2:0 or 4:4:4 #define NVCV_YCUV 7 //!< [Y][UV] Semi-planar 4:2:2 or 4:2:0 (default for 4:2:0) #define NVCV_YCVU 9 //!< [Y][VU] Semi-planar 4:2:2 or 4:2:0 + +//! The following are FOURCC aliases for specific layouts. Note that it is still required to specify the format as well +//! as the layout, e.g. NVCV_YUV420 and NVCV_NV12, even though the NV12 layout is only associated with YUV420 sampling. +#define NVCV_I420 NVCV_YUV //!< [Y][U][V] Planar 4:2:0 +#define NVCV_IYUV NVCV_YUV //!< [Y][U][V] Planar 4:2:0 +#define NVCV_YV12 NVCV_YVU //!< [Y][V][U] Planar 4:2:0 +#define NVCV_NV12 NVCV_YCUV //!< [Y][UV] Semi-planar 4:2:0 (default for 4:2:0) +#define NVCV_NV21 NVCV_YCVU //!< [Y][VU] Semi-planar 4:2:0 #define NVCV_YUY2 NVCV_YUYV //!< [YUYV] Chunky 4:2:2 -#define NVCV_I420 NVCV_YUV //!< [Y][U][V] Planar 4:2:2 or 4:2:0 -#define NVCV_IYUV NVCV_YUV //!< [Y][U][V] Planar 4:2:2 or 4:2:0 -#define NVCV_YV12 NVCV_YVU //!< [Y][V][U] Planar 4:2:2 or 4:2:0 -#define NVCV_NV12 NVCV_YCUV //!< [Y][UV] Semi-planar 4:2:2 or 4:2:0 (default for 4:2:0) -#define NVCV_NV21 NVCV_YCVU //!< [Y][VU] Semi-planar 4:2:2 or 4:2:0 +#define NVCV_I444 NVCV_YUV //!< [Y][U][V] Planar 4:4:4 +#define NVCV_YM24 NVCV_YUV //!< [Y][U][V] Planar 4:4:4 +#define NVCV_YM42 NVCV_YVU //!< [Y][V][U] Planar 4:4:4 +#define NVCV_NV24 NVCV_YCUV //!< [Y][UV] Semi-planar 4:4:4 +#define NVCV_NV42 NVCV_YCVU //!< [Y][VU] Semi-planar 4:4:4 //! The following are ORed together for the colorspace field for YUV. //! NVCV_601 and NVCV_709 describe the color axes of YUV. //! NVCV_VIDEO_RANGE and NVCV_VIDEO_RANGE describe the range, [16, 235] or [0, 255], respectively. //! NVCV_CHROMA_COSITED and NVCV_CHROMA_INTSTITIAL describe the location of the chroma samples. -#define NVCV_601 0 //!< The Rec.601 YUV colorspace, typically used for SD. -#define NVCV_709 1 //!< The Rec.709 YUV colorspace, typically used for HD. -#define NVCV_VIDEO_RANGE 0 //!< The video range is [16, 235]. -#define NVCV_FULL_RANGE 4 //!< The video range is [ 0, 255]. -#define NVCV_CHROMA_COSITED 0 //!< The chroma is sampled at the same location as the luma samples horizontally. -#define NVCV_CHROMA_INTSTITIAL 8 //!< The chroma is sampled between luma samples horizontally. -#define NVCV_CHROMA_MPEG2 NVCV_CHROMA_COSITED +#define NVCV_601 0x00 //!< The Rec.601 YUV colorspace, typically used for SD. +#define NVCV_709 0x01 //!< The Rec.709 YUV colorspace, typically used for HD. +#define NVCV_2020 0x02 //!< The Rec.2020 YUV colorspace. +#define NVCV_VIDEO_RANGE 0x00 //!< The video range is [16, 235]. +#define NVCV_FULL_RANGE 0x04 //!< The video range is [ 0, 255]. +#define NVCV_CHROMA_COSITED 0x00 //!< The chroma is sampled at the same location as the luma samples horizontally. +#define NVCV_CHROMA_INTSTITIAL 0x08 //!< The chroma is sampled between luma samples horizontally. +#define NVCV_CHROMA_TOPLEFT 0x10 //!< The chroma is sampled at the same location as the luma samples horizontally and vertically. +#define NVCV_CHROMA_MPEG2 NVCV_CHROMA_COSITED //!< As is most video. #define NVCV_CHROMA_MPEG1 NVCV_CHROMA_INTSTITIAL +#define NVCV_CHROMA_JPEG NVCV_CHROMA_INTSTITIAL +#define NVCV_CHROMA_H261 NVCV_CHROMA_INTSTITIAL +#define NVCV_CHROMA_INTERSTITIAL NVCV_CHROMA_INTSTITIAL //!< Correct spelling //! This is the value for the gpuMem field or the memSpace argument. -#define NVCV_CPU 0 //!< The buffer is stored in CPU memory. -#define NVCV_GPU 1 //!< The buffer is stored in CUDA memory. -#define NVCV_CUDA 1 //!< The buffer is stored in CUDA memory. +#define NVCV_CPU 0 //!< The buffer is stored in CPU memory. +#define NVCV_GPU 1 //!< The buffer is stored in CUDA memory. +#define NVCV_CUDA 1 //!< The buffer is stored in CUDA memory. #define NVCV_CPU_PINNED 2 //!< The buffer is stored in pinned CPU memory. +#define NVCV_CUDA_ARRAY 3 //!< A CUDA array is used for storage. //! Image descriptor. typedef struct @@ -126,7 +156,7 @@ NvCVImage { unsigned char numComponents; //!< The number of components in each pixel. unsigned char planar; //!< NVCV_CHUNKY, NVCV_PLANAR, NVCV_UYVY, .... unsigned char gpuMem; //!< NVCV_CPU, NVCV_CPU_PINNED, NVCV_CUDA, NVCV_GPU - unsigned char colorspace; //!< an OR of colorspace, range and chroma phase. + unsigned char colorspace; //!< An OR of colorspace, range and chroma phase. unsigned char reserved[2]; //!< For structure padding and future expansion. Set to 0. void *pixels; //!< Pointer to pixel(0,0) in the image. void *deletePtr; //!< Buffer memory to be deleted (can be NULL). @@ -179,8 +209,6 @@ NvCVImage { //! \return NVCV_ERR_MISMATCH if the formats are different //! \return NVCV_ERR_CUDA if a CUDA error occurred //! \return NVCV_ERR_PIXELFORMAT if the pixel format is not yet accommodated. - //! \bug This does not work for planar or semi-planar formats, neither RGB nor YUV. - //! \note This does work for all chunky formats, including UYVY, VYUY, YUYV, YVYU. inline NvCV_Status copyFrom(const NvCVImage *src, int srcX, int srcY, int dstX, int dstY, unsigned width, unsigned height); //! Copy from one image to another. This works for CPU->CPU, CPU->GPU, GPU->GPU, and GPU->CPU. @@ -196,6 +224,22 @@ NvCVImage { } NvCVImage; +//! Integer rectangle. +typedef struct NvCVRect2i { + int x; //!< The left edge of the rectangle. + int y; //!< The top edge of the rectangle. + int width; //!< The width of the rectangle. + int height; //!< The height of the rectangle. +} NvCVRect2i; + + +//! Integer point. +typedef struct NvCVPoint2i { + int x; //!< The horizontal coordinate. + int y; //!< The vertical coordinate +} NvCVPoint2i; + + //! Initialize an image. The C++ constructors can initialize this appropriately. //! This is called by the C++ constructor, but C code should call this explicitly. //! \param[in,out] im the image to initialize. @@ -221,8 +265,12 @@ NvCV_Status NvCV_API NvCVImage_Init(NvCVImage *im, unsigned width, unsigned heig //! \param[in] y the top edge of the sub-image, as coordinate of the full image. //! \param[in] width the desired width of the subImage, in pixels. //! \param[in] height the desired height of the subImage, in pixels. -//! \bug This does not work for planar or semi-planar formats, neither RGB nor YUV. +//! \bug This does not work in general for planar or semi-planar formats, neither RGB nor YUV. +//! However, it does work for all formats with the full image, to make a shallow copy, e.g. +//! NvCVImage_InitView(&subImg, &fullImg, 0, 0, fullImage.width, fullImage.height). +//! Cropping a planar or semi-planar image can be accomplished with NvCVImage_TransferRect(). //! \note This does work for all chunky formats, including UYVY, VYUY, YUYV, YVYU. +//! \sa { NvCVImage_TransferRect } void NvCV_API NvCVImage_InitView(NvCVImage *subImg, NvCVImage *fullImg, int x, int y, unsigned width, unsigned height); @@ -310,36 +358,52 @@ void NvCV_API NvCVImage_ComponentOffsets(NvCVImage_PixelFormat format, int *rOff //! //! If any of the images resides on the GPU, it may run asynchronously, //! so cudaStreamSynchronize() should be called if it is necessary to run synchronously. -//! The following table indicates the currently-implemented conversions: -//! +------------------+-------------+-------------+-------------+-------------+ -//! | | u8 --> u8 | u8 --> f32 | f32 --> u8 | f32 --> f32 | -//! +------------------+-------------+-------------+-------------+-------------+ -//! | Y -- > Y | X | | X | X | -//! | Y -- > A | X | | X | X | -//! | Y -- > RGB | X | X | X | X | -//! | Y -- > RGBA | X | X | X | X | -//! | A -- > Y | X | | X | X | -//! | A -- > A | X | | X | X | -//! | A -- > RGB | X | X | X | X | -//! | A -- > RGBA | X | | | | -//! | RGB -- > Y | X | X | | | -//! | RGB -- > A | X | X | | | -//! | RGB -- > RGB | X | X | X | X | -//! | RGB -- > RGBA | X | X | X | X | -//! | RGBA -- > Y | X | X | | | -//! | RGBA -- > A | | X | | | -//! | RGBA -- > RGB | X | X | X | X | -//! | RGBA -- > RGBA | X | | | | -//! | YUV420 -- > RGB | X | | | | -//! | YUV422 -- > RGB | X | | | | -//! +------------------+-------------+-------------+-------------+-------------+ +//! The following table indicates (with X) the currently-implemented conversions: +//! +-------------------+-------------+-------------+-------------+-------------+ +//! | | u8 --> u8 | u8 --> f32 | f32 --> u8 | f32 --> f32 | +//! +-------------------+-------------+-------------+-------------+-------------+ +//! | Y --> Y | X | | X | X | +//! | Y --> A | X | | X | X | +//! | Y --> RGB | X | X | X | X | +//! | Y --> RGBA | X | X | X | X | +//! | A --> Y | X | | X | X | +//! | A --> A | X | | X | X | +//! | A --> RGB | X | X | X | X | +//! | A --> RGBA | X | | | | +//! | RGB --> Y | X | X | | | +//! | RGB --> A | X | X | | | +//! | RGB --> RGB | X | X | X | X | +//! | RGB --> RGBA | X | X | X | X | +//! | RGBA --> Y | X | X | | | +//! | RGBA --> A | | X | | | +//! | RGBA --> RGB | X | X | X | X | +//! | RGBA --> RGBA | X | X | X | X | +//! | RGB --> YUV420 | X | | X | | +//! | RGBA --> YUV420 | X | | X | | +//! | RGB --> YUV422 | X | | X | | +//! | RGBA --> YUV422 | X | | X | | +//! | RGB --> YUV444 | X | | X | | +//! | RGBA --> YUV444 | X | | X | | +//! | YUV420 --> RGB | X | X | | | +//! | YUV420 --> RGBA | X | X | | | +//! | YUV422 --> RGB | X | X | | | +//! | YUV422 --> RGBA | X | X | | | +//! | YUV444 --> RGB | X | X | | | +//! | YUV444 --> RGBA | X | X | | | +//! +-------------------+-------------+-------------+-------------+-------------+ //! where //! * Either source or destination can be CHUNKY or PLANAR. //! * Either source or destination can reside on the CPU or the GPU. //! * The RGB components are in any order (i.e. RGB or BGR; RGBA or BGRA). -//! * YUV requires that the colorspace field be set manually prior to Transfer. +//! * For RGBA (or BGRA) destinations, most implementations do not change the alpha channel, so it is recommended to +//! set it at initialization time with [cuda]memset(im.pixels, -1, im.pitch * im.height) or +//! [cuda]memset(im.pixels, -1, im.pitch * im.height * im.numComponents) for chunky and planar images respectively. +//! * YUV requires that the colorspace field be set manually prior to Transfer, e.g. typical for layout=NVCV_NV12: +//! image.colorspace = NVCV_709 | NVCV_VIDEO_RANGE | NVCV_CHROMA_INTSTITIAL; +//! * There are also RGBf16-->RGBf32 and RGBf32-->RGBf16 transfers. //! * Additionally, when the src and dst formats are the same, all formats are accommodated on CPU and GPU, -//! and this can be used as a replacement for cudaMemcpy2DAsync() (which it utilizes). +//! and this can be used as a replacement for cudaMemcpy2DAsync() (which it utilizes). This is also true for YUV, +//! whose src and dst must share the same format, layout and colorspace. //! //! When there is some kind of conversion AND the src and dst reside on different processors (CPU, GPU), //! it is necessary to have a temporary GPU buffer, which is reshaped as needed to match the characteristics @@ -368,17 +432,139 @@ NvCV_Status NvCV_API NvCVImage_Transfer( const NvCVImage *src, NvCVImage *dst, float scale, struct CUstream_st *stream, NvCVImage *tmp); -//! Composite one BGRu8 source image over another using the given matte. -//! \param[in] fg the foreground source BGRu8 (or RGBu8) image. -//! \param[in] bg the background source BGRu8 (or RGBu8) image. +//! Transfer a rectangular portion of an image. +//! See NvCVImage_Transfer() for the pixel format combinations that are implemented. +//! \param[in] src the source image. +//! \param[in] srcRect the subRect of the src to be transferred (NULL implies the whole image). +//! \param[out] dst the destination image. +//! \param[in] dstPt location to which the srcRect is to be copied (NULL implies (0,0)). +//! \param[in] scale scale factor applied to the magnitude during transfer, typically 1, 255 or 1/255. +//! \param[in] stream the CUDA stream. +//! \param[in] tmp a staging image. +//! \return NVCV_SUCCESS if the operation was completed successfully. +//! \note The actual transfer region may be smaller, because the rects are clipped against the images. +NvCV_Status NvCV_API NvCVImage_TransferRect( + const NvCVImage *src, const NvCVRect2i *srcRect, NvCVImage *dst, const NvCVPoint2i *dstPt, + float scale, struct CUstream_st *stream, NvCVImage *tmp); + + +//! Transfer from a YUV image. +//! YUVu8 --> RGBu8 and YUVu8 --> RGBf32 are currently available. +//! \param[in] y pointer to pixel(0,0) of the luminance channel. +//! \param[in] yPixBytes the byte stride between y pixels horizontally. +//! \param[in] yPitch the byte stride between y pixels vertically. +//! \param[in] u pointer to pixel(0,0) of the u (Cb) chrominance channel. +//! \param[in] v pointer to pixel(0,0) of the v (Cr) chrominance channel. +//! \param[in] uvPixBytes the byte stride between u or v pixels horizontally. +//! \param[in] uvPitch the byte stride between u or v pixels vertically. +//! \param[in] yuvColorSpace the yuv colorspace, specifying range, chromaticities, and chrominance phase. +//! \param[in] yuvMemSpace the memory space where the pixel buffers reside. +//! \param[out] dst the destination image. +//! \param[in] dstRect the destination rectangle (NULL implies the whole image). +//! \param[in] scale scale factor applied to the magnitude during transfer, typically 1, 255 or 1/255. +//! \param[in] stream the CUDA stream. +//! \param[in] tmp a staging image. +//! \return NVCV_SUCCESS if the operation was completed successfully. +//! \note The actual transfer region may be smaller, because the rects are clipped against the images. +NvCV_Status NvCV_API NvCVImage_TransferFromYUV( + const void *y, int yPixBytes, int yPitch, + const void *u, const void *v, int uvPixBytes, int uvPitch, + NvCVImage_PixelFormat yuvFormat, NvCVImage_ComponentType yuvType, + unsigned yuvColorSpace, unsigned yuvMemSpace, + NvCVImage *dst, const NvCVRect2i *dstRect, float scale, struct CUstream_st *stream, NvCVImage *tmp); + + +//! Transfer to a YUV image. +//! RGBu8 --> YUVu8 and RGBf32 --> YUVu8 are currently available. +//! \param[in] src the source image. +//! \param[in] srcRect the destination rectangle (NULL implies the whole image). +//! \param[out] y pointer to pixel(0,0) of the luminance channel. +//! \param[in] yPixBytes the byte stride between y pixels horizontally. +//! \param[in] yPitch the byte stride between y pixels vertically. +//! \param[out] u pointer to pixel(0,0) of the u (Cb) chrominance channel. +//! \param[out] v pointer to pixel(0,0) of the v (Cr) chrominance channel. +//! \param[in] uvPixBytes the byte stride between u or v pixels horizontally. +//! \param[in] uvPitch the byte stride between u or v pixels vertically. +//! \param[in] yuvColorSpace the yuv colorspace, specifying range, chromaticities, and chrominance phase. +//! \param[in] yuvMemSpace the memory space where the pixel buffers reside. +//! \param[in] scale scale factor applied to the magnitude during transfer, typically 1, 255 or 1/255. +//! \param[in] stream the CUDA stream. +//! \param[in] tmp a staging image. +//! \return NVCV_SUCCESS if the operation was completed successfully. +//! \note The actual transfer region may be smaller, because the rects are clipped against the images. +NvCV_Status NvCV_API NvCVImage_TransferToYUV( + const NvCVImage *src, const NvCVRect2i *srcRect, + const void *y, int yPixBytes, int yPitch, + const void *u, const void *v, int uvPixBytes, int uvPitch, + NvCVImage_PixelFormat yuvFormat, NvCVImage_ComponentType yuvType, + unsigned yuvColorSpace, unsigned yuvMemSpace, + float scale, struct CUstream_st *stream, NvCVImage *tmp); + + +//! Between rendering by a graphics system and Transfer by CUDA, it is necessary to map the texture resource. +//! There is a fair amount of overhead, so its use should be minimized. +//! Every call to NvCVImage_MapResource() should be matched by a subsequent call to NvCVImage_UnmapResource(). +//! \param[in,out] im the image to be mapped. +//! \param[in] stream the stream on which the mapping is to be performed. +//! \return NVCV_SUCCESS is the operation was completed successfully. +NvCV_Status NvCV_API NvCVImage_MapResource(NvCVImage *im, struct CUstream_st *stream); + + +//! After transfer by CUDA, the texture resource must be unmapped in order to be used by the graphics system again. +//! There is a fair amount of overhead, so its use should be minimized. +//! Every call to NvCVImage_UnmapResource() should correspond to a preceding call to NvCVImage_MapResource(). +//! \param[in,out] im the image to be mapped. +//! \param[in] stream the CUDA stream on which the mapping is to be performed. +//! \return NVCV_SUCCESS is the operation was completed successfully. +NvCV_Status NvCV_API NvCVImage_UnmapResource(NvCVImage *im, struct CUstream_st *stream); + + +//! Composite one source image over another using the given matte. +//! This accommodates all RGB and RGBA formats, with u8 and f32 components. +//! \param[in] fg the foreground source image. +//! \param[in] bg the background source image. //! \param[in] mat the matte Yu8 (or Au8) image, indicating where the src should come through. -//! \param[out] dst the destination BGRu8 (or RGBu8) image. This can be the same as fg or bg. +//! \param[out] dst the destination image. This can be the same as fg or bg. +//! \param[in] stream the CUDA stream on which the composition is to be performed. //! \return NVCV_SUCCESS if the operation was successful. //! \return NVCV_ERR_PIXELFORMAT if the pixel format is not accommodated. -//! \bug This is only implemented for 3-component u8 fg, bg and dst, and 1-component u8 mat, -//! where all images are resident on the CPU. +//! \return NVCV_ERR_MISMATCH if either the fg & bg & dst formats do not match, or if fg & bg & dst & mat are not +//! in the same address space (CPU or GPU). +#if RTX_CAMERA_IMAGE == 0 +NvCV_Status NvCV_API NvCVImage_Composite(const NvCVImage *fg, const NvCVImage *bg, const NvCVImage *mat, NvCVImage *dst, + struct CUstream_st *stream); +#else // RTX_CAMERA_IMAGE == 1 // No GPU acceleration NvCV_Status NvCV_API NvCVImage_Composite(const NvCVImage *fg, const NvCVImage *bg, const NvCVImage *mat, NvCVImage *dst); +#endif // RTX_CAMERA_IMAGE == 1 +//! Composite one source image over another using the given matte. +//! Not all pixel format combinations are accommodated. +//! \param[in] fg the foreground source image. +//! \param[in] fgOrg the upper-left corner of the fg image to be composited (NULL implies (0,0)). +//! \param[in] bg the background source image. +//! \param[in] bgOrg the upper-left corner of the bg image to be composited (NULL implies (0,0)). +//! \param[in] mat the matte image, indicating where the src should come through. +//! This determines the size of the rectangle to be composited. +//! If this is multi-channel, the alpha channel is used as the matte. +//! \param[in] mode the composition mode. Only 0 (straight alpha over) is implemented at this time. +//! \param[out] dst the destination image. This can be the same as fg or bg. +//! \param[in] dstOrg the upper-left corner of the dst image to be updated (NULL implies (0,0)). +//! \param[in] stream the CUDA stream on which the composition is to be performed. +//! \note If a smaller region of a matte is desired, a window can be created using +//! NvCVImage_InitView() for chunky or NvCVImage_Init() for planar pixels. +//! \return NVCV_SUCCESS if the operation was successful. +//! \return NVCV_ERR_PIXELFORMAT if the pixel format is not accommodated. +//! \return NVCV_ERR_MISMATCH if either the fg & bg & dst formats do not match, or if fg & bg & dst & mat are not +//! in the same address space (CPU or GPU). +//! \bug Though RGBA destinations are accommodated, the A channel is not updated at all. +//! \todo Accommodate premultiplied alpha, either as a flag in NvCVImage or as a different mode. +//! \todo If the destination has an A channel, update it as per Adobe and Pixar. +NvCV_Status NvCV_API NvCVImage_CompositeRect( + const NvCVImage *fg, const NvCVPoint2i *fgOrg, + const NvCVImage *bg, const NvCVPoint2i *bgOrg, + const NvCVImage *mat, unsigned mode, + NvCVImage *dst, const NvCVPoint2i *dstOrg, + struct CUstream_st *stream); //! Composite a BGRu8 source image over a constant color field using the given matte. //! \param[in] src the source BGRu8 (or RGBu8) image. @@ -464,10 +650,16 @@ NvCVImage::~NvCVImage() { NvCVImage_Dealloc(this); } NvCV_Status NvCVImage::copyFrom(const NvCVImage *src, int srcX, int srcY, int dstX, int dstY, unsigned wd, unsigned ht) { +#if RTX_CAMERA_IMAGE // This only works for chunky images NvCVImage srcView, dstView; NvCVImage_InitView(&srcView, const_cast(src), srcX, srcY, wd, ht); NvCVImage_InitView(&dstView, this, dstX, dstY, wd, ht); return NvCVImage_Transfer(&srcView, &dstView, 1.f, 0, nullptr); +#else // !RTX_CAMERA_IMAGE bug fix for non-chunky images + NvCVRect2i srcRect = { (int)srcX, (int)srcY, (int)wd, (int)ht }; + NvCVPoint2i dstPt = { (int)dstX, (int)dstY }; + return NvCVImage_TransferRect(src, &srcRect, this, &dstPt, 1.f, 0, nullptr); +#endif // RTX_CAMERA_IMAGE } /******************************************************************************** diff --git a/nvvfx/include/nvCVStatus.h b/nvvfx/include/nvCVStatus.h index 17997dc..dd47ba3 100644 --- a/nvvfx/include/nvCVStatus.h +++ b/nvvfx/include/nvCVStatus.h @@ -64,17 +64,33 @@ typedef enum NvCV_Status { NVCV_ERR_UNSUPPORTEDGPU = -17, //!< The GPU is not supported NVCV_ERR_WRONGGPU = -18, //!< The current GPU is not the one selected. NVCV_ERR_UNSUPPORTEDDRIVER = -19, //!< The currently installed graphics driver is not supported + NVCV_ERR_MODELDEPENDENCIES = -20, //!< There is no model with dependencies that match this system + NVCV_ERR_PARSE = -21, //!< There has been a parsing or syntax error while reading a file + NVCV_ERR_MODELSUBSTITUTION = -22, //!< The specified model does not exist and has been substituted. + NVCV_ERR_READ = -23, //!< An error occurred while reading a file. + NVCV_ERR_WRITE = -24, //!< An error occurred while writing a file. + NVCV_ERR_PARAMREADONLY = -25, //!< The selected parameter is read-only. + NVCV_ERR_TRT_ENQUEUE = -26, //!< TensorRT enqueue failed. + NVCV_ERR_TRT_BINDINGS = -27, //!< Unexpected TensorRT bindings. + NVCV_ERR_TRT_CONTEXT = -28, //!< An error occurred while creating a TensorRT context. + NVCV_ERR_TRT_INFER = -29, ///< The was a problem creating the inference engine. + NVCV_ERR_TRT_ENGINE = -30, ///< There was a problem deserializing the inference runtime engine. + NVCV_ERR_NPP = -31, //!< An error has occurred in the NPP library. + NVCV_ERR_CONFIG = -32, //!< No suitable model exists for the specified parameter configuration. - NVCV_ERR_CUDA_MEMORY = -20, //!< There is not enough CUDA memory for the requested operation. - NVCV_ERR_CUDA_VALUE = -21, //!< A CUDA parameter is not within the acceptable range. - NVCV_ERR_CUDA_PITCH = -22, //!< A CUDA pitch is not within the acceptable range. - NVCV_ERR_CUDA_INIT = -23, //!< The CUDA driver and runtime could not be initialized. - NVCV_ERR_CUDA_LAUNCH = -24, //!< The CUDA kernel launch has failed. - NVCV_ERR_CUDA_KERNEL = -25, //!< No suitable kernel image is available for the device. - NVCV_ERR_CUDA_DRIVER = -26, //!< The installed NVIDIA CUDA driver is older than the CUDA runtime library. - NVCV_ERR_CUDA_UNSUPPORTED = -27, //!< The CUDA operation is not supported on the current system or device. - NVCV_ERR_CUDA_ILLEGAL_ADDRESS = -28, //!< CUDA tried to load or store on an invalid memory address. - NVCV_ERR_CUDA = -30, //!< An otherwise unspecified CUDA error has been reported. + NVCV_ERR_DIRECT3D = -99, //!< A Direct3D error has occurred. + + NVCV_ERR_CUDA_BASE = -100, //!< CUDA errors are offset from this value. + NVCV_ERR_CUDA_VALUE = -101, //!< A CUDA parameter is not within the acceptable range. + NVCV_ERR_CUDA_MEMORY = -102, //!< There is not enough CUDA memory for the requested operation. + NVCV_ERR_CUDA_PITCH = -112, //!< A CUDA pitch is not within the acceptable range. + NVCV_ERR_CUDA_INIT = -127, //!< The CUDA driver and runtime could not be initialized. + NVCV_ERR_CUDA_LAUNCH = -819, //!< The CUDA kernel launch has failed. + NVCV_ERR_CUDA_KERNEL = -309, //!< No suitable kernel image is available for the device. + NVCV_ERR_CUDA_DRIVER = -135, //!< The installed NVIDIA CUDA driver is older than the CUDA runtime library. + NVCV_ERR_CUDA_UNSUPPORTED = -901, //!< The CUDA operation is not supported on the current system or device. + NVCV_ERR_CUDA_ILLEGAL_ADDRESS = -800, //!< CUDA tried to load or store on an invalid memory address. + NVCV_ERR_CUDA = -1099, //!< An otherwise unspecified CUDA error has been reported. } NvCV_Status; diff --git a/nvvfx/include/nvTransferD3D.h b/nvvfx/include/nvTransferD3D.h new file mode 100644 index 0000000..e914eb5 --- /dev/null +++ b/nvvfx/include/nvTransferD3D.h @@ -0,0 +1,72 @@ +/*############################################################################### +# +# Copyright(c) 2021 NVIDIA CORPORATION.All Rights Reserved. +# +# NVIDIA CORPORATION and its licensors retain all intellectual property +# and proprietary rights in and to this software, related documentation +# and any modifications thereto.Any use, reproduction, disclosure or +# distribution of this software and related documentation without an express +# license agreement from NVIDIA CORPORATION is strictly prohibited. +# +###############################################################################*/ + +#ifndef __NVTRANSFER_D3D_H__ +#define __NVTRANSFER_D3D_H__ + +#ifndef _WINDOWS_ + #define WIN32_LEAN_AND_MEAN + #include +#endif // _WINDOWS_ +#include +#include "nvCVImage.h" + +#ifdef __cplusplus +extern "C" { +#endif // ___cplusplus + + + +//! Utility to determine the D3D format from the NvCVImage format, type and layout. +//! \param[in] format the pixel format. +//! \param[in] type the component type. +//! \param[in] layout the layout. +//! \param[out] d3dFormat a place to store the corresponding D3D format. +//! \return NVCV_SUCCESS if successful. +NvCV_Status NvCV_API NvCVImage_ToD3DFormat(NvCVImage_PixelFormat format, NvCVImage_ComponentType type, unsigned layout, DXGI_FORMAT *d3dFormat); + + +//! Utility to determine the NvCVImage format, component type and layout from a D3D format. +//! \param[in] d3dFormat the D3D format to translate. +//! \param[out] format a place to store the NvCVImage pixel format. +//! \param[out] type a place to store the NvCVImage component type. +//! \param[out] layout a place to store the NvCVImage layout. +//! \return NVCV_SUCCESS if successful. +NvCV_Status NvCV_API NvCVImage_FromD3DFormat(DXGI_FORMAT d3dFormat, NvCVImage_PixelFormat *format, NvCVImage_ComponentType *type, unsigned char *layout); + + +#ifdef __dxgicommon_h__ + +//! Utility to determine the D3D color space from the NvCVImage color space. +//! \param[in] nvcvColorSpace the NvCVImage colro space. +//! \param[out] pD3dColorSpace a place to store the resultant D3D color space. +//! \return NVCV_SUCCESS if successful. +//! \return NVCV_ERR_PIXELFORMAT if there is no equivalent color space. +NvCV_Status NvCV_API NvCVImage_ToD3DColorSpace(unsigned char nvcvColorSpace, DXGI_COLOR_SPACE_TYPE *pD3dColorSpace); + + +//! Utility to determine the NvCVImage color space from the D3D color space. +//! \param[in] d3dColorSpace the D3D color space. +//! \param[out] pNvcvColorSpace a place to store the resultant NvCVImage color space. +//! \return NVCV_SUCCESS if successful. +//! \return NVCV_ERR_PIXELFORMAT if there is no equivalent color space. +NvCV_Status NvCV_API NvCVImage_FromD3DColorSpace(DXGI_COLOR_SPACE_TYPE d3dColorSpace, unsigned char *pNvcvColorSpace); + +#endif // __dxgicommon_h__ + + +#ifdef __cplusplus +} // extern "C" +#endif // __cplusplus + +#endif // __NVTRANSFER_D3D_H__ + diff --git a/nvvfx/include/nvTransferD3D11.h b/nvvfx/include/nvTransferD3D11.h new file mode 100644 index 0000000..fabf067 --- /dev/null +++ b/nvvfx/include/nvTransferD3D11.h @@ -0,0 +1,44 @@ +/*############################################################################### +# +# Copyright(c) 2021 NVIDIA CORPORATION.All Rights Reserved. +# +# NVIDIA CORPORATION and its licensors retain all intellectual property +# and proprietary rights in and to this software, related documentation +# and any modifications thereto.Any use, reproduction, disclosure or +# distribution of this software and related documentation without an express +# license agreement from NVIDIA CORPORATION is strictly prohibited. +# +###############################################################################*/ + +#ifndef __NVTRANSFER_D3D11_H__ +#define __NVTRANSFER_D3D11_H__ + +#include +#include "nvCVImage.h" +#include "nvTransferD3D.h" // for NvCVImage_ToD3DFormat() and NvCVImage_FromD3DFormat() + +#ifdef __cplusplus +extern "C" { +#endif // ___cplusplus + + + +//! Initialize an NvCVImage from a D3D11 texture. +//! The pixelFormat and component types with be transferred over, and a cudaGraphicsResource will be registered; +//! the NvCVImage destructor will unregister the resource. +//! This is designed to work with NvCVImage_TransferFromArray() (and eventually NvCVImage_Transfer()); +//! however it is necessary to call NvCVImage_MapResource beforehand, and NvCVImage_UnmapResource +//! before allowing D3D to render into it. +//! \param[in,out] im the image to be initialized. +//! \param[in] tx the texture to be used for initialization. +//! \return NVCV_SUCCESS if successful. +NvCV_Status NvCV_API NvCVImage_InitFromD3D11Texture(NvCVImage *im, struct ID3D11Texture2D *tx); + + + +#ifdef __cplusplus +} // extern "C" +#endif // __cplusplus + +#endif // __NVTRANSFER_D3D11_H__ + diff --git a/nvvfx/include/nvVideoEffects.h b/nvvfx/include/nvVideoEffects.h index 5ba4043..9d461bc 100644 --- a/nvvfx/include/nvVideoEffects.h +++ b/nvvfx/include/nvVideoEffects.h @@ -192,13 +192,17 @@ NvCV_Status NvVFX_API NvVFX_CudaStreamDestroy(CUstream stream); // Filter selectors #define NVVFX_FX_TRANSFER "Transfer" +#define NVVFX_FX_GREEN_SCREEN "GreenScreen" // Green Screen +#define NVVFX_FX_BGBLUR "BackgroundBlur" // Background blur #define NVVFX_FX_ARTIFACT_REDUCTION "ArtifactReduction" // Artifact Reduction #define NVVFX_FX_SUPER_RES "SuperRes" // Super Res #define NVVFX_FX_SR_UPSCALE "Upscale" // Super Res Upscale +#define NVVFX_FX_DENOISING "Denoising" // Denoising // Parameter selectors #define NVVFX_INPUT_IMAGE_0 "SrcImage0" //!< There may be multiple input images #define NVVFX_INPUT_IMAGE NVVFX_INPUT_IMAGE_0 //!< but there is usually only one input image +#define NVVFX_INPUT_IMAGE_1 "SrcImage1" //!< Source Image 1 #define NVVFX_OUTPUT_IMAGE_0 "DstImage0" //!< There may be multiple output images #define NVVFX_OUTPUT_IMAGE NVVFX_OUTPUT_IMAGE_0 //!< but there is usually only one output image #define NVVFX_MODEL_DIRECTORY "ModelDir" //!< The directory where the model may be found @@ -206,9 +210,14 @@ NvCV_Status NvVFX_API NvVFX_CudaStreamDestroy(CUstream stream); #define NVVFX_INFO "Info" //!< Get info about the effects #define NVVFX_SCALE "Scale" //!< Scale factor #define NVVFX_STRENGTH "Strength" //!< Strength for different filters +#define NVVFX_STRENGTH_LEVELS "StrengthLevels" //!< Number of strength levels #define NVVFX_MODE "Mode" //!< Mode for different filters #define NVVFX_TEMPORAL "Temporal" //!< Temporal mode: 0=image, 1=video #define NVVFX_GPU "GPU" //!< Preferred GPU (optional) +#define NVVFX_BATCH_SIZE "BatchSize" //!< Batch Size (default 1) +#define NVVFX_MODEL_BATCH "ModelBatch" //!< The preferred batching model to use (default 1) +#define NVVFX_STATE "State" //!< State variable +#define NVVFX_STATE_SIZE "StateSize" //!< Number of bytes needed to store state diff --git a/nvvfx/src/NVVideoEffectsProxy.cpp b/nvvfx/src/NVVideoEffectsProxy.cpp index b4720c1..b7fca4b 100644 --- a/nvvfx/src/NVVideoEffectsProxy.cpp +++ b/nvvfx/src/NVVideoEffectsProxy.cpp @@ -25,31 +25,31 @@ #include "nvVideoEffects.h" #ifdef _WIN32 -#define _WINSOCKAPI_ -#include -#include -#else -#include -typedef void* HMODULE; -typedef void* HANDLE; -typedef void* HINSTANCE; -#endif + #define _WINSOCKAPI_ + #include + #include +#else // !_WIN32 + #include + typedef void* HMODULE; + typedef void* HANDLE; + typedef void* HINSTANCE; +#endif // _WIN32 // Parameter string does not include the file extension #ifdef _WIN32 #define nvLoadLibrary(library) LoadLibrary(TEXT(library ".dll")) -#else +#else // !_WIN32 #define nvLoadLibrary(library) dlopen("lib" library ".so", RTLD_LAZY) -#endif +#endif // _WIN32 inline void* nvGetProcAddress(HINSTANCE handle, const char* proc) { if (nullptr == handle) return nullptr; #ifdef _WIN32 return GetProcAddress(handle, proc); -#else +#else // !_WIN32 return dlsym(handle, proc); -#endif +#endif // _WIN32 } inline int nvFreeLibrary(HINSTANCE handle) { @@ -87,99 +87,6 @@ NvCV_Status NvVFX_API NvVFX_GetVersion(unsigned int* version) { return funcPtr(version); } -NvCV_Status NvVFX_API NvCVImage_Init(NvCVImage* im, unsigned width, unsigned height, int pitch, void* pixels, - NvCVImage_PixelFormat format, NvCVImage_ComponentType type, unsigned isPlanar, - unsigned onGPU) { - static const auto funcPtr = (decltype(NvCVImage_Init)*)nvGetProcAddress(getNvVfxLib(), "NvCVImage_Init"); - - if (nullptr == funcPtr) return NVCV_ERR_LIBRARY; - return funcPtr(im, width, height, pitch, pixels, format, type, isPlanar, onGPU); -} - -void NvVFX_API NvCVImage_InitView(NvCVImage* subImg, NvCVImage* fullImg, int x, int y, unsigned width, - unsigned height) { - static const auto funcPtr = (decltype(NvCVImage_InitView)*)nvGetProcAddress(getNvVfxLib(), "NvCVImage_InitView"); - - if (nullptr != funcPtr) funcPtr(subImg, fullImg, x, y, width, height); -} - -NvCV_Status NvCVImage_Alloc(NvCVImage* im, unsigned width, unsigned height, NvCVImage_PixelFormat format, - NvCVImage_ComponentType type, unsigned isPlanar, unsigned onGPU, unsigned alignment) { - static const auto funcPtr = (decltype(NvCVImage_Alloc)*)nvGetProcAddress(getNvVfxLib(), "NvCVImage_Alloc"); - - if (nullptr == funcPtr) return NVCV_ERR_LIBRARY; - return funcPtr(im, width, height, format, type, isPlanar, onGPU, alignment); -} - -NvCV_Status NvVFX_API NvCVImage_Realloc(NvCVImage* im, unsigned width, unsigned height, - NvCVImage_PixelFormat format, NvCVImage_ComponentType type, - unsigned isPlanar, unsigned onGPU, unsigned alignment) { - static const auto funcPtr = (decltype(NvCVImage_Realloc)*)nvGetProcAddress(getNvVfxLib(), "NvCVImage_Realloc"); - - if (nullptr == funcPtr) return NVCV_ERR_LIBRARY; - return funcPtr(im, width, height, format, type, isPlanar, onGPU, alignment); -} - -void NvVFX_API NvCVImage_Dealloc(NvCVImage* im) { - static const auto funcPtr = (decltype(NvCVImage_Dealloc)*)nvGetProcAddress(getNvVfxLib(), "NvCVImage_Dealloc"); - - if (nullptr != funcPtr) funcPtr(im); -} - -NvCV_Status NvVFX_API NvCVImage_Create(unsigned width, unsigned height, NvCVImage_PixelFormat format, - NvCVImage_ComponentType type, unsigned isPlanar, unsigned onGPU, - unsigned alignment, NvCVImage** out) { - static const auto funcPtr = (decltype(NvCVImage_Create)*)nvGetProcAddress(getNvVfxLib(), "NvCVImage_Create"); - - if (nullptr == funcPtr) return NVCV_ERR_LIBRARY; - return funcPtr(width, height, format, type, isPlanar, onGPU, alignment, out); -} - -void NvVFX_API NvCVImage_Destroy(NvCVImage* im) { - static const auto funcPtr = (decltype(NvCVImage_Destroy)*)nvGetProcAddress(getNvVfxLib(), "NvCVImage_Destroy"); - - if (nullptr != funcPtr) funcPtr(im); -} - -void NvVFX_API NvCVImage_ComponentOffsets(NvCVImage_PixelFormat format, int* rOff, int* gOff, int* bOff, int* aOff, - int* yOff) { - static const auto funcPtr = - (decltype(NvCVImage_ComponentOffsets)*)nvGetProcAddress(getNvVfxLib(), "NvCVImage_ComponentOffsets"); - - if (nullptr != funcPtr) funcPtr(format, rOff, gOff, bOff, aOff, yOff); -} - -NvCV_Status NvVFX_API NvCVImage_Transfer(const NvCVImage* src, NvCVImage* dst, float scale, CUstream_st* stream, - NvCVImage* tmp) { - static const auto funcPtr = (decltype(NvCVImage_Transfer)*)nvGetProcAddress(getNvVfxLib(), "NvCVImage_Transfer"); - - if (nullptr == funcPtr) return NVCV_ERR_LIBRARY; - return funcPtr(src, dst, scale, stream, tmp); -} - -NvCV_Status NvVFX_API NvCVImage_Composite(const NvCVImage* fg, const NvCVImage* bg, const NvCVImage* mat, NvCVImage* dst) { - static const auto funcPtr = (decltype(NvCVImage_Composite)*)nvGetProcAddress(getNvVfxLib(), "NvCVImage_Composite"); - - if (nullptr == funcPtr) return NVCV_ERR_LIBRARY; - return funcPtr(fg, bg, mat, dst); -} - -NvCV_Status NvVFX_API NvCVImage_CompositeOverConstant(const NvCVImage* src, const NvCVImage* mat, - const unsigned char bgColor[3], NvCVImage* dst) { - static const auto funcPtr = - (decltype(NvCVImage_CompositeOverConstant)*)nvGetProcAddress(getNvVfxLib(), "NvCVImage_CompositeOverConstant"); - - if (nullptr == funcPtr) return NVCV_ERR_LIBRARY; - return funcPtr(src, mat, bgColor, dst); -} - -NvCV_Status NvVFX_API NvCVImage_FlipY(const NvCVImage* src, NvCVImage* dst) { - static const auto funcPtr = (decltype(NvCVImage_FlipY)*)nvGetProcAddress(getNvVfxLib(), "NvCVImage_FlipY"); - - if (nullptr == funcPtr) return NVCV_ERR_LIBRARY; - return funcPtr(src, dst); -} - NvCV_Status NvVFX_API NvVFX_CreateEffect(NvVFX_EffectSelector code, NvVFX_Handle* obj) { static const auto funcPtr = (decltype(NvVFX_CreateEffect)*)nvGetProcAddress(getNvVfxLib(), "NvVFX_CreateEffect"); @@ -349,15 +256,3 @@ NvCV_Status NvVFX_API NvVFX_CudaStreamDestroy(CUstream stream) { return funcPtr(stream); } -#ifdef _WIN32 -__declspec(dllexport) const char* __cdecl -#else -const char* -#endif // _WIN32 or linux - NvCV_GetErrorStringFromCode(NvCV_Status code) { - static const auto funcPtr = - (decltype(NvCV_GetErrorStringFromCode)*)nvGetProcAddress(getNvVfxLib(), "NvCV_GetErrorStringFromCode"); - - if (nullptr == funcPtr) return "Cannot find NVVideoEffects DLL or its dependencies"; - return funcPtr(code); -} diff --git a/nvvfx/src/nvCVImageProxy.cpp b/nvvfx/src/nvCVImageProxy.cpp new file mode 100644 index 0000000..f724d7a --- /dev/null +++ b/nvvfx/src/nvCVImageProxy.cpp @@ -0,0 +1,311 @@ +#if defined(linux) || defined(unix) || defined(__linux) +#warning nvCVImageProxy.cpp not ported +#else +/*############################################################################### +# +# Copyright 2020 NVIDIA Corporation +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal in +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +# the Software, and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +###############################################################################*/ +#include +#include "nvCVImage.h" + +#ifdef _WIN32 + #define _WINSOCKAPI_ + #include + #include + #include "nvTransferD3D.h" + #include "nvTransferD3D11.h" +#else // !_WIN32 + #include + typedef void* HMODULE; + typedef void* HANDLE; + typedef void* HINSTANCE; +#endif // _WIN32 + +// Parameter string does not include the file extension +#ifdef _WIN32 +#define nvLoadLibrary(library) LoadLibrary(TEXT(library ".dll")) +#else // !_WIN32 +#define nvLoadLibrary(library) dlopen("lib" library ".so", RTLD_LAZY) +#endif // _WIN32 + + +inline void* nvGetProcAddress(HINSTANCE handle, const char* proc) { + if (nullptr == handle) return nullptr; +#ifdef _WIN32 + return GetProcAddress(handle, proc); +#else // !_WIN32 + return dlsym(handle, proc); +#endif // _WIN32 +} + +inline int nvFreeLibrary(HINSTANCE handle) { +#ifdef _WIN32 + return FreeLibrary(handle); +#else + return dlclose(handle); +#endif +} + +HINSTANCE getNvCVImageLib() { + TCHAR path[MAX_PATH], tmpPath[MAX_PATH], fullPath[MAX_PATH]; + static HINSTANCE nvCVImageLib = NULL; + static bool bSDKPathSet = false; + if (!bSDKPathSet) { + // There can be multiple apps on the system, + // some might include the SDK in the app package and + // others might expect the SDK to be installed in Program Files + GetEnvironmentVariable(TEXT("NV_VIDEO_EFFECTS_PATH"), path, MAX_PATH); + GetEnvironmentVariable(TEXT("NV_AR_SDK_PATH"), tmpPath, MAX_PATH); + if (_tcscmp(path, TEXT("USE_APP_PATH")) && _tcscmp(tmpPath, TEXT("USE_APP_PATH"))) { + // App has not set environment variable to "USE_APP_PATH" + // So pick up the SDK dll and dependencies from Program Files + GetEnvironmentVariable(TEXT("ProgramFiles"), path, MAX_PATH); + size_t max_len = sizeof(fullPath) / sizeof(TCHAR); + _stprintf_s(fullPath, max_len, TEXT("%s\\NVIDIA Corporation\\NVIDIA Video Effects\\"), path); + SetDllDirectory(fullPath); + nvCVImageLib = nvLoadLibrary("NVCVImage"); + if (!nvCVImageLib) { + _stprintf_s(fullPath, max_len, TEXT("%s\\NVIDIA Corporation\\NVIDIA AR SDK\\"), path); + SetDllDirectory(fullPath); + nvCVImageLib = nvLoadLibrary("NVCVImage"); + } + } + bSDKPathSet = true; + } + return nvCVImageLib; +} + +NvCV_Status NvCV_API NvCVImage_Init(NvCVImage* im, unsigned width, unsigned height, int pitch, void* pixels, + NvCVImage_PixelFormat format, NvCVImage_ComponentType type, unsigned isPlanar, + unsigned onGPU) { + static const auto funcPtr = (decltype(NvCVImage_Init)*)nvGetProcAddress(getNvCVImageLib(), "NvCVImage_Init"); + + if (nullptr == funcPtr) return NVCV_ERR_LIBRARY; + return funcPtr(im, width, height, pitch, pixels, format, type, isPlanar, onGPU); +} + +void NvCV_API NvCVImage_InitView(NvCVImage* subImg, NvCVImage* fullImg, int x, int y, unsigned width, + unsigned height) { + static const auto funcPtr = (decltype(NvCVImage_InitView)*)nvGetProcAddress(getNvCVImageLib(), "NvCVImage_InitView"); + + if (nullptr != funcPtr) funcPtr(subImg, fullImg, x, y, width, height); +} + +NvCV_Status NvCV_API NvCVImage_Alloc(NvCVImage* im, unsigned width, unsigned height, NvCVImage_PixelFormat format, + NvCVImage_ComponentType type, unsigned isPlanar, unsigned onGPU, unsigned alignment) { + static const auto funcPtr = (decltype(NvCVImage_Alloc)*)nvGetProcAddress(getNvCVImageLib(), "NvCVImage_Alloc"); + + if (nullptr == funcPtr) return NVCV_ERR_LIBRARY; + return funcPtr(im, width, height, format, type, isPlanar, onGPU, alignment); +} + +NvCV_Status NvCV_API NvCVImage_Realloc(NvCVImage* im, unsigned width, unsigned height, + NvCVImage_PixelFormat format, NvCVImage_ComponentType type, + unsigned isPlanar, unsigned onGPU, unsigned alignment) { + static const auto funcPtr = (decltype(NvCVImage_Realloc)*)nvGetProcAddress(getNvCVImageLib(), "NvCVImage_Realloc"); + + if (nullptr == funcPtr) return NVCV_ERR_LIBRARY; + return funcPtr(im, width, height, format, type, isPlanar, onGPU, alignment); +} + +void NvCV_API NvCVImage_Dealloc(NvCVImage* im) { + static const auto funcPtr = (decltype(NvCVImage_Dealloc)*)nvGetProcAddress(getNvCVImageLib(), "NvCVImage_Dealloc"); + + if (nullptr != funcPtr) funcPtr(im); +} + +NvCV_Status NvCV_API NvCVImage_Create(unsigned width, unsigned height, NvCVImage_PixelFormat format, + NvCVImage_ComponentType type, unsigned isPlanar, unsigned onGPU, + unsigned alignment, NvCVImage** out) { + static const auto funcPtr = (decltype(NvCVImage_Create)*)nvGetProcAddress(getNvCVImageLib(), "NvCVImage_Create"); + + if (nullptr == funcPtr) return NVCV_ERR_LIBRARY; + return funcPtr(width, height, format, type, isPlanar, onGPU, alignment, out); +} + +void NvCV_API NvCVImage_Destroy(NvCVImage* im) { + static const auto funcPtr = (decltype(NvCVImage_Destroy)*)nvGetProcAddress(getNvCVImageLib(), "NvCVImage_Destroy"); + + if (nullptr != funcPtr) funcPtr(im); +} + +void NvCV_API NvCVImage_ComponentOffsets(NvCVImage_PixelFormat format, int* rOff, int* gOff, int* bOff, int* aOff, + int* yOff) { + static const auto funcPtr = + (decltype(NvCVImage_ComponentOffsets)*)nvGetProcAddress(getNvCVImageLib(), "NvCVImage_ComponentOffsets"); + + if (nullptr != funcPtr) funcPtr(format, rOff, gOff, bOff, aOff, yOff); +} + +NvCV_Status NvCV_API NvCVImage_Transfer(const NvCVImage* src, NvCVImage* dst, float scale, CUstream_st* stream, + NvCVImage* tmp) { + static const auto funcPtr = (decltype(NvCVImage_Transfer)*)nvGetProcAddress(getNvCVImageLib(), "NvCVImage_Transfer"); + + if (nullptr == funcPtr) return NVCV_ERR_LIBRARY; + return funcPtr(src, dst, scale, stream, tmp); +} + +NvCV_Status NvCV_API NvCVImage_TransferRect(const NvCVImage *src, const NvCVRect2i *srcRect, NvCVImage *dst, + const NvCVPoint2i *dstPt, float scale, struct CUstream_st *stream, NvCVImage *tmp) { + static const auto funcPtr = (decltype(NvCVImage_TransferRect)*)nvGetProcAddress(getNvCVImageLib(), "NvCVImage_TransferRect"); + + if (nullptr == funcPtr) return NVCV_ERR_LIBRARY; + return funcPtr(src, srcRect, dst, dstPt, scale, stream, tmp); +} + +NvCV_Status NvCV_API NvCVImage_TransferFromYUV(const void *y, int yPixBytes, int yPitch, const void *u, const void *v, + int uvPixBytes, int uvPitch, NvCVImage_PixelFormat yuvFormat, NvCVImage_ComponentType yuvType, unsigned yuvColorSpace, + unsigned yuvMemSpace, NvCVImage *dst, const NvCVRect2i *dstRect, float scale, struct CUstream_st *stream, NvCVImage *tmp) { + static const auto funcPtr = (decltype(NvCVImage_TransferFromYUV)*)nvGetProcAddress(getNvCVImageLib(), "NvCVImage_TransferFromYUV"); + + if (nullptr == funcPtr) return NVCV_ERR_LIBRARY; + return funcPtr(y, yPixBytes, yPitch, u, v, uvPixBytes, uvPitch, yuvFormat, yuvType, yuvColorSpace, yuvMemSpace, dst, + dstRect, scale, stream, tmp); +} + +NvCV_Status NvCV_API NvCVImage_TransferToYUV(const NvCVImage *src, const NvCVRect2i *srcRect, + const void *y, int yPixBytes, int yPitch, const void *u, const void *v, int uvPixBytes, int uvPitch, + NvCVImage_PixelFormat yuvFormat, NvCVImage_ComponentType yuvType, unsigned yuvColorSpace, unsigned yuvMemSpace, + float scale, struct CUstream_st *stream, NvCVImage *tmp) { + static const auto funcPtr = (decltype(NvCVImage_TransferToYUV)*)nvGetProcAddress(getNvCVImageLib(), "NvCVImage_TransferToYUV"); + + if (nullptr == funcPtr) return NVCV_ERR_LIBRARY; + return funcPtr(src, srcRect, y, yPixBytes, yPitch, u, v, uvPixBytes, uvPitch, yuvFormat, yuvType, yuvColorSpace, yuvMemSpace, scale, stream, tmp); +} + +NvCV_Status NvCV_API NvCVImage_MapResource(NvCVImage *im, struct CUstream_st *stream) { + static const auto funcPtr = (decltype(NvCVImage_MapResource)*)nvGetProcAddress(getNvCVImageLib(), "NvCVImage_MapResource"); + + if (nullptr == funcPtr) return NVCV_ERR_LIBRARY; + return funcPtr(im, stream); +} + +NvCV_Status NvCV_API NvCVImage_UnmapResource(NvCVImage *im, struct CUstream_st *stream) { + static const auto funcPtr = (decltype(NvCVImage_UnmapResource)*)nvGetProcAddress(getNvCVImageLib(), "NvCVImage_UnmapResource"); + + if (nullptr == funcPtr) return NVCV_ERR_LIBRARY; + return funcPtr(im, stream); +} + +#if RTX_CAMERA_IMAGE == 0 +NvCV_Status NvCV_API NvCVImage_Composite(const NvCVImage* fg, const NvCVImage* bg, const NvCVImage* mat, NvCVImage* dst, + struct CUstream_st *stream) { + static const auto funcPtr = (decltype(NvCVImage_Composite)*)nvGetProcAddress(getNvCVImageLib(), "NvCVImage_Composite"); + + if (nullptr == funcPtr) return NVCV_ERR_LIBRARY; + return funcPtr(fg, bg, mat, dst, stream); +} +#else // RTX_CAMERA_IMAGE == 1 +NvCV_Status NvCV_API NvCVImage_Composite(const NvCVImage* fg, const NvCVImage* bg, const NvCVImage* mat, NvCVImage* dst) { + static const auto funcPtr = (decltype(NvCVImage_Composite)*)nvGetProcAddress(getNvCVImageLib(), "NvCVImage_Composite"); + + if (nullptr == funcPtr) return NVCV_ERR_LIBRARY; + return funcPtr(fg, bg, mat, dst); +} +#endif // RTX_CAMERA_IMAGE + +NvCV_Status NvCV_API NvCVImage_CompositeRect( + const NvCVImage *fg, const NvCVPoint2i *fgOrg, + const NvCVImage *bg, const NvCVPoint2i *bgOrg, + const NvCVImage *mat, unsigned mode, + NvCVImage *dst, const NvCVPoint2i *dstOrg, + struct CUstream_st *stream) { + static const auto funcPtr = (decltype(NvCVImage_CompositeRect)*)nvGetProcAddress(getNvCVImageLib(), "NvCVImage_CompositeRect"); + + if (nullptr == funcPtr) return NVCV_ERR_LIBRARY; + return funcPtr(fg, fgOrg, bg, bgOrg, mat, mode, dst, dstOrg, stream); +} + +NvCV_Status NvCV_API NvCVImage_CompositeOverConstant(const NvCVImage* src, const NvCVImage* mat, + const unsigned char bgColor[3], NvCVImage* dst) { + static const auto funcPtr = + (decltype(NvCVImage_CompositeOverConstant)*)nvGetProcAddress(getNvCVImageLib(), "NvCVImage_CompositeOverConstant"); + + if (nullptr == funcPtr) return NVCV_ERR_LIBRARY; + return funcPtr(src, mat, bgColor, dst); +} + +NvCV_Status NvCV_API NvCVImage_FlipY(const NvCVImage* src, NvCVImage* dst) { + static const auto funcPtr = (decltype(NvCVImage_FlipY)*)nvGetProcAddress(getNvCVImageLib(), "NvCVImage_FlipY"); + + if (nullptr == funcPtr) return NVCV_ERR_LIBRARY; + return funcPtr(src, dst); +} + +#ifdef _WIN32 +__declspec(dllexport) const char* __cdecl +#else +const char* +#endif // _WIN32 or linux + NvCV_GetErrorStringFromCode(NvCV_Status code) { + static const auto funcPtr = + (decltype(NvCV_GetErrorStringFromCode)*)nvGetProcAddress(getNvCVImageLib(), "NvCV_GetErrorStringFromCode"); + + if (nullptr == funcPtr) return "Cannot find nvCVImage DLL or its dependencies"; + return funcPtr(code); +} + + + +#ifdef _WIN32 // Direct 3D + +NvCV_Status NvCV_API NvCVImage_InitFromD3D11Texture(NvCVImage *im, struct ID3D11Texture2D *tx) { + static const auto funcPtr = (decltype(NvCVImage_InitFromD3D11Texture)*)nvGetProcAddress(getNvCVImageLib(), "NvCVImage_InitFromD3D11Texture"); + + if (nullptr == funcPtr) return NVCV_ERR_LIBRARY; + return funcPtr(im, tx); +} + +NvCV_Status NvCV_API NvCVImage_ToD3DFormat(NvCVImage_PixelFormat format, NvCVImage_ComponentType type, unsigned layout, DXGI_FORMAT *d3dFormat) { + static const auto funcPtr = (decltype(NvCVImage_ToD3DFormat)*)nvGetProcAddress(getNvCVImageLib(), "NvCVImage_ToD3DFormat"); + + if (nullptr == funcPtr) return NVCV_ERR_LIBRARY; + return funcPtr(format, type, layout, d3dFormat); +} + +NvCV_Status NvCV_API NvCVImage_FromD3DFormat(DXGI_FORMAT d3dFormat, NvCVImage_PixelFormat *format, NvCVImage_ComponentType *type, unsigned char *layout) { + static const auto funcPtr = (decltype(NvCVImage_FromD3DFormat)*)nvGetProcAddress(getNvCVImageLib(), "NvCVImage_FromD3DFormat"); + + if (nullptr == funcPtr) return NVCV_ERR_LIBRARY; + return funcPtr(d3dFormat, format, type, layout); +} + +#ifdef __dxgicommon_h__ + +NvCV_Status NvCV_API NvCVImage_ToD3DColorSpace(unsigned char nvcvColorSpace, DXGI_COLOR_SPACE_TYPE *pD3dColorSpace) { + static const auto funcPtr = (decltype(NvCVImage_ToD3DColorSpace)*)nvGetProcAddress(getNvCVImageLib(), "NvCVImage_ToD3DColorSpace"); + + if (nullptr == funcPtr) return NVCV_ERR_LIBRARY; + return funcPtr(nvcvColorSpace, pD3dColorSpace); +} + +NvCV_Status NvCV_API NvCVImage_FromD3DColorSpace(DXGI_COLOR_SPACE_TYPE d3dColorSpace, unsigned char *pNvcvColorSpace) { + static const auto funcPtr = (decltype(NvCVImage_FromD3DColorSpace)*)nvGetProcAddress(getNvCVImageLib(), "NvCVImage_FromD3DColorSpace"); + + if (nullptr == funcPtr) return NVCV_ERR_LIBRARY; + return funcPtr(d3dColorSpace, pNvcvColorSpace); +} + +#endif // __dxgicommon_h__ + +#endif // _WIN32 Direct 3D + +#endif // enabling for this file diff --git a/resources/Denoise.gif b/resources/Denoise.gif new file mode 100644 index 0000000..9d9c011 Binary files /dev/null and b/resources/Denoise.gif differ diff --git a/resources/SR.gif b/resources/SR.gif new file mode 100644 index 0000000..190b70a Binary files /dev/null and b/resources/SR.gif differ diff --git a/resources/superres.gif b/resources/superres.gif deleted file mode 100644 index 50bd18a..0000000 Binary files a/resources/superres.gif and /dev/null differ diff --git a/samples/AigsEffectApp/AigsEffectApp.cpp b/samples/AigsEffectApp/AigsEffectApp.cpp new file mode 100644 index 0000000..4019738 --- /dev/null +++ b/samples/AigsEffectApp/AigsEffectApp.cpp @@ -0,0 +1,826 @@ +/*############################################################################### +# +# Copyright (c) 2020 NVIDIA Corporation +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal in +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +# the Software, and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +###############################################################################*/ +#include +#include +#include +#include + +#include +#include +#include + +#include "nvCVOpenCV.h" +#include "nvVideoEffects.h" +#include "opencv2/opencv.hpp" + +#ifdef _MSC_VER +#define strcasecmp _stricmp +#include +#else // !_MSC_VER +#include +#endif // _MSC_VER + +#define BAIL_IF_ERR(err) \ + do { \ + if (0 != (err)) { \ + goto bail; \ + } \ + } while (0) +#define BAIL_IF_NULL(x, err, code) \ + do { \ + if ((void *)(x) == NULL) { \ + err = code; \ + goto bail; \ + } \ + } while (0) +#define NVCV_ERR_HELP 411 + +#ifdef _WIN32 + #define DEFAULT_CODEC "avc1" +#else // !_WIN32 + #define DEFAULT_CODEC "H264" +#endif // _WIN32 + +bool FLAG_progress = false; +bool FLAG_show = false; +bool FLAG_useOTAU = false; +bool FLAG_verbose = false; +bool FLAG_webcam = false; +int FLAG_compMode = 3 /*compWhite*/; +int FLAG_mode = 0; +float FLAG_blurStrength = 0.5; +std::string FLAG_camRes; +std::string FLAG_codec = DEFAULT_CODEC; +std::string FLAG_inFile; +std::string FLAG_modelDir; +std::string FLAG_outDir; +std::string FLAG_outFile; + +static bool GetFlagArgVal(const char *flag, const char *arg, const char **val) { + if (*arg != '-') return false; + while (*++arg == '-') continue; + const char *s = strchr(arg, '='); + if (s == NULL) { + if (strcmp(flag, arg) != 0) return false; + *val = NULL; + return true; + } + size_t n = s - arg; + if ((strlen(flag) != n) || (strncmp(flag, arg, n) != 0)) return false; + *val = s + 1; + return true; +} + +static bool GetFlagArgVal(const char *flag, const char *arg, std::string *val) { + const char *valStr; + if (!GetFlagArgVal(flag, arg, &valStr)) return false; + val->assign(valStr ? valStr : ""); + return true; +} + +static bool GetFlagArgVal(const char *flag, const char *arg, bool *val) { + const char *valStr; + bool success = GetFlagArgVal(flag, arg, &valStr); + if (success) { + *val = (valStr == NULL || strcasecmp(valStr, "true") == 0 || strcasecmp(valStr, "on") == 0 || + strcasecmp(valStr, "yes") == 0 || strcasecmp(valStr, "1") == 0); + } + return success; +} + +static bool GetFlagArgVal(const char *flag, const char *arg, long *val) { + const char *valStr; + bool success = GetFlagArgVal(flag, arg, &valStr); + if (success) *val = strtol(valStr, NULL, 10); + return success; +} + +static bool GetFlagArgVal(const char *flag, const char *arg, int *val) { + long longVal; + bool success = GetFlagArgVal(flag, arg, &longVal); + if (success) *val = (int)longVal; + return success; +} + +static bool GetFlagArgVal(const char *flag, const char *arg, float *val) { + const char *valStr; + bool success = GetFlagArgVal(flag, arg, &valStr); + if (success) *val = std::stof(valStr); + return success; +} + +static void Usage() { + printf( + "AigsEffectApp [args ...]\n" + " where args is:\n" + " --in_file= input file to be processed\n" + " --out_file= output file to be written\n" + " --webcam use a webcam as input\n" + " --cam_res=[WWWx]HHH specify resolution as height or width x height\n" + " --model_dir= the path to the directory that contains the models\n" + " --codec= the FOURCC code for the desired codec (default " DEFAULT_CODEC ")\n" + " --show display the results in a window\n" + " --progress show progress\n" + " --mode=(0|1) pick one of the green screen modes\n" + " 0 - Best quality\n" + " 1 - Best performance\n" + " --comp_mode choose the composition mode - { compMatte = 0, compLight = 1, compGreen = 2, compWhite = 3, compNone = 4, compBG = 5, compBlur = 6}\n" + " --blur_strength change the blur strength, range is [0, 1]" + ); +} + +static int ParseMyArgs(int argc, char **argv) { + int errs = 0; + for (--argc, ++argv; argc--; ++argv) { + bool help; + const char *arg = *argv; + if (arg[0] != '-') { + continue; + } else if ((arg[1] == '-') && + (GetFlagArgVal("verbose", arg, &FLAG_verbose) || GetFlagArgVal("in", arg, &FLAG_inFile) || + GetFlagArgVal("in_file", arg, &FLAG_inFile) || GetFlagArgVal("out", arg, &FLAG_outFile) || + GetFlagArgVal("out_file", arg, &FLAG_outFile) || GetFlagArgVal("model_dir", arg, &FLAG_modelDir) || + GetFlagArgVal("codec", arg, &FLAG_codec) || GetFlagArgVal("webcam", arg, &FLAG_webcam) || + GetFlagArgVal("cam_res", arg, &FLAG_camRes) || GetFlagArgVal("mode", arg, &FLAG_mode) || + GetFlagArgVal("progress", arg, &FLAG_progress) || GetFlagArgVal("show", arg, &FLAG_show) || + GetFlagArgVal("comp_mode", arg, &FLAG_compMode) || GetFlagArgVal("blur_strength", arg, &FLAG_blurStrength))) { + continue; + } else if (GetFlagArgVal("help", arg, &help)) { + return NVCV_ERR_HELP; + } else if (arg[1] != '-') { + for (++arg; *arg; ++arg) { + if (*arg == 'v') { + FLAG_verbose = true; + } else { + printf("Unknown flag ignored: \"-%c\"\n", *arg); + } + } + continue; + } else { + printf("Unknown flag ignored: \"%s\"\n", arg); + } + } + return errs; +} + +static bool HasSuffix(const char *str, const char *suf) { + size_t strSize = strlen(str), sufSize = strlen(suf); + if (strSize < sufSize) return false; + return (0 == strcasecmp(suf, str + strSize - sufSize)); +} + +static bool HasOneOfTheseSuffixes(const char *str, ...) { + bool matches = false; + const char *suf; + va_list ap; + va_start(ap, str); + while (nullptr != (suf = va_arg(ap, const char *))) { + if (HasSuffix(str, suf)) { + matches = true; + break; + } + } + va_end(ap); + return matches; +} + +static bool IsImageFile(const char *str) { + return HasOneOfTheseSuffixes(str, ".bmp", ".jpg", ".jpeg", ".png", nullptr); +} + +static const char *DurationString(double sc) { + static char buf[16]; + int hr, mn; + hr = (int)(sc / 3600.); + sc -= hr * 3600.; + mn = (int)(sc / 60.); + sc -= mn * 60.; + snprintf(buf, sizeof(buf), "%02d:%02d:%06.3f", hr, mn, sc); + return buf; +} + +struct VideoInfo { + int codec; + int width; + int height; + double frameRate; + long long frameCount; +}; + +static void PrintVideoInfo(const VideoInfo *info, const char *fileName) { + printf( + " file \"%s\"\n" + " codec %.4s\n" + " width %4d\n" + " height %4d\n" + " frame rate %.3f\n" + "frame count %4lld\n" + " duration %s\n", + fileName, (char *)&info->codec, info->width, info->height, info->frameRate, info->frameCount, + info->frameCount ? DurationString(info->frameCount / info->frameRate) : "(webcam)"); +} + +static void GetVideoInfo(cv::VideoCapture &reader, const char *fileName, VideoInfo *info) { + info->codec = (int)reader.get(cv::CAP_PROP_FOURCC); + info->width = (int)reader.get(cv::CAP_PROP_FRAME_WIDTH); + info->height = (int)reader.get(cv::CAP_PROP_FRAME_HEIGHT); + info->frameRate = (double)reader.get(cv::CAP_PROP_FPS); + if(!strcmp(fileName,"webcam")) + info->frameCount = 0; + else + info->frameCount = (long long)reader.get(cv::CAP_PROP_FRAME_COUNT); + if (FLAG_verbose) PrintVideoInfo(info, fileName); +} + +static int StringToFourcc(const std::string &str) { + union chint { + int i; + char c[4]; + }; + chint x = {0}; + for (int n = (str.size() < 4) ? (int)str.size() : 4; n--;) x.c[n] = str[n]; + return x.i; +} + +struct FXApp { + enum Err { + errQuit = +1, // Application errors + errFlag = +2, + errRead = +3, + errWrite = +4, + errNone = NVCV_SUCCESS, // Video Effects SDK errors + errGeneral = NVCV_ERR_GENERAL, + errUnimplemented = NVCV_ERR_UNIMPLEMENTED, + errMemory = NVCV_ERR_MEMORY, + errEffect = NVCV_ERR_EFFECT, + errSelector = NVCV_ERR_SELECTOR, + errBuffer = NVCV_ERR_BUFFER, + errParameter = NVCV_ERR_PARAMETER, + errMismatch = NVCV_ERR_MISMATCH, + errPixelFormat = NVCV_ERR_PIXELFORMAT, + errModel = NVCV_ERR_MODEL, + errLibrary = NVCV_ERR_LIBRARY, + errInitialization = NVCV_ERR_INITIALIZATION, + errFileNotFound = NVCV_ERR_FILE, + errFeatureNotFound = NVCV_ERR_FEATURENOTFOUND, + errMissingInput = NVCV_ERR_MISSINGINPUT, + errResolution = NVCV_ERR_RESOLUTION, + errUnsupportedGPU = NVCV_ERR_UNSUPPORTEDGPU, + errWrongGPU = NVCV_ERR_WRONGGPU, + errCudaMemory = NVCV_ERR_CUDA_MEMORY, // CUDA errors + errCudaValue = NVCV_ERR_CUDA_VALUE, + errCudaPitch = NVCV_ERR_CUDA_PITCH, + errCudaInit = NVCV_ERR_CUDA_INIT, + errCudaLaunch = NVCV_ERR_CUDA_LAUNCH, + errCudaKernel = NVCV_ERR_CUDA_KERNEL, + errCudaDriver = NVCV_ERR_CUDA_DRIVER, + errCudaUnsupported = NVCV_ERR_CUDA_UNSUPPORTED, + errCudaIllegalAddress = NVCV_ERR_CUDA_ILLEGAL_ADDRESS, + errCuda = NVCV_ERR_CUDA, + }; + enum CompMode { compMatte, compLight, compGreen, compWhite, compNone, compBG, compBlur }; + + FXApp() { + _eff = nullptr; + _bgblurEff = nullptr; + _effectName = nullptr; + _inited = false; + _total = 0.0; + _count = 0; + _compMode = compLight; + _showFPS = false; + _stream = nullptr; + _progress = false; + _show = false; + _framePeriod = 0.f; + _lastTime = std::chrono::high_resolution_clock::time_point::min(); + _blurStrength = 0.5f; + } + ~FXApp() { + NvVFX_DestroyEffect(_eff); + NvVFX_DestroyEffect(_bgblurEff); + + if (_stream) { + NvVFX_CudaStreamDestroy(_stream); + } + } + + void setShow(bool show) { _show = show; } + NvCV_Status createAigsEffect(); + void destroyEffect(); + NvCV_Status allocBuffers(unsigned width, unsigned height); + NvCV_Status allocTempBuffers(); + Err processImage(const char *inFile, const char *outFile); + Err processMovie(const char *inFile, const char *outFile); + Err processKey(int key); + void nextCompMode(); + void drawFrameRate(cv::Mat &img); + Err appErrFromVfxStatus(NvCV_Status status) { return (Err)status; } + const char *errorStringFromCode(Err code); + + NvVFX_Handle _eff, _bgblurEff; + cv::Mat _srcImg; + cv::Mat _dstImg; + NvCVImage _srcVFX; + NvCVImage _dstVFX; + bool _show; + bool _inited; + bool _showFPS; + bool _progress; + const char *_effectName; + float _total; + int _count; + CompMode _compMode; + float _framePeriod; + CUstream _stream; + std::chrono::high_resolution_clock::time_point _lastTime; + NvCVImage _srcNvVFXImage; + NvCVImage _dstNvVFXImage; + NvCVImage _blurNvVFXImage; + float _blurStrength; +}; + +const char *FXApp::errorStringFromCode(Err code) { + struct LutEntry { + Err code; + const char *str; + }; + static const LutEntry lut[] = { + {errRead, "There was a problem reading a file"}, + {errWrite, "There was a problem writing a file"}, + {errQuit, "The user chose to quit the application"}, + {errFlag, "There was a problem with the command-line arguments"}, + }; + if ((int)code <= 0) return NvCV_GetErrorStringFromCode((NvCV_Status)code); + for (const LutEntry *p = lut; p != &lut[sizeof(lut) / sizeof(lut[0])]; ++p) + if (p->code == code) return p->str; + return "UNKNOWN ERROR"; +} + +void FXApp::drawFrameRate(cv::Mat &img) { + const float timeConstant = 16.f; + std::chrono::high_resolution_clock::time_point now = std::chrono::high_resolution_clock::now(); + std::chrono::duration dur = std::chrono::duration_cast>(now - _lastTime); + float t = dur.count(); + if (0.f < t && t < 100.f) { + if (_framePeriod) + _framePeriod += (t - _framePeriod) * (1.f / timeConstant); // 1 pole IIR filter + else + _framePeriod = t; + if (_showFPS) { + char buf[32]; + snprintf(buf, sizeof(buf), "%.1f", 1. / _framePeriod); + cv::putText(img, buf, cv::Point(10, img.rows - 10), cv::FONT_HERSHEY_SIMPLEX, 1, cv::Scalar(255, 255, 255), 1); + } + } else { // Ludicrous time interval; reset + _framePeriod = 0.f; // WAKE UP + } + _lastTime = now; +} + +void FXApp::nextCompMode() { + switch (_compMode) { + default: + case compBG: + case compLight: + _compMode = compGreen; + break; + case compMatte: + _compMode = compNone; + break; + case compGreen: + _compMode = compWhite; + break; + case compWhite: + _compMode = compMatte; + break; + case compNone: + _compMode = compBlur; + break; + case compBlur: + _compMode = compLight; + break; + } +} + +FXApp::Err FXApp::processKey(int key) { + static const int ESC_KEY = 27; + switch (key) { + case 'Q': + case 'q': + case ESC_KEY: + return errQuit; + case 'c': + case 'C': + nextCompMode(); + break; + case 'f': + case 'F': + _showFPS = !_showFPS; + break; + case 'p': + case 'P': + case '%': + _progress = !_progress; + break; + case 'm': + _blurStrength += 0.05f; + if (_blurStrength > 1.0) { + _blurStrength = 1.0; + } + break; + case 'n': + _blurStrength -= 0.05f; + if (_blurStrength < 0.0) { + _blurStrength = 0.0; + } + break; + default: + break; + } + return errNone; +} + +NvCV_Status FXApp::createAigsEffect() { + NvCV_Status vfxErr; + + vfxErr = NvVFX_CreateEffect(NVVFX_FX_GREEN_SCREEN, &_eff); + if (NVCV_SUCCESS != vfxErr) { + std::cerr << "Error creating effect \"" << NVVFX_FX_GREEN_SCREEN << "\"\n"; + return vfxErr; + } + _effectName = NVVFX_FX_GREEN_SCREEN; + + if (!FLAG_modelDir.empty()) { + vfxErr = NvVFX_SetString(_eff, NVVFX_MODEL_DIRECTORY, FLAG_modelDir.c_str()); + } + if (vfxErr != NVCV_SUCCESS) { + std::cerr << "Error setting the model path to \"" << FLAG_modelDir << "\"\n"; + return vfxErr; + } + + const char *cstr; // TODO: This is not necessary + vfxErr = NvVFX_GetString(_eff, NVVFX_INFO, &cstr); + if (vfxErr != NVCV_SUCCESS) { + std::cerr << "AIGS modes not found \n" << std::endl; + return vfxErr; + } + + // Choose one mode -> set() -> Load() -> Run() + vfxErr = NvVFX_SetU32(_eff, NVVFX_MODE, FLAG_mode); + if (vfxErr != NVCV_SUCCESS) { + std::cerr << "Error setting the mode \n"; + return vfxErr; + } + + vfxErr = NvVFX_CudaStreamCreate(&_stream); + if (vfxErr != NVCV_SUCCESS) { + std::cerr << "Error creating CUDA stream " << std::endl; + return vfxErr; + } + + vfxErr = NvVFX_SetCudaStream(_eff, NVVFX_CUDA_STREAM, _stream); + if (vfxErr != NVCV_SUCCESS) { + std::cerr << "Error setting up the cuda stream \n"; + return vfxErr; + } + + vfxErr = NvVFX_Load(_eff); + if (vfxErr != NVCV_SUCCESS) { + std::cerr << "Error loading the model \n"; + return vfxErr; + } + + // ------------------ create Background blur effect ------------------ // + vfxErr = NvVFX_CreateEffect(NVVFX_FX_BGBLUR, &_bgblurEff); + if (NVCV_SUCCESS != vfxErr) { + std::cerr << "Error creating effect \"" << NVVFX_FX_BGBLUR << "\"\n"; + return vfxErr; + } + + vfxErr = NvVFX_GetString(_bgblurEff, NVVFX_INFO, &cstr); + if (vfxErr != NVCV_SUCCESS) { + std::cerr << "BGBLUR info not found \n" << std::endl; + return vfxErr; + } + + vfxErr = NvVFX_SetCudaStream(_bgblurEff, NVVFX_CUDA_STREAM, _stream); + if (vfxErr != NVCV_SUCCESS) { + std::cerr << "BGBLUR error setting up the cuda stream \n"; + return vfxErr; + } + + return vfxErr; +} + +void FXApp::destroyEffect() { + NvVFX_DestroyEffect(_eff); + _eff = nullptr; +} + +static void overlay(const cv::Mat &image, const cv::Mat &mask, float alpha, cv::Mat &result) { + cv::Mat maskClr; + cv::cvtColor(mask, maskClr, cv::COLOR_GRAY2BGR); + result = image * (1.f - alpha) + maskClr * alpha; +} + +FXApp::Err FXApp::processImage(const char *inFile, const char *outFile) { + NvCV_Status vfxErr; + bool ok; + cv::Mat result; + + if (!_eff) return errEffect; + _srcImg = cv::imread(inFile); + if (!_srcImg.data) return errRead; + + _dstImg = cv::Mat::zeros(_srcImg.size(), CV_8UC1); + if (!_dstImg.data) return errMemory; + + (void)NVWrapperForCVMat(&_srcImg, &_srcVFX); + (void)NVWrapperForCVMat(&_dstImg, &_dstVFX); + + NvCVImage fxSrcChunkyGPU(_srcImg.cols, _srcImg.rows, NVCV_BGR, NVCV_U8, NVCV_CHUNKY, NVCV_GPU, 1); + NvCVImage fxDstChunkyGPU(_srcImg.cols, _srcImg.rows, NVCV_A, NVCV_U8, NVCV_CHUNKY, NVCV_GPU, 1); + + BAIL_IF_ERR(vfxErr = NvVFX_SetImage(_eff, NVVFX_INPUT_IMAGE, &fxSrcChunkyGPU)); + BAIL_IF_ERR(vfxErr = NvVFX_SetImage(_eff, NVVFX_OUTPUT_IMAGE, &fxDstChunkyGPU)); + BAIL_IF_ERR(vfxErr = NvCVImage_Transfer(&_srcVFX, &fxSrcChunkyGPU, 1.0f, _stream, NULL)); + BAIL_IF_ERR(vfxErr = NvVFX_Run(_eff, 0)); + BAIL_IF_ERR(vfxErr = NvCVImage_Transfer(&fxDstChunkyGPU, &_dstVFX, 1.0f, _stream, NULL)); + + overlay(_srcImg, _dstImg, 0.5, result); + if (!std::string(outFile).empty()) { + ok = cv::imwrite(outFile, result); + if (!ok) { + printf("Error writing: \"%s\"\n", outFile); + return errWrite; + } + ok = cv::imwrite(std::string(outFile) + "_segmentation_mask.png", _dstImg); // save segmentation mask too + if (!ok) { + printf("Error writing: \"%s_segmentation_mask.png\"\n", outFile); + return errWrite; + } + } + if (_show) { + cv::imshow("Output", result); + cv::waitKey(3000); + } +bail: + return (FXApp::Err)vfxErr; +} + +FXApp::Err FXApp::processMovie(const char *inFile, const char *outFile) { + float ms = 0.0f; + FXApp::Err appErr = errNone; + const int camIndex = 0; + NvCV_Status vfxErr = NVCV_SUCCESS; + bool ok; + cv::Mat result; + cv::VideoCapture reader; + cv::VideoWriter writer; + unsigned frameNum; + VideoInfo info; + + if (inFile && !inFile[0]) inFile = nullptr; // Set file paths to NULL if zero length + if (outFile && !outFile[0]) outFile = nullptr; + + if (inFile) { + reader.open(inFile); + } else { + reader.open(camIndex); + if (!FLAG_camRes.empty()) { + int camWidth, camHeight, n; + n = sscanf(FLAG_camRes.c_str(), "%d%*[xX]%d", &camWidth, &camHeight); + switch (n) { + case 2: + break; // We have read both width and height + case 1: + camHeight = camWidth; + camWidth = (int)(camHeight * (16. / 9.) + .5); + break; + default: + camHeight = 0; + camWidth = 0; + break; + } + if (camWidth) reader.set(cv::CAP_PROP_FRAME_WIDTH, camWidth); + if (camHeight) reader.set(cv::CAP_PROP_FRAME_HEIGHT, camHeight); + } + printf("Camera frame: %.0f x %.0f\n", reader.get(cv::CAP_PROP_FRAME_WIDTH), reader.get(cv::CAP_PROP_FRAME_HEIGHT)); + } + if (!reader.isOpened()) { + if (!FLAG_webcam) printf("Error: Could not open video: \"%s\"\n", inFile); + else printf("Error: Webcam not found\n"); + return errRead; + } + + GetVideoInfo(reader, (inFile ? inFile : "webcam"), &info); + + if (outFile) { + ok = writer.open(outFile, StringToFourcc(FLAG_codec), info.frameRate, cv::Size(info.width, info.height)); + if (!ok) { + printf("Cannot open \"%s\" for video writing\n", outFile); + outFile = nullptr; + } + } + + unsigned int width = (unsigned int)reader.get(cv::CAP_PROP_FRAME_WIDTH); + unsigned int height = (unsigned int)reader.get(cv::CAP_PROP_FRAME_HEIGHT); + + // allocate src for GPU + if (!_srcNvVFXImage.pixels) + BAIL_IF_ERR(vfxErr = + NvCVImage_Alloc(&_srcNvVFXImage, width, height, NVCV_BGR, NVCV_U8, NVCV_CHUNKY, NVCV_GPU, 1)); + + // allocate dst for GPU + if (!_dstNvVFXImage.pixels) + BAIL_IF_ERR(vfxErr = + NvCVImage_Alloc(&_dstNvVFXImage, width, height, NVCV_A, NVCV_U8, NVCV_CHUNKY, NVCV_GPU, 1)); + + // allocate blur for GPU + if (!_blurNvVFXImage.pixels) + BAIL_IF_ERR(vfxErr = + NvCVImage_Alloc(&_blurNvVFXImage, width, height, NVCV_BGR, NVCV_U8, NVCV_CHUNKY, NVCV_GPU, 1)); + + for (frameNum = 0; reader.read(_srcImg); ++frameNum) { + if (_srcImg.empty()) printf("Frame %u is empty\n", frameNum); + + _dstImg = cv::Mat::zeros(_srcImg.size(), CV_8UC1); // TODO: Allocate and clear outside of the loop? + BAIL_IF_NULL(_dstImg.data, vfxErr, NVCV_ERR_MEMORY); + + (void)NVWrapperForCVMat(&_srcImg, &_srcVFX); // Ditto + (void)NVWrapperForCVMat(&_dstImg, &_dstVFX); + + BAIL_IF_ERR(vfxErr = NvVFX_SetImage(_eff, NVVFX_INPUT_IMAGE, &_srcNvVFXImage)); + BAIL_IF_ERR(vfxErr = NvVFX_SetImage(_eff, NVVFX_OUTPUT_IMAGE, &_dstNvVFXImage)); + BAIL_IF_ERR(vfxErr = NvCVImage_Transfer(&_srcVFX, &_srcNvVFXImage, 1.0f, _stream, NULL)); + + auto startTime = std::chrono::high_resolution_clock::now(); + BAIL_IF_ERR(vfxErr = NvVFX_Run(_eff, 0)); + auto endTime = std::chrono::high_resolution_clock::now(); + ms = std::chrono::duration(endTime - startTime).count(); + _count += 1; + if (_count > 0) { + // skipping first frame + _total += ms; + } + + BAIL_IF_ERR(vfxErr = NvCVImage_Transfer(&_dstNvVFXImage, &_dstVFX, 1.0f, _stream, NULL)); + + result.create(_srcImg.rows, _srcImg.cols, + CV_8UC3); // Make sure the result is allocated. TODO: allocate outsifde of the loop? + BAIL_IF_NULL(result.data, vfxErr, NVCV_ERR_MEMORY); + result.setTo(cv::Scalar::all(0)); // TODO: This may no longer be necessary since we no longer coerce to 16:9 + switch (_compMode) { + case compNone: + _srcImg.copyTo(result); + break; + case compLight: + if (inFile) { + overlay(_srcImg, _dstImg, 0.5, result); + } else { // If the webcam was cropped, also crop the compositing + cv::Rect rect(0, (_srcImg.rows - _srcVFX.height) / 2, _srcVFX.width, _srcVFX.height); + cv::Mat subResult = result(rect); + overlay(_srcImg(rect), _dstImg(rect), 0.5, subResult); + } + break; + case compGreen: { + const unsigned char bgColor[3] = {0, 255, 0}; + NvCVImage matVFX; + (void)NVWrapperForCVMat(&result, &matVFX); + NvCVImage_CompositeOverConstant(&_srcVFX, &_dstVFX, bgColor, &matVFX); + } break; + case compWhite: { + const unsigned char bgColor[3] = {255, 255, 255}; + NvCVImage matVFX; + (void)NVWrapperForCVMat(&result, &matVFX); + NvCVImage_CompositeOverConstant(&_srcVFX, &_dstVFX, bgColor, &matVFX); + } break; + case compMatte: + cv::cvtColor(_dstImg, result, cv::COLOR_GRAY2BGR); + break; + case compBlur: + BAIL_IF_ERR(vfxErr = NvVFX_SetF32(_bgblurEff, NVVFX_STRENGTH, _blurStrength)); + BAIL_IF_ERR(vfxErr = NvVFX_SetImage(_bgblurEff, NVVFX_INPUT_IMAGE_0, &_srcNvVFXImage)); + BAIL_IF_ERR(vfxErr = NvVFX_SetImage(_bgblurEff, NVVFX_INPUT_IMAGE_1, &_dstNvVFXImage)); + BAIL_IF_ERR(vfxErr = NvVFX_SetImage(_bgblurEff, NVVFX_OUTPUT_IMAGE, &_blurNvVFXImage)); + BAIL_IF_ERR(vfxErr = NvVFX_Load(_bgblurEff)); + BAIL_IF_ERR(vfxErr = NvVFX_Run(_bgblurEff, 0)); + + NvCVImage matVFX; + (void)NVWrapperForCVMat(&result, &matVFX); + BAIL_IF_ERR(vfxErr = NvCVImage_Transfer(&_blurNvVFXImage, &matVFX, 1.0f, _stream, NULL)); + + break; + } + if (outFile) { +#define WRITE_COMPOSITE +#ifdef WRITE_COMPOSITE + writer.write(result); +#else // WRITE_MATTE + writer.write(_dstImg); +#endif // WRITE_MATTE + } + if (_show) { + drawFrameRate(result); + cv::imshow("Output", result); + int key = cv::waitKey(1); + if (key > 0) { + appErr = processKey(key); + if (errQuit == appErr) break; + } + } + if (_progress) { + if(info.frameCount == 0) // no progress for a webcam + fprintf(stderr, "\b\b\b\b???%%"); + else + fprintf(stderr, "\b\b\b\b%3.0f%%", 100.f * frameNum / info.frameCount); + } + } + + if (_progress) fprintf(stderr, "\n"); + reader.release(); + if (outFile) writer.release(); +bail: + // Dealloc + NvCVImage_Dealloc(&(_srcNvVFXImage)); // This is also called in the destructor, ... + NvCVImage_Dealloc(&(_dstNvVFXImage)); // ... so is not necessary except in C code. + NvCVImage_Dealloc(&(_blurNvVFXImage)); + return appErrFromVfxStatus(vfxErr); +} + +int main(int argc, char **argv) { + int nErrs = 0; + FXApp::Err fxErr = FXApp::errNone; + FXApp app; + + nErrs = ParseMyArgs(argc, argv); + if (nErrs) { + Usage(); + return nErrs; + } + + if (FLAG_inFile.empty() && !FLAG_webcam) { + std::cerr << "Please specify --in_file=XXX or --webcam\n"; + ++nErrs; + } + if (FLAG_outFile.empty() && !FLAG_show) { + std::cerr << "Please specify --out_file=XXX or --show\n"; + ++nErrs; + } + + app._progress = FLAG_progress; + app.setShow(FLAG_show); + + app._compMode = static_cast(FLAG_compMode); + app._blurStrength = FLAG_blurStrength; + if (app._blurStrength < 0) { + app._blurStrength = 0; + } + else if (app._blurStrength > 1) { + app._blurStrength = 1; + } + + std::cout << "Processing " << FLAG_inFile << " mode " << FLAG_mode << " models " << FLAG_modelDir << std::endl; + + if (nErrs) { + Usage(); + fxErr = FXApp::errFlag; + } else { + fxErr = app.appErrFromVfxStatus(app.createAigsEffect()); + if (FXApp::errNone == fxErr) { + if (IsImageFile(FLAG_inFile.c_str())) + fxErr = app.processImage(FLAG_inFile.c_str(), FLAG_outFile.c_str()); + else + fxErr = app.processMovie(FLAG_inFile.c_str(), FLAG_outFile.c_str()); + if (fxErr == FXApp::errNone || fxErr == FXApp::errQuit) { + fxErr = FXApp::errNone; // Quitting isn't an error + std::cout << "Processing time averaged over " << app._count << " runs is " + << ((float)app._total) / ((float)app._count - 1) << " ms. " << std::endl; + } + } + } + + if (fxErr) std::cerr << "Error: " << app.errorStringFromCode(fxErr) << std::endl; + return (int)fxErr; +} diff --git a/samples/AigsEffectApp/AigsEffectApp.exe b/samples/AigsEffectApp/AigsEffectApp.exe new file mode 100644 index 0000000..a1feb4c Binary files /dev/null and b/samples/AigsEffectApp/AigsEffectApp.exe differ diff --git a/samples/AigsEffectApp/CMakeLists.txt b/samples/AigsEffectApp/CMakeLists.txt new file mode 100644 index 0000000..3bcce30 --- /dev/null +++ b/samples/AigsEffectApp/CMakeLists.txt @@ -0,0 +1,32 @@ +set(SOURCE_FILES + AigsEffectApp.cpp + ../../nvvfx/src/nvVideoEffectsProxy.cpp + ../../nvvfx/src/nvCVImageProxy.cpp) + +# Set Visual Studio source filters +source_group("Source Files" FILES ${SOURCE_FILES}) + +add_executable(AigsEffectApp ${SOURCE_FILES}) +target_include_directories(AigsEffectApp PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/../utils + ) +target_include_directories(AigsEffectApp PUBLIC + ${SDK_INCLUDES_PATH} + ) + +target_link_libraries(AigsEffectApp PUBLIC + opencv346 + NVVideoEffects + ${CMAKE_CURRENT_SOURCE_DIR}/../external/cuda/lib/x64/cudart.lib + ) + +set(OPENCV_PATH_STR ${CMAKE_CURRENT_SOURCE_DIR}/../external/opencv/bin) +set(VFXSDK_PATH_STR ${CMAKE_CURRENT_SOURCE_DIR}/../../bin) # Also the location for CUDA/NVTRT/libcrypto +set(PATH_STR "PATH=%PATH%" ${VFXSDK_PATH_STR} ${OPENCV_PATH_STR}) +set(CMD_ARG_STR "--show --in_file=\"${CMAKE_CURRENT_SOURCE_DIR}/../input/input_003054.jpg\"") +set_target_properties(AigsEffectApp PROPERTIES + FOLDER SampleApps + VS_DEBUGGER_ENVIRONMENT "${PATH_STR}" + VS_DEBUGGER_COMMAND_ARGUMENTS "${CMD_ARG_STR}" +) diff --git a/samples/AigsEffectApp/run.bat b/samples/AigsEffectApp/run.bat new file mode 100644 index 0000000..f6bdc61 --- /dev/null +++ b/samples/AigsEffectApp/run.bat @@ -0,0 +1,4 @@ +SETLOCAL +SET PATH=%PATH%;..\external\opencv\bin; +REM AigsEffectApp.exe --in_file=..\input\input_003054.jpg --show +AigsEffectApp.exe --webcam --show diff --git a/samples/BatchEffectApp/BatchDenoiseEffectApp.cpp b/samples/BatchEffectApp/BatchDenoiseEffectApp.cpp new file mode 100644 index 0000000..58dd01e --- /dev/null +++ b/samples/BatchEffectApp/BatchDenoiseEffectApp.cpp @@ -0,0 +1,345 @@ +/*############################################################################### +# +# Copyright (c) 2020 NVIDIA Corporation +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal in +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +# the Software, and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +###############################################################################*/ + +#include +#include + +#include +#include +#include "BatchUtilities.h" +#include "nvCVOpenCV.h" +#include "nvVideoEffects.h" +#include "opencv2/opencv.hpp" + +#ifdef _MSC_VER + #define strcasecmp _stricmp +#endif // _MSC_VER + +#define BAIL_IF_ERR(err) do { if (0 != (err)) { goto bail; } } while(0) +#define BAIL_IF_NULL(x, err, code) do { if ((void*)(x) == NULL) { err = code; goto bail; } } while(0) +#define BAIL_IF_FALSE(x, err, code) do { if (!(x)) { err = code; goto bail; } } while(0) +#define BAIL(err, code) do { err = code; goto bail; } while(0) + + +bool FLAG_verbose = false; +float FLAG_strength = 0.f, + FLAG_scale = 1.0; +int FLAG_mode = 0, + FLAG_resolution = 0, + FLAG_batchSize = 8; +std::string FLAG_outFile, + FLAG_modelDir; +std::vector FLAG_inFiles; + +// Set this when using OTA Updates +// This path is used by nvVideoEffectsProxy.cpp to load the SDK dll +// when using OTA Updates +char *g_nvVFXSDKPath = NULL; + +static bool GetFlagArgVal(const char *flag, const char *arg, const char **val) { + if (*arg != '-') + return false; + while (*++arg == '-') + continue; + const char *s = strchr(arg, '='); + if (s == NULL) { + if (strcmp(flag, arg) != 0) + return false; + *val = NULL; + return true; + } + size_t n = s - arg; + if ((strlen(flag) != n) || (strncmp(flag, arg, n) != 0)) + return false; + *val = s + 1; + return true; +} + +static bool GetFlagArgVal(const char *flag, const char *arg, std::string *val) { + const char *valStr; + if (!GetFlagArgVal(flag, arg, &valStr)) + return false; + val->assign(valStr ? valStr : ""); + return true; +} + +static bool GetFlagArgVal(const char *flag, const char *arg, bool *val) { + const char *valStr; + bool success = GetFlagArgVal(flag, arg, &valStr); + if (success) { + *val = (valStr == NULL || + strcasecmp(valStr, "true") == 0 || + strcasecmp(valStr, "on") == 0 || + strcasecmp(valStr, "yes") == 0 || + strcasecmp(valStr, "1") == 0 + ); + } + return success; +} + +static bool GetFlagArgVal(const char *flag, const char *arg, float *val) { + const char *valStr; + bool success = GetFlagArgVal(flag, arg, &valStr); + if (success) + *val = strtof(valStr, NULL); + return success; +} + +static bool GetFlagArgVal(const char *flag, const char *arg, long *val) { + const char *valStr; + bool success = GetFlagArgVal(flag, arg, &valStr); + if (success) + *val = strtol(valStr, NULL, 10); + return success; +} + +static bool GetFlagArgVal(const char *flag, const char *arg, int *val) { + long longVal; + bool success = GetFlagArgVal(flag, arg, &longVal); + if (success) + *val = (int)longVal; + return success; +} + +static void Usage() { + printf( + "BatchDenoiseEffectApp [flags ...] inFile1 [ inFileN ...]\n" + " where flags is:\n" + " --out_file= output video files to be written (a pattern with one %%u or %%d), default \"BatchOut_%%02u.mp4\"\n" + " --strength= strength of denoising [0-1]\n" + " --model_dir= the path to the directory that contains the models\n" + " --batchsize= size of the batch (default: 8)\n" + " --verbose verbose output\n" + " and inFile1 ... are identically sized video files\n" + ); +} + +static int ParseMyArgs(int argc, char **argv) { + int errs = 0; + for (--argc, ++argv; argc--; ++argv) { + bool help; + const char *arg = *argv; + if (arg[0] == '-') { + if (arg[1] == '-') { // double-dash + if (GetFlagArgVal("verbose", arg, &FLAG_verbose) || + GetFlagArgVal("strength", arg, &FLAG_strength) || + GetFlagArgVal("scale", arg, &FLAG_scale) || + GetFlagArgVal("mode", arg, &FLAG_mode) || + GetFlagArgVal("model_dir", arg, &FLAG_modelDir) || + GetFlagArgVal("out_file", arg, &FLAG_outFile) || + GetFlagArgVal("batch_size", arg, &FLAG_batchSize) + ) { + continue; + } else if (GetFlagArgVal("help", arg, &help)) { // --help + Usage(); + errs = 1; + } + } + else { // single dash + for (++arg; *arg; ++arg) { + if (*arg == 'v') { + FLAG_verbose = true; + } else { + printf("Unknown flag ignored: \"-%c\"\n", *arg); + } + } + continue; + } + } + else { // no dash + FLAG_inFiles.push_back(arg); + } + } + return errs; +} + + +class App { +public: + NvVFX_Handle _eff; + NvCVImage _src, _stg, _dst; + CUstream _stream; + unsigned _batchSize; + + + App() : _eff(nullptr), _stream(0), _batchSize(0) {} + ~App() { NvVFX_DestroyEffect(_eff); if (_stream) NvVFX_CudaStreamDestroy(_stream); } + + NvCV_Status init(const char* effectName, unsigned batchSize, const NvCVImage *srcImg) { + NvCV_Status err = NVCV_ERR_UNIMPLEMENTED; + + _batchSize = batchSize; + BAIL_IF_ERR(err = NvVFX_CreateEffect(effectName, &_eff)); + + BAIL_IF_ERR(err = AllocateBatchBuffer(&_src, _batchSize, srcImg->width, srcImg->height, NVCV_BGR, NVCV_F32, NVCV_PLANAR, NVCV_CUDA, 1)); // + BAIL_IF_ERR(err = AllocateBatchBuffer(&_dst, _batchSize, srcImg->width, srcImg->height, NVCV_BGR, NVCV_F32, NVCV_PLANAR, NVCV_CUDA, 1)); // + BAIL_IF_ERR(err = NvVFX_SetString(_eff, NVVFX_MODEL_DIRECTORY, FLAG_modelDir.c_str())); // + + + { // Set parameters. + NvCVImage nth; + BAIL_IF_ERR(err = NvVFX_SetImage(_eff, NVVFX_INPUT_IMAGE, NthImage(0, srcImg->height, &_src, &nth))); // Set the first of the batched images in ... + BAIL_IF_ERR(err = NvVFX_SetImage(_eff, NVVFX_OUTPUT_IMAGE, NthImage(0, _dst.height / _batchSize, &_dst, &nth))); // ... and out + BAIL_IF_ERR(err = NvVFX_CudaStreamCreate(&_stream)); + BAIL_IF_ERR(err = NvVFX_SetCudaStream(_eff, NVVFX_CUDA_STREAM, _stream)); + + BAIL_IF_ERR(err = NvVFX_Load(_eff)); + } + + bail: + return err; + } +}; + + +NvCV_Status BatchProcess(const char* effectName, const std::vector& srcVideos, unsigned batchSize, const char *outfilePattern) { + NvCV_Status err = NVCV_SUCCESS; + App app; + cv::Mat ocv1, ocv2; + NvCVImage nvx1, nvx2; + unsigned srcWidth, srcHeight, dstHeight, i; + + void** arrayOfStates = nullptr; + void** batchOfStates = nullptr; + unsigned int stateSizeInBytes; + + int numOfVideoStreams = srcVideos.size(); + std::vector srcCaptures(numOfVideoStreams); + std::vector dstWriters(numOfVideoStreams); + for (int i = 0; i < numOfVideoStreams; i++) { + srcCaptures[i].open(srcVideos[i]); + if (srcCaptures[i].isOpened()==false) BAIL(err, NVCV_ERR_READ); + + int width, height; + double fps; + width = (int)srcCaptures[i].get(cv::CAP_PROP_FRAME_WIDTH); + height = (int)srcCaptures[i].get(cv::CAP_PROP_FRAME_HEIGHT); + fps = srcCaptures[i].get(cv::CAP_PROP_FPS); + + const int fourcc_h264 = cv::VideoWriter::fourcc('H','2','6','4'); + char fileName[1024]; + snprintf(fileName, sizeof(fileName), outfilePattern, i); + dstWriters[i].open(fileName, fourcc_h264, fps, cv::Size2i(width,height)); + if (dstWriters[i].isOpened() == false) BAIL(err, NVCV_ERR_WRITE); + } + + // Read in the first image, to determine the resolution for init() + BAIL_IF_FALSE(srcVideos.size() > 0, err, NVCV_ERR_MISSINGINPUT); + srcCaptures[0] >> ocv1; + srcCaptures[0].set(cv::CAP_PROP_POS_FRAMES, 0); //resetting to first frame + if (!ocv1.data) { + printf("Cannot read video file \"%s\"\n", srcVideos[0]); + BAIL(err, NVCV_ERR_READ); + } + NVWrapperForCVMat(&ocv1, &nvx1); + srcWidth = nvx1.width; + srcHeight = nvx1.height; + + BAIL_IF_ERR(err = app.init(effectName, batchSize, &nvx1)); // Init effect and buffers + + // Creating state objects, one per stream. + BAIL_IF_ERR(err = NvVFX_GetU32(app._eff, NVVFX_STATE_SIZE, &stateSizeInBytes)); + arrayOfStates = (void**)calloc(numOfVideoStreams, sizeof(void*)); // allocating void* array of numOfVideoStreams elements + for (int i = 0; i < numOfVideoStreams; i++) { + cudaMalloc(&arrayOfStates[i], stateSizeInBytes); + cudaMemsetAsync(arrayOfStates[i], 0, stateSizeInBytes,app._stream); + } + //Creating batch array to hold states + batchOfStates = (void**)calloc(batchSize, sizeof(void*)); + + + dstHeight = app._dst.height / batchSize; + BAIL_IF_ERR(err = NvCVImage_Alloc(&nvx2, app._dst.width, dstHeight, ((app._dst.numComponents == 1) ? NVCV_Y : NVCV_BGR), NVCV_U8, NVCV_CHUNKY, NVCV_CPU, 0)); + CVWrapperForNvCVImage(&nvx2, &ocv2); + for(int j=0;;j++) + { + for (int i = 0; i < batchSize; i++) { + int capIdx = i%numOfVideoStreams; // interlacing frames from different video stream, but can in any order + srcCaptures[capIdx] >> ocv1; + if (ocv1.empty()) goto bail; + batchOfStates[i] = arrayOfStates[capIdx]; + + NVWrapperForCVMat(&ocv1, &nvx1); + if (!(nvx1.width == srcWidth && nvx1.height == srcHeight)) { + printf("Input video file \"%s\" %ux%u does not match %ux%u\n" + "Batching requires all video frames to be of the same size\n", srcVideos[i], nvx1.width, nvx1.height, srcWidth, srcHeight); + BAIL(err, NVCV_ERR_MISMATCH); + } + BAIL_IF_ERR(err = TransferToNthImage(i, &nvx1, &app._src, 1.f / 255.f, app._stream, &app._stg)); + ocv1.release(); + } + + // Run batch + BAIL_IF_ERR(err = NvVFX_SetU32(app._eff, NVVFX_BATCH_SIZE, (unsigned)batchSize)); // The batchSize can change every Run + BAIL_IF_ERR(err = NvVFX_SetObject(app._eff, NVVFX_STATE, (void*)batchOfStates)); // The batch of states can change every Run + BAIL_IF_ERR(err = NvVFX_Run(app._eff, 0)); + + + for (i = 0; i < batchSize; ++i) { + int writerIdx = i % numOfVideoStreams; + BAIL_IF_ERR(err = TransferFromNthImage(i, &app._dst, &nvx2, 255.f, app._stream, &app._stg)); + dstWriters[writerIdx] << ocv2; + } + // NvCVImage_Dealloc() is called in the destructors + } +bail: + if (arrayOfStates) { + for (unsigned i = 0; i < numOfVideoStreams; i++) { + if (arrayOfStates[i]) cudaFree(arrayOfStates[i]); + } + free(arrayOfStates); + } + if (batchOfStates) free(batchOfStates); + + for (auto& cap : srcCaptures) { + if (cap.isOpened()) cap.release(); + } + for (auto& writer : dstWriters) { + if (writer.isOpened()) writer.release(); + } + return err; +} + + +int main(int argc, char** argv) { + int nErrs; + NvCV_Status vfxErr; + + nErrs = ParseMyArgs(argc, argv); + if (nErrs) + return nErrs; + + if (FLAG_outFile.empty()) + FLAG_outFile = "BatchOut_%02u.mp4"; + else if (std::string::npos == FLAG_outFile.find_first_of('%')) + FLAG_outFile.insert(FLAG_outFile.size() - 4, "_%02u"); + + vfxErr = BatchProcess(NVVFX_FX_DENOISING, FLAG_inFiles, FLAG_batchSize, FLAG_outFile.c_str()); + if (NVCV_SUCCESS != vfxErr) { + Usage(); + printf("Error: %s\n", NvCV_GetErrorStringFromCode(vfxErr)); + nErrs = (int)vfxErr; + } + + return nErrs; +} diff --git a/samples/BatchEffectApp/BatchDenoiseEffectApp.exe b/samples/BatchEffectApp/BatchDenoiseEffectApp.exe new file mode 100644 index 0000000..cb9c803 Binary files /dev/null and b/samples/BatchEffectApp/BatchDenoiseEffectApp.exe differ diff --git a/samples/BatchEffectApp/BatchEffectApp.cpp b/samples/BatchEffectApp/BatchEffectApp.cpp new file mode 100644 index 0000000..adae384 --- /dev/null +++ b/samples/BatchEffectApp/BatchEffectApp.cpp @@ -0,0 +1,378 @@ +/*############################################################################### +# +# Copyright (c) 2020 NVIDIA Corporation +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal in +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +# the Software, and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +###############################################################################*/ + +#include +#include + +#include + +#include "BatchUtilities.h" +#include "nvCVOpenCV.h" +#include "nvVideoEffects.h" +#include "opencv2/opencv.hpp" + +#ifdef _MSC_VER + #define strcasecmp _stricmp +#endif // _MSC_VER + +#define BAIL_IF_ERR(err) do { if (0 != (err)) { goto bail; } } while(0) +#define BAIL_IF_NULL(x, err, code) do { if ((void*)(x) == NULL) { err = code; goto bail; } } while(0) +#define BAIL_IF_FALSE(x, err, code) do { if (!(x)) { err = code; goto bail; } } while(0) +#define BAIL(err, code) do { err = code; goto bail; } while(0) + + +bool FLAG_verbose = false; +float FLAG_strength = 0.f, + FLAG_scale = 1.0; +int FLAG_mode = 0, + FLAG_resolution = 0; +std::string FLAG_outFile, + FLAG_modelDir, + FLAG_effect; +std::vector FLAG_inFiles; + +// Set this when using OTA Updates +// This path is used by nvVideoEffectsProxy.cpp to load the SDK dll +// when using OTA Updates +char *g_nvVFXSDKPath = NULL; + +static bool GetFlagArgVal(const char *flag, const char *arg, const char **val) { + if (*arg != '-') + return false; + while (*++arg == '-') + continue; + const char *s = strchr(arg, '='); + if (s == NULL) { + if (strcmp(flag, arg) != 0) + return false; + *val = NULL; + return true; + } + size_t n = s - arg; + if ((strlen(flag) != n) || (strncmp(flag, arg, n) != 0)) + return false; + *val = s + 1; + return true; +} + +static bool GetFlagArgVal(const char *flag, const char *arg, std::string *val) { + const char *valStr; + if (!GetFlagArgVal(flag, arg, &valStr)) + return false; + val->assign(valStr ? valStr : ""); + return true; +} + +static bool GetFlagArgVal(const char *flag, const char *arg, bool *val) { + const char *valStr; + bool success = GetFlagArgVal(flag, arg, &valStr); + if (success) { + *val = (valStr == NULL || + strcasecmp(valStr, "true") == 0 || + strcasecmp(valStr, "on") == 0 || + strcasecmp(valStr, "yes") == 0 || + strcasecmp(valStr, "1") == 0 + ); + } + return success; +} + +static bool GetFlagArgVal(const char *flag, const char *arg, float *val) { + const char *valStr; + bool success = GetFlagArgVal(flag, arg, &valStr); + if (success) + *val = strtof(valStr, NULL); + return success; +} + +static bool GetFlagArgVal(const char *flag, const char *arg, long *val) { + const char *valStr; + bool success = GetFlagArgVal(flag, arg, &valStr); + if (success) + *val = strtol(valStr, NULL, 10); + return success; +} + +static bool GetFlagArgVal(const char *flag, const char *arg, int *val) { + long longVal; + bool success = GetFlagArgVal(flag, arg, &longVal); + if (success) + *val = (int)longVal; + return success; +} + +static void Usage() { + printf( + "BatchEffectApp [flags ...] inFile1 [ inFileN ...]\n" + " where flags is:\n" + " --out_file= output image files to be written, default \"BatchOut_%%02u.png\"\n" + " --effect= the effect to apply\n" + " --strength= strength of an effect, 0 or 1 for super res and artifact reduction,\n" + " and [0.0, 1.0] for upscaling\n" + " --scale= scale factor to be applied: 1.5, 2, 3, maybe 1.3333333\n" + " --resolution= the desired height (either --scale or --resolution may be used)\n" + " --mode= mode 0 or 1\n" + " --model_dir= the path to the directory that contains the models\n" + " --verbose verbose output\n" + " and inFile1 ... are identically sized image files, e.g. png, jpg\n" + ); + + const char* cStr; + NvCV_Status err = NvVFX_GetString(nullptr, NVVFX_INFO, &cStr); + if (NVCV_SUCCESS != err) + printf("Cannot get effects: %s\n", NvCV_GetErrorStringFromCode(err)); + printf("where effects are:\n%s", cStr); +} + +static int ParseMyArgs(int argc, char **argv) { + int errs = 0; + for (--argc, ++argv; argc--; ++argv) { + bool help; + const char *arg = *argv; + if (arg[0] == '-') { + if (arg[1] == '-') { // double-dash + if (GetFlagArgVal("verbose", arg, &FLAG_verbose) || + GetFlagArgVal("effect", arg, &FLAG_effect) || + GetFlagArgVal("strength", arg, &FLAG_strength) || + GetFlagArgVal("scale", arg, &FLAG_scale) || + GetFlagArgVal("mode", arg, &FLAG_mode) || + GetFlagArgVal("model_dir", arg, &FLAG_modelDir) || + GetFlagArgVal("out_file", arg, &FLAG_outFile) + ) { + continue; + } else if (GetFlagArgVal("help", arg, &help)) { // --help + Usage(); + errs = 1; + } + } + else { // single dash + for (++arg; *arg; ++arg) { + if (*arg == 'v') { + FLAG_verbose = true; + } else { + printf("Unknown flag ignored: \"-%c\"\n", *arg); + } + } + continue; + } + } + else { // no dash + FLAG_inFiles.push_back(arg); + } + } + return errs; +} + + +class App { +public: + NvVFX_Handle _eff; + NvCVImage _src, _dst, _stg; + CUstream _stream; + unsigned _batchSize; + + App() : _eff(nullptr), _stream(0), _batchSize(0) {} + ~App() { NvVFX_DestroyEffect(_eff); if (_stream) NvVFX_CudaStreamDestroy(_stream); } + + NvCV_Status init(const char* effectName, unsigned batchSize, const NvCVImage *src) { + NvCV_Status err = NVCV_ERR_UNIMPLEMENTED; + unsigned dw, dh; + + if (FLAG_resolution) { + dw = FLAG_resolution * src->width / src->height, // No rounding + dh = FLAG_resolution; + } + else { + dw = lroundf(src->width * FLAG_scale), + dh = lroundf(src->height * FLAG_scale); + } + + _batchSize = batchSize; + BAIL_IF_ERR(err = NvVFX_CreateEffect(effectName, &_eff)); + + if (!strcmp(effectName, NVVFX_FX_TRANSFER)) { + BAIL_IF_ERR(err = AllocateBatchBuffer(&_src, _batchSize, src->width, src->height, NVCV_RGB, NVCV_U8, NVCV_CHUNKY, NVCV_CUDA, 0)); + BAIL_IF_ERR(err = AllocateBatchBuffer(&_dst, _batchSize, src->width, src->height, NVCV_RGB, NVCV_U8, NVCV_CHUNKY, NVCV_CUDA, 0)); + } +#ifdef NVVFX_FX_SR_UPSCALE + else if (!strcmp(effectName, NVVFX_FX_SR_UPSCALE)) { + BAIL_IF_ERR(err = AllocateBatchBuffer(&_src, _batchSize, src->width, src->height, NVCV_RGBA, NVCV_U8, NVCV_CHUNKY, NVCV_CUDA, 32)); // n*32, n>=0 + BAIL_IF_ERR(err = AllocateBatchBuffer(&_dst, _batchSize, dw, dh, NVCV_RGBA, NVCV_U8, NVCV_CHUNKY, NVCV_CUDA, 32)); + } +#endif // NVVFX_FX_SR_UPSCALE +#ifdef NVVFX_FX_GREEN_SCREEN + else if (!strcmp(effectName, NVVFX_FX_GREEN_SCREEN)) { + BAIL_IF_ERR(err = AllocateBatchBuffer(&_src, _batchSize, src->width, src->height, NVCV_BGR, NVCV_U8, NVCV_CHUNKY, NVCV_CUDA, 1)); + BAIL_IF_ERR(err = AllocateBatchBuffer(&_dst, _batchSize, src->width, src->height, NVCV_Y, NVCV_U8, NVCV_CHUNKY, NVCV_CUDA, 1)); + BAIL_IF_ERR(err = NvVFX_SetString(_eff, NVVFX_MODEL_DIRECTORY, FLAG_modelDir.c_str())); + BAIL_IF_ERR(err = NvVFX_SetU32(_eff, NVVFX_MODE, FLAG_mode)); + } +#endif // NVVFX_FX_GREEN_SCREEN +#ifdef NVVFX_FX_ARTIFACT_REDUCTION + else if (!strcmp(effectName, NVVFX_FX_ARTIFACT_REDUCTION)) { + BAIL_IF_ERR(err = AllocateBatchBuffer(&_src, _batchSize, src->width, src->height, NVCV_BGR, NVCV_F32, NVCV_PLANAR, NVCV_CUDA, 1)); + BAIL_IF_ERR(err = AllocateBatchBuffer(&_dst, _batchSize, src->width, src->height, NVCV_BGR, NVCV_F32, NVCV_PLANAR, NVCV_CUDA, 1)); + BAIL_IF_ERR(err = NvVFX_SetString(_eff, NVVFX_MODEL_DIRECTORY, FLAG_modelDir.c_str())); + BAIL_IF_ERR(err = NvVFX_SetU32(_eff, NVVFX_STRENGTH, (unsigned)FLAG_strength)); + } +#endif // NVVFX_FX_ARTIFACT_REDUCTION +#ifdef NVVFX_FX_SUPER_RES + else if (!strcmp(effectName, NVVFX_FX_SUPER_RES)) { + BAIL_IF_ERR(err = AllocateBatchBuffer(&_src, _batchSize, src->width, src->height, NVCV_BGR, NVCV_F32, NVCV_PLANAR, NVCV_CUDA, 1)); + BAIL_IF_ERR(err = AllocateBatchBuffer(&_dst, _batchSize, dw, dh, NVCV_BGR, NVCV_F32, NVCV_PLANAR, NVCV_CUDA, 1)); + BAIL_IF_ERR(err = NvVFX_SetString(_eff, NVVFX_MODEL_DIRECTORY, FLAG_modelDir.c_str())); + BAIL_IF_ERR(err = NvVFX_SetU32(_eff, NVVFX_STRENGTH, (unsigned)FLAG_strength)); + } +#endif // NVVFX_FX_SUPER_RES + else { + BAIL(err, NVCV_ERR_UNIMPLEMENTED); + } + + { // Set common parameters. + NvCVImage nth; + BAIL_IF_ERR(err = NvVFX_SetImage(_eff, NVVFX_INPUT_IMAGE, NthImage(0, src->height, &_src, &nth))); // Set the first of the batched images in ... + BAIL_IF_ERR(err = NvVFX_SetImage(_eff, NVVFX_OUTPUT_IMAGE, NthImage(0, _dst.height / _batchSize, &_dst, &nth))); // ... and out + BAIL_IF_ERR(err = NvVFX_CudaStreamCreate(&_stream)); + BAIL_IF_ERR(err = NvVFX_SetCudaStream(_eff, NVVFX_CUDA_STREAM, _stream)); + + // The batch size parameter is interpreted at two times: + // (1) during Load(), an appropriate batch-size model is chosen and loaded; + // (2) during Run(), the specified number of images in the batch are processed. + // The optimum throughput results from submitting a batch which is an integral multiple of the batched model + // chosen in Load(). + // + // To request a particular batch-sized model, set the batch size before calling Load(), + // then get the batch size afterward to find out what batch-size model was chosen. If you do not specify the + // desired batchSize before calling Load(), it will choose the batchSize=1 model, since that is the default + // value for batchSize. + // + // After calling Load(), you can subsequently change the batch size to any number, even larger or smaller + // than the batch size of the chosen model. If a larger batch size is chosen, smaller batches are submitted + // until the entire larger batch has been processed. In any event, the batch size should be set at least twice: + // once before Load() and once before the initial Run(). In many server applications, it is expected that + // the batch size is changing constantly as some videos complete and other are added, so setting the batchSize + // before every Run() call would be typical. + unsigned gotBatch; + BAIL_IF_ERR(err = NvVFX_SetU32(_eff, NVVFX_MODEL_BATCH, _batchSize)); // Try to choose a model tuned to this batch size + err = NvVFX_Load(_eff); // This will load a new batched model -- a weighty process + if (!(NVCV_SUCCESS == err || NVCV_ERR_MODELSUBSTITUTION == err)) goto bail; + BAIL_IF_ERR(err = NvVFX_GetU32(_eff, NVVFX_MODEL_BATCH, &gotBatch)); // This tells us the batch size of the chosen model + if (FLAG_verbose && gotBatch != _batchSize) { + printf("Effect %s has no batch=%u model; processing in multiple batches of size %u%s instead\n", + effectName, _batchSize, gotBatch, (gotBatch > 1 ? " or less" : "")); + BAIL_IF_ERR(err = NvVFX_SetU32(_eff, NVVFX_BATCH_SIZE, _batchSize)); // This is lightweight, and usually done each Run + } + } + + bail: + return err; + } +}; + + +NvCV_Status BatchProcessImages(const char* effectName, const std::vector& srcImages, const char *outfilePattern) { + NvCV_Status err = NVCV_SUCCESS; + unsigned batchSize = (unsigned)srcImages.size(); + App app; + cv::Mat ocv; + NvCVImage nvx; + unsigned srcWidth, srcHeight, dstHeight, i; + + // Read in the first image, to determine the resolution for init() + BAIL_IF_FALSE(srcImages.size() > 0, err, NVCV_ERR_MISSINGINPUT); + ocv = cv::imread(srcImages[0]); + if (!ocv.data) { + printf("Cannot read image file \"%s\"\n", srcImages[0]); + BAIL(err, NVCV_ERR_READ); + } + NVWrapperForCVMat(&ocv, &nvx); + srcWidth = nvx.width; + srcHeight = nvx.height; + BAIL_IF_ERR(err = app.init(effectName, batchSize, &nvx)); // Init effect and buffers + + // Transfer the first image to the batch src. + // Note, in all transfers, the scale factor only applies to floating-point pixels. + BAIL_IF_ERR(err = TransferToNthImage(0, &nvx, &app._src, 1.f/255.f, app._stream, &app._stg)); + ocv.release(); + + // Read the remaining images and transfer to the batch src + for (i = 1; i < batchSize; ++i) { + ocv = cv::imread(srcImages[i]); + if (!ocv.data) { + printf("Cannot read image file \"%s\"\n", srcImages[i]); + BAIL(err, NVCV_ERR_READ); + } + NVWrapperForCVMat(&ocv, &nvx); + if (!(nvx.width == srcWidth && nvx.height == srcHeight)) { + printf("Input image file \"%s\" %ux%u does not match %ux%u\n", srcImages[i], nvx.width, nvx.height, srcWidth, srcHeight); + BAIL(err, NVCV_ERR_MISMATCH); + } + BAIL_IF_ERR(err = TransferToNthImage(i, &nvx, &app._src, 1.f / 255.f, app._stream, &app._stg)); + ocv.release(); + } + + // Run batch + BAIL_IF_ERR(err = NvVFX_SetU32(app._eff, NVVFX_BATCH_SIZE, (unsigned)srcImages.size())); // The batchSize can change every Run + BAIL_IF_ERR(err = NvVFX_Run(app._eff, 0)); + + // Retrieve and write images + dstHeight = app._dst.height / batchSize; + BAIL_IF_ERR(err = NvCVImage_Alloc(&nvx, app._dst.width, dstHeight, ((app._dst.numComponents == 1) ? NVCV_Y : NVCV_BGR), NVCV_U8, NVCV_CHUNKY, NVCV_CPU, 0)); + CVWrapperForNvCVImage(&nvx, &ocv); + for (i = 0; i < batchSize; ++i) { + char fileName[1024]; + snprintf(fileName, sizeof(fileName), outfilePattern, i); + BAIL_IF_ERR(err = TransferFromNthImage(i, &app._dst, &nvx, 255.f, app._stream, &app._stg)); + if (!cv::imwrite(fileName, ocv)) { + printf("Cannot write image file \"%s\"\n", fileName); + BAIL(err, NVCV_ERR_WRITE); + } + } + // NvCVImage_Dealloc() is called in the destructors + +bail: + return err; +} + + +int main(int argc, char** argv) { + int nErrs; + NvCV_Status vfxErr; + + nErrs = ParseMyArgs(argc, argv); + if (nErrs) + return nErrs; + + if (FLAG_outFile.empty()) + FLAG_outFile = "BatchOut_%02u.png"; + else if (std::string::npos == FLAG_outFile.find_first_of('%')) + FLAG_outFile.insert(FLAG_outFile.size() - 4, "_%02u"); // assuming .xxx, i.e. .jpg, .png + + vfxErr = BatchProcessImages(FLAG_effect.c_str(), FLAG_inFiles, FLAG_outFile.c_str()); + if (NVCV_SUCCESS != vfxErr) { + printf("Error: %s\n", NvCV_GetErrorStringFromCode(vfxErr)); + nErrs = (int)vfxErr; + } + + return nErrs; +} diff --git a/samples/BatchEffectApp/BatchEffectApp.exe b/samples/BatchEffectApp/BatchEffectApp.exe new file mode 100644 index 0000000..b43fd7f Binary files /dev/null and b/samples/BatchEffectApp/BatchEffectApp.exe differ diff --git a/samples/BatchEffectApp/BatchUtilities.cpp b/samples/BatchEffectApp/BatchUtilities.cpp new file mode 100644 index 0000000..b82a85a --- /dev/null +++ b/samples/BatchEffectApp/BatchUtilities.cpp @@ -0,0 +1,156 @@ +/*############################################################################### +# +# Copyright (c) 2020 NVIDIA Corporation +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal in +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +# the Software, and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +###############################################################################*/ + +#include "BatchUtilities.h" + + +/******************************************************************************** + * AllocateBatchBuffer + ********************************************************************************/ + +NvCV_Status AllocateBatchBuffer(NvCVImage *im, unsigned batchSize, unsigned width, unsigned height, NvCVImage_PixelFormat format, + NvCVImage_ComponentType type, unsigned layout, unsigned memSpace, unsigned alignment) { + return NvCVImage_Alloc(im, width, height * batchSize, format, type, layout, memSpace, alignment); +} + + +/******************************************************************************** + * NthImage + ********************************************************************************/ + +NvCVImage* NthImage(unsigned n, unsigned height, NvCVImage* full, NvCVImage* view) { + unsigned y = height; + if (NVCV_PLANAR & full->planar) { // if not any of the chunky formats + if (NVCV_PLANAR == full->planar) y *= full->numComponents; + else if (NVCV_YUV444 == full->pixelFormat) y *= 3; + else if (NVCV_YUV422 == full->pixelFormat) y *= 2; + else if (NVCV_YUV420 == full->pixelFormat) y = y * 3 / 2; + else y = 0; + } + NvCVImage_InitView(view, full, 0, y * n, full->width, height); + return view; +} + + +/******************************************************************************** + * ComputeImageBytes + ********************************************************************************/ + +int ComputeImageBytes(const NvCVImage* im) { + int imageBytes = im->pitch * (int)im->height; // Correct for all chunky formats + if (NVCV_PLANAR & im->planar) { // if not any of the chunky formats + if (NVCV_PLANAR == im->planar) imageBytes *= (int)im->numComponents; + else if (NVCV_YUV422 == im->pixelFormat) imageBytes *= 2; + else if (NVCV_YUV420 == im->pixelFormat) imageBytes = imageBytes * 3 / 2; + else imageBytes = 0; + } + return imageBytes; +} + + +/******************************************************************************** + * TransferToNthImage + ********************************************************************************/ + +NvCV_Status TransferToNthImage( + unsigned n, const NvCVImage* src, NvCVImage* dstBatch, float scale, struct CUstream_st* stream, NvCVImage* tmp) { + NvCVImage nth; + return NvCVImage_Transfer(src, NthImage(n, src->height, dstBatch, &nth), scale, stream, tmp); +} + + +/******************************************************************************** + * TransferFromNthImage + ********************************************************************************/ + +NvCV_Status TransferFromNthImage( + unsigned n, const NvCVImage* srcBatch, NvCVImage* dst, float scale, struct CUstream_st* stream, NvCVImage* tmp) { + NvCVImage nth; + return NvCVImage_Transfer(NthImage(n, dst->height, const_cast(srcBatch), &nth), dst, scale, stream, tmp); +} + + +/******************************************************************************** + * TransferToBatchImage + * This illustrates the use of the pixel offset method, but the Nth image method could be used instead. + ********************************************************************************/ + +NvCV_Status TransferToBatchImage( + unsigned batchSize, const NvCVImage** srcArray, NvCVImage* dstBatch, float scale, struct CUstream_st* stream, NvCVImage* tmp) { + NvCV_Status err = NVCV_SUCCESS; + NvCVImage nth; + (void)NthImage(0, (**srcArray).height, dstBatch, &nth); + int nextDst = ComputeImageBytes(&nth); + for (; batchSize--; ++srcArray, nth.pixels = (void*)((char*)nth.pixels + nextDst)) + if (NVCV_SUCCESS != (err = NvCVImage_Transfer(*srcArray, &nth, scale, stream, tmp))) + break; + return err; +} + + +/******************************************************************************** + * TransferFromBatchImage + * This illustrates the use of the pixel offset method, but the Nth image method could be used instead. + ********************************************************************************/ + +NvCV_Status TransferFromBatchImage( + unsigned batchSize, const NvCVImage* srcBatch, NvCVImage** dstArray, float scale, struct CUstream_st* stream, NvCVImage* tmp) { + NvCV_Status err = NVCV_SUCCESS; + NvCVImage nth; + (void)NthImage(0, (**dstArray).height, const_cast(srcBatch), &nth); + int nextSrc = ComputeImageBytes(&nth); + for (; batchSize--; nth.pixels = (void*)((char*)nth.pixels + nextSrc), ++dstArray) + if (NVCV_SUCCESS != (err = NvCVImage_Transfer(&nth, *dstArray, scale, stream, tmp))) + break; + return err; +} + + +/******************************************************************************** + * TransferBatchImage + ********************************************************************************/ + +NvCV_Status TransferBatchImage(const NvCVImage *srcBatch, NvCVImage *dstBatch, + unsigned imHeight, unsigned batchSize, float scale, struct CUstream_st *stream) { + NvCV_Status err = NVCV_SUCCESS; + NvCVImage tmp; + + if ((!(srcBatch->planar & NVCV_PLANAR) && !(dstBatch->planar & NVCV_PLANAR)) // both chunky + || (srcBatch->planar == NVCV_PLANAR && dstBatch->planar == NVCV_PLANAR && srcBatch->pixelFormat == dstBatch->pixelFormat) + ) { // This is a fast transfer + err = NvCVImage_Transfer(srcBatch, dstBatch, scale, stream, &tmp); + } + else { // This is guaranteed to be safe for all transfers + NvCVImage subSrc, subDst; + int nextSrc, nextDst, n; + NvCVImage_InitView(&subSrc, const_cast(srcBatch), 0, 0, srcBatch->width, imHeight); + NvCVImage_InitView(&subDst, dstBatch, 0, 0, dstBatch->width, imHeight); + nextSrc = ComputeImageBytes(&subSrc); + nextDst = ComputeImageBytes(&subDst); + for (n = batchSize; n--; subSrc.pixels = (char*)subSrc.pixels + nextSrc, + subDst.pixels = (char*)subDst.pixels + nextDst) + if (NVCV_SUCCESS != (err = NvCVImage_Transfer(&subSrc, &subDst, scale, stream, &tmp))) + break; + } + return err; +} diff --git a/samples/BatchEffectApp/BatchUtilities.h b/samples/BatchEffectApp/BatchUtilities.h new file mode 100644 index 0000000..bf7fe8f --- /dev/null +++ b/samples/BatchEffectApp/BatchUtilities.h @@ -0,0 +1,127 @@ +/*############################################################################### +# +# Copyright (c) 2020 NVIDIA Corporation +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal in +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +# the Software, and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +###############################################################################*/ + +#ifndef __BATCH_UTILITIES__ +#define __BATCH_UTILITIES__ + +#include "nvCVImage.h" + + +//! Allocate a batch buffer. +//! \note All of the arguments are identical to that of NvCVImage_Alloc plus the batchSize. +//! \param[out] im the image to initialize. +//! \param[in] batchSize the number i=of images in the batch. +//! \param[in] width the desired width of each image, in pixels. +//! \param[in] height the desired height of each image, in pixels. +//! \param[in] format the format of the pixels. +//! \param[in] type the type of the components of the pixels. +//! \param[in] layout One of { NVCV_CHUNKY, NVCV_PLANAR } or one of the YUV layouts. +//! \param[in] memSpace Location of the buffer: one of { NVCV_CPU, NVCV_CPU_PINNED, NVCV_GPU, NVCV_CUDA } +//! \param[in] alignment row byte alignment. Choose 0 or a power of 2. +//! 1: yields no gap whatsoever between scanlines; +//! 0: default alignment: 4 on CPU, and cudaMallocPitch's choice on GPU. +//! Other common values are 16 or 32 for cache line size, 32 for texture alignment. +//! \return NVCV_SUCCESS if the operation was successful. +//! \return NVCV_ERR_PIXELFORMAT if the pixel format is not accommodated. +//! \return NVCV_ERR_MEMORY if there is not enough memory to allocate the buffer. +//! \note this simply multiplies height by batchSize and calls NvCVImage_Alloc(). +NvCV_Status AllocateBatchBuffer(NvCVImage* im, unsigned batchSize, unsigned width, unsigned height, + NvCVImage_PixelFormat format, NvCVImage_ComponentType type, unsigned layout, unsigned memSpace, unsigned alignment); + +//! Initialize an image descriptor for the Nth image in a batch. +//! \param[in] n the index of the desired image in the batch. +//! \param[in] height the height of the image +//! \param[in] full the batch image, or the 0th image in the batch. +//! \param[out] view the image descriptor to be initialized to a view of the nth image in the batch. +//! \return a pointer to the nth image view, facilitating the use of NthImage() inline as an argument to a function. +//! \note NvCVImage nth; NvVFX_SetImage(effect, NVVFX_INPUT_IMAGE, NthImage(0, height, batchIn, &nth)); +//! is typically used to set the input image for a batch operation; similarly for output. +NvCVImage* NthImage(unsigned n, unsigned height, NvCVImage* full, NvCVImage* view); + +//! Compute the byte offset between one image in a batch and the next. +//! \param[in] im the image to be measured. +//! \return the increment from one image to the next in a batch. +//! \note this will be negative if the pitch is negative. +int ComputeImageBytes(const NvCVImage* im); + +//! Transfer To the Nth Image in a Batched Image. +//! \param[in] n the index of the batch image to modify. +//! \param[in] src the source image. +//! \param[in] dstBatch the batch destination image. +//! \param[in] scale the pixel scale factor. +//! \param[in] stream the CUDA stream on which to perform the transfer. +//! \param[in] tmp the stage buffer (can be NULL, but can affect performance if needed). +//! \return NVCV_SUCCESS if the operation was successful. +NvCV_Status TransferToNthImage( + unsigned n, const NvCVImage* src, NvCVImage* dstBatch, float scale, struct CUstream_st* stream, NvCVImage* tmp); + +//! Transfer From the Nth Image in a Batched Image. +//! \param[in] n the index of the batch image to read. +//! \param[in] srcBatch the batch source image. +//! \param[in] dst the destination image. +//! \param[in] scale the pixel scale factor. +//! \param[in] stream the CUDA stream on which to perform the transfer. +//! \param[in] tmp the stage buffer (can be NULL, but can affect performance if needed). +//! \return NVCV_SUCCESS if the operation was successful. +NvCV_Status TransferFromNthImage( + unsigned n, const NvCVImage* srcBatch, NvCVImage* dst, float scale, struct CUstream_st* stream, NvCVImage* tmp); + +//! Transfer from a list of source images to a batch image. +//! We use an array of image pointers rather than an array of images +//! in order to more easily accommodate dynamically-changing batches. +//! \param[in] batchSize the number of source images to be transferred to the batch image. +//! \param[in] srcArray array of pointers to the source images. +//! \param[out] dstBatch the batch destination image. +//! \param[in] scale the pixel scale factor. +//! \param[in] stream the CUDA stream. +//! \param[in] tmp the stage buffer (can be NULL, but can affect performance if needed). +//! \return NVCV_SUCCESS if the operation was successful. +NvCV_Status TransferToBatchImage( + unsigned batchSize, const NvCVImage** srcArray, NvCVImage* dstBatch, float scale, struct CUstream_st* stream, NvCVImage* tmp); + +//! Transfer from a batch image to a list of destination images. +//! We use an array of image pointers rather than an array of images +//! in order to more easily accommodate dynamically-changing batches. +//! \param[in] batchSize the number of destination images to be transferred from the batch image. +//! \param[in] srcBatch the batch source image. +//! \param[out] dstArray array of pointers to the source images. +//! \param[in] scale the pixel scale factor. +//! \param[in] stream the CUDA stream. +//! \param[in] tmp the stage buffer (can be NULL, but can affect performance if needed). +//! \return NVCV_SUCCESS if the operation was successful. +NvCV_Status TransferFromBatchImage( + unsigned batchSize, const NvCVImage* srcBatch, NvCVImage** dstArray, float scale, struct CUstream_st* stream, NvCVImage* tmp); + +//! Transfer all images in a batch to another compatible batch of images. +//! \param[in] srcBatch the batch source image. +//! \param[out] dstBatch the batch destination image. +//! \param[in] imHeight the height of each image in the batch. +//! \param[in] batchSize the number of images in the batch. +//! \param[in] scale the pixel scale factor. +//! \param[in] stream the CUDA stream. +//! \return NVCV_SUCCESS if the operation was successful. +NvCV_Status TransferBatchImage(const NvCVImage* srcBatch, NvCVImage* dstBatch, + unsigned imHeight, unsigned batchSize, float scale, struct CUstream_st* stream); + + +#endif // __BATCH_UTILITIES__ diff --git a/samples/BatchEffectApp/CMakeLists.txt b/samples/BatchEffectApp/CMakeLists.txt new file mode 100644 index 0000000..bb0078f --- /dev/null +++ b/samples/BatchEffectApp/CMakeLists.txt @@ -0,0 +1,91 @@ +set(SOURCE_FILES + BatchEffectApp.cpp + BatchUtilities.cpp + ../../nvvfx/src/nvVideoEffectsProxy.cpp + ../../nvvfx/src/nvCVImageProxy.cpp) + +# Set Visual Studio source filters +source_group("Source Files" FILES ${SOURCE_FILES}) + +add_executable(BatchEffectApp ${SOURCE_FILES}) +target_include_directories(BatchEffectApp PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/../utils + ) +target_include_directories(BatchEffectApp PUBLIC + ${SDK_INCLUDES_PATH} + ) + +if(MSVC) + target_link_libraries(BatchEffectApp PUBLIC + opencv346 + NVVideoEffects + ${CMAKE_CURRENT_SOURCE_DIR}/../external/cuda/lib/x64/cudart.lib + ) + + set(OPENCV_PATH_STR ${CMAKE_CURRENT_SOURCE_DIR}/../external/opencv/bin) + set(VFXSDK_PATH_STR ${CMAKE_CURRENT_SOURCE_DIR}/../../bin) # Also the location for CUDA/NVTRT/libcrypto + set(PATH_STR "PATH=%PATH%" ${VFXSDK_PATH_STR} ${OPENCV_PATH_STR}) + set(CMD_ARG_STR "--show --in_file=\"${CMAKE_CURRENT_SOURCE_DIR}/../input/input_003054.jpg\"") + set_target_properties(BatchEffectApp PROPERTIES + FOLDER SampleApps + VS_DEBUGGER_ENVIRONMENT "${PATH_STR}" + VS_DEBUGGER_COMMAND_ARGUMENTS "${CMD_ARG_STR}" + ) +else() + + target_link_libraries(BatchEffectApp PUBLIC + NVVideoEffects + NVCVImage + OpenCV + TensorRT + CUDA + ) +endif() + +#Batch denoise effect +set(SOURCE_FILES + BatchDenoiseEffectApp.cpp + BatchUtilities.cpp + ../../nvvfx/src/nvVideoEffectsProxy.cpp + ../../nvvfx/src/nvCVImageProxy.cpp) + +# Set Visual Studio source filters +source_group("Source Files" FILES ${SOURCE_FILES}) + +add_executable(BatchDenoiseEffectApp ${SOURCE_FILES}) +target_include_directories(BatchDenoiseEffectApp PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/../utils + ) +target_include_directories(BatchDenoiseEffectApp PUBLIC + ${SDK_INCLUDES_PATH} + ) + +if(MSVC) + target_link_libraries(BatchDenoiseEffectApp PUBLIC + opencv346 + NVVideoEffects + ${CMAKE_CURRENT_SOURCE_DIR}/../external/cuda/lib/x64/cudart.lib + ) + target_include_directories(BatchDenoiseEffectApp PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../external/cuda/include) + + set(OPENCV_PATH_STR ${CMAKE_CURRENT_SOURCE_DIR}/../external/opencv/bin) + set(VFXSDK_PATH_STR ${CMAKE_CURRENT_SOURCE_DIR}/../../bin) # Also the location for CUDA/NVTRT/libcrypto + set(PATH_STR "PATH=%PATH%" ${VFXSDK_PATH_STR} ${OPENCV_PATH_STR}) + set(CMD_ARG_STR "video1.mp4 video2.mp4") + set_target_properties(BatchDenoiseEffectApp PROPERTIES + FOLDER SampleApps + VS_DEBUGGER_ENVIRONMENT "${PATH_STR}" + VS_DEBUGGER_COMMAND_ARGUMENTS "${CMD_ARG_STR}" + ) +else() + + target_link_libraries(BatchDenoiseEffectApp PUBLIC + NVVideoEffects + NVCVImage + OpenCV + TensorRT + CUDA + ) +endif() diff --git a/samples/BatchEffectApp/run.bat b/samples/BatchEffectApp/run.bat new file mode 100644 index 0000000..cc804b6 --- /dev/null +++ b/samples/BatchEffectApp/run.bat @@ -0,0 +1,7 @@ +SETLOCAL +SET PATH=%PATH%;..\external\opencv\bin; +SET IMAGE_LIST=..\input\LeFret_000900.jpg ..\input\LeFret_001400.jpg ..\input\LeFret_003400.jpg ..\input\LeFret_012300.jpg +BatchEffectApp.exe --effect=GreenScreen --out_file=GreenScreen_%%04u.png %IMAGE_LIST% +BatchEffectApp.exe --effect=ArtifactReduction --out_file=ArtifactReduction_%%04u.png %IMAGE_LIST% +BatchEffectApp.exe --effect=SuperRes --out_file=SuperRes_%%04u.png --scale=1.5 %IMAGE_LIST% +BatchEffectApp.exe --effect=Upscale --out_file=Upscale_%%04u.png --scale=1.5 %IMAGE_LIST% \ No newline at end of file diff --git a/samples/CMakeLists.txt b/samples/CMakeLists.txt index 861a8d5..abae9e9 100644 --- a/samples/CMakeLists.txt +++ b/samples/CMakeLists.txt @@ -1,4 +1,7 @@ # Sample apps add_subdirectory(external) -add_subdirectory(UpscalePipelineApp) # Artifact Reduction and Upscale -add_subdirectory(VideoEffectsApp) # Artifact Reduction and Super Res +add_subdirectory(UpscalePipelineApp) # Artifact Reduction and Upscale +add_subdirectory(VideoEffectsApp) # Artifact Reduction and Super Res +add_subdirectory(AigsEffectApp) # Green Screen +add_subdirectory(BatchEffectApp) +add_subdirectory(DenoiseEffectApp) diff --git a/samples/DenoiseEffectApp/CMakeLists.txt b/samples/DenoiseEffectApp/CMakeLists.txt new file mode 100644 index 0000000..c0d8b95 --- /dev/null +++ b/samples/DenoiseEffectApp/CMakeLists.txt @@ -0,0 +1,35 @@ +set(SOURCE_FILES DenoiseEffectApp.cpp ../../nvvfx/src/nvVideoEffectsProxy.cpp ../../nvvfx/src/nvCVImageProxy.cpp) + +# Set Visual Studio source filters +source_group("Source Files" FILES ${SOURCE_FILES}) + +add_executable(DenoiseEffectApp ${SOURCE_FILES}) +target_include_directories(DenoiseEffectApp PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../utils) +target_include_directories(DenoiseEffectApp PUBLIC ${SDK_INCLUDES_PATH}) + +if(MSVC) + target_link_libraries(DenoiseEffectApp PUBLIC + opencv346 + NVVideoEffects + ${CMAKE_CURRENT_SOURCE_DIR}/../external/cuda/lib/x64/cudart.lib + ) + target_include_directories(DenoiseEffectApp PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../external/cuda/include) + set(OPENCV_PATH_STR ${CMAKE_CURRENT_SOURCE_DIR}/../external/opencv/bin) + set(VFXSDK_PATH_STR ${CMAKE_CURRENT_SOURCE_DIR}/../../bin) # Also the location for CUDA/NVTRT/libcrypto + set(PATH_STR "PATH=%PATH%" ${VFXSDK_PATH_STR} ${OPENCV_PATH_STR}) + set(CMD_ARG_STR "--model_dir=\"${CMAKE_CURRENT_SOURCE_DIR}/../../bin/models\" --show --webcam") + set_target_properties(DenoiseEffectApp PROPERTIES + FOLDER SampleApps + VS_DEBUGGER_ENVIRONMENT "${PATH_STR}" + VS_DEBUGGER_COMMAND_ARGUMENTS "${CMD_ARG_STR}" + ) +else() + + target_link_libraries(DenoiseEffectApp PUBLIC + NVVideoEffects + NVCVImage + OpenCV + TensorRT + CUDA + ) +endif() diff --git a/samples/DenoiseEffectApp/DenoiseEffectApp.cpp b/samples/DenoiseEffectApp/DenoiseEffectApp.cpp new file mode 100644 index 0000000..ff9a07c --- /dev/null +++ b/samples/DenoiseEffectApp/DenoiseEffectApp.cpp @@ -0,0 +1,684 @@ +/*############################################################################### +# +# Copyright (c) 2020 NVIDIA Corporation +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal in +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +# the Software, and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +###############################################################################*/ +#include +#include +#include + +#include +#include +#include +#include +#include "nvCVOpenCV.h" +#include "nvVideoEffects.h" +#include "opencv2/opencv.hpp" + + +#ifdef _MSC_VER + #define strcasecmp _stricmp + #include +#else // !_MSC_VER + #include +#endif // _MSC_VER + +#define BAIL_IF_ERR(err) do { if (0 != (err)) { goto bail; } } while(0) +#define BAIL_IF_NULL(x, err, code) do { if ((void*)(x) == NULL) { err = code; goto bail; } } while(0) +#define NVCV_ERR_HELP 411 + +#ifdef _WIN32 + #define DEFAULT_CODEC "avc1" +#else // !_WIN32 + #define DEFAULT_CODEC "H264" +#endif // _WIN32 + + +bool FLAG_debug = false, + FLAG_verbose = false, + FLAG_show = false, + FLAG_progress = false, + FLAG_webcam = false; +float FLAG_strength = 0.f; +std::string FLAG_codec = DEFAULT_CODEC, + FLAG_camRes = "1280x720", + FLAG_inFile, + FLAG_outFile, + FLAG_outDir, + FLAG_modelDir; + + +// Set this when using OTA Updates +// This path is used by nvVideoEffectsProxy.cpp to load the SDK dll +// when using OTA Updates +char *g_nvVFXSDKPath = NULL; + +static bool GetFlagArgVal(const char *flag, const char *arg, const char **val) { + if (*arg != '-') + return false; + while (*++arg == '-') + continue; + const char *s = strchr(arg, '='); + if (s == NULL) { + if (strcmp(flag, arg) != 0) + return false; + *val = NULL; + return true; + } + size_t n = s - arg; + if ((strlen(flag) != n) || (strncmp(flag, arg, n) != 0)) + return false; + *val = s + 1; + return true; +} + +static bool GetFlagArgVal(const char *flag, const char *arg, std::string *val) { + const char *valStr; + if (!GetFlagArgVal(flag, arg, &valStr)) + return false; + val->assign(valStr ? valStr : ""); + return true; +} + +static bool GetFlagArgVal(const char *flag, const char *arg, bool *val) { + const char *valStr; + bool success = GetFlagArgVal(flag, arg, &valStr); + if (success) { + *val = (valStr == NULL || + strcasecmp(valStr, "true") == 0 || + strcasecmp(valStr, "on") == 0 || + strcasecmp(valStr, "yes") == 0 || + strcasecmp(valStr, "1") == 0 + ); + } + return success; +} + +static bool GetFlagArgVal(const char *flag, const char *arg, float *val) { + const char *valStr; + bool success = GetFlagArgVal(flag, arg, &valStr); + if (success) + *val = strtof(valStr, NULL); + return success; +} + +static bool GetFlagArgVal(const char *flag, const char *arg, long *val) { + const char *valStr; + bool success = GetFlagArgVal(flag, arg, &valStr); + if (success) + *val = strtol(valStr, NULL, 10); + return success; +} + +static bool GetFlagArgVal(const char *flag, const char *arg, int *val) { + long longVal; + bool success = GetFlagArgVal(flag, arg, &longVal); + if (success) + *val = (int)longVal; + return success; +} + +static void Usage() { + printf( + "DenoiseEffectApp [args ...]\n" + " where args is:\n" + " --in_file= input file to be processed (can be an image but the best denoising performance is observed on videos)\n" + " --webcam use a webcam as the input\n" + " --out_file= output file to be written\n" + " --show display the results in a window (for webcam, it is always true)\n" + " --strength= strength of an effect [0-1]\n" + " --model_dir= the path to the directory that contains the models\n" + " --codec= the fourcc code for the desired codec (default " DEFAULT_CODEC ")\n" + " --progress show progress\n" + " --verbose verbose output\n" + " --debug print extra debugging information\n" + ); +} + +static int ParseMyArgs(int argc, char **argv) { + int errs = 0; + for (--argc, ++argv; argc--; ++argv) { + bool help; + const char *arg = *argv; + if (arg[0] != '-') { + continue; + } else if ((arg[1] == '-') && + ( GetFlagArgVal("verbose", arg, &FLAG_verbose) || + GetFlagArgVal("in", arg, &FLAG_inFile) || + GetFlagArgVal("in_file", arg, &FLAG_inFile) || + GetFlagArgVal("out", arg, &FLAG_outFile) || + GetFlagArgVal("out_file", arg, &FLAG_outFile) || + GetFlagArgVal("show", arg, &FLAG_show) || + GetFlagArgVal("webcam", arg, &FLAG_webcam) || + GetFlagArgVal("cam_res", arg, &FLAG_camRes) || + GetFlagArgVal("strength", arg, &FLAG_strength) || + GetFlagArgVal("model_dir", arg, &FLAG_modelDir) || + GetFlagArgVal("codec", arg, &FLAG_codec) || + GetFlagArgVal("progress", arg, &FLAG_progress) || + GetFlagArgVal("debug", arg, &FLAG_debug) + )) { + continue; + } else if (GetFlagArgVal("help", arg, &help)) { + return NVCV_ERR_HELP; + } else if (arg[1] != '-') { + for (++arg; *arg; ++arg) { + if (*arg == 'v') { + FLAG_verbose = true; + } else { + printf("Unknown flag ignored: \"-%c\"\n", *arg); + } + } + continue; + } else { + printf("Unknown flag ignored: \"%s\"\n", arg); + } + } + return errs; +} + +static bool HasSuffix(const char *str, const char *suf) { + size_t strSize = strlen(str), + sufSize = strlen(suf); + if (strSize < sufSize) + return false; + return (0 == strcasecmp(suf, str + strSize - sufSize)); +} + +static bool HasOneOfTheseSuffixes(const char *str, ...) { + bool matches = false; + const char *suf; + va_list ap; + va_start(ap, str); + while (nullptr != (suf = va_arg(ap, const char*))) { + if (HasSuffix(str, suf)) { + matches = true; + break; + } + } + va_end(ap); + return matches; +} + +static bool IsImageFile(const char *str) { + return HasOneOfTheseSuffixes(str, ".bmp", ".jpg", ".jpeg", ".png", nullptr); +} + +static const char* DurationString(double sc) { + static char buf[16]; + int hr, mn; + hr = (int)(sc / 3600.); + sc -= hr * 3600.; + mn = (int)(sc / 60.); + sc -= mn * 60.; + snprintf(buf, sizeof(buf), "%02d:%02d:%06.3f", hr, mn, sc); + return buf; +} + +struct VideoInfo { + int codec; + int width; + int height; + double frameRate; + long long frameCount; +}; + +static void GetVideoInfo(cv::VideoCapture& reader, const char *fileName, VideoInfo *info) { + info->codec = (int)reader.get(cv::CAP_PROP_FOURCC); + info->width = (int)reader.get(cv::CAP_PROP_FRAME_WIDTH); + info->height = (int)reader.get(cv::CAP_PROP_FRAME_HEIGHT); + info->frameRate = (double)reader.get(cv::CAP_PROP_FPS); + info->frameCount = (long long)reader.get(cv::CAP_PROP_FRAME_COUNT); + if (FLAG_verbose) + printf( + " file \"%s\"\n" + " codec %.4s\n" + " width %4d\n" + " height %4d\n" + " frame rate %.3f\n" + "frame count %4lld\n" + " duration %s\n", + fileName, (char*)&info->codec, info->width, info->height, info->frameRate, info->frameCount, + DurationString(info->frameCount / info->frameRate) + ); +} + +static int StringToFourcc(const std::string& str) { + union chint { int i; char c[4]; }; + chint x = { 0 }; + for (int n = (str.size() < 4) ? (int)str.size() : 4; n--;) + x.c[n] = str[n]; + return x.i; +} + +struct FXApp { + enum Err { + errQuit = +1, // Application errors + errFlag = +2, + errRead = +3, + errWrite = +4, + errNone = NVCV_SUCCESS, // Video Effects SDK errors + errGeneral = NVCV_ERR_GENERAL, + errUnimplemented = NVCV_ERR_UNIMPLEMENTED, + errMemory = NVCV_ERR_MEMORY, + errEffect = NVCV_ERR_EFFECT, + errSelector = NVCV_ERR_SELECTOR, + errBuffer = NVCV_ERR_BUFFER, + errParameter = NVCV_ERR_PARAMETER, + errMismatch = NVCV_ERR_MISMATCH, + errPixelFormat = NVCV_ERR_PIXELFORMAT, + errModel = NVCV_ERR_MODEL, + errLibrary = NVCV_ERR_LIBRARY, + errInitialization = NVCV_ERR_INITIALIZATION, + errFileNotFound = NVCV_ERR_FILE, + errFeatureNotFound = NVCV_ERR_FEATURENOTFOUND, + errMissingInput = NVCV_ERR_MISSINGINPUT, + errResolution = NVCV_ERR_RESOLUTION, + errUnsupportedGPU = NVCV_ERR_UNSUPPORTEDGPU, + errWrongGPU = NVCV_ERR_WRONGGPU, + errUnsupportedDriver = NVCV_ERR_UNSUPPORTEDDRIVER, + errCudaMemory = NVCV_ERR_CUDA_MEMORY, // CUDA errors + errCudaValue = NVCV_ERR_CUDA_VALUE, + errCudaPitch = NVCV_ERR_CUDA_PITCH, + errCudaInit = NVCV_ERR_CUDA_INIT, + errCudaLaunch = NVCV_ERR_CUDA_LAUNCH, + errCudaKernel = NVCV_ERR_CUDA_KERNEL, + errCudaDriver = NVCV_ERR_CUDA_DRIVER, + errCudaUnsupported = NVCV_ERR_CUDA_UNSUPPORTED, + errCudaIllegalAddress = NVCV_ERR_CUDA_ILLEGAL_ADDRESS, + errCuda = NVCV_ERR_CUDA, + }; + + FXApp() { _eff = nullptr; _effectName = nullptr; _inited = false; _showFPS = false; _progress = false; + _show = false; _enableEffect = true, _drawVisualization = true, _framePeriod = 0.f; } + ~FXApp() { NvVFX_DestroyEffect(_eff); } + + void setShow(bool show) { _show = show; } + Err createEffect(const char *effectSelector, const char *modelDir); + void destroyEffect(); + NvCV_Status allocBuffers(unsigned width, unsigned height); + NvCV_Status allocTempBuffers(); + Err processImage(const char *inFile, const char *outFile); + Err processMovie(const char *inFile, const char *outFile); + Err initCamera(cv::VideoCapture& cap); + Err processKey(int key); + void drawFrameRate(cv::Mat& img); + void drawEffectStatus(cv::Mat& img); + Err appErrFromVfxStatus(NvCV_Status status) { return (Err)status; } + const char* errorStringFromCode(Err code); + + NvVFX_Handle _eff; + cv::Mat _srcImg; + cv::Mat _dstImg; + NvCVImage _srcGpuBuf; + NvCVImage _dstGpuBuf; + NvCVImage _srcVFX; + NvCVImage _dstVFX; + NvCVImage _tmpVFX; // We use the same temporary buffer for source and dst, since it auto-shapes as needed + bool _show; + bool _inited; + bool _showFPS; + bool _progress; + bool _enableEffect; + bool _drawVisualization; + const char* _effectName; + float _framePeriod; + std::chrono::high_resolution_clock::time_point _lastTime; +}; + +const char* FXApp::errorStringFromCode(Err code) { + struct LutEntry { Err code; const char *str; }; + static const LutEntry lut[] = { + { errRead, "There was a problem reading a file" }, + { errWrite, "There was a problem writing a file" }, + { errQuit, "The user chose to quit the application" }, + { errFlag, "There was a problem with the command-line arguments" }, + }; + if ((int)code <= 0) return NvCV_GetErrorStringFromCode((NvCV_Status)code); + for (const LutEntry *p = lut; p != &lut[sizeof(lut) / sizeof(lut[0])]; ++p) + if (p->code == code) return p->str; + return "UNKNOWN ERROR"; +} + +void FXApp::drawFrameRate(cv::Mat &img) { + const float timeConstant = 16.f; + std::chrono::high_resolution_clock::time_point now = std::chrono::high_resolution_clock::now(); + std::chrono::duration dur = std::chrono::duration_cast>(now - _lastTime); + float t = dur.count(); + if (0.f < t && t < 100.f) { + if (_framePeriod) + _framePeriod += (t - _framePeriod) * (1.f / timeConstant); // 1 pole IIR filter + else + _framePeriod = t; + if (_showFPS) { + char buf[32]; + snprintf(buf, sizeof(buf), "%.1f", 1. / _framePeriod); + cv::putText(img, buf, cv::Point(10, img.rows - 10), cv::FONT_HERSHEY_SIMPLEX, 1, cv::Scalar(255, 255, 255), 1); + } + } else { // Ludicrous time interval; reset + _framePeriod = 0.f; // WAKE UP + } + _lastTime = now; +} + +FXApp::Err FXApp::processKey(int key) { + static const int ESC_KEY = 27; + switch (key) { + case 'Q': case 'q': case ESC_KEY: + return errQuit; + case 'f': case 'F': + _showFPS = !_showFPS; + break; + case 'p': case 'P': case '%': + _progress = !_progress; + case 'e': case 'E': + _enableEffect = !_enableEffect; + break; + case 'd': case'D': + if (FLAG_webcam) + _drawVisualization = !_drawVisualization; + break; + default: + break; + } + return errNone; +} + +FXApp::Err FXApp::initCamera(cv::VideoCapture& cap) { + const int camIndex = 0; + cap.open(camIndex); + if (!FLAG_camRes.empty()) { + int camWidth, camHeight, n; + n = sscanf(FLAG_camRes.c_str(), "%d%*[xX]%d", &camWidth, &camHeight); + switch (n) { + case 2: + break; // We have read both width and height + case 1: + camHeight = camWidth; + camWidth = (int)(camHeight * (16. / 9.) + .5); + break; + default: + camHeight = 0; + camWidth = 0; + break; + } + + if (camWidth) cap.set(cv::CAP_PROP_FRAME_WIDTH, camWidth); + if (camHeight) cap.set(cv::CAP_PROP_FRAME_HEIGHT, camHeight); + int actualCamWidth = cap.get(cv::CAP_PROP_FRAME_WIDTH); + int actualCamHeight = cap.get(cv::CAP_PROP_FRAME_HEIGHT); + if (camWidth != actualCamWidth || camHeight != actualCamHeight) { + printf("The requested resolution of %d x %d is not available and has been subsituted by %d x %d.\n", camWidth, camHeight, actualCamWidth, actualCamHeight); + } + } + return errNone; +} + +void FXApp::drawEffectStatus(cv::Mat& img) { + char buf[32]; + snprintf(buf, sizeof(buf), "Effect: %s", _enableEffect ? "on" : "off"); + cv::putText(img, buf, cv::Point(10, img.rows - 40), cv::FONT_HERSHEY_SIMPLEX, 1, cv::Scalar(255, 255, 255), 1); +} + +FXApp::Err FXApp::createEffect(const char *effectSelector, const char *modelDir) { + NvCV_Status vfxErr; + BAIL_IF_ERR(vfxErr = NvVFX_CreateEffect(effectSelector, &_eff)); + _effectName = effectSelector; + if (modelDir[0] != '\0'){ + BAIL_IF_ERR(vfxErr = NvVFX_SetString(_eff, NVVFX_MODEL_DIRECTORY, modelDir)); + } +bail: + return appErrFromVfxStatus(vfxErr); +} + +void FXApp::destroyEffect() { + NvVFX_DestroyEffect(_eff); + _eff = nullptr; +} + +// Allocate one temp buffer to be used for input and output. Reshaping of the temp buffer in NvCVImage_Transfer() is done automatically, +// and is very low overhead. We expect the destination to be largest, so we allocate that first to minimize reallocs probablistically. +// Then we Realloc for the source to get the union of the two. +// This could alternately be done at runtime by feeding in an empty temp NvCVImage, but there are advantages to allocating all memory at load time. +NvCV_Status FXApp::allocTempBuffers() { + NvCV_Status vfxErr; + BAIL_IF_ERR(vfxErr = NvCVImage_Alloc( &_tmpVFX, _dstVFX.width, _dstVFX.height, _dstVFX.pixelFormat, _dstVFX.componentType, _dstVFX.planar, NVCV_GPU, 0)); + BAIL_IF_ERR(vfxErr = NvCVImage_Realloc(&_tmpVFX, _srcVFX.width, _srcVFX.height, _srcVFX.pixelFormat, _srcVFX.componentType, _srcVFX.planar, NVCV_GPU, 0)); +bail: + return vfxErr; +} + +NvCV_Status FXApp::allocBuffers(unsigned width, unsigned height) { + NvCV_Status vfxErr = NVCV_SUCCESS; + + if (_inited) + return NVCV_SUCCESS; + + if (!_srcImg.data) { + _srcImg.create(height, width, CV_8UC3); // src CPU + BAIL_IF_NULL(_srcImg.data, vfxErr, NVCV_ERR_MEMORY); + } + + _dstImg.create(_srcImg.rows, _srcImg.cols, _srcImg.type()); // + BAIL_IF_NULL(_dstImg.data, vfxErr, NVCV_ERR_MEMORY); // + BAIL_IF_ERR(vfxErr = NvCVImage_Alloc(&_srcGpuBuf, _srcImg.cols, _srcImg.rows, NVCV_BGR, NVCV_F32, NVCV_PLANAR, NVCV_GPU, 1)); // src GPU + BAIL_IF_ERR(vfxErr = NvCVImage_Alloc(&_dstGpuBuf, _srcImg.cols, _srcImg.rows, NVCV_BGR, NVCV_F32, NVCV_PLANAR, NVCV_GPU, 1)); //dst GPU + + NVWrapperForCVMat(&_srcImg, &_srcVFX); // _srcVFX is an alias for _srcImg + NVWrapperForCVMat(&_dstImg, &_dstVFX); // _dstVFX is an alias for _dstImg + + //#define ALLOC_TEMP_BUFFERS_AT_RUN_TIME // Deferring temp buffer allocation is easier + #ifndef ALLOC_TEMP_BUFFERS_AT_RUN_TIME // Allocating temp buffers at load time avoids run time hiccups + BAIL_IF_ERR(vfxErr = allocTempBuffers()); // This uses _srcVFX and _dstVFX and allocates one buffer to be a temporary for src and dst + #endif // ALLOC_TEMP_BUFFERS_AT_RUN_TIME + + _inited = true; + +bail: + return vfxErr; +} + +FXApp::Err FXApp::processImage(const char *inFile, const char *outFile) { + CUstream stream = 0; + NvCV_Status vfxErr; + + void* state = nullptr; + void* stateArray[1]; + + if (!_eff) + return errEffect; + _srcImg = cv::imread(inFile); + if (!_srcImg.data) + return errRead; + + BAIL_IF_ERR(vfxErr = allocBuffers(_srcImg.cols, _srcImg.rows)); + BAIL_IF_ERR(vfxErr = NvCVImage_Transfer(&_srcVFX, &_srcGpuBuf, 1.f / 255.f, stream, &_tmpVFX)); // _srcVFX--> _tmpVFX --> _srcGpuBuf + BAIL_IF_ERR(vfxErr = NvVFX_SetImage(_eff, NVVFX_INPUT_IMAGE, &_srcGpuBuf)); + BAIL_IF_ERR(vfxErr = NvVFX_SetImage(_eff, NVVFX_OUTPUT_IMAGE, &_dstGpuBuf)); + BAIL_IF_ERR(vfxErr = NvVFX_SetF32(_eff, NVVFX_STRENGTH, FLAG_strength)); + + unsigned int stateSizeInBytes; + BAIL_IF_ERR(vfxErr = NvVFX_GetU32(_eff, NVVFX_STATE_SIZE, &stateSizeInBytes)); + cudaMalloc(&state, stateSizeInBytes); + cudaMemsetAsync(state, 0, stateSizeInBytes, stream); + stateArray[0] = state; + BAIL_IF_ERR(vfxErr = NvVFX_SetObject(_eff, NVVFX_STATE, (void*)stateArray)); + + BAIL_IF_ERR(vfxErr = NvVFX_Load(_eff)); + BAIL_IF_ERR(vfxErr = NvVFX_Run(_eff, 0)); + BAIL_IF_ERR(vfxErr = NvCVImage_Transfer(&_dstGpuBuf, &_dstVFX, 255.f, stream, &_tmpVFX)); + + if (outFile && outFile[0]) { + if (!cv::imwrite(outFile, _dstImg)) { + printf("Error writing: \"%s\"\n", outFile); + return errWrite; + } + } + if (_show) { + cv::imshow("Output", _dstImg); + cv::waitKey(3000); + } +bail: + if (state) cudaFree(state); // release state memory + return appErrFromVfxStatus(vfxErr); +} + +FXApp::Err FXApp::processMovie(const char *inFile, const char *outFile) { + const int fourcc_h264 = cv::VideoWriter::fourcc('H','2','6','4'); + CUstream stream = 0; + FXApp::Err appErr = errNone; + bool ok; + cv::VideoCapture reader; + cv::VideoWriter writer; + NvCV_Status vfxErr; + unsigned frameNum; + VideoInfo info; + + void* state = nullptr; + void* stateArray[1]; + + if (inFile && !inFile[0]) inFile = nullptr; // Set file paths to NULL if zero length + + if (!FLAG_webcam && inFile) { + reader.open(inFile); + } else { + appErr = initCamera(reader); + if (appErr != errNone) + return appErr; + } + + if (!reader.isOpened()) { + if (!FLAG_webcam) printf("Error: Could not open video: \"%s\"\n", inFile); + else printf("Error: Webcam not found\n"); + return errRead; + } + + GetVideoInfo(reader, (inFile ? inFile : "webcam"), &info); + if (!(fourcc_h264 == info.codec || cv::VideoWriter::fourcc('a', 'v', 'c', '1') == info.codec)) // avc1 is alias for h264 + printf("Filters only target H264 videos, not %.4s\n", (char*)&info.codec); + + BAIL_IF_ERR(vfxErr = allocBuffers(info.width, info.height)); + + if (outFile && !outFile[0]) outFile = nullptr; + if (outFile) { + ok = writer.open(outFile, StringToFourcc(FLAG_codec), info.frameRate, cv::Size(_dstVFX.width, _dstVFX.height)); + if (!ok) { + printf("Cannot open \"%s\" for video writing\n", outFile); + outFile = nullptr; + if (!_show) + return errWrite; + } + } + + BAIL_IF_ERR(vfxErr = NvVFX_SetImage(_eff, NVVFX_INPUT_IMAGE, &_srcGpuBuf)); + BAIL_IF_ERR(vfxErr = NvVFX_SetImage(_eff, NVVFX_OUTPUT_IMAGE, &_dstGpuBuf)); + BAIL_IF_ERR(vfxErr = NvVFX_SetF32(_eff, NVVFX_STRENGTH, FLAG_strength)); + + unsigned int stateSizeInBytes; + BAIL_IF_ERR(vfxErr = NvVFX_GetU32(_eff, NVVFX_STATE_SIZE, &stateSizeInBytes)); + cudaMalloc(&state, stateSizeInBytes); + cudaMemsetAsync(state, 0, stateSizeInBytes, stream); + stateArray[0] = state; + BAIL_IF_ERR(vfxErr = NvVFX_SetObject(_eff, NVVFX_STATE, (void*)stateArray)); + + BAIL_IF_ERR(vfxErr = NvVFX_Load(_eff)); + + for (frameNum = 0; reader.read(_srcImg); frameNum++) { + if (_enableEffect) { + BAIL_IF_ERR(vfxErr = NvCVImage_Transfer(&_srcVFX, &_srcGpuBuf, 1.f / 255.f, stream, &_tmpVFX)); + BAIL_IF_ERR(vfxErr = NvVFX_Run(_eff, 0)); + BAIL_IF_ERR(vfxErr = NvCVImage_Transfer(&_dstGpuBuf, &_dstVFX, 255.f, stream, &_tmpVFX)); + } else { + BAIL_IF_ERR(vfxErr = NvCVImage_Transfer(&_srcVFX, &_dstVFX, 1.f, stream, &_tmpVFX)); + cudaMemsetAsync(state, 0, stateSizeInBytes, stream);// reset state by setting to 0 + } + + if (outFile) + writer.write(_dstImg); + + if (_show) { + if (_drawVisualization) drawEffectStatus(_dstImg); + drawFrameRate(_dstImg); + cv::imshow("Output", _dstImg); + int key= cv::waitKey(1); + if (key > 0) { + appErr = processKey(key); + if (errQuit == appErr) + break; + } + } + if (_progress) + fprintf(stderr, "\b\b\b\b%3.0f%%", 100.f * frameNum / info.frameCount); + } + + if (_progress) fprintf(stderr, "\n"); + reader.release(); + if (outFile) + writer.release(); +bail: + if (state) cudaFree(state); // release state memory + return appErrFromVfxStatus(vfxErr); +} + +int main(int argc, char **argv) { + FXApp::Err fxErr = FXApp::errNone; + int nErrs; + FXApp app; + + nErrs = ParseMyArgs(argc, argv); + if (nErrs) + std::cerr << nErrs << " command line syntax problems\n"; + + if (FLAG_webcam) { + // If webcam is on, enable showing the results and turn off displaying the progress + if (FLAG_progress) FLAG_progress = !FLAG_progress; + if (!FLAG_show) FLAG_show = !FLAG_show; + } + if (FLAG_inFile.empty() && !FLAG_webcam) { + std::cerr << "Please specify --in_file=XXX or --webcam=true\n"; + ++nErrs; + } + if (FLAG_outFile.empty() && !FLAG_show) { + std::cerr << "Please specify --out_file=XXX or --show\n"; + ++nErrs; + } + app._progress = FLAG_progress; + app.setShow(FLAG_show); + + if (nErrs) { + Usage(); + fxErr = FXApp::errFlag; + } + else { + fxErr = app.createEffect(NVVFX_FX_DENOISING, FLAG_modelDir.c_str()); + if (FXApp::errNone != fxErr) { + std::cerr << "Error creating effect\n"; + } + else { + if (IsImageFile(FLAG_inFile.c_str())) + fxErr = app.processImage(FLAG_inFile.c_str(), FLAG_outFile.c_str()); + else + fxErr = app.processMovie(FLAG_inFile.c_str(), FLAG_outFile.c_str()); + } + } + + if (fxErr) + std::cerr << "Error: " << app.errorStringFromCode(fxErr) << std::endl; + return (int)fxErr; +} diff --git a/samples/DenoiseEffectApp/DenoiseEffectApp.exe b/samples/DenoiseEffectApp/DenoiseEffectApp.exe new file mode 100644 index 0000000..4f78833 Binary files /dev/null and b/samples/DenoiseEffectApp/DenoiseEffectApp.exe differ diff --git a/samples/DenoiseEffectApp/run.bat b/samples/DenoiseEffectApp/run.bat new file mode 100644 index 0000000..34c0cd2 --- /dev/null +++ b/samples/DenoiseEffectApp/run.bat @@ -0,0 +1,4 @@ +SETLOCAL +SET PATH=%PATH%;..\external\opencv\bin; +DenoiseEffectApp.exe --webcam --strength=0 --show +DenoiseEffectApp.exe --webcam --strength=1 --show \ No newline at end of file diff --git a/samples/UpscalePipelineApp/CMakeLists.txt b/samples/UpscalePipelineApp/CMakeLists.txt index b9068a6..8ee5f68 100644 --- a/samples/UpscalePipelineApp/CMakeLists.txt +++ b/samples/UpscalePipelineApp/CMakeLists.txt @@ -1,4 +1,4 @@ -set(SOURCE_FILES UpscalePipeline.cpp ../../nvvfx/src/nvVideoEffectsProxy.cpp) +set(SOURCE_FILES UpscalePipeline.cpp ../../nvvfx/src/nvVideoEffectsProxy.cpp ../../nvvfx/src/nvCVImageProxy.cpp) # Set Visual Studio source filters source_group("Source Files" FILES ${SOURCE_FILES}) @@ -32,6 +32,7 @@ else() target_link_libraries(UpscalePipelineApp PUBLIC NVVideoEffects + NVCVImage OpenCV TensorRT CUDA diff --git a/samples/UpscalePipelineApp/UpscalePipeline.cpp b/samples/UpscalePipelineApp/UpscalePipeline.cpp index 7715914..2ef943d 100644 --- a/samples/UpscalePipelineApp/UpscalePipeline.cpp +++ b/samples/UpscalePipelineApp/UpscalePipeline.cpp @@ -26,6 +26,7 @@ #include #include +#include #include "nvCVOpenCV.h" #include "nvVideoEffects.h" @@ -51,6 +52,13 @@ #define BAIL_IF_ERR(err) do { if (0 != (err)) { goto bail; } } while(0) #define BAIL_IF_NULL(x, err, code) do { if ((void*)(x) == NULL) { err = code; goto bail; } } while(0) +#define NVCV_ERR_HELP 411 + +#ifdef _WIN32 +#define DEFAULT_CODEC "avc1" +#else // !_WIN32 +#define DEFAULT_CODEC "H264" +#endif // _WIN32 bool FLAG_debug = false, @@ -60,7 +68,7 @@ bool FLAG_debug = false, int FLAG_resolution = 0, FLAG_arStrength = 0; float FLAG_upscaleStrength = 0.2f; -std::string FLAG_codec = "H264", +std::string FLAG_codec = DEFAULT_CODEC, FLAG_inFile, FLAG_outFile, FLAG_outDir, @@ -143,7 +151,7 @@ static void Usage() { " --resolution= the desired height of the output\n" " --out_height= the desired height of the output\n" " --model_dir= the path to the directory that contains the models\n" - " --codec= the fourcc code for the desired codec (default \"H264\")\n" + " --codec= the fourcc code for the desired codec (default " DEFAULT_CODEC ")\n" " --progress show progress\n" " --verbose verbose output\n" " --debug print extra debugging information\n" @@ -174,7 +182,7 @@ static int ParseMyArgs(int argc, char **argv) { )) { continue; } else if (GetFlagArgVal("help", arg, &help)) { - Usage(); + return NVCV_ERR_HELP; } else if (arg[1] != '-') { for (++arg; *arg; ++arg) { if (*arg == 'v') { @@ -267,6 +275,10 @@ static int StringToFourcc(const std::string& str) { struct FXApp { enum Err { + errQuit = +1, // Application errors + errFlag = +2, + errRead = +3, + errWrite = +4, errNone = NVCV_SUCCESS, // Video Effects SDK errors errGeneral = NVCV_ERR_GENERAL, errUnimplemented = NVCV_ERR_UNIMPLEMENTED, @@ -286,6 +298,7 @@ struct FXApp { errResolution = NVCV_ERR_RESOLUTION, errUnsupportedGPU = NVCV_ERR_UNSUPPORTEDGPU, errWrongGPU = NVCV_ERR_WRONGGPU, + errUnsupportedDriver = NVCV_ERR_UNSUPPORTEDDRIVER, errCudaMemory = NVCV_ERR_CUDA_MEMORY, // CUDA errors errCudaValue = NVCV_ERR_CUDA_VALUE, errCudaPitch = NVCV_ERR_CUDA_PITCH, @@ -296,10 +309,6 @@ struct FXApp { errCudaUnsupported = NVCV_ERR_CUDA_UNSUPPORTED, errCudaIllegalAddress = NVCV_ERR_CUDA_ILLEGAL_ADDRESS, errCuda = NVCV_ERR_CUDA, - errQuit = -50, // Application errors - errFlag = -51, - errRead = -52, - errWrite = -53, }; FXApp() { _arEff = nullptr; _upscaleEff = nullptr; _inited = false; _showFPS = false; _progress = false; @@ -346,11 +355,9 @@ const char* FXApp::errorStringFromCode(Err code) { { errQuit, "The user chose to quit the application" }, { errFlag, "There was a problem with the command-line arguments" }, }; - if ((int)code >= (int)errCuda) - return NvCV_GetErrorStringFromCode((NvCV_Status)code); + if ((int)code <= 0) return NvCV_GetErrorStringFromCode((NvCV_Status)code); for (const LutEntry *p = lut; p != &lut[sizeof(lut) / sizeof(lut[0])]; ++p) - if (p->code == code) - return p->str; + if (p->code == code) return p->str; return "UNKNOWN ERROR"; } @@ -447,10 +454,10 @@ NvCV_Status FXApp::allocBuffers(unsigned width, unsigned height) { BAIL_IF_ERR(vfxErr = NvCVImage_Alloc(&_interGpuBGRf32pl, _srcImg.cols, _srcImg.rows, NVCV_BGR, NVCV_F32, NVCV_PLANAR, NVCV_GPU, 1)); // intermediate GPU BAIL_IF_ERR(vfxErr = NvVFX_SetF32(_upscaleEff, NVVFX_STRENGTH, FLAG_upscaleStrength)); BAIL_IF_ERR(vfxErr = NvCVImage_Alloc(&_interGpuRGBAu8, _srcImg.cols, _srcImg.rows, NVCV_RGBA, NVCV_U8, - NVCV_INTERLEAVED, NVCV_GPU, 1)); // intermediate GPU + NVCV_INTERLEAVED, NVCV_GPU, 32)); // intermediate GPU BAIL_IF_ERR(vfxErr = NvCVImage_Alloc(&_dstGpuBuf, _dstImg.cols, _dstImg.rows, NVCV_RGBA, NVCV_U8, NVCV_INTERLEAVED, - NVCV_GPU, 1)); // dst GPU + NVCV_GPU, 32)); // dst GPU NVWrapperForCVMat(&_srcImg, &_srcVFX); // _srcVFX is an alias for _srcImg NVWrapperForCVMat(&_dstImg, &_dstVFX); // _dstVFX is an alias for _dstImg @@ -510,7 +517,7 @@ bail: } FXApp::Err FXApp::processMovie(const char *inFile, const char *outFile) { - const int fourcc_h264 = CV_FOURCC('H','2','6','4'); + const int fourcc_h264 = cv::VideoWriter::fourcc('H','2','6','4'); CUstream stream = 0; FXApp::Err appErr = errNone; bool ok; @@ -526,7 +533,7 @@ FXApp::Err FXApp::processMovie(const char *inFile, const char *outFile) { } GetVideoInfo(reader, inFile, &info); - if (!(fourcc_h264 == info.codec || CV_FOURCC('a','v','c','1') == info.codec)) // avc1 is alias for h264 + if (!(fourcc_h264 == info.codec || cv::VideoWriter::fourcc('a','v','c','1') == info.codec)) // avc1 is alias for h264 printf("Filters only target H264 videos, not %.4s\n", (char*)&info.codec); BAIL_IF_ERR(vfxErr = allocBuffers(info.width, info.height)); diff --git a/samples/UpscalePipelineApp/UpscalePipelineApp.exe b/samples/UpscalePipelineApp/UpscalePipelineApp.exe index 7fc0e56..6121f65 100644 Binary files a/samples/UpscalePipelineApp/UpscalePipelineApp.exe and b/samples/UpscalePipelineApp/UpscalePipelineApp.exe differ diff --git a/samples/VideoEffectsApp/CMakeLists.txt b/samples/VideoEffectsApp/CMakeLists.txt index 98eadcb..c81d875 100644 --- a/samples/VideoEffectsApp/CMakeLists.txt +++ b/samples/VideoEffectsApp/CMakeLists.txt @@ -1,4 +1,4 @@ -set(SOURCE_FILES VideoEffectsApp.cpp ../../nvvfx/src/nvVideoEffectsProxy.cpp) +set(SOURCE_FILES VideoEffectsApp.cpp ../../nvvfx/src/nvVideoEffectsProxy.cpp ../../nvvfx/src/nvCVImageProxy.cpp) # Set Visual Studio source filters source_group("Source Files" FILES ${SOURCE_FILES}) @@ -6,18 +6,30 @@ source_group("Source Files" FILES ${SOURCE_FILES}) add_executable(VideoEffectsApp ${SOURCE_FILES}) target_include_directories(VideoEffectsApp PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../utils) target_include_directories(VideoEffectsApp PUBLIC ${SDK_INCLUDES_PATH}) -target_link_libraries(VideoEffectsApp PUBLIC - opencv346 - NVVideoEffects - ${CMAKE_CURRENT_SOURCE_DIR}/../external/cuda/lib/x64/cudart.lib -) -set(OPENCV_PATH_STR ${CMAKE_CURRENT_SOURCE_DIR}/../external/opencv/bin) -set(VFXSDK_PATH_STR ${CMAKE_CURRENT_SOURCE_DIR}/../../bin) # Also the location for CUDA/NVTRT/libcrypto -set(PATH_STR "PATH=%PATH%" ${VFXSDK_PATH_STR} ${OPENCV_PATH_STR}) -set(CMD_ARG_STR "--model_dir=\"${CMAKE_CURRENT_SOURCE_DIR}/../../bin/models\" --show --effect=SuperRes --resolution=1080 --in_file=\"${CMAKE_CURRENT_SOURCE_DIR}/../input/input2.png\"") -set_target_properties(VideoEffectsApp PROPERTIES - FOLDER SampleApps - VS_DEBUGGER_ENVIRONMENT "${PATH_STR}" - VS_DEBUGGER_COMMAND_ARGUMENTS "${CMD_ARG_STR}" -) +if(MSVC) + target_link_libraries(VideoEffectsApp PUBLIC + opencv346 + NVVideoEffects + ${CMAKE_CURRENT_SOURCE_DIR}/../external/cuda/lib/x64/cudart.lib + ) + + set(OPENCV_PATH_STR ${CMAKE_CURRENT_SOURCE_DIR}/../external/opencv/bin) + set(VFXSDK_PATH_STR ${CMAKE_CURRENT_SOURCE_DIR}/../../bin) # Also the location for CUDA/NVTRT/libcrypto + set(PATH_STR "PATH=%PATH%" ${VFXSDK_PATH_STR} ${OPENCV_PATH_STR}) + set(CMD_ARG_STR "--model_dir=\"${CMAKE_CURRENT_SOURCE_DIR}/../../bin/models\" --show --effect=SuperRes --resolution=1080 --in_file=\"${CMAKE_CURRENT_SOURCE_DIR}/../input/input2.png\"") + set_target_properties(VideoEffectsApp PROPERTIES + FOLDER SampleApps + VS_DEBUGGER_ENVIRONMENT "${PATH_STR}" + VS_DEBUGGER_COMMAND_ARGUMENTS "${CMD_ARG_STR}" + ) +else() + + target_link_libraries(VideoEffectsApp PUBLIC + NVVideoEffects + NVCVImage + OpenCV + TensorRT + CUDA + ) +endif() diff --git a/samples/VideoEffectsApp/VideoEffectsApp.cpp b/samples/VideoEffectsApp/VideoEffectsApp.cpp index a8cde32..00310dd 100644 --- a/samples/VideoEffectsApp/VideoEffectsApp.cpp +++ b/samples/VideoEffectsApp/VideoEffectsApp.cpp @@ -26,6 +26,7 @@ #include #include +#include #include "nvCVOpenCV.h" #include "nvVideoEffects.h" @@ -41,15 +42,24 @@ #define BAIL_IF_ERR(err) do { if (0 != (err)) { goto bail; } } while(0) #define BAIL_IF_NULL(x, err, code) do { if ((void*)(x) == NULL) { err = code; goto bail; } } while(0) +#define NVCV_ERR_HELP 411 + +#ifdef _WIN32 + #define DEFAULT_CODEC "avc1" +#else // !_WIN32 + #define DEFAULT_CODEC "H264" +#endif // _WIN32 bool FLAG_debug = false, FLAG_verbose = false, FLAG_show = false, - FLAG_progress = false; + FLAG_progress = false, + FLAG_webcam = false; float FLAG_strength = 0.f; int FLAG_resolution = 0; -std::string FLAG_codec = "H264", +std::string FLAG_codec = DEFAULT_CODEC, + FLAG_camRes = "1280x720", FLAG_inFile, FLAG_outFile, FLAG_outDir, @@ -126,18 +136,26 @@ static void Usage() { "VideoEffectsApp [args ...]\n" " where args is:\n" " --in_file= input file to be processed\n" + " --webcam use a webcam as the input\n" " --out_file= output file to be written\n" " --effect= the effect to apply\n" - " --show display the results in a window\n" + " --show display the results in a window (for webcam, it is always true)\n" " --strength= strength of an effect, 0 or 1 for super res and artifact reduction,\n" " and [0.0, 1.0] for upscaling\n" + " --cam_res=[WWWx]HHH specify camera resolution as height or width x height\n" + " supports 720 and 1080 resolutions (default \"720\") \n" " --resolution= the desired height of the output\n" " --model_dir= the path to the directory that contains the models\n" - " --codec= the fourcc code for the desired codec (default \"H264\")\n" + " --codec= the fourcc code for the desired codec (default " DEFAULT_CODEC ")\n" " --progress show progress\n" " --verbose verbose output\n" " --debug print extra debugging information\n" ); + const char* cStr; + NvCV_Status err = NvVFX_GetString(nullptr, NVVFX_INFO, &cStr); + if (NVCV_SUCCESS != err) + printf("Cannot get effects: %s\n", NvCV_GetErrorStringFromCode(err)); + printf("where effects are:\n%s", cStr); } static int ParseMyArgs(int argc, char **argv) { @@ -155,6 +173,8 @@ static int ParseMyArgs(int argc, char **argv) { GetFlagArgVal("out_file", arg, &FLAG_outFile) || GetFlagArgVal("effect", arg, &FLAG_effect) || GetFlagArgVal("show", arg, &FLAG_show) || + GetFlagArgVal("webcam", arg, &FLAG_webcam) || + GetFlagArgVal("cam_res", arg, &FLAG_camRes) || GetFlagArgVal("strength", arg, &FLAG_strength) || GetFlagArgVal("resolution", arg, &FLAG_resolution) || GetFlagArgVal("model_dir", arg, &FLAG_modelDir) || @@ -164,7 +184,7 @@ static int ParseMyArgs(int argc, char **argv) { )) { continue; } else if (GetFlagArgVal("help", arg, &help)) { - Usage(); + return NVCV_ERR_HELP; } else if (arg[1] != '-') { for (++arg; *arg; ++arg) { if (*arg == 'v') { @@ -257,7 +277,11 @@ static int StringToFourcc(const std::string& str) { struct FXApp { enum Err { - errNone = NVCV_SUCCESS, // Video Effects SDK errors + errQuit = +1, // Application errors + errFlag = +2, + errRead = +3, + errWrite = +4, + errNone = NVCV_SUCCESS, // Video Effects SDK errors errGeneral = NVCV_ERR_GENERAL, errUnimplemented = NVCV_ERR_UNIMPLEMENTED, errMemory = NVCV_ERR_MEMORY, @@ -276,7 +300,8 @@ struct FXApp { errResolution = NVCV_ERR_RESOLUTION, errUnsupportedGPU = NVCV_ERR_UNSUPPORTEDGPU, errWrongGPU = NVCV_ERR_WRONGGPU, - errCudaMemory = NVCV_ERR_CUDA_MEMORY, // CUDA errors + errUnsupportedDriver = NVCV_ERR_UNSUPPORTEDDRIVER, + errCudaMemory = NVCV_ERR_CUDA_MEMORY, // CUDA errors errCudaValue = NVCV_ERR_CUDA_VALUE, errCudaPitch = NVCV_ERR_CUDA_PITCH, errCudaInit = NVCV_ERR_CUDA_INIT, @@ -286,14 +311,10 @@ struct FXApp { errCudaUnsupported = NVCV_ERR_CUDA_UNSUPPORTED, errCudaIllegalAddress = NVCV_ERR_CUDA_ILLEGAL_ADDRESS, errCuda = NVCV_ERR_CUDA, - errQuit = -50, // Application errors - errFlag = -51, - errRead = -52, - errWrite = -53, }; FXApp() { _eff = nullptr; _effectName = nullptr; _inited = false; _showFPS = false; _progress = false; - _show = false; _framePeriod = 0.f; } + _show = false; _enableEffect = true, _drawVisualization = true, _framePeriod = 0.f; } ~FXApp() { NvVFX_DestroyEffect(_eff); } void setShow(bool show) { _show = show; } @@ -303,8 +324,10 @@ struct FXApp { NvCV_Status allocTempBuffers(); Err processImage(const char *inFile, const char *outFile); Err processMovie(const char *inFile, const char *outFile); + Err initCamera(cv::VideoCapture& cap); Err processKey(int key); void drawFrameRate(cv::Mat& img); + void drawEffectStatus(cv::Mat& img); Err appErrFromVfxStatus(NvCV_Status status) { return (Err)status; } const char* errorStringFromCode(Err code); @@ -320,6 +343,8 @@ struct FXApp { bool _inited; bool _showFPS; bool _progress; + bool _enableEffect; + bool _drawVisualization; const char* _effectName; float _framePeriod; std::chrono::high_resolution_clock::time_point _lastTime; @@ -333,11 +358,9 @@ const char* FXApp::errorStringFromCode(Err code) { { errQuit, "The user chose to quit the application" }, { errFlag, "There was a problem with the command-line arguments" }, }; - if ((int)code >= (int)errCuda) - return NvCV_GetErrorStringFromCode((NvCV_Status)code); + if ((int)code <= 0) return NvCV_GetErrorStringFromCode((NvCV_Status)code); for (const LutEntry *p = lut; p != &lut[sizeof(lut) / sizeof(lut[0])]; ++p) - if (p->code == code) - return p->str; + if (p->code == code) return p->str; return "UNKNOWN ERROR"; } @@ -372,6 +395,11 @@ FXApp::Err FXApp::processKey(int key) { break; case 'p': case 'P': case '%': _progress = !_progress; + case 'e': case 'E': + break; + case 'd': case'D': + if (FLAG_webcam) + _drawVisualization = !_drawVisualization; break; default: break; @@ -379,6 +407,41 @@ FXApp::Err FXApp::processKey(int key) { return errNone; } +FXApp::Err FXApp::initCamera(cv::VideoCapture& cap) { + const int camIndex = 0; + cap.open(camIndex); + if (!FLAG_camRes.empty()) { + int camWidth, camHeight, n; + n = sscanf(FLAG_camRes.c_str(), "%d%*[xX]%d", &camWidth, &camHeight); + switch (n) { + case 2: + break; // We have read both width and height + case 1: + camHeight = camWidth; + camWidth = (int)(camHeight * (16. / 9.) + .5); + break; + default: + camHeight = 0; + camWidth = 0; + break; + } + + if (camWidth) cap.set(cv::CAP_PROP_FRAME_WIDTH, camWidth); + if (camHeight) cap.set(cv::CAP_PROP_FRAME_HEIGHT, camHeight); + if (camWidth != cap.get(cv::CAP_PROP_FRAME_WIDTH) || camHeight != cap.get(cv::CAP_PROP_FRAME_HEIGHT)) { + printf("Error: Camera does not support %d x %d resolution\n", camWidth, camHeight); + return errGeneral; + } + } + return errNone; +} + +void FXApp::drawEffectStatus(cv::Mat& img) { + char buf[32]; + snprintf(buf, sizeof(buf), "Effect: %s", _enableEffect ? "on" : "off"); + cv::putText(img, buf, cv::Point(10, img.rows - 40), cv::FONT_HERSHEY_SIMPLEX, 1, cv::Scalar(255, 255, 255), 1); +} + FXApp::Err FXApp::createEffect(const char *effectSelector, const char *modelDir) { NvCV_Status vfxErr; BAIL_IF_ERR(vfxErr = NvVFX_CreateEffect(effectSelector, &_eff)); @@ -407,6 +470,15 @@ bail: return vfxErr; } +static NvCV_Status CheckScaleIsotropy(const NvCVImage *src, const NvCVImage *dst) { + if (src->width * dst->height != src->height * dst->width) { + printf("%ux%u --> %ux%u: different scale for width and height is not supported\n", + src->width, src->height, dst->width, dst->height); + return NVCV_ERR_RESOLUTION; + } + return NVCV_SUCCESS; +} + NvCV_Status FXApp::allocBuffers(unsigned width, unsigned height) { NvCV_Status vfxErr = NVCV_SUCCESS; @@ -439,6 +511,7 @@ NvCV_Status FXApp::allocBuffers(unsigned width, unsigned height) { BAIL_IF_NULL(_dstImg.data, vfxErr, NVCV_ERR_MEMORY); BAIL_IF_ERR(vfxErr = NvCVImage_Alloc(&_srcGpuBuf, _srcImg.cols, _srcImg.rows, NVCV_BGR, NVCV_F32, NVCV_PLANAR, NVCV_GPU, 1)); // src GPU BAIL_IF_ERR(vfxErr = NvCVImage_Alloc(&_dstGpuBuf, _dstImg.cols, _dstImg.rows, NVCV_BGR, NVCV_F32, NVCV_PLANAR, NVCV_GPU, 1)); // dst GPU + BAIL_IF_ERR(vfxErr = CheckScaleIsotropy(&_srcGpuBuf, &_dstGpuBuf)); } else if (!strcmp(_effectName, NVVFX_FX_SR_UPSCALE)) { if (!FLAG_resolution) { @@ -451,9 +524,10 @@ NvCV_Status FXApp::allocBuffers(unsigned width, unsigned height) { _dstImg.create(FLAG_resolution, dstWidth, _srcImg.type()); // dst CPU BAIL_IF_NULL(_dstImg.data, vfxErr, NVCV_ERR_MEMORY); BAIL_IF_ERR(vfxErr = NvCVImage_Alloc(&_srcGpuBuf, _srcImg.cols, _srcImg.rows, NVCV_RGBA, NVCV_U8, NVCV_INTERLEAVED, - NVCV_GPU, 1)); // src GPU + NVCV_GPU, 32)); // src GPU BAIL_IF_ERR(vfxErr = NvCVImage_Alloc(&_dstGpuBuf, _dstImg.cols, _dstImg.rows, NVCV_RGBA, NVCV_U8, NVCV_INTERLEAVED, - NVCV_GPU, 1)); // dst GPU + NVCV_GPU, 32)); // dst GPU + BAIL_IF_ERR(vfxErr = CheckScaleIsotropy(&_srcGpuBuf, &_dstGpuBuf)); } NVWrapperForCVMat(&_srcImg, &_srcVFX); // _srcVFX is an alias for _srcImg NVWrapperForCVMat(&_dstImg, &_dstVFX); // _dstVFX is an alias for _dstImg @@ -511,23 +585,34 @@ bail: } FXApp::Err FXApp::processMovie(const char *inFile, const char *outFile) { - const int fourcc_h264 = CV_FOURCC('H','2','6','4'); + const int fourcc_h264 = cv::VideoWriter::fourcc('H','2','6','4'); CUstream stream = 0; FXApp::Err appErr = errNone; bool ok; + cv::VideoCapture reader; cv::VideoWriter writer; NvCV_Status vfxErr; unsigned frameNum; VideoInfo info; - cv::VideoCapture reader(inFile); + if (inFile && !inFile[0]) inFile = nullptr; // Set file paths to NULL if zero length + + if (!FLAG_webcam && inFile) { + reader.open(inFile); + } else { + appErr = initCamera(reader); + if (appErr != errNone) + return appErr; + } + if (!reader.isOpened()) { - printf("Error: Could not open video: \"%s\"\n", inFile); + if (!FLAG_webcam) printf("Error: Could not open video: \"%s\"\n", inFile); + else printf("Error: Webcam not found\n"); return errRead; } - GetVideoInfo(reader, inFile, &info); - if (!(fourcc_h264 == info.codec || CV_FOURCC('a', 'v', 'c', '1') == info.codec)) // avc1 is alias for h264 + GetVideoInfo(reader, (inFile ? inFile : "webcam"), &info); + if (!(fourcc_h264 == info.codec || cv::VideoWriter::fourcc('a', 'v', 'c', '1') == info.codec)) // avc1 is alias for h264 printf("Filters only target H264 videos, not %.4s\n", (char*)&info.codec); BAIL_IF_ERR(vfxErr = allocBuffers(info.width, info.height)); @@ -559,9 +644,13 @@ FXApp::Err FXApp::processMovie(const char *inFile, const char *outFile) { } // _srcVFX --> _srcTmpVFX --> _srcGpuBuf --> _dstGpuBuf --> _dstTmpVFX --> _dstVFX - BAIL_IF_ERR(vfxErr = NvCVImage_Transfer(&_srcVFX, &_srcGpuBuf, 1.f/255.f, stream, &_tmpVFX)); - BAIL_IF_ERR(vfxErr = NvVFX_Run(_eff, 0)); - BAIL_IF_ERR(vfxErr = NvCVImage_Transfer(&_dstGpuBuf, &_dstVFX, 255.f, stream, &_tmpVFX)); + if (_enableEffect) { + BAIL_IF_ERR(vfxErr = NvCVImage_Transfer(&_srcVFX, &_srcGpuBuf, 1.f / 255.f, stream, &_tmpVFX)); + BAIL_IF_ERR(vfxErr = NvVFX_Run(_eff, 0)); + BAIL_IF_ERR(vfxErr = NvCVImage_Transfer(&_dstGpuBuf, &_dstVFX, 255.f, stream, &_tmpVFX)); + } else { + BAIL_IF_ERR(vfxErr = NvCVImage_Transfer(&_srcVFX, &_dstVFX, 1.f / 255.f, stream, &_tmpVFX)); + } if (outFile) writer.write(_dstImg); @@ -602,9 +691,13 @@ int main(int argc, char **argv) { NvVFX_GetString(nullptr, NVVFX_INFO, &cstr); std::cerr << "Effects:" << std::endl << cstr << std::endl; } - - if (FLAG_inFile.empty()) { - std::cerr << "Please specify --in_file=XXX\n"; + if (FLAG_webcam) { + // If webcam is on, enable showing the results and turn off displaying the progress + if (FLAG_progress) FLAG_progress = !FLAG_progress; + if (!FLAG_show) FLAG_show = !FLAG_show; + } + if (FLAG_inFile.empty() && !FLAG_webcam) { + std::cerr << "Please specify --in_file=XXX or --webcam=true\n"; ++nErrs; } if (FLAG_outFile.empty() && !FLAG_show) { diff --git a/samples/VideoEffectsApp/VideoEffectsApp.exe b/samples/VideoEffectsApp/VideoEffectsApp.exe index a4fd058..3464870 100644 Binary files a/samples/VideoEffectsApp/VideoEffectsApp.exe and b/samples/VideoEffectsApp/VideoEffectsApp.exe differ diff --git a/samples/external/CMakeLists.txt b/samples/external/CMakeLists.txt index 2678370..47d4497 100644 --- a/samples/external/CMakeLists.txt +++ b/samples/external/CMakeLists.txt @@ -3,28 +3,58 @@ ####################### if(MSVC) - if(CMAKE_CL_64) - set(OpenCV_ARCH x64) - elseif((CMAKE_GENERATOR MATCHES "ARM") OR ("${arch_hint}" STREQUAL "ARM") OR (CMAKE_VS_EFFECTIVE_PLATFORMS MATCHES "ARM|arm")) - # see Modules/CmakeGenericSystem.cmake - set(OpenCV_ARCH ARM) - else() - set(OpenCV_ARCH x86) - endif() - if(MSVC_VERSION GREATER_EQUAL 1920) - #set(OpenCV_RUNTIME vc16) - message("No Visual Studio 2019 OpenCV library available; trying 2017 library instead") - set(OpenCV_RUNTIME vc15) - elseif(MSVC_VERSION GREATER_EQUAL 1910) - set(OpenCV_RUNTIME vc15) - elseif(MSVC_VERSION GREATER_EQUAL 1900) - set(OpenCV_RUNTIME vc14) - else() - message("MSVC_VERSION ${MSVC_VERSION} is not accommodated") - endif() -endif() + if(CMAKE_CL_64) + set(OpenCV_ARCH x64) + elseif((CMAKE_GENERATOR MATCHES "ARM") OR ("${arch_hint}" STREQUAL "ARM") OR (CMAKE_VS_EFFECTIVE_PLATFORMS MATCHES "ARM|arm")) + # see Modules/CmakeGenericSystem.cmake + set(OpenCV_ARCH ARM) + else() + set(OpenCV_ARCH x86) + endif() + if(MSVC_VERSION GREATER_EQUAL 1920) + #set(OpenCV_RUNTIME vc16) + message("No Visual Studio 2019 OpenCV library available; trying 2017 library instead") + set(OpenCV_RUNTIME vc15) + elseif(MSVC_VERSION GREATER_EQUAL 1910) + set(OpenCV_RUNTIME vc15) + elseif(MSVC_VERSION GREATER_EQUAL 1900) + set(OpenCV_RUNTIME vc14) + else() + message("MSVC_VERSION ${MSVC_VERSION} is not accommodated") + endif() -add_library(opencv346 INTERFACE) -set(OpenCV_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/opencv/include ${CMAKE_CURRENT_SOURCE_DIR}/opencv/include/opencv2) -target_include_directories(opencv346 INTERFACE ${OpenCV_INCLUDE_DIR}) -target_link_libraries(opencv346 INTERFACE optimized ${CMAKE_CURRENT_SOURCE_DIR}/opencv/lib/opencv_world346.lib) \ No newline at end of file + add_library(opencv346 INTERFACE) + set(OpenCV_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/opencv/include ${CMAKE_CURRENT_SOURCE_DIR}/opencv/include/opencv2) + target_include_directories(opencv346 INTERFACE ${OpenCV_INCLUDE_DIR}) + target_link_libraries(opencv346 INTERFACE optimized ${CMAKE_CURRENT_SOURCE_DIR}/opencv/lib/opencv_world346.lib) +else() + + find_package(OpenCV REQUIRED + PATHS /usr /usr/local + PATH_SUFFIXES share/OpenCV share/opencv4) + add_library(OpenCV INTERFACE) + target_include_directories(OpenCV INTERFACE ${OpenCV_INCLUDE_DIRS}) + target_link_libraries(OpenCV INTERFACE ${OpenCV_LIBRARIES}) + + message("OpenCV_INCLUDE_DIRS ${OpenCV_INCLUDE_DIRS}") + message("OpenCV_LIBRARIES ${OpenCV_LIBRARIES}") + message("OpenCV_LIBS ${OpenCV_LIBS}") + + find_package(CUDA 11.1 REQUIRED) + add_library(CUDA INTERFACE) + target_include_directories(CUDA INTERFACE ${CUDA_INCLUDE_DIRS}) + target_link_libraries(CUDA INTERFACE "${CUDA_LIBRARIES};cuda") + + message("CUDA_INCLUDE_DIRS ${CUDA_INCLUDE_DIRS}") + message("CUDA_LIBRARIES ${CUDA_LIBRARIES}") + + find_package(TensorRT 7 REQUIRED) + add_library(TensorRT INTERFACE) + target_include_directories(TensorRT INTERFACE ${TensorRT_INCLUDE_DIRS}) + target_link_libraries(TensorRT INTERFACE ${TensorRT_LIBRARIES}) + + message("TensorRT_INCLUDE_DIRS ${TensorRT_INCLUDE_DIRS}") + message("TensorRT_LIBRARIES ${TensorRT_LIBRARIES}") + + +endif() diff --git a/samples/input/LeFret_000900.jpg b/samples/input/LeFret_000900.jpg new file mode 100644 index 0000000..e429653 Binary files /dev/null and b/samples/input/LeFret_000900.jpg differ diff --git a/samples/input/LeFret_001400.jpg b/samples/input/LeFret_001400.jpg new file mode 100644 index 0000000..5d2acf3 Binary files /dev/null and b/samples/input/LeFret_001400.jpg differ diff --git a/samples/input/LeFret_003400.jpg b/samples/input/LeFret_003400.jpg new file mode 100644 index 0000000..5c22a7e Binary files /dev/null and b/samples/input/LeFret_003400.jpg differ diff --git a/samples/input/LeFret_012300.jpg b/samples/input/LeFret_012300.jpg new file mode 100644 index 0000000..ce0e9aa Binary files /dev/null and b/samples/input/LeFret_012300.jpg differ diff --git a/samples/utils/nvCVOpenCV.h b/samples/utils/nvCVOpenCV.h index cdfbe1a..ddb9274 100644 --- a/samples/utils/nvCVOpenCV.h +++ b/samples/utils/nvCVOpenCV.h @@ -52,7 +52,7 @@ inline void CVImageSet(cv::Mat *cvIm, int width, int height, int numComps, int c // Wrap an NvCVImage in a cv::Mat inline void CVWrapperForNvCVImage(const NvCVImage *nvcvIm, cv::Mat *cvIm) { static const char cvType[] = { 7, 0, 2, 3, 7, 7, 4, 5, 7, 7, 6 }; - CVImageSet(cvIm, nvcvIm->width, nvcvIm->height, nvcvIm->numComponents, cvType[(int)nvcvIm->pixelFormat], nvcvIm->componentBytes, nvcvIm->pixels, nvcvIm->pitch); + CVImageSet(cvIm, nvcvIm->width, nvcvIm->height, nvcvIm->numComponents, cvType[(int)nvcvIm->componentType], nvcvIm->componentBytes, nvcvIm->pixels, nvcvIm->pitch); } // Wrap a cv::Mat in an NvCVImage. diff --git a/version.h b/version.h index 7bd4988..4308db3 100644 --- a/version.h +++ b/version.h @@ -1,10 +1,10 @@ #define NVIDIA_VIDEOEFFECTS_SDK_VERSION_MAJOR 0 -#define NVIDIA_VIDEOEFFECTS_SDK_VERSION_MINOR 5 -#define NVIDIA_VIDEOEFFECTS_SDK_VERSION_RELEASE 1 +#define NVIDIA_VIDEOEFFECTS_SDK_VERSION_MINOR 6 +#define NVIDIA_VIDEOEFFECTS_SDK_VERSION_RELEASE 4 -#define NVIDIA_VIDEOEFFECTS_SDK_VERSION 0,5,1,0 -#define NVIDIA_VIDEOEFFECTS_SDK_VERSION_MAJOR_MINOR 0,5 -#define NVIDIA_VIDEOEFFECTS_SDK_VERSION_STRING "0.5.1.0" -#define NVIDIA_VIDEOEFFECTS_SDK_VERSION_STRING_SHORT "0.5.1" -#define NVIDIA_VIDEOEFFECTS_SDK_VERSION_STRING_MAJOR_MINOR "0.5" +#define NVIDIA_VIDEOEFFECTS_SDK_VERSION 0,6,4,0 +#define NVIDIA_VIDEOEFFECTS_SDK_VERSION_MAJOR_MINOR 0,6 +#define NVIDIA_VIDEOEFFECTS_SDK_VERSION_STRING "0.6.4.0" +#define NVIDIA_VIDEOEFFECTS_SDK_VERSION_STRING_SHORT "0.6.4" +#define NVIDIA_VIDEOEFFECTS_SDK_VERSION_STRING_MAJOR_MINOR "0.6"