diff --git a/CHANGELOG b/CHANGELOG index e717cda..e6a9547 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,14 +1,3 @@ -Changelog (v0.7.1.0) +Changelog (v0.7.2.0) -------------------- - - NvCVImage_Transfer() now sets alpha to 255 or 1.0f when doing RGB -> RGBA - - NvCVImage_CompositeRect() has a premultiplied alpha mode added - - AI Green Screen filter - - Introduced NVVFX_MAX_INPUT_WIDTH, NVVFX_MAX_INPUT_HEIGHT parameter selectors and getters to set the maximum input resolution - for NVVFX_FX_GREEN_SCREEN pipeline to avoid lazy memory allocations - - Introduced the following parameter selectors and APIs for NVVFX_FX_GREEN_SCREEN to improve the quality of the model - - NVVFX_MAX_NUMBER_STREAMS parameter selector to set the maximum number of streams - - NVVFX_STATE, NVVFX_STATE_COUNT parameter selectors - - NvVFX_AllocateState, NvVFX_DeallocateState, NvVFX_ResetState, NvVFX_SetStateObjectHandleArray APIs - - BatchEffectApp does not support NVVFX_FX_GREEN_SCREEN anymore. Use BatchAigsEffectApp as the sample app for batch input - - Migrated to TensorRT 8.4.2.2 - - Migrated to CUDA 11.6u1 + - AI Green Screen - Latency improvements \ No newline at end of file diff --git a/nvvfx/include/nvCVImage.h b/nvvfx/include/nvCVImage.h index 543df6e..3906119 100644 --- a/nvvfx/include/nvCVImage.h +++ b/nvvfx/include/nvCVImage.h @@ -538,6 +538,7 @@ NvCV_Status NvCV_API NvCVImage_TransferToYUV( //! Composite one source image over another using the given matte. //! This accommodates all RGB and RGBA formats, with u8 and f32 components. +//! If the bg has alpha, then the dst alpha is updated for use in subsequent composition. //! \param[in] fg the foreground source image. //! \param[in] bg the background source image. //! \param[in] mat the matte Yu8 (or Au8) image, indicating where the src should come through. @@ -547,7 +548,6 @@ NvCV_Status NvCV_API NvCVImage_TransferToYUV( //! \return NVCV_ERR_PIXELFORMAT if the pixel format is not accommodated. //! \return NVCV_ERR_MISMATCH if either the fg & bg & dst formats do not match, or if fg & bg & dst & mat are not //! in the same address space (CPU or GPU). -//! \bug Though RGBA destinations are accommodated, the A channel is not updated at all. #if RTX_CAMERA_IMAGE == 0 NvCV_Status NvCV_API NvCVImage_Composite(const NvCVImage *fg, const NvCVImage *bg, const NvCVImage *mat, NvCVImage *dst, struct CUstream_st *stream); @@ -556,8 +556,11 @@ NvCV_Status NvCV_API NvCVImage_Composite(const NvCVImage *fg, const NvCVImage *b #endif // RTX_CAMERA_IMAGE == 1 -//! Composite one source image over another using the given matte. +//! Composite one source image rectangular region over another using the given matte. //! This accommodates all RGB and RGBA formats, with u8 and f32 components. +//! If the bg has alpha, then the dst alpha is updated for use in subsequent composition. +//! If the background is not opaque, it is recommended that all images be premultiplied by alpha, +//! and mode 1 composition be used, to yield the most meaningful composite matte. //! \param[in] fg the foreground source image. //! \param[in] fgOrg the upper-left corner of the fg image to be composited (NULL implies (0,0)). //! \param[in] bg the background source image. @@ -570,14 +573,11 @@ NvCV_Status NvCV_API NvCVImage_Composite(const NvCVImage *fg, const NvCVImage *b //! \param[in] dstOrg the upper-left corner of the dst image to be updated (NULL implies (0,0)). //! \param[in] stream the CUDA stream on which the composition is to be performed. //! \note If a smaller region of a matte is desired, a window can be created using -//! NvCVImage_InitView() for chunky or NvCVImage_Init() for planar pixels. +//! NvCVImage_InitView() for chunky pixels, as illustrated below in NvCVImage_CompositeRectA(). //! \return NVCV_SUCCESS if the operation was successful. //! \return NVCV_ERR_PIXELFORMAT if the pixel format is not accommodated. //! \return NVCV_ERR_MISMATCH if either the fg & bg & dst formats do not match, or if fg & bg & dst & mat are not //! in the same address space (CPU or GPU). -//! \bug Though RGBA destinations are accommodated, the A channel is not updated at all. -//! \todo Accommodate premultiplied alpha, either as a flag in NvCVImage or as a different mode. -//! \todo If the destination has an A channel, update it as per Adobe and Pixar. NvCV_Status NvCV_API NvCVImage_CompositeRect( const NvCVImage *fg, const NvCVPoint2i *fgOrg, const NvCVImage *bg, const NvCVPoint2i *bgOrg, @@ -586,6 +586,41 @@ NvCV_Status NvCV_API NvCVImage_CompositeRect( struct CUstream_st *stream); +//! Composite one RGBA or BGRA source image rectangular region over another RGB, BGR, RGBA or BGRA region. +//! This accommodates all RGB and RGBA formats, with u8 and f32 components. +//! If the bg has alpha, then the dst alpha is updated for use in subsequent composition. +//! If the background is not opaque, it is recommended that all images be premultiplied by alpha, +//! and mode 1 composition be used, to yield the most meaningful composite matte. +//! \param[in] fg the foreground RGBA or BGRA source image. +//! \param[in] fgRect a sub-rectangle of the fg image (NULL implies the whole image). +//! \param[in] bg the background source image. +//! \param[in] bgOrg the upper-left corner of the bg image to be composited (NULL implies (0,0)). +//! \param[in] mode the composition mode: 0 (straight alpha over) or 1 (premultiplied alpha over). +//! \param[out] dst the destination image. This can be the same as fg or bg. +//! \param[in] dstOrg the upper-left corner of the dst image to be updated (NULL implies (0,0)). +//! \param[in] stream the CUDA stream on which the composition is to be performed. +//! \return NVCV_SUCCESS if the operation was successful. +//! \return NVCV_ERR_PIXELFORMAT if the pixel format is not accommodated. +//! \return NVCV_ERR_MISMATCH if either the fg & bg & dst formats do not match, or if fg & bg & dst & mat are not +//! in the same address space (CPU or GPU). +//! \bug fgRect will only work for chunky images, not planar. +#ifdef __cplusplus +inline NvCV_Status NvCVImage_CompositeRectA( + const NvCVImage *fg, const NvCVRect2i *fgRect, + const NvCVImage *bg, const NvCVPoint2i *bgOrg, + unsigned mode, + NvCVImage *dst, const NvCVPoint2i *dstOrg, + struct CUstream_st *stream +) { + if (fgRect) { + NvCVImage fgView(const_cast(fg), fgRect->x, fgRect->y, fgRect->width, fgRect->height); + return NvCVImage_CompositeRect(&fgView, nullptr, bg, bgOrg, &fgView, mode, dst, dstOrg, stream); + } + return NvCVImage_CompositeRect(fg, nullptr, bg, bgOrg, fg, mode, dst, dstOrg, stream); +} +#endif // __cplusplus + + //! Composite a source image over a constant color field using the given matte. //! \param[in] src the source image. //! \param[in] mat the matte image, indicating where the src should come through. @@ -598,7 +633,6 @@ NvCV_Status NvCV_API NvCVImage_CompositeRect( //! \return NVCV_ERR_MISMATCH if fg & mat & dst & bgColor are not in the same address space (CPU or GPU). //! \note The bgColor must remain valid until complete; this is an important consideration especially if //! the buffers are on the GPU and NvCVImage_CompositeOverConstant() runs asynchronously. -//! \bug Though RGBA destinations are accommodated, the A channel is not updated at all. NvCV_Status NvCV_API NvCVImage_CompositeOverConstant( #if RTX_CAMERA_IMAGE == 0 const NvCVImage *src, const NvCVImage *mat, const void *bgColor, NvCVImage *dst, struct CUstream_st *stream diff --git a/nvvfx/include/nvCVStatus.h b/nvvfx/include/nvCVStatus.h index 2931a9a..f560edc 100644 --- a/nvvfx/include/nvCVStatus.h +++ b/nvvfx/include/nvCVStatus.h @@ -83,6 +83,7 @@ typedef enum NvCV_Status { NVCV_ERR_OBJECTNOTFOUND = -36, //!< The specified object was not found. NVCV_ERR_SINGULAR = -37, //!< A mathematical singularity has been encountered. NVCV_ERR_NOTHINGRENDERED = -38, //!< Nothing was rendered in the specified region. + NVCV_ERR_CONVERGENCE = -39, //!< An iteration did not converge satisfactorily. NVCV_ERR_OPENGL = -98, //!< An OpenGL error has occurred. NVCV_ERR_DIRECT3D = -99, //!< A Direct3D error has occurred. diff --git a/samples/AigsEffectApp/AigsEffectApp.cpp b/samples/AigsEffectApp/AigsEffectApp.cpp index d914467..5b9a059 100644 --- a/samples/AigsEffectApp/AigsEffectApp.cpp +++ b/samples/AigsEffectApp/AigsEffectApp.cpp @@ -622,6 +622,18 @@ static void overlay(const cv::Mat &image, const cv::Mat &mask, float alpha, cv:: result = image * (1.f - alpha) + maskClr * alpha; } +static NvCV_Status WriteRGBA(const NvCVImage *bgr, const NvCVImage *a, const std::string& name) { + NvCV_Status err; + NvCVImage bgra(bgr->width, bgr->height, NVCV_BGRA, NVCV_U8); + NvCVImage aa(const_cast(a), 0, 0, a->width, a->height); + aa.pixelFormat = NVCV_A; // This could be Y but we make sure it is interpreted as alpha + err = NvCVImage_Transfer(bgr, &bgra, 0, 0, 0); if (NVCV_SUCCESS != err) return err; + err = NvCVImage_Transfer(&aa, &bgra, 0, 0, 0); if (NVCV_SUCCESS != err) return err; + cv::Mat ocv; + CVWrapperForNvCVImage(&bgra, &ocv); + return cv::imwrite(name, ocv) ? NVCV_SUCCESS : NVCV_ERR_WRITE; +} + FXApp::Err FXApp::processImage(const char *inFile, const char *outFile) { NvCV_Status vfxErr; bool ok; @@ -675,10 +687,10 @@ FXApp::Err FXApp::processImage(const char *inFile, const char *outFile) { if (!std::string(outFile).empty()) { if(IsLossyImageFile(outFile)) fprintf(stderr, "WARNING: JPEG output file format will reduce image quality\n"); - ok = cv::imwrite(outFile, result); - if (!ok) { - printf("Error writing: \"%s\"\n", outFile); - return errWrite; + vfxErr = WriteRGBA(&_srcVFX, &_dstVFX, outFile); + if (NVCV_SUCCESS != vfxErr) { + printf("%s: \"%s\"\n", NvCV_GetErrorStringFromCode(vfxErr), outFile); + goto bail; } ok = cv::imwrite(std::string(outFile) + "_segmentation_mask.png", _dstImg); // save segmentation mask too if (!ok) { diff --git a/samples/AigsEffectApp/AigsEffectApp.exe b/samples/AigsEffectApp/AigsEffectApp.exe index 48bc6ab..a2c4e86 100644 Binary files a/samples/AigsEffectApp/AigsEffectApp.exe and b/samples/AigsEffectApp/AigsEffectApp.exe differ diff --git a/samples/BatchEffectApp/BatchAigsEffectApp.exe b/samples/BatchEffectApp/BatchAigsEffectApp.exe index cb18344..d2189c3 100644 Binary files a/samples/BatchEffectApp/BatchAigsEffectApp.exe and b/samples/BatchEffectApp/BatchAigsEffectApp.exe differ diff --git a/samples/BatchEffectApp/BatchDenoiseEffectApp.exe b/samples/BatchEffectApp/BatchDenoiseEffectApp.exe index 9af42f9..f7c9580 100644 Binary files a/samples/BatchEffectApp/BatchDenoiseEffectApp.exe and b/samples/BatchEffectApp/BatchDenoiseEffectApp.exe differ diff --git a/samples/BatchEffectApp/BatchEffectApp.exe b/samples/BatchEffectApp/BatchEffectApp.exe index 0582115..786d9e3 100644 Binary files a/samples/BatchEffectApp/BatchEffectApp.exe and b/samples/BatchEffectApp/BatchEffectApp.exe differ diff --git a/samples/DenoiseEffectApp/DenoiseEffectApp.exe b/samples/DenoiseEffectApp/DenoiseEffectApp.exe index 9994382..1a78d11 100644 Binary files a/samples/DenoiseEffectApp/DenoiseEffectApp.exe and b/samples/DenoiseEffectApp/DenoiseEffectApp.exe differ diff --git a/samples/UpscalePipelineApp/UpscalePipeline.cpp b/samples/UpscalePipelineApp/UpscalePipeline.cpp index 3776ddd..f750246 100644 --- a/samples/UpscalePipelineApp/UpscalePipeline.cpp +++ b/samples/UpscalePipelineApp/UpscalePipeline.cpp @@ -414,9 +414,6 @@ FXApp::Err FXApp::createEffects(const char *modelDir, NvVFX_EffectSelector first BAIL_IF_ERR(vfxErr = NvVFX_CreateEffect(first, &_arEff)); BAIL_IF_ERR(vfxErr = NvVFX_SetString(_arEff, NVVFX_MODEL_DIRECTORY, modelDir)); BAIL_IF_ERR(vfxErr = NvVFX_CreateEffect(second, &_upscaleEff)); - if (modelDir[0] != '\0'){ - BAIL_IF_ERR(vfxErr = NvVFX_SetString(_upscaleEff, NVVFX_MODEL_DIRECTORY, modelDir)); - } bail: return appErrFromVfxStatus(vfxErr); } diff --git a/samples/UpscalePipelineApp/UpscalePipelineApp.exe b/samples/UpscalePipelineApp/UpscalePipelineApp.exe index 1d25885..131780f 100644 Binary files a/samples/UpscalePipelineApp/UpscalePipelineApp.exe and b/samples/UpscalePipelineApp/UpscalePipelineApp.exe differ diff --git a/samples/VideoEffectsApp/VideoEffectsApp.cpp b/samples/VideoEffectsApp/VideoEffectsApp.cpp index 2954193..20392a4 100644 --- a/samples/VideoEffectsApp/VideoEffectsApp.cpp +++ b/samples/VideoEffectsApp/VideoEffectsApp.cpp @@ -459,7 +459,8 @@ FXApp::Err FXApp::createEffect(const char *effectSelector, const char *modelDir) NvCV_Status vfxErr; BAIL_IF_ERR(vfxErr = NvVFX_CreateEffect(effectSelector, &_eff)); _effectName = effectSelector; - if (modelDir[0] != '\0'){ + // Do not set NVVFX_MODEL_DIRECTORY for NVVFX_FX_SR_UPSCALE feature as it is not a valid selector for that feature + if (modelDir[0] != '\0' && strcmp(_effectName, NVVFX_FX_SR_UPSCALE)){ BAIL_IF_ERR(vfxErr = NvVFX_SetString(_eff, NVVFX_MODEL_DIRECTORY, modelDir)); } bail: diff --git a/samples/VideoEffectsApp/VideoEffectsApp.exe b/samples/VideoEffectsApp/VideoEffectsApp.exe index 5974b7c..752ec06 100644 Binary files a/samples/VideoEffectsApp/VideoEffectsApp.exe and b/samples/VideoEffectsApp/VideoEffectsApp.exe differ diff --git a/version.h b/version.h index 19928fa..c038568 100644 --- a/version.h +++ b/version.h @@ -24,12 +24,12 @@ #define NVIDIA_VIDEOEFFECTS_SDK_VERSION_MAJOR 0 #define NVIDIA_VIDEOEFFECTS_SDK_VERSION_MINOR 7 -#define NVIDIA_VIDEOEFFECTS_SDK_VERSION_RELEASE 1 +#define NVIDIA_VIDEOEFFECTS_SDK_VERSION_RELEASE 2 #define NVIDIA_VIDEOEFFECTS_SDK_VERSION_BUILD 0 -#define NVIDIA_VIDEOEFFECTS_SDK_VERSION 0,7,1,0 +#define NVIDIA_VIDEOEFFECTS_SDK_VERSION 0,7,2,0 #define NVIDIA_VIDEOEFFECTS_SDK_VERSION_MAJOR_MINOR 0,7 -#define NVIDIA_VIDEOEFFECTS_SDK_VERSION_STRING "0.7.1.0" -#define NVIDIA_VIDEOEFFECTS_SDK_VERSION_STRING_SHORT "0.7.1" +#define NVIDIA_VIDEOEFFECTS_SDK_VERSION_STRING "0.7.2.0" +#define NVIDIA_VIDEOEFFECTS_SDK_VERSION_STRING_SHORT "0.7.2" #define NVIDIA_VIDEOEFFECTS_SDK_VERSION_STRING_MAJOR_MINOR "0.7"