v0.7.2.0 Release

v0.7.2.0 Release

CHANGELOG

@@ -1,14 +1,3 @@
-Changelog (v0.7.1.0)
+Changelog (v0.7.2.0)
 --------------------
- - NvCVImage_Transfer() now sets alpha to 255 or 1.0f when doing RGB -> RGBA
- - NvCVImage_CompositeRect() has a premultiplied alpha mode added
- - AI Green Screen filter
-    - Introduced NVVFX_MAX_INPUT_WIDTH, NVVFX_MAX_INPUT_HEIGHT parameter selectors and getters to set the maximum input resolution
-        for NVVFX_FX_GREEN_SCREEN pipeline to avoid lazy memory allocations
-    - Introduced the following parameter selectors and APIs for NVVFX_FX_GREEN_SCREEN to improve the quality of the model
-    - NVVFX_MAX_NUMBER_STREAMS parameter selector to set the maximum number of streams
-    - NVVFX_STATE, NVVFX_STATE_COUNT parameter selectors
-    - NvVFX_AllocateState, NvVFX_DeallocateState, NvVFX_ResetState, NvVFX_SetStateObjectHandleArray APIs
-    - BatchEffectApp does not support NVVFX_FX_GREEN_SCREEN anymore. Use BatchAigsEffectApp as the sample app for batch input
- - Migrated to TensorRT 8.4.2.2
- - Migrated to CUDA 11.6u1
+  - AI Green Screen - Latency improvements

nvvfx/include/nvCVImage.h

@@ -538,6 +538,7 @@ NvCV_Status NvCV_API NvCVImage_TransferToYUV(
 
 //! Composite one source image over another using the given matte.
 //! This accommodates all RGB and RGBA formats, with u8 and f32 components.
+//! If the bg has alpha, then the dst alpha is updated for use in subsequent composition.
 //! \param[in]  fg      the foreground source image.
 //! \param[in]  bg      the background source image.
 //! \param[in]  mat     the matte  Yu8   (or Au8)   image, indicating where the src should come through.
@@ -547,7 +548,6 @@ NvCV_Status NvCV_API NvCVImage_TransferToYUV(
 //! \return NVCV_ERR_PIXELFORMAT if the pixel format is not accommodated.
 //! \return NVCV_ERR_MISMATCH    if either the fg & bg & dst formats do not match, or if fg & bg & dst & mat are not
 //!                              in the same address space (CPU or GPU).
-//! \bug  Though RGBA destinations are accommodated, the A channel is not updated at all.
 #if RTX_CAMERA_IMAGE == 0
 NvCV_Status NvCV_API NvCVImage_Composite(const NvCVImage *fg, const NvCVImage *bg, const NvCVImage *mat, NvCVImage *dst,
             struct CUstream_st *stream);
@@ -556,8 +556,11 @@ NvCV_Status NvCV_API NvCVImage_Composite(const NvCVImage *fg, const NvCVImage *b
 #endif // RTX_CAMERA_IMAGE == 1
 
 
-//! Composite one source image over another using the given matte.
+//! Composite one source image rectangular region over another using the given matte.
 //! This accommodates all RGB and RGBA formats, with u8 and f32 components.
+//! If the bg has alpha, then the dst alpha is updated for use in subsequent composition.
+//! If the background is not opaque, it is recommended that all images be premultiplied by alpha,
+//! and mode 1 composition be used, to yield the most meaningful composite matte.
 //! \param[in]      fg      the foreground source image.
 //! \param[in]      fgOrg   the upper-left corner of the fg image to be composited (NULL implies (0,0)).
 //! \param[in]      bg      the background source image.
@@ -570,14 +573,11 @@ NvCV_Status NvCV_API NvCVImage_Composite(const NvCVImage *fg, const NvCVImage *b
 //! \param[in]      dstOrg  the upper-left corner of the dst image to be updated (NULL implies (0,0)).
 //! \param[in]      stream  the CUDA stream on which the composition is to be performed.
 //! \note   If a smaller region of a matte is desired, a window can be created using
-//!         NvCVImage_InitView() for chunky or NvCVImage_Init() for planar pixels.
+//!         NvCVImage_InitView() for chunky pixels, as illustrated below in NvCVImage_CompositeRectA().
 //! \return NVCV_SUCCESS         if the operation was successful.
 //! \return NVCV_ERR_PIXELFORMAT if the pixel format is not accommodated.
 //! \return NVCV_ERR_MISMATCH    if either the fg & bg & dst formats do not match, or if fg & bg & dst & mat are not
 //!                              in the same address space (CPU or GPU).
-//! \bug  Though RGBA destinations are accommodated, the A channel is not updated at all.
-//! \todo Accommodate premultiplied alpha, either as a flag in NvCVImage or as a different mode.
-//! \todo If the destination has an A channel, update it as per Adobe and Pixar.
 NvCV_Status NvCV_API NvCVImage_CompositeRect(
     const NvCVImage *fg,  const NvCVPoint2i *fgOrg,
     const NvCVImage *bg,  const NvCVPoint2i *bgOrg,
@@ -586,6 +586,41 @@ NvCV_Status NvCV_API NvCVImage_CompositeRect(
     struct CUstream_st *stream);
 
 
+//! Composite one RGBA or BGRA source image rectangular region over another RGB, BGR, RGBA or BGRA region.
+//! This accommodates all RGB and RGBA formats, with u8 and f32 components.
+//! If the bg has alpha, then the dst alpha is updated for use in subsequent composition.
+//! If the background is not opaque, it is recommended that all images be premultiplied by alpha,
+//! and mode 1 composition be used, to yield the most meaningful composite matte.
+//! \param[in]      fg      the foreground RGBA or BGRA source image.
+//! \param[in]      fgRect  a sub-rectangle of the fg image (NULL implies the whole image).
+//! \param[in]      bg      the background source image.
+//! \param[in]      bgOrg   the upper-left corner of the bg image to be composited (NULL implies (0,0)).
+//! \param[in]      mode    the composition mode: 0 (straight alpha over) or 1 (premultiplied alpha over).
+//! \param[out]     dst     the destination image. This can be the same as fg or bg.
+//! \param[in]      dstOrg  the upper-left corner of the dst image to be updated (NULL implies (0,0)).
+//! \param[in]      stream  the CUDA stream on which the composition is to be performed.
+//! \return NVCV_SUCCESS         if the operation was successful.
+//! \return NVCV_ERR_PIXELFORMAT if the pixel format is not accommodated.
+//! \return NVCV_ERR_MISMATCH    if either the fg & bg & dst formats do not match, or if fg & bg & dst & mat are not
+//!                              in the same address space (CPU or GPU).
+//! \bug  fgRect will only work for chunky images, not planar.
+#ifdef __cplusplus
+inline NvCV_Status NvCVImage_CompositeRectA(
+    const NvCVImage *fg, const NvCVRect2i  *fgRect,
+    const NvCVImage *bg, const NvCVPoint2i *bgOrg,
+    unsigned mode,
+    NvCVImage       *dst, const NvCVPoint2i *dstOrg,
+    struct CUstream_st *stream
+) {
+  if (fgRect) {
+    NvCVImage fgView(const_cast<NvCVImage*>(fg), fgRect->x, fgRect->y, fgRect->width, fgRect->height);
+    return NvCVImage_CompositeRect(&fgView, nullptr, bg, bgOrg, &fgView, mode, dst, dstOrg, stream);
+  }
+  return NvCVImage_CompositeRect(fg, nullptr, bg, bgOrg, fg, mode, dst, dstOrg, stream);
+}
+#endif // __cplusplus
+
+
 //! Composite a source image over a constant color field using the given matte.
 //! \param[in]      src     the source image.
 //! \param[in]      mat     the matte  image, indicating where the src should come through.
@@ -598,7 +633,6 @@ NvCV_Status NvCV_API NvCVImage_CompositeRect(
 //! \return NVCV_ERR_MISMATCH    if fg & mat & dst & bgColor are not in the same address space (CPU or GPU).
 //! \note   The bgColor must remain valid until complete; this is an important consideration especially if
 //!         the buffers are on the GPU and NvCVImage_CompositeOverConstant() runs asynchronously.
-//! \bug    Though RGBA destinations are accommodated, the A channel is not updated at all.
 NvCV_Status NvCV_API NvCVImage_CompositeOverConstant(
 #if RTX_CAMERA_IMAGE == 0
     const NvCVImage *src, const NvCVImage *mat, const void *bgColor, NvCVImage *dst, struct CUstream_st *stream

nvvfx/include/nvCVStatus.h

@@ -83,6 +83,7 @@ typedef enum NvCV_Status {
   NVCV_ERR_OBJECTNOTFOUND        = -36,  //!< The specified object was not found.
   NVCV_ERR_SINGULAR              = -37,  //!< A mathematical singularity has been encountered.
   NVCV_ERR_NOTHINGRENDERED       = -38,  //!< Nothing was rendered in the specified region.
+  NVCV_ERR_CONVERGENCE           = -39,  //!< An iteration did not converge satisfactorily.
 
   NVCV_ERR_OPENGL                = -98,  //!< An OpenGL error has occurred.
   NVCV_ERR_DIRECT3D              = -99,  //!< A Direct3D error has occurred.

samples/AigsEffectApp/AigsEffectApp.cpp

@@ -622,6 +622,18 @@ static void overlay(const cv::Mat &image, const cv::Mat &mask, float alpha, cv::
   result = image * (1.f - alpha) + maskClr * alpha;
 }
 
+static NvCV_Status WriteRGBA(const NvCVImage *bgr, const NvCVImage *a, const std::string& name) {
+  NvCV_Status err;
+  NvCVImage bgra(bgr->width, bgr->height, NVCV_BGRA, NVCV_U8);
+  NvCVImage aa(const_cast<NvCVImage*>(a), 0, 0, a->width, a->height);
+  aa.pixelFormat = NVCV_A;  // This could be Y but we make sure it is interpreted as alpha
+  err = NvCVImage_Transfer(bgr, &bgra, 0, 0, 0);   if (NVCV_SUCCESS != err) return err;
+  err = NvCVImage_Transfer(&aa, &bgra, 0, 0, 0);   if (NVCV_SUCCESS != err) return err;
+  cv::Mat ocv;
+  CVWrapperForNvCVImage(&bgra, &ocv);
+  return cv::imwrite(name, ocv) ? NVCV_SUCCESS : NVCV_ERR_WRITE;
+}
+
 FXApp::Err FXApp::processImage(const char *inFile, const char *outFile) {
   NvCV_Status vfxErr;
   bool ok;
@@ -675,10 +687,10 @@ FXApp::Err FXApp::processImage(const char *inFile, const char *outFile) {
   if (!std::string(outFile).empty()) {
     if(IsLossyImageFile(outFile))
       fprintf(stderr, "WARNING: JPEG output file format will reduce image quality\n");
-    ok = cv::imwrite(outFile, result);
-    if (!ok) {
-      printf("Error writing: \"%s\"\n", outFile);
-      return errWrite;
+    vfxErr = WriteRGBA(&_srcVFX, &_dstVFX, outFile);
+    if (NVCV_SUCCESS != vfxErr) {
+      printf("%s: \"%s\"\n", NvCV_GetErrorStringFromCode(vfxErr), outFile);
+      goto bail;
     }
     ok = cv::imwrite(std::string(outFile) + "_segmentation_mask.png", _dstImg);  // save segmentation mask too
     if (!ok) {

samples/UpscalePipelineApp/UpscalePipeline.cpp

@@ -414,9 +414,6 @@ FXApp::Err FXApp::createEffects(const char *modelDir, NvVFX_EffectSelector first
   BAIL_IF_ERR(vfxErr = NvVFX_CreateEffect(first, &_arEff));
   BAIL_IF_ERR(vfxErr = NvVFX_SetString(_arEff, NVVFX_MODEL_DIRECTORY, modelDir));
   BAIL_IF_ERR(vfxErr = NvVFX_CreateEffect(second, &_upscaleEff));
-  if (modelDir[0] != '\0'){
-    BAIL_IF_ERR(vfxErr = NvVFX_SetString(_upscaleEff, NVVFX_MODEL_DIRECTORY, modelDir));
-  }
 bail:
   return appErrFromVfxStatus(vfxErr);
 }

samples/VideoEffectsApp/VideoEffectsApp.cpp

@@ -459,7 +459,8 @@ FXApp::Err FXApp::createEffect(const char *effectSelector, const char *modelDir)
   NvCV_Status vfxErr;
   BAIL_IF_ERR(vfxErr = NvVFX_CreateEffect(effectSelector, &_eff));
   _effectName = effectSelector;
-  if (modelDir[0] != '\0'){
+  // Do not set NVVFX_MODEL_DIRECTORY for NVVFX_FX_SR_UPSCALE feature as it is not a valid selector for that feature
+  if (modelDir[0] != '\0' && strcmp(_effectName, NVVFX_FX_SR_UPSCALE)){
     BAIL_IF_ERR(vfxErr = NvVFX_SetString(_eff, NVVFX_MODEL_DIRECTORY, modelDir));
   }
 bail:

version.h

@@ -24,12 +24,12 @@
 
 #define NVIDIA_VIDEOEFFECTS_SDK_VERSION_MAJOR 0
 #define NVIDIA_VIDEOEFFECTS_SDK_VERSION_MINOR 7
-#define NVIDIA_VIDEOEFFECTS_SDK_VERSION_RELEASE 1
+#define NVIDIA_VIDEOEFFECTS_SDK_VERSION_RELEASE 2
 #define NVIDIA_VIDEOEFFECTS_SDK_VERSION_BUILD 0
 
 
-#define NVIDIA_VIDEOEFFECTS_SDK_VERSION 0,7,1,0
+#define NVIDIA_VIDEOEFFECTS_SDK_VERSION 0,7,2,0
 #define NVIDIA_VIDEOEFFECTS_SDK_VERSION_MAJOR_MINOR 0,7
-#define NVIDIA_VIDEOEFFECTS_SDK_VERSION_STRING "0.7.1.0"
-#define NVIDIA_VIDEOEFFECTS_SDK_VERSION_STRING_SHORT "0.7.1"
+#define NVIDIA_VIDEOEFFECTS_SDK_VERSION_STRING "0.7.2.0"
+#define NVIDIA_VIDEOEFFECTS_SDK_VERSION_STRING_SHORT "0.7.2"
 #define NVIDIA_VIDEOEFFECTS_SDK_VERSION_STRING_MAJOR_MINOR "0.7"