vf_d3d11vpp: use ID3D11VideoContext1 interface for setting color space

This allows to use DXGI enums for color spaces, which supports PQ
transfers and so on. Old D3D11_VIDEO_PROCESSOR_COLOR_SPACE is limited
and doesn't support HDR formats.

video/filter/vf_d3d11vpp.c

@@ -18,6 +18,7 @@
 #include <assert.h>
 #include <windows.h>
 #include <d3d11.h>
+#include <d3d11_1.h>
 
 #include <libavutil/hwcontext.h>
 #include <libavutil/hwcontext_d3d11va.h>
@@ -34,6 +35,7 @@
 #include "video/hwdec.h"
 #include "video/mp_image.h"
 #include "video/mp_image_pool.h"
+#include "video/out/gpu/d3d11_helpers.h"
 
 // For video processor extensions identifiers reference see:
 // https://chromium.googlesource.com/chromium/src/+/5f354f38/ui/gl/swap_chain_presenter.cc
@@ -374,16 +376,35 @@ static int recreate_video_proc(struct mp_filter *vf)
                                                             D3D11_VIDEO_PROCESSOR_OUTPUT_RATE_NORMAL,
                                                          FALSE, 0);
 
-    D3D11_VIDEO_PROCESSOR_COLOR_SPACE csp = {
-        .YCbCr_Matrix = p->params.repr.sys != PL_COLOR_SYSTEM_BT_601,
-        .Nominal_Range = p->params.repr.levels == PL_COLOR_LEVELS_LIMITED ? 1 : 2,
-    };
-    ID3D11VideoContext_VideoProcessorSetStreamColorSpace(p->video_ctx,
-                                                         p->video_proc,
-                                                         0, &csp);
-    ID3D11VideoContext_VideoProcessorSetOutputColorSpace(p->video_ctx,
-                                                         p->video_proc,
-                                                         &csp);
+    mp_image_params_guess_csp(&p->params);
+    mp_image_params_guess_csp(&p->out_params);
+
+    ID3D11VideoContext1 *video_ctx1;
+    hr = ID3D11VideoContext_QueryInterface(p->video_ctx, &IID_ID3D11VideoContext1, (void **)&video_ctx1);
+    if (SUCCEEDED(hr)) {
+        DXGI_COLOR_SPACE_TYPE in = mp_params_to_dxgi_colorspace(vf->log, &p->params);
+        DXGI_COLOR_SPACE_TYPE out = mp_params_to_dxgi_colorspace(vf->log, &p->out_params);
+        if (in != out)
+            MP_VERBOSE(vf, "Converting %s to %s.\n", d3d11_get_csp_name(in), d3d11_get_csp_name(out));
+        ID3D11VideoContext1_VideoProcessorSetStreamColorSpace1(video_ctx1,
+                                                               p->video_proc,
+                                                               0, in);
+        ID3D11VideoContext1_VideoProcessorSetOutputColorSpace1(video_ctx1,
+                                                               p->video_proc,
+                                                               out);
+        SAFE_RELEASE(video_ctx1);
+    } else {
+        D3D11_VIDEO_PROCESSOR_COLOR_SPACE csp = {
+            .YCbCr_Matrix = p->params.repr.sys != PL_COLOR_SYSTEM_BT_601,
+            .Nominal_Range = p->params.repr.levels == PL_COLOR_LEVELS_LIMITED ? 1 : 2,
+        };
+        ID3D11VideoContext_VideoProcessorSetStreamColorSpace(p->video_ctx,
+                                                            p->video_proc,
+                                                            0, &csp);
+        ID3D11VideoContext_VideoProcessorSetOutputColorSpace(p->video_ctx,
+                                                            p->video_proc,
+                                                            &csp);
+    }
 
     switch (p->opts->scaling_mode) {
     case SCALING_INTEL_VSR:

video/out/gpu/d3d11_helpers.c

@@ -192,7 +192,7 @@ static const char *d3d11_get_format_name(DXGI_FORMAT fmt)
     }
 }
 
-static const char *d3d11_get_csp_name(DXGI_COLOR_SPACE_TYPE csp)
+const char *d3d11_get_csp_name(DXGI_COLOR_SPACE_TYPE csp)
 {
     switch (csp) {
     D3D11_DXGI_ENUM(DXGI_COLOR_SPACE_, RGB_FULL_G22_NONE_P709);
@@ -1140,3 +1140,90 @@ void mp_d3d11_get_debug_interfaces(struct mp_log *log, IDXGIDebug **debug,
         return;
     }
 }
+
+DXGI_COLOR_SPACE_TYPE mp_params_to_dxgi_colorspace(struct mp_log *log,
+                                                   const struct mp_image_params *params)
+{
+    bool is_rgb  = params->repr.sys == PL_COLOR_SYSTEM_RGB;
+    bool full    = params->repr.levels == PL_COLOR_LEVELS_FULL;
+    bool topleft = params->chroma_location == PL_CHROMA_TOP_LEFT;
+    bool p601    = params->color.primaries == PL_COLOR_PRIM_BT_601_525 ||
+                   params->color.primaries == PL_COLOR_PRIM_BT_601_625;
+    bool p2020   = params->color.primaries == PL_COLOR_PRIM_BT_2020;
+
+    if (is_rgb) {
+        switch (params->color.transfer) {
+        case PL_COLOR_TRC_LINEAR:
+            return DXGI_COLOR_SPACE_RGB_FULL_G10_NONE_P709;
+        case PL_COLOR_TRC_PQ:
+            if (!p2020) {
+                mp_warn(log, "PQ transfer with non-BT.2020 primaries is not "
+                            "supported by DXGI, using P2020 entry.\n");
+            }
+            return full ? DXGI_COLOR_SPACE_RGB_FULL_G2084_NONE_P2020
+                        : DXGI_COLOR_SPACE_RGB_STUDIO_G2084_NONE_P2020;
+        case PL_COLOR_TRC_GAMMA24:
+            return p2020 ? DXGI_COLOR_SPACE_RGB_STUDIO_G24_NONE_P2020
+                         : DXGI_COLOR_SPACE_RGB_STUDIO_G24_NONE_P709;
+        case PL_COLOR_TRC_HLG:
+            mp_warn(log, "HLG RGB is not supported by DXGI, "
+                        "falling back to G22.\n");
+            MP_FALLTHROUGH;
+        default:
+            if (p2020) {
+                return full ? DXGI_COLOR_SPACE_RGB_FULL_G22_NONE_P2020
+                            : DXGI_COLOR_SPACE_RGB_STUDIO_G22_NONE_P2020;
+            }
+            return full ? DXGI_COLOR_SPACE_RGB_FULL_G22_NONE_P709
+                        : DXGI_COLOR_SPACE_RGB_STUDIO_G22_NONE_P709;
+        }
+    }
+
+    switch (params->color.transfer) {
+    case PL_COLOR_TRC_PQ:
+        if (!p2020) {
+            mp_warn(log, "PQ transfer with non-BT.2020 primaries is not "
+                         "supported by DXGI, using P2020 entry.\n");
+        }
+        if (full) {
+            mp_warn(log, "Full-range PQ YCbCr is not supported by DXGI, "
+                         "using studio entry.\n");
+        }
+        return topleft ? DXGI_COLOR_SPACE_YCBCR_STUDIO_G2084_TOPLEFT_P2020
+                       : DXGI_COLOR_SPACE_YCBCR_STUDIO_G2084_LEFT_P2020;
+    case PL_COLOR_TRC_HLG:
+        if (!p2020) {
+            mp_warn(log, "HLG transfer with non-BT.2020 primaries is not "
+                         "supported by DXGI, using P2020 entry.\n");
+        }
+        return full ? DXGI_COLOR_SPACE_YCBCR_FULL_GHLG_TOPLEFT_P2020
+                    : DXGI_COLOR_SPACE_YCBCR_STUDIO_GHLG_TOPLEFT_P2020;
+    case PL_COLOR_TRC_GAMMA24:
+        if (full) {
+            mp_warn(log, "Full-range G24 YCbCr is not supported by DXGI, "
+                         "using studio entry.\n");
+        }
+        if (p2020) {
+            return topleft ? DXGI_COLOR_SPACE_YCBCR_STUDIO_G24_TOPLEFT_P2020
+                           : DXGI_COLOR_SPACE_YCBCR_STUDIO_G24_LEFT_P2020;
+        }
+        return DXGI_COLOR_SPACE_YCBCR_STUDIO_G24_LEFT_P709;
+    default:
+        if (p601) {
+            return full ? DXGI_COLOR_SPACE_YCBCR_FULL_G22_LEFT_P601
+                        : DXGI_COLOR_SPACE_YCBCR_STUDIO_G22_LEFT_P601;
+        }
+        if (p2020) {
+            if (topleft) {
+                if (full)
+                    mp_warn(log, "Full-range top-left chroma BT.2020 G22 is not "
+                                 "supported by DXGI, using studio entry.\n");
+                return DXGI_COLOR_SPACE_YCBCR_STUDIO_G22_TOPLEFT_P2020;
+            }
+            return full ? DXGI_COLOR_SPACE_YCBCR_FULL_G22_LEFT_P2020
+                        : DXGI_COLOR_SPACE_YCBCR_STUDIO_G22_LEFT_P2020;
+        }
+        return full ? DXGI_COLOR_SPACE_YCBCR_FULL_G22_LEFT_P709
+                    : DXGI_COLOR_SPACE_YCBCR_STUDIO_G22_LEFT_P709;
+    }
+}

video/out/gpu/d3d11_helpers.h

@@ -121,4 +121,9 @@ bool mp_d3d11_create_swapchain(ID3D11Device *dev, struct mp_log *log,
 void mp_d3d11_get_debug_interfaces(struct mp_log *log, IDXGIDebug **debug,
                                    IDXGIInfoQueue **iqueue);
 
+const char *d3d11_get_csp_name(DXGI_COLOR_SPACE_TYPE csp);
+
+DXGI_COLOR_SPACE_TYPE mp_params_to_dxgi_colorspace(struct mp_log *log,
+                                                   const struct mp_image_params *params);
+
 #endif