vf_d3d11vpp: create own device if the VO provides none

DOCS/man/vf.rst

@@ -718,10 +718,24 @@ Available mpv-only filters are:
             Apply high quality VDPAU scaling (needs capable hardware).
 
 ``d3d11vpp``
-    Direct3D 11 video post-processing. Requires a D3D11 context and works best
-    with hardware decoding. Software frames are automatically uploaded to hardware
+    Direct3D 11 video post-processing. Works best with a D3D11 context and
+    hardware decoding. Software frames are automatically uploaded to hardware
     for processing.
 
+    If the video output provides no D3D11 device, for example with
+    ``--gpu-api=vulkan``, the filter creates its own device. Every frame is
+    then copied between the two devices, directly on the GPU when the formats
+    involved allow it and through system memory otherwise, which costs memory
+    bandwidth and adds latency. Prefer ``--gpu-api=d3d11`` when using this
+    filter.
+
+    In that fallback mode, frames from a different hardware decoder, such as
+    the CUDA frames ``nvdec`` produces, are routed through system memory on
+    their way to the D3D11 device, which costs yet another copy.
+    ``--hwdec=d3d11va`` cannot be used at all, because the decoder picks its
+    device before this filter is created, so it falls back to software
+    decoding.
+
     ``format``
         Convert to the selected image format, e.g., nv12, p010, etc. (default: don't change).
         Format names can be queried with ``--vf=d3d11vpp=format=help``.

video/filter/vf_d3d11vpp.c

@@ -115,6 +115,10 @@ struct priv {
     AVBufferRef *av_device_ref;
     AVBufferRef *hw_pool;
 
+    // Set only if the device was created by us instead of taken from the VO.
+    struct mp_hwdec_devices *hwdec_devs;
+    struct mp_hwdec_ctx own_hwctx;
+
     struct mp_refqueue *queue;
 
     UINT num_past_views;
@@ -697,6 +701,9 @@ static void uninit(struct mp_filter *vf)
     flush_frames(vf);
     talloc_free(p->queue);
     av_buffer_unref(&p->hw_pool);
+
+    if (p->hwdec_devs)
+        hwdec_devices_remove(p->hwdec_devs, &p->own_hwctx);
     av_buffer_unref(&p->av_device_ref);
 
     if (p->video_ctx)
@@ -757,9 +764,68 @@ static struct mp_filter *vf_d3d11vpp_create(struct mp_filter *parent,
     struct mp_hwdec_ctx *hwctx =
         hwdec_devices_get_by_imgfmt_and_type(info->hwdec_devs, IMGFMT_D3D11,
                                              AV_HWDEVICE_TYPE_D3D11VA);
-    if (!hwctx || !hwctx->av_device_ref)
-        goto fail;
-    p->av_device_ref = av_buffer_ref(hwctx->av_device_ref);
+    bool from_vo = hwctx && hwctx->av_device_ref &&
+                   !(hwctx->driver_name &&
+                     !strcmp(hwctx->driver_name, vf_d3d11vpp_filter.name));
+
+    if (from_vo) {
+        p->av_device_ref = av_buffer_ref(hwctx->av_device_ref);
+    } else {
+        // The VO does not provide a D3D11 device (e.g. --gpu-api=vulkan).
+        // Share one with any other instance of this filter, so that frames
+        // stay usable across a chain of them, or create one if we are first.
+        if (hwctx && hwctx->av_device_ref) {
+            p->av_device_ref = av_buffer_ref(hwctx->av_device_ref);
+        } else {
+            // Prefer deriving from the VO's device, so that we land on the
+            // same adapter it is rendering on. Creating a device standalone
+            // picks whichever adapter enumerates first, which is not
+            // necessarily the same GPU on a multi-adapter system.
+            struct hwdec_imgfmt_request vk_params = {
+                .imgfmt = IMGFMT_VULKAN,
+                .probing = false,
+            };
+            hwdec_devices_request_for_img_fmt(info->hwdec_devs, &vk_params);
+            struct mp_hwdec_ctx *vkctx =
+                hwdec_devices_get_by_imgfmt_and_type(info->hwdec_devs,
+                                                     IMGFMT_VULKAN,
+                                                     AV_HWDEVICE_TYPE_VULKAN);
+            if (vkctx && vkctx->av_device_ref) {
+                int ret = av_hwdevice_ctx_create_derived(&p->av_device_ref,
+                                                         AV_HWDEVICE_TYPE_D3D11VA,
+                                                         vkctx->av_device_ref, 0);
+                if (ret < 0) {
+                    MP_VERBOSE(f, "Could not derive a D3D11 device from the "
+                                  "VO's Vulkan device: %s\n", av_err2str(ret));
+                    p->av_device_ref = NULL;
+                }
+            }
+            if (!p->av_device_ref) {
+                const struct hwcontext_fns *fns =
+                    hwdec_get_hwcontext_fns(AV_HWDEVICE_TYPE_D3D11VA);
+                if (!fns || !fns->create_dev)
+                    goto fail;
+                struct hwcontext_create_dev_params dev_params = {0};
+                p->av_device_ref = fns->create_dev(f->global, f->log, &dev_params);
+                if (!p->av_device_ref)
+                    goto fail;
+            }
+            MP_WARN(f, "No D3D11 video output, using a separate device. "
+                       "Frames are copied through system memory, which is "
+                       "slow. Use --gpu-api=d3d11 to avoid this.\n");
+        }
+        // Advertise the device so that the generic upload/download filters can
+        // move frames in and out of it. Every instance registers its own entry
+        // even when sharing a device: filters are created before the old ones
+        // are destroyed, so the instance we borrowed from may go away first.
+        p->own_hwctx = (struct mp_hwdec_ctx){
+            .driver_name = vf_d3d11vpp_filter.name,
+            .av_device_ref = p->av_device_ref,
+            .hw_imgfmt = IMGFMT_D3D11,
+        };
+        p->hwdec_devs = info->hwdec_devs;
+        hwdec_devices_add(p->hwdec_devs, &p->own_hwctx);
+    }
     AVHWDeviceContext *avhwctx = (void *)p->av_device_ref->data;
     AVD3D11VADeviceContext *d3dctx = avhwctx->hwctx;