f_autoconvert: keep hw frames on the GPU when possible

filters/f_autoconvert.c

@@ -230,38 +230,50 @@ static bool build_image_converter(struct mp_autoconvert *c, struct mp_log *log,
             mp_err(log, "Failed to create HW uploader for format %s\n",
                    mp_imgfmt_to_name(src_fmt));
         }
-    } else if (dst_all_hw && num_fmts > 0) {
+    } else if ((dst_all_hw || !imgfmt_is_sw) && num_fmts > 0) {
+        // All targets are hw, or the source is hw and staying on the GPU may
+        // be possible, which beats downloading.
         bool upload_created = false;
         int sw_fmt = imgfmt_is_sw ? img->imgfmt : img->params.hw_subfmt;
-
         for (int i = 0; i < num_fmts; i++) {
+            if (!IMGFMT_IS_HWACCEL(fmts[i]))
+                continue;
             // We can probably use this! Very lazy and very approximate.
             struct mp_hwupload upload = mp_hwupload_create(conv, fmts[i],
                                                            sw_fmt, false);
-            if (upload.successful_init) {
-                mp_info(log, "HW-uploading to %s\n", mp_imgfmt_to_name(fmts[i]));
-                filters[2] = upload.f;
-                hwupload_fmt = upload.selected_sw_imgfmt;
-                fmts = &hwupload_fmt;
-                num_fmts = hwupload_fmt ? 1 : 0;
-                hw_to_sw = false;
-
-                // We cannot do format conversions when transferring between
-                // two hardware devices, so reject this format if that would be
-                // required.
-                if (!imgfmt_is_sw && hwupload_fmt != sw_fmt) {
-                    mp_err(log, "Format %s is not supported by %s\n",
-                           mp_imgfmt_to_name(sw_fmt),
-                           mp_imgfmt_to_name(p->imgfmts[i]));
-                    continue;
-                }
-                upload_created = true;
-                break;
+            if (!upload.successful_init)
+                continue;
+            // For a hw source this is a move between two devices, and the
+            // uploader cannot convert sw formats on the way. Whether the
+            // move works can only be tested with a real frame; do that when
+            // a sw target exists and the result decides against downloading.
+            // With only hw targets there is no alternative route, so insert
+            // the uploader untested and let the first frame decide.
+            if (!imgfmt_is_sw &&
+                (upload.selected_sw_imgfmt != sw_fmt ||
+                 (dst_have_sw && !mp_hwupload_probe_hw_to_hw(&upload, img)))) {
+                mp_verbose(log, "Cannot keep %s frames on the GPU as %s\n",
+                           mp_imgfmt_to_name(img->imgfmt),
+                           mp_imgfmt_to_name(fmts[i]));
+                talloc_free(upload.f);
+                continue;
             }
+            mp_info(log, "HW-uploading to %s\n", mp_imgfmt_to_name(fmts[i]));
+            filters[2] = upload.f;
+            hwupload_fmt = upload.selected_sw_imgfmt;
+            fmts = &hwupload_fmt;
+            num_fmts = hwupload_fmt ? 1 : 0;
+            hw_to_sw = false;
+            upload_created = true;
+            break;
         }
-        if (!upload_created) {
+        if (!upload_created && dst_all_hw) {
+            // Nothing else can produce the hw target formats. Failing here
+            // keeps the unconvertible frames away from consumers that would
+            // blindly dereference their native handles.
             mp_err(log, "Failed to create HW uploader for format %s\n",
                    mp_imgfmt_to_name(sw_fmt));
+            goto fail;
         }
     }
 
@@ -291,8 +303,8 @@ static bool build_image_converter(struct mp_autoconvert *c, struct mp_log *log,
         force_sws_params |= !mp_image_params_equal(&imgpar, &p->imgparams);
         need_sws |= force_sws_params;
     }
-    if (!imgfmt_is_sw && dst_all_hw) {
-        // This is a hw -> hw upload, so the sw format must already be
+    if (!imgfmt_is_sw && !hw_to_sw) {
+        // This is a direct hw -> hw upload, so the sw format must already be
         // mutually understood. No conversion can be done.
         need_sws = false;
     }

filters/f_hwtransfer.c

@@ -68,6 +68,36 @@ static const struct hwmap_pairs hwmap_pairs[] = {
     {0}
 };
 
+bool mp_hwupload_probe_hw_to_hw(struct mp_hwupload *u, struct mp_image *src)
+{
+    if (!u->f || !src || !src->hwctx)
+        return false;
+    struct priv *p = u->f->priv;
+
+    bool map_images = false;
+    for (int n = 0; n < p->num_map_fmts; n++) {
+        if (src->imgfmt == p->map_fmts[n]) {
+            map_images = true;
+            break;
+        }
+    }
+
+    // Mirror hwupload_process(): same device, same pool parameters, same
+    // map or transfer choice, so the test cannot diverge from what the
+    // uploader will do with the real frames.
+    AVBufferRef *frames = NULL;
+    if (!mp_update_av_hw_frames_pool(&frames, p->av_device_ctx, p->hw_imgfmt,
+                                     src->params.hw_subfmt, src->w, src->h,
+                                     src->imgfmt == IMGFMT_CUDA))
+        return false;
+    struct mp_image *dst = map_images ? mp_av_pool_image_hw_map(frames, src)
+                                      : mp_av_pool_image_hw_upload(frames, src);
+    bool ok = !!dst;
+    talloc_free(dst);
+    av_buffer_unref(&frames);
+    return ok;
+}
+
 /**
  * @brief Find the closest supported format when hw uploading
  *

filters/f_hwtransfer.h

@@ -18,6 +18,13 @@ struct mp_hwupload {
 struct mp_hwupload mp_hwupload_create(struct mp_filter *parent, int hw_imgfmt,
                                        int sw_imgfmt, bool src_is_same_hw);
 
+// Whether the created uploader can move the hw frame src to its hw format
+// without going through system memory, by mapping it or by a direct libavutil
+// transfer on the GPU. Support cannot be queried, so it is tested with src
+// against the uploader's own device and pool parameters. Only valid for
+// uploaders created with src_is_same_hw=false.
+bool mp_hwupload_probe_hw_to_hw(struct mp_hwupload *u, struct mp_image *src);
+
 // A filter which downloads sw frames from hw. Ignores sw frames.
 struct mp_hwdownload {
     struct mp_filter *f;

video/mp_image_pool.c

@@ -411,6 +411,9 @@ bool mp_update_av_hw_frames_pool(struct AVBufferRef **hw_frames_ctx,
         if (format == AV_PIX_FMT_D3D11) {
             AVD3D11VAFramesContext *d3d11va_frames_ctx = hw_frames->hwctx;
             d3d11va_frames_ctx->BindFlags |= D3D11_BIND_RENDER_TARGET | D3D11_BIND_SHADER_RESOURCE;
+            // Shareable textures are cheap and allow libavutil to transfer
+            // frames directly to another device, e.g. d3d11 -> vulkan.
+            d3d11va_frames_ctx->MiscFlags |= D3D11_RESOURCE_MISC_SHARED;
         }
 #endif
 

video/out/hwdec/hwdec_cuda.c

@@ -165,6 +165,28 @@ static void cuda_uninit(struct ra_hwdec *hw)
 #undef CHECK_CU
 #define CHECK_CU(x) check_cu((mapper)->owner, (x), #x)
 
+/* Bit packed formats have no per component description the RA could give,
+ * but the same trick the d3d11va format table uses works here: the texture
+ * format carrying the layout is known by name, and both the copies and the
+ * interop move whole texels anyway. */
+static const struct ra_format *packed_plane_format(struct ra *ra, int imgfmt)
+{
+    static const char *const x2bgr10[] = {"rgb10_a2", "rgb10a2", NULL};
+    const char *const *names = NULL;
+
+    switch (imgfmt) {
+    case IMGFMT_X2BGR10:
+        names = x2bgr10;
+        break;
+    }
+    for (int i = 0; names && names[i]; i++) {
+        const struct ra_format *fmt = ra_find_named_format(ra, names[i]);
+        if (fmt)
+            return fmt;
+    }
+    return NULL;
+}
+
 static int mapper_init(struct ra_hwdec_mapper *mapper)
 {
     struct cuda_hw_priv *p_owner = mapper->owner->priv;
@@ -184,8 +206,17 @@ static int mapper_init(struct ra_hwdec_mapper *mapper)
 
     struct ra_imgfmt_desc desc;
     if (!ra_get_imgfmt_desc(mapper->ra, imgfmt, &desc)) {
-        MP_ERR(mapper, "Unsupported format: %s\n", mp_imgfmt_to_name(imgfmt));
-        return -1;
+        const struct ra_format *fmt = packed_plane_format(mapper->ra, imgfmt);
+        if (!fmt) {
+            MP_ERR(mapper, "Unsupported format: %s\n", mp_imgfmt_to_name(imgfmt));
+            return -1;
+        }
+        desc = (struct ra_imgfmt_desc) {
+            .num_planes = 1,
+            .planes     = { fmt },
+            .chroma_w   = 1,
+            .chroma_h   = 1,
+        };
     }
 
     ret = CHECK_CU(cu->cuCtxPushCurrent(p->display_ctx));

video/out/hwdec/hwdec_vulkan.c

@@ -215,10 +215,10 @@ static int mapper_init(struct ra_hwdec_mapper *mapper)
 
     mp_image_set_params(&p->layout, &mapper->dst_params);
 
-    struct ra_imgfmt_desc desc = {0};
-    if (!ra_get_imgfmt_desc(mapper->ra, mapper->dst_params.imgfmt, &desc))
-        return -1;
-
+    /* No RA format description check: the mapper does not use it, a Vulkan
+     * hwdec device only exists on libplacebo RAs (vulkan_init requires one),
+     * and those map planes by their Vulkan format, which works for formats
+     * the description cannot express, like the bit packed IMGFMT_X2BGR10. */
     return 0;
 }