wayland: properly use tranche_formats when getting compositor formats

It's a bit roundabout, but we were doing this incorrectly before. The
format table alone isn't good enough because it is possible the the
compositor may advertise different formats for a specific device which
is a subset of the format table. What we need to do is add formats based
on the array in tranche_formats. Note that it is possible for the
compositor to send multiple tranches (was not able to simulate this but
it's allowed by the protocol). For mpv, we only care about the very
first one which is supposed to be the most preferred one. The compositor
can also send the entire chain of events (main device, format_table,
tranches, etc.) all over again. We handle this in the wayland code, but
handling this in mpv's core code isn't done. e.g. say a format in use
was suddenly no longer supported. We ideally should do a full reconfig
in this case, but that gets complicated and is pretty niche so save that
for another day. Multiple GPUs isn't taken into account either. We just
pick the first one. Not strictly correct, but close enough for us.
Fixes #14544.

video/out/wayland_common.c

@@ -1363,6 +1363,12 @@ static void format_table(void *data,
 {
     struct vo_wayland_state *wl = data;
 
+    if (wl->compositor_format_size) {
+        munmap(wl->compositor_format_map, wl->compositor_format_size);
+        wl->compositor_format_map = NULL;
+        wl->compositor_format_size = 0;
+    }
+
     void *map = mmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0);
     close(fd);
 
@@ -1377,32 +1383,62 @@ static void main_device(void *data,
                         struct wl_array *device)
 {
     struct vo_wayland_state *wl = data;
+    wl->add_tranche = true;
 
-    // Despite being an array, the protocol specifically states there can only be
-    // one main device so break as soon as we get one.
-    dev_t *id;
-    wl_array_for_each(id, device) {
-        memcpy(&wl->main_device_id, id, sizeof(dev_t));
-        break;
-    }
-    get_gpu_drm_formats(wl);
 }
 
 static void tranche_done(void *data,
                          struct zwp_linux_dmabuf_feedback_v1 *zwp_linux_dmabuf_feedback_v1)
 {
+    struct vo_wayland_state *wl = data;
+    wl->add_tranche = false;
 }
 
 static void tranche_target_device(void *data,
                                   struct zwp_linux_dmabuf_feedback_v1 *zwp_linux_dmabuf_feedback_v1,
                                   struct wl_array *device)
 {
+    struct vo_wayland_state *wl = data;
+    // Only use the first tranche device we get.
+    if (wl->add_tranche) {
+        dev_t *id;
+        wl_array_for_each(id, device) {
+            memcpy(&wl->target_device_id, id, sizeof(dev_t));
+            break;
+        }
+        get_gpu_drm_formats(wl);
+    }
 }
 
 static void tranche_formats(void *data,
                             struct zwp_linux_dmabuf_feedback_v1 *zwp_linux_dmabuf_feedback_v1,
                             struct wl_array *indices)
 {
+    struct vo_wayland_state *wl = data;
+
+    // Only grab formats from the first tranche and ignore the rest.
+    if (!wl->add_tranche)
+        return;
+
+    // Should never happen.
+    if (!wl->compositor_format_map) {
+        MP_WARN(wl, "Compositor did not send a format and modifier table!\n");
+        return;
+    }
+
+    const compositor_format *formats = wl->compositor_format_map;
+    MP_RESIZE_ARRAY(wl, wl->compositor_formats, indices->size);
+    wl->num_compositor_formats = 0;
+    uint16_t *index;
+    wl_array_for_each(index, indices) {
+        MP_TARRAY_APPEND(wl, wl->compositor_formats, wl->num_compositor_formats,
+                         (struct drm_format) {
+                            formats[*index].format,
+                            formats[*index].modifier,
+                        });
+        MP_DBG(wl, "Compositor supports drm format: '%s(%016" PRIx64 ")'\n",
+               mp_tag_str(formats[*index].format), formats[*index].modifier);
+    }
 }
 
 static void tranche_flags(void *data,
@@ -1799,7 +1835,7 @@ static void get_gpu_drm_formats(struct vo_wayland_state *wl)
     drmModePlaneRes *res = NULL;
     drmModePlane *plane = NULL;
 
-    if (drmGetDeviceFromDevId(wl->main_device_id, 0, &device) != 0) {
+    if (drmGetDeviceFromDevId(wl->target_device_id, 0, &device) != 0) {
         MP_WARN(wl, "Unable to get drm device from device id: %s\n", mp_strerror(errno));
         goto done;
     }

video/out/wayland_common.h

@@ -30,6 +30,11 @@ typedef struct {
     uint64_t modifier;
 } compositor_format;
 
+struct drm_format {
+    uint32_t format;
+    uint64_t modifier;
+};
+
 struct vo_wayland_state {
     struct m_config_cache   *opts_cache;
     struct mp_log           *log;
@@ -102,11 +107,14 @@ struct vo_wayland_state {
     struct zwp_idle_inhibitor_v1 *idle_inhibitor;
 
     /* linux-dmabuf */
-    dev_t main_device_id;
+    dev_t target_device_id;
     struct zwp_linux_dmabuf_v1 *dmabuf;
     struct zwp_linux_dmabuf_feedback_v1 *dmabuf_feedback;
+    bool add_tranche;
     compositor_format *compositor_format_map;
     uint32_t compositor_format_size;
+    struct drm_format *compositor_formats;
+    int num_compositor_formats;
     uint32_t *gpu_formats;
     int num_gpu_formats;
 

video/out/wldmabuf/ra_wldmabuf.c

@@ -32,7 +32,7 @@ bool ra_compatible_format(struct ra *ra, int imgfmt, uint32_t drm_format, uint64
 {
     struct priv *p = ra->priv;
     struct vo_wayland_state *wl = p->vo->wl;
-    const compositor_format *formats = wl->compositor_format_map;
+    struct drm_format *formats = wl->compositor_formats;
 
 
     // If we were able to make the DRM query, filter out the GPU formats.
@@ -50,7 +50,7 @@ bool ra_compatible_format(struct ra *ra, int imgfmt, uint32_t drm_format, uint64
     }
 
     // Always check if the compositor supports the format.
-    for (int i = 0; i < wl->compositor_format_size / sizeof(compositor_format); i++) {
+    for (int i = 0; i < wl->num_compositor_formats; ++i) {
         if (drm_format == formats[i].format && modifier == formats[i].modifier)
             return true;
     }