x11: separate window creation and configuration

This gets rid of an old hack, VOFLAG_HIDDEN. Although handling of it has
been sane for a while, it used to cause much pain, and is still
unintuitive and weird even today.

The main reason for this hack is that OpenGL selects a X11 Visual for
you, and you're supposed to use this Visual when creating the X window
for the OpenGL context. Which means the X window can't be created early
in the common X11 init code, but the OpenGL code needs to do something
before that. API-wise you need separate functions for X11 init and X11
window creation. The VOFLAG_HIDDEN hack conflated window creation and
the entrypoint for resizing on video resolution change into one
function, vo_x11_config_vo_window(). This required all platform backends
to handle this flag, even if they didn't need this mechanism.

Wayland still uses this for minor reasons (alpha support?), so the
wayland backend must be changed before the flag can be entirely removed.

video/out/opengl/x11.c

@@ -240,7 +240,8 @@ static bool config_window_x11(struct MPGLContext *ctx, int flags)
     glXGetFBConfigAttrib(vo->x11->display, fbc, GLX_GREEN_SIZE, &ctx->depth_g);
     glXGetFBConfigAttrib(vo->x11->display, fbc, GLX_BLUE_SIZE, &ctx->depth_b);
 
-    vo_x11_config_vo_window(vo, glx_ctx->vinfo, flags | VOFLAG_HIDDEN, "gl");
+    if (!vo_x11_create_vo_window(vo, glx_ctx->vinfo, "gl"))
+        return false;
 
     bool success = false;
     if (!(flags & VOFLAG_GLES)) {
@@ -265,8 +266,7 @@ static int glx_init(struct MPGLContext *ctx, int vo_flags)
 
 static int glx_reconfig(struct MPGLContext *ctx, int flags)
 {
-    struct glx_context *glx_ctx = ctx->priv;
-    vo_x11_config_vo_window(ctx->vo, glx_ctx->vinfo, flags, "gl");
+    vo_x11_config_vo_window(ctx->vo);
     return 0;
 }
 

video/out/opengl/x11egl.c

@@ -142,7 +142,10 @@ static int mpegl_init(struct MPGLContext *ctx, int flags)
         goto uninit;
     }
 
-    vo_x11_config_vo_window(vo, vi, flags | VOFLAG_HIDDEN, "gl");
+    if (!vo_x11_create_vo_window(vo, vi, "gl")) {
+        XFree(vi);
+        goto uninit;
+    }
 
     XFree(vi);
 
@@ -166,7 +169,7 @@ uninit:
 
 static int mpegl_reconfig(struct MPGLContext *ctx, int flags)
 {
-    vo_x11_config_vo_window(ctx->vo, NULL, flags, "gl");
+    vo_x11_config_vo_window(ctx->vo);
     return 0;
 }
 

video/out/vo_vaapi.c

@@ -158,7 +158,7 @@ static int reconfig(struct vo *vo, struct mp_image_params *params, int flags)
 
     free_video_specific(p);
 
-    vo_x11_config_vo_window(vo, NULL, flags, "vaapi");
+    vo_x11_config_vo_window(vo);
 
     if (params->imgfmt != IMGFMT_VAAPI) {
         if (!alloc_swdec_surfaces(p, params->w, params->h, params->imgfmt))
@@ -577,6 +577,9 @@ static int preinit(struct vo *vo)
     if (!vo_x11_init(vo))
         goto fail;
 
+    if (!vo_x11_create_vo_window(vo, NULL, "vaapi"))
+        goto fail;
+
     p->display = vaGetDisplay(vo->x11->display);
     if (!p->display)
         goto fail;

video/out/vo_vdpau.c

@@ -524,7 +524,7 @@ static int reconfig(struct vo *vo, struct mp_image_params *params, int flags)
 
     free_video_specific(vo);
 
-    vo_x11_config_vo_window(vo, NULL, flags, "vdpau");
+    vo_x11_config_vo_window(vo);
 
     if (initialize_vdpau_objects(vo) < 0)
         return -1;
@@ -1040,6 +1040,11 @@ static int preinit(struct vo *vo)
     if (!vo_x11_init(vo))
         return -1;
 
+    if (!vo_x11_create_vo_window(vo, NULL, "vdpau")) {
+        vo_x11_uninit(vo);
+        return -1;
+    }
+
     vc->mpvdp = mp_vdpau_create_device_x11(vo->log, vo->x11->display, false);
     if (!vc->mpvdp) {
         vo_x11_uninit(vo);

video/out/vo_x11.c

@@ -196,7 +196,7 @@ static int reconfig(struct vo *vo, struct mp_image_params *fmt, int flags)
 
     p->sws->src = *fmt;
 
-    vo_x11_config_vo_window(vo, NULL, flags, "x11");
+    vo_x11_config_vo_window(vo);
 
     if (!resize(vo))
         return -1;
@@ -402,7 +402,9 @@ static int preinit(struct vo *vo)
 
     MP_VERBOSE(vo, "selected visual: %d\n", (int)p->vinfo.visualid);
 
-    vo_x11_config_vo_window(vo, &p->vinfo, VOFLAG_HIDDEN, "x11");
+    if (!vo_x11_create_vo_window(vo, &p->vinfo, "x11"))
+        goto error;
+
     p->gc = XCreateGC(x11->display, x11->window, 0, NULL);
     return 0;
 

video/out/vo_xv.c

@@ -489,7 +489,7 @@ static int reconfig(struct vo *vo, struct mp_image_params *params, int flags)
     if (!ctx->xv_format)
         return -1;
 
-    vo_x11_config_vo_window(vo, NULL, flags, "xv");
+    vo_x11_config_vo_window(vo);
 
     if (!ctx->f_gc && !ctx->vo_gc) {
         ctx->f_gc = XCreateGC(x11->display, x11->window, 0, 0);
@@ -761,6 +761,9 @@ static int preinit(struct vo *vo)
     if (!vo_x11_init(vo))
         return -1;
 
+    if (!vo_x11_create_vo_window(vo, NULL, "xv"))
+        goto error;
+
     struct vo_x11_state *x11 = vo->x11;
 
     /* check for Xvideo extension */

video/out/x11_common.c

@@ -1477,21 +1477,40 @@ static void wait_until_mapped(struct vo *vo)
     }
 }
 
-/* Create and setup a window suitable for display
- * vis: Visual to use for creating the window (NULL for default)
- * x, y: position of window (might be ignored)
- * width, height: size of window
- * flags: flags for window creation (VOFLAG_*)
- * classname: name to use for the X11 classhint
- *
- * If the window already exists, it just moves and resizes it.
- */
-void vo_x11_config_vo_window(struct vo *vo, XVisualInfo *vis, int flags,
+// Create the X11 window. There is only 1, and it must be created before
+// vo_x11_config_vo_window() is called. vis can be NULL for default.
+bool vo_x11_create_vo_window(struct vo *vo, XVisualInfo *vis,
                              const char *classname)
+{
+    struct vo_x11_state *x11 = vo->x11;
+    assert(!x11->window);
+
+    if (x11->parent) {
+        if (x11->parent == x11->rootwin) {
+            x11->window = x11->rootwin;
+            x11->pseudo_mapped = true;
+            XSelectInput(x11->display, x11->window, StructureNotifyMask);
+        } else {
+            XSelectInput(x11->display, x11->parent, StructureNotifyMask);
+        }
+    }
+    if (x11->window == None) {
+        vo_x11_create_window(vo, vis, (struct mp_rect){.x1 = 320, .y1 = 200 });
+        vo_x11_classhint(vo, x11->window, classname);
+        x11->window_hidden = true;
+    }
+
+    return !!x11->window;
+}
+
+// Resize the window (e.g. new file, or video resolution change)
+void vo_x11_config_vo_window(struct vo *vo)
 {
     struct mp_vo_opts *opts = vo->opts;
     struct vo_x11_state *x11 = vo->x11;
 
+    assert(x11->window);
+
     vo_x11_update_screeninfo(vo);
 
     struct vo_win_geometry geo;
@@ -1501,25 +1520,9 @@ void vo_x11_config_vo_window(struct vo *vo, XVisualInfo *vis, int flags,
     struct mp_rect rc = geo.win;
 
     if (x11->parent) {
-        if (x11->parent == x11->rootwin) {
-            x11->window = x11->rootwin;
-            x11->pseudo_mapped = true;
-            XSelectInput(x11->display, x11->window, StructureNotifyMask);
-        } else {
-            XSelectInput(x11->display, x11->parent, StructureNotifyMask);
-        }
         vo_x11_update_geometry(vo);
         rc = (struct mp_rect){0, 0, RC_W(x11->winrc), RC_H(x11->winrc)};
     }
-    if (x11->window == None) {
-        vo_x11_create_window(vo, vis, rc);
-        vo_x11_classhint(vo, x11->window, classname);
-        x11->window_hidden = true;
-        x11->winrc = geo.win;
-    }
-
-    if (flags & VOFLAG_HIDDEN)
-        return;
 
     bool reset_size = x11->old_dw != RC_W(rc) || x11->old_dh != RC_H(rc);
     x11->old_dw = RC_W(rc);

video/out/x11_common.h

@@ -125,8 +125,9 @@ int vo_x11_init(struct vo *vo);
 void vo_x11_uninit(struct vo *vo);
 int vo_x11_check_events(struct vo *vo);
 bool vo_x11_screen_is_composited(struct vo *vo);
-void vo_x11_config_vo_window(struct vo *vo, XVisualInfo *vis, int flags,
+bool vo_x11_create_vo_window(struct vo *vo, XVisualInfo *vis,
                              const char *classname);
+void vo_x11_config_vo_window(struct vo *vo);
 int vo_x11_control(struct vo *vo, int *events, int request, void *arg);
 
 #endif /* MPLAYER_X11_COMMON_H */