Windows: use the GUI subsystem, attach to console

This is necessary to start mpv without forcing a console window,
but also breaks console usability. A workaround is to call mpv
from a wrapper process that uses the console subsystem and helps
redirecting the standard streams and WriteConsole output to where
they belong.

osdep/io.c

@@ -327,4 +327,26 @@ char *mp_getenv(const char *name)
     return NULL;
 }
 
+void mp_attach_console(void)
+{
+    if (AttachConsole(ATTACH_PARENT_PROCESS)) {
+        // We have been started by something with a console window.
+        // Redirect output streams to that console's low-level handles,
+        // so we can actually use WriteConsole later on.
+
+        int hConHandle;
+        intptr_t hStdio;
+
+        hStdio = (intptr_t)GetStdHandle(STD_OUTPUT_HANDLE);
+        hConHandle = _open_osfhandle(hStdio, _O_TEXT);
+        *stdout = *_fdopen(hConHandle, "w");
+        setvbuf(stdout, NULL, _IONBF, 0);
+
+        hStdio = (intptr_t)GetStdHandle(STD_ERROR_HANDLE);
+        hConHandle = _open_osfhandle(hStdio, _O_TEXT);
+        *stderr = *_fdopen(hConHandle, "w");
+        setvbuf(stderr, NULL, _IONBF, 0);
+    }
+}
+
 #endif // __MINGW32__

osdep/io.h

@@ -77,6 +77,7 @@ struct dirent *mp_readdir(DIR *dir);
 int mp_closedir(DIR *dir);
 int mp_mkdir(const char *path, int mode);
 char *mp_getenv(const char *name);
+void mp_attach_console(void);
 
 // NOTE: stat is not overridden with mp_stat, because MinGW-w64 defines it as
 //       macro.

player/main.c

@@ -249,6 +249,7 @@ static void osdep_preinit(int *p_argc, char ***p_argv)
         talloc_enable_leak_report();
 
 #ifdef __MINGW32__
+    mp_attach_console();
     mp_get_converted_argv(p_argc, p_argv);
 #endif
 

waftools/detections/compiler.py

@@ -36,7 +36,7 @@ def __add_mingw_flags__(ctx):
     ctx.env.CFLAGS += ['-DBYTE_ORDER=1234']
     ctx.env.CFLAGS += ['-DLITLE_ENDIAN=1234']
     ctx.env.CFLAGS += ['-DBIG_ENDIAN=4321']
-    ctx.env.LAST_LINKFLAGS += ['-mconsole']
+    ctx.env.LAST_LINKFLAGS += ['-mwindows']
 
 def __add_cygwin_flags__(ctx):
     ctx.env.CFLAGS += ['-mwin32']