From c5bc784da611ee929e0da7c20a410e03e6540613 Mon Sep 17 00:00:00 2001
From: Oliver Kurth <okurth@vmware.com>
Date: Fri, 26 Jan 2018 15:04:36 -0800
Subject: [PATCH] Set X11 as the backend for gtk3 of open-vm-tools

Open-vm-tools defaults to gtk3.  For distros like Fedora 27, Wayland is
the default display server.  With no restriction on the backend, plugins
will try to use Wayland as the backend of Gtk+3.  As a result,
gdk_display_get_default() returns a Wayland display;
gdk_display_get_default_group() also returns a Wayland window.  Applying
GDK_WINDOW_XID() on the Wayland window will result in the crash reported
on recent Linux releases that default to a Wayland display server.

As X11-specific code is widely used in tools plugins, a migratation to
Wayland in the short term is not possible.  So, as a compromised solution,
plugins can be forced to run on XWayland, which is the compatible mode
of Wayland for legacy X11 clients.

gdk_set_allowed_backends() only applies when flag GTK3 is defined, and Gtk
version cwgreater than 3.10.
---
 open-vm-tools/services/plugins/desktopEvents/x11Lock.c   | 12 +++++++++++-
 open-vm-tools/services/plugins/dndcp/copyPasteDnDX11.cpp | 14 +++++++++++++-
 2 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/open-vm-tools/services/plugins/desktopEvents/x11Lock.c b/open-vm-tools/services/plugins/desktopEvents/x11Lock.c
index 572234ac..613b6946 100644
--- a/open-vm-tools/services/plugins/desktopEvents/x11Lock.c
+++ b/open-vm-tools/services/plugins/desktopEvents/x11Lock.c
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2010-2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 2010-2018 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU Lesser General Public License as published
@@ -364,6 +364,16 @@ X11Lock_Init(ToolsAppCtx *ctx,
    g_set_prgname(VMUSER_TITLE);
    argv[0] = VMUSER_TITLE;
 
+#if GTK_MAJOR_VERSION > 3 || (GTK_MAJOR_VERSION == 3 && GTK_MINOR_VERSION >= 10)
+   /*
+    * On recent distros, Wayland is the default display server. If the obtained
+    * display or window is a wayland one, applying X11 specific functions on them
+    * will result in crashes. Before migrating the X11 specific code to Wayland,
+    * force using X11 as the backend of Gtk+3. gdk_set_allowed_backends() is
+    * introduced since Gtk+3.10 and Wayland is supported from Gtk+3.10.
+    */
+   gdk_set_allowed_backends("x11");
+#endif
    /* XXX: is calling gtk_init() multiple times safe? */
    gtk_init(&argc, (char ***) &argv);
 
diff --git a/open-vm-tools/services/plugins/dndcp/copyPasteDnDX11.cpp b/open-vm-tools/services/plugins/dndcp/copyPasteDnDX11.cpp
index fffdc402..4c2f3d8d 100644
--- a/open-vm-tools/services/plugins/dndcp/copyPasteDnDX11.cpp
+++ b/open-vm-tools/services/plugins/dndcp/copyPasteDnDX11.cpp
@@ -1,5 +1,5 @@
 /*********************************************************
- * Copyright (C) 2010-2016 VMware, Inc. All rights reserved.
+ * Copyright (C) 2010-2018 VMware, Inc. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU Lesser General Public License as published
@@ -211,6 +211,18 @@ gboolean
 CopyPasteDnDX11::Init(ToolsAppCtx *ctx)
 {
    TRACE_CALL();
+
+#if GTK_MAJOR_VERSION > 3 || (GTK_MAJOR_VERSION == 3 && GTK_MINOR_VERSION >= 10)
+   /*
+    * On recent distros, Wayland is the default display server. If the obtained
+    * display or window is a wayland one, applying X11 specific functions on them
+    * will result in crashes. Before migrating the X11 specific code to Wayland,
+    * force using X11 as the backend of Gtk+3. gdk_set_allowed_backends() is
+    * introduced since Gtk+3.10 and Wayland is supported from Gtk+3.10.
+    */
+   gdk_set_allowed_backends("x11");
+#endif
+
    CopyPasteDnDWrapper *wrapper = CopyPasteDnDWrapper::GetInstance();
 
    ASSERT(ctx);
