vmware/xorg: Properly detect overlay support

src/gallium/winsys/drm/vmware/xorg/vmw_driver.h

@@ -73,6 +73,8 @@ void vmw_video_stop_all(ScrnInfoPtr pScrn, struct vmw_driver *vmw);
  * vmw_ioctl.c
  */
 
+int vmw_ioctl_supports_overlay(struct vmw_driver *vmw);
+
 int vmw_ioctl_cursor_bypass(struct vmw_driver *vmw, int xhot, int yhot);
 
 struct vmw_dma_buffer * vmw_ioctl_buffer_create(struct vmw_driver *vmw,

src/gallium/winsys/drm/vmware/xorg/vmw_ioctl.c

@@ -56,6 +56,37 @@ struct vmw_dma_buffer
     uint32_t size;
 };
 
+static int
+vmw_ioctl_get_param(struct vmw_driver *vmw, uint32_t param, uint64_t *out)
+{
+    struct drm_vmw_getparam_arg gp_arg;
+    int ret;
+
+    memset(&gp_arg, 0, sizeof(gp_arg));
+    gp_arg.param = param;
+    ret = drmCommandWriteRead(vmw->fd, DRM_VMW_GET_PARAM,
+	    &gp_arg, sizeof(gp_arg));
+
+    if (ret == 0) {
+	*out = gp_arg.value;
+    }
+
+    return ret;
+}
+
+int
+vmw_ioctl_supports_overlay(struct vmw_driver *vmw)
+{
+    uint64_t value;
+    int ret;
+
+    ret = vmw_ioctl_get_param(vmw, DRM_VMW_PARAM_OVERLAY_IOCTL, &value);
+    if (ret)
+	return ret;
+
+    return value ? 0 : -ENOSYS;
+}
+
 int
 vmw_ioctl_cursor_bypass(struct vmw_driver *vmw, int xhot, int yhot)
 {

src/gallium/winsys/drm/vmware/xorg/vmw_video.c

@@ -276,6 +276,11 @@ vmw_video_init(ScrnInfoPtr pScrn, struct vmw_driver *vmw)
 
     debug_printf("%s: enter\n", __func__);
 
+    if (vmw_ioctl_supports_overlay(vmw) != 0) {
+        debug_printf("No overlay ioctl support\n");
+        return FALSE;
+    }
+
     numAdaptors = xf86XVListGenericAdaptors(pScrn, &overlayAdaptors);
 
     newAdaptor = vmw_video_init_adaptor(pScrn, vmw);