Changeset 79817 in vbox

Timestamp:

Jul 16, 2019 5:56:18 PM (5 years ago)

Author:

vboxsync

Message:

Additions/3D/win/VBoxGL: fix for r132149: check if the surface format is supported only for render targets. bugref:9483

File:

• trunk/src/VBox/Additions/3D/win/VBoxGL/GaDrvEnvKMT.cpp (modified) (2 diffs)

trunk/src/VBox/Additions/3D/win/VBoxGL/GaDrvEnvKMT.cpp

@@ -406 +406 @@
     uint32_t                          cbReq;
 
-    /* First check if the format is supported. */
-    D3DDDIFORMAT const ddiFormat = svgaToD3DDDIFormat((SVGA3dSurfaceFormat)pCreateParms->format);
-    if (ddiFormat == D3DDDIFMT_UNKNOWN)
-        return -1;
-
     /* Size of the SVGA request data */
     cbReq = sizeof(GASURFCREATE) + cSizes * sizeof(GASURFSIZE);
@@ -448 +443 @@
         if (pCreateParms->flags & SVGA3D_SURFACE_HINT_RENDERTARGET)
         {
-            GAWDDMSURFACEINFO *pSurfaceInfo = (GAWDDMSURFACEINFO *)malloc(sizeof(GAWDDMSURFACEINFO));
-            if (pSurfaceInfo)
+            /* First check if the format is supported. */
+            D3DDDIFORMAT const ddiFormat = svgaToD3DDDIFormat((SVGA3dSurfaceFormat)pCreateParms->format);
+            if (ddiFormat != D3DDDIFMT_UNKNOWN)
             {
-                memset(pSurfaceInfo, 0, sizeof(GAWDDMSURFACEINFO));
-
-                VBOXWDDM_ALLOCINFO wddmAllocInfo;
-                memset(&wddmAllocInfo, 0, sizeof(wddmAllocInfo));
-
-                wddmAllocInfo.enmType             = VBOXWDDM_ALLOC_TYPE_UMD_RC_GENERIC;
-                wddmAllocInfo.fFlags.RenderTarget = 1;
-                wddmAllocInfo.hSharedHandle       = 0;
-                wddmAllocInfo.hostID              = pData->u32Sid;
-                wddmAllocInfo.SurfDesc.slicePitch = 0;
-                wddmAllocInfo.SurfDesc.depth      = paSizes[0].cDepth;
-                wddmAllocInfo.SurfDesc.width      = paSizes[0].cWidth;
-                wddmAllocInfo.SurfDesc.height     = paSizes[0].cHeight;
-                wddmAllocInfo.SurfDesc.format     = ddiFormat;
-                wddmAllocInfo.SurfDesc.VidPnSourceId = 0;
-                wddmAllocInfo.SurfDesc.bpp        = vboxWddmCalcBitsPerPixel(wddmAllocInfo.SurfDesc.format);
-                wddmAllocInfo.SurfDesc.pitch      = vboxWddmCalcPitch(wddmAllocInfo.SurfDesc.width,
-                                                                      wddmAllocInfo.SurfDesc.format);
-                wddmAllocInfo.SurfDesc.cbSize     = vboxWddmCalcSize(wddmAllocInfo.SurfDesc.pitch,
-                                                                     wddmAllocInfo.SurfDesc.height,
-                                                                     wddmAllocInfo.SurfDesc.format);
-                wddmAllocInfo.SurfDesc.d3dWidth   = vboxWddmCalcWidthForPitch(wddmAllocInfo.SurfDesc.pitch,
-                                                                              wddmAllocInfo.SurfDesc.format);
-
-                D3DDDI_ALLOCATIONINFO AllocationInfo;
-                memset(&AllocationInfo, 0, sizeof(AllocationInfo));
-                // AllocationInfo.hAllocation           = NULL;
-                // AllocationInfo.pSystemMem            = NULL;
-                AllocationInfo.pPrivateDriverData    = &wddmAllocInfo;
-                AllocationInfo.PrivateDriverDataSize = sizeof(wddmAllocInfo);
-
-                D3DKMT_CREATEALLOCATION CreateAllocation;
-                memset(&CreateAllocation, 0, sizeof(CreateAllocation));
-                CreateAllocation.hDevice         = pThis->mKmtCallbacks.hDevice;
-                CreateAllocation.NumAllocations  = 1;
-                CreateAllocation.pAllocationInfo = &AllocationInfo;
-
-                Status = pThis->mKmtCallbacks.d3dkmt->pfnD3DKMTCreateAllocation(&CreateAllocation);
-                if (Status == STATUS_SUCCESS)
+                GAWDDMSURFACEINFO *pSurfaceInfo = (GAWDDMSURFACEINFO *)malloc(sizeof(GAWDDMSURFACEINFO));
+                if (pSurfaceInfo)
                 {
-                    pSurfaceInfo->Core.Key    = pData->u32Sid;
-                    pSurfaceInfo->hAllocation = AllocationInfo.hAllocation;
-                    if (!RTAvlU32Insert(&pThis->mSurfaceTree, &pSurfaceInfo->Core))
+                    memset(pSurfaceInfo, 0, sizeof(GAWDDMSURFACEINFO));
+
+                    VBOXWDDM_ALLOCINFO wddmAllocInfo;
+                    memset(&wddmAllocInfo, 0, sizeof(wddmAllocInfo));
+
+                    wddmAllocInfo.enmType             = VBOXWDDM_ALLOC_TYPE_UMD_RC_GENERIC;
+                    wddmAllocInfo.fFlags.RenderTarget = 1;
+                    wddmAllocInfo.hSharedHandle       = 0;
+                    wddmAllocInfo.hostID              = pData->u32Sid;
+                    wddmAllocInfo.SurfDesc.slicePitch = 0;
+                    wddmAllocInfo.SurfDesc.depth      = paSizes[0].cDepth;
+                    wddmAllocInfo.SurfDesc.width      = paSizes[0].cWidth;
+                    wddmAllocInfo.SurfDesc.height     = paSizes[0].cHeight;
+                    wddmAllocInfo.SurfDesc.format     = ddiFormat;
+                    wddmAllocInfo.SurfDesc.VidPnSourceId = 0;
+                    wddmAllocInfo.SurfDesc.bpp        = vboxWddmCalcBitsPerPixel(wddmAllocInfo.SurfDesc.format);
+                    wddmAllocInfo.SurfDesc.pitch      = vboxWddmCalcPitch(wddmAllocInfo.SurfDesc.width,
+                                                                          wddmAllocInfo.SurfDesc.format);
+                    wddmAllocInfo.SurfDesc.cbSize     = vboxWddmCalcSize(wddmAllocInfo.SurfDesc.pitch,
+                                                                         wddmAllocInfo.SurfDesc.height,
+                                                                         wddmAllocInfo.SurfDesc.format);
+                    wddmAllocInfo.SurfDesc.d3dWidth   = vboxWddmCalcWidthForPitch(wddmAllocInfo.SurfDesc.pitch,
+                                                                                  wddmAllocInfo.SurfDesc.format);
+
+                    D3DDDI_ALLOCATIONINFO AllocationInfo;
+                    memset(&AllocationInfo, 0, sizeof(AllocationInfo));
+                    // AllocationInfo.hAllocation           = NULL;
+                    // AllocationInfo.pSystemMem            = NULL;
+                    AllocationInfo.pPrivateDriverData    = &wddmAllocInfo;
+                    AllocationInfo.PrivateDriverDataSize = sizeof(wddmAllocInfo);
+
+                    D3DKMT_CREATEALLOCATION CreateAllocation;
+                    memset(&CreateAllocation, 0, sizeof(CreateAllocation));
+                    CreateAllocation.hDevice         = pThis->mKmtCallbacks.hDevice;
+                    CreateAllocation.NumAllocations  = 1;
+                    CreateAllocation.pAllocationInfo = &AllocationInfo;
+
+                    Status = pThis->mKmtCallbacks.d3dkmt->pfnD3DKMTCreateAllocation(&CreateAllocation);
+                    if (Status == STATUS_SUCCESS)
                     {
-                        Status = STATUS_NOT_SUPPORTED;
+                        pSurfaceInfo->Core.Key    = pData->u32Sid;
+                        pSurfaceInfo->hAllocation = AllocationInfo.hAllocation;
+                        if (!RTAvlU32Insert(&pThis->mSurfaceTree, &pSurfaceInfo->Core))
+                        {
+                            Status = STATUS_NOT_SUPPORTED;
+                        }
+                    }
+
+                    if (Status != STATUS_SUCCESS)
+                    {
+                        free(pSurfaceInfo);
                     }
                 }
-
-                if (Status != STATUS_SUCCESS)
+                else
                 {
-                    free(pSurfaceInfo);
+                    Status = STATUS_NOT_SUPPORTED;
                 }
             }
             else
             {
+                /* Unsupported render target format. */
                 Status = STATUS_NOT_SUPPORTED;
             }