Changeset 68812 in vbox

Timestamp:

Sep 21, 2017 3:43:12 PM (7 years ago)

Author:

vboxsync

Message:

VBoxGuest/solaris: Use ldi_ioctl instead of VBoxGuestIDC. ldi_open_by_name and ldi_close calls the open & close entry points with the FKLYR flag set, in which case we'll be creating a kernel session rather than a user one. Adjusted vgdrvSolarisIOCtl to handle kernel sessions.

Location:

trunk/src/VBox/Additions/common/VBoxGuest

Files:

• VBoxGuest-solaris.c (modified) (8 diffs)
• lib/VBoxGuestR0LibIdc-solaris.cpp (modified) (3 diffs)

trunk/src/VBox/Additions/common/VBoxGuest/VBoxGuest-solaris.c

@@ -483 +483 @@
 /**
  * User context entry points
- */
-static int vgdrvSolarisOpen(dev_t *pDev, int fFlag, int fType, cred_t *pCred)
+ *
+ * @remarks fFlags are the flags passed to open() or to ldi_open_by_name.  In
+ *          the latter case the FKLYR flag is added to indicate that the caller
+ *          is a kernel component rather than user land.
+ */
+static int vgdrvSolarisOpen(dev_t *pDev, int fFlags, int fType, cred_t *pCred)
 {
     int                 rc;
@@ -517 +521 @@
      * Create a new session.
      */
-    rc = VGDrvCommonCreateUserSession(&g_DevExt, &pSession);
+    if (!(fFlags & FKLYR))
+        rc = VGDrvCommonCreateUserSession(&g_DevExt, &pSession);
+    else
+        rc = VGDrvCommonCreateKernelSession(&g_DevExt, &pSession);
     if (RT_SUCCESS(rc))
     {
-        pState->pvProcRef = proc_ref();
+        if (!(fFlags & FKLYR))
+            pState->pvProcRef = proc_ref();
+        else
+            pState->pvProcRef = NULL;
         pState->pSession = pSession;
         *pDev = makedevice(getmajor(*pDev), iOpenInstance);
@@ -547 +557 @@
     }
 
-    proc_unref(pState->pvProcRef);
+    if (pState->pvProcRef != NULL)
+    {
+        proc_unref(pState->pvProcRef);
+        pState->pvProcRef = NULL;
+    }
     pSession = pState->pSession;
     pState->pSession = NULL;
@@ -561 +575 @@
      * Close the session.
      */
-    VGDrvCommonCloseSession(&g_DevExt, pSession);
+    if (pSession)
+        VGDrvCommonCloseSession(&g_DevExt, pSession);
     return 0;
 }
@@ -591 +606 @@
     return 0;
 }
+
+
+/** @def IOCPARM_LEN
+ * Gets the length from the ioctl number.
+ * This is normally defined by sys/ioccom.h on BSD systems...
+ */
+#ifndef IOCPARM_LEN
+# define IOCPARM_LEN(x)     ( ((x) >> 16) & IOCPARM_MASK )
+#endif
 
 
@@ -633 +657 @@
     }
 
+    /*
+     * It's kind of simple if this is a kernel session, take slow path if user land.
+     */
+    if (pSession->R0Process == NIL_RTR0PROCESS)
+    {
+        if (IOCPARM_LEN(iCmd) == sizeof(VBGLREQHDR))
+        {
+            PVBGLREQHDR pHdr = (PVBGLREQHDR)iArgs;
+            int rc;
+            if (iCmd != VBGL_IOCTL_IDC_DISCONNECT)
+                rc =VGDrvCommonIoCtl(iCmd, &g_DevExt, pSession, pHdr, RT_MAX(pHdr->cbIn, pHdr->cbOut));
+            else
+            {
+                pState->pSession = NULL;
+                rc = VGDrvCommonIoCtl(iCmd, &g_DevExt, pSession, pHdr, RT_MAX(pHdr->cbIn, pHdr->cbOut));
+                if (RT_FAILURE(rc))
+                    pState->pSession = pSession;
+            }
+            return rc;
+        }
+    }
+
     return vgdrvSolarisIOCtlSlow(pSession, iCmd, Mode, iArgs);
 }
-
-
-/** @def IOCPARM_LEN
- * Gets the length from the ioctl number.
- * This is normally defined by sys/ioccom.h on BSD systems...
- */
-#ifndef IOCPARM_LEN
-# define IOCPARM_LEN(x)     ( ((x) >> 16) & IOCPARM_MASK )
-#endif
 
 
@@ -755 +792 @@
 
 
+#if 0
 /**
  * @note This code is duplicated on other platforms with variations, so please
@@ -797 +835 @@
     return rc;
 }
-
+#endif
 
 

trunk/src/VBox/Additions/common/VBoxGuest/lib/VBoxGuestR0LibIdc-solaris.cpp

@@ -41 +41 @@
     ldi_handle_t hDev   = NULL;
     ldi_ident_t  hIdent = ldi_ident_from_anon();
-    /* Note! ldi_open_by_name actually opens the device and ends up creating a useless user session.
-             Wonder if there is any way to detect ldi_open_by_name and do I/O controls via hDev... */
-    int rc = ldi_open_by_name(VBOXGUEST_DEVICE_NAME, FREAD, kcred, &hDev, hIdent);
+    int rc = ldi_open_by_name((char *)VBOXGUEST_DEVICE_NAME, FREAD, kcred, &hDev, hIdent);
     ldi_ident_release(hIdent);
     if (rc == 0)
     {
-        rc = VBoxGuestIDC(NULL, VBGL_IOCTL_IDC_CONNECT, &pReq->Hdr, sizeof(*pReq));
+        pHandle->s.hDev = hDev;
+        rc = VbglR0IdcCallRaw(pHandle, VBGL_IOCTL_IDC_CONNECT, &pReq->Hdr, sizeof(*pReq));
         if (RT_SUCCESS(rc) && RT_SUCCESS(pReq->Hdr.rc))
-        {
-            pHandle->s.hDev = hDev;
             return VINF_SUCCESS;
-        }
         ldi_close(hDev, FREAD, kcred);
     }
@@ -64 +60 @@
 int VBOXCALL vbglR0IdcNativeClose(PVBGLIDCHANDLE pHandle, PVBGLIOCIDCDISCONNECT pReq)
 {
-    int rc = VBoxGuestIDC(pHandle->s.pvSession, VBGL_IOCTL_IDC_DISCONNECT, &pReq->Hdr, sizeof(*pReq));
+    int rc = VbglR0IdcCallRaw(pHandle, VBGL_IOCTL_IDC_DISCONNECT, &pReq->Hdr, sizeof(*pReq));
     if (RT_SUCCESS(rc) && RT_SUCCESS(pReq->Hdr.rc))
     {
@@ -85 +81 @@
 DECLR0VBGL(int) VbglR0IdcCallRaw(PVBGLIDCHANDLE pHandle, uintptr_t uReq, PVBGLREQHDR pReqHdr, uint32_t cbReq)
 {
+#if 0
     return VBoxGuestIDC(pHandle->s.pvSession, uReq, pReqHdr, cbReq);
+#else
+    int iIgn;
+    int rc = ldi_ioctl(pHandle->s.hDev, uReq, (intptr_t)pReqHdr, FKIOCTL | FNATIVE, kcred, &iIgn);
+    if (rc == 0)
+        return VINF_SUCCESS;
+    return RTErrConvertFromErrno(rc);
+#endif
 }