Index: /trunk/include/VBox/pdmcommon.h
===================================================================
--- /trunk/include/VBox/pdmcommon.h	(revision 24730)
+++ /trunk/include/VBox/pdmcommon.h	(revision 24730)
@@ -0,0 +1,103 @@
+/** @file
+ * PDM - Pluggable Device Manager, Common Definitions & Types. (VMM)
+ */
+
+/*
+ * Copyright (C) 2006-2007 Sun Microsystems, Inc.
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ *
+ * The contents of this file may alternatively be used under the terms
+ * of the Common Development and Distribution License Version 1.0
+ * (CDDL) only, as it comes in the "COPYING.CDDL" file of the
+ * VirtualBox OSE distribution, in which case the provisions of the
+ * CDDL are applicable instead of those of the GPL.
+ *
+ * You may elect to license modified versions of this file under the
+ * terms and conditions of either the GPL or the CDDL or both.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
+ * Clara, CA 95054 USA or visit http://www.sun.com if you need
+ * additional information or have any questions.
+ */
+
+#ifndef ___VBox_pdmcommon_h
+#define ___VBox_pdmcommon_h
+
+/** @defgroup grp_pdm_common    Common Definitions & Types
+ * @ingroup grp_pdm
+ *
+ * Not all the types here are "common", they are here to work around header
+ * ordering issues.
+ *
+ * @{
+ */
+
+/** PDM Attach/Detach Callback Flags.
+ * Used by PDMDeviceAttach, PDMDeviceDetach, PDMDriverAttach, PDMDriverDetach,
+ * FNPDMDEVATTACH, FNPDMDEVDETACH, FNPDMDRVATTACH, FNPDMDRVDETACH and
+ * FNPDMDRVCONSTRUCT.
+ @{ */
+/** The attach/detach command is not a hotplug event. */
+#define PDM_TACH_FLAGS_NOT_HOT_PLUG     RT_BIT_32(0)
+/* @} */
+
+
+/**
+ * Is asynchronous handling of suspend or power off notification completed?
+ *
+ * This is called to check whether the USB device has quiesced.  Don't deadlock.
+ * Avoid blocking.  Do NOT wait for anything.
+ *
+ * @returns true if done, false if more work to be done.
+ *
+ * @param   pUsbIns             The USB device instance.
+ *
+ * @thread  EMT(0)
+ */
+typedef DECLCALLBACK(bool) FNPDMUSBASYNCNOTIFY(PPDMUSBINS pUsbIns);
+/** Pointer to a FNPDMUSBASYNCNOTIFY. */
+typedef FNPDMUSBASYNCNOTIFY *PFNPDMUSBASYNCNOTIFY;
+
+/**
+ * Is asynchronous handling of suspend or power off notification completed?
+ *
+ * This is called to check whether the device has quiesced.  Don't deadlock.
+ * Avoid blocking.  Do NOT wait for anything.
+ *
+ * @returns true if done, false if more work to be done.
+ *
+ * @param   pDevIns             The device instance.
+ *
+ * @thread  EMT(0)
+ */
+typedef DECLCALLBACK(bool) FNPDMDEVASYNCNOTIFY(PPDMDEVINS pDevIns);
+/** Pointer to a FNPDMDEVASYNCNOTIFY. */
+typedef FNPDMDEVASYNCNOTIFY *PFNPDMDEVASYNCNOTIFY;
+
+/**
+ * Is asynchronous handling of suspend or power off notification completed?
+ *
+ * This is called to check whether the driver has quiesced.  Don't deadlock.
+ * Avoid blocking.  Do NOT wait for anything.
+ *
+ * @returns true if done, false if more work to be done.
+ *
+ * @param   pDrvIns             The driver instance.
+ *
+ * @thread  EMT(0)
+ */
+typedef DECLCALLBACK(bool) FNPDMDRVASYNCNOTIFY(PPDMDRVINS pDrvIns);
+/** Pointer to a FNPDMDRVASYNCNOTIFY. */
+typedef FNPDMDRVASYNCNOTIFY *PFNPDMDRVASYNCNOTIFY;
+
+/** @} */
+
+#endif
+
Index: /trunk/include/VBox/pdmdev.h
===================================================================
--- /trunk/include/VBox/pdmdev.h	(revision 24729)
+++ /trunk/include/VBox/pdmdev.h	(revision 24730)
@@ -36,5 +36,5 @@
 #include <VBox/pdmifs.h>
 #include <VBox/pdmins.h>
-#include <VBox/pdmdevdrv.h>
+#include <VBox/pdmcommon.h>
 #include <VBox/iom.h>
 #include <VBox/tm.h>
@@ -151,4 +151,5 @@
  * @returns VBox status.
  * @param   pDevIns     The device instance data.
+ * @thread  EMT(0)
  */
 typedef DECLCALLBACK(void)  FNPDMDEVSUSPEND(PPDMDEVINS pDevIns);
@@ -170,4 +171,5 @@
  *
  * @param   pDevIns     The device instance data.
+ * @thread  EMT(0)
  */
 typedef DECLCALLBACK(void)   FNPDMDEVPOWEROFF(PPDMDEVINS pDevIns);
@@ -406,5 +408,5 @@
 #define PDM_IRQ_LEVEL_HIGH                      RT_BIT(0)
 /** Deassert the IRQ (can assume value 0). */
-#define PDM_IRQ_LEVEL_LOW           0
+#define PDM_IRQ_LEVEL_LOW                       0
 /** flip-flop - assert and then deassert it again immediately. */
 #define PDM_IRQ_LEVEL_FLIP_FLOP                 (RT_BIT(1) | PDM_IRQ_LEVEL_HIGH)
@@ -2405,4 +2407,17 @@
     DECLR3CALLBACKMEMBER(int, pfnPhysGCPtr2GCPhys, (PPDMDEVINS pDevIns, RTGCPTR GCPtr, PRTGCPHYS pGCPhys));
 
+    /**
+     * Set up asynchronous handling of a suspend or power off notification.
+     *
+     * This shall only be called when getting the notification.  It must be called
+     * for each one.
+     *
+     * @returns VBox status code.
+     * @param   pDevIns             The device instance.
+     * @param   pfnAsyncNotify      The callback.
+     * @thread  EMT(0)
+     */
+    DECLR3CALLBACKMEMBER(int, pfnSetAsyncNotification, (PPDMDEVINS pDevIns, PFNPDMDEVASYNCNOTIFY pfnAsyncNotify));
+
     /** Space reserved for future members.
      * @{ */
@@ -3716,4 +3731,12 @@
 
 /**
+ * @copydoc PDMDEVHLPR3::pfnSetAsyncNotification
+ */
+DECLINLINE(int) PDMDevHlpSetAsyncNotification(PPDMDEVINS pDevIns, PFNPDMDEVASYNCNOTIFY pfnAsyncNotify)
+{
+    return pDevIns->pDevHlpR3->pfnSetAsyncNotification(pDevIns, pfnAsyncNotify);
+}
+
+/**
  * @copydoc PDMDEVHLPR3::pfnVMState
  */
Index: unk/include/VBox/pdmdevdrv.h
===================================================================
--- /trunk/include/VBox/pdmdevdrv.h	(revision 24729)
+++ 	(revision )
@@ -1,50 +1,0 @@
-/** @file
- * PDM - Pluggable Device Manager, Common Device & Driver Definitions. (VMM)
- */
-
-/*
- * Copyright (C) 2006-2007 Sun Microsystems, Inc.
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- *
- * The contents of this file may alternatively be used under the terms
- * of the Common Development and Distribution License Version 1.0
- * (CDDL) only, as it comes in the "COPYING.CDDL" file of the
- * VirtualBox OSE distribution, in which case the provisions of the
- * CDDL are applicable instead of those of the GPL.
- *
- * You may elect to license modified versions of this file under the
- * terms and conditions of either the GPL or the CDDL or both.
- *
- * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
- * Clara, CA 95054 USA or visit http://www.sun.com if you need
- * additional information or have any questions.
- */
-
-#ifndef ___VBox_pdmdevdrv_h
-#define ___VBox_pdmdevdrv_h
-
-/** @defgroup grp_pdm_devdrv    Common Device & Driver Definitions
- * @ingroup grp_pdm
- * @{
- */
-
-/** PDM Attach/Detach Callback Flags.
- * Used by PDMDeviceAttach, PDMDeviceDetach, PDMDriverAttach, PDMDriverDetach,
- * FNPDMDEVATTACH, FNPDMDEVDETACH, FNPDMDRVATTACH, FNPDMDRVDETACH and 
- * FNPDMDRVCONSTRUCT. 
- @{ */ 
-/** The attach/detach command is not a hotplug event. */
-#define PDM_TACH_FLAGS_NOT_HOT_PLUG     RT_BIT_32(0)
-/* @} */
-
-/** @} */
-
-#endif
-
Index: /trunk/include/VBox/pdmdrv.h
===================================================================
--- /trunk/include/VBox/pdmdrv.h	(revision 24729)
+++ /trunk/include/VBox/pdmdrv.h	(revision 24730)
@@ -36,5 +36,5 @@
 #include <VBox/pdmifs.h>
 #include <VBox/pdmins.h>
-#include <VBox/pdmdevdrv.h>
+#include <VBox/pdmcommon.h>
 #include <VBox/tm.h>
 #include <VBox/ssm.h>
@@ -698,4 +698,17 @@
 
     /**
+     * Set up asynchronous handling of a suspend or power off notification.
+     *
+     * This shall only be called when getting the notification.  It must be called
+     * for each one.
+     *
+     * @returns VBox status code.
+     * @param   pDrvIns             The driver instance.
+     * @param   pfnAsyncNotify      The callback.
+     * @thread  EMT(0)
+     */
+    DECLR3CALLBACKMEMBER(int, pfnSetAsyncNotification, (PPDMDRVINS pDrvIns, PFNPDMDRVASYNCNOTIFY pfnAsyncNotify));
+
+    /**
      * Creates a PDM thread.
      *
Index: /trunk/include/VBox/pdmusb.h
===================================================================
--- /trunk/include/VBox/pdmusb.h	(revision 24729)
+++ /trunk/include/VBox/pdmusb.h	(revision 24730)
@@ -35,4 +35,5 @@
 #include <VBox/pdmthread.h>
 #include <VBox/pdmifs.h>
+#include <VBox/pdmcommon.h>
 #include <VBox/tm.h>
 #include <VBox/ssm.h>
@@ -374,5 +375,7 @@
 /** @} */
 
+
 #ifdef IN_RING3
+
 /**
  * PDM USB Device API.
@@ -570,4 +573,17 @@
      */
     DECLR3CALLBACKMEMBER(VMSTATE, pfnVMState, (PPDMUSBINS pUsbIns));
+
+    /**
+     * Set up asynchronous handling of a suspend or power off notification.
+     *
+     * This shall only be called when getting the notification.  It must be called
+     * for each one.
+     *
+     * @returns VBox status code.
+     * @param   pUSBIns             The USB device instance.
+     * @param   pfnAsyncNotify      The callback.
+     * @thread  EMT(0)
+     */
+    DECLR3CALLBACKMEMBER(int, pfnSetAsyncNotification, (PPDMUSBINS pUSbIns, PFNPDMUSBASYNCNOTIFY pfnAsyncNotify));
 
     /** Just a safety precaution. */
@@ -607,5 +623,5 @@
         PDMUSBINSINT            s;
 #endif
-        uint8_t                 padding[HC_ARCH_BITS == 32 ? 64 : 96];
+        uint8_t                 padding[HC_ARCH_BITS == 32 ? 96 : 128];
     } Internal;
 
Index: /trunk/src/VBox/VMM/PDM.cpp
===================================================================
--- /trunk/src/VBox/VMM/PDM.cpp	(revision 24729)
+++ /trunk/src/VBox/VMM/PDM.cpp	(revision 24730)
@@ -912,4 +912,80 @@
 
 /**
+ * Worker for PDMR3PowerOn that deals with one driver.
+ *
+ * @param   pDrvIns             The driver instance.
+ * @param   pszDeviceName       The parent device name.
+ * @param   iDevInstance        The parent device instance number.
+ * @param   iLun                The parent LUN number.
+ */
+DECLINLINE(bool) pdmR3PowerOnDrv(PPDMDRVINS pDrvIns, const char *pszDeviceName, uint32_t iDevInstance, uint32_t iLun)
+{
+    Assert(pDrvIns->Internal.s.fVMSuspended);
+    if (pDrvIns->pDrvReg->pfnPowerOn)
+    {
+        LogFlow(("PDMR3PowerOn: Notifying - driver '%s'/%d on LUN#%d of device '%s'/%d\n",
+                 pDrvIns->pDrvReg->szDriverName, pDrvIns->iInstance, iLun, pszDeviceName, iDevInstance));
+        int rc = VINF_SUCCESS; pDrvIns->pDrvReg->pfnPowerOn(pDrvIns);
+        if (RT_FAILURE(rc))
+        {
+            LogRel(("PDMR3PowerOn: driver '%s'/%d on LUN#%d of device '%s'/%d -> %Rrc\n",
+                    pDrvIns->pDrvReg->szDriverName, pDrvIns->iInstance, iLun, pszDeviceName, iDevInstance, rc));
+            return rc;
+        }
+    }
+    pDrvIns->Internal.s.fVMSuspended = false;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Worker for PDMR3PowerOn that deals with one USB device instance.
+ *
+ * @returns VBox status code.
+ * @param   pUsbIns             The USB device instance.
+ */
+DECLINLINE(int) pdmR3PowerOnUsb(PPDMUSBINS pUsbIns)
+{
+    Assert(pUsbIns->Internal.s.fVMSuspended);
+    if (pUsbIns->pUsbReg->pfnVMPowerOn)
+    {
+        LogFlow(("PDMR3PowerOn: Notifying - device '%s'/%d\n", pUsbIns->pUsbReg->szDeviceName, pUsbIns->iInstance));
+        int rc = VINF_SUCCESS; pUsbIns->pUsbReg->pfnVMPowerOn(pUsbIns);
+        if (RT_FAILURE(rc))
+        {
+            LogRel(("PDMR3PowerOn: device '%s'/%d -> %Rrc\n", pUsbIns->pUsbReg->szDeviceName, pUsbIns->iInstance, rc));
+            return rc;
+        }
+    }
+    pUsbIns->Internal.s.fVMSuspended = false;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Worker for PDMR3PowerOn that deals with one device instance.
+ *
+ * @returns VBox status code.
+ * @param   pDevIns             The device instance.
+ */
+DECLINLINE(int) pdmR3PowerOnDev(PPDMDEVINS pDevIns)
+{
+    Assert(pDevIns->Internal.s.fIntFlags & PDMDEVINSINT_FLAGS_SUSPENDED);
+    if (pDevIns->pDevReg->pfnPowerOn)
+    {
+        LogFlow(("PDMR3PowerOn: Notifying - device '%s'/%d\n", pDevIns->pDevReg->szDeviceName, pDevIns->iInstance));
+        int rc = VINF_SUCCESS; pDevIns->pDevReg->pfnPowerOn(pDevIns);
+        if (RT_FAILURE(rc))
+        {
+            LogRel(("PDMR3PowerOn: device '%s'/%d -> %Rrc\n", pDevIns->pDevReg->szDeviceName, pDevIns->iInstance, rc));
+            return rc;
+        }
+    }
+    pDevIns->Internal.s.fIntFlags &= ~PDMDEVINSINT_FLAGS_SUSPENDED;
+    return VINF_SUCCESS;
+}
+
+
+/**
  * This function will notify all the devices and their
  * attached drivers about the VM now being powered on.
@@ -922,45 +998,25 @@
 
     /*
-     * Iterate the device instances.
-     * The attached drivers are processed first.
-     */
-    for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3)
-    {
-        for (PPDMLUN pLun = pDevIns->Internal.s.pLunsR3; pLun; pLun = pLun->pNext)
-            /** @todo Inverse the order here? */
-            for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
-                if (pDrvIns->pDrvReg->pfnPowerOn)
-                {
-                    LogFlow(("PDMR3PowerOn: Notifying - driver '%s'/%d on LUN#%d of device '%s'/%d\n",
-                             pDrvIns->pDrvReg->szDriverName, pDrvIns->iInstance, pLun->iLun, pDevIns->pDevReg->szDeviceName, pDevIns->iInstance));
-                    pDrvIns->pDrvReg->pfnPowerOn(pDrvIns);
-                }
-
-        if (pDevIns->pDevReg->pfnPowerOn)
-        {
-            LogFlow(("PDMR3PowerOn: Notifying - device '%s'/%d\n",
-                     pDevIns->pDevReg->szDeviceName, pDevIns->iInstance));
-            pDevIns->pDevReg->pfnPowerOn(pDevIns);
-        }
+     * Iterate thru the device instances and USB device instances,
+     * processing the drivers associated with those.
+     */
+    int rc = VINF_SUCCESS;
+    for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances;  pDevIns && RT_SUCCESS(rc);  pDevIns = pDevIns->Internal.s.pNextR3)
+    {
+        for (PPDMLUN pLun = pDevIns->Internal.s.pLunsR3;  pLun && RT_SUCCESS(rc);  pLun = pLun->pNext)
+            for (PPDMDRVINS pDrvIns = pLun->pTop;  pDrvIns && RT_SUCCESS(rc);  pDrvIns = pDrvIns->Internal.s.pDown)
+                rc = pdmR3PowerOnDrv(pDrvIns, pDevIns->pDevReg->szDeviceName, pDevIns->iInstance, pLun->iLun);
+        if (RT_SUCCESS(rc))
+            rc = pdmR3PowerOnDev(pDevIns);
     }
 
 #ifdef VBOX_WITH_USB
-    for (PPDMUSBINS pUsbIns = pVM->pdm.s.pUsbInstances; pUsbIns; pUsbIns = pUsbIns->Internal.s.pNext)
-    {
-        for (PPDMLUN pLun = pUsbIns->Internal.s.pLuns; pLun; pLun = pLun->pNext)
-            for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
-                if (pDrvIns->pDrvReg->pfnPowerOn)
-                {
-                    LogFlow(("PDMR3PowerOn: Notifying - driver '%s'/%d on LUN#%d of usb device '%s'/%d\n",
-                             pDrvIns->pDrvReg->szDriverName, pDrvIns->iInstance, pLun->iLun, pUsbIns->pUsbReg->szDeviceName, pUsbIns->iInstance));
-                    pDrvIns->pDrvReg->pfnPowerOn(pDrvIns);
-                }
-
-        if (pUsbIns->pUsbReg->pfnVMPowerOn)
-        {
-            LogFlow(("PDMR3PowerOn: Notifying - device '%s'/%d\n",
-                     pUsbIns->pUsbReg->szDeviceName, pUsbIns->iInstance));
-            pUsbIns->pUsbReg->pfnVMPowerOn(pUsbIns);
-        }
+    for (PPDMUSBINS pUsbIns = pVM->pdm.s.pUsbInstances;  pUsbIns && RT_SUCCESS(rc);  pUsbIns = pUsbIns->Internal.s.pNext)
+    {
+        for (PPDMLUN pLun = pUsbIns->Internal.s.pLuns;  pLun && RT_SUCCESS(rc);  pLun = pLun->pNext)
+            for (PPDMDRVINS pDrvIns = pLun->pTop;  pDrvIns && RT_SUCCESS(rc);  pDrvIns = pDrvIns->Internal.s.pDown)
+                rc = pdmR3PowerOnDrv(pDrvIns, pUsbIns->pUsbReg->szDeviceName, pUsbIns->iInstance, pLun->iLun);
+        if (RT_SUCCESS(rc))
+            rc = pdmR3PowerOnUsb(pUsbIns);
     }
 #endif
@@ -969,7 +1025,15 @@
      * Resume all threads.
      */
-    pdmR3ThreadResumeAll(pVM);
-
-    LogFlow(("PDMR3PowerOn: returns void\n"));
+    if (RT_SUCCESS(rc))
+        pdmR3ThreadResumeAll(pVM);
+
+    /*
+     * On failure, clean up via PDMR3Suspend.
+     */
+    if (RT_FAILURE(rc))
+        PDMR3Suspend(pVM);
+
+    LogFlow(("PDMR3PowerOn: returns %Rrc\n", rc));
+    return /*rc*/;
 }
 
@@ -1050,8 +1114,115 @@
 
 /**
- * This function will notify all the devices and their
- * attached drivers about the VM now being suspended.
- *
- * @param   pVM     VM Handle.
+ * Worker for PDMR3Suspend that deals with one driver.
+ *
+ * @param   pDrvIns             The driver instance.
+ * @param   pcAsync             The asynchronous suspend notification counter.
+ * @param   pszDeviceName       The parent device name.
+ * @param   iDevInstance        The parent device instance number.
+ * @param   iLun                The parent LUN number.
+ */
+DECLINLINE(bool) pdmR3SuspendDrv(PPDMDRVINS pDrvIns, unsigned *pcAsync,
+                                 const char *pszDeviceName, uint32_t iDevInstance, uint32_t iLun)
+{
+    if (!pDrvIns->Internal.s.fVMSuspended)
+    {
+        pDrvIns->Internal.s.fVMSuspended = true;
+        if (pDrvIns->pDrvReg->pfnSuspend)
+        {
+            if (!pDrvIns->Internal.s.pfnAsyncNotify)
+            {
+                LogFlow(("PDMR3Suspend: Notifying - driver '%s'/%d on LUN#%d of device '%s'/%d\n",
+                         pDrvIns->pDrvReg->szDriverName, pDrvIns->iInstance, iLun, pszDeviceName, iDevInstance));
+                pDrvIns->pDrvReg->pfnSuspend(pDrvIns);
+            }
+            else if (pDrvIns->Internal.s.pfnAsyncNotify(pDrvIns))
+            {
+                pDrvIns->Internal.s.pfnAsyncNotify = false;
+                LogFlow(("PDMR3Suspend: Async notification completed - driver '%s'/%d on LUN#%d of device '%s'/%d\n",
+                         pDrvIns->pDrvReg->szDriverName, pDrvIns->iInstance, iLun, pszDeviceName, iDevInstance));
+            }
+            if (pDrvIns->Internal.s.pfnAsyncNotify)
+            {
+                pDrvIns->Internal.s.fVMSuspended = false;
+                (*pcAsync)++;
+                return false;
+            }
+        }
+    }
+    return true;
+}
+
+
+/**
+ * Worker for PDMR3Suspend that deals with one USB device instance.
+ *
+ * @param   pUsbIns             The USB device instance.
+ * @param   pcAsync             The asynchronous suspend notification counter.
+ */
+DECLINLINE(void) pdmR3SuspendUsb(PPDMUSBINS pUsbIns, unsigned *pcAsync)
+{
+    if (!pUsbIns->Internal.s.fVMSuspended)
+    {
+        pUsbIns->Internal.s.fVMSuspended = true;
+        if (pUsbIns->pUsbReg->pfnVMSuspend)
+        {
+            if (!pUsbIns->Internal.s.pfnAsyncNotify)
+            {
+                LogFlow(("PDMR3Suspend: Notifying - device '%s'/%d\n", pUsbIns->pUsbReg->szDeviceName, pUsbIns->iInstance));
+                pUsbIns->pUsbReg->pfnVMSuspend(pUsbIns);
+            }
+            else if (pUsbIns->Internal.s.pfnAsyncNotify(pUsbIns))
+            {
+                LogFlow(("PDMR3Suspend: Async notification completed - device '%s'/%d\n", pUsbIns->pUsbReg->szDeviceName, pUsbIns->iInstance));
+                pUsbIns->Internal.s.pfnAsyncNotify = NULL;
+            }
+            if (pUsbIns->Internal.s.pfnAsyncNotify)
+            {
+                pUsbIns->Internal.s.fVMSuspended = false;
+                (*pcAsync)++;
+            }
+        }
+    }
+}
+
+
+/**
+ * Worker for PDMR3Suspend that deals with one device instance.
+ *
+ * @param   pDevIns             The device instance.
+ * @param   pcAsync             The asynchronous suspend notification counter.
+ */
+DECLINLINE(void) pdmR3SuspendDev(PPDMDEVINS pDevIns, unsigned *pcAsync)
+{
+    if (!(pDevIns->Internal.s.fIntFlags & PDMDEVINSINT_FLAGS_SUSPENDED))
+    {
+        pDevIns->Internal.s.fIntFlags |= PDMDEVINSINT_FLAGS_SUSPENDED;
+        if (pDevIns->pDevReg->pfnSuspend)
+        {
+            if (!pDevIns->Internal.s.pfnAsyncNotify)
+            {
+                LogFlow(("PDMR3Suspend: Notifying - device '%s'/%d\n", pDevIns->pDevReg->szDeviceName, pDevIns->iInstance));
+                pDevIns->pDevReg->pfnSuspend(pDevIns);
+            }
+            else if (pDevIns->Internal.s.pfnAsyncNotify(pDevIns))
+            {
+                LogFlow(("PDMR3Suspend: Async notification completed - device '%s'/%d\n", pDevIns->pDevReg->szDeviceName, pDevIns->iInstance));
+                pDevIns->Internal.s.pfnAsyncNotify = NULL;
+            }
+            if (pDevIns->Internal.s.pfnAsyncNotify)
+            {
+                pDevIns->Internal.s.fIntFlags &= ~PDMDEVINSINT_FLAGS_SUSPENDED;
+                (*pcAsync)++;
+            }
+        }
+    }
+}
+
+
+/**
+ * This function will notify all the devices and their attached drivers about
+ * the VM now being suspended.
+ *
+ * @param   pVM     The VM Handle.
  * @thread  EMT(0)
  */
@@ -1062,60 +1233,67 @@
 
     /*
-     * Iterate the device instances.
-     * The attached drivers are processed first.
-     */
-    for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3)
+     * The outer loop repeats until there are no more async requests.
+     *
+     * Note! We depend on the suspended indicators to be in the desired state
+     *       and we do not reset them before starting because this allows
+     *       PDMR3PowerOn and PDMR3Resume to use PDMR3Suspend for cleaning up
+     *       on failure.
+     */
+    unsigned cAsync;
+    for (;;)
     {
         /*
-         * Some devices need to be notified first that the VM is suspended to ensure that that there are no pending
-         * requests from the guest which are still processed. Calling the drivers before these requests are finished
-         * might lead to errors otherwise. One example is the SATA controller which might still have I/O requests
-         * pending. But DrvVD sets the files into readonly mode and every request will fail then.
+         * Iterate thru the device instances and USB device instances,
+         * processing the drivers associated with those.
+         *
+         * The attached drivers are normally processed first.  Some devices
+         * (like DevAHCI) though needs to be notified before the drivers so
+         * that it doesn't kick off any new requests after the drivers stopped
+         * taking any. (DrvVD changes to read-only in this particular case.)
          */
-        if (pDevIns->pDevReg->pfnSuspend && (pDevIns->pDevReg->fFlags & PDM_DEVREG_FLAGS_FIRST_SUSPEND_NOTIFICATION))
-        {
-            LogFlow(("PDMR3Suspend: Notifying - device '%s'/%d\n",
-                     pDevIns->pDevReg->szDeviceName, pDevIns->iInstance));
-            pDevIns->pDevReg->pfnSuspend(pDevIns);
-        }
-
-        for (PPDMLUN pLun = pDevIns->Internal.s.pLunsR3; pLun; pLun = pLun->pNext)
-            for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
-                if (pDrvIns->pDrvReg->pfnSuspend)
-                {
-                    LogFlow(("PDMR3Suspend: Notifying - driver '%s'/%d on LUN#%d of device '%s'/%d\n",
-                             pDrvIns->pDrvReg->szDriverName, pDrvIns->iInstance, pLun->iLun, pDevIns->pDevReg->szDeviceName, pDevIns->iInstance));
-                    pDrvIns->pDrvReg->pfnSuspend(pDrvIns);
-                }
-
-        /* Don't call the suspend notification again if it was already called. */
-        if (pDevIns->pDevReg->pfnSuspend && !(pDevIns->pDevReg->fFlags & PDM_DEVREG_FLAGS_FIRST_SUSPEND_NOTIFICATION))
-        {
-            LogFlow(("PDMR3Suspend: Notifying - device '%s'/%d\n",
-                     pDevIns->pDevReg->szDeviceName, pDevIns->iInstance));
-            pDevIns->pDevReg->pfnSuspend(pDevIns);
-        }
-    }
+        cAsync = 0;
+        for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3)
+        {
+            unsigned const cAsyncStart = cAsync;
+
+            if (pDevIns->pDevReg->fFlags & PDM_DEVREG_FLAGS_FIRST_SUSPEND_NOTIFICATION)
+                pdmR3SuspendDev(pDevIns, &cAsync);
+
+            if (cAsync == cAsyncStart)
+                for (PPDMLUN pLun = pDevIns->Internal.s.pLunsR3; pLun; pLun = pLun->pNext)
+                    for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
+                        if (!pdmR3SuspendDrv(pDrvIns, &cAsync, pDevIns->pDevReg->szDeviceName, pDevIns->iInstance, pLun->iLun))
+                            break;
+
+            if (    cAsync == cAsyncStart
+                && !(pDevIns->pDevReg->fFlags & PDM_DEVREG_FLAGS_FIRST_SUSPEND_NOTIFICATION))
+                pdmR3SuspendDev(pDevIns, &cAsync);
+        }
 
 #ifdef VBOX_WITH_USB
-    for (PPDMUSBINS pUsbIns = pVM->pdm.s.pUsbInstances; pUsbIns; pUsbIns = pUsbIns->Internal.s.pNext)
-    {
-        for (PPDMLUN pLun = pUsbIns->Internal.s.pLuns; pLun; pLun = pLun->pNext)
-            for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
-                if (pDrvIns->pDrvReg->pfnSuspend)
-                {
-                    LogFlow(("PDMR3Suspend: Notifying - driver '%s'/%d on LUN#%d of usb device '%s'/%d\n",
-                             pDrvIns->pDrvReg->szDriverName, pDrvIns->iInstance, pLun->iLun, pUsbIns->pUsbReg->szDeviceName, pUsbIns->iInstance));
-                    pDrvIns->pDrvReg->pfnSuspend(pDrvIns);
-                }
-
-        if (pUsbIns->pUsbReg->pfnVMSuspend)
-        {
-            LogFlow(("PDMR3Suspend: Notifying - device '%s'/%d\n",
-                     pUsbIns->pUsbReg->szDeviceName, pUsbIns->iInstance));
-            pUsbIns->pUsbReg->pfnVMSuspend(pUsbIns);
-        }
-    }
+        for (PPDMUSBINS pUsbIns = pVM->pdm.s.pUsbInstances; pUsbIns; pUsbIns = pUsbIns->Internal.s.pNext)
+        {
+            unsigned const cAsyncStart = cAsync;
+
+            for (PPDMLUN pLun = pUsbIns->Internal.s.pLuns; pLun; pLun = pLun->pNext)
+                for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
+                    if (!pdmR3SuspendDrv(pDrvIns, &cAsync, pUsbIns->pUsbReg->szDeviceName, pUsbIns->iInstance, pLun->iLun))
+                        break;
+
+            if (cAsync == cAsyncStart)
+                pdmR3SuspendUsb(pUsbIns, &cAsync);
+        }
 #endif
+        if (!cAsync)
+            break;
+
+        /*
+         * Process requests.
+         */
+        /** @todo This is utterly nuts and completely unsafe... will get back to it in a
+         *        bit I hope... */
+        int rc = VMR3ReqProcessU(pVM->pUVM, VMCPUID_ANY);
+        AssertReleaseRC(rc == VINF_SUCCESS);
+    }
 
     /*
@@ -1125,4 +1303,80 @@
 
     LogFlow(("PDMR3Suspend: returns void\n"));
+}
+
+
+/**
+ * Worker for PDMR3Resume that deals with one driver.
+ *
+ * @param   pDrvIns             The driver instance.
+ * @param   pszDeviceName       The parent device name.
+ * @param   iDevInstance        The parent device instance number.
+ * @param   iLun                The parent LUN number.
+ */
+DECLINLINE(bool) pdmR3ResumeDrv(PPDMDRVINS pDrvIns, const char *pszDeviceName, uint32_t iDevInstance, uint32_t iLun)
+{
+    Assert(pDrvIns->Internal.s.fVMSuspended);
+    if (pDrvIns->pDrvReg->pfnResume)
+    {
+        LogFlow(("PDMR3Resume: Notifying - driver '%s'/%d on LUN#%d of device '%s'/%d\n",
+                 pDrvIns->pDrvReg->szDriverName, pDrvIns->iInstance, iLun, pszDeviceName, iDevInstance));
+        int rc = VINF_SUCCESS; pDrvIns->pDrvReg->pfnResume(pDrvIns);
+        if (RT_FAILURE(rc))
+        {
+            LogRel(("PDMR3Resume: driver '%s'/%d on LUN#%d of device '%s'/%d -> %Rrc\n",
+                    pDrvIns->pDrvReg->szDriverName, pDrvIns->iInstance, iLun, pszDeviceName, iDevInstance, rc));
+            return rc;
+        }
+    }
+    pDrvIns->Internal.s.fVMSuspended = false;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Worker for PDMR3Resume that deals with one USB device instance.
+ *
+ * @returns VBox status code.
+ * @param   pUsbIns             The USB device instance.
+ */
+DECLINLINE(int) pdmR3ResumeUsb(PPDMUSBINS pUsbIns)
+{
+    Assert(pUsbIns->Internal.s.fVMSuspended);
+    if (pUsbIns->pUsbReg->pfnVMResume)
+    {
+        LogFlow(("PDMR3Resume: Notifying - device '%s'/%d\n", pUsbIns->pUsbReg->szDeviceName, pUsbIns->iInstance));
+        int rc = VINF_SUCCESS; pUsbIns->pUsbReg->pfnVMResume(pUsbIns);
+        if (RT_FAILURE(rc))
+        {
+            LogRel(("PDMR3Resume: device '%s'/%d -> %Rrc\n", pUsbIns->pUsbReg->szDeviceName, pUsbIns->iInstance, rc));
+            return rc;
+        }
+    }
+    pUsbIns->Internal.s.fVMSuspended = false;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Worker for PDMR3Resume that deals with one device instance.
+ *
+ * @returns VBox status code.
+ * @param   pDevIns             The device instance.
+ */
+DECLINLINE(int) pdmR3ResumeDev(PPDMDEVINS pDevIns)
+{
+    Assert(pDevIns->Internal.s.fIntFlags & PDMDEVINSINT_FLAGS_SUSPENDED);
+    if (pDevIns->pDevReg->pfnResume)
+    {
+        LogFlow(("PDMR3Resume: Notifying - device '%s'/%d\n", pDevIns->pDevReg->szDeviceName, pDevIns->iInstance));
+        int rc = VINF_SUCCESS; pDevIns->pDevReg->pfnResume(pDevIns);
+        if (RT_FAILURE(rc))
+        {
+            LogRel(("PDMR3Resume: device '%s'/%d -> %Rrc\n", pDevIns->pDevReg->szDeviceName, pDevIns->iInstance, rc));
+            return rc;
+        }
+    }
+    pDevIns->Internal.s.fIntFlags &= ~PDMDEVINSINT_FLAGS_SUSPENDED;
+    return VINF_SUCCESS;
 }
 
@@ -1139,44 +1393,25 @@
 
     /*
-     * Iterate the device instances.
-     * The attached drivers are processed first.
-     */
-    for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3)
-    {
-        for (PPDMLUN pLun = pDevIns->Internal.s.pLunsR3; pLun; pLun = pLun->pNext)
-            for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
-                if (pDrvIns->pDrvReg->pfnResume)
-                {
-                    LogFlow(("PDMR3Resume: Notifying - driver '%s'/%d on LUN#%d of device '%s'/%d\n",
-                             pDrvIns->pDrvReg->szDriverName, pDrvIns->iInstance, pLun->iLun, pDevIns->pDevReg->szDeviceName, pDevIns->iInstance));
-                    pDrvIns->pDrvReg->pfnResume(pDrvIns);
-                }
-
-        if (pDevIns->pDevReg->pfnResume)
-        {
-            LogFlow(("PDMR3Resume: Notifying - device '%s'/%d\n",
-                     pDevIns->pDevReg->szDeviceName, pDevIns->iInstance));
-            pDevIns->pDevReg->pfnResume(pDevIns);
-        }
+     * Iterate thru the device instances and USB device instances,
+     * processing the drivers associated with those.
+     */
+    int rc = VINF_SUCCESS;
+    for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances;  pDevIns && RT_SUCCESS(rc);  pDevIns = pDevIns->Internal.s.pNextR3)
+    {
+        for (PPDMLUN pLun = pDevIns->Internal.s.pLunsR3;  pLun && RT_SUCCESS(rc);  pLun    = pLun->pNext)
+            for (PPDMDRVINS pDrvIns = pLun->pTop;  pDrvIns && RT_SUCCESS(rc);  pDrvIns = pDrvIns->Internal.s.pDown)
+                rc = pdmR3ResumeDrv(pDrvIns, pDevIns->pDevReg->szDeviceName, pDevIns->iInstance, pLun->iLun);
+        if (RT_SUCCESS(rc))
+            rc = pdmR3ResumeDev(pDevIns);
     }
 
 #ifdef VBOX_WITH_USB
-    for (PPDMUSBINS pUsbIns = pVM->pdm.s.pUsbInstances; pUsbIns; pUsbIns = pUsbIns->Internal.s.pNext)
-    {
-        for (PPDMLUN pLun = pUsbIns->Internal.s.pLuns; pLun; pLun = pLun->pNext)
-            for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
-                if (pDrvIns->pDrvReg->pfnResume)
-                {
-                    LogFlow(("PDMR3Resume: Notifying - driver '%s'/%d on LUN#%d of usb device '%s'/%d\n",
-                             pDrvIns->pDrvReg->szDriverName, pDrvIns->iInstance, pLun->iLun, pUsbIns->pUsbReg->szDeviceName, pUsbIns->iInstance));
-                    pDrvIns->pDrvReg->pfnResume(pDrvIns);
-                }
-
-        if (pUsbIns->pUsbReg->pfnVMResume)
-        {
-            LogFlow(("PDMR3Resume: Notifying - device '%s'/%d\n",
-                     pUsbIns->pUsbReg->szDeviceName, pUsbIns->iInstance));
-            pUsbIns->pUsbReg->pfnVMResume(pUsbIns);
-        }
+    for (PPDMUSBINS pUsbIns = pVM->pdm.s.pUsbInstances;  pUsbIns && RT_SUCCESS(rc);  pUsbIns = pUsbIns->Internal.s.pNext)
+    {
+        for (PPDMLUN pLun = pUsbIns->Internal.s.pLuns;  pLun && RT_SUCCESS(rc);  pLun = pLun->pNext)
+            for (PPDMDRVINS pDrvIns = pLun->pTop;  pDrvIns && RT_SUCCESS(rc);  pDrvIns = pDrvIns->Internal.s.pDown)
+                rc = pdmR3ResumeDrv(pDrvIns, pUsbIns->pUsbReg->szDeviceName, pUsbIns->iInstance, pLun->iLun);
+        if (RT_SUCCESS(rc))
+            rc = pdmR3ResumeUsb(pUsbIns);
     }
 #endif
@@ -1185,7 +1420,122 @@
      * Resume all threads.
      */
-    pdmR3ThreadResumeAll(pVM);
-
-    LogFlow(("PDMR3Resume: returns void\n"));
+    if (RT_SUCCESS(rc))
+        pdmR3ThreadResumeAll(pVM);
+
+    /*
+     * On failure, clean up via PDMR3Suspend.
+     */
+    if (RT_FAILURE(rc))
+        PDMR3Suspend(pVM);
+
+    LogFlow(("PDMR3Resume: returns %Rrc\n", rc));
+    return /*rc*/;
+}
+
+
+/**
+ * Worker for PDMR3PowerOff that deals with one driver.
+ *
+ * @param   pDrvIns             The driver instance.
+ * @param   pcAsync             The asynchronous power off notification counter.
+ * @param   pszDeviceName       The parent device name.
+ * @param   iDevInstance        The parent device instance number.
+ * @param   iLun                The parent LUN number.
+ */
+DECLINLINE(bool) pdmR3PowerOffDrv(PPDMDRVINS pDrvIns, unsigned *pcAsync,
+                                  const char *pszDeviceName, uint32_t iDevInstance, uint32_t iLun)
+{
+    if (!pDrvIns->Internal.s.fVMSuspended)
+    {
+        pDrvIns->Internal.s.fVMSuspended = true;
+        if (pDrvIns->pDrvReg->pfnSuspend)
+        {
+            if (!pDrvIns->Internal.s.pfnAsyncNotify)
+            {
+                LogFlow(("PDMR3PowerOff: Notifying - driver '%s'/%d on LUN#%d of device '%s'/%d\n",
+                         pDrvIns->pDrvReg->szDriverName, pDrvIns->iInstance, iLun, pszDeviceName, iDevInstance));
+                pDrvIns->pDrvReg->pfnPowerOff(pDrvIns);
+            }
+            else if (pDrvIns->Internal.s.pfnAsyncNotify(pDrvIns))
+            {
+                pDrvIns->Internal.s.pfnAsyncNotify = false;
+                LogFlow(("PDMR3PowerOff: Async notification completed - driver '%s'/%d on LUN#%d of device '%s'/%d\n",
+                         pDrvIns->pDrvReg->szDriverName, pDrvIns->iInstance, iLun, pszDeviceName, iDevInstance));
+            }
+            if (pDrvIns->Internal.s.pfnAsyncNotify)
+            {
+                pDrvIns->Internal.s.fVMSuspended = false;
+                (*pcAsync)++;
+                return false;
+            }
+        }
+    }
+    return true;
+}
+
+
+/**
+ * Worker for PDMR3PowerOff that deals with one USB device instance.
+ *
+ * @param   pUsbIns             The USB device instance.
+ * @param   pcAsync             The asynchronous power off notification counter.
+ */
+DECLINLINE(void) pdmR3PowerOffUsb(PPDMUSBINS pUsbIns, unsigned *pcAsync)
+{
+    if (!pUsbIns->Internal.s.fVMSuspended)
+    {
+        pUsbIns->Internal.s.fVMSuspended = true;
+        if (pUsbIns->pUsbReg->pfnVMPowerOff)
+        {
+            if (!pUsbIns->Internal.s.pfnAsyncNotify)
+            {
+                LogFlow(("PDMR3PowerOff: Notifying - device '%s'/%d\n", pUsbIns->pUsbReg->szDeviceName, pUsbIns->iInstance));
+                pUsbIns->pUsbReg->pfnVMPowerOff(pUsbIns);
+            }
+            else if (pUsbIns->Internal.s.pfnAsyncNotify(pUsbIns))
+            {
+                LogFlow(("PDMR3PowerOff: Async notification completed - device '%s'/%d\n", pUsbIns->pUsbReg->szDeviceName, pUsbIns->iInstance));
+                pUsbIns->Internal.s.pfnAsyncNotify = NULL;
+            }
+            if (pUsbIns->Internal.s.pfnAsyncNotify)
+            {
+                pUsbIns->Internal.s.fVMSuspended = false;
+                (*pcAsync)++;
+            }
+        }
+    }
+}
+
+
+/**
+ * Worker for PDMR3PowerOff that deals with one device instance.
+ *
+ * @param   pDevIns             The device instance.
+ * @param   pcAsync             The asynchronous power off notification counter.
+ */
+DECLINLINE(void) pdmR3PowerOffDev(PPDMDEVINS pDevIns, unsigned *pcAsync)
+{
+    if (!(pDevIns->Internal.s.fIntFlags & PDMDEVINSINT_FLAGS_SUSPENDED))
+    {
+        pDevIns->Internal.s.fIntFlags |= PDMDEVINSINT_FLAGS_SUSPENDED;
+        if (pDevIns->pDevReg->pfnSuspend)
+        {
+            if (!pDevIns->Internal.s.pfnAsyncNotify)
+            {
+                LogFlow(("PDMR3PowerOff: Notifying - device '%s'/%d\n", pDevIns->pDevReg->szDeviceName, pDevIns->iInstance));
+                pDevIns->pDevReg->pfnPowerOff(pDevIns);
+            }
+            else if (pDevIns->Internal.s.pfnAsyncNotify(pDevIns))
+            {
+                LogFlow(("PDMR3PowerOff: Async notification completed - device '%s'/%d\n", pDevIns->pDevReg->szDeviceName, pDevIns->iInstance));
+                pDevIns->Internal.s.pfnAsyncNotify = NULL;
+            }
+            if (pDevIns->Internal.s.pfnAsyncNotify)
+            {
+                pDevIns->Internal.s.fIntFlags &= ~PDMDEVINSINT_FLAGS_SUSPENDED;
+                (*pcAsync)++;
+            }
+        }
+    }
 }
 
@@ -1202,54 +1552,62 @@
 
     /*
-     * Iterate the device instances.
-     * The attached drivers are processed first.
-     */
-    for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3)
-    {
-
-        if (pDevIns->pDevReg->pfnPowerOff && (pDevIns->pDevReg->fFlags & PDM_DEVREG_FLAGS_FIRST_POWEROFF_NOTIFICATION))
-        {
-            LogFlow(("PDMR3PowerOff: Notifying - device '%s'/%d\n",
-                     pDevIns->pDevReg->szDeviceName, pDevIns->iInstance));
-            pDevIns->pDevReg->pfnPowerOff(pDevIns);
-        }
-
-        for (PPDMLUN pLun = pDevIns->Internal.s.pLunsR3; pLun; pLun = pLun->pNext)
-            for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
-                if (pDrvIns->pDrvReg->pfnPowerOff)
-                {
-                    LogFlow(("PDMR3PowerOff: Notifying - driver '%s'/%d on LUN#%d of device '%s'/%d\n",
-                             pDrvIns->pDrvReg->szDriverName, pDrvIns->iInstance, pLun->iLun, pDevIns->pDevReg->szDeviceName, pDevIns->iInstance));
-                    pDrvIns->pDrvReg->pfnPowerOff(pDrvIns);
-                }
-
-        if (pDevIns->pDevReg->pfnPowerOff && !(pDevIns->pDevReg->fFlags & PDM_DEVREG_FLAGS_FIRST_POWEROFF_NOTIFICATION))
-        {
-            LogFlow(("PDMR3PowerOff: Notifying - device '%s'/%d\n",
-                     pDevIns->pDevReg->szDeviceName, pDevIns->iInstance));
-            pDevIns->pDevReg->pfnPowerOff(pDevIns);
-        }
-    }
+     * The outer loop repeats until there are no more async requests.
+     */
+    unsigned cAsync;
+    for (;;)
+    {
+        /*
+         * Iterate thru the device instances and USB device instances,
+         * processing the drivers associated with those.
+         *
+         * The attached drivers are normally processed first.  Some devices
+         * (like DevAHCI) though needs to be notified before the drivers so
+         * that it doesn't kick off any new requests after the drivers stopped
+         * taking any. (DrvVD changes to read-only in this particular case.)
+         */
+        cAsync = 0;
+        for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3)
+        {
+            unsigned const cAsyncStart = cAsync;
+
+            if (pDevIns->pDevReg->fFlags & PDM_DEVREG_FLAGS_FIRST_POWEROFF_NOTIFICATION)
+                pdmR3PowerOffDev(pDevIns, &cAsync);
+
+            if (cAsync == cAsyncStart)
+                for (PPDMLUN pLun = pDevIns->Internal.s.pLunsR3; pLun; pLun = pLun->pNext)
+                    for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
+                        if (!pdmR3PowerOffDrv(pDrvIns, &cAsync, pDevIns->pDevReg->szDeviceName, pDevIns->iInstance, pLun->iLun))
+                            break;
+
+            if (    cAsync == cAsyncStart
+                && !(pDevIns->pDevReg->fFlags & PDM_DEVREG_FLAGS_FIRST_POWEROFF_NOTIFICATION))
+                pdmR3PowerOffDev(pDevIns, &cAsync);
+        }
 
 #ifdef VBOX_WITH_USB
-    for (PPDMUSBINS pUsbIns = pVM->pdm.s.pUsbInstances; pUsbIns; pUsbIns = pUsbIns->Internal.s.pNext)
-    {
-        for (PPDMLUN pLun = pUsbIns->Internal.s.pLuns; pLun; pLun = pLun->pNext)
-            for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
-                if (pDrvIns->pDrvReg->pfnPowerOff)
-                {
-                    LogFlow(("PDMR3PowerOff: Notifying - driver '%s'/%d on LUN#%d of usb device '%s'/%d\n",
-                             pDrvIns->pDrvReg->szDriverName, pDrvIns->iInstance, pLun->iLun, pUsbIns->pUsbReg->szDeviceName, pUsbIns->iInstance));
-                    pDrvIns->pDrvReg->pfnPowerOff(pDrvIns);
-                }
-
-        if (pUsbIns->pUsbReg->pfnVMPowerOff)
-        {
-            LogFlow(("PDMR3PowerOff: Notifying - device '%s'/%d\n",
-                     pUsbIns->pUsbReg->szDeviceName, pUsbIns->iInstance));
-            pUsbIns->pUsbReg->pfnVMPowerOff(pUsbIns);
-        }
-    }
+        for (PPDMUSBINS pUsbIns = pVM->pdm.s.pUsbInstances; pUsbIns; pUsbIns = pUsbIns->Internal.s.pNext)
+        {
+            unsigned const cAsyncStart = cAsync;
+
+            for (PPDMLUN pLun = pUsbIns->Internal.s.pLuns; pLun; pLun = pLun->pNext)
+                for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
+                    if (!pdmR3PowerOffDrv(pDrvIns, &cAsync, pUsbIns->pUsbReg->szDeviceName, pUsbIns->iInstance, pLun->iLun))
+                        break;
+
+            if (cAsync == cAsyncStart)
+                pdmR3PowerOffUsb(pUsbIns, &cAsync);
+        }
 #endif
+        if (!cAsync)
+            break;
+
+        /*
+         * Process requests.
+         */
+        /** @todo This is utterly nuts and completely unsafe... will get back to it in a
+         *        bit I hope... */
+        int rc = VMR3ReqProcessU(pVM->pUVM, VMCPUID_ANY);
+        AssertReleaseRC(rc == VINF_SUCCESS);
+    }
 
     /*
Index: /trunk/src/VBox/VMM/PDMDevHlp.cpp
===================================================================
--- /trunk/src/VBox/VMM/PDMDevHlp.cpp	(revision 24729)
+++ /trunk/src/VBox/VMM/PDMDevHlp.cpp	(revision 24730)
@@ -887,4 +887,6 @@
                         pNew->Internal.s.pVM            = pVM;
                         //pNew->Internal.s.fDetaching     = false;
+                        pNew->Internal.s.fVMSuspended   = true;
+                        //pNew->Internal.s.pfnAsyncNotify = NULL;
                         pNew->Internal.s.pCfgHandle     = pNode;
                         pNew->pDrvHlp                   = &g_pdmR3DrvHlp;
@@ -2324,4 +2326,31 @@
     LogFlow(("pdmR3DevHlp_PhysGCPtr2GCPhys: caller='%s'/%d: returns %Rrc *pGCPhys=%RGp\n", pDevIns->pDevReg->szDeviceName, pDevIns->iInstance, rc, *pGCPhys));
 
+    return rc;
+}
+
+
+/** @copydoc PDMDEVHLPR3::pfnSetAsyncNotification */
+static DECLCALLBACK(int) pdmR3DevHlp_SetAsyncNotification(PPDMDEVINS pDevIns, PFNPDMDEVASYNCNOTIFY pfnAsyncNotify)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT0(pDevIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DevHlp_SetAsyncNotification: caller='%s'/%d: pfnAsyncNotify\n", pDevIns->pDevReg->szDeviceName, pDevIns->iInstance, pfnAsyncNotify));
+
+    int rc = VINF_SUCCESS;
+    AssertStmt(pfnAsyncNotify, rc = VERR_INVALID_PARAMETER);
+    AssertStmt(!pDevIns->Internal.s.pfnAsyncNotify, rc = VERR_WRONG_ORDER);
+    AssertStmt(pDevIns->Internal.s.fIntFlags & PDMDEVINSINT_FLAGS_SUSPENDED, rc = VERR_WRONG_ORDER);
+    VMSTATE enmVMState = VMR3GetState(pDevIns->Internal.s.pVMR3);
+    AssertStmt(   enmVMState == VMSTATE_SUSPENDING
+               || enmVMState == VMSTATE_SUSPENDING_EXT_LS
+               || enmVMState == VMSTATE_SUSPENDING_LS
+               || enmVMState == VMSTATE_POWERING_OFF
+               || enmVMState == VMSTATE_POWERING_OFF_LS,
+               rc = VERR_INVALID_STATE);
+
+    if (RT_SUCCESS(rc))
+        pDevIns->Internal.s.pfnAsyncNotify = pfnAsyncNotify;
+
+    LogFlow(("pdmR3DevHlp_SetAsyncNotification: caller='%s'/%d: returns %Rrc\n", pDevIns->pDevReg->szDeviceName, pDevIns->iInstance, rc));
     return rc;
 }
@@ -2836,4 +2865,5 @@
     pdmR3DevHlp_PDMThreadCreate,
     pdmR3DevHlp_PhysGCPtr2GCPhys,
+    pdmR3DevHlp_SetAsyncNotification,
     0,
     0,
@@ -3301,4 +3331,5 @@
     pdmR3DevHlp_PDMThreadCreate,
     pdmR3DevHlp_PhysGCPtr2GCPhys,
+    pdmR3DevHlp_SetAsyncNotification,
     0,
     0,
Index: /trunk/src/VBox/VMM/PDMDevice.cpp
===================================================================
--- /trunk/src/VBox/VMM/PDMDevice.cpp	(revision 24729)
+++ /trunk/src/VBox/VMM/PDMDevice.cpp	(revision 24730)
@@ -325,4 +325,5 @@
         //pDevIns->Internal.s.pPciDeviceRC        = 0;
         //pDevIns->Internal.s.pPciBusRC           = 0;
+        pDevIns->Internal.s.fIntFlags           = PDMDEVINSINT_FLAGS_SUSPENDED;
         pDevIns->pDevHlpR3                      = fTrusted ? &g_pdmR3DevHlpTrusted : &g_pdmR3DevHlpUnTrusted;
         pDevIns->pDevHlpRC                      = pDevHlpRC;
Index: /trunk/src/VBox/VMM/PDMDriver.cpp
===================================================================
--- /trunk/src/VBox/VMM/PDMDriver.cpp	(revision 24729)
+++ /trunk/src/VBox/VMM/PDMDriver.cpp	(revision 24730)
@@ -536,4 +536,6 @@
                             pNew->Internal.s.pVM            = pVM;
                             pNew->Internal.s.fDetaching     = false;
+                            pNew->Internal.s.fVMSuspended   = true;
+                            pNew->Internal.s.pfnAsyncNotify = NULL;
                             pNew->Internal.s.pCfgHandle     = pNode;
                             pNew->pDrvHlp                   = &g_pdmR3DrvHlp;
@@ -1017,4 +1019,31 @@
 
 
+/** @copydoc PDMDRVHLP::pfnSetAsyncNotification */
+static DECLCALLBACK(int) pdmR3DrvHlp_SetAsyncNotification(PPDMDRVINS pDrvIns, PFNPDMDRVASYNCNOTIFY pfnAsyncNotify)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    VM_ASSERT_EMT0(pDrvIns->Internal.s.pVM);
+    LogFlow(("pdmR3DrvHlp_SetAsyncNotification: caller='%s'/%d: pfnAsyncNotify=%p\n", pDrvIns->pDrvReg->szDriverName, pDrvIns->iInstance, pfnAsyncNotify));
+
+    int rc = VINF_SUCCESS;
+    AssertStmt(pfnAsyncNotify, rc = VERR_INVALID_PARAMETER);
+    AssertStmt(!pDrvIns->Internal.s.pfnAsyncNotify, rc = VERR_WRONG_ORDER);
+    AssertStmt(pDrvIns->Internal.s.fVMSuspended, rc = VERR_WRONG_ORDER);
+    VMSTATE enmVMState = VMR3GetState(pDrvIns->Internal.s.pVM);
+    AssertStmt(   enmVMState == VMSTATE_SUSPENDING
+               || enmVMState == VMSTATE_SUSPENDING_EXT_LS
+               || enmVMState == VMSTATE_SUSPENDING_LS
+               || enmVMState == VMSTATE_POWERING_OFF
+               || enmVMState == VMSTATE_POWERING_OFF_LS,
+               rc = VERR_INVALID_STATE);
+
+    if (RT_SUCCESS(rc))
+        pDrvIns->Internal.s.pfnAsyncNotify = pfnAsyncNotify;
+
+    LogFlow(("pdmR3DrvHlp_SetAsyncNotification: caller='%s'/%d: returns %Rrc\n", pDrvIns->pDrvReg->szDriverName, pDrvIns->iInstance, rc));
+    return rc;
+}
+
+
 /** @copydoc PDMDRVHLP::pfnPDMThreadCreate */
 static DECLCALLBACK(int) pdmR3DrvHlp_PDMThreadCreate(PPDMDRVINS pDrvIns, PPPDMTHREAD ppThread, void *pvUser, PFNPDMTHREADDRV pfnThread,
@@ -1084,4 +1113,5 @@
     pdmR3DrvHlp_SUPCallVMMR0Ex,
     pdmR3DrvHlp_USBRegisterHub,
+    pdmR3DrvHlp_SetAsyncNotification,
     pdmR3DrvHlp_PDMThreadCreate,
 #ifdef VBOX_WITH_PDM_ASYNC_COMPLETION
Index: /trunk/src/VBox/VMM/PDMInternal.h
===================================================================
--- /trunk/src/VBox/VMM/PDMInternal.h	(revision 24729)
+++ /trunk/src/VBox/VMM/PDMInternal.h	(revision 24730)
@@ -23,5 +23,4 @@
 #define ___PDMInternal_h
 
-#include <VBox/cdefs.h>
 #include <VBox/types.h>
 #include <VBox/param.h>
@@ -30,4 +29,5 @@
 #include <VBox/vusb.h>
 #include <VBox/pdmasynccompletion.h>
+#include <VBox/pdmcommon.h>
 #include <iprt/assert.h>
 #include <iprt/critsect.h>
@@ -101,4 +101,7 @@
     /** Pointer to the list of logical units associated with the device. (FIFO) */
     R3PTRTYPE(PPDMLUN)              pLunsR3;
+    /** Pointer to the asynchronous notification callback set while in
+     * FNPDMDEVSUSPEND or FNPDMDEVPOWEROFF. */
+    R3PTRTYPE(PFNPDMDEVASYNCNOTIFY) pfnAsyncNotify;
     /** Configuration handle to the instance node. */
     R3PTRTYPE(PCFGMNODE)            pCfgHandle;
@@ -117,6 +120,4 @@
     /** R0 pointer to associated PCI bus structure. */
     R0PTRTYPE(PPDMPCIBUS)           pPciBusR0;
-    /** Alignment padding. */
-    RTR0PTR                         Alignment0;
 
     /** RC pointer to the VM this instance was created for. */
@@ -135,4 +136,10 @@
 /** Used by pdmR3Load to mark device instances it found in the saved state. */
 #define PDMDEVINSINT_FLAGS_FOUND         RT_BIT_32(0)
+/** Indicates that the device hasn't been powered on or resumed.
+ * This is used by PDMR3PowerOn, PDMR3Resume, PDMR3Suspend and PDMR3PowerOff
+ * to make sure each device gets exactly one notification for each of those
+ * events.  PDMR3Resume and PDMR3PowerOn also makes use of it to bail out on
+ * a failure (already resumed/powered-on devices are suspended). */
+#define PDMDEVINSINT_FLAGS_SUSPENDED     RT_BIT_32(1)
 /** @} */
 
@@ -171,7 +178,10 @@
     /** The port number that we're connected to. */
     uint32_t                        iPort;
-#if HC_ARCH_BITS == 64
-    uint32_t                        Alignment0;
-#endif
+    /** Indicates that the driver hasn't been powered on or resumed.
+     * See PDMDEVINSINT_FLAGS_SUSPENDED. */
+    bool                            fVMSuspended;
+    /** Pointer to the asynchronous notification callback set while in
+     * FNPDMDEVSUSPEND or FNPDMDEVPOWEROFF. */
+    R3PTRTYPE(PFNPDMUSBASYNCNOTIFY) pfnAsyncNotify;
 } PDMUSBINSINT;
 
@@ -197,4 +207,10 @@
      * (Helps detect potential recursive detaching.) */
     bool                            fDetaching;
+    /** Indicates that the driver hasn't been powered on or resumed.
+     * See PDMDEVINSINT_FLAGS_SUSPENDED. */
+    bool                            fVMSuspended;
+    /** Pointer to the asynchronous notification callback set while in
+     * PDMUSBREG::pfnVMSuspend or PDMUSBREG::pfnVMPowerOff. */
+    R3PTRTYPE(PFNPDMDRVASYNCNOTIFY) pfnAsyncNotify;
     /** Configuration handle to the instance node. */
     PCFGMNODE                       pCfgHandle;
Index: /trunk/src/VBox/VMM/VM.cpp
===================================================================
--- /trunk/src/VBox/VMM/VM.cpp	(revision 24729)
+++ /trunk/src/VBox/VMM/VM.cpp	(revision 24730)
@@ -1210,5 +1210,5 @@
     int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING | VMMEMTRENDEZVOUS_FLAGS_STOP_ON_ERROR,
                                 vmR3PowerOn, NULL);
-    LogFlow(("VMR3Suspend: returns %Rrc\n", rc));
+    LogFlow(("VMR3PowerOn: returns %Rrc\n", rc));
     return rc;
 }
Index: /trunk/src/VBox/VMM/testcase/tstVMStructGC.cpp
===================================================================
--- /trunk/src/VBox/VMM/testcase/tstVMStructGC.cpp	(revision 24729)
+++ /trunk/src/VBox/VMM/testcase/tstVMStructGC.cpp	(revision 24730)
@@ -356,4 +356,5 @@
     GEN_CHECK_OFF(PDMDEVINSINT, pVMRC);
     GEN_CHECK_OFF(PDMDEVINSINT, pLunsR3);
+    GEN_CHECK_OFF(PDMDEVINSINT, pfnAsyncNotify);
     GEN_CHECK_OFF(PDMDEVINSINT, pCfgHandle);
     GEN_CHECK_OFF(PDMDEVINSINT, pPciDeviceR3);
@@ -363,4 +364,18 @@
     GEN_CHECK_OFF(PDMDEVINSINT, pPciBusR0);
     GEN_CHECK_OFF(PDMDEVINSINT, pPciBusRC);
+    GEN_CHECK_OFF(PDMDEVINSINT, fIntFlags);
+    GEN_CHECK_OFF(PDMDEVINS, u32Version);
+    GEN_CHECK_OFF(PDMDEVINS, iInstance);
+    GEN_CHECK_OFF(PDMDEVINS, pDevHlpRC);
+    GEN_CHECK_OFF(PDMDEVINS, pvInstanceDataRC);
+    GEN_CHECK_OFF(PDMDEVINS, pDevHlpR0);
+    GEN_CHECK_OFF(PDMDEVINS, pvInstanceDataR0);
+    GEN_CHECK_OFF(PDMDEVINS, pDevHlpR3);
+    GEN_CHECK_OFF(PDMDEVINS, pvInstanceDataR3);
+    GEN_CHECK_OFF(PDMDEVINS, pDevReg);
+    GEN_CHECK_OFF(PDMDEVINS, pCfgHandle);
+    GEN_CHECK_OFF(PDMDEVINS, IBase);
+    GEN_CHECK_OFF(PDMDEVINS, Internal);
+    GEN_CHECK_OFF(PDMDEVINS, achInstanceData);
     GEN_CHECK_SIZE(PDMCRITSECTINT);
     GEN_CHECK_OFF(PDMCRITSECTINT, Core);
