Index: /trunk/src/VBox/HostDrivers/Support/SUPDrv.c =================================================================== --- /trunk/src/VBox/HostDrivers/Support/SUPDrv.c (revision 37061) +++ /trunk/src/VBox/HostDrivers/Support/SUPDrv.c (revision 37062) @@ -322,4 +322,6 @@ { "RTMpIsCpuOnline", (void *)RTMpIsCpuOnline }, { "RTMpIsCpuWorkPending", (void *)RTMpIsCpuWorkPending }, + { "RTMpNotificationRegister", (void *)RTMpNotificationRegister }, + { "RTMpNotificationDeregister", (void *)RTMpNotificationDeregister }, { "RTMpOnAll", (void *)RTMpOnAll }, { "RTMpOnOthers", (void *)RTMpOnOthers }, @@ -4990,4 +4992,38 @@ /** + * Helper for finding the CPU index from the CPU Id. + * + * @param pGip The GIP. + * @param idCpu The CPU ID. + * + * @returns Index of the CPU in the cache set. + */ +static inline uint32_t supdrvGipCpuIndexFromCpuId(PSUPGLOBALINFOPAGE pGip, RTCPUID idCpu) +{ + uint32_t i = 0; + + /* + * Find our entry, or allocate one if not found. + * ASSUMES that CPU IDs are constant. + */ + for (i = 0; i < pGip->cCpus; i++) + if (pGip->aCPUs[i].idCpu == idCpu) + break; + + if (i >= pGip->cCpus) + for (i = 0; i < pGip->cCpus; i++) + { + bool fRc; + ASMAtomicCmpXchgSize(&pGip->aCPUs[i].idCpu, idCpu, NIL_RTCPUID, fRc); + if (fRc) + break; + } + + AssertRelease(i < pGip->cCpus); + return i; +} + + +/** * The calling CPU should be accounted as online, update GIP accordingly. * @@ -4999,9 +5035,9 @@ static void supdrvGipMpEventOnline(PSUPGLOBALINFOPAGE pGip, RTCPUID idCpu) { - int iCpuSet; - uint8_t idApic; - uint32_t i; - - Assert(idCpu == RTMpCpuId()); + int iCpuSet = 0; + uint16_t idApic = UINT16_MAX; + uint32_t i = 0; + + AssertRelease(idCpu == RTMpCpuId()); Assert(pGip->cPossibleCpus == RTMpGetCount()); @@ -5019,22 +5055,5 @@ } - /* - * Find our entry, or allocate one if not found. - * ASSUMES that CPU IDs are constant. - */ - for (i = 0; i < pGip->cCpus; i++) - if (pGip->aCPUs[i].idCpu == idCpu) - break; - - if (i >= pGip->cCpus) - for (i = 0; i < pGip->cCpus; i++) - { - bool fRc; - ASMAtomicCmpXchgSize(&pGip->aCPUs[i].idCpu, idCpu, NIL_RTCPUID, fRc); - if (fRc) - break; - } - - AssertReturnVoid(i < pGip->cCpus); + i = supdrvGipCpuIndexFromCpuId(pGip, idCpu); /* @@ -5045,5 +5064,4 @@ ASMAtomicUoWriteS16(&pGip->aCPUs[i].iCpuSet, (int16_t)iCpuSet); ASMAtomicUoWriteSize(&pGip->aCPUs[i].idCpu, idCpu); - ASMAtomicWriteSize(&pGip->aCPUs[i].enmState, SUPGIPCPUSTATE_ONLINE); /* @@ -5052,4 +5070,6 @@ ASMAtomicWriteU16(&pGip->aiCpuFromApicId[idApic], i); ASMAtomicWriteU16(&pGip->aiCpuFromCpuSetIdx[iCpuSet], i); + + ASMAtomicWriteSize(&pGip->aCPUs[i].enmState, SUPGIPCPUSTATE_ONLINE); } @@ -5076,6 +5096,6 @@ Assert(RTCpuSetIsMemberByIndex(&pGip->PossibleCpuSet, iCpuSet)); + RTCpuSetDelByIndex(&pGip->OnlineCpuSet, iCpuSet); ASMAtomicWriteSize(&pGip->aCPUs[i].enmState, SUPGIPCPUSTATE_OFFLINE); - RTCpuSetDelByIndex(&pGip->OnlineCpuSet, iCpuSet); } @@ -5095,4 +5115,6 @@ PSUPDRVDEVEXT pDevExt = (PSUPDRVDEVEXT)pvUser; PSUPGLOBALINFOPAGE pGip = pDevExt->pGip; + + AssertRelease(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); /* @@ -5359,5 +5381,5 @@ pGip->aCPUs[i].idCpu = NIL_RTCPUID; pGip->aCPUs[i].iCpuSet = -1; - pGip->aCPUs[i].idApic = UINT8_MAX; + pGip->aCPUs[i].idApic = UINT16_MAX; /* @@ -5618,4 +5640,15 @@ RTCPUID idCpu, uint8_t idApic, uint64_t iTick) { + /* + * Avoid a potential race when a CPU online notification doesn't fire on the onlined CPU + * but the tick creeps in before the event notification is run. + */ + if (iTick == 1) + { + uint32_t i = supdrvGipCpuIndexFromCpuId(pGip, idCpu); + if (pGip->aCPUs[i].enmState == SUPGIPCPUSTATE_OFFLINE) + supdrvGipMpEventOnline(pGip, idCpu); + } + unsigned iCpu = pGip->aiCpuFromApicId[idApic]; Index: /trunk/src/VBox/Runtime/r0drv/solaris/vbi/mp-r0drv-solaris.c =================================================================== --- /trunk/src/VBox/Runtime/r0drv/solaris/vbi/mp-r0drv-solaris.c (revision 37061) +++ /trunk/src/VBox/Runtime/r0drv/solaris/vbi/mp-r0drv-solaris.c (revision 37062) @@ -174,4 +174,11 @@ PRTMPARGS pArgs = (PRTMPARGS)(uArg); + /* + * Solaris CPU cross calls execute on offline CPUs too. Check our CPU cache + * set and ignore if it's offline. + */ + if (!RTMpIsCpuOnline(RTMpCpuId())) + return 0; + pArgs->pfnWorker(RTMpCpuId(), pArgs->pvUser1, pArgs->pvUser2); @@ -279,4 +286,7 @@ return VERR_CPU_NOT_FOUND; + if (RT_UNLIKELY(!RTMpIsCpuOnline(idCpu))) + return RTMpIsCpuPresent(idCpu) ? VERR_CPU_OFFLINE : VERR_CPU_NOT_FOUND; + Args.pfnWorker = pfnWorker; Args.pvUser1 = pvUser1; Index: /trunk/src/VBox/Runtime/r0drv/solaris/vbi/mpnotification-r0drv-solaris.c =================================================================== --- /trunk/src/VBox/Runtime/r0drv/solaris/vbi/mpnotification-r0drv-solaris.c (revision 37061) +++ /trunk/src/VBox/Runtime/r0drv/solaris/vbi/mpnotification-r0drv-solaris.c (revision 37062) @@ -34,4 +34,5 @@ #include #include +#include #include "r0drv/mp-r0drv.h" @@ -49,19 +50,49 @@ -static void rtMpNotificationSolarisCallback(void *pvUser, int iCpu, int online) +static void rtMpNotificationSolarisOnCurrentCpu(void *pvArgs, void *uIgnored1, void *uIgnored2) { - NOREF(pvUser); + NOREF(uIgnored1); + NOREF(uIgnored2); - /* ASSUMES iCpu == RTCPUID */ + PRTMPARGS pArgs = (PRTMPARGS)(pvArgs); + AssertRelease(pArgs && pArgs->idCpu == RTMpCpuId()); + Assert(pArgs->pvUser2); + Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); + + int online = *(int *)pArgs->pvUser2; if (online) { - RTCpuSetAdd(&g_rtMpSolarisCpuSet, iCpu); - rtMpNotificationDoCallbacks(RTMPEVENT_ONLINE, iCpu); + RTCpuSetAdd(&g_rtMpSolarisCpuSet, pArgs->idCpu); + rtMpNotificationDoCallbacks(RTMPEVENT_ONLINE, pArgs->idCpu); } else { - RTCpuSetDel(&g_rtMpSolarisCpuSet, iCpu); - rtMpNotificationDoCallbacks(RTMPEVENT_OFFLINE, iCpu); + RTCpuSetDel(&g_rtMpSolarisCpuSet, pArgs->idCpu); + rtMpNotificationDoCallbacks(RTMPEVENT_OFFLINE, pArgs->idCpu); } +} + + +static void rtMpNotificationSolarisCallback(void *pvUser, int iCpu, int online) +{ + vbi_preempt_disable(); + + RTMPARGS Args; + RT_ZERO(Args); + Args.pvUser1 = pvUser; + Args.pvUser2 = &online; + Args.idCpu = iCpu; + + /* + * If we're not on the target CPU, schedule (synchronous) the event notification callback + * to run on the target CPU i.e. the one pertaining to the MP event. + */ + bool fRunningOnTargetCpu = iCpu == RTMpCpuId(); /* ASSUMES iCpu == RTCPUID */ + if (fRunningOnTargetCpu) + rtMpNotificationSolarisOnCurrentCpu(&Args, NULL /* pvIgnored1 */, NULL /* pvIgnored2 */); + else + vbi_execute_on_one(rtMpNotificationSolarisOnCurrentCpu, &Args, iCpu); + + vbi_preempt_enable(); } Index: /trunk/src/VBox/VMM/VMMR0/HWACCMR0.cpp =================================================================== --- /trunk/src/VBox/VMM/VMMR0/HWACCMR0.cpp (revision 37061) +++ /trunk/src/VBox/VMM/VMMR0/HWACCMR0.cpp (revision 37062) @@ -51,4 +51,5 @@ static bool hwaccmR0IsSubjectToVmxPreemptionTimerErratum(void); static DECLCALLBACK(void) hwaccmR0PowerCallback(RTPOWEREVENT enmEvent, void *pvUser); +static DECLCALLBACK(void) hwaccmR0CpuCallback(RTMPEVENT enmEvent, RTCPUID idCpu, void *pvData); /******************************************************************************* @@ -498,4 +499,7 @@ if (!HWACCMR0Globals.vmx.fUsingSUPR0EnableVTx) { + rc = RTMpNotificationRegister(hwaccmR0CpuCallback, 0); + AssertRC(rc); + rc = RTPowerNotificationRegister(hwaccmR0PowerCallback, 0); AssertRC(rc); @@ -575,4 +579,7 @@ if (!HWACCMR0Globals.vmx.fUsingSUPR0EnableVTx) { + /* Doesn't really matter if this fails. */ + rc = RTMpNotificationDeregister(hwaccmR0CpuCallback, 0); + AssertRC(rc); rc = RTPowerNotificationDeregister(hwaccmR0PowerCallback, 0); AssertRC(rc); @@ -732,6 +739,5 @@ Assert(!HWACCMR0Globals.aCpuInfo[i].pMemObj); - /** @todo this is rather dangerous if cpus can be taken offline; we don't care for now */ - if (RTMpIsCpuOnline(i)) + if (RTMpIsCpuPossible(RTMpCpuId())) { rc = RTR0MemObjAllocCont(&HWACCMR0Globals.aCpuInfo[i].pMemObj, 1 << PAGE_SHIFT, true /* executable R0 mapping */); @@ -748,4 +754,5 @@ #endif } + HWACCMR0Globals.aCpuInfo[i].fConfigured = false; } if (HWACCMR0Globals.fGlobalInit) @@ -798,4 +805,5 @@ if (!pCpu->pMemObj) { + LogRel(("HWACCMR0: hwaccmR0EnableCpu failed idCpu=%d.\n", idCpu)); AssertFailed(); return VERR_INTERNAL_ERROR; @@ -883,4 +891,35 @@ hwaccmR0FirstRcSetStatus(pFirstRc, hwaccmR0DisableCpu(idCpu)); } + + +/** + * Callback function invoked when a cpu goes online or offline. + * + * @param enmEvent The Mp event. + * @param idCpu The identifier for the CPU the function is called on. + * @param pvData Opaque data (PVM pointer). + */ +static DECLCALLBACK(void) hwaccmR0CpuCallback(RTMPEVENT enmEvent, RTCPUID idCpu, void *pvData) +{ + /* + * We only care about uninitializing a CPU that is going offline. When a + * CPU comes online, the initialization is done lazily in HWACCMR0Enter(). + */ + AssertRelease(idCpu == RTMpCpuId()); + Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); + switch (enmEvent) + { + case RTMPEVENT_OFFLINE: + { + int rc = hwaccmR0DisableCpu(idCpu); + AssertRC(rc); + break; + } + + default: + break; + } +} + /** @@ -1129,6 +1168,6 @@ * * @returns VBox status code. - * @param pVM The VM to operate on. - * @param pVCpu VMCPUD id. + * @param pVM The VM to operate on. + * @param pVCpu VMCPU handle. */ VMMR0DECL(int) HWACCMR0Enter(PVM pVM, PVMCPU pVCpu) @@ -1163,6 +1202,7 @@ pVM->hwaccm.s.u64RegisterMask = UINT64_C(0xFFFFFFFF); - /* Enable VT-x or AMD-V if local init is required. */ - if (!HWACCMR0Globals.fGlobalInit) + /* Enable VT-x or AMD-V if local init is required, or enable if it's a freshly onlined CPU. */ + if ( !pCpu->fConfigured + || !HWACCMR0Globals.fGlobalInit) { rc = hwaccmR0EnableCpu(pVM, idCpu); @@ -1198,6 +1238,6 @@ * * @returns VBox status code. - * @param pVM The VM to operate on. - * @param pVCpu VMCPUD id. + * @param pVM The VM to operate on. + * @param pVCpu VMCPU handle. */ VMMR0DECL(int) HWACCMR0Leave(PVM pVM, PVMCPU pVCpu)