Changeset 101163 in vbox

Timestamp:

Sep 18, 2023 8:44:24 PM (12 months ago)

Author:

vboxsync

Message:

VMM/IEM: Experimental recompilation of threaded blocks into native code on linux.amd64. bugref:10370

Location:

trunk/src/VBox/VMM

Files:

• Config.kmk (modified) (1 diff)
• Makefile.kmk (modified) (1 diff)
• VMMAll/IEMAllN8veRecompiler.cpp (added)
• VMMAll/IEMAllThrdFuncsBltIn.cpp (modified) (1 diff)
• VMMAll/IEMAllThrdRecompiler.cpp (modified) (16 diffs)
• VMMR3/IEMR3.cpp (modified) (3 diffs)
• include/IEMInline.h (modified) (6 diffs)
• include/IEMInternal.h (modified) (9 diffs)

trunk/src/VBox/VMM/Config.kmk

@@ -70 +70 @@
 ifdef VBOX_WITH_IEM_RECOMPILER # Selectely removing hwvirt doesn't work yet with the recompiler. Python code doesn't check #ifdefs.
  VMM_COMMON_DEFS += VBOX_WITH_IEM_RECOMPILER IEM_WITH_CODE_TLB IEM_WITH_DATA_TLB
+ ifdef VBOX_WITH_IEM_NATIVE_RECOMPILER
+  VMM_COMMON_DEFS += VBOX_WITH_IEM_NATIVE_RECOMPILER
+ endif
  VMM_COMMON_DEFS += VBOX_WITH_NESTED_HWVIRT_VMX
  #ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT # Busted with TLB enabled.

trunk/src/VBox/VMM/Makefile.kmk

@@ -258 +258 @@
         VMMAll/IEMAllThrdFuncs.cpp \
         VMMAll/IEMAllThrdFuncsBltIn.cpp
+ ifdef VBOX_WITH_IEM_NATIVE_RECOMPILER
+  VBoxVMM_SOURCES += \
+        VMMAll/IEMAllN8veRecompiler.cpp
+ endif
  VMMAll/IEMAllThrdFuncs.cpp_CXXFLAGS.win = /bigobj
 endif

trunk/src/VBox/VMM/VMMAll/IEMAllThrdFuncsBltIn.cpp

@@ -73 +73 @@
 static VBOXSTRICTRC iemThreadeFuncWorkerObsoleteTb(PVMCPUCC pVCpu)
 {
-    iemThreadedTbObsolete(pVCpu, pVCpu->iem.s.pCurTbR3);
+    /* We set fSafeToFree to false where as we're being called in the context
+       of a TB callback function, which for native TBs means we cannot release
+       the executable memory till we've returned our way back to iemTbExec as
+       that return path codes via the native code generated for the TB. */
+    iemThreadedTbObsolete(pVCpu, pVCpu->iem.s.pCurTbR3, false /*fSafeToFree*/);
     return VINF_IEM_REEXEC_BREAK;
 }

trunk/src/VBox/VMM/VMMAll/IEMAllThrdRecompiler.cpp

@@ -115 +115 @@
 *   Internal Functions                                                                                                           *
 *********************************************************************************************************************************/
-static VBOXSTRICTRC iemThreadedTbExec(PVMCPUCC pVCpu, PIEMTB pTb);
 static void         iemTbAllocatorFree(PVMCPUCC pVCpu, PIEMTB pTb);
 
@@ -650 +649 @@
                 if (pTb->x86.fAttr == (uint16_t)pVCpu->cpum.GstCtx.cs.Attr.u)
                 {
-                    pTb->cUsed++;
-                    pTb->msLastUsed = pVCpu->iem.s.msRecompilerPollNow;
                     STAM_COUNTER_INC(&pTbCache->cLookupHits);
                     AssertMsg(cLeft > 0, ("%d\n", cLeft));
+
+                    pTb->msLastUsed = pVCpu->iem.s.msRecompilerPollNow;
+                    pTb->cUsed++;
+#ifdef VBOX_WITH_IEM_NATIVE_RECOMPILER
+                    if ((pTb->fFlags & IEMTB_F_TYPE_NATIVE) || pTb->cUsed != 16)
+                        return pTb;
+                    return iemNativeRecompile(pVCpu, pTb);
+#else
                     return pTb;
+#endif
                 }
                 Log11(("TB miss: CS: %#x, wanted %#x\n", pTb->x86.fAttr, (uint16_t)pVCpu->cpum.GstCtx.cs.Attr.u));
@@ -706 +712 @@
  *
  * @returns VBox status code.
- * @param   pVM         The VM handle.
- * @param   cInitialTbs The initial number of translation blocks to
- *                      preallocator.
- * @param   cMaxTbs     The max number of translation blocks allowed.
+ * @param   pVM             The VM handle.
+ * @param   cInitialTbs     The initial number of translation blocks to
+ *                          preallocator.
+ * @param   cMaxTbs         The max number of translation blocks allowed.
+ * @param   cbInitialExec   The initial size of the executable memory allocator.
+ * @param   cbMaxExec       The max size of the executable memory allocator.
+ * @param   cbChunkExec     The chunk size for executable memory allocator. Zero
+ *                          or UINT32_MAX for automatically determining this.
  * @thread  EMT
  */
-DECLCALLBACK(int) iemTbInit(PVMCC pVM, uint32_t cInitialTbs, uint32_t cMaxTbs)
+DECLCALLBACK(int) iemTbInit(PVMCC pVM, uint32_t cInitialTbs, uint32_t cMaxTbs,
+                            uint64_t cbInitialExec, uint64_t cbMaxExec, uint32_t cbChunkExec)
 {
     PVMCPUCC pVCpu = VMMGetCpu(pVM);
@@ -820 +831 @@
     pVCpu->iem.s.pTbCacheR3 = pTbCache;
 
+    /*
+     * Initialize the native executable memory allocator.
+     */
+#ifdef VBOX_WITH_IEM_NATIVE_RECOMPILER
+    int rc = iemExecMemAllocatorInit(pVCpu, cbMaxExec, cbInitialExec, cbChunkExec);
+    AssertLogRelRCReturn(rc, rc);
+#else
+    RT_NOREF(cbMaxExec, cbInitialExec, cbChunkExec);
+#endif
+
     return VINF_SUCCESS;
 }
@@ -849 +870 @@
             RTMemFree(pTb->Thrd.paCalls);
             break;
+#ifdef VBOX_WITH_IEM_NATIVE_RECOMPILER
         case IEMTB_F_TYPE_NATIVE:
             pTbAllocator->cNativeTbs -= 1;
-            RTMemFree(pTb->Native.pbCode); /// @todo native: fix me
+            iemExecMemAllocatorFree(pVCpu, pTb->Native.paInstructions,
+                                    pTb->Native.cInstructions * sizeof(pTb->Native.paInstructions[0]));
             break;
+#endif
         default:
             AssertFailed();
@@ -880 +904 @@
  * @param   pVCpu   The cross context virtual CPU structure of the calling
  *                  thread.
- * @param   pTb     The translation block to free..
+ * @param   pTb     The translation block to free.
  * @thread  EMT(pVCpu)
  */
@@ -899 +923 @@
      */
     iemTbAllocatorFreeInner(pVCpu, pTbAllocator, pTb, idxChunk, (uint32_t)idxInChunk);
+}
+
+
+/**
+ * Schedules a native TB for freeing when it's not longer being executed and
+ * part of the caller's call stack.
+ *
+ * The TB will be removed from the translation block cache, though, so it isn't
+ * possible to executed it again and the IEMTB::pNext member can be used to link
+ * it together with other TBs awaiting freeing.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure of the calling
+ *                  thread.
+ * @param   pTb     The translation block to schedule for freeing.
+ */
+static void iemTbAlloctorScheduleForFree(PVMCPUCC pVCpu, PIEMTB pTb)
+{
+    /*
+     * Validate state.
+     */
+    PIEMTBALLOCATOR const pTbAllocator = pVCpu->iem.s.pTbAllocatorR3;
+    Assert(pTbAllocator && pTbAllocator->uMagic == IEMTBALLOCATOR_MAGIC);
+    Assert(pTb->idxAllocChunk < pTbAllocator->cAllocatedChunks);
+    Assert((uintptr_t)(pTb - pTbAllocator->aChunks[pTb->idxAllocChunk].paTbs) < pTbAllocator->cTbsPerChunk);
+    Assert(ASMBitTest(&pTbAllocator->bmAllocated,
+                      IEMTBALLOC_IDX_MAKE(pTbAllocator, pTb->idxAllocChunk,
+                                          (uintptr_t)(pTb - pTbAllocator->aChunks[pTb->idxAllocChunk].paTbs))));
+    Assert((pTb->fFlags & IEMTB_F_TYPE_MASK) == IEMTB_F_TYPE_NATIVE);
+
+    /*
+     * Remove it from the cache and prepend it to the allocator's todo list.
+     */
+    iemTbCacheRemove(pVCpu->iem.s.pTbCacheR3, pTb);
+
+    pTb->pNext = pTbAllocator->pDelayedFreeHead;
+    pTbAllocator->pDelayedFreeHead = pTb;
+}
+
+
+/**
+ * Processes the delayed frees.
+ *
+ * This is called by the allocator function as well as the native recompile
+ * function before making any TB or executable memory allocations respectively.
+ */
+void iemTbAllocatorProcessDelayedFrees(PVMCPU pVCpu, PIEMTBALLOCATOR pTbAllocator)
+{
+    PIEMTB pTb = pTbAllocator->pDelayedFreeHead;
+    pTbAllocator->pDelayedFreeHead = NULL;
+    while (pTb)
+    {
+        PIEMTB const pTbNext = pTb->pNext;
+        Assert(pVCpu->iem.s.pCurTbR3 != pTb);
+        iemTbAlloctorScheduleForFree(pVCpu, pTb);
+        pTb = pTbNext;
+    }
 }
 
@@ -1078 +1158 @@
     Assert(pTbAllocator && pTbAllocator->uMagic == IEMTBALLOCATOR_MAGIC);
 
+    /* Free any pending TBs before we proceed. */
+    if (!pTbAllocator->pDelayedFreeHead)
+    { /* probably likely */ }
+    else
+        iemTbAllocatorProcessDelayedFrees(pVCpu, pTbAllocator);
+
     /* If the allocator is full, take slow code path.*/
     if (RT_LIKELY(pTbAllocator->cInUseTbs < pTbAllocator->cTotalTbs))
@@ -1084 +1170 @@
 }
 
+
+
+/*********************************************************************************************************************************
+*   Threaded Recompiler Core                                                                                                     *
+*********************************************************************************************************************************/
 
 /**
@@ -1258 +1349 @@
  * Called by opcode verifier functions when they detect a problem.
  */
-void iemThreadedTbObsolete(PVMCPUCC pVCpu, PIEMTB pTb)
-{
-    iemTbAllocatorFree(pVCpu, pTb);
+void iemThreadedTbObsolete(PVMCPUCC pVCpu, PIEMTB pTb, bool fSafeToFree)
+{
+    /* Unless it's safe, we can only immediately free threaded TB, as we will
+       have more code left to execute in native TBs when fSafeToFree == false.  */
+    if (fSafeToFree || (pTb->fFlags & IEMTB_F_TYPE_THREADED))
+        iemTbAllocatorFree(pVCpu, pTb);
+    else
+        iemTbAlloctorScheduleForFree(pVCpu, pTb);
 }
 
@@ -2113 +2209 @@
 
 
+
+/*********************************************************************************************************************************
+*   Recompiled Execution Core                                                                                                    *
+*********************************************************************************************************************************/
+
 /**
  * Executes a translation block.
@@ -2121 +2222 @@
  * @param   pTb     The translation block to execute.
  */
-static VBOXSTRICTRC iemThreadedTbExec(PVMCPUCC pVCpu, PIEMTB pTb) IEM_NOEXCEPT_MAY_LONGJMP
-{
-    /* Check the opcodes in the first page before starting execution. */
+static VBOXSTRICTRC iemTbExec(PVMCPUCC pVCpu, PIEMTB pTb) IEM_NOEXCEPT_MAY_LONGJMP
+{
+    /*
+     * Check the opcodes in the first page before starting execution.
+     */
     Assert(!(pVCpu->iem.s.GCPhysInstrBuf & (RTGCPHYS)GUEST_PAGE_OFFSET_MASK));
     Assert(pTb->aRanges[0].cbOpcodes <= pVCpu->iem.s.cbInstrBufTotal - pVCpu->iem.s.offInstrNextByte);
@@ -2131 +2234 @@
     {
         Log7(("TB obsolete: %p GCPhys=%RGp\n", pTb, pTb->GCPhysPc));
-        iemThreadedTbObsolete(pVCpu, pTb);
+        iemThreadedTbObsolete(pVCpu, pTb, true /*fSafeToFree*/);
         return VINF_SUCCESS;
     }
 
-    /* Set the current TB so CIMPL function may get at it. */
+    /*
+     * Set the current TB so CIMPL functions may get at it.
+     */
     pVCpu->iem.s.pCurTbR3 = pTb;
-    pVCpu->iem.s.cTbExec++;
-
-    /* The execution loop. */
-#ifdef LOG_ENABLED
-    uint64_t             uRipPrev   = UINT64_MAX;
-#endif
-    PCIEMTHRDEDCALLENTRY pCallEntry = pTb->Thrd.paCalls;
-    uint32_t             cCallsLeft = pTb->Thrd.cCalls;
-    while (cCallsLeft-- > 0)
-    {
-#ifdef LOG_ENABLED
-        if (pVCpu->cpum.GstCtx.rip != uRipPrev)
-        {
-            uRipPrev = pVCpu->cpum.GstCtx.rip;
-            iemThreadedLogCurInstr(pVCpu, "EX");
-        }
-        Log9(("%04x:%08RX64: #%d/%d - %d %s\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
-              pTb->Thrd.cCalls - cCallsLeft - 1, pCallEntry->idxInstr, pCallEntry->enmFunction,
-              g_apszIemThreadedFunctions[pCallEntry->enmFunction]));
-#endif
-        VBOXSTRICTRC const rcStrict = g_apfnIemThreadedFunctions[pCallEntry->enmFunction](pVCpu,
-                                                                                          pCallEntry->auParams[0],
-                                                                                          pCallEntry->auParams[1],
-                                                                                          pCallEntry->auParams[2]);
+
+    /*
+     * Execute the block.
+     */
+#ifdef VBOX_WITH_IEM_NATIVE_RECOMPILER
+    if (pTb->fFlags & IEMTB_F_TYPE_NATIVE)
+    {
+        pVCpu->iem.s.cTbExecNative++;
+        typedef IEM_DECL_IMPL_TYPE(int, FNIEMNATIVETB, (PVMCPUCC pVCpu, PIEMTB pTb));
+# ifdef LOG_ENABLED
+        iemThreadedLogCurInstr(pVCpu, "EXn");
+# endif
+        VBOXSTRICTRC const rcStrict = ((FNIEMNATIVETB *)pTb->Native.paInstructions)(pVCpu, pTb);
         if (RT_LIKELY(   rcStrict == VINF_SUCCESS
                       && pVCpu->iem.s.rcPassUp == VINF_SUCCESS /** @todo this isn't great. */))
-            pCallEntry++;
+        { /* likely */ }
         else
         {
-            pVCpu->iem.s.cInstructions += pCallEntry->idxInstr; /* This may be one short, but better than zero. */
-            pVCpu->iem.s.pCurTbR3       = NULL;
+            /* pVCpu->iem.s.cInstructions is incremented by iemNativeHlpExecStatusCodeFiddling. */
+            pVCpu->iem.s.pCurTbR3 = NULL;
             STAM_REL_COUNTER_INC(&pVCpu->iem.s.StatTbExecBreaks);
 
-            /* Some status codes are just to get us out of this loop and
-               continue in a different translation block. */
+            /* VINF_IEM_REEXEC_BREAK should be treated as VINF_SUCCESS as it's
+               only to break out of TB execution early. */
             if (rcStrict == VINF_IEM_REEXEC_BREAK)
                 return iemExecStatusCodeFiddling(pVCpu, VINF_SUCCESS);
             return iemExecStatusCodeFiddling(pVCpu, rcStrict);
+        }
+    }
+    else
+#endif /* VBOX_WITH_IEM_NATIVE_RECOMPILER */
+    {
+        /*
+         * The threaded execution loop.
+         */
+        pVCpu->iem.s.cTbExecThreaded++;
+#ifdef LOG_ENABLED
+        uint64_t             uRipPrev   = UINT64_MAX;
+#endif
+        PCIEMTHRDEDCALLENTRY pCallEntry = pTb->Thrd.paCalls;
+        uint32_t             cCallsLeft = pTb->Thrd.cCalls;
+        while (cCallsLeft-- > 0)
+        {
+#ifdef LOG_ENABLED
+            if (pVCpu->cpum.GstCtx.rip != uRipPrev)
+            {
+                uRipPrev = pVCpu->cpum.GstCtx.rip;
+                iemThreadedLogCurInstr(pVCpu, "EXt");
+            }
+            Log9(("%04x:%08RX64: #%d/%d - %d %s\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
+                  pTb->Thrd.cCalls - cCallsLeft - 1, pCallEntry->idxInstr, pCallEntry->enmFunction,
+                  g_apszIemThreadedFunctions[pCallEntry->enmFunction]));
+#endif
+            VBOXSTRICTRC const rcStrict = g_apfnIemThreadedFunctions[pCallEntry->enmFunction](pVCpu,
+                                                                                              pCallEntry->auParams[0],
+                                                                                              pCallEntry->auParams[1],
+                                                                                              pCallEntry->auParams[2]);
+            if (RT_LIKELY(   rcStrict == VINF_SUCCESS
+                          && pVCpu->iem.s.rcPassUp == VINF_SUCCESS /** @todo this isn't great. */))
+                pCallEntry++;
+            else
+            {
+                pVCpu->iem.s.cInstructions += pCallEntry->idxInstr; /* This may be one short, but better than zero. */
+                pVCpu->iem.s.pCurTbR3       = NULL;
+                STAM_REL_COUNTER_INC(&pVCpu->iem.s.StatTbExecBreaks);
+
+                /* VINF_IEM_REEXEC_BREAK should be treated as VINF_SUCCESS as it's
+                   only to break out of TB execution early. */
+                if (rcStrict == VINF_IEM_REEXEC_BREAK)
+                    return iemExecStatusCodeFiddling(pVCpu, VINF_SUCCESS);
+                return iemExecStatusCodeFiddling(pVCpu, rcStrict);
+            }
         }
     }
@@ -2320 +2459 @@
                 pTb = iemTbCacheLookup(pVCpu, pTbCache, GCPhysPc, fExtraFlags);
                 if (pTb)
-                {
-                    if (pTb->fFlags & IEMTB_F_TYPE_THREADED)
-                        rcStrict = iemThreadedTbExec(pVCpu, pTb);
-                    else
-                        AssertFailedStmt(rcStrict = VERR_INTERNAL_ERROR_4);
-                }
+                    rcStrict = iemTbExec(pVCpu, pTb);
                 else
                     rcStrict = iemThreadedCompile(pVM, pVCpu, GCPhysPc, fExtraFlags);
@@ -2346 +2480 @@
                         if (RT_LIKELY(   (iIterations & cPollRate) != 0
                                       || !TMTimerPollBoolWith32BitMilliTS(pVM, pVCpu, &pVCpu->iem.s.msRecompilerPollNow)))
-                        {
-
-                        }
+                            pTb = NULL; /* Clear it before looping so iemTbCacheLookup can safely do native recompilation. */
                         else
                             return VINF_SUCCESS;
@@ -2365 +2497 @@
                 iemMemRollback(pVCpu);
 
-#if 0 /** @todo do we need to clean up anything? */
-            /* If pTb isn't NULL we're in iemThreadedTbExec. */
+#if 0 /** @todo do we need to clean up anything?  If not, we can drop the pTb = NULL some lines up and change the scope. */
+            /* If pTb isn't NULL we're in iemTbExec. */
             if (!pTb)
             {

trunk/src/VBox/VMM/VMMR3/IEMR3.cpp

@@ -140 +140 @@
                           "InitialTbCount value %u (%#x) is higher than the MaxTbCount value %u (%#x)",
                           cInitialTbs, cInitialTbs, cMaxTbs, cMaxTbs);
-#endif
+
+    /** @cfgm{/IEM/MaxExecMem, uint64_t, 512 MiB}
+     * Max executable memory for recompiled code per EMT. */
+    uint64_t cbMaxExec = 0;
+    rc = CFGMR3QueryU64Def(pIem, "MaxExecMem", &cbMaxExec, _512M);
+    AssertLogRelRCReturn(rc, rc);
+    if (cbMaxExec < _1M || cbMaxExec > 16*_1G64)
+        return VMSetError(pVM, VERR_OUT_OF_RANGE, RT_SRC_POS,
+                          "MaxExecMem value %'RU64 (%#RX64) is out of range (min %'RU64, max %'RU64)",
+                          cbMaxExec, cbMaxExec, (uint64_t)_1M, 16*_1G64);
+
+    /** @cfgm{/IEM/ExecChunkSize, uint32_t, 0 (auto)}
+     * The executable memory allocator chunk size. */
+    uint32_t cbChunkExec = 0;
+    rc = CFGMR3QueryU32Def(pIem, "ExecChunkSize", &cbChunkExec, 0);
+    AssertLogRelRCReturn(rc, rc);
+    if (cbChunkExec != 0 && cbChunkExec != UINT32_MAX && (cbChunkExec < _1M || cbChunkExec > _256M))
+        return VMSetError(pVM, VERR_OUT_OF_RANGE, RT_SRC_POS,
+                          "ExecChunkSize value %'RU32 (%#RX32) is out of range (min %'RU32, max %'RU32)",
+                          cbChunkExec, cbChunkExec, _1M, _256M);
+
+    /** @cfgm{/IEM/InitialExecMemSize, uint64_t, 1}
+     * The initial executable memory allocator size (per EMT).  The value is
+     * rounded up to the nearest chunk size, so 1 byte means one chunk. */
+    uint64_t cbInitialExec = 0;
+    rc = CFGMR3QueryU64Def(pIem, "InitialExecMemSize", &cbInitialExec, 0);
+    AssertLogRelRCReturn(rc, rc);
+    if (cbInitialExec > cbMaxExec)
+        return VMSetError(pVM, VERR_OUT_OF_RANGE, RT_SRC_POS,
+                          "InitialExecMemSize value %'RU64 (%#RX64) is out of range (max %'RU64)",
+                          cbInitialExec, cbInitialExec, cbMaxExec);
+
+#endif /* VBOX_WITH_IEM_RECOMPILER*/
 
     /*
@@ -228 +260 @@
      * the allocations.
      */
-    rc = VMR3ReqCallWait(pVM, VMCPUID_ALL, (PFNRT)iemTbInit, 3, pVM, cInitialTbs, cMaxTbs);
+    rc = VMR3ReqCallWait(pVM, VMCPUID_ALL, (PFNRT)iemTbInit, 6,
+                         pVM, cInitialTbs, cMaxTbs, cbInitialExec, cbMaxExec, cbChunkExec);
     AssertLogRelRCReturn(rc, rc);
 #endif
@@ -286 +319 @@
 
 #ifdef VBOX_WITH_IEM_RECOMPILER
-        STAMR3RegisterF(pVM, (void *)&pVCpu->iem.s.cTbExec,             STAMTYPE_U64_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
-                        "Executed translation block",                   "/IEM/CPU%u/re/cTbExec", idCpu);
+        STAMR3RegisterF(pVM, (void *)&pVCpu->iem.s.cTbExecNative,       STAMTYPE_U64_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
+                        "Executed native translation block",            "/IEM/CPU%u/re/cTbExecNative", idCpu);
+        STAMR3RegisterF(pVM, (void *)&pVCpu->iem.s.cTbExecThreaded,     STAMTYPE_U64_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
+                        "Executed threaded translation block",          "/IEM/CPU%u/re/cTbExecThreaded", idCpu);
         STAMR3RegisterF(pVM, (void *)&pVCpu->iem.s.StatTbExecBreaks,    STAMTYPE_COUNTER,   STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,
                         "Times TB execution was interrupted/broken off", "/IEM/CPU%u/re/cTbExecBreaks", idCpu);

trunk/src/VBox/VMM/include/IEMInline.h

@@ -332 +332 @@
 }
 
+#ifdef VBOX_INCLUDED_vmm_dbgf_h /* VM::dbgf.ro.cEnabledHwBreakpoints is only accessible if VBox/vmm/dbgf.h is included. */
 
 /**
@@ -392 +393 @@
 }
 
+#endif /* VBOX_INCLUDED_vmm_dbgf_h */
+
 
 #ifndef IEM_WITH_OPAQUE_DECODER_STATE
 
 # if defined(VBOX_INCLUDED_vmm_dbgf_h) || defined(DOXYGEN_RUNNING) /* dbgf.ro.cEnabledHwBreakpoints */
+
 /**
  * Initializes the execution state.
@@ -458 +462 @@
 #  endif /* VBOX_STRICT */
 }
-# endif /* VBOX_INCLUDED_vmm_dbgf_h */
-
-
-# if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)
+
+
+#  if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)
 /**
  * Performs a minimal reinitialization of the execution state.
@@ -477 +480 @@
     iemOpcodeFlushHeavy(pVCpu, cbInstr);
 }
-# endif
-
+#  endif
+
+# endif /* VBOX_INCLUDED_vmm_dbgf_h || DOXYGEN_RUNNING */
 
 /**
@@ -3564 +3568 @@
 
 
+# ifdef XAPIC_OFF_END /* Requires VBox/apic.h to be included before IEMInline.h. */
 /**
  * Sets virtual-APIC write emulation as pending.
@@ -3588 +3593 @@
         VMCPU_FF_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE);
 }
+# endif /* XAPIC_OFF_END */
 
 #endif /* VBOX_WITH_NESTED_HWVIRT_VMX */

trunk/src/VBox/VMM/include/IEMInternal.h

@@ -742 +742 @@
 AssertCompile(!(IEM_F_MODE_X86_64BIT              & IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK));
 
+/** Native instruction type for use with the native code generator.
+ * This is a byte (uint8_t) for x86 and amd64 and uint32_t for the other(s). */
+#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
+typedef uint8_t IEMNATIVEINSTR;
+#else
+typedef uint32_t IEMNATIVEINSTR;
+#endif
+/** Pointer to a native instruction unit. */
+typedef IEMNATIVEINSTR *PIEMNATIVEINSTR;
+
 /**
  * A call for the threaded call table.
@@ -826 +836 @@
         struct
         {
-            uint8_t            *pbCode;
-            /** Amount of code that pbCode points to. */
-            uint32_t            cbAllocated;
+            /** The native instructions. */
+            PIEMNATIVEINSTR     paInstructions;
+            /** Number of instructions pointed to by paInstructions. */
+            uint32_t            cInstructions;
         } Native;
         /** Generic view for zeroing when freeing. */
@@ -974 +985 @@
     /** Statistics: Time spend pruning. */
     STAMPROFILE     StatPrune;
+
+    /** The delayed free list (see iemTbAlloctorScheduleForFree). */
+    PIEMTB          pDelayedFreeHead;
 
     /** Allocation chunks. */
@@ -1368 +1382 @@
      * This is allocated once with maxed-out sizes and re-used afterwards. */
     R3PTRTYPE(PIEMTB)       pThrdCompileTbR3;
-    /** Fixed TB used for native recompilation.
-     * This is allocated once and re-used afterwards, growing individual
-     * components as needed. */
-    R3PTRTYPE(PIEMTB)       pNativeCompileTbR3;
     /** Pointer to the ring-3 TB cache for this EMT. */
     R3PTRTYPE(PIEMTBCACHE)  pTbCacheR3;
@@ -1378 +1388 @@
      * RIP to opcode bytes stored in the TB (AMD-V / VT-x). */
     uint64_t                uCurTbStartPc;
-    /** Number of TBs executed. */
-    uint64_t                cTbExec;
+    /** Number of threaded TBs executed. */
+    uint64_t                cTbExecThreaded;
+    /** Number of native TBs executed. */
+    uint64_t                cTbExecNative;
     /** Whether we need to check the opcode bytes for the current instruction.
      * This is set by a previous instruction if it modified memory or similar.  */
@@ -1414 +1426 @@
     /** Pointer to the ring-3 TB allocator for this EMT. */
     R3PTRTYPE(PIEMTBALLOCATOR) pTbAllocatorR3;
-    /* Alignment. */
-    uint64_t                auAlignment10[7];
+    /** Pointer to the ring-3 executable memory allocator for this EMT. */
+    R3PTRTYPE(struct IEMEXECMEMALLOCATOR *) pExecMemAllocatorR3;
+
+    /** Native recompiler state for ring-3. */
+    struct IEMRECOMPILERSTATE
+    {
+        /** Size of the buffer that pbNativeRecompileBufR3 points to in
+         * IEMNATIVEINSTR units. */
+        uint32_t                            cInstrBufAlloc;
+        uint32_t                            uPadding; /* We don't keep track of this here... */
+        /** Fixed temporary code buffer for native recompilation. */
+        R3PTRTYPE(PIEMNATIVEINSTR)          pInstrBuf;
+
+        /** Actual number of labels in paLabels. */
+        uint32_t                            cLabels;
+        /** Max number of entries allowed in paLabels before reallocating it. */
+        uint32_t                            cLabelsAlloc;
+        /** Labels defined while recompiling (referenced by fixups). */
+        R3PTRTYPE(struct IEMNATIVELABEL *)  paLabels;
+
+        /** Actual number of fixups paFixups. */
+        uint32_t                            cFixups;
+        /** Max number of entries allowed in paFixups before reallocating it. */
+        uint32_t                            cFixupsAlloc;
+        /** Buffer used by the recompiler for recording fixups when generating code. */
+        R3PTRTYPE(struct IEMNATIVEFIXUP *)  paFixups;
+    } Native;
+
+//    /* Alignment. */
+//    uint64_t                auAlignment10[1];
     /** Statistics: Times TB execution was broken off before reaching the end. */
     STAMCOUNTER             StatTbExecBreaks;
@@ -1430 +1470 @@
     /** Threaded TB statistics: Number of calls per TB. */
     STAMPROFILE             StatTbThreadedCalls;
+    /** Native TB statistics: Native code size per TB. */
+    STAMPROFILE             StatTbNativeCode;
+    /** Native TB statistics: Profiling native recompilation. */
+    STAMPROFILE             StatNativeRecompilation;
     /** @} */
 
@@ -5290 +5334 @@
 extern const PFNIEMOP g_apfnIemThreadedRecompilerVecMap3[1024];
 
-DECLCALLBACK(int) iemTbInit(PVMCC pVM, uint32_t cInitialTbs, uint32_t cMaxTbs);
-void            iemThreadedTbObsolete(PVMCPUCC pVCpu, PIEMTB pTb);
+DECLCALLBACK(int)   iemTbInit(PVMCC pVM, uint32_t cInitialTbs, uint32_t cMaxTbs,
+                              uint64_t cbInitialExec, uint64_t cbMaxExec, uint32_t cbChunkExec);
+void                iemThreadedTbObsolete(PVMCPUCC pVCpu, PIEMTB pTb, bool fSafeToFree);
+void                iemTbAllocatorProcessDelayedFrees(PVMCPU pVCpu, PIEMTBALLOCATOR pTbAllocator);
+
 
 /** @todo FNIEMTHREADEDFUNC and friends may need more work... */
@@ -5348 +5395 @@
 bool iemThreadedCompileBeginEmitCallsComplications(PVMCPUCC pVCpu, PIEMTB pTb);
 
+/* Native recompiler public bits: */
+PIEMTB iemNativeRecompile(PVMCPUCC pVCpu, PIEMTB pTb);
+int    iemExecMemAllocatorInit(PVMCPU pVCpu, uint64_t cbMax, uint64_t cbInitial, uint32_t cbChunk);
+void   iemExecMemAllocatorFree(PVMCPU pVCpu, void *pv, size_t cb);
+
 
 /** @} */