Changeset 27404 in vbox

Timestamp:

Mar 16, 2010 1:34:17 PM (15 years ago)

Author:

vboxsync

Message:

alloc-ef.cpp: fix fence in front case no man's area filling (had a rounding error leading to heap corruption), and additionally fill and check the alignment padding area after the allocated buffer in the fence after the block case.

File:

• trunk/src/VBox/Runtime/r3/alloc-ef.cpp (modified) (3 diffs)

trunk/src/VBox/Runtime/r3/alloc-ef.cpp

@@ -292 +292 @@
 #endif
     }
-    cb = RT_ALIGN_Z(cb, RTALLOC_EFENCE_ALIGNMENT);
+#ifndef RTALLOC_EFENCE_IN_FRONT
+    /* Alignment decreases fence accuracy, but this is at least partially
+     * counteracted by filling and checking the alignment padding. When the
+     * fence is in front then then no extra alignment is needed. */
+    size_t cbAlign = RT_ALIGN_Z(cb, RTALLOC_EFENCE_ALIGNMENT);
+#endif
 
 #ifdef RTALLOC_EFENCE_TRACE
@@ -321 +326 @@
         void *pv = (char *)pvEFence + RTALLOC_EFENCE_SIZE;
 #ifdef RTALLOC_EFENCE_NOMAN_FILLER
-        memset((char *)pv + cb, RTALLOC_EFENCE_NOMAN_FILLER, PAGE_SIZE - cb % PAGE_SIZE);
+        memset((char *)pv + cb, RTALLOC_EFENCE_NOMAN_FILLER, cbBlock - RTALLOC_EFENCE_SIZE - cb);
 #endif
         #else
         void *pvEFence = (char *)pvBlock + (cbBlock - RTALLOC_EFENCE_SIZE);
-        void *pv = (char *)pvEFence - cb;
+        void *pv = (char *)pvEFence - cbAlign;
 #ifdef RTALLOC_EFENCE_NOMAN_FILLER
-        memset(pvBlock, RTALLOC_EFENCE_NOMAN_FILLER, cbBlock - RTALLOC_EFENCE_SIZE - cb);
+        memset(pvBlock, RTALLOC_EFENCE_NOMAN_FILLER, cbBlock - RTALLOC_EFENCE_SIZE - cbAlign);
+        memset((char *)pv + cb, RTALLOC_EFENCE_NOMAN_FILLER, cbAlign - cb);
 #endif
         #endif
@@ -396 +402 @@
          */
 # ifdef RTALLOC_EFENCE_IN_FRONT
-        void *pvNoMan = (char *)pv + pBlock->cb;
-# else
-        void *pvNoMan = (void *)((uintptr_t)pv & ~PAGE_OFFSET_MASK);
-# endif
-        void *p = ASMMemIsAll8(pvNoMan,
+        void *p = ASMMemIsAll8((char *)pv + pBlock->cb,
                                RT_ALIGN_Z(pBlock->cb, PAGE_SIZE) - pBlock->cb,
                                RTALLOC_EFENCE_NOMAN_FILLER);
+# else
+        /* Alignment must match allocation alignment in rtMemAlloc(). */
+        size_t cbAlign = RT_ALIGN_Z(pBlock->cb, RTALLOC_EFENCE_ALIGNMENT);
+        void *p = ASMMemIsAll8((char *)pv + pBlock->cb,
+                               cbAlign - pBlock->cb,
+                               RTALLOC_EFENCE_NOMAN_FILLER);
+        if (p)
+            RTAssertDoPanic();
+        p = ASMMemIsAll8((void *)((uintptr_t)pv & ~PAGE_OFFSET_MASK),
+                         RT_ALIGN_Z(cbAlign, PAGE_SIZE) - cbAlign,
+                         RTALLOC_EFENCE_NOMAN_FILLER);
+# endif
         if (p)
             RTAssertDoPanic();