source: vbox/trunk/src/VBox/Storage/VD.cpp@ 99739

/* $Id: VD.cpp 99739 2023-05-11 01:01:08Z vboxsync $ */
/** @file
 * VD - Virtual disk container implementation.
 */

/*
 * Copyright (C) 2006-2023 Oracle and/or its affiliates.
 *
 * This file is part of VirtualBox base platform packages, as
 * available from https://www.virtualbox.org.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation, in version 3 of the
 * License.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, see <https://www.gnu.org/licenses>.
 *
 * SPDX-License-Identifier: GPL-3.0-only
 */


/*********************************************************************************************************************************
*   Header Files                                                                                                                 *
*********************************************************************************************************************************/
#define LOG_GROUP LOG_GROUP_VD
#include <VBox/vd.h>
#include <VBox/err.h>
#include <VBox/sup.h>
#include <VBox/log.h>

#include <iprt/alloc.h>
#include <iprt/assert.h>
#include <iprt/uuid.h>
#include <iprt/file.h>
#include <iprt/string.h>
#include <iprt/asm.h>
#include <iprt/param.h>
#include <iprt/path.h>
#include <iprt/sg.h>
#include <iprt/semaphore.h>
#include <iprt/vector.h>

#include "VDInternal.h"

/** Buffer size used for merging images. */
#define VD_MERGE_BUFFER_SIZE    (16 * _1M)

/** Maximum number of segments in one I/O task. */
#define VD_IO_TASK_SEGMENTS_MAX 64

/** Threshold after not recently used blocks are removed from the list. */
#define VD_DISCARD_REMOVE_THRESHOLD (10 * _1M) /** @todo experiment */

/**
 * VD async I/O interface storage descriptor.
 */
typedef struct VDIIOFALLBACKSTORAGE
{
    /** File handle. */
    RTFILE              File;
    /** Completion callback. */
    PFNVDCOMPLETED      pfnCompleted;
    /** Thread for async access. */
    RTTHREAD            ThreadAsync;
} VDIIOFALLBACKSTORAGE, *PVDIIOFALLBACKSTORAGE;

/**
 * uModified bit flags.
 */
#define VD_IMAGE_MODIFIED_FLAG                  RT_BIT(0)
#define VD_IMAGE_MODIFIED_FIRST                 RT_BIT(1)
#define VD_IMAGE_MODIFIED_DISABLE_UUID_UPDATE   RT_BIT(2)


# define VD_IS_LOCKED(a_pDisk) \
    do \
    { \
        NOREF(a_pDisk); \
        AssertMsg((a_pDisk)->fLocked, \
                  ("Lock not held\n"));\
    } while(0)

/**
 * VBox parent read descriptor, used internally for compaction.
 */
typedef struct VDPARENTSTATEDESC
{
    /** Pointer to disk descriptor. */
    PVDISK pDisk;
    /** Pointer to image descriptor. */
    PVDIMAGE pImage;
} VDPARENTSTATEDESC, *PVDPARENTSTATEDESC;

/**
 * Transfer direction.
 */
typedef enum VDIOCTXTXDIR
{
    /** Read */
    VDIOCTXTXDIR_READ = 0,
    /** Write */
    VDIOCTXTXDIR_WRITE,
    /** Flush */
    VDIOCTXTXDIR_FLUSH,
    /** Discard */
    VDIOCTXTXDIR_DISCARD,
    /** 32bit hack */
    VDIOCTXTXDIR_32BIT_HACK = 0x7fffffff
} VDIOCTXTXDIR, *PVDIOCTXTXDIR;

/** Transfer function */
typedef DECLCALLBACKTYPE(int, FNVDIOCTXTRANSFER ,(PVDIOCTX pIoCtx));
/** Pointer to a transfer function. */
typedef FNVDIOCTXTRANSFER *PFNVDIOCTXTRANSFER;

/**
 * I/O context
 */
typedef struct VDIOCTX
{
    /** Pointer to the next I/O context. */
    struct VDIOCTX * volatile    pIoCtxNext;
    /** Disk this is request is for. */
    PVDISK                       pDisk;
    /** Return code. */
    int                          rcReq;
    /** Various flags for the I/O context. */
    uint32_t                     fFlags;
    /** Number of data transfers currently pending. */
    volatile uint32_t            cDataTransfersPending;
    /** How many meta data transfers are pending. */
    volatile uint32_t            cMetaTransfersPending;
    /** Flag whether the request finished */
    volatile bool                fComplete;
    /** Temporary allocated memory which is freed
     * when the context completes. */
    void                        *pvAllocation;
    /** Transfer function. */
    PFNVDIOCTXTRANSFER           pfnIoCtxTransfer;
    /** Next transfer part after the current one completed. */
    PFNVDIOCTXTRANSFER           pfnIoCtxTransferNext;
    /** Transfer direction */
    VDIOCTXTXDIR                 enmTxDir;
    /** Request type dependent data. */
    union
    {
        /** I/O request (read/write). */
        struct
        {
            /** Number of bytes left until this context completes. */
            volatile uint32_t    cbTransferLeft;
            /** Current offset */
            volatile uint64_t    uOffset;
            /** Number of bytes to transfer */
            volatile size_t      cbTransfer;
            /** Current image in the chain. */
            PVDIMAGE             pImageCur;
            /** Start image to read from. pImageCur is reset to this
             *  value after it reached the first image in the chain. */
            PVDIMAGE             pImageStart;
            /** S/G buffer */
            RTSGBUF              SgBuf;
            /** Number of bytes to clear in the buffer before the current read. */
            size_t               cbBufClear;
            /** Number of images to read. */
            unsigned             cImagesRead;
            /** Override for the parent image to start reading from. */
            PVDIMAGE             pImageParentOverride;
            /** Original offset of the transfer - required for filtering read requests. */
            uint64_t             uOffsetXferOrig;
            /** Original size of the transfer - required for fitlering read requests. */
            size_t               cbXferOrig;
        } Io;
        /** Discard requests. */
        struct
        {
            /** Pointer to the range descriptor array. */
            PCRTRANGE            paRanges;
            /** Number of ranges in the array. */
            unsigned             cRanges;
            /** Range descriptor index which is processed. */
            unsigned             idxRange;
            /** Start offset to discard currently. */
            uint64_t             offCur;
            /** How many bytes left to discard in the current range. */
            size_t               cbDiscardLeft;
            /** How many bytes to discard in the current block (<= cbDiscardLeft). */
            size_t               cbThisDiscard;
            /** Discard block handled currently. */
            PVDDISCARDBLOCK      pBlock;
        } Discard;
    } Req;
    /** Parent I/O context if any. Sets the type of the context (root/child) */
    PVDIOCTX                     pIoCtxParent;
    /** Type dependent data (root/child) */
    union
    {
        /** Root data */
        struct
        {
            /** Completion callback */
            PFNVDASYNCTRANSFERCOMPLETE   pfnComplete;
            /** User argument 1 passed on completion. */
            void                        *pvUser1;
            /** User argument 2 passed on completion. */
            void                        *pvUser2;
        } Root;
        /** Child data */
        struct
        {
            /** Saved start offset */
            uint64_t                     uOffsetSaved;
            /** Saved transfer size */
            size_t                       cbTransferLeftSaved;
            /** Number of bytes transferred from the parent if this context completes. */
            size_t                       cbTransferParent;
            /** Number of bytes to pre read */
            size_t                       cbPreRead;
            /** Number of bytes to post read. */
            size_t                       cbPostRead;
            /** Number of bytes to write left in the parent. */
            size_t                       cbWriteParent;
            /** Write type dependent data. */
            union
            {
                /** Optimized */
                struct
                {
                    /** Bytes to fill to satisfy the block size. Not part of the virtual disk. */
                    size_t               cbFill;
                    /** Bytes to copy instead of reading from the parent */
                    size_t               cbWriteCopy;
                    /** Bytes to read from the image. */
                    size_t               cbReadImage;
                } Optimized;
            } Write;
        } Child;
    } Type;
} VDIOCTX;

/** Default flags for an I/O context, i.e. unblocked and async. */
#define VDIOCTX_FLAGS_DEFAULT                         (0)
/** Flag whether the context is blocked. */
#define VDIOCTX_FLAGS_BLOCKED                RT_BIT_32(0)
/** Flag whether the I/O context is using synchronous I/O. */
#define VDIOCTX_FLAGS_SYNC                   RT_BIT_32(1)
/** Flag whether the read should update the cache. */
#define VDIOCTX_FLAGS_READ_UPDATE_CACHE      RT_BIT_32(2)
/** Flag whether free blocks should be zeroed.
 * If false and no image has data for sepcified
 * range VERR_VD_BLOCK_FREE is returned for the I/O context.
 * Note that unallocated blocks are still zeroed
 * if at least one image has valid data for a part
 * of the range.
 */
#define VDIOCTX_FLAGS_ZERO_FREE_BLOCKS       RT_BIT_32(3)
/** Don't free the I/O context when complete because
 * it was alloacted elsewhere (stack, ...). */
#define VDIOCTX_FLAGS_DONT_FREE              RT_BIT_32(4)
/** Don't set the modified flag for this I/O context when writing. */
#define VDIOCTX_FLAGS_DONT_SET_MODIFIED_FLAG RT_BIT_32(5)
/** The write filter was applied already and shouldn't be applied a second time.
 * Used at the beginning of vdWriteHelperAsync() because it might be called
 * multiple times.
 */
#define VDIOCTX_FLAGS_WRITE_FILTER_APPLIED   RT_BIT_32(6)

/** NIL I/O context pointer value. */
#define NIL_VDIOCTX ((PVDIOCTX)0)

/**
 * List node for deferred I/O contexts.
 */
typedef struct VDIOCTXDEFERRED
{
    /** Node in the list of deferred requests.
     * A request can be deferred if the image is growing
     * and the request accesses the same range or if
     * the backend needs to read or write metadata from the disk
     * before it can continue. */
    RTLISTNODE NodeDeferred;
    /** I/O context this entry points to. */
    PVDIOCTX   pIoCtx;
} VDIOCTXDEFERRED, *PVDIOCTXDEFERRED;

/**
 * I/O task.
 */
typedef struct VDIOTASK
{
    /** Next I/O task waiting in the list. */
    struct VDIOTASK * volatile   pNext;
    /** Storage this task belongs to. */
    PVDIOSTORAGE                 pIoStorage;
    /** Optional completion callback. */
    PFNVDXFERCOMPLETED           pfnComplete;
    /** Opaque user data. */
    void                        *pvUser;
    /** Completion status code for the task. */
    int                          rcReq;
    /** Flag whether this is a meta data transfer. */
    bool                         fMeta;
    /** Type dependent data. */
    union
    {
        /** User data transfer. */
        struct
        {
            /** Number of bytes this task transferred. */
            uint32_t             cbTransfer;
            /** Pointer to the I/O context the task belongs. */
            PVDIOCTX             pIoCtx;
        } User;
        /** Meta data transfer. */
        struct
        {
            /** Meta transfer this task is for. */
            PVDMETAXFER          pMetaXfer;
        } Meta;
    } Type;
} VDIOTASK;

/**
 * Storage handle.
 */
typedef struct VDIOSTORAGE
{
    /** Image I/O state this storage handle belongs to. */
    PVDIO                        pVDIo;
    /** AVL tree for pending async metadata transfers. */
    PAVLRFOFFTREE                pTreeMetaXfers;
    /** Storage handle */
    void                        *pStorage;
} VDIOSTORAGE;

/**
 *  Metadata transfer.
 *
 *  @note This entry can't be freed if either the list is not empty or
 *  the reference counter is not 0.
 *  The assumption is that the backends don't need to read huge amounts of
 *  metadata to complete a transfer so the additional memory overhead should
 *  be relatively small.
 */
typedef struct VDMETAXFER
{
    /** AVL core for fast search (the file offset is the key) */
    AVLRFOFFNODECORE Core;
    /** I/O storage for this transfer. */
    PVDIOSTORAGE     pIoStorage;
    /** Flags. */
    uint32_t         fFlags;
    /** List of I/O contexts waiting for this metadata transfer to complete. */
    RTLISTNODE       ListIoCtxWaiting;
    /** Number of references to this entry. */
    unsigned         cRefs;
    /** Size of the data stored with this entry. */
    size_t           cbMeta;
    /** Shadow buffer which is used in case a write is still active and other
     * writes update the shadow buffer. */
    uint8_t         *pbDataShw;
    /** List of I/O contexts updating the shadow buffer while there is a write
     * in progress. */
    RTLISTNODE       ListIoCtxShwWrites;
    /** Data stored - variable size. */
    uint8_t          abData[1];
} VDMETAXFER;

/**
 * The transfer direction for the metadata.
 */
#define VDMETAXFER_TXDIR_MASK  0x3
#define VDMETAXFER_TXDIR_NONE  0x0
#define VDMETAXFER_TXDIR_WRITE 0x1
#define VDMETAXFER_TXDIR_READ  0x2
#define VDMETAXFER_TXDIR_FLUSH 0x3
#define VDMETAXFER_TXDIR_GET(flags)      ((flags) & VDMETAXFER_TXDIR_MASK)
#define VDMETAXFER_TXDIR_SET(flags, dir) ((flags) = (flags & ~VDMETAXFER_TXDIR_MASK) | (dir))

/** Forward declaration of the async discard helper. */
static DECLCALLBACK(int) vdDiscardHelperAsync(PVDIOCTX pIoCtx);
static DECLCALLBACK(int) vdWriteHelperAsync(PVDIOCTX pIoCtx);
static void vdDiskProcessBlockedIoCtx(PVDISK pDisk);
static int vdDiskUnlock(PVDISK pDisk, PVDIOCTX pIoCtxRc);
static DECLCALLBACK(void) vdIoCtxSyncComplete(void *pvUser1, void *pvUser2, int rcReq);

/**
 * internal: issue error message.
 */
static int vdError(PVDISK pDisk, int rc, RT_SRC_POS_DECL,
                   const char *pszFormat, ...)
{
    va_list va;
    va_start(va, pszFormat);
    if (pDisk->pInterfaceError)
        pDisk->pInterfaceError->pfnError(pDisk->pInterfaceError->Core.pvUser, rc, RT_SRC_POS_ARGS, pszFormat, va);
    va_end(va);
    return rc;
}

/**
 * internal: thread synchronization, start read.
 */
DECLINLINE(int) vdThreadStartRead(PVDISK pDisk)
{
    int rc = VINF_SUCCESS;
    if (RT_UNLIKELY(pDisk->pInterfaceThreadSync))
        rc = pDisk->pInterfaceThreadSync->pfnStartRead(pDisk->pInterfaceThreadSync->Core.pvUser);
    return rc;
}

/**
 * internal: thread synchronization, finish read.
 */
DECLINLINE(int) vdThreadFinishRead(PVDISK pDisk)
{
    int rc = VINF_SUCCESS;
    if (RT_UNLIKELY(pDisk->pInterfaceThreadSync))
        rc = pDisk->pInterfaceThreadSync->pfnFinishRead(pDisk->pInterfaceThreadSync->Core.pvUser);
    return rc;
}

/**
 * internal: thread synchronization, start write.
 */
DECLINLINE(int) vdThreadStartWrite(PVDISK pDisk)
{
    int rc = VINF_SUCCESS;
    if (RT_UNLIKELY(pDisk->pInterfaceThreadSync))
        rc = pDisk->pInterfaceThreadSync->pfnStartWrite(pDisk->pInterfaceThreadSync->Core.pvUser);
    return rc;
}

/**
 * internal: thread synchronization, finish write.
 */
DECLINLINE(int) vdThreadFinishWrite(PVDISK pDisk)
{
    int rc = VINF_SUCCESS;
    if (RT_UNLIKELY(pDisk->pInterfaceThreadSync))
        rc = pDisk->pInterfaceThreadSync->pfnFinishWrite(pDisk->pInterfaceThreadSync->Core.pvUser);
    return rc;
}

/**
 * internal: add image structure to the end of images list.
 */
static void vdAddImageToList(PVDISK pDisk, PVDIMAGE pImage)
{
    pImage->pPrev = NULL;
    pImage->pNext = NULL;

    if (pDisk->pBase)
    {
        Assert(pDisk->cImages > 0);
        pImage->pPrev = pDisk->pLast;
        pDisk->pLast->pNext = pImage;
        pDisk->pLast = pImage;
    }
    else
    {
        Assert(pDisk->cImages == 0);
        pDisk->pBase = pImage;
        pDisk->pLast = pImage;
    }

    pDisk->cImages++;
}

/**
 * internal: remove image structure from the images list.
 */
static void vdRemoveImageFromList(PVDISK pDisk, PVDIMAGE pImage)
{
    Assert(pDisk->cImages > 0);

    if (pImage->pPrev)
        pImage->pPrev->pNext = pImage->pNext;
    else
        pDisk->pBase = pImage->pNext;

    if (pImage->pNext)
        pImage->pNext->pPrev = pImage->pPrev;
    else
        pDisk->pLast = pImage->pPrev;

    pImage->pPrev = NULL;
    pImage->pNext = NULL;

    pDisk->cImages--;
}

/**
 * Release a referene to the filter decrementing the counter and destroying the filter
 * when the counter reaches zero.
 *
 * @returns The new reference count.
 * @param   pFilter    The filter to release.
 */
static uint32_t vdFilterRelease(PVDFILTER pFilter)
{
    uint32_t cRefs = ASMAtomicDecU32(&pFilter->cRefs);
    if (!cRefs)
    {
        pFilter->pBackend->pfnDestroy(pFilter->pvBackendData);
        RTMemFree(pFilter);
    }

    return cRefs;
}

/**
 * Increments the reference counter of the given filter.
 *
 * @return The new reference count.
 * @param  pFilter    The filter.
 */
static uint32_t vdFilterRetain(PVDFILTER pFilter)
{
    return ASMAtomicIncU32(&pFilter->cRefs);
}

/**
 * internal: find image by index into the images list.
 */
static PVDIMAGE vdGetImageByNumber(PVDISK pDisk, unsigned nImage)
{
    PVDIMAGE pImage = pDisk->pBase;
    if (nImage == VD_LAST_IMAGE)
        return pDisk->pLast;
    while (pImage && nImage)
    {
        pImage = pImage->pNext;
        nImage--;
    }
    return pImage;
}

/**
 * Creates a new region list from the given one converting to match the flags if necessary.
 *
 * @returns VBox status code.
 * @param   pRegionList     The region list to convert from.
 * @param   fFlags          The flags for the new region list.
 * @param   ppRegionList    Where to store the new region list on success.
 */
static int vdRegionListConv(PCVDREGIONLIST pRegionList, uint32_t fFlags, PPVDREGIONLIST ppRegionList)
{
    int rc = VINF_SUCCESS;
    PVDREGIONLIST pRegionListNew = (PVDREGIONLIST)RTMemDup(pRegionList,
                                                           RT_UOFFSETOF_DYN(VDREGIONLIST, aRegions[pRegionList->cRegions]));
    if (RT_LIKELY(pRegionListNew))
    {
        /* Do we have to convert anything? */
        if (pRegionList->fFlags != fFlags)
        {
            uint64_t offRegionNext = 0;

            pRegionListNew->fFlags = fFlags;
            for (unsigned i = 0; i < pRegionListNew->cRegions; i++)
            {
                PVDREGIONDESC pRegion = &pRegionListNew->aRegions[i];

                if (   (fFlags & VD_REGION_LIST_F_LOC_SIZE_BLOCKS)
                    && !(pRegionList->fFlags & VD_REGION_LIST_F_LOC_SIZE_BLOCKS))
                {
                    Assert(!(pRegion->cRegionBlocksOrBytes % pRegion->cbBlock));

                    /* Convert from bytes to logical blocks. */
                    pRegion->offRegion            = offRegionNext;
                    pRegion->cRegionBlocksOrBytes = pRegion->cRegionBlocksOrBytes / pRegion->cbBlock;
                    offRegionNext += pRegion->cRegionBlocksOrBytes;
                }
                else
                {
                    /* Convert from logical blocks to bytes. */
                    pRegion->offRegion            = offRegionNext;
                    pRegion->cRegionBlocksOrBytes = pRegion->cRegionBlocksOrBytes * pRegion->cbBlock;
                    offRegionNext += pRegion->cRegionBlocksOrBytes;
                }
            }
        }

        *ppRegionList = pRegionListNew;
    }
    else
        rc = VERR_NO_MEMORY;

    return rc;
}

/**
 * Returns the virtual size of the image in bytes.
 *
 * @returns Size of the given image in bytes.
 * @param   pImage          The image to get the size from.
 */
static uint64_t vdImageGetSize(PVDIMAGE pImage)
{
    uint64_t cbImage = 0;

    if (pImage->cbImage == VD_IMAGE_SIZE_UNINITIALIZED)
    {
        PCVDREGIONLIST pRegionList = NULL;
        int rc = pImage->Backend->pfnQueryRegions(pImage->pBackendData, &pRegionList);
        if (RT_SUCCESS(rc))
        {
            if (pRegionList->fFlags & VD_REGION_LIST_F_LOC_SIZE_BLOCKS)
            {
                PVDREGIONLIST pRegionListConv = NULL;
                rc = vdRegionListConv(pRegionList, 0, &pRegionListConv);
                if (RT_SUCCESS(rc))
                {
                    for (uint32_t i = 0; i < pRegionListConv->cRegions; i++)
                        cbImage += pRegionListConv->aRegions[i].cRegionBlocksOrBytes;

                    VDRegionListFree(pRegionListConv);
                }
            }
            else
                for (uint32_t i = 0; i < pRegionList->cRegions; i++)
                    cbImage += pRegionList->aRegions[i].cRegionBlocksOrBytes;

            AssertPtr(pImage->Backend->pfnRegionListRelease);
            pImage->Backend->pfnRegionListRelease(pImage->pBackendData, pRegionList);
            pImage->cbImage = cbImage; /* Cache the value. */
        }
    }
    else
        cbImage = pImage->cbImage;

    return cbImage;
}

/**
 * Applies the filter chain to the given write request.
 *
 * @returns VBox status code.
 * @param   pDisk    The HDD container.
 * @param   uOffset  The start offset of the write.
 * @param   cbWrite  Number of bytes to write.
 * @param   pIoCtx   The I/O context associated with the request.
 */
static int vdFilterChainApplyWrite(PVDISK pDisk, uint64_t uOffset, size_t cbWrite,
                                   PVDIOCTX pIoCtx)
{
    int rc = VINF_SUCCESS;

    VD_IS_LOCKED(pDisk);

    PVDFILTER pFilter;
    RTListForEach(&pDisk->ListFilterChainWrite, pFilter, VDFILTER, ListNodeChainWrite)
    {
        rc = pFilter->pBackend->pfnFilterWrite(pFilter->pvBackendData, uOffset, cbWrite, pIoCtx);
        if (RT_FAILURE(rc))
            break;
        /* Reset S/G buffer for the next filter. */
        RTSgBufReset(&pIoCtx->Req.Io.SgBuf);
    }

    return rc;
}

/**
 * Applies the filter chain to the given read request.
 *
 * @returns VBox status code.
 * @param   pDisk    The HDD container.
 * @param   uOffset  The start offset of the read.
 * @param   cbRead   Number of bytes read.
 * @param   pIoCtx   The I/O context associated with the request.
 */
static int vdFilterChainApplyRead(PVDISK pDisk, uint64_t uOffset, size_t cbRead,
                                  PVDIOCTX pIoCtx)
{
    int rc = VINF_SUCCESS;

    VD_IS_LOCKED(pDisk);

    /* Reset buffer before starting. */
    RTSgBufReset(&pIoCtx->Req.Io.SgBuf);

    PVDFILTER pFilter;
    RTListForEach(&pDisk->ListFilterChainRead, pFilter, VDFILTER, ListNodeChainRead)
    {
        rc = pFilter->pBackend->pfnFilterRead(pFilter->pvBackendData, uOffset, cbRead, pIoCtx);
        if (RT_FAILURE(rc))
            break;
        /* Reset S/G buffer for the next filter. */
        RTSgBufReset(&pIoCtx->Req.Io.SgBuf);
    }

    return rc;
}

DECLINLINE(void) vdIoCtxRootComplete(PVDISK pDisk, PVDIOCTX pIoCtx)
{
    if (   RT_SUCCESS(pIoCtx->rcReq)
        && pIoCtx->enmTxDir == VDIOCTXTXDIR_READ)
        pIoCtx->rcReq = vdFilterChainApplyRead(pDisk, pIoCtx->Req.Io.uOffsetXferOrig,
                                               pIoCtx->Req.Io.cbXferOrig, pIoCtx);

    pIoCtx->Type.Root.pfnComplete(pIoCtx->Type.Root.pvUser1,
                                  pIoCtx->Type.Root.pvUser2,
                                  pIoCtx->rcReq);
}

/**
 * Initialize the structure members of a given I/O context.
 */
DECLINLINE(void) vdIoCtxInit(PVDIOCTX pIoCtx, PVDISK pDisk, VDIOCTXTXDIR enmTxDir,
                             uint64_t uOffset, size_t cbTransfer, PVDIMAGE pImageStart,
                             PCRTSGBUF pSgBuf, void *pvAllocation,
                             PFNVDIOCTXTRANSFER pfnIoCtxTransfer, uint32_t fFlags)
{
    pIoCtx->pDisk                 = pDisk;
    pIoCtx->enmTxDir              = enmTxDir;
    pIoCtx->Req.Io.cbTransferLeft = (uint32_t)cbTransfer; Assert((uint32_t)cbTransfer == cbTransfer);
    pIoCtx->Req.Io.uOffset        = uOffset;
    pIoCtx->Req.Io.cbTransfer     = cbTransfer;
    pIoCtx->Req.Io.pImageStart    = pImageStart;
    pIoCtx->Req.Io.pImageCur      = pImageStart;
    pIoCtx->Req.Io.cbBufClear     = 0;
    pIoCtx->Req.Io.pImageParentOverride = NULL;
    pIoCtx->Req.Io.uOffsetXferOrig      = uOffset;
    pIoCtx->Req.Io.cbXferOrig           = cbTransfer;
    pIoCtx->cDataTransfersPending = 0;
    pIoCtx->cMetaTransfersPending = 0;
    pIoCtx->fComplete             = false;
    pIoCtx->fFlags                = fFlags;
    pIoCtx->pvAllocation          = pvAllocation;
    pIoCtx->pfnIoCtxTransfer      = pfnIoCtxTransfer;
    pIoCtx->pfnIoCtxTransferNext  = NULL;
    pIoCtx->rcReq                 = VINF_SUCCESS;
    pIoCtx->pIoCtxParent          = NULL;

    /* There is no S/G list for a flush request. */
    if (   enmTxDir != VDIOCTXTXDIR_FLUSH
        && enmTxDir != VDIOCTXTXDIR_DISCARD)
        RTSgBufClone(&pIoCtx->Req.Io.SgBuf, pSgBuf);
    else
        memset(&pIoCtx->Req.Io.SgBuf, 0, sizeof(RTSGBUF));
}

/**
 * Internal: Tries to read the desired range from the given cache.
 *
 * @returns VBox status code.
 * @retval  VERR_VD_BLOCK_FREE if the block is not in the cache.
 *          pcbRead will be set to the number of bytes not in the cache.
 *          Everything thereafter might be in the cache.
 * @param   pCache   The cache to read from.
 * @param   uOffset  Offset of the virtual disk to read.
 * @param   cbRead   How much to read.
 * @param   pIoCtx   The I/O context to read into.
 * @param   pcbRead  Where to store the number of bytes actually read.
 *                   On success this indicates the number of bytes read from the cache.
 *                   If VERR_VD_BLOCK_FREE is returned this gives the number of bytes
 *                   which are not in the cache.
 *                   In both cases everything beyond this value
 *                   might or might not be in the cache.
 */
static int vdCacheReadHelper(PVDCACHE pCache, uint64_t uOffset,
                             size_t cbRead, PVDIOCTX pIoCtx, size_t *pcbRead)
{
    int rc = VINF_SUCCESS;

    LogFlowFunc(("pCache=%#p uOffset=%llu pIoCtx=%p cbRead=%zu pcbRead=%#p\n",
                 pCache, uOffset, pIoCtx, cbRead, pcbRead));

    AssertPtr(pCache);
    AssertPtr(pcbRead);

    rc = pCache->Backend->pfnRead(pCache->pBackendData, uOffset, cbRead,
                                  pIoCtx, pcbRead);

    LogFlowFunc(("returns rc=%Rrc pcbRead=%zu\n", rc, *pcbRead));
    return rc;
}

/**
 * Internal: Writes data for the given block into the cache.
 *
 * @returns VBox status code.
 * @param   pCache     The cache to write to.
 * @param   uOffset    Offset of the virtual disk to write to the cache.
 * @param   cbWrite    How much to write.
 * @param   pIoCtx     The I/O context to write from.
 * @param   pcbWritten How much data could be written, optional.
 */
static int vdCacheWriteHelper(PVDCACHE pCache, uint64_t uOffset, size_t cbWrite,
                              PVDIOCTX pIoCtx, size_t *pcbWritten)
{
    int rc = VINF_SUCCESS;

    LogFlowFunc(("pCache=%#p uOffset=%llu pIoCtx=%p cbWrite=%zu pcbWritten=%#p\n",
                 pCache, uOffset, pIoCtx, cbWrite, pcbWritten));

    AssertPtr(pCache);
    AssertPtr(pIoCtx);
    Assert(cbWrite > 0);

    if (pcbWritten)
        rc = pCache->Backend->pfnWrite(pCache->pBackendData, uOffset, cbWrite,
                                       pIoCtx, pcbWritten);
    else
    {
        size_t cbWritten = 0;

        do
        {
            rc = pCache->Backend->pfnWrite(pCache->pBackendData, uOffset, cbWrite,
                                           pIoCtx, &cbWritten);
            uOffset += cbWritten;
            cbWrite -= cbWritten;
        } while (   cbWrite
                 && (   RT_SUCCESS(rc)
                     || rc == VERR_VD_ASYNC_IO_IN_PROGRESS));
    }

    LogFlowFunc(("returns rc=%Rrc pcbWritten=%zu\n",
                 rc, pcbWritten ? *pcbWritten : cbWrite));
    return rc;
}

/**
 * Creates a new empty discard state.
 *
 * @returns Pointer to the new discard state or NULL if out of memory.
 */
static PVDDISCARDSTATE vdDiscardStateCreate(void)
{
    PVDDISCARDSTATE pDiscard = (PVDDISCARDSTATE)RTMemAllocZ(sizeof(VDDISCARDSTATE));

    if (pDiscard)
    {
        RTListInit(&pDiscard->ListLru);
        pDiscard->pTreeBlocks = (PAVLRU64TREE)RTMemAllocZ(sizeof(AVLRU64TREE));
        if (!pDiscard->pTreeBlocks)
        {
            RTMemFree(pDiscard);
            pDiscard = NULL;
        }
    }

    return pDiscard;
}

/**
 * Removes the least recently used blocks from the waiting list until
 * the new value is reached.
 *
 * @returns VBox status code.
 * @param   pDisk              VD disk container.
 * @param   pDiscard           The discard state.
 * @param   cbDiscardingNew    How many bytes should be waiting on success.
 *                             The number of bytes waiting can be less.
 */
static int vdDiscardRemoveBlocks(PVDISK pDisk, PVDDISCARDSTATE pDiscard, size_t cbDiscardingNew)
{
    int rc = VINF_SUCCESS;

    LogFlowFunc(("pDisk=%#p pDiscard=%#p cbDiscardingNew=%zu\n",
                 pDisk, pDiscard, cbDiscardingNew));

    while (pDiscard->cbDiscarding > cbDiscardingNew)
    {
        PVDDISCARDBLOCK pBlock = RTListGetLast(&pDiscard->ListLru, VDDISCARDBLOCK, NodeLru);

        Assert(!RTListIsEmpty(&pDiscard->ListLru));

        /* Go over the allocation bitmap and mark all discarded sectors as unused. */
        uint64_t offStart = pBlock->Core.Key;
        uint32_t idxStart = 0;
        size_t cbLeft = pBlock->cbDiscard;
        bool fAllocated = ASMBitTest(pBlock->pbmAllocated, idxStart);
        uint32_t cSectors = (uint32_t)(pBlock->cbDiscard / 512);

        while (cbLeft > 0)
        {
            int32_t idxEnd;
            size_t cbThis = cbLeft;

            if (fAllocated)
            {
                /* Check for the first unallocated bit. */
                idxEnd = ASMBitNextClear(pBlock->pbmAllocated, cSectors, idxStart);
                if (idxEnd != -1)
                {
                    cbThis = (idxEnd - idxStart) * 512;
                    fAllocated = false;
                }
            }
            else
            {
                /* Mark as unused and check for the first set bit. */
                idxEnd = ASMBitNextSet(pBlock->pbmAllocated, cSectors, idxStart);
                if (idxEnd != -1)
                    cbThis = (idxEnd - idxStart) * 512;


                VDIOCTX IoCtx;
                vdIoCtxInit(&IoCtx, pDisk, VDIOCTXTXDIR_DISCARD, 0, 0, NULL,
                            NULL, NULL, NULL, VDIOCTX_FLAGS_SYNC);
                rc = pDisk->pLast->Backend->pfnDiscard(pDisk->pLast->pBackendData,
                                                            &IoCtx, offStart, cbThis, NULL,
                                                            NULL, &cbThis, NULL,
                                                            VD_DISCARD_MARK_UNUSED);
                if (RT_FAILURE(rc))
                    break;

                fAllocated = true;
            }

            idxStart  = idxEnd;
            offStart += cbThis;
            cbLeft   -= cbThis;
        }

        if (RT_FAILURE(rc))
            break;

        PVDDISCARDBLOCK pBlockRemove = (PVDDISCARDBLOCK)RTAvlrU64RangeRemove(pDiscard->pTreeBlocks, pBlock->Core.Key);
        Assert(pBlockRemove == pBlock); NOREF(pBlockRemove);
        RTListNodeRemove(&pBlock->NodeLru);

        pDiscard->cbDiscarding -= pBlock->cbDiscard;
        RTMemFree(pBlock->pbmAllocated);
        RTMemFree(pBlock);
    }

    Assert(RT_FAILURE(rc) || pDiscard->cbDiscarding <= cbDiscardingNew);

    LogFlowFunc(("returns rc=%Rrc\n", rc));
    return rc;
}

/**
 * Destroys the current discard state, writing any waiting blocks to the image.
 *
 * @returns VBox status code.
 * @param   pDisk    VD disk container.
 */
static int vdDiscardStateDestroy(PVDISK pDisk)
{
    int rc = VINF_SUCCESS;

    if (pDisk->pDiscard)
    {
        rc = vdDiscardRemoveBlocks(pDisk, pDisk->pDiscard, 0 /* Remove all blocks. */);
        AssertRC(rc);
        RTMemFree(pDisk->pDiscard->pTreeBlocks);
        RTMemFree(pDisk->pDiscard);
        pDisk->pDiscard = NULL;
    }

    return rc;
}

/**
 * Marks the given range as allocated in the image.
 * Required if there are discards in progress and a write to a block which can get discarded
 * is written to.
 *
 * @returns VBox status code.
 * @param   pDisk    VD container data.
 * @param   uOffset  First byte to mark as allocated.
 * @param   cbRange  Number of bytes to mark as allocated.
 */
static int vdDiscardSetRangeAllocated(PVDISK pDisk, uint64_t uOffset, size_t cbRange)
{
    PVDDISCARDSTATE pDiscard = pDisk->pDiscard;
    int rc = VINF_SUCCESS;

    if (pDiscard)
    {
        do
        {
            size_t cbThisRange = cbRange;
            PVDDISCARDBLOCK pBlock = (PVDDISCARDBLOCK)RTAvlrU64RangeGet(pDiscard->pTreeBlocks, uOffset);

            if (pBlock)
            {
                int32_t idxStart, idxEnd;

                Assert(!(cbThisRange % 512));
                Assert(!((uOffset - pBlock->Core.Key) % 512));

                cbThisRange = RT_MIN(cbThisRange, pBlock->Core.KeyLast - uOffset + 1);

                idxStart = (uOffset - pBlock->Core.Key) / 512;
                idxEnd = idxStart + (int32_t)(cbThisRange / 512);
                ASMBitSetRange(pBlock->pbmAllocated, idxStart, idxEnd);
            }
            else
            {
                pBlock = (PVDDISCARDBLOCK)RTAvlrU64GetBestFit(pDiscard->pTreeBlocks, uOffset, true);
                if (pBlock)
                    cbThisRange = RT_MIN(cbThisRange, pBlock->Core.Key - uOffset);
            }

            Assert(cbRange >= cbThisRange);

            uOffset += cbThisRange;
            cbRange -= cbThisRange;
        } while (cbRange != 0);
    }

    return rc;
}

DECLINLINE(PVDIOCTX) vdIoCtxAlloc(PVDISK pDisk, VDIOCTXTXDIR enmTxDir,
                                  uint64_t uOffset, size_t cbTransfer,
                                  PVDIMAGE pImageStart,PCRTSGBUF pSgBuf,
                                  void *pvAllocation, PFNVDIOCTXTRANSFER pfnIoCtxTransfer,
                                  uint32_t fFlags)
{
    PVDIOCTX pIoCtx = NULL;

    pIoCtx = (PVDIOCTX)RTMemCacheAlloc(pDisk->hMemCacheIoCtx);
    if (RT_LIKELY(pIoCtx))
    {
        vdIoCtxInit(pIoCtx, pDisk, enmTxDir, uOffset, cbTransfer, pImageStart,
                    pSgBuf, pvAllocation, pfnIoCtxTransfer, fFlags);
    }

    return pIoCtx;
}

DECLINLINE(PVDIOCTX) vdIoCtxRootAlloc(PVDISK pDisk, VDIOCTXTXDIR enmTxDir,
                                      uint64_t uOffset, size_t cbTransfer,
                                      PVDIMAGE pImageStart, PCRTSGBUF pSgBuf,
                                      PFNVDASYNCTRANSFERCOMPLETE pfnComplete,
                                      void *pvUser1, void *pvUser2,
                                      void *pvAllocation,
                                      PFNVDIOCTXTRANSFER pfnIoCtxTransfer,
                                      uint32_t fFlags)
{
    PVDIOCTX pIoCtx = vdIoCtxAlloc(pDisk, enmTxDir, uOffset, cbTransfer, pImageStart,
                                   pSgBuf, pvAllocation, pfnIoCtxTransfer, fFlags);

    if (RT_LIKELY(pIoCtx))
    {
        pIoCtx->pIoCtxParent          = NULL;
        pIoCtx->Type.Root.pfnComplete = pfnComplete;
        pIoCtx->Type.Root.pvUser1     = pvUser1;
        pIoCtx->Type.Root.pvUser2     = pvUser2;
    }

    LogFlow(("Allocated root I/O context %#p\n", pIoCtx));
    return pIoCtx;
}

DECLINLINE(void) vdIoCtxDiscardInit(PVDIOCTX pIoCtx, PVDISK pDisk, PCRTRANGE paRanges,
                                    unsigned cRanges, PFNVDASYNCTRANSFERCOMPLETE pfnComplete,
                                    void *pvUser1, void *pvUser2, void *pvAllocation,
                                    PFNVDIOCTXTRANSFER pfnIoCtxTransfer, uint32_t fFlags)
{
    pIoCtx->pIoCtxNext                = NULL;
    pIoCtx->pDisk                     = pDisk;
    pIoCtx->enmTxDir                  = VDIOCTXTXDIR_DISCARD;
    pIoCtx->cDataTransfersPending     = 0;
    pIoCtx->cMetaTransfersPending     = 0;
    pIoCtx->fComplete                 = false;
    pIoCtx->fFlags                    = fFlags;
    pIoCtx->pvAllocation              = pvAllocation;
    pIoCtx->pfnIoCtxTransfer          = pfnIoCtxTransfer;
    pIoCtx->pfnIoCtxTransferNext      = NULL;
    pIoCtx->rcReq                     = VINF_SUCCESS;
    pIoCtx->Req.Discard.paRanges      = paRanges;
    pIoCtx->Req.Discard.cRanges       = cRanges;
    pIoCtx->Req.Discard.idxRange      = 0;
    pIoCtx->Req.Discard.cbDiscardLeft = 0;
    pIoCtx->Req.Discard.offCur        = 0;
    pIoCtx->Req.Discard.cbThisDiscard = 0;

    pIoCtx->pIoCtxParent          = NULL;
    pIoCtx->Type.Root.pfnComplete = pfnComplete;
    pIoCtx->Type.Root.pvUser1     = pvUser1;
    pIoCtx->Type.Root.pvUser2     = pvUser2;
}

DECLINLINE(PVDIOCTX) vdIoCtxDiscardAlloc(PVDISK pDisk, PCRTRANGE paRanges,
                                         unsigned cRanges,
                                         PFNVDASYNCTRANSFERCOMPLETE pfnComplete,
                                         void *pvUser1, void *pvUser2,
                                         void *pvAllocation,
                                         PFNVDIOCTXTRANSFER pfnIoCtxTransfer,
                                         uint32_t fFlags)
{
    PVDIOCTX pIoCtx = NULL;

    pIoCtx = (PVDIOCTX)RTMemCacheAlloc(pDisk->hMemCacheIoCtx);
    if (RT_LIKELY(pIoCtx))
    {
        vdIoCtxDiscardInit(pIoCtx, pDisk, paRanges, cRanges, pfnComplete, pvUser1,
                           pvUser2, pvAllocation, pfnIoCtxTransfer, fFlags);
    }

    LogFlow(("Allocated discard I/O context %#p\n", pIoCtx));
    return pIoCtx;
}

DECLINLINE(PVDIOCTX) vdIoCtxChildAlloc(PVDISK pDisk, VDIOCTXTXDIR enmTxDir,
                                       uint64_t uOffset, size_t cbTransfer,
                                       PVDIMAGE pImageStart, PCRTSGBUF pSgBuf,
                                       PVDIOCTX pIoCtxParent, size_t cbTransferParent,
                                       size_t cbWriteParent, void *pvAllocation,
                                       PFNVDIOCTXTRANSFER pfnIoCtxTransfer)
{
    PVDIOCTX pIoCtx = vdIoCtxAlloc(pDisk, enmTxDir, uOffset, cbTransfer, pImageStart,
                                   pSgBuf, pvAllocation, pfnIoCtxTransfer, pIoCtxParent->fFlags & ~VDIOCTX_FLAGS_DONT_FREE);

    AssertPtr(pIoCtxParent);
    Assert(!pIoCtxParent->pIoCtxParent);

    if (RT_LIKELY(pIoCtx))
    {
        pIoCtx->pIoCtxParent                   = pIoCtxParent;
        pIoCtx->Type.Child.uOffsetSaved        = uOffset;
        pIoCtx->Type.Child.cbTransferLeftSaved = cbTransfer;
        pIoCtx->Type.Child.cbTransferParent    = cbTransferParent;
        pIoCtx->Type.Child.cbWriteParent       = cbWriteParent;
    }

    LogFlow(("Allocated child I/O context %#p\n", pIoCtx));
    return pIoCtx;
}

DECLINLINE(PVDIOTASK) vdIoTaskUserAlloc(PVDIOSTORAGE pIoStorage, PFNVDXFERCOMPLETED pfnComplete, void *pvUser, PVDIOCTX pIoCtx, uint32_t cbTransfer)
{
    PVDIOTASK pIoTask = NULL;

    pIoTask = (PVDIOTASK)RTMemCacheAlloc(pIoStorage->pVDIo->pDisk->hMemCacheIoTask);
    if (pIoTask)
    {
        pIoTask->pIoStorage           = pIoStorage;
        pIoTask->pfnComplete          = pfnComplete;
        pIoTask->pvUser               = pvUser;
        pIoTask->fMeta                = false;
        pIoTask->Type.User.cbTransfer = cbTransfer;
        pIoTask->Type.User.pIoCtx     = pIoCtx;
    }

    return pIoTask;
}

DECLINLINE(PVDIOTASK) vdIoTaskMetaAlloc(PVDIOSTORAGE pIoStorage, PFNVDXFERCOMPLETED pfnComplete, void *pvUser, PVDMETAXFER pMetaXfer)
{
    PVDIOTASK pIoTask = NULL;

    pIoTask = (PVDIOTASK)RTMemCacheAlloc(pIoStorage->pVDIo->pDisk->hMemCacheIoTask);
    if (pIoTask)
    {
        pIoTask->pIoStorage          = pIoStorage;
        pIoTask->pfnComplete         = pfnComplete;
        pIoTask->pvUser              = pvUser;
        pIoTask->fMeta               = true;
        pIoTask->Type.Meta.pMetaXfer = pMetaXfer;
    }

    return pIoTask;
}

DECLINLINE(void) vdIoCtxFree(PVDISK pDisk, PVDIOCTX pIoCtx)
{
    Log(("Freeing I/O context %#p\n", pIoCtx));

    if (!(pIoCtx->fFlags & VDIOCTX_FLAGS_DONT_FREE))
    {
        if (pIoCtx->pvAllocation)
            RTMemFree(pIoCtx->pvAllocation);
#ifdef DEBUG
        memset(&pIoCtx->pDisk, 0xff, sizeof(void *));
#endif
        RTMemCacheFree(pDisk->hMemCacheIoCtx, pIoCtx);
    }
}

DECLINLINE(void) vdIoTaskFree(PVDISK pDisk, PVDIOTASK pIoTask)
{
#ifdef DEBUG
    memset(pIoTask, 0xff, sizeof(VDIOTASK));
#endif
    RTMemCacheFree(pDisk->hMemCacheIoTask, pIoTask);
}

DECLINLINE(void) vdIoCtxChildReset(PVDIOCTX pIoCtx)
{
    AssertPtr(pIoCtx->pIoCtxParent);

    RTSgBufReset(&pIoCtx->Req.Io.SgBuf);
    pIoCtx->Req.Io.uOffset        = pIoCtx->Type.Child.uOffsetSaved;
    pIoCtx->Req.Io.cbTransferLeft = (uint32_t)pIoCtx->Type.Child.cbTransferLeftSaved;
    Assert((uint32_t)pIoCtx->Type.Child.cbTransferLeftSaved == pIoCtx->Type.Child.cbTransferLeftSaved);
}

DECLINLINE(PVDMETAXFER) vdMetaXferAlloc(PVDIOSTORAGE pIoStorage, uint64_t uOffset, size_t cb)
{
    PVDMETAXFER pMetaXfer = (PVDMETAXFER)RTMemAlloc(RT_UOFFSETOF_DYN(VDMETAXFER, abData[cb]));

    if (RT_LIKELY(pMetaXfer))
    {
        pMetaXfer->Core.Key     = uOffset;
        pMetaXfer->Core.KeyLast = uOffset + cb - 1;
        pMetaXfer->fFlags       = VDMETAXFER_TXDIR_NONE;
        pMetaXfer->cbMeta       = cb;
        pMetaXfer->pIoStorage   = pIoStorage;
        pMetaXfer->cRefs        = 0;
        pMetaXfer->pbDataShw    = NULL;
        RTListInit(&pMetaXfer->ListIoCtxWaiting);
        RTListInit(&pMetaXfer->ListIoCtxShwWrites);
    }
    return pMetaXfer;
}

DECLINLINE(void) vdIoCtxAddToWaitingList(volatile PVDIOCTX *ppList, PVDIOCTX pIoCtx)
{
    /* Put it on the waiting list. */
    PVDIOCTX pNext = ASMAtomicUoReadPtrT(ppList, PVDIOCTX);
    PVDIOCTX pHeadOld;
    pIoCtx->pIoCtxNext = pNext;
    while (!ASMAtomicCmpXchgExPtr(ppList, pIoCtx, pNext, &pHeadOld))
    {
        pNext = pHeadOld;
        Assert(pNext != pIoCtx);
        pIoCtx->pIoCtxNext = pNext;
        ASMNopPause();
    }
}

DECLINLINE(void) vdIoCtxDefer(PVDISK pDisk, PVDIOCTX pIoCtx)
{
    LogFlowFunc(("Deferring I/O context pIoCtx=%#p\n", pIoCtx));

    Assert(!pIoCtx->pIoCtxParent && !(pIoCtx->fFlags & VDIOCTX_FLAGS_BLOCKED));
    pIoCtx->fFlags |= VDIOCTX_FLAGS_BLOCKED;
    vdIoCtxAddToWaitingList(&pDisk->pIoCtxBlockedHead, pIoCtx);
}

static size_t vdIoCtxCopy(PVDIOCTX pIoCtxDst, PVDIOCTX pIoCtxSrc, size_t cbData)
{
    return RTSgBufCopy(&pIoCtxDst->Req.Io.SgBuf, &pIoCtxSrc->Req.Io.SgBuf, cbData);
}

#if 0 /* unused */
static int vdIoCtxCmp(PVDIOCTX pIoCtx1, PVDIOCTX pIoCtx2, size_t cbData)
{
    return RTSgBufCmp(&pIoCtx1->Req.Io.SgBuf, &pIoCtx2->Req.Io.SgBuf, cbData);
}
#endif

static size_t vdIoCtxCopyTo(PVDIOCTX pIoCtx, const uint8_t *pbData, size_t cbData)
{
    return RTSgBufCopyFromBuf(&pIoCtx->Req.Io.SgBuf, pbData, cbData);
}

static size_t vdIoCtxCopyFrom(PVDIOCTX pIoCtx, uint8_t *pbData, size_t cbData)
{
    return RTSgBufCopyToBuf(&pIoCtx->Req.Io.SgBuf, pbData, cbData);
}

static size_t vdIoCtxSet(PVDIOCTX pIoCtx, uint8_t ch, size_t cbData)
{
    return RTSgBufSet(&pIoCtx->Req.Io.SgBuf, ch, cbData);
}

/**
 * Returns whether the given I/O context has completed.
 *
 * @returns Flag whether the I/O context is complete.
 * @param   pIoCtx          The I/O context to check.
 */
DECLINLINE(bool) vdIoCtxIsComplete(PVDIOCTX pIoCtx)
{
    if (   !pIoCtx->cMetaTransfersPending
        && !pIoCtx->cDataTransfersPending
        && !pIoCtx->pfnIoCtxTransfer)
        return true;

    /*
     * We complete the I/O context in case of an error
     * if there is no I/O task pending.
     */
    if (   RT_FAILURE(pIoCtx->rcReq)
        && !pIoCtx->cMetaTransfersPending
        && !pIoCtx->cDataTransfersPending)
        return true;

    return false;
}

/**
 * Returns whether the given I/O context is blocked due to a metadata transfer
 * or because the backend blocked it.
 *
 * @returns Flag whether the I/O context is blocked.
 * @param   pIoCtx          The I/O context to check.
 */
DECLINLINE(bool) vdIoCtxIsBlocked(PVDIOCTX pIoCtx)
{
    /* Don't change anything if there is a metadata transfer pending or we are blocked. */
    if (   pIoCtx->cMetaTransfersPending
        || (pIoCtx->fFlags & VDIOCTX_FLAGS_BLOCKED))
        return true;

    return false;
}

/**
 * Process the I/O context, core method which assumes that the I/O context
 * acquired the lock.
 *
 * @returns VBox status code.
 * @param   pIoCtx    I/O context to process.
 */
static int vdIoCtxProcessLocked(PVDIOCTX pIoCtx)
{
    int rc = VINF_SUCCESS;

    VD_IS_LOCKED(pIoCtx->pDisk);

    LogFlowFunc(("pIoCtx=%#p\n", pIoCtx));

    if (!vdIoCtxIsComplete(pIoCtx))
    {
        if (!vdIoCtxIsBlocked(pIoCtx))
        {
            if (pIoCtx->pfnIoCtxTransfer)
            {
                /* Call the transfer function advancing to the next while there is no error. */
                while (   pIoCtx->pfnIoCtxTransfer
                       && !pIoCtx->cMetaTransfersPending
                       && RT_SUCCESS(rc))
                {
                    LogFlowFunc(("calling transfer function %#p\n", pIoCtx->pfnIoCtxTransfer));
                    rc = pIoCtx->pfnIoCtxTransfer(pIoCtx);

                    /* Advance to the next part of the transfer if the current one succeeded. */
                    if (RT_SUCCESS(rc))
                    {
                        pIoCtx->pfnIoCtxTransfer = pIoCtx->pfnIoCtxTransferNext;
                        pIoCtx->pfnIoCtxTransferNext = NULL;
                    }
                }
            }

            if (   RT_SUCCESS(rc)
                && !pIoCtx->cMetaTransfersPending
                && !pIoCtx->cDataTransfersPending
                && !(pIoCtx->fFlags & VDIOCTX_FLAGS_BLOCKED))
                rc = VINF_VD_ASYNC_IO_FINISHED;
            else if (   RT_SUCCESS(rc)
                     || rc == VERR_VD_NOT_ENOUGH_METADATA
                     || rc == VERR_VD_IOCTX_HALT)
                rc = VERR_VD_ASYNC_IO_IN_PROGRESS;
            else if (   RT_FAILURE(rc)
                        && (rc != VERR_VD_ASYNC_IO_IN_PROGRESS))
            {
                ASMAtomicCmpXchgS32(&pIoCtx->rcReq, rc, VINF_SUCCESS);

                /*
                 * The I/O context completed if we have an error and there is no data
                 * or meta data transfer pending.
                 */
                if (   !pIoCtx->cMetaTransfersPending
                    && !pIoCtx->cDataTransfersPending)
                    rc = VINF_VD_ASYNC_IO_FINISHED;
                else
                    rc = VERR_VD_ASYNC_IO_IN_PROGRESS;
            }
        }
        else
            rc = VERR_VD_ASYNC_IO_IN_PROGRESS;
    }
    else
        rc = VINF_VD_ASYNC_IO_FINISHED;

    LogFlowFunc(("pIoCtx=%#p rc=%Rrc cDataTransfersPending=%u cMetaTransfersPending=%u fComplete=%RTbool\n",
                 pIoCtx, rc, pIoCtx->cDataTransfersPending, pIoCtx->cMetaTransfersPending,
                 pIoCtx->fComplete));

    return rc;
}

/**
 * Processes the list of waiting I/O contexts.
 *
 * @returns VBox status code, only valid if pIoCtxRc is not NULL, treat as void
 *          function otherwise.
 * @param   pDisk    The disk structure.
 * @param   pIoCtxRc An I/O context handle which waits on the list. When processed
 *                   The status code is returned. NULL if there is no I/O context
 *                   to return the status code for.
 */
static int vdDiskProcessWaitingIoCtx(PVDISK pDisk, PVDIOCTX pIoCtxRc)
{
    int rc = VERR_VD_ASYNC_IO_IN_PROGRESS;

    LogFlowFunc(("pDisk=%#p pIoCtxRc=%#p\n", pDisk, pIoCtxRc));

    VD_IS_LOCKED(pDisk);

    /* Get the waiting list and process it in FIFO order. */
    PVDIOCTX pIoCtxHead = ASMAtomicXchgPtrT(&pDisk->pIoCtxHead, NULL, PVDIOCTX);

    /* Reverse it. */
    PVDIOCTX pCur = pIoCtxHead;
    pIoCtxHead = NULL;
    while (pCur)
    {
        PVDIOCTX pInsert = pCur;
        pCur = pCur->pIoCtxNext;
        pInsert->pIoCtxNext = pIoCtxHead;
        pIoCtxHead = pInsert;
    }

    /* Process now. */
    pCur = pIoCtxHead;
    while (pCur)
    {
        int rcTmp;
        PVDIOCTX pTmp = pCur;

        pCur = pCur->pIoCtxNext;
        pTmp->pIoCtxNext = NULL;

        /*
         * Need to clear the sync flag here if there is a new I/O context
         * with it set and the context is not given in pIoCtxRc.
         * This happens most likely on a different thread and that one shouldn't
         * process the context synchronously.
         *
         * The thread who issued the context will wait on the event semaphore
         * anyway which is signalled when the completion handler is called.
         */
        if (   pTmp->fFlags & VDIOCTX_FLAGS_SYNC
            && pTmp != pIoCtxRc)
            pTmp->fFlags &= ~VDIOCTX_FLAGS_SYNC;

        rcTmp = vdIoCtxProcessLocked(pTmp);
        if (pTmp == pIoCtxRc)
        {
            if (   rcTmp == VINF_VD_ASYNC_IO_FINISHED
                && RT_SUCCESS(pTmp->rcReq)
                && pTmp->enmTxDir == VDIOCTXTXDIR_READ)
            {
                   int rc2 = vdFilterChainApplyRead(pDisk, pTmp->Req.Io.uOffsetXferOrig,
                                                    pTmp->Req.Io.cbXferOrig, pTmp);
                    if (RT_FAILURE(rc2))
                        rcTmp = rc2;
            }

            /* The given I/O context was processed, pass the return code to the caller. */
            if (   rcTmp == VINF_VD_ASYNC_IO_FINISHED
                && (pTmp->fFlags & VDIOCTX_FLAGS_SYNC))
                rc = pTmp->rcReq;
            else
                rc = rcTmp;
        }
        else if (   rcTmp == VINF_VD_ASYNC_IO_FINISHED
                 && ASMAtomicCmpXchgBool(&pTmp->fComplete, true, false))
        {
            LogFlowFunc(("Waiting I/O context completed pTmp=%#p\n", pTmp));
            vdThreadFinishWrite(pDisk);

            bool fFreeCtx = RT_BOOL(!(pTmp->fFlags & VDIOCTX_FLAGS_DONT_FREE));
            vdIoCtxRootComplete(pDisk, pTmp);

            if (fFreeCtx)
                vdIoCtxFree(pDisk, pTmp);
        }
    }

    LogFlowFunc(("returns rc=%Rrc\n", rc));
    return rc;
}

/**
 * Processes the list of blocked I/O contexts.
 *
 * @param   pDisk    The disk structure.
 */
static void vdDiskProcessBlockedIoCtx(PVDISK pDisk)
{
    LogFlowFunc(("pDisk=%#p\n", pDisk));

    VD_IS_LOCKED(pDisk);

    /* Get the waiting list and process it in FIFO order. */
    PVDIOCTX pIoCtxHead = ASMAtomicXchgPtrT(&pDisk->pIoCtxBlockedHead, NULL, PVDIOCTX);

    /* Reverse it. */
    PVDIOCTX pCur = pIoCtxHead;
    pIoCtxHead = NULL;
    while (pCur)
    {
        PVDIOCTX pInsert = pCur;
        pCur = pCur->pIoCtxNext;
        pInsert->pIoCtxNext = pIoCtxHead;
        pIoCtxHead = pInsert;
    }

    /* Process now. */
    pCur = pIoCtxHead;
    while (pCur)
    {
        int rc;
        PVDIOCTX pTmp = pCur;

        pCur = pCur->pIoCtxNext;
        pTmp->pIoCtxNext = NULL;

        Assert(!pTmp->pIoCtxParent);
        Assert(pTmp->fFlags & VDIOCTX_FLAGS_BLOCKED);
        pTmp->fFlags &= ~VDIOCTX_FLAGS_BLOCKED;

        rc = vdIoCtxProcessLocked(pTmp);
        if (   rc == VINF_VD_ASYNC_IO_FINISHED
            && ASMAtomicCmpXchgBool(&pTmp->fComplete, true, false))
        {
            LogFlowFunc(("Waiting I/O context completed pTmp=%#p\n", pTmp));
            vdThreadFinishWrite(pDisk);

            bool fFreeCtx = RT_BOOL(!(pTmp->fFlags & VDIOCTX_FLAGS_DONT_FREE));
            vdIoCtxRootComplete(pDisk, pTmp);
            if (fFreeCtx)
                vdIoCtxFree(pDisk, pTmp);
        }
    }

    LogFlowFunc(("returns\n"));
}

/**
 * Processes the I/O context trying to lock the criticial section.
 * The context is deferred if the critical section is busy.
 *
 * @returns VBox status code.
 * @param   pIoCtx    The I/O context to process.
 */
static int vdIoCtxProcessTryLockDefer(PVDIOCTX pIoCtx)
{
    int rc = VINF_SUCCESS;
    PVDISK pDisk = pIoCtx->pDisk;

    Log(("Defer pIoCtx=%#p\n", pIoCtx));

    /* Put it on the waiting list first. */
    vdIoCtxAddToWaitingList(&pDisk->pIoCtxHead, pIoCtx);

    if (ASMAtomicCmpXchgBool(&pDisk->fLocked, true, false))
    {
        /* Leave it again, the context will be processed just before leaving the lock. */
        LogFlowFunc(("Successfully acquired the lock\n"));
        rc = vdDiskUnlock(pDisk, pIoCtx);
    }
    else
    {
        LogFlowFunc(("Lock is held\n"));
        rc = VERR_VD_ASYNC_IO_IN_PROGRESS;
    }

    return rc;
}

/**
 * Process the I/O context in a synchronous manner, waiting
 * for it to complete.
 *
 * @returns VBox status code of the completed request.
 * @param   pIoCtx            The sync I/O context.
 * @param   hEventComplete    Event sempahore to wait on for completion.
 */
static int vdIoCtxProcessSync(PVDIOCTX pIoCtx, RTSEMEVENT hEventComplete)
{
    int rc = VINF_SUCCESS;
    PVDISK pDisk = pIoCtx->pDisk;

    LogFlowFunc(("pIoCtx=%p\n", pIoCtx));

    AssertMsg(pIoCtx->fFlags & (VDIOCTX_FLAGS_SYNC | VDIOCTX_FLAGS_DONT_FREE),
              ("I/O context is not marked as synchronous\n"));

    rc = vdIoCtxProcessTryLockDefer(pIoCtx);
    if (rc == VINF_VD_ASYNC_IO_FINISHED)
        rc = VINF_SUCCESS;

    if (rc == VERR_VD_ASYNC_IO_IN_PROGRESS)
    {
        rc = RTSemEventWait(hEventComplete, RT_INDEFINITE_WAIT);
        AssertRC(rc);
    }

    rc = pIoCtx->rcReq;
    vdIoCtxFree(pDisk, pIoCtx);

    return rc;
}

DECLINLINE(bool) vdIoCtxIsDiskLockOwner(PVDISK pDisk, PVDIOCTX pIoCtx)
{
    return pDisk->pIoCtxLockOwner == pIoCtx;
}

static int vdIoCtxLockDisk(PVDISK pDisk, PVDIOCTX pIoCtx)
{
    int rc = VINF_SUCCESS;

    VD_IS_LOCKED(pDisk);

    LogFlowFunc(("pDisk=%#p pIoCtx=%#p\n", pDisk, pIoCtx));

    if (!ASMAtomicCmpXchgPtr(&pDisk->pIoCtxLockOwner, pIoCtx, NIL_VDIOCTX))
    {
        Assert(pDisk->pIoCtxLockOwner != pIoCtx); /* No nesting allowed. */
        vdIoCtxDefer(pDisk, pIoCtx);
        rc = VERR_VD_ASYNC_IO_IN_PROGRESS;
    }

    LogFlowFunc(("returns -> %Rrc\n", rc));
    return rc;
}

static void vdIoCtxUnlockDisk(PVDISK pDisk, PVDIOCTX pIoCtx, bool fProcessBlockedReqs)
{
    RT_NOREF1(pIoCtx);
    LogFlowFunc(("pDisk=%#p pIoCtx=%#p fProcessBlockedReqs=%RTbool\n",
                 pDisk, pIoCtx, fProcessBlockedReqs));

    VD_IS_LOCKED(pDisk);

    LogFlow(("Unlocking disk lock owner is %#p\n", pDisk->pIoCtxLockOwner));
    Assert(pDisk->pIoCtxLockOwner == pIoCtx);
    ASMAtomicXchgPtrT(&pDisk->pIoCtxLockOwner, NIL_VDIOCTX, PVDIOCTX);

    if (fProcessBlockedReqs)
    {
        /* Process any blocked writes if the current request didn't caused another growing. */
        vdDiskProcessBlockedIoCtx(pDisk);
    }

    LogFlowFunc(("returns\n"));
}

/**
 * Internal: Reads a given amount of data from the image chain of the disk.
 **/
static int vdDiskReadHelper(PVDISK pDisk, PVDIMAGE pImage, PVDIMAGE pImageParentOverride,
                            uint64_t uOffset, size_t cbRead, PVDIOCTX pIoCtx, size_t *pcbThisRead)
{
    RT_NOREF1(pDisk);
    int rc = VINF_SUCCESS;
    size_t cbThisRead = cbRead;

    AssertPtr(pcbThisRead);

    *pcbThisRead = 0;

    /*
     * Try to read from the given image.
     * If the block is not allocated read from override chain if present.
     */
    rc = pImage->Backend->pfnRead(pImage->pBackendData,
                                  uOffset, cbThisRead, pIoCtx,
                                  &cbThisRead);

    if (rc == VERR_VD_BLOCK_FREE)
    {
        for (PVDIMAGE pCurrImage = pImageParentOverride ? pImageParentOverride : pImage->pPrev;
             pCurrImage != NULL && rc == VERR_VD_BLOCK_FREE;
             pCurrImage = pCurrImage->pPrev)
        {
            rc = pCurrImage->Backend->pfnRead(pCurrImage->pBackendData,
                                              uOffset, cbThisRead, pIoCtx,
                                              &cbThisRead);
        }
    }

    if (RT_SUCCESS(rc) || rc == VERR_VD_BLOCK_FREE)
        *pcbThisRead = cbThisRead;

    return rc;
}

/**
 * internal: read the specified amount of data in whatever blocks the backend
 * will give us - async version.
 */
static DECLCALLBACK(int) vdReadHelperAsync(PVDIOCTX pIoCtx)
{
    int rc;
    PVDISK pDisk                = pIoCtx->pDisk;
    size_t cbToRead               = pIoCtx->Req.Io.cbTransfer;
    uint64_t uOffset              = pIoCtx->Req.Io.uOffset;
    PVDIMAGE pCurrImage           = pIoCtx->Req.Io.pImageCur;
    PVDIMAGE pImageParentOverride = pIoCtx->Req.Io.pImageParentOverride;
    unsigned cImagesRead          = pIoCtx->Req.Io.cImagesRead;
    size_t cbThisRead;

    /*
     * Check whether there is a full block write in progress which was not allocated.
     * Defer I/O if the range interferes but only if it does not belong to the
     * write doing the allocation.
     */
    if (   pDisk->pIoCtxLockOwner != NIL_VDIOCTX
        && uOffset >= pDisk->uOffsetStartLocked
        && uOffset < pDisk->uOffsetEndLocked
        && (   !pIoCtx->pIoCtxParent
            || pIoCtx->pIoCtxParent != pDisk->pIoCtxLockOwner))
    {
        Log(("Interferring read while allocating a new block => deferring read\n"));
        vdIoCtxDefer(pDisk, pIoCtx);
        return VERR_VD_ASYNC_IO_IN_PROGRESS;
    }

    /* Loop until all reads started or we have a backend which needs to read metadata. */
    do
    {
        /* Search for image with allocated block. Do not attempt to read more
         * than the previous reads marked as valid. Otherwise this would return
         * stale data when different block sizes are used for the images. */
        cbThisRead = cbToRead;

        if (   pDisk->pCache
            && !pImageParentOverride)
        {
            rc = vdCacheReadHelper(pDisk->pCache, uOffset, cbThisRead,
                                   pIoCtx, &cbThisRead);
            if (rc == VERR_VD_BLOCK_FREE)
            {
                rc = vdDiskReadHelper(pDisk, pCurrImage, NULL, uOffset, cbThisRead,
                                      pIoCtx, &cbThisRead);

                /* If the read was successful, write the data back into the cache. */
                if (   RT_SUCCESS(rc)
                    && pIoCtx->fFlags & VDIOCTX_FLAGS_READ_UPDATE_CACHE)
                {
                    rc = vdCacheWriteHelper(pDisk->pCache, uOffset, cbThisRead,
                                            pIoCtx, NULL);
                }
            }
        }
        else
        {
            /*
             * Try to read from the given image.
             * If the block is not allocated read from override chain if present.
             */
            rc = pCurrImage->Backend->pfnRead(pCurrImage->pBackendData,
                                              uOffset, cbThisRead, pIoCtx,
                                              &cbThisRead);

            if (   rc == VERR_VD_BLOCK_FREE
                && cImagesRead != 1)
            {
                unsigned cImagesToProcess = cImagesRead;

                pCurrImage = pImageParentOverride ? pImageParentOverride : pCurrImage->pPrev;
                pIoCtx->Req.Io.pImageParentOverride = NULL;

                while (pCurrImage && rc == VERR_VD_BLOCK_FREE)
                {
                    rc = pCurrImage->Backend->pfnRead(pCurrImage->pBackendData,
                                                      uOffset, cbThisRead,
                                                      pIoCtx, &cbThisRead);
                    if (cImagesToProcess == 1)
                        break;
                    else if (cImagesToProcess > 0)
                        cImagesToProcess--;

                    if (rc == VERR_VD_BLOCK_FREE)
                        pCurrImage = pCurrImage->pPrev;
                }
            }
        }

        /* The task state will be updated on success already, don't do it here!. */
        if (rc == VERR_VD_BLOCK_FREE)
        {
            /* No image in the chain contains the data for the block. */
            ASMAtomicSubU32(&pIoCtx->Req.Io.cbTransferLeft, (uint32_t)cbThisRead); Assert(cbThisRead == (uint32_t)cbThisRead);

            /* Fill the free space with 0 if we are told to do so
             * or a previous read returned valid data. */
            if (pIoCtx->fFlags & VDIOCTX_FLAGS_ZERO_FREE_BLOCKS)
                vdIoCtxSet(pIoCtx, '\0', cbThisRead);
            else
                pIoCtx->Req.Io.cbBufClear += cbThisRead;

            if (pIoCtx->Req.Io.pImageCur->uOpenFlags & VD_OPEN_FLAGS_INFORM_ABOUT_ZERO_BLOCKS)
                rc = VINF_VD_NEW_ZEROED_BLOCK;
            else
                rc = VINF_SUCCESS;
        }
        else if (rc == VERR_VD_IOCTX_HALT)
        {
            uOffset  += cbThisRead;
            cbToRead -= cbThisRead;
            pIoCtx->fFlags |= VDIOCTX_FLAGS_BLOCKED;
        }
        else if (   RT_SUCCESS(rc)
                 || rc == VERR_VD_ASYNC_IO_IN_PROGRESS)
        {
            /* First not free block, fill the space before with 0. */
            if (   pIoCtx->Req.Io.cbBufClear
                && !(pIoCtx->fFlags & VDIOCTX_FLAGS_ZERO_FREE_BLOCKS))
            {
                RTSGBUF SgBuf;
                RTSgBufClone(&SgBuf, &pIoCtx->Req.Io.SgBuf);
                RTSgBufReset(&SgBuf);
                RTSgBufSet(&SgBuf, 0, pIoCtx->Req.Io.cbBufClear);
                pIoCtx->Req.Io.cbBufClear = 0;
                pIoCtx->fFlags |= VDIOCTX_FLAGS_ZERO_FREE_BLOCKS;
            }
            rc = VINF_SUCCESS;
        }

        if (RT_FAILURE(rc))
            break;

        cbToRead -= cbThisRead;
        uOffset  += cbThisRead;
        pCurrImage = pIoCtx->Req.Io.pImageStart; /* Start with the highest image in the chain. */
    } while (cbToRead != 0 && RT_SUCCESS(rc));

    if (   rc == VERR_VD_NOT_ENOUGH_METADATA
        || rc == VERR_VD_IOCTX_HALT)
    {
        /* Save the current state. */
        pIoCtx->Req.Io.uOffset    = uOffset;
        pIoCtx->Req.Io.cbTransfer = cbToRead;
        pIoCtx->Req.Io.pImageCur  = pCurrImage ? pCurrImage : pIoCtx->Req.Io.pImageStart;
    }

    return (!(pIoCtx->fFlags & VDIOCTX_FLAGS_ZERO_FREE_BLOCKS))
           ? VERR_VD_BLOCK_FREE
           : rc;
}

/**
 * internal: parent image read wrapper for compacting.
 */
static DECLCALLBACK(int) vdParentRead(void *pvUser, uint64_t uOffset, void *pvBuf,
                                      size_t cbRead)
{
    PVDPARENTSTATEDESC pParentState = (PVDPARENTSTATEDESC)pvUser;

    /** @todo
     * Only used for compaction so far which is not possible to mix with async I/O.
     * Needs to be changed if we want to support online compaction of images.
     */
    bool fLocked = ASMAtomicXchgBool(&pParentState->pDisk->fLocked, true);
    AssertMsgReturn(!fLocked,
                    ("Calling synchronous parent read while another thread holds the disk lock\n"),
                    VERR_VD_INVALID_STATE);

    /* Fake an I/O context. */
    RTSGSEG Segment;
    RTSGBUF SgBuf;
    VDIOCTX IoCtx;

    Segment.pvSeg = pvBuf;
    Segment.cbSeg = cbRead;
    RTSgBufInit(&SgBuf, &Segment, 1);
    vdIoCtxInit(&IoCtx, pParentState->pDisk, VDIOCTXTXDIR_READ, uOffset, cbRead, pParentState->pImage,
                &SgBuf, NULL, NULL, VDIOCTX_FLAGS_SYNC | VDIOCTX_FLAGS_ZERO_FREE_BLOCKS);
    int rc = vdReadHelperAsync(&IoCtx);
    ASMAtomicXchgBool(&pParentState->pDisk->fLocked, false);
    return rc;
}

/**
 * Extended version of vdReadHelper(), implementing certain optimizations
 * for image cloning.
 *
 * @returns VBox status code.
 * @param   pDisk                   The disk to read from.
 * @param   pImage                  The image to start reading from.
 * @param   pImageParentOverride    The parent image to read from
 *                                  if the starting image returns a free block.
 *                                  If NULL is passed the real parent of the image
 *                                  in the chain is used.
 * @param   uOffset                 Offset in the disk to start reading from.
 * @param   pvBuf                   Where to store the read data.
 * @param   cbRead                  How much to read.
 * @param   fZeroFreeBlocks         Flag whether free blocks should be zeroed.
 *                                  If false and no image has data for sepcified
 *                                  range VERR_VD_BLOCK_FREE is returned.
 *                                  Note that unallocated blocks are still zeroed
 *                                  if at least one image has valid data for a part
 *                                  of the range.
 * @param   fUpdateCache            Flag whether to update the attached cache if
 *                                  available.
 * @param   cImagesRead             Number of images in the chain to read until
 *                                  the read is cut off. A value of 0 disables the cut off.
 */
static int vdReadHelperEx(PVDISK pDisk, PVDIMAGE pImage, PVDIMAGE pImageParentOverride,
                          uint64_t uOffset, void *pvBuf, size_t cbRead,
                          bool fZeroFreeBlocks, bool fUpdateCache, unsigned cImagesRead)
{
    int rc = VINF_SUCCESS;
    uint32_t fFlags = VDIOCTX_FLAGS_SYNC | VDIOCTX_FLAGS_DONT_FREE;
    RTSGSEG Segment;
    RTSGBUF SgBuf;
    VDIOCTX IoCtx;
    RTSEMEVENT hEventComplete = NIL_RTSEMEVENT;

    rc = RTSemEventCreate(&hEventComplete);
    if (RT_FAILURE(rc))
        return rc;

    if (fZeroFreeBlocks)
        fFlags |= VDIOCTX_FLAGS_ZERO_FREE_BLOCKS;
    if (fUpdateCache)
        fFlags |= VDIOCTX_FLAGS_READ_UPDATE_CACHE;

    Segment.pvSeg = pvBuf;
    Segment.cbSeg = cbRead;
    RTSgBufInit(&SgBuf, &Segment, 1);
    vdIoCtxInit(&IoCtx, pDisk, VDIOCTXTXDIR_READ, uOffset, cbRead, pImage, &SgBuf,
                NULL, vdReadHelperAsync, fFlags);

    IoCtx.Req.Io.pImageParentOverride = pImageParentOverride;
    IoCtx.Req.Io.cImagesRead = cImagesRead;
    IoCtx.Type.Root.pfnComplete = vdIoCtxSyncComplete;
    IoCtx.Type.Root.pvUser1     = pDisk;
    IoCtx.Type.Root.pvUser2     = hEventComplete;
    rc = vdIoCtxProcessSync(&IoCtx, hEventComplete);
    RTSemEventDestroy(hEventComplete);
    return rc;
}

/**
 * internal: read the specified amount of data in whatever blocks the backend
 * will give us.
 */
static int vdReadHelper(PVDISK pDisk, PVDIMAGE pImage, uint64_t uOffset,
                        void *pvBuf, size_t cbRead, bool fUpdateCache)
{
    return vdReadHelperEx(pDisk, pImage, NULL, uOffset, pvBuf, cbRead,
                          true /* fZeroFreeBlocks */, fUpdateCache, 0);
}

/**
 * internal: mark the disk as not modified.
 */
static void vdResetModifiedFlag(PVDISK pDisk)
{
    if (pDisk->uModified & VD_IMAGE_MODIFIED_FLAG)
    {
        /* generate new last-modified uuid */
        if (!(pDisk->uModified & VD_IMAGE_MODIFIED_DISABLE_UUID_UPDATE))
        {
            RTUUID Uuid;

            RTUuidCreate(&Uuid);
            pDisk->pLast->Backend->pfnSetModificationUuid(pDisk->pLast->pBackendData,
                                                          &Uuid);

            if (pDisk->pCache)
                pDisk->pCache->Backend->pfnSetModificationUuid(pDisk->pCache->pBackendData,
                                                               &Uuid);
        }

        pDisk->uModified &= ~VD_IMAGE_MODIFIED_FLAG;
    }
}

/**
 * internal: mark the disk as modified.
 */
static void vdSetModifiedFlag(PVDISK pDisk)
{
    pDisk->uModified |= VD_IMAGE_MODIFIED_FLAG;
    if (pDisk->uModified & VD_IMAGE_MODIFIED_FIRST)
    {
        pDisk->uModified &= ~VD_IMAGE_MODIFIED_FIRST;

        /* First modify, so create a UUID and ensure it's written to disk. */
        vdResetModifiedFlag(pDisk);

        if (!(pDisk->uModified & VD_IMAGE_MODIFIED_DISABLE_UUID_UPDATE))
        {
            VDIOCTX IoCtx;
            vdIoCtxInit(&IoCtx, pDisk, VDIOCTXTXDIR_FLUSH, 0, 0, NULL,
                        NULL, NULL, NULL, VDIOCTX_FLAGS_SYNC);
            pDisk->pLast->Backend->pfnFlush(pDisk->pLast->pBackendData, &IoCtx);
        }
    }
}

/**
 * internal: write buffer to the image, taking care of block boundaries and
 * write optimizations.
 */
static int vdWriteHelperEx(PVDISK pDisk, PVDIMAGE pImage,
                           PVDIMAGE pImageParentOverride, uint64_t uOffset,
                           const void *pvBuf, size_t cbWrite,
                           uint32_t fFlags, unsigned cImagesRead)
{
    int rc = VINF_SUCCESS;
    RTSGSEG Segment;
    RTSGBUF SgBuf;
    VDIOCTX IoCtx;
    RTSEMEVENT hEventComplete = NIL_RTSEMEVENT;

    rc = RTSemEventCreate(&hEventComplete);
    if (RT_FAILURE(rc))
        return rc;

    fFlags |= VDIOCTX_FLAGS_SYNC | VDIOCTX_FLAGS_DONT_FREE;

    Segment.pvSeg = (void *)pvBuf;
    Segment.cbSeg = cbWrite;
    RTSgBufInit(&SgBuf, &Segment, 1);
    vdIoCtxInit(&IoCtx, pDisk, VDIOCTXTXDIR_WRITE, uOffset, cbWrite, pImage, &SgBuf,
                NULL, vdWriteHelperAsync, fFlags);

    IoCtx.Req.Io.pImageParentOverride = pImageParentOverride;
    IoCtx.Req.Io.cImagesRead = cImagesRead;
    IoCtx.pIoCtxParent          = NULL;
    IoCtx.Type.Root.pfnComplete = vdIoCtxSyncComplete;
    IoCtx.Type.Root.pvUser1     = pDisk;
    IoCtx.Type.Root.pvUser2     = hEventComplete;
    if (RT_SUCCESS(rc))
        rc = vdIoCtxProcessSync(&IoCtx, hEventComplete);

    RTSemEventDestroy(hEventComplete);
    return rc;
}

/**
 * internal: write buffer to the image, taking care of block boundaries and
 * write optimizations.
 */
static int vdWriteHelper(PVDISK pDisk, PVDIMAGE pImage, uint64_t uOffset,
                         const void *pvBuf, size_t cbWrite, uint32_t fFlags)
{
    return vdWriteHelperEx(pDisk, pImage, NULL, uOffset, pvBuf, cbWrite,
                           fFlags, 0);
}

/**
 * Internal: Copies the content of one disk to another one applying optimizations
 * to speed up the copy process if possible.
 */
static int vdCopyHelper(PVDISK pDiskFrom, PVDIMAGE pImageFrom, PVDISK pDiskTo,
                        uint64_t cbSize, unsigned cImagesFromRead, unsigned cImagesToRead,
                        bool fSuppressRedundantIo, PVDINTERFACEPROGRESS pIfProgress,
                        PVDINTERFACEPROGRESS pDstIfProgress)
{
    int rc = VINF_SUCCESS;
    int rc2;
    uint64_t uOffset = 0;
    uint64_t cbRemaining = cbSize;
    void *pvBuf = NULL;
    bool fLockReadFrom = false;
    bool fLockWriteTo = false;
    bool fBlockwiseCopy = false;
    unsigned uProgressOld = 0;

    LogFlowFunc(("pDiskFrom=%#p pImageFrom=%#p pDiskTo=%#p cbSize=%llu cImagesFromRead=%u cImagesToRead=%u fSuppressRedundantIo=%RTbool pIfProgress=%#p pDstIfProgress=%#p\n",
                 pDiskFrom, pImageFrom, pDiskTo, cbSize, cImagesFromRead, cImagesToRead, fSuppressRedundantIo, pDstIfProgress, pDstIfProgress));

    if (   (fSuppressRedundantIo || (cImagesFromRead > 0))
        && RTListIsEmpty(&pDiskFrom->ListFilterChainRead))
        fBlockwiseCopy = true;

    /* Allocate tmp buffer. */
    pvBuf = RTMemTmpAlloc(VD_MERGE_BUFFER_SIZE);
    if (!pvBuf)
        return rc;

    do
    {
        size_t cbThisRead = RT_MIN(VD_MERGE_BUFFER_SIZE, cbRemaining);

        /* Note that we don't attempt to synchronize cross-disk accesses.
         * It wouldn't be very difficult to do, just the lock order would
         * need to be defined somehow to prevent deadlocks. Postpone such
         * magic as there is no use case for this. */

        rc2 = vdThreadStartRead(pDiskFrom);
        AssertRC(rc2);
        fLockReadFrom = true;

        if (fBlockwiseCopy)
        {
            RTSGSEG SegmentBuf;
            RTSGBUF SgBuf;
            VDIOCTX IoCtx;

            SegmentBuf.pvSeg = pvBuf;
            SegmentBuf.cbSeg = VD_MERGE_BUFFER_SIZE;
            RTSgBufInit(&SgBuf, &SegmentBuf, 1);
            vdIoCtxInit(&IoCtx, pDiskFrom, VDIOCTXTXDIR_READ, 0, 0, NULL,
                        &SgBuf, NULL, NULL, VDIOCTX_FLAGS_SYNC);

            /* Read the source data. */
            rc = pImageFrom->Backend->pfnRead(pImageFrom->pBackendData,
                                              uOffset, cbThisRead, &IoCtx,
                                              &cbThisRead);

            if (   rc == VERR_VD_BLOCK_FREE
                && cImagesFromRead != 1)
            {
                unsigned cImagesToProcess = cImagesFromRead;

                for (PVDIMAGE pCurrImage = pImageFrom->pPrev;
                     pCurrImage != NULL && rc == VERR_VD_BLOCK_FREE;
                     pCurrImage = pCurrImage->pPrev)
                {
                    rc = pCurrImage->Backend->pfnRead(pCurrImage->pBackendData,
                                                           uOffset, cbThisRead,
                                                           &IoCtx, &cbThisRead);
                    if (cImagesToProcess == 1)
                        break;
                    else if (cImagesToProcess > 0)
                        cImagesToProcess--;
                }
            }
        }
        else
            rc = vdReadHelper(pDiskFrom, pImageFrom, uOffset, pvBuf, cbThisRead,
                              false /* fUpdateCache */);

        if (RT_FAILURE(rc) && rc != VERR_VD_BLOCK_FREE)
            break;

        rc2 = vdThreadFinishRead(pDiskFrom);
        AssertRC(rc2);
        fLockReadFrom = false;

        if (rc != VERR_VD_BLOCK_FREE)
        {
            rc2 = vdThreadStartWrite(pDiskTo);
            AssertRC(rc2);
            fLockWriteTo = true;

            /* Only do collapsed I/O if we are copying the data blockwise. */
            rc = vdWriteHelperEx(pDiskTo, pDiskTo->pLast, NULL, uOffset, pvBuf,
                                 cbThisRead, VDIOCTX_FLAGS_DONT_SET_MODIFIED_FLAG /* fFlags */,
                                 fBlockwiseCopy ? cImagesToRead : 0);
            if (RT_FAILURE(rc))
                break;

            rc2 = vdThreadFinishWrite(pDiskTo);
            AssertRC(rc2);
            fLockWriteTo = false;
        }
        else /* Don't propagate the error to the outside */
            rc = VINF_SUCCESS;

        uOffset += cbThisRead;
        cbRemaining -= cbThisRead;

        unsigned uProgressNew = uOffset * 99 / cbSize;
        if (uProgressNew != uProgressOld)
        {
            uProgressOld = uProgressNew;

            if (pIfProgress && pIfProgress->pfnProgress)
            {
                rc = pIfProgress->pfnProgress(pIfProgress->Core.pvUser,
                                              uProgressOld);
                if (RT_FAILURE(rc))
                    break;
            }
            if (pDstIfProgress && pDstIfProgress->pfnProgress)
            {
                rc = pDstIfProgress->pfnProgress(pDstIfProgress->Core.pvUser,
                                                 uProgressOld);
                if (RT_FAILURE(rc))
                    break;
            }
        }
    } while (uOffset < cbSize);

    RTMemFree(pvBuf);

    if (fLockReadFrom)
    {
        rc2 = vdThreadFinishRead(pDiskFrom);
        AssertRC(rc2);
    }

    if (fLockWriteTo)
    {
        rc2 = vdThreadFinishWrite(pDiskTo);
        AssertRC(rc2);
    }

    LogFlowFunc(("returns rc=%Rrc\n", rc));
    return rc;
}

/**
 * Flush helper async version.
 */
static DECLCALLBACK(int) vdSetModifiedHelperAsync(PVDIOCTX pIoCtx)
{
    int rc = VINF_SUCCESS;
    PVDIMAGE pImage = pIoCtx->Req.Io.pImageCur;

    rc = pImage->Backend->pfnFlush(pImage->pBackendData, pIoCtx);
    if (rc == VERR_VD_ASYNC_IO_IN_PROGRESS)
        rc = VINF_SUCCESS;

    return rc;
}

/**
 * internal: mark the disk as modified - async version.
 */
static int vdSetModifiedFlagAsync(PVDISK pDisk, PVDIOCTX pIoCtx)
{
    int rc = VINF_SUCCESS;

    VD_IS_LOCKED(pDisk);

    pDisk->uModified |= VD_IMAGE_MODIFIED_FLAG;
    if (pDisk->uModified & VD_IMAGE_MODIFIED_FIRST)
    {
        rc = vdIoCtxLockDisk(pDisk, pIoCtx);
        if (RT_SUCCESS(rc))
        {
            pDisk->uModified &= ~VD_IMAGE_MODIFIED_FIRST;

            /* First modify, so create a UUID and ensure it's written to disk. */
            vdResetModifiedFlag(pDisk);

            if (!(pDisk->uModified & VD_IMAGE_MODIFIED_DISABLE_UUID_UPDATE))
            {
                PVDIOCTX pIoCtxFlush = vdIoCtxChildAlloc(pDisk, VDIOCTXTXDIR_FLUSH,
                                                         0, 0, pDisk->pLast,
                                                         NULL, pIoCtx, 0, 0, NULL,
                                                         vdSetModifiedHelperAsync);

                if (pIoCtxFlush)
                {
                    rc = vdIoCtxProcessLocked(pIoCtxFlush);
                    if (rc == VINF_VD_ASYNC_IO_FINISHED)
                    {
                        vdIoCtxUnlockDisk(pDisk, pIoCtx, false /* fProcessDeferredReqs */);
                        vdIoCtxFree(pDisk, pIoCtxFlush);
                    }
                    else if (rc == VERR_VD_ASYNC_IO_IN_PROGRESS)
                    {
                        ASMAtomicIncU32(&pIoCtx->cDataTransfersPending);
                        pIoCtx->fFlags |= VDIOCTX_FLAGS_BLOCKED;
                    }
                    else /* Another error */
                        vdIoCtxFree(pDisk, pIoCtxFlush);
                }
                else
                    rc = VERR_NO_MEMORY;
            }
        }
    }

    return rc;
}

static DECLCALLBACK(int) vdWriteHelperCommitAsync(PVDIOCTX pIoCtx)
{
    int rc             = VINF_SUCCESS;
    PVDIMAGE pImage    = pIoCtx->Req.Io.pImageStart;
    size_t cbPreRead   = pIoCtx->Type.Child.cbPreRead;
    size_t cbPostRead  = pIoCtx->Type.Child.cbPostRead;
    size_t cbThisWrite = pIoCtx->Type.Child.cbTransferParent;

    LogFlowFunc(("pIoCtx=%#p\n", pIoCtx));
    rc = pImage->Backend->pfnWrite(pImage->pBackendData,
                                   pIoCtx->Req.Io.uOffset - cbPreRead,
                                   cbPreRead + cbThisWrite + cbPostRead,
                                   pIoCtx, NULL, &cbPreRead, &cbPostRead, 0);
    Assert(rc != VERR_VD_BLOCK_FREE);
    Assert(rc == VERR_VD_NOT_ENOUGH_METADATA || cbPreRead == 0);
    Assert(rc == VERR_VD_NOT_ENOUGH_METADATA || cbPostRead == 0);
    if (rc == VERR_VD_ASYNC_IO_IN_PROGRESS)
        rc = VINF_SUCCESS;
    else if (rc == VERR_VD_IOCTX_HALT)
    {
        pIoCtx->fFlags |= VDIOCTX_FLAGS_BLOCKED;
        rc = VINF_SUCCESS;
    }

    LogFlowFunc(("returns rc=%Rrc\n", rc));
    return rc;
}

static DECLCALLBACK(int) vdWriteHelperOptimizedCmpAndWriteAsync(PVDIOCTX pIoCtx)
{
    int rc = VINF_SUCCESS;
    size_t cbThisWrite    = 0;
    size_t cbPreRead      = pIoCtx->Type.Child.cbPreRead;
    size_t cbPostRead     = pIoCtx->Type.Child.cbPostRead;
    size_t cbWriteCopy    = pIoCtx->Type.Child.Write.Optimized.cbWriteCopy;
    size_t cbFill         = pIoCtx->Type.Child.Write.Optimized.cbFill;
    size_t cbReadImage    = pIoCtx->Type.Child.Write.Optimized.cbReadImage;
    PVDIOCTX pIoCtxParent = pIoCtx->pIoCtxParent;

    LogFlowFunc(("pIoCtx=%#p\n", pIoCtx));

    AssertPtr(pIoCtxParent);
    Assert(!pIoCtxParent->pIoCtxParent);
    Assert(!pIoCtx->Req.Io.cbTransferLeft && !pIoCtx->cMetaTransfersPending);

    vdIoCtxChildReset(pIoCtx);
    cbThisWrite = pIoCtx->Type.Child.cbTransferParent;
    RTSgBufAdvance(&pIoCtx->Req.Io.SgBuf, cbPreRead);

    /* Check if the write would modify anything in this block. */
    if (!RTSgBufCmp(&pIoCtx->Req.Io.SgBuf, &pIoCtxParent->Req.Io.SgBuf, cbThisWrite))
    {
        RTSGBUF SgBufSrcTmp;

        RTSgBufClone(&SgBufSrcTmp, &pIoCtxParent->Req.Io.SgBuf);
        RTSgBufAdvance(&SgBufSrcTmp, cbThisWrite);
        RTSgBufAdvance(&pIoCtx->Req.Io.SgBuf, cbThisWrite);

        if (!cbWriteCopy || !RTSgBufCmp(&pIoCtx->Req.Io.SgBuf, &SgBufSrcTmp, cbWriteCopy))
        {
            /* Block is completely unchanged, so no need to write anything. */
            LogFlowFunc(("Block didn't changed\n"));
            ASMAtomicWriteU32(&pIoCtx->Req.Io.cbTransferLeft, 0);
            RTSgBufAdvance(&pIoCtxParent->Req.Io.SgBuf, cbThisWrite);
            return VINF_VD_ASYNC_IO_FINISHED;
        }
    }

    /* Copy the data to the right place in the buffer. */
    RTSgBufReset(&pIoCtx->Req.Io.SgBuf);
    RTSgBufAdvance(&pIoCtx->Req.Io.SgBuf, cbPreRead);
    vdIoCtxCopy(pIoCtx, pIoCtxParent, cbThisWrite);

    /* Handle the data that goes after the write to fill the block. */
    if (cbPostRead)
    {
        /* Now assemble the remaining data. */
        if (cbWriteCopy)
        {
            /*
             * The S/G buffer of the parent needs to be cloned because
             * it is not allowed to modify the state.
             */
            RTSGBUF SgBufParentTmp;

            RTSgBufClone(&SgBufParentTmp, &pIoCtxParent->Req.Io.SgBuf);
            RTSgBufCopy(&pIoCtx->Req.Io.SgBuf, &SgBufParentTmp, cbWriteCopy);
        }

        /* Zero out the remainder of this block. Will never be visible, as this
         * is beyond the limit of the image. */
        if (cbFill)
        {
            RTSgBufAdvance(&pIoCtx->Req.Io.SgBuf, cbReadImage);
            vdIoCtxSet(pIoCtx, '\0', cbFill);
        }
    }

    /* Write the full block to the virtual disk. */
    RTSgBufReset(&pIoCtx->Req.Io.SgBuf);
    pIoCtx->pfnIoCtxTransferNext = vdWriteHelperCommitAsync;

    return rc;
}

static DECLCALLBACK(int) vdWriteHelperOptimizedPreReadAsync(PVDIOCTX pIoCtx)
{
    int rc = VINF_SUCCESS;

    LogFlowFunc(("pIoCtx=%#p\n", pIoCtx));

    pIoCtx->fFlags |= VDIOCTX_FLAGS_ZERO_FREE_BLOCKS;

    if (   pIoCtx->Req.Io.cbTransferLeft
        && !pIoCtx->cDataTransfersPending)
        rc = vdReadHelperAsync(pIoCtx);

    if (   (   RT_SUCCESS(rc)
            || (rc == VERR_VD_ASYNC_IO_IN_PROGRESS))
        && (   pIoCtx->Req.Io.cbTransferLeft
            || pIoCtx->cMetaTransfersPending))
        rc = VERR_VD_ASYNC_IO_IN_PROGRESS;
    else
        pIoCtx->pfnIoCtxTransferNext = vdWriteHelperOptimizedCmpAndWriteAsync;

    return rc;
}

/**
 * internal: write a complete block (only used for diff images), taking the
 * remaining data from parent images. This implementation optimizes out writes
 * that do not change the data relative to the state as of the parent images.
 * All backends which support differential/growing images support this - async version.
 */
static DECLCALLBACK(int) vdWriteHelperOptimizedAsync(PVDIOCTX pIoCtx)
{
    PVDISK pDisk = pIoCtx->pDisk;
    uint64_t uOffset   = pIoCtx->Type.Child.uOffsetSaved;
    size_t cbThisWrite = pIoCtx->Type.Child.cbTransferParent;
    size_t cbPreRead   = pIoCtx->Type.Child.cbPreRead;
    size_t cbPostRead  = pIoCtx->Type.Child.cbPostRead;
    size_t cbWrite     = pIoCtx->Type.Child.cbWriteParent;
    size_t cbFill = 0;
    size_t cbWriteCopy = 0;
    size_t cbReadImage = 0;

    LogFlowFunc(("pIoCtx=%#p\n", pIoCtx));

    AssertPtr(pIoCtx->pIoCtxParent);
    Assert(!pIoCtx->pIoCtxParent->pIoCtxParent);

    if (cbPostRead)
    {
        /* Figure out how much we cannot read from the image, because
         * the last block to write might exceed the nominal size of the
         * image for technical reasons. */
        if (uOffset + cbThisWrite + cbPostRead > pDisk->cbSize)
            cbFill = uOffset + cbThisWrite + cbPostRead - pDisk->cbSize;

        /* If we have data to be written, use that instead of reading
         * data from the image. */
        if (cbWrite > cbThisWrite)
            cbWriteCopy = RT_MIN(cbWrite - cbThisWrite, cbPostRead);

        /* The rest must be read from the image. */
        cbReadImage = cbPostRead - cbWriteCopy - cbFill;
    }

    pIoCtx->Type.Child.Write.Optimized.cbFill      = cbFill;
    pIoCtx->Type.Child.Write.Optimized.cbWriteCopy = cbWriteCopy;
    pIoCtx->Type.Child.Write.Optimized.cbReadImage = cbReadImage;

    /* Read the entire data of the block so that we can compare whether it will
     * be modified by the write or not. */
    size_t cbTmp = cbPreRead + cbThisWrite + cbPostRead - cbFill; Assert(cbTmp == (uint32_t)cbTmp);
    pIoCtx->Req.Io.cbTransferLeft = (uint32_t)cbTmp;
    pIoCtx->Req.Io.cbTransfer     = pIoCtx->Req.Io.cbTransferLeft;
    pIoCtx->Req.Io.uOffset       -= cbPreRead;

    /* Next step */
    pIoCtx->pfnIoCtxTransferNext = vdWriteHelperOptimizedPreReadAsync;
    return VINF_SUCCESS;
}

static DECLCALLBACK(int) vdWriteHelperStandardReadImageAsync(PVDIOCTX pIoCtx)
{
    int rc = VINF_SUCCESS;

    LogFlowFunc(("pIoCtx=%#p\n", pIoCtx));

    pIoCtx->fFlags |= VDIOCTX_FLAGS_ZERO_FREE_BLOCKS;

    if (   pIoCtx->Req.Io.cbTransferLeft
        && !pIoCtx->cDataTransfersPending)
        rc = vdReadHelperAsync(pIoCtx);

    if (   RT_SUCCESS(rc)
        && (   pIoCtx->Req.Io.cbTransferLeft
            || pIoCtx->cMetaTransfersPending))
        rc = VERR_VD_ASYNC_IO_IN_PROGRESS;
    else
    {
        size_t cbFill = pIoCtx->Type.Child.Write.Optimized.cbFill;

        /* Zero out the remainder of this block. Will never be visible, as this
         * is beyond the limit of the image. */
        if (cbFill)
            vdIoCtxSet(pIoCtx, '\0', cbFill);

        /* Write the full block to the virtual disk. */
        RTSgBufReset(&pIoCtx->Req.Io.SgBuf);

        vdIoCtxChildReset(pIoCtx);
        pIoCtx->pfnIoCtxTransferNext = vdWriteHelperCommitAsync;
    }

    return rc;
}

static DECLCALLBACK(int) vdWriteHelperStandardAssemble(PVDIOCTX pIoCtx)
{
    int rc = VINF_SUCCESS;
    size_t cbPostRead  = pIoCtx->Type.Child.cbPostRead;
    size_t cbThisWrite = pIoCtx->Type.Child.cbTransferParent;
    PVDIOCTX pIoCtxParent = pIoCtx->pIoCtxParent;

    LogFlowFunc(("pIoCtx=%#p\n", pIoCtx));

    vdIoCtxCopy(pIoCtx, pIoCtxParent, cbThisWrite);
    if (cbPostRead)
    {
        size_t cbFill = pIoCtx->Type.Child.Write.Optimized.cbFill;
        size_t cbWriteCopy = pIoCtx->Type.Child.Write.Optimized.cbWriteCopy;
        size_t cbReadImage = pIoCtx->Type.Child.Write.Optimized.cbReadImage;

        /* Now assemble the remaining data. */
        if (cbWriteCopy)
        {
            /*
             * The S/G buffer of the parent needs to be cloned because
             * it is not allowed to modify the state.
             */
            RTSGBUF SgBufParentTmp;

            RTSgBufClone(&SgBufParentTmp, &pIoCtxParent->Req.Io.SgBuf);
            RTSgBufCopy(&pIoCtx->Req.Io.SgBuf, &SgBufParentTmp, cbWriteCopy);
        }

        if (cbReadImage)
        {
            /* Read remaining data. */
            pIoCtx->pfnIoCtxTransferNext = vdWriteHelperStandardReadImageAsync;

            /* Read the data that goes before the write to fill the block. */
            pIoCtx->Req.Io.cbTransferLeft = (uint32_t)cbReadImage; Assert(cbReadImage == (uint32_t)cbReadImage);
            pIoCtx->Req.Io.cbTransfer     = pIoCtx->Req.Io.cbTransferLeft;
            pIoCtx->Req.Io.uOffset       += cbWriteCopy;
        }
        else
        {
            /* Zero out the remainder of this block. Will never be visible, as this
             * is beyond the limit of the image. */
            if (cbFill)
                vdIoCtxSet(pIoCtx, '\0', cbFill);

            /* Write the full block to the virtual disk. */
            RTSgBufReset(&pIoCtx->Req.Io.SgBuf);
            vdIoCtxChildReset(pIoCtx);
            pIoCtx->pfnIoCtxTransferNext = vdWriteHelperCommitAsync;
        }
    }
    else
    {
        /* Write the full block to the virtual disk. */
        RTSgBufReset(&pIoCtx->Req.Io.SgBuf);
        vdIoCtxChildReset(pIoCtx);
        pIoCtx->pfnIoCtxTransferNext = vdWriteHelperCommitAsync;
    }

    return rc;
}

static DECLCALLBACK(int) vdWriteHelperStandardPreReadAsync(PVDIOCTX pIoCtx)
{
    int rc = VINF_SUCCESS;

    LogFlowFunc(("pIoCtx=%#p\n", pIoCtx));

    pIoCtx->fFlags |= VDIOCTX_FLAGS_ZERO_FREE_BLOCKS;

    if (   pIoCtx->Req.Io.cbTransferLeft
        && !pIoCtx->cDataTransfersPending)
        rc = vdReadHelperAsync(pIoCtx);

    if (   RT_SUCCESS(rc)
        && (   pIoCtx->Req.Io.cbTransferLeft
            || pIoCtx->cMetaTransfersPending))
        rc = VERR_VD_ASYNC_IO_IN_PROGRESS;
     else
        pIoCtx->pfnIoCtxTransferNext = vdWriteHelperStandardAssemble;

    return rc;
}

static DECLCALLBACK(int) vdWriteHelperStandardAsync(PVDIOCTX pIoCtx)
{
    PVDISK pDisk = pIoCtx->pDisk;
    uint64_t uOffset   = pIoCtx->Type.Child.uOffsetSaved;
    size_t cbThisWrite = pIoCtx->Type.Child.cbTransferParent;
    size_t cbPreRead   = pIoCtx->Type.Child.cbPreRead;
    size_t cbPostRead  = pIoCtx->Type.Child.cbPostRead;
    size_t cbWrite     = pIoCtx->Type.Child.cbWriteParent;
    size_t cbFill = 0;
    size_t cbWriteCopy = 0;
    size_t cbReadImage = 0;

    LogFlowFunc(("pIoCtx=%#p\n", pIoCtx));

    AssertPtr(pIoCtx->pIoCtxParent);
    Assert(!pIoCtx->pIoCtxParent->pIoCtxParent);

    /* Calculate the amount of data to read that goes after the write to fill the block. */
    if (cbPostRead)
    {
        /* If we have data to be written, use that instead of reading
         * data from the image. */
        if (cbWrite > cbThisWrite)
            cbWriteCopy = RT_MIN(cbWrite - cbThisWrite, cbPostRead);
        else
            cbWriteCopy = 0;

        /* Figure out how much we cannot read from the image, because
         * the last block to write might exceed the nominal size of the
         * image for technical reasons. */
        if (uOffset + cbThisWrite + cbPostRead > pDisk->cbSize)
            cbFill = uOffset + cbThisWrite + cbPostRead - pDisk->cbSize;

        /* The rest must be read from the image. */
        cbReadImage = cbPostRead - cbWriteCopy - cbFill;
    }

    pIoCtx->Type.Child.Write.Optimized.cbFill      = cbFill;
    pIoCtx->Type.Child.Write.Optimized.cbWriteCopy = cbWriteCopy;
    pIoCtx->Type.Child.Write.Optimized.cbReadImage = cbReadImage;

    /* Next step */
    if (cbPreRead)
    {
        pIoCtx->pfnIoCtxTransferNext = vdWriteHelperStandardPreReadAsync;

        /* Read the data that goes before the write to fill the block. */
        pIoCtx->Req.Io.cbTransferLeft = (uint32_t)cbPreRead; Assert(cbPreRead == (uint32_t)cbPreRead);
        pIoCtx->Req.Io.cbTransfer     = pIoCtx->Req.Io.cbTransferLeft;
        pIoCtx->Req.Io.uOffset       -= cbPreRead;
    }
    else
        pIoCtx->pfnIoCtxTransferNext = vdWriteHelperStandardAssemble;

    return VINF_SUCCESS;
}

/**
 * internal: write buffer to the image, taking care of block boundaries and
 * write optimizations - async version.
 */
static DECLCALLBACK(int) vdWriteHelperAsync(PVDIOCTX pIoCtx)
{
    int rc;
    size_t cbWrite   = pIoCtx->Req.Io.cbTransfer;
    uint64_t uOffset = pIoCtx->Req.Io.uOffset;
    PVDIMAGE pImage  = pIoCtx->Req.Io.pImageCur;
    PVDISK pDisk   = pIoCtx->pDisk;
    unsigned fWrite;
    size_t cbThisWrite;
    size_t cbPreRead, cbPostRead;

    /* Apply write filter chain here if it was not done already. */
    if (!(pIoCtx->fFlags & VDIOCTX_FLAGS_WRITE_FILTER_APPLIED))
    {
        rc = vdFilterChainApplyWrite(pDisk, uOffset, cbWrite, pIoCtx);
        if (RT_FAILURE(rc))
            return rc;
        pIoCtx->fFlags |= VDIOCTX_FLAGS_WRITE_FILTER_APPLIED;
    }

    if (!(pIoCtx->fFlags & VDIOCTX_FLAGS_DONT_SET_MODIFIED_FLAG))
    {
        rc = vdSetModifiedFlagAsync(pDisk, pIoCtx);
        if (RT_FAILURE(rc)) /* Includes I/O in progress. */
            return rc;
    }

    rc = vdDiscardSetRangeAllocated(pDisk, uOffset, cbWrite);
    if (RT_FAILURE(rc))
        return rc;

    /* Loop until all written. */
    do
    {
        /* Try to write the possibly partial block to the last opened image.
         * This works when the block is already allocated in this image or
         * if it is a full-block write (and allocation isn't suppressed below).
         * For image formats which don't support zero blocks, it's beneficial
         * to avoid unnecessarily allocating unchanged blocks. This prevents
         * unwanted expanding of images. VMDK is an example. */
        cbThisWrite = cbWrite;

        /*
         * Check whether there is a full block write in progress which was not allocated.
         * Defer I/O if the range interferes.
         */
        if (   pDisk->pIoCtxLockOwner != NIL_VDIOCTX
            && uOffset >= pDisk->uOffsetStartLocked
            && uOffset < pDisk->uOffsetEndLocked)
        {
            Log(("Interferring write while allocating a new block => deferring write\n"));
            vdIoCtxDefer(pDisk, pIoCtx);
            rc = VERR_VD_ASYNC_IO_IN_PROGRESS;
            break;
        }

        fWrite =   (pImage->uOpenFlags & VD_OPEN_FLAGS_HONOR_SAME)
                 ? 0 : VD_WRITE_NO_ALLOC;
        rc = pImage->Backend->pfnWrite(pImage->pBackendData, uOffset, cbThisWrite,
                                       pIoCtx, &cbThisWrite, &cbPreRead, &cbPostRead,
                                       fWrite);
        if (rc == VERR_VD_BLOCK_FREE)
        {
            /* Lock the disk .*/
            rc = vdIoCtxLockDisk(pDisk, pIoCtx);
            if (RT_SUCCESS(rc))
            {
                /*
                 * Allocate segment and buffer in one go.
                 * A bit hackish but avoids the need to allocate memory twice.
                 */
                PRTSGBUF pTmp = (PRTSGBUF)RTMemAlloc(cbPreRead + cbThisWrite + cbPostRead + sizeof(RTSGSEG) + sizeof(RTSGBUF));
                AssertBreakStmt(pTmp, rc = VERR_NO_MEMORY);
                PRTSGSEG pSeg = (PRTSGSEG)(pTmp + 1);

                pSeg->pvSeg = pSeg + 1;
                pSeg->cbSeg = cbPreRead + cbThisWrite + cbPostRead;
                RTSgBufInit(pTmp, pSeg, 1);

                PVDIOCTX pIoCtxWrite = vdIoCtxChildAlloc(pDisk, VDIOCTXTXDIR_WRITE,
                                                         uOffset, pSeg->cbSeg, pImage,
                                                         pTmp,
                                                         pIoCtx, cbThisWrite,
                                                         cbWrite,
                                                         pTmp,
                                                           (pImage->uOpenFlags & VD_OPEN_FLAGS_HONOR_SAME)
                                                         ? vdWriteHelperStandardAsync
                                                         : vdWriteHelperOptimizedAsync);
                if (!pIoCtxWrite)
                {
                    RTMemTmpFree(pTmp);
                    rc = VERR_NO_MEMORY;
                    break;
                }

                LogFlowFunc(("Disk is growing because of pIoCtx=%#p pIoCtxWrite=%#p\n",
                             pIoCtx, pIoCtxWrite));

                /* Save the current range for the growing operation to check for intersecting requests later. */
                pDisk->uOffsetStartLocked = uOffset - cbPreRead;
                pDisk->uOffsetEndLocked = uOffset + cbThisWrite + cbPostRead;

                pIoCtxWrite->Type.Child.cbPreRead  = cbPreRead;
                pIoCtxWrite->Type.Child.cbPostRead = cbPostRead;
                pIoCtxWrite->Req.Io.pImageParentOverride = pIoCtx->Req.Io.pImageParentOverride;

                /* Process the write request */
                rc = vdIoCtxProcessLocked(pIoCtxWrite);

                if (RT_FAILURE(rc) && (rc != VERR_VD_ASYNC_IO_IN_PROGRESS))
                {
                    vdIoCtxUnlockDisk(pDisk, pIoCtx, false /* fProcessDeferredReqs*/ );
                    vdIoCtxFree(pDisk, pIoCtxWrite);
                    break;
                }
                else if (   rc == VINF_VD_ASYNC_IO_FINISHED
                         && ASMAtomicCmpXchgBool(&pIoCtxWrite->fComplete, true, false))
                {
                    LogFlow(("Child write request completed\n"));
                    Assert(pIoCtx->Req.Io.cbTransferLeft >= cbThisWrite);
                    Assert(cbThisWrite == (uint32_t)cbThisWrite);
                    rc = pIoCtxWrite->rcReq;
                    ASMAtomicSubU32(&pIoCtx->Req.Io.cbTransferLeft, (uint32_t)cbThisWrite);
                    vdIoCtxUnlockDisk(pDisk, pIoCtx, false /* fProcessDeferredReqs*/ );
                    vdIoCtxFree(pDisk, pIoCtxWrite);
                }
                else
                {
                    LogFlow(("Child write pending\n"));
                    ASMAtomicIncU32(&pIoCtx->cDataTransfersPending);
                    pIoCtx->fFlags |= VDIOCTX_FLAGS_BLOCKED;
                    rc = VERR_VD_ASYNC_IO_IN_PROGRESS;
                    cbWrite -= cbThisWrite;
                    uOffset += cbThisWrite;
                    break;
                }
            }
            else
            {
                rc = VERR_VD_ASYNC_IO_IN_PROGRESS;
                break;
            }
        }

        if (rc == VERR_VD_IOCTX_HALT)
        {
            cbWrite -= cbThisWrite;
            uOffset += cbThisWrite;
            pIoCtx->fFlags |= VDIOCTX_FLAGS_BLOCKED;
            break;
        }
        else if (rc == VERR_VD_NOT_ENOUGH_METADATA)
            break;

        cbWrite -= cbThisWrite;
        uOffset += cbThisWrite;
    } while (cbWrite != 0 && (RT_SUCCESS(rc) || rc == VERR_VD_ASYNC_IO_IN_PROGRESS));

    if (   rc == VERR_VD_ASYNC_IO_IN_PROGRESS
        || rc == VERR_VD_NOT_ENOUGH_METADATA
        || rc == VERR_VD_IOCTX_HALT)
    {
        /*
         * Tell the caller that we don't need to go back here because all
         * writes are initiated.
         */
        if (   !cbWrite
            && rc != VERR_VD_IOCTX_HALT)
            rc = VINF_SUCCESS;

        pIoCtx->Req.Io.uOffset    = uOffset;
        pIoCtx->Req.Io.cbTransfer = cbWrite;
    }

    return rc;
}

/**
 * Flush helper async version.
 */
static DECLCALLBACK(int) vdFlushHelperAsync(PVDIOCTX pIoCtx)
{
    int rc = VINF_SUCCESS;
    PVDISK pDisk = pIoCtx->pDisk;
    PVDIMAGE pImage = pIoCtx->Req.Io.pImageCur;

    rc = vdIoCtxLockDisk(pDisk, pIoCtx);
    if (RT_SUCCESS(rc))
    {
        /* Mark the whole disk as locked. */
        pDisk->uOffsetStartLocked = 0;
        pDisk->uOffsetEndLocked = UINT64_C(0xffffffffffffffff);

        vdResetModifiedFlag(pDisk);
        rc = pImage->Backend->pfnFlush(pImage->pBackendData, pIoCtx);
        if (   (   RT_SUCCESS(rc)
                || rc == VERR_VD_ASYNC_IO_IN_PROGRESS
                || rc == VERR_VD_IOCTX_HALT)
            && pDisk->pCache)
        {
            rc = pDisk->pCache->Backend->pfnFlush(pDisk->pCache->pBackendData, pIoCtx);
            if (   RT_SUCCESS(rc)
                || (   rc != VERR_VD_ASYNC_IO_IN_PROGRESS
                    && rc != VERR_VD_IOCTX_HALT))
                vdIoCtxUnlockDisk(pDisk, pIoCtx, true /* fProcessBlockedReqs */);
            else if (rc != VERR_VD_IOCTX_HALT)
                rc = VINF_SUCCESS;
        }
        else if (rc == VERR_VD_ASYNC_IO_IN_PROGRESS)
            rc = VINF_SUCCESS;
        else if (rc != VERR_VD_IOCTX_HALT)/* Some other error. */
            vdIoCtxUnlockDisk(pDisk, pIoCtx, true /* fProcessBlockedReqs */);
    }

    return rc;
}

/**
 * Async discard helper - discards a whole block which is recorded in the block
 * tree.
 *
 * @returns VBox status code.
 * @param   pIoCtx    The I/O context to operate on.
 */
static DECLCALLBACK(int) vdDiscardWholeBlockAsync(PVDIOCTX pIoCtx)
{
    int rc = VINF_SUCCESS;
    PVDISK pDisk = pIoCtx->pDisk;
    PVDDISCARDSTATE pDiscard = pDisk->pDiscard;
    PVDDISCARDBLOCK pBlock = pIoCtx->Req.Discard.pBlock;
    size_t cbPreAllocated, cbPostAllocated, cbActuallyDiscarded;

    LogFlowFunc(("pIoCtx=%#p\n", pIoCtx));

    AssertPtr(pBlock);

    rc = pDisk->pLast->Backend->pfnDiscard(pDisk->pLast->pBackendData, pIoCtx,
                                                pBlock->Core.Key, pBlock->cbDiscard,
                                                &cbPreAllocated, &cbPostAllocated,
                                                &cbActuallyDiscarded, NULL, 0);
    Assert(rc != VERR_VD_DISCARD_ALIGNMENT_NOT_MET);
    Assert(!cbPreAllocated);
    Assert(!cbPostAllocated);
    Assert(cbActuallyDiscarded == pBlock->cbDiscard || RT_FAILURE(rc));

    /* Remove the block on success. */
    if (   RT_SUCCESS(rc)
        || rc == VERR_VD_ASYNC_IO_IN_PROGRESS)
    {
        PVDDISCARDBLOCK pBlockRemove = (PVDDISCARDBLOCK)RTAvlrU64RangeRemove(pDiscard->pTreeBlocks, pBlock->Core.Key);
        Assert(pBlockRemove == pBlock); RT_NOREF1(pBlockRemove);

        pDiscard->cbDiscarding -= pBlock->cbDiscard;
        RTListNodeRemove(&pBlock->NodeLru);
        RTMemFree(pBlock->pbmAllocated);
        RTMemFree(pBlock);
        pIoCtx->Req.Discard.pBlock = NULL;/* Safety precaution. */
        pIoCtx->pfnIoCtxTransferNext = vdDiscardHelperAsync; /* Next part. */
        rc = VINF_SUCCESS;
    }

    LogFlowFunc(("returns rc=%Rrc\n", rc));
    return rc;
}

/**
 * Removes the least recently used blocks from the waiting list until
 * the new value is reached - version for async I/O.
 *
 * @returns VBox status code.
 * @param   pDisk              VD disk container.
 * @param   pIoCtx             The I/O context associated with this discard operation.
 * @param   cbDiscardingNew    How many bytes should be waiting on success.
 *                             The number of bytes waiting can be less.
 */
static int vdDiscardRemoveBlocksAsync(PVDISK pDisk, PVDIOCTX pIoCtx, size_t cbDiscardingNew)
{
    int rc = VINF_SUCCESS;
    PVDDISCARDSTATE pDiscard = pDisk->pDiscard;

    LogFlowFunc(("pDisk=%#p pDiscard=%#p cbDiscardingNew=%zu\n",
                 pDisk, pDiscard, cbDiscardingNew));

    while (pDiscard->cbDiscarding > cbDiscardingNew)
    {
        PVDDISCARDBLOCK pBlock = RTListGetLast(&pDiscard->ListLru, VDDISCARDBLOCK, NodeLru);

        Assert(!RTListIsEmpty(&pDiscard->ListLru));

        /* Go over the allocation bitmap and mark all discarded sectors as unused. */
        uint64_t offStart = pBlock->Core.Key;
        uint32_t idxStart = 0;
        size_t cbLeft = pBlock->cbDiscard;
        bool fAllocated = ASMBitTest(pBlock->pbmAllocated, idxStart);
        uint32_t cSectors = (uint32_t)(pBlock->cbDiscard / 512);

        while (cbLeft > 0)
        {
            int32_t idxEnd;
            size_t cbThis = cbLeft;

            if (fAllocated)
            {
                /* Check for the first unallocated bit. */
                idxEnd = ASMBitNextClear(pBlock->pbmAllocated, cSectors, idxStart);
                if (idxEnd != -1)
                {
                    cbThis = (idxEnd - idxStart) * 512;
                    fAllocated = false;
                }
            }
            else
            {
                /* Mark as unused and check for the first set bit. */
                idxEnd = ASMBitNextSet(pBlock->pbmAllocated, cSectors, idxStart);
                if (idxEnd != -1)
                    cbThis = (idxEnd - idxStart) * 512;

                rc = pDisk->pLast->Backend->pfnDiscard(pDisk->pLast->pBackendData, pIoCtx,
                                                            offStart, cbThis, NULL, NULL, &cbThis,
                                                            NULL, VD_DISCARD_MARK_UNUSED);
                if (      RT_FAILURE(rc)
                    && rc != VERR_VD_ASYNC_IO_IN_PROGRESS)
                    break;

                fAllocated = true;
            }

            idxStart  = idxEnd;
            offStart += cbThis;
            cbLeft   -= cbThis;
        }

        if (   RT_FAILURE(rc)
            && rc != VERR_VD_ASYNC_IO_IN_PROGRESS)
            break;

        PVDDISCARDBLOCK pBlockRemove = (PVDDISCARDBLOCK)RTAvlrU64RangeRemove(pDiscard->pTreeBlocks, pBlock->Core.Key);
        Assert(pBlockRemove == pBlock); NOREF(pBlockRemove);
        RTListNodeRemove(&pBlock->NodeLru);

        pDiscard->cbDiscarding -= pBlock->cbDiscard;
        RTMemFree(pBlock->pbmAllocated);
        RTMemFree(pBlock);
    }

    if (rc == VERR_VD_ASYNC_IO_IN_PROGRESS)
        rc = VINF_SUCCESS;

    Assert(RT_FAILURE(rc) || pDiscard->cbDiscarding <= cbDiscardingNew);

    LogFlowFunc(("returns rc=%Rrc\n", rc));
    return rc;
}

/**
 * Async discard helper - discards the current range if there is no matching
 * block in the tree.
 *
 * @returns VBox status code.
 * @param   pIoCtx    The I/O context to operate on.
 */
static DECLCALLBACK(int) vdDiscardCurrentRangeAsync(PVDIOCTX pIoCtx)
{
    PVDISK        pDisk         = pIoCtx->pDisk;
    PVDDISCARDSTATE pDiscard      = pDisk->pDiscard;
    uint64_t        offStart      = pIoCtx->Req.Discard.offCur;
    size_t          cbThisDiscard = pIoCtx->Req.Discard.cbThisDiscard;
    void *pbmAllocated = NULL;
    size_t cbPreAllocated, cbPostAllocated;
    int rc = VINF_SUCCESS;

    LogFlowFunc(("pIoCtx=%#p\n", pIoCtx));

    /* No block found, try to discard using the backend first. */
    rc = pDisk->pLast->Backend->pfnDiscard(pDisk->pLast->pBackendData, pIoCtx,
                                                offStart, cbThisDiscard, &cbPreAllocated,
                                                &cbPostAllocated, &cbThisDiscard,
                                                &pbmAllocated, 0);
    if (rc == VERR_VD_DISCARD_ALIGNMENT_NOT_MET)
    {
        /* Create new discard block. */
        PVDDISCARDBLOCK pBlock = (PVDDISCARDBLOCK)RTMemAllocZ(sizeof(VDDISCARDBLOCK));
        if (pBlock)
        {
            pBlock->Core.Key     = offStart - cbPreAllocated;
            pBlock->Core.KeyLast = offStart + cbThisDiscard + cbPostAllocated - 1;
            pBlock->cbDiscard    = cbPreAllocated + cbThisDiscard + cbPostAllocated;
            pBlock->pbmAllocated = pbmAllocated;
            bool fInserted = RTAvlrU64Insert(pDiscard->pTreeBlocks, &pBlock->Core);
            Assert(fInserted); NOREF(fInserted);

            RTListPrepend(&pDiscard->ListLru, &pBlock->NodeLru);
            pDiscard->cbDiscarding += pBlock->cbDiscard;

            Assert(pIoCtx->Req.Discard.cbDiscardLeft >= cbThisDiscard);
            pIoCtx->Req.Discard.cbDiscardLeft -= cbThisDiscard;
            pIoCtx->Req.Discard.offCur        += cbThisDiscard;
            pIoCtx->Req.Discard.cbThisDiscard = cbThisDiscard;

            if (pDiscard->cbDiscarding > VD_DISCARD_REMOVE_THRESHOLD)
                rc = vdDiscardRemoveBlocksAsync(pDisk, pIoCtx, VD_DISCARD_REMOVE_THRESHOLD);
            else
                rc = VINF_SUCCESS;

            if (RT_SUCCESS(rc))
                pIoCtx->pfnIoCtxTransferNext = vdDiscardHelperAsync; /* Next part. */
        }
        else
        {
            RTMemFree(pbmAllocated);
            rc = VERR_NO_MEMORY;
        }
    }
    else if (   RT_SUCCESS(rc)
             || rc == VERR_VD_ASYNC_IO_IN_PROGRESS) /* Save state and andvance to next range. */
    {
        Assert(pIoCtx->Req.Discard.cbDiscardLeft >= cbThisDiscard);
        pIoCtx->Req.Discard.cbDiscardLeft -= cbThisDiscard;
        pIoCtx->Req.Discard.offCur        += cbThisDiscard;
        pIoCtx->Req.Discard.cbThisDiscard  = cbThisDiscard;
        pIoCtx->pfnIoCtxTransferNext       = vdDiscardHelperAsync;
        rc = VINF_SUCCESS;
    }

    LogFlowFunc(("returns rc=%Rrc\n", rc));
    return rc;
}

/**
 * Async discard helper - entry point.
 *
 * @returns VBox status code.
 * @param   pIoCtx    The I/O context to operate on.
 */
static DECLCALLBACK(int) vdDiscardHelperAsync(PVDIOCTX pIoCtx)
{
    int rc             = VINF_SUCCESS;
    PVDISK  pDisk    = pIoCtx->pDisk;
    PCRTRANGE paRanges = pIoCtx->Req.Discard.paRanges;
    unsigned  cRanges  = pIoCtx->Req.Discard.cRanges;
    PVDDISCARDSTATE pDiscard = pDisk->pDiscard;

    LogFlowFunc(("pIoCtx=%#p\n", pIoCtx));

    /* Check if the I/O context processed all ranges. */
    if (   pIoCtx->Req.Discard.idxRange == cRanges
        && !pIoCtx->Req.Discard.cbDiscardLeft)
    {
        LogFlowFunc(("All ranges discarded, completing\n"));
        vdIoCtxUnlockDisk(pDisk, pIoCtx, true /* fProcessDeferredReqs*/);
        return VINF_SUCCESS;
    }

    if (pDisk->pIoCtxLockOwner != pIoCtx)
        rc = vdIoCtxLockDisk(pDisk, pIoCtx);

    if (RT_SUCCESS(rc))
    {
        uint64_t offStart      = pIoCtx->Req.Discard.offCur;
        size_t   cbDiscardLeft = pIoCtx->Req.Discard.cbDiscardLeft;
        size_t   cbThisDiscard;

        pDisk->uOffsetStartLocked = offStart;
        pDisk->uOffsetEndLocked = offStart + cbDiscardLeft;

        if (RT_UNLIKELY(!pDiscard))
        {
            pDiscard = vdDiscardStateCreate();
            if (!pDiscard)
                return VERR_NO_MEMORY;

            pDisk->pDiscard = pDiscard;
        }

        if (!pIoCtx->Req.Discard.cbDiscardLeft)
        {
            offStart      = paRanges[pIoCtx->Req.Discard.idxRange].offStart;
            cbDiscardLeft = paRanges[pIoCtx->Req.Discard.idxRange].cbRange;
            LogFlowFunc(("New range descriptor loaded (%u) offStart=%llu cbDiscard=%zu\n",
                         pIoCtx->Req.Discard.idxRange, offStart, cbDiscardLeft));
            pIoCtx->Req.Discard.idxRange++;
        }

        /* Look for a matching block in the AVL tree first. */
        PVDDISCARDBLOCK pBlock = (PVDDISCARDBLOCK)RTAvlrU64GetBestFit(pDiscard->pTreeBlocks, offStart, false);
        if (!pBlock || pBlock->Core.KeyLast < offStart)
        {
            PVDDISCARDBLOCK pBlockAbove = (PVDDISCARDBLOCK)RTAvlrU64GetBestFit(pDiscard->pTreeBlocks, offStart, true);

            /* Clip range to remain in the current block. */
            if (pBlockAbove)
                cbThisDiscard = RT_MIN(cbDiscardLeft, pBlockAbove->Core.KeyLast - offStart + 1);
            else
                cbThisDiscard = cbDiscardLeft;

            Assert(!(cbThisDiscard % 512));
            pIoCtx->Req.Discard.pBlock   = NULL;
            pIoCtx->pfnIoCtxTransferNext = vdDiscardCurrentRangeAsync;
        }
        else
        {
            /* Range lies partly in the block, update allocation bitmap. */
            int32_t idxStart, idxEnd;

            cbThisDiscard = RT_MIN(cbDiscardLeft, pBlock->Core.KeyLast - offStart + 1);

            AssertPtr(pBlock);

            Assert(!(cbThisDiscard % 512));
            Assert(!((offStart - pBlock->Core.Key) % 512));

            idxStart = (offStart - pBlock->Core.Key) / 512;
            idxEnd = idxStart + (int32_t)(cbThisDiscard / 512);

            ASMBitClearRange(pBlock->pbmAllocated, idxStart, idxEnd);

            cbDiscardLeft -= cbThisDiscard;
            offStart      += cbThisDiscard;

            /* Call the backend to discard the block if it is completely unallocated now. */
            if (ASMBitFirstSet((volatile void *)pBlock->pbmAllocated, (uint32_t)(pBlock->cbDiscard / 512)) == -1)
            {
                pIoCtx->Req.Discard.pBlock   = pBlock;
                pIoCtx->pfnIoCtxTransferNext = vdDiscardWholeBlockAsync;
                rc = VINF_SUCCESS;
            }
            else
            {
                RTListNodeRemove(&pBlock->NodeLru);
                RTListPrepend(&pDiscard->ListLru, &pBlock->NodeLru);

                /* Start with next range. */
                pIoCtx->pfnIoCtxTransferNext = vdDiscardHelperAsync;
                rc = VINF_SUCCESS;
            }
        }

        /* Save state in the context. */
        pIoCtx->Req.Discard.offCur        = offStart;
        pIoCtx->Req.Discard.cbDiscardLeft = cbDiscardLeft;
        pIoCtx->Req.Discard.cbThisDiscard = cbThisDiscard;
    }

    LogFlowFunc(("returns rc=%Rrc\n", rc));
    return rc;
}

/**
 * VD async I/O interface open callback.
 */
static DECLCALLBACK(int) vdIOOpenFallback(void *pvUser, const char *pszLocation,
                                          uint32_t fOpen, PFNVDCOMPLETED pfnCompleted,
                                          void **ppStorage)
{
    RT_NOREF1(pvUser);
    PVDIIOFALLBACKSTORAGE pStorage = (PVDIIOFALLBACKSTORAGE)RTMemAllocZ(sizeof(VDIIOFALLBACKSTORAGE));

    if (!pStorage)
        return VERR_NO_MEMORY;

    pStorage->pfnCompleted = pfnCompleted;

    /* Open the file. */
    int rc = RTFileOpen(&pStorage->File, pszLocation, fOpen);
    if (RT_SUCCESS(rc))
    {
        *ppStorage = pStorage;
        return VINF_SUCCESS;
    }

    RTMemFree(pStorage);
    return rc;
}

/**
 * VD async I/O interface close callback.
 */
static DECLCALLBACK(int) vdIOCloseFallback(void *pvUser, void *pvStorage)
{
    RT_NOREF1(pvUser);
    PVDIIOFALLBACKSTORAGE pStorage = (PVDIIOFALLBACKSTORAGE)pvStorage;

    RTFileClose(pStorage->File);
    RTMemFree(pStorage);
    return VINF_SUCCESS;
}

static DECLCALLBACK(int) vdIODeleteFallback(void *pvUser, const char *pcszFilename)
{
    RT_NOREF1(pvUser);
    return RTFileDelete(pcszFilename);
}

static DECLCALLBACK(int) vdIOMoveFallback(void *pvUser, const char *pcszSrc, const char *pcszDst, unsigned fMove)
{
    RT_NOREF1(pvUser);
    return RTFileMove(pcszSrc, pcszDst, fMove);
}

static DECLCALLBACK(int) vdIOGetFreeSpaceFallback(void *pvUser, const char *pcszFilename, int64_t *pcbFreeSpace)
{
    RT_NOREF1(pvUser);
    return RTFsQuerySizes(pcszFilename, NULL, pcbFreeSpace, NULL, NULL);
}

static DECLCALLBACK(int) vdIOGetModificationTimeFallback(void *pvUser, const char *pcszFilename, PRTTIMESPEC pModificationTime)
{
    RT_NOREF1(pvUser);
    RTFSOBJINFO info;
    int rc = RTPathQueryInfo(pcszFilename, &info, RTFSOBJATTRADD_NOTHING);
    if (RT_SUCCESS(rc))
        *pModificationTime = info.ModificationTime;
    return rc;
}

/**
 * VD async I/O interface callback for retrieving the file size.
 */
static DECLCALLBACK(int) vdIOGetSizeFallback(void *pvUser, void *pvStorage, uint64_t *pcbSize)
{
    RT_NOREF1(pvUser);
    PVDIIOFALLBACKSTORAGE pStorage = (PVDIIOFALLBACKSTORAGE)pvStorage;

    return RTFileQuerySize(pStorage->File, pcbSize);
}

/**
 * VD async I/O interface callback for setting the file size.
 */
static DECLCALLBACK(int) vdIOSetSizeFallback(void *pvUser, void *pvStorage, uint64_t cbSize)
{
    RT_NOREF1(pvUser);
    PVDIIOFALLBACKSTORAGE pStorage = (PVDIIOFALLBACKSTORAGE)pvStorage;

    return RTFileSetSize(pStorage->File, cbSize);
}

/**
 * VD async I/O interface callback for setting the file allocation size.
 */
static DECLCALLBACK(int) vdIOSetAllocationSizeFallback(void *pvUser, void *pvStorage, uint64_t cbSize,
                                                       uint32_t fFlags)
{
    RT_NOREF2(pvUser, fFlags);
    PVDIIOFALLBACKSTORAGE pStorage = (PVDIIOFALLBACKSTORAGE)pvStorage;

    return RTFileSetAllocationSize(pStorage->File, cbSize, RTFILE_ALLOC_SIZE_F_DEFAULT);
}

/**
 * VD async I/O interface callback for a synchronous write to the file.
 */
static DECLCALLBACK(int) vdIOWriteSyncFallback(void *pvUser, void *pvStorage, uint64_t uOffset,
                                               const void *pvBuf, size_t cbWrite, size_t *pcbWritten)
{
    RT_NOREF1(pvUser);
    PVDIIOFALLBACKSTORAGE pStorage = (PVDIIOFALLBACKSTORAGE)pvStorage;

    return RTFileWriteAt(pStorage->File, uOffset, pvBuf, cbWrite, pcbWritten);
}

/**
 * VD async I/O interface callback for a synchronous read from the file.
 */
static DECLCALLBACK(int) vdIOReadSyncFallback(void *pvUser, void *pvStorage, uint64_t uOffset,
                                              void *pvBuf, size_t cbRead, size_t *pcbRead)
{
    RT_NOREF1(pvUser);
    PVDIIOFALLBACKSTORAGE pStorage = (PVDIIOFALLBACKSTORAGE)pvStorage;

    return RTFileReadAt(pStorage->File, uOffset, pvBuf, cbRead, pcbRead);
}

/**
 * VD async I/O interface callback for a synchronous flush of the file data.
 */
static DECLCALLBACK(int) vdIOFlushSyncFallback(void *pvUser, void *pvStorage)
{
    RT_NOREF1(pvUser);
    PVDIIOFALLBACKSTORAGE pStorage = (PVDIIOFALLBACKSTORAGE)pvStorage;

    return RTFileFlush(pStorage->File);
}

/**
 * Internal - Continues an I/O context after
 * it was halted because of an active transfer.
 */
static int vdIoCtxContinue(PVDIOCTX pIoCtx, int rcReq)
{
    PVDISK pDisk = pIoCtx->pDisk;
    int rc = VINF_SUCCESS;

    VD_IS_LOCKED(pDisk);

    if (RT_FAILURE(rcReq))
        ASMAtomicCmpXchgS32(&pIoCtx->rcReq, rcReq, VINF_SUCCESS);

    if (!(pIoCtx->fFlags & VDIOCTX_FLAGS_BLOCKED))
    {
        /* Continue the transfer */
        rc = vdIoCtxProcessLocked(pIoCtx);

        if (   rc == VINF_VD_ASYNC_IO_FINISHED
            && ASMAtomicCmpXchgBool(&pIoCtx->fComplete, true, false))
        {
            LogFlowFunc(("I/O context completed pIoCtx=%#p\n", pIoCtx));
            bool fFreeCtx = RT_BOOL(!(pIoCtx->fFlags & VDIOCTX_FLAGS_DONT_FREE));
            if (pIoCtx->pIoCtxParent)
            {
                PVDIOCTX pIoCtxParent = pIoCtx->pIoCtxParent;

                Assert(!pIoCtxParent->pIoCtxParent);
                if (RT_FAILURE(pIoCtx->rcReq))
                    ASMAtomicCmpXchgS32(&pIoCtxParent->rcReq, pIoCtx->rcReq, VINF_SUCCESS);

                ASMAtomicDecU32(&pIoCtxParent->cDataTransfersPending);

                if (pIoCtx->enmTxDir == VDIOCTXTXDIR_WRITE)
                {
                    LogFlowFunc(("I/O context transferred %u bytes for the parent pIoCtxParent=%p\n",
                                 pIoCtx->Type.Child.cbTransferParent, pIoCtxParent));

                    /* Update the parent state. */
                    Assert(pIoCtxParent->Req.Io.cbTransferLeft >= pIoCtx->Type.Child.cbTransferParent);
                    ASMAtomicSubU32(&pIoCtxParent->Req.Io.cbTransferLeft, (uint32_t)pIoCtx->Type.Child.cbTransferParent);
                }
                else
                    Assert(pIoCtx->enmTxDir == VDIOCTXTXDIR_FLUSH);

                /*
                 * A completed child write means that we finished growing the image.
                 * We have to process any pending writes now.
                 */
                vdIoCtxUnlockDisk(pDisk, pIoCtxParent, false /* fProcessDeferredReqs */);

                /* Unblock the parent */
                pIoCtxParent->fFlags &= ~VDIOCTX_FLAGS_BLOCKED;

                rc = vdIoCtxProcessLocked(pIoCtxParent);

                if (   rc == VINF_VD_ASYNC_IO_FINISHED
                    && ASMAtomicCmpXchgBool(&pIoCtxParent->fComplete, true, false))
                {
                    LogFlowFunc(("Parent I/O context completed pIoCtxParent=%#p rcReq=%Rrc\n", pIoCtxParent, pIoCtxParent->rcReq));
                    bool fFreeParentCtx = RT_BOOL(!(pIoCtxParent->fFlags & VDIOCTX_FLAGS_DONT_FREE));
                    vdIoCtxRootComplete(pDisk, pIoCtxParent);
                    vdThreadFinishWrite(pDisk);

                    if (fFreeParentCtx)
                        vdIoCtxFree(pDisk, pIoCtxParent);
                    vdDiskProcessBlockedIoCtx(pDisk);
                }
                else if (!vdIoCtxIsDiskLockOwner(pDisk, pIoCtx))
                {
                    /* Process any pending writes if the current request didn't caused another growing. */
                    vdDiskProcessBlockedIoCtx(pDisk);
                }
            }
            else
            {
                if (pIoCtx->enmTxDir == VDIOCTXTXDIR_FLUSH)
                {
                    vdIoCtxUnlockDisk(pDisk, pIoCtx, true /* fProcessDerredReqs */);
                    vdThreadFinishWrite(pDisk);
                }
                else if (   pIoCtx->enmTxDir == VDIOCTXTXDIR_WRITE
                         || pIoCtx->enmTxDir == VDIOCTXTXDIR_DISCARD)
                    vdThreadFinishWrite(pDisk);
                else
                {
                    Assert(pIoCtx->enmTxDir == VDIOCTXTXDIR_READ);
                    vdThreadFinishRead(pDisk);
                }

                LogFlowFunc(("I/O context completed pIoCtx=%#p rcReq=%Rrc\n", pIoCtx, pIoCtx->rcReq));
                vdIoCtxRootComplete(pDisk, pIoCtx);
            }

            if (fFreeCtx)
                vdIoCtxFree(pDisk, pIoCtx);
        }
    }

    return VINF_SUCCESS;
}

/**
 * Internal - Called when user transfer completed.
 */
static int vdUserXferCompleted(PVDIOSTORAGE pIoStorage, PVDIOCTX pIoCtx,
                               PFNVDXFERCOMPLETED pfnComplete, void *pvUser,
                               size_t cbTransfer, int rcReq)
{
    int rc = VINF_SUCCESS;
    PVDISK pDisk = pIoCtx->pDisk;

    LogFlowFunc(("pIoStorage=%#p pIoCtx=%#p pfnComplete=%#p pvUser=%#p cbTransfer=%zu rcReq=%Rrc\n",
                 pIoStorage, pIoCtx, pfnComplete, pvUser, cbTransfer, rcReq));

    VD_IS_LOCKED(pDisk);

    Assert(pIoCtx->Req.Io.cbTransferLeft >= cbTransfer);
    ASMAtomicSubU32(&pIoCtx->Req.Io.cbTransferLeft, (uint32_t)cbTransfer); Assert(cbTransfer == (uint32_t)cbTransfer);
    ASMAtomicDecU32(&pIoCtx->cDataTransfersPending);

    if (pfnComplete)
        rc = pfnComplete(pIoStorage->pVDIo->pBackendData, pIoCtx, pvUser, rcReq);

    if (RT_SUCCESS(rc))
        rc = vdIoCtxContinue(pIoCtx, rcReq);
    else if (rc == VERR_VD_ASYNC_IO_IN_PROGRESS)
        rc = VINF_SUCCESS;

    return rc;
}

static void vdIoCtxContinueDeferredList(PVDIOSTORAGE pIoStorage, PRTLISTANCHOR pListWaiting,
                                        PFNVDXFERCOMPLETED pfnComplete, void *pvUser, int rcReq)
{
    LogFlowFunc(("pIoStorage=%#p pListWaiting=%#p pfnComplete=%#p pvUser=%#p rcReq=%Rrc\n",
                 pIoStorage, pListWaiting, pfnComplete, pvUser, rcReq));

    /* Go through the waiting list and continue the I/O contexts. */
    while (!RTListIsEmpty(pListWaiting))
    {
        int rc = VINF_SUCCESS;
        PVDIOCTXDEFERRED pDeferred = RTListGetFirst(pListWaiting, VDIOCTXDEFERRED, NodeDeferred);
        PVDIOCTX pIoCtx = pDeferred->pIoCtx;
        RTListNodeRemove(&pDeferred->NodeDeferred);

        RTMemFree(pDeferred);
        ASMAtomicDecU32(&pIoCtx->cMetaTransfersPending);

        if (pfnComplete)
            rc = pfnComplete(pIoStorage->pVDIo->pBackendData, pIoCtx, pvUser, rcReq);

        LogFlow(("Completion callback for I/O context %#p returned %Rrc\n", pIoCtx, rc));

        if (RT_SUCCESS(rc))
        {
            rc = vdIoCtxContinue(pIoCtx, rcReq);
            AssertRC(rc);
        }
        else
            Assert(rc == VERR_VD_ASYNC_IO_IN_PROGRESS);
    }
}

/**
 * Internal - Called when a meta transfer completed.
 */
static int vdMetaXferCompleted(PVDIOSTORAGE pIoStorage, PFNVDXFERCOMPLETED pfnComplete, void *pvUser,
                               PVDMETAXFER pMetaXfer, int rcReq)
{
    PVDISK pDisk = pIoStorage->pVDIo->pDisk;
    RTLISTANCHOR ListIoCtxWaiting;
    bool fFlush;

    LogFlowFunc(("pIoStorage=%#p pfnComplete=%#p pvUser=%#p pMetaXfer=%#p rcReq=%Rrc\n",
                 pIoStorage, pfnComplete, pvUser, pMetaXfer, rcReq));

    VD_IS_LOCKED(pDisk);

    fFlush = VDMETAXFER_TXDIR_GET(pMetaXfer->fFlags) == VDMETAXFER_TXDIR_FLUSH;

    if (!fFlush)
    {
        RTListMove(&ListIoCtxWaiting, &pMetaXfer->ListIoCtxWaiting);

        if (RT_FAILURE(rcReq))
        {
            /* Remove from the AVL tree. */
            LogFlow(("Removing meta xfer=%#p\n", pMetaXfer));
            bool fRemoved = RTAvlrFileOffsetRemove(pIoStorage->pTreeMetaXfers, pMetaXfer->Core.Key) != NULL;
            Assert(fRemoved); NOREF(fRemoved);
            /* If this was a write check if there is a shadow buffer with updated data. */
            if (pMetaXfer->pbDataShw)
            {
                Assert(VDMETAXFER_TXDIR_GET(pMetaXfer->fFlags) == VDMETAXFER_TXDIR_WRITE);
                Assert(!RTListIsEmpty(&pMetaXfer->ListIoCtxShwWrites));
                RTListConcatenate(&ListIoCtxWaiting, &pMetaXfer->ListIoCtxShwWrites);
                RTMemFree(pMetaXfer->pbDataShw);
                pMetaXfer->pbDataShw = NULL;
            }
            RTMemFree(pMetaXfer);
        }
        else
        {
            /* Increase the reference counter to make sure it doesn't go away before the last context is processed. */
            pMetaXfer->cRefs++;
        }
    }
    else
        RTListMove(&ListIoCtxWaiting, &pMetaXfer->ListIoCtxWaiting);

    VDMETAXFER_TXDIR_SET(pMetaXfer->fFlags, VDMETAXFER_TXDIR_NONE);
    vdIoCtxContinueDeferredList(pIoStorage, &ListIoCtxWaiting, pfnComplete, pvUser, rcReq);

    /*
     * If there is a shadow buffer and the previous write was successful update with the
     * new data and trigger a new write.
     */
    if (   pMetaXfer->pbDataShw
        && RT_SUCCESS(rcReq)
        && VDMETAXFER_TXDIR_GET(pMetaXfer->fFlags) == VDMETAXFER_TXDIR_NONE)
    {
        LogFlowFunc(("pMetaXfer=%#p Updating from shadow buffer and triggering new write\n", pMetaXfer));
        memcpy(pMetaXfer->abData, pMetaXfer->pbDataShw, pMetaXfer->cbMeta);
        RTMemFree(pMetaXfer->pbDataShw);
        pMetaXfer->pbDataShw = NULL;
        Assert(!RTListIsEmpty(&pMetaXfer->ListIoCtxShwWrites));

        /* Setup a new I/O write. */
        PVDIOTASK pIoTask = vdIoTaskMetaAlloc(pIoStorage, pfnComplete, pvUser, pMetaXfer);
        if (RT_LIKELY(pIoTask))
        {
            void *pvTask = NULL;
            RTSGSEG Seg;

            Seg.cbSeg = pMetaXfer->cbMeta;
            Seg.pvSeg = pMetaXfer->abData;

            VDMETAXFER_TXDIR_SET(pMetaXfer->fFlags, VDMETAXFER_TXDIR_WRITE);
            rcReq = pIoStorage->pVDIo->pInterfaceIo->pfnWriteAsync(pIoStorage->pVDIo->pInterfaceIo->Core.pvUser,
                                                                   pIoStorage->pStorage,
                                                                   pMetaXfer->Core.Key, &Seg, 1,
                                                                   pMetaXfer->cbMeta, pIoTask,
                                                                   &pvTask);
            if (   RT_SUCCESS(rcReq)
                || rcReq != VERR_VD_ASYNC_IO_IN_PROGRESS)
            {
                VDMETAXFER_TXDIR_SET(pMetaXfer->fFlags, VDMETAXFER_TXDIR_NONE);
                vdIoTaskFree(pDisk, pIoTask);
            }
            else
                RTListMove(&pMetaXfer->ListIoCtxWaiting, &pMetaXfer->ListIoCtxShwWrites);
        }
        else
            rcReq = VERR_NO_MEMORY;

        /* Cleanup if there was an error or the request completed already. */
        if (rcReq != VERR_VD_ASYNC_IO_IN_PROGRESS)
            vdIoCtxContinueDeferredList(pIoStorage, &pMetaXfer->ListIoCtxShwWrites, pfnComplete, pvUser, rcReq);
    }

    /* Remove if not used anymore. */
    if (!fFlush)
    {
        pMetaXfer->cRefs--;
        if (!pMetaXfer->cRefs && RTListIsEmpty(&pMetaXfer->ListIoCtxWaiting))
        {
            /* Remove from the AVL tree. */
            LogFlow(("Removing meta xfer=%#p\n", pMetaXfer));
            bool fRemoved = RTAvlrFileOffsetRemove(pIoStorage->pTreeMetaXfers, pMetaXfer->Core.Key) != NULL;
            Assert(fRemoved); NOREF(fRemoved);
            RTMemFree(pMetaXfer);
        }
    }
    else if (fFlush)
        RTMemFree(pMetaXfer);

    return VINF_SUCCESS;
}

/**
 * Processes a list of waiting I/O tasks. The disk lock must be held by caller.
 *
 * @param   pDisk    The disk to process the list for.
 */
static void vdIoTaskProcessWaitingList(PVDISK pDisk)
{
    LogFlowFunc(("pDisk=%#p\n", pDisk));

    VD_IS_LOCKED(pDisk);

    PVDIOTASK pHead = ASMAtomicXchgPtrT(&pDisk->pIoTasksPendingHead, NULL, PVDIOTASK);

    Log(("I/O task list cleared\n"));

    /* Reverse order. */
    PVDIOTASK pCur = pHead;
    pHead = NULL;
    while (pCur)
    {
        PVDIOTASK pInsert = pCur;
        pCur = pCur->pNext;
        pInsert->pNext = pHead;
        pHead = pInsert;
    }

    while (pHead)
    {
        PVDIOSTORAGE pIoStorage = pHead->pIoStorage;

        if (!pHead->fMeta)
            vdUserXferCompleted(pIoStorage, pHead->Type.User.pIoCtx,
                                pHead->pfnComplete, pHead->pvUser,
                                pHead->Type.User.cbTransfer, pHead->rcReq);
        else
            vdMetaXferCompleted(pIoStorage, pHead->pfnComplete, pHead->pvUser,
                                pHead->Type.Meta.pMetaXfer, pHead->rcReq);

        pCur = pHead;
        pHead = pHead->pNext;
        vdIoTaskFree(pDisk, pCur);
    }
}

/**
 * Process any I/O context on the halted list.
 *
 * @param   pDisk    The disk.
 */
static void vdIoCtxProcessHaltedList(PVDISK pDisk)
{
    LogFlowFunc(("pDisk=%#p\n", pDisk));

    VD_IS_LOCKED(pDisk);

    /* Get the waiting list and process it in FIFO order. */
    PVDIOCTX pIoCtxHead = ASMAtomicXchgPtrT(&pDisk->pIoCtxHaltedHead, NULL, PVDIOCTX);

    /* Reverse it. */
    PVDIOCTX pCur = pIoCtxHead;
    pIoCtxHead = NULL;
    while (pCur)
    {
        PVDIOCTX pInsert = pCur;
        pCur = pCur->pIoCtxNext;
        pInsert->pIoCtxNext = pIoCtxHead;
        pIoCtxHead = pInsert;
    }

    /* Process now. */
    pCur = pIoCtxHead;
    while (pCur)
    {
        PVDIOCTX pTmp = pCur;

        pCur = pCur->pIoCtxNext;
        pTmp->pIoCtxNext = NULL;

        /* Continue */
        pTmp->fFlags &= ~VDIOCTX_FLAGS_BLOCKED;
        vdIoCtxContinue(pTmp, pTmp->rcReq);
    }
}

/**
 * Unlock the disk and process pending tasks.
 *
 * @returns VBox status code.
 * @param   pDisk    The disk to unlock.
 * @param   pIoCtxRc The I/O context to get the status code from, optional.
 */
static int vdDiskUnlock(PVDISK pDisk, PVDIOCTX pIoCtxRc)
{
    int rc = VINF_SUCCESS;

    VD_IS_LOCKED(pDisk);

    /*
     * Process the list of waiting I/O tasks first
     * because they might complete I/O contexts.
     * Same for the list of halted I/O contexts.
     * Afterwards comes the list of new I/O contexts.
     */
    vdIoTaskProcessWaitingList(pDisk);
    vdIoCtxProcessHaltedList(pDisk);
    rc = vdDiskProcessWaitingIoCtx(pDisk, pIoCtxRc);
    ASMAtomicXchgBool(&pDisk->fLocked, false);

    /*
     * Need to check for new I/O tasks and waiting I/O contexts now
     * again as other threads might added them while we processed
     * previous lists.
     */
    while (   ASMAtomicUoReadPtrT(&pDisk->pIoCtxHead, PVDIOCTX) != NULL
           || ASMAtomicUoReadPtrT(&pDisk->pIoTasksPendingHead, PVDIOTASK) != NULL
           || ASMAtomicUoReadPtrT(&pDisk->pIoCtxHaltedHead, PVDIOCTX) != NULL)
    {
        /* Try lock disk again. */
        if (ASMAtomicCmpXchgBool(&pDisk->fLocked, true, false))
        {
            vdIoTaskProcessWaitingList(pDisk);
            vdIoCtxProcessHaltedList(pDisk);
            vdDiskProcessWaitingIoCtx(pDisk, NULL);
            ASMAtomicXchgBool(&pDisk->fLocked, false);
        }
        else /* Let the other thread everything when he unlocks the disk. */
            break;
    }

    return rc;
}

/**
 * Try to lock the disk to complete pressing of the I/O task.
 * The completion is deferred if the disk is locked already.
 *
 * @param   pIoTask  The I/O task to complete.
 */
static void vdXferTryLockDiskDeferIoTask(PVDIOTASK pIoTask)
{
    PVDIOSTORAGE pIoStorage = pIoTask->pIoStorage;
    PVDISK pDisk = pIoStorage->pVDIo->pDisk;

    Log(("Deferring I/O task pIoTask=%p\n", pIoTask));

    /* Put it on the waiting list. */
    PVDIOTASK pNext = ASMAtomicUoReadPtrT(&pDisk->pIoTasksPendingHead, PVDIOTASK);
    PVDIOTASK pHeadOld;
    pIoTask->pNext = pNext;
    while (!ASMAtomicCmpXchgExPtr(&pDisk->pIoTasksPendingHead, pIoTask, pNext, &pHeadOld))
    {
        pNext = pHeadOld;
        Assert(pNext != pIoTask);
        pIoTask->pNext = pNext;
        ASMNopPause();
    }

    if (ASMAtomicCmpXchgBool(&pDisk->fLocked, true, false))
    {
        /* Release disk lock, it will take care of processing all lists. */
        vdDiskUnlock(pDisk, NULL);
    }
}

static DECLCALLBACK(int) vdIOIntReqCompleted(void *pvUser, int rcReq)
{
    PVDIOTASK pIoTask = (PVDIOTASK)pvUser;

    LogFlowFunc(("Task completed pIoTask=%#p\n", pIoTask));

    pIoTask->rcReq = rcReq;
    vdXferTryLockDiskDeferIoTask(pIoTask);
    return VINF_SUCCESS;
}

/**
 * VD I/O interface callback for opening a file.
 */
static DECLCALLBACK(int) vdIOIntOpen(void *pvUser, const char *pszLocation,
                                     unsigned uOpenFlags, PPVDIOSTORAGE ppIoStorage)
{
    int rc = VINF_SUCCESS;
    PVDIO pVDIo             = (PVDIO)pvUser;
    PVDIOSTORAGE pIoStorage = (PVDIOSTORAGE)RTMemAllocZ(sizeof(VDIOSTORAGE));

    if (!pIoStorage)
        return VERR_NO_MEMORY;

    /* Create the AVl tree. */
    pIoStorage->pTreeMetaXfers = (PAVLRFOFFTREE)RTMemAllocZ(sizeof(AVLRFOFFTREE));
    if (pIoStorage->pTreeMetaXfers)
    {
        rc = pVDIo->pInterfaceIo->pfnOpen(pVDIo->pInterfaceIo->Core.pvUser,
                                          pszLocation, uOpenFlags,
                                          vdIOIntReqCompleted,
                                          &pIoStorage->pStorage);
        if (RT_SUCCESS(rc))
        {
            pIoStorage->pVDIo = pVDIo;
            *ppIoStorage = pIoStorage;
            return VINF_SUCCESS;
        }

        RTMemFree(pIoStorage->pTreeMetaXfers);
    }
    else
        rc = VERR_NO_MEMORY;

    RTMemFree(pIoStorage);
    return rc;
}

static DECLCALLBACK(int) vdIOIntTreeMetaXferDestroy(PAVLRFOFFNODECORE pNode, void *pvUser)
{
    RT_NOREF2(pNode, pvUser);
    AssertMsgFailed(("Tree should be empty at this point!\n"));
    return VINF_SUCCESS;
}

static DECLCALLBACK(int) vdIOIntClose(void *pvUser, PVDIOSTORAGE pIoStorage)
{
    int rc = VINF_SUCCESS;
    PVDIO pVDIo = (PVDIO)pvUser;

    /* We free everything here, even if closing the file failed for some reason. */
    rc = pVDIo->pInterfaceIo->pfnClose(pVDIo->pInterfaceIo->Core.pvUser, pIoStorage->pStorage);
    RTAvlrFileOffsetDestroy(pIoStorage->pTreeMetaXfers, vdIOIntTreeMetaXferDestroy, NULL);
    RTMemFree(pIoStorage->pTreeMetaXfers);
    RTMemFree(pIoStorage);
    return rc;
}

static DECLCALLBACK(int) vdIOIntDelete(void *pvUser, const char *pcszFilename)
{
    PVDIO pVDIo = (PVDIO)pvUser;
    return pVDIo->pInterfaceIo->pfnDelete(pVDIo->pInterfaceIo->Core.pvUser,
                                          pcszFilename);
}

static DECLCALLBACK(int) vdIOIntMove(void *pvUser, const char *pcszSrc, const char *pcszDst,
                                     unsigned fMove)
{
    PVDIO pVDIo = (PVDIO)pvUser;
    return pVDIo->pInterfaceIo->pfnMove(pVDIo->pInterfaceIo->Core.pvUser,
                                        pcszSrc, pcszDst, fMove);
}

static DECLCALLBACK(int) vdIOIntGetFreeSpace(void *pvUser, const char *pcszFilename,
                                             int64_t *pcbFreeSpace)
{
    PVDIO pVDIo = (PVDIO)pvUser;
    return pVDIo->pInterfaceIo->pfnGetFreeSpace(pVDIo->pInterfaceIo->Core.pvUser,
                                                pcszFilename, pcbFreeSpace);
}

static DECLCALLBACK(int) vdIOIntGetModificationTime(void *pvUser, const char *pcszFilename,
                                                    PRTTIMESPEC pModificationTime)
{
    PVDIO pVDIo = (PVDIO)pvUser;
    return pVDIo->pInterfaceIo->pfnGetModificationTime(pVDIo->pInterfaceIo->Core.pvUser,
                                                       pcszFilename, pModificationTime);
}

static DECLCALLBACK(int) vdIOIntGetSize(void *pvUser, PVDIOSTORAGE pIoStorage,
                                        uint64_t *pcbSize)
{
    PVDIO pVDIo = (PVDIO)pvUser;
    return pVDIo->pInterfaceIo->pfnGetSize(pVDIo->pInterfaceIo->Core.pvUser,
                                           pIoStorage->pStorage, pcbSize);
}

static DECLCALLBACK(int) vdIOIntSetSize(void *pvUser, PVDIOSTORAGE pIoStorage,
                                        uint64_t cbSize)
{
    PVDIO pVDIo = (PVDIO)pvUser;
    return pVDIo->pInterfaceIo->pfnSetSize(pVDIo->pInterfaceIo->Core.pvUser,
                                           pIoStorage->pStorage, cbSize);
}

static DECLCALLBACK(int) vdIOIntSetAllocationSize(void *pvUser, PVDIOSTORAGE pIoStorage,
                                                  uint64_t cbSize, uint32_t fFlags,
                                                  PVDINTERFACEPROGRESS pIfProgress,
                                                  unsigned uPercentStart, unsigned uPercentSpan)
{
    PVDIO pVDIo = (PVDIO)pvUser;
    int rc = pVDIo->pInterfaceIo->pfnSetAllocationSize(pVDIo->pInterfaceIo->Core.pvUser,
                                                       pIoStorage->pStorage, cbSize, fFlags);
    if (rc == VERR_NOT_SUPPORTED)
    {
        /* Fallback if the underlying medium does not support optimized storage allocation. */
        uint64_t cbSizeCur = 0;
        rc = pVDIo->pInterfaceIo->pfnGetSize(pVDIo->pInterfaceIo->Core.pvUser,
                                             pIoStorage->pStorage, &cbSizeCur);
        if (RT_SUCCESS(rc))
        {
            if (cbSizeCur < cbSize)
            {
                const size_t cbBuf = 128 * _1K;
                void *pvBuf = RTMemTmpAllocZ(cbBuf);
                if (RT_LIKELY(pvBuf))
                {
                    uint64_t cbFill = cbSize - cbSizeCur;
                    uint64_t uOff = 0;

                    /* Write data to all blocks. */
                    while (   uOff < cbFill
                           && RT_SUCCESS(rc))
                    {
                        size_t cbChunk = (size_t)RT_MIN(cbFill - uOff, cbBuf);

                        rc = pVDIo->pInterfaceIo->pfnWriteSync(pVDIo->pInterfaceIo->Core.pvUser,
                                                               pIoStorage->pStorage, cbSizeCur + uOff,
                                                               pvBuf, cbChunk, NULL);
                        if (RT_SUCCESS(rc))
                        {
                            uOff += cbChunk;

                            rc = vdIfProgress(pIfProgress, uPercentStart + uOff * uPercentSpan / cbFill);
                        }
                    }

                    RTMemTmpFree(pvBuf);
                }
                else
                    rc = VERR_NO_MEMORY;
            }
            else if (cbSizeCur > cbSize)
                rc = pVDIo->pInterfaceIo->pfnSetSize(pVDIo->pInterfaceIo->Core.pvUser,
                                                     pIoStorage->pStorage, cbSize);
        }
    }

    if (RT_SUCCESS(rc))
        rc = vdIfProgress(pIfProgress, uPercentStart + uPercentSpan);

    return rc;
}

static DECLCALLBACK(int) vdIOIntReadUser(void *pvUser, PVDIOSTORAGE pIoStorage, uint64_t uOffset,
                                         PVDIOCTX pIoCtx, size_t cbRead)
{
    int rc = VINF_SUCCESS;
    PVDIO    pVDIo = (PVDIO)pvUser;
    PVDISK pDisk = pVDIo->pDisk;

    LogFlowFunc(("pvUser=%#p pIoStorage=%#p uOffset=%llu pIoCtx=%#p cbRead=%u\n",
                 pvUser, pIoStorage, uOffset, pIoCtx, cbRead));

    /** @todo Enable check for sync I/O later. */
    if (!(pIoCtx->fFlags & VDIOCTX_FLAGS_SYNC))
        VD_IS_LOCKED(pDisk);

    Assert(cbRead > 0);

    if (   (pIoCtx->fFlags & VDIOCTX_FLAGS_SYNC)
        || !pVDIo->pInterfaceIo->pfnReadAsync)
    {
        RTSGSEG Seg;
        unsigned cSegments = 1;
        size_t cbTaskRead = 0;

        /* Synchronous I/O contexts only have one buffer segment. */
        AssertMsgReturn(pIoCtx->Req.Io.SgBuf.cSegs == 1,
                        ("Invalid number of buffer segments for synchronous I/O context"),
                        VERR_INVALID_PARAMETER);

        cbTaskRead = RTSgBufSegArrayCreate(&pIoCtx->Req.Io.SgBuf, &Seg, &cSegments, cbRead);
        Assert(cbRead == cbTaskRead);
        Assert(cSegments == 1);
        rc = pVDIo->pInterfaceIo->pfnReadSync(pVDIo->pInterfaceIo->Core.pvUser,
                                              pIoStorage->pStorage, uOffset,
                                              Seg.pvSeg, cbRead, NULL);
        if (RT_SUCCESS(rc))
        {
            Assert(cbRead == (uint32_t)cbRead);
            ASMAtomicSubU32(&pIoCtx->Req.Io.cbTransferLeft, (uint32_t)cbRead);
        }
    }
    else
    {
        /* Build the S/G array and spawn a new I/O task */
        while (cbRead)
        {
            RTSGSEG  aSeg[VD_IO_TASK_SEGMENTS_MAX];
            unsigned cSegments  = VD_IO_TASK_SEGMENTS_MAX;
            size_t   cbTaskRead = RTSgBufSegArrayCreate(&pIoCtx->Req.Io.SgBuf, aSeg, &cSegments, cbRead);

            Assert(cSegments > 0);
            Assert(cbTaskRead > 0);
            AssertMsg(cbTaskRead <= cbRead, ("Invalid number of bytes to read\n"));

            LogFlow(("Reading %u bytes into %u segments\n", cbTaskRead, cSegments));

#ifdef RT_STRICT
            for (unsigned i = 0; i < cSegments; i++)
                    AssertMsg(aSeg[i].pvSeg && !(aSeg[i].cbSeg % 512),
                              ("Segment %u is invalid\n", i));
#endif

            Assert(cbTaskRead == (uint32_t)cbTaskRead);
            PVDIOTASK pIoTask = vdIoTaskUserAlloc(pIoStorage, NULL, NULL, pIoCtx, (uint32_t)cbTaskRead);

            if (!pIoTask)
                return VERR_NO_MEMORY;

            ASMAtomicIncU32(&pIoCtx->cDataTransfersPending);

            void *pvTask;
            Log(("Spawning pIoTask=%p pIoCtx=%p\n", pIoTask, pIoCtx));
            rc = pVDIo->pInterfaceIo->pfnReadAsync(pVDIo->pInterfaceIo->Core.pvUser,
                                                   pIoStorage->pStorage, uOffset,
                                                   aSeg, cSegments, cbTaskRead, pIoTask,
                                                   &pvTask);
            if (RT_SUCCESS(rc))
            {
                AssertMsg(cbTaskRead <= pIoCtx->Req.Io.cbTransferLeft, ("Impossible!\n"));
                ASMAtomicSubU32(&pIoCtx->Req.Io.cbTransferLeft, (uint32_t)cbTaskRead);
                ASMAtomicDecU32(&pIoCtx->cDataTransfersPending);
                vdIoTaskFree(pDisk, pIoTask);
            }
            else if (rc != VERR_VD_ASYNC_IO_IN_PROGRESS)
            {
                ASMAtomicDecU32(&pIoCtx->cDataTransfersPending);
                vdIoTaskFree(pDisk, pIoTask);
                break;
            }

            uOffset += cbTaskRead;
            cbRead  -= cbTaskRead;
        }
    }

    LogFlowFunc(("returns rc=%Rrc\n", rc));
    return rc;
}

static DECLCALLBACK(int) vdIOIntWriteUser(void *pvUser, PVDIOSTORAGE pIoStorage, uint64_t uOffset,
                                          PVDIOCTX pIoCtx, size_t cbWrite, PFNVDXFERCOMPLETED pfnComplete,
                                          void *pvCompleteUser)
{
    int rc = VINF_SUCCESS;
    PVDIO    pVDIo = (PVDIO)pvUser;
    PVDISK pDisk = pVDIo->pDisk;

    LogFlowFunc(("pvUser=%#p pIoStorage=%#p uOffset=%llu pIoCtx=%#p cbWrite=%u\n",
                 pvUser, pIoStorage, uOffset, pIoCtx, cbWrite));

    /** @todo Enable check for sync I/O later. */
    if (!(pIoCtx->fFlags & VDIOCTX_FLAGS_SYNC))
        VD_IS_LOCKED(pDisk);

    Assert(cbWrite > 0);

    if (   (pIoCtx->fFlags & VDIOCTX_FLAGS_SYNC)
        || !pVDIo->pInterfaceIo->pfnWriteAsync)
    {
        RTSGSEG Seg;
        unsigned cSegments = 1;
        size_t cbTaskWrite = 0;

        /* Synchronous I/O contexts only have one buffer segment. */
        AssertMsgReturn(pIoCtx->Req.Io.SgBuf.cSegs == 1,
                        ("Invalid number of buffer segments for synchronous I/O context"),
                        VERR_INVALID_PARAMETER);

        cbTaskWrite = RTSgBufSegArrayCreate(&pIoCtx->Req.Io.SgBuf, &Seg, &cSegments, cbWrite);
        Assert(cbWrite == cbTaskWrite);
        Assert(cSegments == 1);
        rc = pVDIo->pInterfaceIo->pfnWriteSync(pVDIo->pInterfaceIo->Core.pvUser,
                                              pIoStorage->pStorage, uOffset,
                                              Seg.pvSeg, cbWrite, NULL);
        if (RT_SUCCESS(rc))
        {
            Assert(pIoCtx->Req.Io.cbTransferLeft >= cbWrite);
            ASMAtomicSubU32(&pIoCtx->Req.Io.cbTransferLeft, (uint32_t)cbWrite);
        }
    }
    else
    {
        /* Build the S/G array and spawn a new I/O task */
        while (cbWrite)
        {
            RTSGSEG  aSeg[VD_IO_TASK_SEGMENTS_MAX];
            unsigned cSegments   = VD_IO_TASK_SEGMENTS_MAX;
            size_t   cbTaskWrite = 0;

            cbTaskWrite = RTSgBufSegArrayCreate(&pIoCtx->Req.Io.SgBuf, aSeg, &cSegments, cbWrite);

            Assert(cSegments > 0);
            Assert(cbTaskWrite > 0);
            AssertMsg(cbTaskWrite <= cbWrite, ("Invalid number of bytes to write\n"));

            LogFlow(("Writing %u bytes from %u segments\n", cbTaskWrite, cSegments));

#ifdef DEBUG
            for (unsigned i = 0; i < cSegments; i++)
                    AssertMsg(aSeg[i].pvSeg && !(aSeg[i].cbSeg % 512),
                              ("Segment %u is invalid\n", i));
#endif

            Assert(cbTaskWrite == (uint32_t)cbTaskWrite);
            PVDIOTASK pIoTask = vdIoTaskUserAlloc(pIoStorage, pfnComplete, pvCompleteUser, pIoCtx, (uint32_t)cbTaskWrite);

            if (!pIoTask)
                return VERR_NO_MEMORY;

            ASMAtomicIncU32(&pIoCtx->cDataTransfersPending);

            void *pvTask;
            Log(("Spawning pIoTask=%p pIoCtx=%p\n", pIoTask, pIoCtx));
            rc = pVDIo->pInterfaceIo->pfnWriteAsync(pVDIo->pInterfaceIo->Core.pvUser,
                                                    pIoStorage->pStorage,
                                                    uOffset, aSeg, cSegments,
                                                    cbTaskWrite, pIoTask, &pvTask);
            if (RT_SUCCESS(rc))
            {
                AssertMsg(cbTaskWrite <= pIoCtx->Req.Io.cbTransferLeft, ("Impossible!\n"));
                ASMAtomicSubU32(&pIoCtx->Req.Io.cbTransferLeft, (uint32_t)cbTaskWrite);
                ASMAtomicDecU32(&pIoCtx->cDataTransfersPending);
                vdIoTaskFree(pDisk, pIoTask);
            }
            else if (rc != VERR_VD_ASYNC_IO_IN_PROGRESS)
            {
                ASMAtomicDecU32(&pIoCtx->cDataTransfersPending);
                vdIoTaskFree(pDisk, pIoTask);
                break;
            }

            uOffset += cbTaskWrite;
            cbWrite -= cbTaskWrite;
        }
    }

    LogFlowFunc(("returns rc=%Rrc\n", rc));
    return rc;
}

static DECLCALLBACK(int) vdIOIntReadMeta(void *pvUser, PVDIOSTORAGE pIoStorage, uint64_t uOffset,
                                         void *pvBuf, size_t cbRead, PVDIOCTX pIoCtx,
                                         PPVDMETAXFER ppMetaXfer, PFNVDXFERCOMPLETED pfnComplete,
                                         void *pvCompleteUser)
{
    PVDIO pVDIo     = (PVDIO)pvUser;
    PVDISK pDisk  = pVDIo->pDisk;
    int rc = VINF_SUCCESS;
    RTSGSEG Seg;
    PVDIOTASK pIoTask;
    PVDMETAXFER pMetaXfer = NULL;
    void *pvTask = NULL;

    LogFlowFunc(("pvUser=%#p pIoStorage=%#p uOffset=%llu pvBuf=%#p cbRead=%u\n",
                 pvUser, pIoStorage, uOffset, pvBuf, cbRead));

    AssertMsgReturn(   pIoCtx
                    || (!ppMetaXfer && !pfnComplete && !pvCompleteUser),
                    ("A synchronous metadata read is requested but the parameters are wrong\n"),
                    VERR_INVALID_POINTER);

    /** @todo Enable check for sync I/O later. */
    if (   pIoCtx
        && !(pIoCtx->fFlags & VDIOCTX_FLAGS_SYNC))
        VD_IS_LOCKED(pDisk);

    if (   !pIoCtx
        || pIoCtx->fFlags & VDIOCTX_FLAGS_SYNC
        || !pVDIo->pInterfaceIo->pfnReadAsync)
    {
        /* Handle synchronous metadata I/O. */
        /** @todo Integrate with metadata transfers below. */
        rc = pVDIo->pInterfaceIo->pfnReadSync(pVDIo->pInterfaceIo->Core.pvUser,
                                               pIoStorage->pStorage, uOffset,
                                               pvBuf, cbRead, NULL);
        if (ppMetaXfer)
            *ppMetaXfer = NULL;
    }
    else
    {
        pMetaXfer = (PVDMETAXFER)RTAvlrFileOffsetGet(pIoStorage->pTreeMetaXfers, uOffset);
        if (!pMetaXfer)
        {
#ifdef RT_STRICT
            pMetaXfer = (PVDMETAXFER)RTAvlrFileOffsetGetBestFit(pIoStorage->pTreeMetaXfers, uOffset, false /* fAbove */);
            AssertMsg(!pMetaXfer || (pMetaXfer->Core.Key + (RTFOFF)pMetaXfer->cbMeta <= (RTFOFF)uOffset),
                      ("Overlapping meta transfers!\n"));
#endif

            /* Allocate a new meta transfer. */
            pMetaXfer = vdMetaXferAlloc(pIoStorage, uOffset, cbRead);
            if (!pMetaXfer)
                return VERR_NO_MEMORY;

            pIoTask = vdIoTaskMetaAlloc(pIoStorage, pfnComplete, pvCompleteUser, pMetaXfer);
            if (!pIoTask)
            {
                RTMemFree(pMetaXfer);
                return VERR_NO_MEMORY;
            }

            Seg.cbSeg = cbRead;
            Seg.pvSeg = pMetaXfer->abData;

            VDMETAXFER_TXDIR_SET(pMetaXfer->fFlags, VDMETAXFER_TXDIR_READ);
            rc = pVDIo->pInterfaceIo->pfnReadAsync(pVDIo->pInterfaceIo->Core.pvUser,
                                                   pIoStorage->pStorage,
                                                   uOffset, &Seg, 1,
                                                   cbRead, pIoTask, &pvTask);

            if (RT_SUCCESS(rc) || rc == VERR_VD_ASYNC_IO_IN_PROGRESS)
            {
                bool fInserted = RTAvlrFileOffsetInsert(pIoStorage->pTreeMetaXfers, &pMetaXfer->Core);
                Assert(fInserted); NOREF(fInserted);
            }
            else
                RTMemFree(pMetaXfer);

            if (RT_SUCCESS(rc))
            {
                VDMETAXFER_TXDIR_SET(pMetaXfer->fFlags, VDMETAXFER_TXDIR_NONE);
                vdIoTaskFree(pDisk, pIoTask);
            }
            else if (rc == VERR_VD_ASYNC_IO_IN_PROGRESS && !pfnComplete)
                rc = VERR_VD_NOT_ENOUGH_METADATA;
        }

        Assert(RT_VALID_PTR(pMetaXfer) || RT_FAILURE(rc));

        if (RT_SUCCESS(rc) || rc == VERR_VD_NOT_ENOUGH_METADATA || rc == VERR_VD_ASYNC_IO_IN_PROGRESS)
        {
            /* If it is pending add the request to the list. */
            if (VDMETAXFER_TXDIR_GET(pMetaXfer->fFlags) == VDMETAXFER_TXDIR_READ)
            {
                PVDIOCTXDEFERRED pDeferred = (PVDIOCTXDEFERRED)RTMemAllocZ(sizeof(VDIOCTXDEFERRED));
                AssertPtr(pDeferred);

                RTListInit(&pDeferred->NodeDeferred);
                pDeferred->pIoCtx = pIoCtx;

                ASMAtomicIncU32(&pIoCtx->cMetaTransfersPending);
                RTListAppend(&pMetaXfer->ListIoCtxWaiting, &pDeferred->NodeDeferred);
                rc = VERR_VD_NOT_ENOUGH_METADATA;
            }
            else
            {
                /* Transfer the data. */
                pMetaXfer->cRefs++;
                Assert(pMetaXfer->cbMeta >= cbRead);
                Assert(pMetaXfer->Core.Key == (RTFOFF)uOffset);
                if (pMetaXfer->pbDataShw)
                    memcpy(pvBuf, pMetaXfer->pbDataShw, cbRead);
                else
                    memcpy(pvBuf, pMetaXfer->abData, cbRead);
                *ppMetaXfer = pMetaXfer;
            }
        }
    }

    LogFlowFunc(("returns rc=%Rrc\n", rc));
    return rc;
}

static DECLCALLBACK(int) vdIOIntWriteMeta(void *pvUser, PVDIOSTORAGE pIoStorage, uint64_t uOffset,
                                          const void *pvBuf, size_t cbWrite, PVDIOCTX pIoCtx,
                                          PFNVDXFERCOMPLETED pfnComplete, void *pvCompleteUser)
{
    PVDIO    pVDIo = (PVDIO)pvUser;
    PVDISK pDisk = pVDIo->pDisk;
    int rc = VINF_SUCCESS;
    RTSGSEG Seg;
    PVDIOTASK pIoTask;
    PVDMETAXFER pMetaXfer = NULL;
    bool fInTree = false;
    void *pvTask = NULL;

    LogFlowFunc(("pvUser=%#p pIoStorage=%#p uOffset=%llu pvBuf=%#p cbWrite=%u\n",
                 pvUser, pIoStorage, uOffset, pvBuf, cbWrite));

    AssertMsgReturn(   pIoCtx
                    || (!pfnComplete && !pvCompleteUser),
                    ("A synchronous metadata write is requested but the parameters are wrong\n"),
                    VERR_INVALID_POINTER);

    /** @todo Enable check for sync I/O later. */
    if (   pIoCtx
        && !(pIoCtx->fFlags & VDIOCTX_FLAGS_SYNC))
        VD_IS_LOCKED(pDisk);

    if (   !pIoCtx
        || pIoCtx->fFlags & VDIOCTX_FLAGS_SYNC
        || !pVDIo->pInterfaceIo->pfnWriteAsync)
    {
        /* Handle synchronous metadata I/O. */
        /** @todo Integrate with metadata transfers below. */
        rc = pVDIo->pInterfaceIo->pfnWriteSync(pVDIo->pInterfaceIo->Core.pvUser,
                                               pIoStorage->pStorage, uOffset,
                                               pvBuf, cbWrite, NULL);
    }
    else
    {
        pMetaXfer = (PVDMETAXFER)RTAvlrFileOffsetGet(pIoStorage->pTreeMetaXfers, uOffset);
        if (!pMetaXfer)
        {
            /* Allocate a new meta transfer. */
            pMetaXfer = vdMetaXferAlloc(pIoStorage, uOffset, cbWrite);
            if (!pMetaXfer)
                return VERR_NO_MEMORY;
        }
        else
        {
            Assert(pMetaXfer->cbMeta >= cbWrite);
            Assert(pMetaXfer->Core.Key == (RTFOFF)uOffset);
            fInTree = true;
        }

        if (VDMETAXFER_TXDIR_GET(pMetaXfer->fFlags) == VDMETAXFER_TXDIR_NONE)
        {
            pIoTask = vdIoTaskMetaAlloc(pIoStorage, pfnComplete, pvCompleteUser, pMetaXfer);
            if (!pIoTask)
            {
                RTMemFree(pMetaXfer);
                return VERR_NO_MEMORY;
            }

            memcpy(pMetaXfer->abData, pvBuf, cbWrite);
            Seg.cbSeg = cbWrite;
            Seg.pvSeg = pMetaXfer->abData;

            ASMAtomicIncU32(&pIoCtx->cMetaTransfersPending);

            VDMETAXFER_TXDIR_SET(pMetaXfer->fFlags, VDMETAXFER_TXDIR_WRITE);
            rc = pVDIo->pInterfaceIo->pfnWriteAsync(pVDIo->pInterfaceIo->Core.pvUser,
                                                    pIoStorage->pStorage,
                                                    uOffset, &Seg, 1, cbWrite, pIoTask,
                                                    &pvTask);
            if (RT_SUCCESS(rc))
            {
                VDMETAXFER_TXDIR_SET(pMetaXfer->fFlags, VDMETAXFER_TXDIR_NONE);
                ASMAtomicDecU32(&pIoCtx->cMetaTransfersPending);
                vdIoTaskFree(pDisk, pIoTask);
                if (fInTree && !pMetaXfer->cRefs)
                {
                    LogFlow(("Removing meta xfer=%#p\n", pMetaXfer));
                    bool fRemoved = RTAvlrFileOffsetRemove(pIoStorage->pTreeMetaXfers, pMetaXfer->Core.Key) != NULL;
                    AssertMsg(fRemoved, ("Metadata transfer wasn't removed\n")); NOREF(fRemoved);
                    RTMemFree(pMetaXfer);
                    pMetaXfer = NULL;
                }
            }
            else if (rc == VERR_VD_ASYNC_IO_IN_PROGRESS)
            {
                PVDIOCTXDEFERRED pDeferred = (PVDIOCTXDEFERRED)RTMemAllocZ(sizeof(VDIOCTXDEFERRED));
                AssertPtr(pDeferred);

                RTListInit(&pDeferred->NodeDeferred);
                pDeferred->pIoCtx = pIoCtx;

                if (!fInTree)
                {
                    bool fInserted = RTAvlrFileOffsetInsert(pIoStorage->pTreeMetaXfers, &pMetaXfer->Core);
                    Assert(fInserted); NOREF(fInserted);
                }

                RTListAppend(&pMetaXfer->ListIoCtxWaiting, &pDeferred->NodeDeferred);
            }
            else
            {
                RTMemFree(pMetaXfer);
                pMetaXfer = NULL;
            }
        }
        else
        {
            /* I/O is in progress, update shadow buffer and add to waiting list. */
            Assert(VDMETAXFER_TXDIR_GET(pMetaXfer->fFlags) == VDMETAXFER_TXDIR_WRITE);
            if (!pMetaXfer->pbDataShw)
            {
                /* Allocate shadow buffer and set initial state. */
                LogFlowFunc(("pMetaXfer=%#p Creating shadow buffer\n", pMetaXfer));
                pMetaXfer->pbDataShw = (uint8_t *)RTMemAlloc(pMetaXfer->cbMeta);
                if (RT_LIKELY(pMetaXfer->pbDataShw))
                    memcpy(pMetaXfer->pbDataShw, pMetaXfer->abData, pMetaXfer->cbMeta);
                else
                    rc = VERR_NO_MEMORY;
            }

            if (RT_SUCCESS(rc))
            {
                /* Update with written data and append to waiting list. */
                PVDIOCTXDEFERRED pDeferred = (PVDIOCTXDEFERRED)RTMemAllocZ(sizeof(VDIOCTXDEFERRED));
                if (pDeferred)
                {
                    LogFlowFunc(("pMetaXfer=%#p Updating shadow buffer\n", pMetaXfer));

                    RTListInit(&pDeferred->NodeDeferred);
                    pDeferred->pIoCtx = pIoCtx;
                    ASMAtomicIncU32(&pIoCtx->cMetaTransfersPending);
                    memcpy(pMetaXfer->pbDataShw, pvBuf, cbWrite);
                    RTListAppend(&pMetaXfer->ListIoCtxShwWrites, &pDeferred->NodeDeferred);
                }
                else
                {
                    /*
                     * Free shadow buffer if there is no one depending on it, i.e.
                     * we just allocated it.
                     */
                    if (RTListIsEmpty(&pMetaXfer->ListIoCtxShwWrites))
                    {
                        RTMemFree(pMetaXfer->pbDataShw);
                        pMetaXfer->pbDataShw = NULL;
                    }
                    rc = VERR_NO_MEMORY;
                }
            }
        }
    }

    LogFlowFunc(("returns rc=%Rrc\n", rc));
    return rc;
}

static DECLCALLBACK(void) vdIOIntMetaXferRelease(void *pvUser, PVDMETAXFER pMetaXfer)
{
    PVDIO    pVDIo = (PVDIO)pvUser;
    PVDISK pDisk = pVDIo->pDisk;
    PVDIOSTORAGE pIoStorage;

    /*
     * It is possible that we get called with a NULL metadata xfer handle
     * for synchronous I/O. Just exit.
     */
    if (!pMetaXfer)
        return;

    pIoStorage = pMetaXfer->pIoStorage;

    VD_IS_LOCKED(pDisk);

    Assert(   VDMETAXFER_TXDIR_GET(pMetaXfer->fFlags) == VDMETAXFER_TXDIR_NONE
           || VDMETAXFER_TXDIR_GET(pMetaXfer->fFlags) == VDMETAXFER_TXDIR_WRITE);
    Assert(pMetaXfer->cRefs > 0);

    pMetaXfer->cRefs--;
    if (   !pMetaXfer->cRefs
        && RTListIsEmpty(&pMetaXfer->ListIoCtxWaiting)
        && VDMETAXFER_TXDIR_GET(pMetaXfer->fFlags) == VDMETAXFER_TXDIR_NONE)
    {
        /* Free the meta data entry. */
        LogFlow(("Removing meta xfer=%#p\n", pMetaXfer));
        bool fRemoved = RTAvlrFileOffsetRemove(pIoStorage->pTreeMetaXfers, pMetaXfer->Core.Key) != NULL;
        AssertMsg(fRemoved, ("Metadata transfer wasn't removed\n")); NOREF(fRemoved);

        RTMemFree(pMetaXfer);
    }
}

static DECLCALLBACK(int) vdIOIntFlush(void *pvUser, PVDIOSTORAGE pIoStorage, PVDIOCTX pIoCtx,
                                      PFNVDXFERCOMPLETED pfnComplete, void *pvCompleteUser)
{
    PVDIO    pVDIo = (PVDIO)pvUser;
    PVDISK pDisk = pVDIo->pDisk;
    int rc = VINF_SUCCESS;
    PVDIOTASK pIoTask;
    PVDMETAXFER pMetaXfer = NULL;
    void *pvTask = NULL;

    LogFlowFunc(("pvUser=%#p pIoStorage=%#p pIoCtx=%#p\n",
                 pvUser, pIoStorage, pIoCtx));

    AssertMsgReturn(   pIoCtx
                    || (!pfnComplete && !pvCompleteUser),
                    ("A synchronous metadata write is requested but the parameters are wrong\n"),
                    VERR_INVALID_POINTER);

    /** @todo Enable check for sync I/O later. */
    if (   pIoCtx
        && !(pIoCtx->fFlags & VDIOCTX_FLAGS_SYNC))
        VD_IS_LOCKED(pDisk);

    if (pVDIo->fIgnoreFlush)
        return VINF_SUCCESS;

    if (   !pIoCtx
        || pIoCtx->fFlags & VDIOCTX_FLAGS_SYNC
        || !pVDIo->pInterfaceIo->pfnFlushAsync)
    {
        /* Handle synchronous flushes. */
        /** @todo Integrate with metadata transfers below. */
        rc = pVDIo->pInterfaceIo->pfnFlushSync(pVDIo->pInterfaceIo->Core.pvUser,
                                               pIoStorage->pStorage);
    }
    else
    {
        /* Allocate a new meta transfer. */
        pMetaXfer = vdMetaXferAlloc(pIoStorage, 0, 0);
        if (!pMetaXfer)
            return VERR_NO_MEMORY;

        pIoTask = vdIoTaskMetaAlloc(pIoStorage, pfnComplete, pvUser, pMetaXfer);
        if (!pIoTask)
        {
            RTMemFree(pMetaXfer);
            return VERR_NO_MEMORY;
        }

        ASMAtomicIncU32(&pIoCtx->cMetaTransfersPending);

        PVDIOCTXDEFERRED pDeferred = (PVDIOCTXDEFERRED)RTMemAllocZ(sizeof(VDIOCTXDEFERRED));
        AssertPtr(pDeferred);

        RTListInit(&pDeferred->NodeDeferred);
        pDeferred->pIoCtx = pIoCtx;

        RTListAppend(&pMetaXfer->ListIoCtxWaiting, &pDeferred->NodeDeferred);
        VDMETAXFER_TXDIR_SET(pMetaXfer->fFlags, VDMETAXFER_TXDIR_FLUSH);
        rc = pVDIo->pInterfaceIo->pfnFlushAsync(pVDIo->pInterfaceIo->Core.pvUser,
                                                pIoStorage->pStorage,
                                                pIoTask, &pvTask);
        if (RT_SUCCESS(rc))
        {
            VDMETAXFER_TXDIR_SET(pMetaXfer->fFlags, VDMETAXFER_TXDIR_NONE);
            ASMAtomicDecU32(&pIoCtx->cMetaTransfersPending);
            vdIoTaskFree(pDisk, pIoTask);
            RTMemFree(pDeferred);
            RTMemFree(pMetaXfer);
        }
        else if (rc != VERR_VD_ASYNC_IO_IN_PROGRESS)
            RTMemFree(pMetaXfer);
    }

    LogFlowFunc(("returns rc=%Rrc\n", rc));
    return rc;
}

static DECLCALLBACK(size_t) vdIOIntIoCtxCopyTo(void *pvUser, PVDIOCTX pIoCtx,
                                               const void *pvBuf, size_t cbBuf)
{
    PVDIO    pVDIo = (PVDIO)pvUser;
    PVDISK pDisk = pVDIo->pDisk;
    size_t cbCopied = 0;

    /** @todo Enable check for sync I/O later. */
    if (!(pIoCtx->fFlags & VDIOCTX_FLAGS_SYNC))
        VD_IS_LOCKED(pDisk);

    cbCopied = vdIoCtxCopyTo(pIoCtx, (uint8_t *)pvBuf, cbBuf);
    Assert(cbCopied == cbBuf);

    /// @todo Assert(pIoCtx->Req.Io.cbTransferLeft >= cbCopied); - triggers with vdCopyHelper/dmgRead.
    ASMAtomicSubU32(&pIoCtx->Req.Io.cbTransferLeft, (uint32_t)cbCopied);

    return cbCopied;
}

static DECLCALLBACK(size_t) vdIOIntIoCtxCopyFrom(void *pvUser, PVDIOCTX pIoCtx,
                                                 void *pvBuf, size_t cbBuf)
{
    PVDIO    pVDIo = (PVDIO)pvUser;
    PVDISK pDisk = pVDIo->pDisk;
    size_t cbCopied = 0;

    /** @todo Enable check for sync I/O later. */
    if (!(pIoCtx->fFlags & VDIOCTX_FLAGS_SYNC))
        VD_IS_LOCKED(pDisk);

    cbCopied = vdIoCtxCopyFrom(pIoCtx, (uint8_t *)pvBuf, cbBuf);
    Assert(cbCopied == cbBuf);

    /// @todo Assert(pIoCtx->Req.Io.cbTransferLeft > cbCopied); - triggers with vdCopyHelper/dmgRead.
    ASMAtomicSubU32(&pIoCtx->Req.Io.cbTransferLeft, (uint32_t)cbCopied);

    return cbCopied;
}

static DECLCALLBACK(size_t) vdIOIntIoCtxSet(void *pvUser, PVDIOCTX pIoCtx, int ch, size_t cb)
{
    PVDIO    pVDIo = (PVDIO)pvUser;
    PVDISK pDisk = pVDIo->pDisk;
    size_t cbSet = 0;

    /** @todo Enable check for sync I/O later. */
    if (!(pIoCtx->fFlags & VDIOCTX_FLAGS_SYNC))
        VD_IS_LOCKED(pDisk);

    cbSet = vdIoCtxSet(pIoCtx, ch, cb);
    Assert(cbSet == cb);

    /// @todo Assert(pIoCtx->Req.Io.cbTransferLeft >= cbSet); - triggers with vdCopyHelper/dmgRead.
    ASMAtomicSubU32(&pIoCtx->Req.Io.cbTransferLeft, (uint32_t)cbSet);

    return cbSet;
}

static DECLCALLBACK(size_t) vdIOIntIoCtxSegArrayCreate(void *pvUser, PVDIOCTX pIoCtx,
                                                       PRTSGSEG paSeg, unsigned *pcSeg,
                                                       size_t cbData)
{
    PVDIO    pVDIo = (PVDIO)pvUser;
    PVDISK pDisk = pVDIo->pDisk;
    size_t cbCreated = 0;

    /** @todo It is possible that this gets called from a filter plugin
     * outside of the disk lock. Refine assertion or remove completely. */
#if 0
    /** @todo Enable check for sync I/O later. */
    if (!(pIoCtx->fFlags & VDIOCTX_FLAGS_SYNC))
        VD_IS_LOCKED(pDisk);
#else
    NOREF(pDisk);
#endif

    cbCreated = RTSgBufSegArrayCreate(&pIoCtx->Req.Io.SgBuf, paSeg, pcSeg, cbData);
    Assert(!paSeg || cbData == cbCreated);

    return cbCreated;
}

static DECLCALLBACK(void) vdIOIntIoCtxCompleted(void *pvUser, PVDIOCTX pIoCtx, int rcReq,
                                                size_t cbCompleted)
{
    PVDIO    pVDIo = (PVDIO)pvUser;
    PVDISK pDisk = pVDIo->pDisk;

    LogFlowFunc(("pvUser=%#p pIoCtx=%#p rcReq=%Rrc cbCompleted=%zu\n",
                 pvUser, pIoCtx, rcReq, cbCompleted));

    /*
     * Grab the disk critical section to avoid races with other threads which
     * might still modify the I/O context.
     * Example is that iSCSI is doing an asynchronous write but calls us already
     * while the other thread is still hanging in vdWriteHelperAsync and couldn't update
     * the blocked state yet.
     * It can overwrite the state to true before we call vdIoCtxContinue and the
     * the request would hang indefinite.
     */
    ASMAtomicCmpXchgS32(&pIoCtx->rcReq, rcReq, VINF_SUCCESS);
    Assert(pIoCtx->Req.Io.cbTransferLeft >= cbCompleted);
    ASMAtomicSubU32(&pIoCtx->Req.Io.cbTransferLeft, (uint32_t)cbCompleted);

    /* Set next transfer function if the current one finished.
     * @todo: Find a better way to prevent vdIoCtxContinue from calling the current helper again. */
    if (!pIoCtx->Req.Io.cbTransferLeft)
    {
        pIoCtx->pfnIoCtxTransfer = pIoCtx->pfnIoCtxTransferNext;
        pIoCtx->pfnIoCtxTransferNext = NULL;
    }

    vdIoCtxAddToWaitingList(&pDisk->pIoCtxHaltedHead, pIoCtx);
    if (ASMAtomicCmpXchgBool(&pDisk->fLocked, true, false))
    {
        /* Immediately drop the lock again, it will take care of processing the list. */
        vdDiskUnlock(pDisk, NULL);
    }
}

static DECLCALLBACK(bool) vdIOIntIoCtxIsSynchronous(void *pvUser, PVDIOCTX pIoCtx)
{
    NOREF(pvUser);
    return !!(pIoCtx->fFlags & VDIOCTX_FLAGS_SYNC);
}

static DECLCALLBACK(bool) vdIOIntIoCtxIsZero(void *pvUser, PVDIOCTX pIoCtx, size_t cbCheck,
                                             bool fAdvance)
{
    NOREF(pvUser);

    bool fIsZero = RTSgBufIsZero(&pIoCtx->Req.Io.SgBuf, cbCheck);
    if (fIsZero && fAdvance)
        RTSgBufAdvance(&pIoCtx->Req.Io.SgBuf, cbCheck);

    return fIsZero;
}

static DECLCALLBACK(size_t) vdIOIntIoCtxGetDataUnitSize(void *pvUser, PVDIOCTX pIoCtx)
{
    RT_NOREF1(pIoCtx);
    PVDIO    pVDIo = (PVDIO)pvUser;
    PVDISK pDisk = pVDIo->pDisk;
    size_t cbSector = 0;

    PVDIMAGE pImage = vdGetImageByNumber(pDisk, VD_LAST_IMAGE);
    AssertPtrReturn(pImage, 0);

    PCVDREGIONLIST pRegionList = NULL;
    int rc = pImage->Backend->pfnQueryRegions(pImage->pBackendData, &pRegionList);
    if (RT_SUCCESS(rc))
    {
        cbSector = pRegionList->aRegions[0].cbBlock;

        AssertPtr(pImage->Backend->pfnRegionListRelease);
        pImage->Backend->pfnRegionListRelease(pImage->pBackendData, pRegionList);
    }

    return cbSector;
}

/**
 * VD I/O interface callback for opening a file (limited version for VDGetFormat).
 */
static DECLCALLBACK(int) vdIOIntOpenLimited(void *pvUser, const char *pszLocation,
                                            uint32_t fOpen, PPVDIOSTORAGE ppIoStorage)
{
    int rc = VINF_SUCCESS;
    PVDINTERFACEIO pInterfaceIo = (PVDINTERFACEIO)pvUser;
    PVDIOSTORAGE pIoStorage = (PVDIOSTORAGE)RTMemAllocZ(sizeof(VDIOSTORAGE));

    if (!pIoStorage)
        return VERR_NO_MEMORY;

    rc = pInterfaceIo->pfnOpen(NULL, pszLocation, fOpen, NULL, &pIoStorage->pStorage);
    if (RT_SUCCESS(rc))
        *ppIoStorage = pIoStorage;
    else
        RTMemFree(pIoStorage);

    return rc;
}

static DECLCALLBACK(int) vdIOIntCloseLimited(void *pvUser, PVDIOSTORAGE pIoStorage)
{
    PVDINTERFACEIO pInterfaceIo = (PVDINTERFACEIO)pvUser;
    int rc = pInterfaceIo->pfnClose(NULL, pIoStorage->pStorage);

    RTMemFree(pIoStorage);
    return rc;
}

static DECLCALLBACK(int) vdIOIntDeleteLimited(void *pvUser, const char *pcszFilename)
{
    PVDINTERFACEIO pInterfaceIo = (PVDINTERFACEIO)pvUser;
    return pInterfaceIo->pfnDelete(NULL, pcszFilename);
}

static DECLCALLBACK(int) vdIOIntMoveLimited(void *pvUser, const char *pcszSrc,
                                            const char *pcszDst, unsigned fMove)
{
    PVDINTERFACEIO pInterfaceIo = (PVDINTERFACEIO)pvUser;
    return pInterfaceIo->pfnMove(NULL, pcszSrc, pcszDst, fMove);
}

static DECLCALLBACK(int) vdIOIntGetFreeSpaceLimited(void *pvUser, const char *pcszFilename,
                                                    int64_t *pcbFreeSpace)
{
    PVDINTERFACEIO pInterfaceIo = (PVDINTERFACEIO)pvUser;
    return pInterfaceIo->pfnGetFreeSpace(NULL, pcszFilename, pcbFreeSpace);
}

static DECLCALLBACK(int) vdIOIntGetModificationTimeLimited(void *pvUser,
                                                           const char *pcszFilename,
                                                           PRTTIMESPEC pModificationTime)
{
    PVDINTERFACEIO pInterfaceIo = (PVDINTERFACEIO)pvUser;
    return pInterfaceIo->pfnGetModificationTime(NULL, pcszFilename, pModificationTime);
}

static DECLCALLBACK(int) vdIOIntGetSizeLimited(void *pvUser, PVDIOSTORAGE pIoStorage,
                                               uint64_t *pcbSize)
{
    PVDINTERFACEIO pInterfaceIo = (PVDINTERFACEIO)pvUser;
    return pInterfaceIo->pfnGetSize(NULL, pIoStorage->pStorage, pcbSize);
}

static DECLCALLBACK(int) vdIOIntSetSizeLimited(void *pvUser, PVDIOSTORAGE pIoStorage,
                                               uint64_t cbSize)
{
    PVDINTERFACEIO pInterfaceIo = (PVDINTERFACEIO)pvUser;
    return pInterfaceIo->pfnSetSize(NULL, pIoStorage->pStorage, cbSize);
}

static DECLCALLBACK(int) vdIOIntWriteUserLimited(void *pvUser, PVDIOSTORAGE pStorage,
                                                 uint64_t uOffset, PVDIOCTX pIoCtx,
                                                 size_t cbWrite,
                                                 PFNVDXFERCOMPLETED pfnComplete,
                                                 void *pvCompleteUser)
{
    NOREF(pvUser);
    NOREF(pStorage);
    NOREF(uOffset);
    NOREF(pIoCtx);
    NOREF(cbWrite);
    NOREF(pfnComplete);
    NOREF(pvCompleteUser);
    AssertMsgFailedReturn(("This needs to be implemented when called\n"), VERR_NOT_IMPLEMENTED);
}

static DECLCALLBACK(int) vdIOIntReadUserLimited(void *pvUser, PVDIOSTORAGE pStorage,
                                                uint64_t uOffset, PVDIOCTX pIoCtx,
                                                size_t cbRead)
{
    NOREF(pvUser);
    NOREF(pStorage);
    NOREF(uOffset);
    NOREF(pIoCtx);
    NOREF(cbRead);
    AssertMsgFailedReturn(("This needs to be implemented when called\n"), VERR_NOT_IMPLEMENTED);
}

static DECLCALLBACK(int) vdIOIntWriteMetaLimited(void *pvUser, PVDIOSTORAGE pStorage,
                                                 uint64_t uOffset, const void *pvBuffer,
                                                 size_t cbBuffer, PVDIOCTX pIoCtx,
                                                 PFNVDXFERCOMPLETED pfnComplete,
                                                 void *pvCompleteUser)
{
    PVDINTERFACEIO pInterfaceIo = (PVDINTERFACEIO)pvUser;

    AssertMsgReturn(!pIoCtx && !pfnComplete && !pvCompleteUser,
                    ("Async I/O not implemented for the limited interface"),
                    VERR_NOT_SUPPORTED);

    return pInterfaceIo->pfnWriteSync(NULL, pStorage->pStorage, uOffset, pvBuffer, cbBuffer, NULL);
}

static DECLCALLBACK(int) vdIOIntReadMetaLimited(void *pvUser, PVDIOSTORAGE pStorage,
                                                uint64_t uOffset, void *pvBuffer,
                                                size_t cbBuffer, PVDIOCTX pIoCtx,
                                                PPVDMETAXFER ppMetaXfer,
                                                PFNVDXFERCOMPLETED pfnComplete,
                                                void *pvCompleteUser)
{
    PVDINTERFACEIO pInterfaceIo = (PVDINTERFACEIO)pvUser;

    AssertMsgReturn(!pIoCtx && !ppMetaXfer && !pfnComplete && !pvCompleteUser,
                    ("Async I/O not implemented for the limited interface"),
                    VERR_NOT_SUPPORTED);

    return pInterfaceIo->pfnReadSync(NULL, pStorage->pStorage, uOffset, pvBuffer, cbBuffer, NULL);
}

#if 0 /* unsed */
static int vdIOIntMetaXferReleaseLimited(void *pvUser, PVDMETAXFER pMetaXfer)
{
    /* This is a NOP in this case. */
    NOREF(pvUser);
    NOREF(pMetaXfer);
    return VINF_SUCCESS;
}
#endif

static DECLCALLBACK(int) vdIOIntFlushLimited(void *pvUser, PVDIOSTORAGE pStorage,
                                             PVDIOCTX pIoCtx,
                                             PFNVDXFERCOMPLETED pfnComplete,
                                             void *pvCompleteUser)
{
    PVDINTERFACEIO pInterfaceIo = (PVDINTERFACEIO)pvUser;

    AssertMsgReturn(!pIoCtx && !pfnComplete && !pvCompleteUser,
                    ("Async I/O not implemented for the limited interface"),
                    VERR_NOT_SUPPORTED);

    return pInterfaceIo->pfnFlushSync(NULL, pStorage->pStorage);
}

/**
 * internal: send output to the log (unconditionally).
 */
static DECLCALLBACK(int) vdLogMessage(void *pvUser, const char *pszFormat, va_list args)
{
    NOREF(pvUser);
    RTLogPrintfV(pszFormat, args);
    return VINF_SUCCESS;
}

DECLINLINE(int) vdMessageWrapper(PVDISK pDisk, const char *pszFormat, ...)
{
    va_list va;
    va_start(va, pszFormat);
    int rc = pDisk->pInterfaceError->pfnMessage(pDisk->pInterfaceError->Core.pvUser,
                                                pszFormat, va);
    va_end(va);
    return rc;
}


/**
 * internal: adjust PCHS geometry
 */
static void vdFixupPCHSGeometry(PVDGEOMETRY pPCHS, uint64_t cbSize)
{
    /* Fix broken PCHS geometry. Can happen for two reasons: either the backend
     * mixes up PCHS and LCHS, or the application used to create the source
     * image has put garbage in it. Additionally, if the PCHS geometry covers
     * more than the image size, set it back to the default. */
    if (   pPCHS->cHeads > 16
        || pPCHS->cSectors > 63
        || pPCHS->cCylinders == 0
        || (uint64_t)pPCHS->cHeads * pPCHS->cSectors * pPCHS->cCylinders * 512 > cbSize)
    {
        Assert(!(RT_MIN(cbSize / 512 / 16 / 63, 16383) - (uint32_t)RT_MIN(cbSize / 512 / 16 / 63, 16383)));
        pPCHS->cCylinders = (uint32_t)RT_MIN(cbSize / 512 / 16 / 63, 16383);
        pPCHS->cHeads = 16;
        pPCHS->cSectors = 63;
    }
}

/**
 * internal: adjust LCHS geometry
 */
static void vdFixupLCHSGeometry(PVDGEOMETRY pLCHS, uint64_t cbSize)
{
    /* Fix broken LCHS geometry. Can happen for two reasons: either the backend
     * mixes up PCHS and LCHS, or the application used to create the source
     * image has put garbage in it. The fix in this case is to clear the LCHS
     * geometry to trigger autodetection when it is used next. If the geometry
     * already says "please autodetect" (cylinders=0) keep it. */
    if (   (   pLCHS->cHeads > 255
            || pLCHS->cHeads == 0
            || pLCHS->cSectors > 63
            || pLCHS->cSectors == 0)
        && pLCHS->cCylinders != 0)
    {
        pLCHS->cCylinders = 0;
        pLCHS->cHeads = 0;
        pLCHS->cSectors = 0;
    }
    /* Always recompute the number of cylinders stored in the LCHS
     * geometry if it isn't set to "autotedetect" at the moment.
     * This is very useful if the destination image size is
     * larger or smaller than the source image size. Do not modify
     * the number of heads and sectors. Windows guests hate it. */
    if (   pLCHS->cCylinders != 0
        && pLCHS->cHeads != 0 /* paranoia */
        && pLCHS->cSectors != 0 /* paranoia */)
    {
        Assert(!(RT_MIN(cbSize / 512 / pLCHS->cHeads / pLCHS->cSectors, 1024) - (uint32_t)RT_MIN(cbSize / 512 / pLCHS->cHeads / pLCHS->cSectors, 1024)));
        pLCHS->cCylinders = (uint32_t)RT_MIN(cbSize / 512 / pLCHS->cHeads / pLCHS->cSectors, 1024);
    }
}

/**
 * Sets the I/O callbacks of the given interface to the fallback methods
 *
 * @param   pIfIo    The I/O interface to setup.
 */
static void vdIfIoFallbackCallbacksSetup(PVDINTERFACEIO pIfIo)
{
    pIfIo->pfnOpen                = vdIOOpenFallback;
    pIfIo->pfnClose               = vdIOCloseFallback;
    pIfIo->pfnDelete              = vdIODeleteFallback;
    pIfIo->pfnMove                = vdIOMoveFallback;
    pIfIo->pfnGetFreeSpace        = vdIOGetFreeSpaceFallback;
    pIfIo->pfnGetModificationTime = vdIOGetModificationTimeFallback;
    pIfIo->pfnGetSize             = vdIOGetSizeFallback;
    pIfIo->pfnSetSize             = vdIOSetSizeFallback;
    pIfIo->pfnSetAllocationSize   = vdIOSetAllocationSizeFallback;
    pIfIo->pfnReadSync            = vdIOReadSyncFallback;
    pIfIo->pfnWriteSync           = vdIOWriteSyncFallback;
    pIfIo->pfnFlushSync           = vdIOFlushSyncFallback;
    pIfIo->pfnReadAsync           = NULL;
    pIfIo->pfnWriteAsync          = NULL;
    pIfIo->pfnFlushAsync          = NULL;
}

/**
 * Sets the internal I/O callbacks of the given interface.
 *
 * @param   pIfIoInt    The internal I/O interface to setup.
 */
static void vdIfIoIntCallbacksSetup(PVDINTERFACEIOINT pIfIoInt)
{
    pIfIoInt->pfnOpen                 = vdIOIntOpen;
    pIfIoInt->pfnClose                = vdIOIntClose;
    pIfIoInt->pfnDelete               = vdIOIntDelete;
    pIfIoInt->pfnMove                 = vdIOIntMove;
    pIfIoInt->pfnGetFreeSpace         = vdIOIntGetFreeSpace;
    pIfIoInt->pfnGetModificationTime  = vdIOIntGetModificationTime;
    pIfIoInt->pfnGetSize              = vdIOIntGetSize;
    pIfIoInt->pfnSetSize              = vdIOIntSetSize;
    pIfIoInt->pfnSetAllocationSize    = vdIOIntSetAllocationSize;
    pIfIoInt->pfnReadUser             = vdIOIntReadUser;
    pIfIoInt->pfnWriteUser            = vdIOIntWriteUser;
    pIfIoInt->pfnReadMeta             = vdIOIntReadMeta;
    pIfIoInt->pfnWriteMeta            = vdIOIntWriteMeta;
    pIfIoInt->pfnMetaXferRelease      = vdIOIntMetaXferRelease;
    pIfIoInt->pfnFlush                = vdIOIntFlush;
    pIfIoInt->pfnIoCtxCopyFrom        = vdIOIntIoCtxCopyFrom;
    pIfIoInt->pfnIoCtxCopyTo          = vdIOIntIoCtxCopyTo;
    pIfIoInt->pfnIoCtxSet             = vdIOIntIoCtxSet;
    pIfIoInt->pfnIoCtxSegArrayCreate  = vdIOIntIoCtxSegArrayCreate;
    pIfIoInt->pfnIoCtxCompleted       = vdIOIntIoCtxCompleted;
    pIfIoInt->pfnIoCtxIsSynchronous   = vdIOIntIoCtxIsSynchronous;
    pIfIoInt->pfnIoCtxIsZero          = vdIOIntIoCtxIsZero;
    pIfIoInt->pfnIoCtxGetDataUnitSize = vdIOIntIoCtxGetDataUnitSize;
}

/**
 * Internally used completion handler for synchronous I/O contexts.
 */
static DECLCALLBACK(void) vdIoCtxSyncComplete(void *pvUser1, void *pvUser2, int rcReq)
{
    RT_NOREF2(pvUser1, rcReq);
    RTSEMEVENT hEvent = (RTSEMEVENT)pvUser2;

    RTSemEventSignal(hEvent);
}


VBOXDDU_DECL(int) VDInit(void)
{
    int rc = vdPluginInit();
    LogRel(("VD: VDInit finished with %Rrc\n", rc));
    return rc;
}


VBOXDDU_DECL(int) VDShutdown(void)
{
    return vdPluginTerm();
}


VBOXDDU_DECL(int) VDPluginLoadFromFilename(const char *pszFilename)
{
    if (!vdPluginIsInitialized())
    {
        int rc = VDInit();
        if (RT_FAILURE(rc))
            return rc;
    }

    return vdPluginLoadFromFilename(pszFilename);
}

/**
 * Load all plugins from a given path.
 *
 * @returns VBox statuse code.
 * @param   pszPath         The path to load plugins from.
 */
VBOXDDU_DECL(int) VDPluginLoadFromPath(const char *pszPath)
{
    if (!vdPluginIsInitialized())
    {
        int rc = VDInit();
        if (RT_FAILURE(rc))
            return rc;
    }

    return vdPluginLoadFromPath(pszPath);
}


VBOXDDU_DECL(int) VDPluginUnloadFromFilename(const char *pszFilename)
{
    if (!vdPluginIsInitialized())
    {
        int rc = VDInit();
        if (RT_FAILURE(rc))
            return rc;
    }

    return vdPluginUnloadFromFilename(pszFilename);
}


VBOXDDU_DECL(int) VDPluginUnloadFromPath(const char *pszPath)
{
    if (!vdPluginIsInitialized())
    {
        int rc = VDInit();
        if (RT_FAILURE(rc))
            return rc;
    }

    return vdPluginUnloadFromPath(pszPath);
}


VBOXDDU_DECL(int) VDBackendInfo(unsigned cEntriesAlloc, PVDBACKENDINFO pEntries,
                                unsigned *pcEntriesUsed)
{
    int rc = VINF_SUCCESS;

    LogFlowFunc(("cEntriesAlloc=%u pEntries=%#p pcEntriesUsed=%#p\n", cEntriesAlloc, pEntries, pcEntriesUsed));
    /* Check arguments. */
    AssertMsgReturn(cEntriesAlloc, ("cEntriesAlloc=%u\n", cEntriesAlloc), VERR_INVALID_PARAMETER);
    AssertPtrReturn(pEntries, VERR_INVALID_POINTER);
    AssertPtrReturn(pcEntriesUsed, VERR_INVALID_POINTER);
    if (!vdPluginIsInitialized())
        VDInit();

    uint32_t cBackends = vdGetImageBackendCount();
    if (cEntriesAlloc < cBackends)
    {
        *pcEntriesUsed = cBackends;
        return VERR_BUFFER_OVERFLOW;
    }

    for (unsigned i = 0; i < cBackends; i++)
    {
        PCVDIMAGEBACKEND pBackend;
        rc = vdQueryImageBackend(i, &pBackend);
        AssertRC(rc);

        pEntries[i].pszBackend         = pBackend->pszBackendName;
        pEntries[i].uBackendCaps       = pBackend->uBackendCaps;
        pEntries[i].paFileExtensions   = pBackend->paFileExtensions;
        pEntries[i].paConfigInfo       = pBackend->paConfigInfo;
        pEntries[i].pfnComposeLocation = pBackend->pfnComposeLocation;
        pEntries[i].pfnComposeName     = pBackend->pfnComposeName;
    }

    LogFlowFunc(("returns %Rrc *pcEntriesUsed=%u\n", rc, cBackends));
    *pcEntriesUsed = cBackends;
    return rc;
}


VBOXDDU_DECL(int) VDBackendInfoOne(const char *pszBackend, PVDBACKENDINFO pEntry)
{
    LogFlowFunc(("pszBackend=%#p pEntry=%#p\n", pszBackend, pEntry));
    /* Check arguments. */
    AssertPtrReturn(pszBackend, VERR_INVALID_POINTER);
    AssertPtrReturn(pEntry, VERR_INVALID_POINTER);
    if (!vdPluginIsInitialized())
        VDInit();

    PCVDIMAGEBACKEND pBackend;
    int rc = vdFindImageBackend(pszBackend, &pBackend);
    if (RT_SUCCESS(rc))
    {
        pEntry->pszBackend       = pBackend->pszBackendName;
        pEntry->uBackendCaps     = pBackend->uBackendCaps;
        pEntry->paFileExtensions = pBackend->paFileExtensions;
        pEntry->paConfigInfo     = pBackend->paConfigInfo;
    }

    return rc;
}


VBOXDDU_DECL(int) VDFilterInfo(unsigned cEntriesAlloc, PVDFILTERINFO pEntries,
                               unsigned *pcEntriesUsed)
{
    int rc = VINF_SUCCESS;

    LogFlowFunc(("cEntriesAlloc=%u pEntries=%#p pcEntriesUsed=%#p\n", cEntriesAlloc, pEntries, pcEntriesUsed));
    /* Check arguments. */
    AssertMsgReturn(cEntriesAlloc,
                    ("cEntriesAlloc=%u\n", cEntriesAlloc),
                    VERR_INVALID_PARAMETER);
    AssertPtrReturn(pEntries, VERR_INVALID_POINTER);
    AssertPtrReturn(pcEntriesUsed, VERR_INVALID_POINTER);
    if (!vdPluginIsInitialized())
        VDInit();

    uint32_t cBackends = vdGetFilterBackendCount();
    if (cEntriesAlloc < cBackends)
    {
        *pcEntriesUsed = cBackends;
        return VERR_BUFFER_OVERFLOW;
    }

    for (unsigned i = 0; i < cBackends; i++)
    {
        PCVDFILTERBACKEND pBackend;
        rc = vdQueryFilterBackend(i, &pBackend);
        pEntries[i].pszFilter    = pBackend->pszBackendName;
        pEntries[i].paConfigInfo = pBackend->paConfigInfo;
    }

    LogFlowFunc(("returns %Rrc *pcEntriesUsed=%u\n", rc, cBackends));
    *pcEntriesUsed = cBackends;
    return rc;
}


VBOXDDU_DECL(int) VDFilterInfoOne(const char *pszFilter, PVDFILTERINFO pEntry)
{
    LogFlowFunc(("pszFilter=%#p pEntry=%#p\n", pszFilter, pEntry));
    /* Check arguments. */
    AssertPtrReturn(pszFilter, VERR_INVALID_POINTER);
    AssertPtrReturn(pEntry, VERR_INVALID_POINTER);
    if (!vdPluginIsInitialized())
        VDInit();

    PCVDFILTERBACKEND pBackend;
    int rc = vdFindFilterBackend(pszFilter, &pBackend);
    if (RT_SUCCESS(rc))
    {
        pEntry->pszFilter    = pBackend->pszBackendName;
        pEntry->paConfigInfo = pBackend->paConfigInfo;
    }

    return rc;
}


VBOXDDU_DECL(int) VDCreate(PVDINTERFACE pVDIfsDisk, VDTYPE enmType, PVDISK *ppDisk)
{
    int rc = VINF_SUCCESS;
    PVDISK pDisk = NULL;

    LogFlowFunc(("pVDIfsDisk=%#p\n", pVDIfsDisk));
    /* Check arguments. */
    AssertPtrReturn(ppDisk, VERR_INVALID_POINTER);

    do
    {
        pDisk = (PVDISK)RTMemAllocZ(sizeof(VDISK));
        if (pDisk)
        {
            pDisk->u32Signature            = VDISK_SIGNATURE;
            pDisk->enmType                 = enmType;
            pDisk->cImages                 = 0;
            pDisk->pBase                   = NULL;
            pDisk->pLast                   = NULL;
            pDisk->cbSize                  = 0;
            pDisk->PCHSGeometry.cCylinders = 0;
            pDisk->PCHSGeometry.cHeads     = 0;
            pDisk->PCHSGeometry.cSectors   = 0;
            pDisk->LCHSGeometry.cCylinders = 0;
            pDisk->LCHSGeometry.cHeads     = 0;
            pDisk->LCHSGeometry.cSectors   = 0;
            pDisk->pVDIfsDisk              = pVDIfsDisk;
            pDisk->pInterfaceError         = NULL;
            pDisk->pInterfaceThreadSync    = NULL;
            pDisk->pIoCtxLockOwner         = NULL;
            pDisk->pIoCtxHead              = NULL;
            pDisk->fLocked                 = false;
            pDisk->hMemCacheIoCtx          = NIL_RTMEMCACHE;
            pDisk->hMemCacheIoTask         = NIL_RTMEMCACHE;
            RTListInit(&pDisk->ListFilterChainWrite);
            RTListInit(&pDisk->ListFilterChainRead);

            /* Create the I/O ctx cache */
            rc = RTMemCacheCreate(&pDisk->hMemCacheIoCtx, sizeof(VDIOCTX), 0, UINT32_MAX,
                                  NULL, NULL, NULL, 0);
            if (RT_FAILURE(rc))
                break;

            /* Create the I/O task cache */
            rc = RTMemCacheCreate(&pDisk->hMemCacheIoTask, sizeof(VDIOTASK), 0, UINT32_MAX,
                                  NULL, NULL, NULL, 0);
            if (RT_FAILURE(rc))
                break;

            pDisk->pInterfaceError      = VDIfErrorGet(pVDIfsDisk);
            pDisk->pInterfaceThreadSync = VDIfThreadSyncGet(pVDIfsDisk);

            *ppDisk = pDisk;
        }
        else
        {
            rc = VERR_NO_MEMORY;
            break;
        }
    } while (0);

    if (   RT_FAILURE(rc)
        && pDisk)
    {
        if (pDisk->hMemCacheIoCtx != NIL_RTMEMCACHE)
            RTMemCacheDestroy(pDisk->hMemCacheIoCtx);
        if (pDisk->hMemCacheIoTask != NIL_RTMEMCACHE)
            RTMemCacheDestroy(pDisk->hMemCacheIoTask);
    }

    LogFlowFunc(("returns %Rrc (pDisk=%#p)\n", rc, pDisk));
    return rc;
}


VBOXDDU_DECL(int) VDDestroy(PVDISK pDisk)
{
    int rc = VINF_SUCCESS;
    LogFlowFunc(("pDisk=%#p\n", pDisk));
    do
    {
        /* sanity check */
        AssertPtrBreak(pDisk);
        AssertMsg(pDisk->u32Signature == VDISK_SIGNATURE, ("u32Signature=%08x\n", pDisk->u32Signature));
        Assert(!pDisk->fLocked);

        rc = VDCloseAll(pDisk);
        int rc2 = VDFilterRemoveAll(pDisk);
        if (RT_SUCCESS(rc))
            rc = rc2;

        RTMemCacheDestroy(pDisk->hMemCacheIoCtx);
        RTMemCacheDestroy(pDisk->hMemCacheIoTask);
        RTMemFree