Video recording not working

[gim]

Since 5.0 video recording not working anymore in VBoxHeadless mode.

Step to reproduce:

VBoxManage createvm --name "test" --register
VBoxManage modifyvm "test" --memory 512 --acpi on --boot1 dvd
VBoxManage modifyvm "test" --hostonlyadapter1 vboxnet0
VBoxManage modifyvm "test" --nic1 hostonly
VBoxManage modifyvm "test" --ostype Debian
VBoxManage createhd --filename test.vdi --size 10000
VBoxManage storagectl "test" --name "IDE Controller" --add ide
VBoxManage storageattach "test" --storagectl "IDE Controller" --port 0 --device 0 --type hdd --medium test.vdi
VBoxManage modifyvm "test" --videocap on --videocapfile video.webm

VBoxManage startvm "test" --type headless

So video.webm contains only header and footer (after poweroffing) of the VPX format.

After digging a little I found that video not writing because pFramebuffer always is NULL:

...
                if (   !pFBInfo->pFramebuffer.isNull()          // <-------- HERE
                    && !pFBInfo->fDisabled)
                {
                    rc = VERR_NOT_SUPPORTED;
                    if (   pFBInfo->fVBVAEnabled
                        && pFBInfo->pu8FramebufferVRAM)
                    {
                        rc = VideoRecCopyToIntBuf(pDisplay->mpVideoRecCtx, uScreenId, 0, 0,
                                                  BitmapFormat_BGR,
                                                  pFBInfo->u16BitsPerPixel,
                                                  pFBInfo->u32LineSize, pFBInfo->w, pFBInfo->h,
                                                  pFBInfo->pu8FramebufferVRAM, u64Now);
...

(VirtualBox-5.0.20\src\VBox\Main\src-client\DisplayImpl.cpp) So VideoRecCopyToIntBuf will never called and as a result will never trigging write video.

After digging a littel more I found that pFramebuffer doesn't attach anymore in 5.0:

...
#ifdef VBOX_WITH_VPX
        if (fVideoRec)
        {
            CHECK_ERROR_BREAK(machine, COMSETTER(VideoCaptureFile)(Bstr(szMpegFile).raw()));
            CHECK_ERROR_BREAK(machine, COMSETTER(VideoCaptureWidth)(ulFrameWidth));
            CHECK_ERROR_BREAK(machine, COMSETTER(VideoCaptureHeight)(ulFrameHeight));
            CHECK_ERROR_BREAK(machine, COMSETTER(VideoCaptureRate)(ulBitRate));
            CHECK_ERROR_BREAK(machine, COMSETTER(VideoCaptureEnabled)(TRUE));
        }
#endif /* defined(VBOX_WITH_VPX) */

// ---- PROBABLY HERE ATTACH? ---- 

        /* get the machine debugger (isn't necessarily available) */
        ComPtr <IMachineDebugger> machineDebugger;
        console->COMGETTER(Debugger)(machineDebugger.asOutParam());
        if (machineDebugger)
        {
            Log(("Machine debugger available!\n"));
        }
...

(VirtualBox-5.0.20\src\VBox\Frontends\VBoxHeadless\VBoxHeadless.cpp)

Versus 4.3.xx:

#ifdef VBOX_WITH_VPX
        if (fVideoRec)
        {
            CHECK_ERROR_BREAK(machine, COMSETTER(VideoCaptureFile)(Bstr(szMpegFile).raw()));
            CHECK_ERROR_BREAK(machine, COMSETTER(VideoCaptureWidth)(ulFrameWidth));
            CHECK_ERROR_BREAK(machine, COMSETTER(VideoCaptureHeight)(ulFrameHeight));
            CHECK_ERROR_BREAK(machine, COMSETTER(VideoCaptureRate)(ulBitRate));
            CHECK_ERROR_BREAK(machine, COMSETTER(VideoCaptureEnabled)(TRUE));
        }
#endif /* defined(VBOX_WITH_VPX) */
        ULONG cMonitors = 1;
        machine->COMGETTER(MonitorCount)(&cMonitors);

        unsigned uScreenId;
        for (uScreenId = 0; uScreenId < cMonitors; uScreenId++)
        {
            VRDPFramebuffer *pVRDPFramebuffer = new VRDPFramebuffer();
            if (!pVRDPFramebuffer)
            {
                RTPrintf("Error: could not create framebuffer object %d\n", uScreenId);
                break;
            }
            pVRDPFramebuffer->AddRef();
            display->SetFramebuffer(uScreenId, pVRDPFramebuffer);
        }
        if (uScreenId < cMonitors)
        {
            break;
        }

        // fill in remaining slots with null framebuffers
        for (uScreenId = 0; uScreenId < cMonitors; uScreenId++)
        {
            ComPtr<IFramebuffer> fb;
            LONG xOrigin, yOrigin;
            HRESULT hrc2 = display->GetFramebuffer(uScreenId,
                                                   fb.asOutParam(),
                                                   &xOrigin, &yOrigin);
            if (hrc2 == S_OK && fb.isNull())
            {
                NullFB *pNullFB =  new NullFB();
                pNullFB->AddRef();
                pNullFB->init();
                display->SetFramebuffer(uScreenId, pNullFB);
            }
        }

(src\VBox\Frontends\VBoxHeadless\VBoxHeadless.cpp)

[gim]

After this patch video recording working fine. But you need check this for sure...

[gim]

AttachFramebuffer fix

[gim]

After re-examination detected problem image video size (resize)...

[Frank Mehnert]

Confirmed.

[Frank Mehnert]

Should be fixed in 5.0.26.