FTM.cpp@ 32190

Last change on this file since 32190 was 32179, checked in by vboxsync, 15 years ago
More stats
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 35.2 KB

Line
1	/* $Id: FTM.cpp 32179 2010-09-01 14:32:27Z vboxsync $ */
2	/** @file
3	* FTM - Fault Tolerance Manager
4	*/
5
6	/*
7	* Copyright (C) 2010 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*******************************************************************************
20	* Header Files *
21	*******************************************************************************/
22	#define LOG_GROUP LOG_GROUP_FTM
23	#include "FTMInternal.h"
24	#include <VBox/vm.h>
25	#include <VBox/vmm.h>
26	#include <VBox/err.h>
27	#include <VBox/param.h>
28	#include <VBox/ssm.h>
29	#include <VBox/log.h>
30	#include <VBox/pgm.h>
31
32	#include <iprt/assert.h>
33	#include <iprt/thread.h>
34	#include <iprt/string.h>
35	#include <iprt/mem.h>
36	#include <iprt/tcp.h>
37	#include <iprt/socket.h>
38	#include <iprt/semaphore.h>
39	#include <iprt/asm.h>
40
41	#include <include/internal/vm.h>
42	#include <include/internal/em.h>
43
44	/*******************************************************************************
45	* Structures and Typedefs *
46	*******************************************************************************/
47
48	/**
49	* TCP stream header.
50	*
51	* This is an extra layer for fixing the problem with figuring out when the SSM
52	* stream ends.
53	*/
54	typedef struct FTMTCPHDR
55	{
56	/** Magic value. */
57	uint32_t u32Magic;
58	/** The size of the data block following this header.
59	* 0 indicates the end of the stream, while UINT32_MAX indicates
60	* cancelation. */
61	uint32_t cb;
62	} FTMTCPHDR;
63	/** Magic value for FTMTCPHDR::u32Magic. (Egberto Gismonti Amin) */
64	#define FTMTCPHDR_MAGIC UINT32_C(0x19471205)
65	/** The max block size. */
66	#define FTMTCPHDR_MAX_SIZE UINT32_C(0x00fffff8)
67
68	/**
69	* TCP stream header.
70	*
71	* This is an extra layer for fixing the problem with figuring out when the SSM
72	* stream ends.
73	*/
74	typedef struct FTMTCPHDRMEM
75	{
76	/** Magic value. */
77	uint32_t u32Magic;
78	/** Size (Uncompressed) of the pages following the header. */
79	uint32_t cbPageRange;
80	/** GC Physical address of the page(s) to sync. */
81	RTGCPHYS GCPhys;
82	/** The size of the data block following this header.
83	* 0 indicates the end of the stream, while UINT32_MAX indicates
84	* cancelation. */
85	uint32_t cb;
86	} FTMTCPHDRMEM;
87
88	/*******************************************************************************
89	* Global Variables *
90	*******************************************************************************/
91	static const char g_szWelcome[] = "VirtualBox-Fault-Tolerance-Sync-1.0\n";
92
93	/**
94	* Initializes the FTM.
95	*
96	* @returns VBox status code.
97	* @param pVM The VM to operate on.
98	*/
99	VMMR3DECL(int) FTMR3Init(PVM pVM)
100	{
101	/*
102	* Assert alignment and sizes.
103	*/
104	AssertCompile(sizeof(pVM->ftm.s) <= sizeof(pVM->ftm.padding));
105	AssertCompileMemberAlignment(FTM, CritSect, sizeof(uintptr_t));
106
107	/** @todo saved state for master nodes! */
108	pVM->ftm.s.pszAddress = NULL;
109	pVM->ftm.s.pszPassword = NULL;
110	pVM->fFaultTolerantMaster = false;
111	pVM->ftm.s.fIsStandbyNode = false;
112	pVM->ftm.s.standby.hServer = NIL_RTTCPSERVER;
113	pVM->ftm.s.master.hShutdownEvent = NIL_RTSEMEVENT;
114	pVM->ftm.s.hSocket = NIL_RTSOCKET;
115
116	/*
117	* Initialize the PGM critical section.
118	*/
119	int rc = PDMR3CritSectInit(pVM, &pVM->ftm.s.CritSect, RT_SRC_POS, "FTM");
120	AssertRCReturn(rc, rc);
121
122	/*
123	* Register statistics.
124	*/
125	STAM_REL_REG(pVM, &pVM->ftm.s.StatReceivedMem, STAMTYPE_COUNTER, "/FT/Received/Mem", STAMUNIT_BYTES, "The amount of memory pages that was received.");
126	STAM_REL_REG(pVM, &pVM->ftm.s.StatReceivedState, STAMTYPE_COUNTER, "/FT/Received/State", STAMUNIT_BYTES, "The amount of state information that was received.");
127	STAM_REL_REG(pVM, &pVM->ftm.s.StatSentMem, STAMTYPE_COUNTER, "/FT/Sent/Mem", STAMUNIT_BYTES, "The amount of memory pages that was sent.");
128	STAM_REL_REG(pVM, &pVM->ftm.s.StatSentState, STAMTYPE_COUNTER, "/FT/Sent/State", STAMUNIT_BYTES, "The amount of state information that was sent.");
129	STAM_REL_REG(pVM, &pVM->ftm.s.StatDeltaVM, STAMTYPE_COUNTER, "/FT/Sync/DeltaVM", STAMUNIT_OCCURENCES, "Number of delta vm syncs.");
130	STAM_REL_REG(pVM, &pVM->ftm.s.StatFullSync, STAMTYPE_COUNTER, "/FT/Sync/Full", STAMUNIT_OCCURENCES, "Number of full vm syncs.");
131	STAM_REL_REG(pVM, &pVM->ftm.s.StatDeltaMem, STAMTYPE_COUNTER, "/FT/Sync/DeltaMem", STAMUNIT_OCCURENCES, "Number of delta mem syncs.");
132	STAM_REL_REG(pVM, &pVM->ftm.s.StatCheckpointStorage, STAMTYPE_COUNTER, "/FT/Checkpoint/Storage", STAMUNIT_OCCURENCES, "Number of storage checkpoints.");
133	STAM_REL_REG(pVM, &pVM->ftm.s.StatCheckpointNetwork, STAMTYPE_COUNTER, "/FT/Checkpoint/Network", STAMUNIT_OCCURENCES, "Number of network checkpoints.");
134	#ifdef VBOX_WITH_STATISTICS
135	STAM_REG(pVM, &pVM->ftm.s.StatCheckpoint, STAMTYPE_PROFILE, "/FT/Checkpoint", STAMUNIT_TICKS_PER_CALL, "Profiling of FTMR3SetCheckpoint.");
136	#endif
137	return VINF_SUCCESS;
138	}
139
140	/**
141	* Terminates the FTM.
142	*
143	* Termination means cleaning up and freeing all resources,
144	* the VM itself is at this point powered off or suspended.
145	*
146	* @returns VBox status code.
147	* @param pVM The VM to operate on.
148	*/
149	VMMR3DECL(int) FTMR3Term(PVM pVM)
150	{
151	if (pVM->ftm.s.master.hShutdownEvent != NIL_RTSEMEVENT)
152	{
153	RTSemEventDestroy(pVM->ftm.s.master.hShutdownEvent);
154	pVM->ftm.s.master.hShutdownEvent = NIL_RTSEMEVENT;
155	}
156	if (pVM->ftm.s.hSocket != NIL_RTSOCKET)
157	{
158	RTTcpClientClose(pVM->ftm.s.hSocket);
159	pVM->ftm.s.hSocket = NIL_RTSOCKET;
160	}
161	if (pVM->ftm.s.standby.hServer)
162	{
163	RTTcpServerDestroy(pVM->ftm.s.standby.hServer);
164	pVM->ftm.s.standby.hServer = NULL;
165	}
166	if (pVM->ftm.s.pszAddress)
167	RTMemFree(pVM->ftm.s.pszAddress);
168	if (pVM->ftm.s.pszPassword)
169	RTMemFree(pVM->ftm.s.pszPassword);
170
171	pVM->ftm.s.pszAddress = NULL;
172	pVM->ftm.s.pszPassword = NULL;
173
174	PDMR3CritSectDelete(&pVM->ftm.s.CritSect);
175	return VINF_SUCCESS;
176	}
177
178
179	static int ftmR3TcpWriteACK(PVM pVM)
180	{
181	int rc = RTTcpWrite(pVM->ftm.s.hSocket, "ACK\n", sizeof("ACK\n") - 1);
182	if (RT_FAILURE(rc))
183	{
184	LogRel(("FTSync: RTTcpWrite(,ACK,) -> %Rrc\n", rc));
185	}
186	return rc;
187	}
188
189
190	static int ftmR3TcpWriteNACK(PVM pVM, int32_t rc2, const char *pszMsgText = NULL)
191	{
192	char szMsg[256];
193	size_t cch;
194	if (pszMsgText && *pszMsgText)
195	{
196	cch = RTStrPrintf(szMsg, sizeof(szMsg), "NACK=%d;%s\n", rc2, pszMsgText);
197	for (size_t off = 6; off + 1 < cch; off++)
198	if (szMsg[off] == '\n')
199	szMsg[off] = '\r';
200	}
201	else
202	cch = RTStrPrintf(szMsg, sizeof(szMsg), "NACK=%d\n", rc2);
203	int rc = RTTcpWrite(pVM->ftm.s.hSocket, szMsg, cch);
204	if (RT_FAILURE(rc))
205	LogRel(("FTSync: RTTcpWrite(,%s,%zu) -> %Rrc\n", szMsg, cch, rc));
206	return rc;
207	}
208
209	/**
210	* Reads a string from the socket.
211	*
212	* @returns VBox status code.
213	*
214	* @param pState The teleporter state structure.
215	* @param pszBuf The output buffer.
216	* @param cchBuf The size of the output buffer.
217	*
218	*/
219	static int ftmR3TcpReadLine(PVM pVM, char *pszBuf, size_t cchBuf)
220	{
221	char *pszStart = pszBuf;
222	RTSOCKET Sock = pVM->ftm.s.hSocket;
223
224	AssertReturn(cchBuf > 1, VERR_INTERNAL_ERROR);
225	*pszBuf = '\0';
226
227	/* dead simple approach. */
228	for (;;)
229	{
230	char ch;
231	int rc = RTTcpRead(Sock, &ch, sizeof(ch), NULL);
232	if (RT_FAILURE(rc))
233	{
234	LogRel(("FTSync: RTTcpRead -> %Rrc while reading string ('%s')\n", rc, pszStart));
235	return rc;
236	}
237	if ( ch == '\n'
238	\|\| ch == '\0')
239	return VINF_SUCCESS;
240	if (cchBuf <= 1)
241	{
242	LogRel(("FTSync: String buffer overflow: '%s'\n", pszStart));
243	return VERR_BUFFER_OVERFLOW;
244	}
245	*pszBuf++ = ch;
246	*pszBuf = '\0';
247	cchBuf--;
248	}
249	}
250
251	/**
252	* Reads an ACK or NACK.
253	*
254	* @returns VBox status code.
255	* @param pVM The VM to operate on.
256	* @param pszWhich Which ACK is this this?
257	* @param pszNAckMsg Optional NACK message.
258	*/
259	static int ftmR3TcpReadACK(PVM pVM, const char pszWhich, const char pszNAckMsg = NULL)
260	{
261	char szMsg[256];
262	int rc = ftmR3TcpReadLine(pVM, szMsg, sizeof(szMsg));
263	if (RT_FAILURE(rc))
264	return rc;
265
266	if (!strcmp(szMsg, "ACK"))
267	return VINF_SUCCESS;
268
269	if (!strncmp(szMsg, "NACK=", sizeof("NACK=") - 1))
270	{
271	char *pszMsgText = strchr(szMsg, ';');
272	if (pszMsgText)
273	*pszMsgText++ = '\0';
274
275	int32_t vrc2;
276	rc = RTStrToInt32Full(&szMsg[sizeof("NACK=") - 1], 10, &vrc2);
277	if (rc == VINF_SUCCESS)
278	{
279	/*
280	* Well formed NACK, transform it into an error.
281	*/
282	if (pszNAckMsg)
283	{
284	LogRel(("FTSync: %s: NACK=%Rrc (%d)\n", pszWhich, vrc2, vrc2));
285	return VERR_INTERNAL_ERROR;
286	}
287
288	if (pszMsgText)
289	{
290	pszMsgText = RTStrStrip(pszMsgText);
291	for (size_t off = 0; pszMsgText[off]; off++)
292	if (pszMsgText[off] == '\r')
293	pszMsgText[off] = '\n';
294
295	LogRel(("FTSync: %s: NACK=%Rrc (%d) - '%s'\n", pszWhich, vrc2, vrc2, pszMsgText));
296	}
297	return VERR_INTERNAL_ERROR_2;
298	}
299
300	if (pszMsgText)
301	pszMsgText[-1] = ';';
302	}
303	return VERR_INTERNAL_ERROR_3;
304	}
305
306	/**
307	* Submitts a command to the destination and waits for the ACK.
308	*
309	* @returns VBox status code.
310	*
311	* @param pVM The VM to operate on.
312	* @param pszCommand The command.
313	* @param fWaitForAck Whether to wait for the ACK.
314	*/
315	static int ftmR3TcpSubmitCommand(PVM pVM, const char *pszCommand, bool fWaitForAck = true)
316	{
317	int rc = RTTcpSgWriteL(pVM->ftm.s.hSocket, 2, pszCommand, strlen(pszCommand), "\n", sizeof("\n") - 1);
318	if (RT_FAILURE(rc))
319	return rc;
320	if (!fWaitForAck)
321	return VINF_SUCCESS;
322	return ftmR3TcpReadACK(pVM, pszCommand);
323	}
324
325	/**
326	* @copydoc SSMSTRMOPS::pfnWrite
327	*/
328	static DECLCALLBACK(int) ftmR3TcpOpWrite(void pvUser, uint64_t offStream, const void pvBuf, size_t cbToWrite)
329	{
330	PVM pVM = (PVM)pvUser;
331
332	AssertReturn(cbToWrite > 0, VINF_SUCCESS);
333	AssertReturn(cbToWrite < UINT32_MAX, VERR_OUT_OF_RANGE);
334	AssertReturn(pVM->fFaultTolerantMaster, VERR_INVALID_HANDLE);
335
336	for (;;)
337	{
338	FTMTCPHDR Hdr;
339	Hdr.u32Magic = FTMTCPHDR_MAGIC;
340	Hdr.cb = RT_MIN((uint32_t)cbToWrite, FTMTCPHDR_MAX_SIZE);
341	int rc = RTTcpSgWriteL(pVM->ftm.s.hSocket, 2, &Hdr, sizeof(Hdr), pvBuf, (size_t)Hdr.cb);
342	if (RT_FAILURE(rc))
343	{
344	LogRel(("FTSync/TCP: Write error: %Rrc (cb=%#x)\n", rc, Hdr.cb));
345	return rc;
346	}
347	pVM->ftm.s.StatSentState.c += Hdr.cb + sizeof(Hdr);
348	pVM->ftm.s.syncstate.uOffStream += Hdr.cb;
349	if (Hdr.cb == cbToWrite)
350	return VINF_SUCCESS;
351
352	/* advance */
353	cbToWrite -= Hdr.cb;
354	pvBuf = (uint8_t const *)pvBuf + Hdr.cb;
355	}
356	}
357
358
359	/**
360	* Selects and poll for close condition.
361	*
362	* We can use a relatively high poll timeout here since it's only used to get
363	* us out of error paths. In the normal cause of events, we'll get a
364	* end-of-stream header.
365	*
366	* @returns VBox status code.
367	*
368	* @param pState The teleporter state data.
369	*/
370	static int ftmR3TcpReadSelect(PVM pVM)
371	{
372	int rc;
373	do
374	{
375	rc = RTTcpSelectOne(pVM->ftm.s.hSocket, 1000);
376	if (RT_FAILURE(rc) && rc != VERR_TIMEOUT)
377	{
378	pVM->ftm.s.syncstate.fIOError = true;
379	LogRel(("FTSync/TCP: Header select error: %Rrc\n", rc));
380	break;
381	}
382	if (pVM->ftm.s.syncstate.fStopReading)
383	{
384	rc = VERR_EOF;
385	break;
386	}
387	} while (rc == VERR_TIMEOUT);
388	return rc;
389	}
390
391
392	/**
393	* @copydoc SSMSTRMOPS::pfnRead
394	*/
395	static DECLCALLBACK(int) ftmR3TcpOpRead(void pvUser, uint64_t offStream, void pvBuf, size_t cbToRead, size_t *pcbRead)
396	{
397	PVM pVM = (PVM)pvUser;
398	AssertReturn(!pVM->fFaultTolerantMaster, VERR_INVALID_HANDLE);
399
400	for (;;)
401	{
402	int rc;
403
404	/*
405	* Check for various conditions and may have been signalled.
406	*/
407	if (pVM->ftm.s.syncstate.fEndOfStream)
408	return VERR_EOF;
409	if (pVM->ftm.s.syncstate.fStopReading)
410	return VERR_EOF;
411	if (pVM->ftm.s.syncstate.fIOError)
412	return VERR_IO_GEN_FAILURE;
413
414	/*
415	* If there is no more data in the current block, read the next
416	* block header.
417	*/
418	if (!pVM->ftm.s.syncstate.cbReadBlock)
419	{
420	rc = ftmR3TcpReadSelect(pVM);
421	if (RT_FAILURE(rc))
422	return rc;
423	FTMTCPHDR Hdr;
424	rc = RTTcpRead(pVM->ftm.s.hSocket, &Hdr, sizeof(Hdr), NULL);
425	if (RT_FAILURE(rc))
426	{
427	pVM->ftm.s.syncstate.fIOError = true;
428	LogRel(("FTSync/TCP: Header read error: %Rrc\n", rc));
429	return rc;
430	}
431	pVM->ftm.s.StatReceivedState.c += sizeof(Hdr);
432
433	if (RT_UNLIKELY( Hdr.u32Magic != FTMTCPHDR_MAGIC
434	\|\| Hdr.cb > FTMTCPHDR_MAX_SIZE
435	\|\| Hdr.cb == 0))
436	{
437	if ( Hdr.u32Magic == FTMTCPHDR_MAGIC
438	&& ( Hdr.cb == 0
439	\|\| Hdr.cb == UINT32_MAX)
440	)
441	{
442	pVM->ftm.s.syncstate.fEndOfStream = true;
443	pVM->ftm.s.syncstate.cbReadBlock = 0;
444	return Hdr.cb ? VERR_SSM_CANCELLED : VERR_EOF;
445	}
446	pVM->ftm.s.syncstate.fIOError = true;
447	LogRel(("FTSync/TCP: Invalid block: u32Magic=%#x cb=%#x\n", Hdr.u32Magic, Hdr.cb));
448	return VERR_IO_GEN_FAILURE;
449	}
450
451	pVM->ftm.s.syncstate.cbReadBlock = Hdr.cb;
452	if (pVM->ftm.s.syncstate.fStopReading)
453	return VERR_EOF;
454	}
455
456	/*
457	* Read more data.
458	*/
459	rc = ftmR3TcpReadSelect(pVM);
460	if (RT_FAILURE(rc))
461	return rc;
462
463	uint32_t cb = (uint32_t)RT_MIN(pVM->ftm.s.syncstate.cbReadBlock, cbToRead);
464	rc = RTTcpRead(pVM->ftm.s.hSocket, pvBuf, cb, pcbRead);
465	if (RT_FAILURE(rc))
466	{
467	pVM->ftm.s.syncstate.fIOError = true;
468	LogRel(("FTSync/TCP: Data read error: %Rrc (cb=%#x)\n", rc, cb));
469	return rc;
470	}
471	if (pcbRead)
472	{
473	cb = (uint32_t)*pcbRead;
474	pVM->ftm.s.StatReceivedState.c += cb;
475	pVM->ftm.s.syncstate.uOffStream += cb;
476	pVM->ftm.s.syncstate.cbReadBlock -= cb;
477	return VINF_SUCCESS;
478	}
479	pVM->ftm.s.StatReceivedState.c += cb;
480	pVM->ftm.s.syncstate.uOffStream += cb;
481	pVM->ftm.s.syncstate.cbReadBlock -= cb;
482	if (cbToRead == cb)
483	return VINF_SUCCESS;
484
485	/* Advance to the next block. */
486	cbToRead -= cb;
487	pvBuf = (uint8_t *)pvBuf + cb;
488	}
489	}
490
491
492	/**
493	* @copydoc SSMSTRMOPS::pfnSeek
494	*/
495	static DECLCALLBACK(int) ftmR3TcpOpSeek(void pvUser, int64_t offSeek, unsigned uMethod, uint64_t poffActual)
496	{
497	return VERR_NOT_SUPPORTED;
498	}
499
500
501	/**
502	* @copydoc SSMSTRMOPS::pfnTell
503	*/
504	static DECLCALLBACK(uint64_t) ftmR3TcpOpTell(void *pvUser)
505	{
506	PVM pVM = (PVM)pvUser;
507	return pVM->ftm.s.syncstate.uOffStream;
508	}
509
510
511	/**
512	* @copydoc SSMSTRMOPS::pfnSize
513	*/
514	static DECLCALLBACK(int) ftmR3TcpOpSize(void pvUser, uint64_t pcb)
515	{
516	return VERR_NOT_SUPPORTED;
517	}
518
519
520	/**
521	* @copydoc SSMSTRMOPS::pfnIsOk
522	*/
523	static DECLCALLBACK(int) ftmR3TcpOpIsOk(void *pvUser)
524	{
525	PVM pVM = (PVM)pvUser;
526
527	if (pVM->fFaultTolerantMaster)
528	{
529	/* Poll for incoming NACKs and errors from the other side */
530	int rc = RTTcpSelectOne(pVM->ftm.s.hSocket, 0);
531	if (rc != VERR_TIMEOUT)
532	{
533	if (RT_SUCCESS(rc))
534	{
535	LogRel(("FTSync/TCP: Incoming data detect by IsOk, assuming it is a cancellation NACK.\n"));
536	rc = VERR_SSM_CANCELLED;
537	}
538	else
539	LogRel(("FTSync/TCP: RTTcpSelectOne -> %Rrc (IsOk).\n", rc));
540	return rc;
541	}
542	}
543
544	return VINF_SUCCESS;
545	}
546
547
548	/**
549	* @copydoc SSMSTRMOPS::pfnClose
550	*/
551	static DECLCALLBACK(int) ftmR3TcpOpClose(void *pvUser, bool fCanceled)
552	{
553	PVM pVM = (PVM)pvUser;
554
555	if (pVM->fFaultTolerantMaster)
556	{
557	FTMTCPHDR EofHdr;
558	EofHdr.u32Magic = FTMTCPHDR_MAGIC;
559	EofHdr.cb = fCanceled ? UINT32_MAX : 0;
560	int rc = RTTcpWrite(pVM->ftm.s.hSocket, &EofHdr, sizeof(EofHdr));
561	if (RT_FAILURE(rc))
562	{
563	LogRel(("FTSync/TCP: EOF Header write error: %Rrc\n", rc));
564	return rc;
565	}
566	}
567	else
568	{
569	ASMAtomicWriteBool(&pVM->ftm.s.syncstate.fStopReading, true);
570	}
571
572	return VINF_SUCCESS;
573	}
574
575
576	/**
577	* Method table for a TCP based stream.
578	*/
579	static SSMSTRMOPS const g_ftmR3TcpOps =
580	{
581	SSMSTRMOPS_VERSION,
582	ftmR3TcpOpWrite,
583	ftmR3TcpOpRead,
584	ftmR3TcpOpSeek,
585	ftmR3TcpOpTell,
586	ftmR3TcpOpSize,
587	ftmR3TcpOpIsOk,
588	ftmR3TcpOpClose,
589	SSMSTRMOPS_VERSION
590	};
591
592	/**
593	* VMR3ReqCallWait callback
594	*
595	* @param pVM The VM handle.
596	*
597	*/
598	static DECLCALLBACK(void) ftmR3WriteProtectMemory(PVM pVM)
599	{
600	int rc = PGMR3PhysWriteProtectRAM(pVM);
601	AssertRC(rc);
602	}
603
604	/**
605	* Sync the VM state partially or fully
606	*
607	* @returns VBox status code.
608	* @param pVM The VM handle.
609	* @param enmState Which state to sync
610	*/
611	static int ftmR3PerformSync(PVM pVM, FTMSYNCSTATE enmState)
612	{
613	int rc;
614	bool fFullSync = false;
615
616	if (enmState != FTMSYNCSTATE_DELTA_MEMORY)
617	{
618	rc = VMR3Suspend(pVM);
619	AssertRCReturn(rc, rc);
620	/** Hack alert as EM is responsible for dealing with the suspend state. We must do this here ourselves, but only for this EMT.*/
621	if (VM_IS_EMT(pVM))
622	EMR3NotifySuspend(pVM);
623	}
624
625	switch (enmState)
626	{
627	case FTMSYNCSTATE_FULL:
628	fFullSync = true;
629	/* no break */
630	case FTMSYNCSTATE_DELTA_VM:
631	{
632	bool fSuspended = false;
633
634	STAM_REL_COUNTER_INC((fFullSync) ? &pVM->ftm.s.StatFullSync : &pVM->ftm.s.StatDeltaVM);
635
636	RTSocketRetain(pVM->ftm.s.hSocket); /* For concurrent access by I/O thread and EMT. */
637
638	/* Reset the sync state. */
639	pVM->ftm.s.syncstate.uOffStream = 0;
640	pVM->ftm.s.syncstate.cbReadBlock = 0;
641	pVM->ftm.s.syncstate.fStopReading = false;
642	pVM->ftm.s.syncstate.fIOError = false;
643	pVM->ftm.s.syncstate.fEndOfStream = false;
644
645	rc = ftmR3TcpSubmitCommand(pVM, (fFullSync) ? "full-sync" : "checkpoint");
646	AssertRC(rc);
647
648	pVM->ftm.s.fDeltaLoadSaveActive = (fFullSync == false);
649	rc = VMR3SaveFT(pVM, &g_ftmR3TcpOps, pVM, &fSuspended);
650	pVM->ftm.s.fDeltaLoadSaveActive = false;
651	AssertRC(rc);
652
653	rc = ftmR3TcpReadACK(pVM, (fFullSync) ? "full-sync-complete" : "checkpoint-complete");
654	AssertRC(rc);
655
656	RTSocketRelease(pVM->ftm.s.hSocket);
657	break;
658	}
659
660	case FTMSYNCSTATE_DELTA_MEMORY:
661	/* Nothing to do as we sync the memory in an async thread; no need to block EMT. */
662	STAM_REL_COUNTER_INC(&pVM->ftm.s.StatDeltaMem);
663	break;
664	}
665
666	/* Write protect all memory. */
667	rc = VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)ftmR3WriteProtectMemory, 1, pVM);
668	AssertRCReturn(rc, rc);
669
670	if (enmState != FTMSYNCSTATE_DELTA_MEMORY)
671	{
672	rc = VMR3Resume(pVM);
673	AssertRCReturn(rc, rc);
674
675	/** Hack alert as EM is responsible for dealing with the suspend state. We must do this here ourselves, but only for this EMT.*/
676	if (VM_IS_EMT(pVM))
677	EMR3NotifyResume(pVM);
678	}
679	return VINF_SUCCESS;
680	}
681
682	/**
683	* PGMR3PhysEnumDirtyFTPages callback for syncing dirty physical pages
684	*
685	* @param pVM VM Handle.
686	* @param GCPhys GC physical address
687	* @param pRange HC virtual address of the page(s)
688	* @param cbRange Size of the dirty range in bytes.
689	* @param pvUser User argument
690	*/
691	static DECLCALLBACK(int) ftmR3SyncDirtyPage(PVM pVM, RTGCPHYS GCPhys, uint8_t pRange, unsigned cbRange, void pvUser)
692	{
693	FTMTCPHDRMEM Hdr;
694	Hdr.u32Magic = FTMTCPHDR_MAGIC;
695	Hdr.GCPhys = GCPhys;
696	Hdr.cbPageRange = cbRange;
697	Hdr.cb = cbRange;
698	/** @todo compress page(s). */
699	int rc = RTTcpSgWriteL(pVM->ftm.s.hSocket, 2, &Hdr, sizeof(Hdr), pRange, (size_t)Hdr.cb);
700	if (RT_FAILURE(rc))
701	{
702	LogRel(("FTSync/TCP: Write error (ftmR3SyncDirtyPage): %Rrc (cb=%#x)\n", rc, Hdr.cb));
703	return rc;
704	}
705	pVM->ftm.s.StatSentMem.c += Hdr.cb + sizeof(Hdr);
706	return VINF_SUCCESS;
707	}
708
709	/**
710	* Thread function which starts syncing process for this master VM
711	*
712	* @param Thread The thread id.
713	* @param pvUser Not used
714	* @return VINF_SUCCESS (ignored).
715	*
716	*/
717	static DECLCALLBACK(int) ftmR3MasterThread(RTTHREAD Thread, void *pvUser)
718	{
719	int rc = VINF_SUCCESS;
720	PVM pVM = (PVM)pvUser;
721
722	for (;;)
723	{
724	/*
725	* Try connect to the standby machine.
726	*/
727	Log(("ftmR3MasterThread: client connect to %s %d\n", pVM->ftm.s.pszAddress, pVM->ftm.s.uPort));
728	rc = RTTcpClientConnect(pVM->ftm.s.pszAddress, pVM->ftm.s.uPort, &pVM->ftm.s.hSocket);
729	if (RT_SUCCESS(rc))
730	{
731	Log(("ftmR3MasterThread: CONNECTED\n"));
732
733	/* Disable Nagle. */
734	rc = RTTcpSetSendCoalescing(pVM->ftm.s.hSocket, false /fEnable/);
735	AssertRC(rc);
736
737	/* Read and check the welcome message. */
738	char szLine[RT_MAX(128, sizeof(g_szWelcome))];
739	RT_ZERO(szLine);
740	rc = RTTcpRead(pVM->ftm.s.hSocket, szLine, sizeof(g_szWelcome) - 1, NULL);
741	if ( RT_SUCCESS(rc)
742	&& !strcmp(szLine, g_szWelcome))
743	{
744	/* password */
745	if (pVM->ftm.s.pszPassword)
746	rc = RTTcpWrite(pVM->ftm.s.hSocket, pVM->ftm.s.pszPassword, strlen(pVM->ftm.s.pszPassword));
747
748	if (RT_SUCCESS(rc))
749	{
750	/* ACK */
751	rc = ftmR3TcpReadACK(pVM, "password", "Invalid password");
752	if (RT_SUCCESS(rc))
753	{
754	/** todo: verify VM config. */
755	break;
756	}
757	}
758	}
759	/* Failed, so don't bother anymore. */
760	return VINF_SUCCESS;
761	}
762	rc = RTSemEventWait(pVM->ftm.s.master.hShutdownEvent, 1000 /* 1 second */);
763	if (rc != VERR_TIMEOUT)
764	return VINF_SUCCESS; /* told to quit */
765	}
766
767	/* Successfully initialized the connection to the standby node.
768	* Start the sync process.
769	*/
770
771	/* First sync all memory and write protect everything so
772	* we can send changed pages later on.
773	*/
774
775	rc = ftmR3PerformSync(pVM, FTMSYNCSTATE_FULL);
776
777	for (;;)
778	{
779	rc = RTSemEventWait(pVM->ftm.s.master.hShutdownEvent, pVM->ftm.s.uInterval);
780	if (rc != VERR_TIMEOUT)
781	break; /* told to quit */
782
783	if (!pVM->ftm.s.fCheckpointingActive)
784	{
785	rc = PDMCritSectEnter(&pVM->ftm.s.CritSect, VERR_SEM_BUSY);
786	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
787
788	rc = ftmR3TcpSubmitCommand(pVM, "mem-sync");
789	AssertRC(rc);
790
791	/* sync the changed memory with the standby node. */
792	rc = ftmR3PerformSync(pVM, FTMSYNCSTATE_DELTA_MEMORY);
793
794	/* Enumerate all dirty pages and send them to the standby VM. */
795	rc = PGMR3PhysEnumDirtyFTPages(pVM, ftmR3SyncDirtyPage, NULL /* pvUser */);
796	AssertRC(rc);
797
798	/* Send last memory header to signal the end. */
799	FTMTCPHDRMEM Hdr;
800	Hdr.u32Magic = FTMTCPHDR_MAGIC;
801	Hdr.GCPhys = 0;
802	Hdr.cbPageRange = 0;
803	Hdr.cb = 0;
804	rc = RTTcpSgWriteL(pVM->ftm.s.hSocket, 1, &Hdr, sizeof(Hdr));
805	if (RT_FAILURE(rc))
806	LogRel(("FTSync/TCP: Write error (ftmR3MasterThread): %Rrc (cb=%#x)\n", rc, Hdr.cb));
807
808	rc = ftmR3TcpReadACK(pVM, "mem-sync-complete");
809	AssertRC(rc);
810
811	PDMCritSectLeave(&pVM->ftm.s.CritSect);
812	}
813	}
814	return rc;
815	}
816
817	/**
818	* Listen for incoming traffic destined for the standby VM.
819	*
820	* @copydoc FNRTTCPSERVE
821	*
822	* @returns VINF_SUCCESS or VERR_TCP_SERVER_STOP.
823	*/
824	static DECLCALLBACK(int) ftmR3StandbyServeConnection(RTSOCKET Sock, void *pvUser)
825	{
826	PVM pVM = (PVM)pvUser;
827
828	pVM->ftm.s.hSocket = Sock;
829
830	/*
831	* Disable Nagle.
832	*/
833	int rc = RTTcpSetSendCoalescing(Sock, false /fEnable/);
834	AssertRC(rc);
835
836	/* Send the welcome message to the master node. */
837	rc = RTTcpWrite(Sock, g_szWelcome, sizeof(g_szWelcome) - 1);
838	if (RT_FAILURE(rc))
839	{
840	LogRel(("Teleporter: Failed to write welcome message: %Rrc\n", rc));
841	return VINF_SUCCESS;
842	}
843
844	/*
845	* Password.
846	*/
847	const char *pszPassword = pVM->ftm.s.pszPassword;
848	if (pszPassword)
849	{
850	unsigned off = 0;
851	while (pszPassword[off])
852	{
853	char ch;
854	rc = RTTcpRead(Sock, &ch, sizeof(ch), NULL);
855	if ( RT_FAILURE(rc)
856	\|\| pszPassword[off] != ch)
857	{
858	if (RT_FAILURE(rc))
859	LogRel(("FTSync: Password read failure (off=%u): %Rrc\n", off, rc));
860	else
861	LogRel(("FTSync: Invalid password (off=%u)\n", off));
862	ftmR3TcpWriteNACK(pVM, VERR_AUTHENTICATION_FAILURE);
863	return VINF_SUCCESS;
864	}
865	off++;
866	}
867	}
868	rc = ftmR3TcpWriteACK(pVM);
869	if (RT_FAILURE(rc))
870	return VINF_SUCCESS;
871
872	/** todo: verify VM config. */
873
874	/*
875	* Stop the server.
876	*
877	* Note! After this point we must return VERR_TCP_SERVER_STOP, while prior
878	* to it we must not return that value!
879	*/
880	RTTcpServerShutdown(pVM->ftm.s.standby.hServer);
881
882	/*
883	* Command processing loop.
884	*/
885	bool fDone = false;
886	for (;;)
887	{
888	bool fFullSync = false;
889	char szCmd[128];
890
891	rc = ftmR3TcpReadLine(pVM, szCmd, sizeof(szCmd));
892	if (RT_FAILURE(rc))
893	break;
894
895	if (!strcmp(szCmd, "mem-sync"))
896	{
897	rc = ftmR3TcpWriteACK(pVM);
898	AssertRC(rc);
899	if (RT_FAILURE(rc))
900	continue;
901
902	while (true)
903	{
904	FTMTCPHDRMEM Hdr;
905	void *pPage;
906
907	/* Read memory header. */
908	rc = RTTcpRead(pVM->ftm.s.hSocket, &Hdr, sizeof(Hdr), NULL);
909	if (RT_FAILURE(rc))
910	{
911	Log(("RTTcpRead failed with %Rrc\n", rc));
912	break;
913	}
914	pVM->ftm.s.StatReceivedMem.c += sizeof(Hdr);
915
916	if (Hdr.cb == 0)
917	break; /* end of sync. */
918
919	Assert(Hdr.cb == Hdr.cbPageRange); /** @todo uncompress */
920
921	/* Allocate memory to hold the page(s). */
922	pPage = RTMemAlloc(Hdr.cbPageRange);
923	AssertBreak(pPage);
924
925	/* Fetch the page(s). */
926	rc = RTTcpRead(pVM->ftm.s.hSocket, pPage, Hdr.cb, NULL);
927	if (RT_FAILURE(rc))
928	{
929	Log(("RTTcpRead page data (%d bytes) failed with %Rrc\n", Hdr.cb, rc));
930	break;
931	}
932	pVM->ftm.s.StatReceivedMem.c += Hdr.cb;
933
934	/* Update the guest memory of the standby VM. */
935	#if 1
936	rc = PGMR3PhysWriteExternal(pVM, Hdr.GCPhys, pPage, Hdr.cbPageRange, "FTMemSync");
937	#else
938	rc = PGMPhysWrite(pVM, Hdr.GCPhys, pPage, Hdr.cbPageRange);
939	#endif
940	AssertRC(rc);
941
942	RTMemFree(pPage);
943	}
944
945	rc = ftmR3TcpWriteACK(pVM);
946	AssertRC(rc);
947	}
948	else
949	if ( !strcmp(szCmd, "checkpoint")
950	\|\| !strcmp(szCmd, "full-sync")
951	\|\| (fFullSync = true)) /* intended assignment */
952	{
953	rc = ftmR3TcpWriteACK(pVM);
954	AssertRC(rc);
955	if (RT_FAILURE(rc))
956	continue;
957
958	RTSocketRetain(pVM->ftm.s.hSocket); /* For concurrent access by I/O thread and EMT. */
959
960	/* Reset the sync state. */
961	pVM->ftm.s.syncstate.uOffStream = 0;
962	pVM->ftm.s.syncstate.cbReadBlock = 0;
963	pVM->ftm.s.syncstate.fStopReading = false;
964	pVM->ftm.s.syncstate.fIOError = false;
965	pVM->ftm.s.syncstate.fEndOfStream = false;
966
967	pVM->ftm.s.fDeltaLoadSaveActive = (fFullSync == false);
968	rc = VMR3LoadFromStream(pVM, &g_ftmR3TcpOps, pVM, NULL, NULL);
969	pVM->ftm.s.fDeltaLoadSaveActive = false;
970	RTSocketRelease(pVM->ftm.s.hSocket);
971	AssertRC(rc);
972	if (RT_FAILURE(rc))
973	{
974	LogRel(("FTSync: VMR3LoadFromStream -> %Rrc\n", rc));
975	ftmR3TcpWriteNACK(pVM, rc);
976	continue;
977	}
978
979	/* The EOS might not have been read, make sure it is. */
980	pVM->ftm.s.syncstate.fStopReading = false;
981	size_t cbRead;
982	rc = ftmR3TcpOpRead(pVM, pVM->ftm.s.syncstate.uOffStream, szCmd, 1, &cbRead);
983	if (rc != VERR_EOF)
984	{
985	LogRel(("FTSync: Draining teleporterTcpOpRead -> %Rrc\n", rc));
986	ftmR3TcpWriteNACK(pVM, rc);
987	continue;
988	}
989
990	rc = ftmR3TcpWriteACK(pVM);
991	AssertRC(rc);
992	}
993	}
994	LogFlowFunc(("returns mRc=%Rrc\n", rc));
995	return VERR_TCP_SERVER_STOP;
996	}
997
998	/**
999	* Powers on the fault tolerant virtual machine.
1000	*
1001	* @returns VBox status code.
1002	*
1003	* @param pVM The VM to operate on.
1004	* @param fMaster FT master or standby
1005	* @param uInterval FT sync interval
1006	* @param pszAddress Standby VM address
1007	* @param uPort Standby VM port
1008	* @param pszPassword FT password (NULL for none)
1009	*
1010	* @thread Any thread.
1011	* @vmstate Created
1012	* @vmstateto PoweringOn+Running (master), PoweringOn+Running_FT (standby)
1013	*/
1014	VMMR3DECL(int) FTMR3PowerOn(PVM pVM, bool fMaster, unsigned uInterval, const char pszAddress, unsigned uPort, const char pszPassword)
1015	{
1016	int rc = VINF_SUCCESS;
1017
1018	VMSTATE enmVMState = VMR3GetState(pVM);
1019	AssertMsgReturn(enmVMState == VMSTATE_CREATED,
1020	("%s\n", VMR3GetStateName(enmVMState)),
1021	VERR_INTERNAL_ERROR_4);
1022	AssertReturn(pszAddress, VERR_INVALID_PARAMETER);
1023
1024	if (pVM->ftm.s.uInterval)
1025	pVM->ftm.s.uInterval = uInterval;
1026	else
1027	pVM->ftm.s.uInterval = 50; /* standard sync interval of 50ms */
1028
1029	pVM->ftm.s.uPort = uPort;
1030	pVM->ftm.s.pszAddress = RTStrDup(pszAddress);
1031	if (pszPassword)
1032	pVM->ftm.s.pszPassword = RTStrDup(pszPassword);
1033	if (fMaster)
1034	{
1035	rc = RTSemEventCreate(&pVM->ftm.s.master.hShutdownEvent);
1036	if (RT_FAILURE(rc))
1037	return rc;
1038
1039	rc = RTThreadCreate(NULL, ftmR3MasterThread, pVM,
1040	0, RTTHREADTYPE_IO /* higher than normal priority */, 0, "ftmMaster");
1041	if (RT_FAILURE(rc))
1042	return rc;
1043
1044	pVM->fFaultTolerantMaster = true;
1045	if (PGMIsUsingLargePages(pVM))
1046	{
1047	/* Must disable large page usage as 2 MB pages are too big to write monitor. */
1048	LogRel(("FTSync: disabling large page usage.\n"));
1049	PGMSetLargePageUsage(pVM, false);
1050	}
1051	/** @todo might need to disable page fusion as well */
1052
1053	return VMR3PowerOn(pVM);
1054	}
1055	else
1056	{
1057	/* standby */
1058	rc = RTTcpServerCreateEx(pszAddress, uPort, &pVM->ftm.s.standby.hServer);
1059	if (RT_FAILURE(rc))
1060	return rc;
1061	pVM->ftm.s.fIsStandbyNode = true;
1062
1063	rc = RTTcpServerListen(pVM->ftm.s.standby.hServer, ftmR3StandbyServeConnection, pVM);
1064	/** @todo deal with the exit code to check if we should activate this standby VM. */
1065
1066	if (pVM->ftm.s.standby.hServer)
1067	{
1068	RTTcpServerDestroy(pVM->ftm.s.standby.hServer);
1069	pVM->ftm.s.standby.hServer = NULL;
1070	}
1071	if (rc == VERR_TCP_SERVER_SHUTDOWN)
1072	rc = VINF_SUCCESS; /* ignore this error; the standby process was cancelled. */
1073	}
1074	return rc;
1075	}
1076
1077	/**
1078	* Powers off the fault tolerant virtual machine (standby).
1079	*
1080	* @returns VBox status code.
1081	*
1082	* @param pVM The VM to operate on.
1083	*/
1084	VMMR3DECL(int) FTMR3CancelStandby(PVM pVM)
1085	{
1086	AssertReturn(!pVM->fFaultTolerantMaster, VERR_NOT_SUPPORTED);
1087	Assert(pVM->ftm.s.standby.hServer);
1088
1089	return RTTcpServerShutdown(pVM->ftm.s.standby.hServer);
1090	}
1091
1092
1093	/**
1094	* Performs a full sync to the standby node
1095	*
1096	* @returns VBox status code.
1097	*
1098	* @param pVM The VM to operate on.
1099	* @param enmCheckpoint Checkpoint type
1100	*/
1101	VMMR3DECL(int) FTMR3SetCheckpoint(PVM pVM, FTMCHECKPOINTTYPE enmCheckpoint)
1102	{
1103	int rc;
1104
1105	if (!pVM->fFaultTolerantMaster)
1106	return VINF_SUCCESS;
1107
1108	switch (enmCheckpoint)
1109	{
1110	case FTMCHECKPOINTTYPE_NETWORK:
1111	STAM_REL_COUNTER_INC(&pVM->ftm.s.StatCheckpointNetwork);
1112	break;
1113
1114	case FTMCHECKPOINTTYPE_STORAGE:
1115	STAM_REL_COUNTER_INC(&pVM->ftm.s.StatCheckpointStorage);
1116	break;
1117
1118	default:
1119	break;
1120	}
1121	pVM->ftm.s.fCheckpointingActive = true;
1122	if (VM_IS_EMT(pVM))
1123	{
1124	PVMCPU pVCpu = VMMGetCpu(pVM);
1125
1126	/* We must take special care here as the memory sync is competing with us and requires a responsive EMT. */
1127	while ((rc = PDMCritSectTryEnter(&pVM->ftm.s.CritSect)) == VERR_SEM_BUSY)
1128	{
1129	if (VM_FF_ISPENDING(pVM, VM_FF_EMT_RENDEZVOUS))
1130	{
1131	rc = VMMR3EmtRendezvousFF(pVM, pVCpu);
1132	AssertRC(rc);
1133	}
1134
1135	if (VM_FF_ISPENDING(pVM, VM_FF_REQUEST))
1136	{
1137	rc = VMR3ReqProcessU(pVM->pUVM, VMCPUID_ANY);
1138	AssertRC(rc);
1139	}
1140	}
1141	}
1142	else
1143	rc = PDMCritSectEnter(&pVM->ftm.s.CritSect, VERR_SEM_BUSY);
1144
1145	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
1146
1147	STAM_PROFILE_START(&pVM->ftm.s.StatCheckpoint, a);
1148
1149	/* Sync state + changed memory with the standby node. */
1150	rc = ftmR3PerformSync(pVM, FTMSYNCSTATE_DELTA_VM);
1151
1152	STAM_PROFILE_STOP(&pVM->ftm.s.StatCheckpoint, a);
1153
1154	PDMCritSectLeave(&pVM->ftm.s.CritSect);
1155	pVM->ftm.s.fCheckpointingActive = false;
1156
1157	return rc;
1158	}

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/FTM.cpp@ 32190

Download in other formats: