VirtualBox

source: vbox/trunk/src/VBox/Runtime/r0drv/nt/mp-r0drv-nt.cpp

Last change on this file was 98103, checked in by vboxsync, 17 months ago

Copyright year updates by scm.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 77.2 KB
Line 
1/* $Id: mp-r0drv-nt.cpp 98103 2023-01-17 14:15:46Z vboxsync $ */
2/** @file
3 * IPRT - Multiprocessor, Ring-0 Driver, NT.
4 */
5
6/*
7 * Copyright (C) 2008-2023 Oracle and/or its affiliates.
8 *
9 * This file is part of VirtualBox base platform packages, as
10 * available from https://www.virtualbox.org.
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation, in version 3 of the
15 * License.
16 *
17 * This program is distributed in the hope that it will be useful, but
18 * WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 * General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, see <https://www.gnu.org/licenses>.
24 *
25 * The contents of this file may alternatively be used under the terms
26 * of the Common Development and Distribution License Version 1.0
27 * (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
28 * in the VirtualBox distribution, in which case the provisions of the
29 * CDDL are applicable instead of those of the GPL.
30 *
31 * You may elect to license modified versions of this file under the
32 * terms and conditions of either the GPL or the CDDL or both.
33 *
34 * SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
35 */
36
37
38/*********************************************************************************************************************************
39* Header Files *
40*********************************************************************************************************************************/
41#include "the-nt-kernel.h"
42
43#include <iprt/mp.h>
44#include <iprt/cpuset.h>
45#include <iprt/err.h>
46#include <iprt/asm.h>
47#include <iprt/log.h>
48#include <iprt/mem.h>
49#include <iprt/time.h>
50#include "r0drv/mp-r0drv.h"
51#include "symdb.h"
52#include "internal-r0drv-nt.h"
53#include "internal/mp.h"
54
55
56/*********************************************************************************************************************************
57* Structures and Typedefs *
58*********************************************************************************************************************************/
59typedef enum
60{
61 RT_NT_CPUID_SPECIFIC,
62 RT_NT_CPUID_PAIR,
63 RT_NT_CPUID_OTHERS,
64 RT_NT_CPUID_ALL
65} RT_NT_CPUID;
66
67
68/**
69 * Used by the RTMpOnSpecific.
70 */
71typedef struct RTMPNTONSPECIFICARGS
72{
73 /** Set if we're executing. */
74 bool volatile fExecuting;
75 /** Set when done executing. */
76 bool volatile fDone;
77 /** Number of references to this heap block. */
78 uint32_t volatile cRefs;
79 /** Event that the calling thread is waiting on. */
80 KEVENT DoneEvt;
81 /** The deferred procedure call object. */
82 KDPC Dpc;
83 /** The callback argument package. */
84 RTMPARGS CallbackArgs;
85} RTMPNTONSPECIFICARGS;
86/** Pointer to an argument/state structure for RTMpOnSpecific on NT. */
87typedef RTMPNTONSPECIFICARGS *PRTMPNTONSPECIFICARGS;
88
89
90/*********************************************************************************************************************************
91* Defined Constants And Macros *
92*********************************************************************************************************************************/
93/** Inactive bit for g_aidRtMpNtByCpuSetIdx. */
94#define RTMPNT_ID_F_INACTIVE RT_BIT_32(31)
95
96
97/*********************************************************************************************************************************
98* Global Variables *
99*********************************************************************************************************************************/
100/** Maximum number of processor groups. */
101uint32_t g_cRtMpNtMaxGroups;
102/** Maximum number of processors. */
103uint32_t g_cRtMpNtMaxCpus;
104/** Number of active processors. */
105uint32_t volatile g_cRtMpNtActiveCpus;
106/** The NT CPU set.
107 * KeQueryActiveProcssors() cannot be called at all IRQLs and therefore we'll
108 * have to cache it. Fortunately, NT doesn't really support taking CPUs offline,
109 * and taking them online was introduced with W2K8 where it is intended for virtual
110 * machines and not real HW. We update this, g_cRtMpNtActiveCpus and
111 * g_aidRtMpNtByCpuSetIdx from the rtR0NtMpProcessorChangeCallback.
112 */
113RTCPUSET g_rtMpNtCpuSet;
114
115/** Static per group info.
116 * @remarks With 256 groups this takes up 33KB. */
117static struct
118{
119 /** The max CPUs in the group. */
120 uint16_t cMaxCpus;
121 /** The number of active CPUs at the time of initialization. */
122 uint16_t cActiveCpus;
123 /** CPU set indexes for each CPU in the group. */
124 int16_t aidxCpuSetMembers[64];
125} g_aRtMpNtCpuGroups[256];
126/** Maps CPU set indexes to RTCPUID.
127 * Inactive CPUs has bit 31 set (RTMPNT_ID_F_INACTIVE) so we can identify them
128 * and shuffle duplicates during CPU hotplugging. We assign temporary IDs to
129 * the inactive CPUs starting at g_cRtMpNtMaxCpus - 1, ASSUMING that active
130 * CPUs has IDs from 0 to g_cRtMpNtActiveCpus. */
131RTCPUID g_aidRtMpNtByCpuSetIdx[RTCPUSET_MAX_CPUS];
132/** The handle of the rtR0NtMpProcessorChangeCallback registration. */
133static PVOID g_pvMpCpuChangeCallback = NULL;
134/** Size of the KAFFINITY_EX structure.
135 * This increased from 20 to 32 bitmap words in the 2020 H2 windows 10 release
136 * (i.e. 1280 to 2048 CPUs). We expect this to increase in the future. */
137static size_t g_cbRtMpNtKaffinityEx = RT_UOFFSETOF(KAFFINITY_EX, Bitmap)
138 + RT_SIZEOFMEMB(KAFFINITY_EX, Bitmap[0]) * 256;
139/** The size value of the KAFFINITY_EX structure. */
140static uint16_t g_cRtMpNtKaffinityExEntries = 256;
141
142
143/*********************************************************************************************************************************
144* Internal Functions *
145*********************************************************************************************************************************/
146static VOID __stdcall rtR0NtMpProcessorChangeCallback(void *pvUser, PKE_PROCESSOR_CHANGE_NOTIFY_CONTEXT pChangeCtx,
147 PNTSTATUS prcOperationStatus);
148static int rtR0NtInitQueryGroupRelations(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX **ppInfo);
149
150
151
152/**
153 * Initalizes multiprocessor globals (called by rtR0InitNative).
154 *
155 * @returns IPRT status code.
156 * @param pOsVerInfo Version information.
157 */
158DECLHIDDEN(int) rtR0MpNtInit(RTNTSDBOSVER const *pOsVerInfo)
159{
160#define MY_CHECK_BREAK(a_Check, a_DbgPrintArgs) \
161 AssertMsgBreakStmt(a_Check, a_DbgPrintArgs, DbgPrint a_DbgPrintArgs; rc = VERR_INTERNAL_ERROR_4 )
162#define MY_CHECK_RETURN(a_Check, a_DbgPrintArgs, a_rcRet) \
163 AssertMsgReturnStmt(a_Check, a_DbgPrintArgs, DbgPrint a_DbgPrintArgs, a_rcRet)
164#define MY_CHECK(a_Check, a_DbgPrintArgs) \
165 AssertMsgStmt(a_Check, a_DbgPrintArgs, DbgPrint a_DbgPrintArgs; rc = VERR_INTERNAL_ERROR_4 )
166
167 /*
168 * API combination checks.
169 */
170 MY_CHECK_RETURN(!g_pfnrtKeSetTargetProcessorDpcEx || g_pfnrtKeGetProcessorNumberFromIndex,
171 ("IPRT: Fatal: Missing KeSetTargetProcessorDpcEx without KeGetProcessorNumberFromIndex!\n"),
172 VERR_SYMBOL_NOT_FOUND);
173
174 /*
175 * Get max number of processor groups.
176 *
177 * We may need to upadjust this number below, because windows likes to keep
178 * all options open when it comes to hotplugged CPU group assignments. A
179 * server advertising up to 64 CPUs in the ACPI table will get a result of
180 * 64 from KeQueryMaximumGroupCount. That makes sense. However, when windows
181 * server 2012 does a two processor group setup for it, the sum of the
182 * GroupInfo[*].MaximumProcessorCount members below is 128. This is probably
183 * because windows doesn't want to make decisions grouping of hotpluggable CPUs.
184 * So, we need to bump the maximum count to 128 below do deal with this as we
185 * want to have valid CPU set indexes for all potential CPUs - how could we
186 * otherwise use the RTMpGetSet() result and also RTCpuSetCount(RTMpGetSet())
187 * should equal RTMpGetCount().
188 */
189 if (g_pfnrtKeQueryMaximumGroupCount)
190 {
191 g_cRtMpNtMaxGroups = g_pfnrtKeQueryMaximumGroupCount();
192 MY_CHECK_RETURN(g_cRtMpNtMaxGroups <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxGroups > 0,
193 ("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u\n", g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS),
194 VERR_MP_TOO_MANY_CPUS);
195 }
196 else
197 g_cRtMpNtMaxGroups = 1;
198
199 /*
200 * Get max number CPUs.
201 * This also defines the range of NT CPU indexes, RTCPUID and index into RTCPUSET.
202 */
203 if (g_pfnrtKeQueryMaximumProcessorCountEx)
204 {
205 g_cRtMpNtMaxCpus = g_pfnrtKeQueryMaximumProcessorCountEx(ALL_PROCESSOR_GROUPS);
206 MY_CHECK_RETURN(g_cRtMpNtMaxCpus <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxCpus > 0,
207 ("IPRT: Fatal: g_cRtMpNtMaxCpus=%u, max %u [KeQueryMaximumProcessorCountEx]\n",
208 g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS),
209 VERR_MP_TOO_MANY_CPUS);
210 }
211 else if (g_pfnrtKeQueryMaximumProcessorCount)
212 {
213 g_cRtMpNtMaxCpus = g_pfnrtKeQueryMaximumProcessorCount();
214 MY_CHECK_RETURN(g_cRtMpNtMaxCpus <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxCpus > 0,
215 ("IPRT: Fatal: g_cRtMpNtMaxCpus=%u, max %u [KeQueryMaximumProcessorCount]\n",
216 g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS),
217 VERR_MP_TOO_MANY_CPUS);
218 }
219 else if (g_pfnrtKeQueryActiveProcessors)
220 {
221 KAFFINITY fActiveProcessors = g_pfnrtKeQueryActiveProcessors();
222 MY_CHECK_RETURN(fActiveProcessors != 0,
223 ("IPRT: Fatal: KeQueryActiveProcessors returned 0!\n"),
224 VERR_INTERNAL_ERROR_2);
225 g_cRtMpNtMaxCpus = 0;
226 do
227 {
228 g_cRtMpNtMaxCpus++;
229 fActiveProcessors >>= 1;
230 } while (fActiveProcessors);
231 }
232 else
233 g_cRtMpNtMaxCpus = KeNumberProcessors;
234
235 /*
236 * Just because we're a bit paranoid about getting something wrong wrt to the
237 * kernel interfaces, we try 16 times to get the KeQueryActiveProcessorCountEx
238 * and KeQueryLogicalProcessorRelationship information to match up.
239 */
240 for (unsigned cTries = 0;; cTries++)
241 {
242 /*
243 * Get number of active CPUs.
244 */
245 if (g_pfnrtKeQueryActiveProcessorCountEx)
246 {
247 g_cRtMpNtActiveCpus = g_pfnrtKeQueryActiveProcessorCountEx(ALL_PROCESSOR_GROUPS);
248 MY_CHECK_RETURN(g_cRtMpNtActiveCpus <= g_cRtMpNtMaxCpus && g_cRtMpNtActiveCpus > 0,
249 ("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u [KeQueryActiveProcessorCountEx]\n",
250 g_cRtMpNtMaxGroups, g_cRtMpNtMaxCpus),
251 VERR_MP_TOO_MANY_CPUS);
252 }
253 else if (g_pfnrtKeQueryActiveProcessorCount)
254 {
255 g_cRtMpNtActiveCpus = g_pfnrtKeQueryActiveProcessorCount(NULL);
256 MY_CHECK_RETURN(g_cRtMpNtActiveCpus <= g_cRtMpNtMaxCpus && g_cRtMpNtActiveCpus > 0,
257 ("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u [KeQueryActiveProcessorCount]\n",
258 g_cRtMpNtMaxGroups, g_cRtMpNtMaxCpus),
259 VERR_MP_TOO_MANY_CPUS);
260 }
261 else
262 g_cRtMpNtActiveCpus = g_cRtMpNtMaxCpus;
263
264 /*
265 * Query the details for the groups to figure out which CPUs are online as
266 * well as the NT index limit.
267 */
268 for (unsigned i = 0; i < RT_ELEMENTS(g_aidRtMpNtByCpuSetIdx); i++)
269#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
270 g_aidRtMpNtByCpuSetIdx[i] = NIL_RTCPUID;
271#else
272 g_aidRtMpNtByCpuSetIdx[i] = i < g_cRtMpNtMaxCpus ? i : NIL_RTCPUID;
273#endif
274 for (unsigned idxGroup = 0; idxGroup < RT_ELEMENTS(g_aRtMpNtCpuGroups); idxGroup++)
275 {
276 g_aRtMpNtCpuGroups[idxGroup].cMaxCpus = 0;
277 g_aRtMpNtCpuGroups[idxGroup].cActiveCpus = 0;
278 for (unsigned idxMember = 0; idxMember < RT_ELEMENTS(g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers); idxMember++)
279 g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = -1;
280 }
281
282 if (g_pfnrtKeQueryLogicalProcessorRelationship)
283 {
284 MY_CHECK_RETURN(g_pfnrtKeGetProcessorIndexFromNumber,
285 ("IPRT: Fatal: Found KeQueryLogicalProcessorRelationship but not KeGetProcessorIndexFromNumber!\n"),
286 VERR_SYMBOL_NOT_FOUND);
287 MY_CHECK_RETURN(g_pfnrtKeGetProcessorNumberFromIndex,
288 ("IPRT: Fatal: Found KeQueryLogicalProcessorRelationship but not KeGetProcessorIndexFromNumber!\n"),
289 VERR_SYMBOL_NOT_FOUND);
290 MY_CHECK_RETURN(g_pfnrtKeSetTargetProcessorDpcEx,
291 ("IPRT: Fatal: Found KeQueryLogicalProcessorRelationship but not KeSetTargetProcessorDpcEx!\n"),
292 VERR_SYMBOL_NOT_FOUND);
293
294 SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *pInfo = NULL;
295 int rc = rtR0NtInitQueryGroupRelations(&pInfo);
296 if (RT_FAILURE(rc))
297 return rc;
298
299 MY_CHECK(pInfo->Group.MaximumGroupCount == g_cRtMpNtMaxGroups,
300 ("IPRT: Fatal: MaximumGroupCount=%u != g_cRtMpNtMaxGroups=%u!\n",
301 pInfo->Group.MaximumGroupCount, g_cRtMpNtMaxGroups));
302 MY_CHECK(pInfo->Group.ActiveGroupCount > 0 && pInfo->Group.ActiveGroupCount <= g_cRtMpNtMaxGroups,
303 ("IPRT: Fatal: ActiveGroupCount=%u != g_cRtMpNtMaxGroups=%u!\n",
304 pInfo->Group.ActiveGroupCount, g_cRtMpNtMaxGroups));
305
306 /*
307 * First we need to recalc g_cRtMpNtMaxCpus (see above).
308 */
309 uint32_t cMaxCpus = 0;
310 uint32_t idxGroup;
311 for (idxGroup = 0; RT_SUCCESS(rc) && idxGroup < pInfo->Group.ActiveGroupCount; idxGroup++)
312 {
313 const PROCESSOR_GROUP_INFO *pGrpInfo = &pInfo->Group.GroupInfo[idxGroup];
314 MY_CHECK_BREAK(pGrpInfo->MaximumProcessorCount <= MAXIMUM_PROC_PER_GROUP,
315 ("IPRT: Fatal: MaximumProcessorCount=%u\n", pGrpInfo->MaximumProcessorCount));
316 MY_CHECK_BREAK(pGrpInfo->ActiveProcessorCount <= pGrpInfo->MaximumProcessorCount,
317 ("IPRT: Fatal: ActiveProcessorCount=%u > MaximumProcessorCount=%u\n",
318 pGrpInfo->ActiveProcessorCount, pGrpInfo->MaximumProcessorCount));
319 cMaxCpus += pGrpInfo->MaximumProcessorCount;
320 }
321 if (cMaxCpus > g_cRtMpNtMaxCpus && RT_SUCCESS(rc))
322 {
323 DbgPrint("IPRT: g_cRtMpNtMaxCpus=%u -> %u\n", g_cRtMpNtMaxCpus, cMaxCpus);
324#ifndef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
325 uint32_t i = RT_MIN(cMaxCpus, RT_ELEMENTS(g_aidRtMpNtByCpuSetIdx));
326 while (i-- > g_cRtMpNtMaxCpus)
327 g_aidRtMpNtByCpuSetIdx[i] = i;
328#endif
329 g_cRtMpNtMaxCpus = cMaxCpus;
330 if (g_cRtMpNtMaxGroups > RTCPUSET_MAX_CPUS)
331 {
332 MY_CHECK(g_cRtMpNtMaxGroups <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxGroups > 0,
333 ("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u\n", g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS));
334 rc = VERR_MP_TOO_MANY_CPUS;
335 }
336 }
337
338 /*
339 * Calc online mask, partition IDs and such.
340 *
341 * Also check ASSUMPTIONS:
342 *
343 * 1. Processor indexes going from 0 and up to
344 * KeQueryMaximumProcessorCountEx(ALL_PROCESSOR_GROUPS) - 1.
345 *
346 * 2. Currently valid processor indexes, i.e. accepted by
347 * KeGetProcessorIndexFromNumber & KeGetProcessorNumberFromIndex, goes
348 * from 0 thru KeQueryActiveProcessorCountEx(ALL_PROCESSOR_GROUPS) - 1.
349 *
350 * 3. PROCESSOR_GROUP_INFO::MaximumProcessorCount gives the number of
351 * relevant bits in the ActiveProcessorMask (from LSB).
352 *
353 * 4. Active processor count found in KeQueryLogicalProcessorRelationship
354 * output matches what KeQueryActiveProcessorCountEx(ALL) returns.
355 *
356 * 5. Active + inactive processor counts in same does not exceed
357 * KeQueryMaximumProcessorCountEx(ALL).
358 *
359 * Note! Processor indexes are assigned as CPUs come online and are not
360 * preallocated according to group maximums. Since CPUS are only taken
361 * online and never offlined, this means that internal CPU bitmaps are
362 * never sparse and no time is wasted scanning unused bits.
363 *
364 * Unfortunately, it means that ring-3 cannot easily guess the index
365 * assignments when hotswapping is used, and must use GIP when available.
366 */
367 RTCpuSetEmpty(&g_rtMpNtCpuSet);
368 uint32_t cInactive = 0;
369 uint32_t cActive = 0;
370 uint32_t idxCpuMax = 0;
371 uint32_t idxCpuSetNextInactive = g_cRtMpNtMaxCpus - 1;
372 for (idxGroup = 0; RT_SUCCESS(rc) && idxGroup < pInfo->Group.ActiveGroupCount; idxGroup++)
373 {
374 const PROCESSOR_GROUP_INFO *pGrpInfo = &pInfo->Group.GroupInfo[idxGroup];
375 MY_CHECK_BREAK(pGrpInfo->MaximumProcessorCount <= MAXIMUM_PROC_PER_GROUP,
376 ("IPRT: Fatal: MaximumProcessorCount=%u\n", pGrpInfo->MaximumProcessorCount));
377 MY_CHECK_BREAK(pGrpInfo->ActiveProcessorCount <= pGrpInfo->MaximumProcessorCount,
378 ("IPRT: Fatal: ActiveProcessorCount=%u > MaximumProcessorCount=%u\n",
379 pGrpInfo->ActiveProcessorCount, pGrpInfo->MaximumProcessorCount));
380
381 g_aRtMpNtCpuGroups[idxGroup].cMaxCpus = pGrpInfo->MaximumProcessorCount;
382 g_aRtMpNtCpuGroups[idxGroup].cActiveCpus = pGrpInfo->ActiveProcessorCount;
383
384 for (uint32_t idxMember = 0; idxMember < pGrpInfo->MaximumProcessorCount; idxMember++)
385 {
386 PROCESSOR_NUMBER ProcNum;
387 ProcNum.Group = (USHORT)idxGroup;
388 ProcNum.Number = (UCHAR)idxMember;
389 ProcNum.Reserved = 0;
390 ULONG idxCpu = g_pfnrtKeGetProcessorIndexFromNumber(&ProcNum);
391 if (idxCpu != INVALID_PROCESSOR_INDEX)
392 {
393 MY_CHECK_BREAK(idxCpu < g_cRtMpNtMaxCpus && idxCpu < RTCPUSET_MAX_CPUS, /* ASSUMPTION #1 */
394 ("IPRT: Fatal: idxCpu=%u >= g_cRtMpNtMaxCpus=%u (RTCPUSET_MAX_CPUS=%u)\n",
395 idxCpu, g_cRtMpNtMaxCpus, RTCPUSET_MAX_CPUS));
396 if (idxCpu > idxCpuMax)
397 idxCpuMax = idxCpu;
398 g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = idxCpu;
399#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
400 g_aidRtMpNtByCpuSetIdx[idxCpu] = RTMPCPUID_FROM_GROUP_AND_NUMBER(idxGroup, idxMember);
401#endif
402
403 ProcNum.Group = UINT16_MAX;
404 ProcNum.Number = UINT8_MAX;
405 ProcNum.Reserved = UINT8_MAX;
406 NTSTATUS rcNt = g_pfnrtKeGetProcessorNumberFromIndex(idxCpu, &ProcNum);
407 MY_CHECK_BREAK(NT_SUCCESS(rcNt),
408 ("IPRT: Fatal: KeGetProcessorNumberFromIndex(%u,) -> %#x!\n", idxCpu, rcNt));
409 MY_CHECK_BREAK(ProcNum.Group == idxGroup && ProcNum.Number == idxMember,
410 ("IPRT: Fatal: KeGetProcessorXxxxFromYyyy roundtrip error for %#x! Group: %u vs %u, Number: %u vs %u\n",
411 idxCpu, ProcNum.Group, idxGroup, ProcNum.Number, idxMember));
412
413 if (pGrpInfo->ActiveProcessorMask & RT_BIT_64(idxMember))
414 {
415 RTCpuSetAddByIndex(&g_rtMpNtCpuSet, idxCpu);
416 cActive++;
417 }
418 else
419 cInactive++; /* (This is a little unexpected, but not important as long as things add up below.) */
420 }
421 else
422 {
423 /* Must be not present / inactive when KeGetProcessorIndexFromNumber fails. */
424 MY_CHECK_BREAK(!(pGrpInfo->ActiveProcessorMask & RT_BIT_64(idxMember)),
425 ("IPRT: Fatal: KeGetProcessorIndexFromNumber(%u/%u) failed but CPU is active! cMax=%u cActive=%u fActive=%p\n",
426 idxGroup, idxMember, pGrpInfo->MaximumProcessorCount, pGrpInfo->ActiveProcessorCount,
427 pGrpInfo->ActiveProcessorMask));
428 cInactive++;
429 if (idxCpuSetNextInactive >= g_cRtMpNtActiveCpus)
430 {
431 g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = idxCpuSetNextInactive;
432#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
433 g_aidRtMpNtByCpuSetIdx[idxCpuSetNextInactive] = RTMPCPUID_FROM_GROUP_AND_NUMBER(idxGroup, idxMember)
434 | RTMPNT_ID_F_INACTIVE;
435#endif
436 idxCpuSetNextInactive--;
437 }
438 }
439 }
440 }
441
442 MY_CHECK(cInactive + cActive <= g_cRtMpNtMaxCpus, /* ASSUMPTION #5 (not '==' because of inactive groups) */
443 ("IPRT: Fatal: cInactive=%u + cActive=%u > g_cRtMpNtMaxCpus=%u\n", cInactive, cActive, g_cRtMpNtMaxCpus));
444
445 /* Deal with inactive groups using KeQueryMaximumProcessorCountEx or as
446 best as we can by as best we can by stipulating maximum member counts
447 from the previous group. */
448 if ( RT_SUCCESS(rc)
449 && idxGroup < pInfo->Group.MaximumGroupCount)
450 {
451 uint16_t cInactiveLeft = g_cRtMpNtMaxCpus - (cInactive + cActive);
452 while (idxGroup < pInfo->Group.MaximumGroupCount)
453 {
454 uint32_t cMaxMembers = 0;
455 if (g_pfnrtKeQueryMaximumProcessorCountEx)
456 cMaxMembers = g_pfnrtKeQueryMaximumProcessorCountEx(idxGroup);
457 if (cMaxMembers != 0 || cInactiveLeft == 0)
458 AssertStmt(cMaxMembers <= cInactiveLeft, cMaxMembers = cInactiveLeft);
459 else
460 {
461 uint16_t cGroupsLeft = pInfo->Group.MaximumGroupCount - idxGroup;
462 cMaxMembers = pInfo->Group.GroupInfo[idxGroup - 1].MaximumProcessorCount;
463 while (cMaxMembers * cGroupsLeft < cInactiveLeft)
464 cMaxMembers++;
465 if (cMaxMembers > cInactiveLeft)
466 cMaxMembers = cInactiveLeft;
467 }
468
469 g_aRtMpNtCpuGroups[idxGroup].cMaxCpus = (uint16_t)cMaxMembers;
470 g_aRtMpNtCpuGroups[idxGroup].cActiveCpus = 0;
471 for (uint16_t idxMember = 0; idxMember < cMaxMembers; idxMember++)
472 if (idxCpuSetNextInactive >= g_cRtMpNtActiveCpus)
473 {
474 g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = idxCpuSetNextInactive;
475#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
476 g_aidRtMpNtByCpuSetIdx[idxCpuSetNextInactive] = RTMPCPUID_FROM_GROUP_AND_NUMBER(idxGroup, idxMember)
477 | RTMPNT_ID_F_INACTIVE;
478#endif
479 idxCpuSetNextInactive--;
480 }
481 cInactiveLeft -= cMaxMembers;
482 idxGroup++;
483 }
484 }
485
486 /* We're done with pInfo now, free it so we can start returning when assertions fail. */
487 RTMemFree(pInfo);
488 if (RT_FAILURE(rc)) /* MY_CHECK_BREAK sets rc. */
489 return rc;
490 MY_CHECK_RETURN(cActive >= g_cRtMpNtActiveCpus,
491 ("IPRT: Fatal: cActive=%u < g_cRtMpNtActiveCpus=%u - CPUs removed?\n", cActive, g_cRtMpNtActiveCpus),
492 VERR_INTERNAL_ERROR_3);
493 MY_CHECK_RETURN(idxCpuMax < cActive, /* ASSUMPTION #2 */
494 ("IPRT: Fatal: idCpuMax=%u >= cActive=%u! Unexpected CPU index allocation. CPUs removed?\n",
495 idxCpuMax, cActive),
496 VERR_INTERNAL_ERROR_4);
497
498 /* Retry if CPUs were added. */
499 if ( cActive != g_cRtMpNtActiveCpus
500 && cTries < 16)
501 continue;
502 MY_CHECK_RETURN(cActive == g_cRtMpNtActiveCpus, /* ASSUMPTION #4 */
503 ("IPRT: Fatal: cActive=%u != g_cRtMpNtActiveCpus=%u\n", cActive, g_cRtMpNtActiveCpus),
504 VERR_INTERNAL_ERROR_5);
505 }
506 else
507 {
508 /* Legacy: */
509 MY_CHECK_RETURN(g_cRtMpNtMaxGroups == 1, ("IPRT: Fatal: Missing KeQueryLogicalProcessorRelationship!\n"),
510 VERR_SYMBOL_NOT_FOUND);
511
512 /** @todo Is it possible that the affinity mask returned by
513 * KeQueryActiveProcessors is sparse? */
514 if (g_pfnrtKeQueryActiveProcessors)
515 RTCpuSetFromU64(&g_rtMpNtCpuSet, g_pfnrtKeQueryActiveProcessors());
516 else if (g_cRtMpNtMaxCpus < 64)
517 RTCpuSetFromU64(&g_rtMpNtCpuSet, (UINT64_C(1) << g_cRtMpNtMaxCpus) - 1);
518 else
519 {
520 MY_CHECK_RETURN(g_cRtMpNtMaxCpus == 64, ("IPRT: Fatal: g_cRtMpNtMaxCpus=%u, expect 64 or less\n", g_cRtMpNtMaxCpus),
521 VERR_MP_TOO_MANY_CPUS);
522 RTCpuSetFromU64(&g_rtMpNtCpuSet, UINT64_MAX);
523 }
524
525 g_aRtMpNtCpuGroups[0].cMaxCpus = g_cRtMpNtMaxCpus;
526 g_aRtMpNtCpuGroups[0].cActiveCpus = g_cRtMpNtMaxCpus;
527 for (unsigned i = 0; i < g_cRtMpNtMaxCpus; i++)
528 {
529 g_aRtMpNtCpuGroups[0].aidxCpuSetMembers[i] = i;
530#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
531 g_aidRtMpNtByCpuSetIdx[i] = RTMPCPUID_FROM_GROUP_AND_NUMBER(0, i);
532#endif
533 }
534 }
535
536 /*
537 * Register CPU hot plugging callback (it also counts active CPUs).
538 */
539 Assert(g_pvMpCpuChangeCallback == NULL);
540 if (g_pfnrtKeRegisterProcessorChangeCallback)
541 {
542 MY_CHECK_RETURN(g_pfnrtKeDeregisterProcessorChangeCallback,
543 ("IPRT: Fatal: KeRegisterProcessorChangeCallback without KeDeregisterProcessorChangeCallback!\n"),
544 VERR_SYMBOL_NOT_FOUND);
545
546 RTCPUSET const ActiveSetCopy = g_rtMpNtCpuSet;
547 RTCpuSetEmpty(&g_rtMpNtCpuSet);
548 uint32_t const cActiveCpus = g_cRtMpNtActiveCpus;
549 g_cRtMpNtActiveCpus = 0;
550
551 g_pvMpCpuChangeCallback = g_pfnrtKeRegisterProcessorChangeCallback(rtR0NtMpProcessorChangeCallback, NULL /*pvUser*/,
552 KE_PROCESSOR_CHANGE_ADD_EXISTING);
553 if (g_pvMpCpuChangeCallback)
554 {
555 if (cActiveCpus == g_cRtMpNtActiveCpus)
556 { /* likely */ }
557 else
558 {
559 g_pfnrtKeDeregisterProcessorChangeCallback(g_pvMpCpuChangeCallback);
560 if (cTries < 16)
561 {
562 /* Retry if CPUs were added. */
563 MY_CHECK_RETURN(g_cRtMpNtActiveCpus >= cActiveCpus,
564 ("IPRT: Fatal: g_cRtMpNtActiveCpus=%u < cActiveCpus=%u! CPUs removed?\n",
565 g_cRtMpNtActiveCpus, cActiveCpus),
566 VERR_INTERNAL_ERROR_2);
567 MY_CHECK_RETURN(g_cRtMpNtActiveCpus <= g_cRtMpNtMaxCpus,
568 ("IPRT: Fatal: g_cRtMpNtActiveCpus=%u > g_cRtMpNtMaxCpus=%u!\n",
569 g_cRtMpNtActiveCpus, g_cRtMpNtMaxCpus),
570 VERR_INTERNAL_ERROR_2);
571 continue;
572 }
573 MY_CHECK_RETURN(0, ("IPRT: Fatal: g_cRtMpNtActiveCpus=%u cActiveCpus=%u\n", g_cRtMpNtActiveCpus, cActiveCpus),
574 VERR_INTERNAL_ERROR_3);
575 }
576 }
577 else
578 {
579 AssertFailed();
580 g_rtMpNtCpuSet = ActiveSetCopy;
581 g_cRtMpNtActiveCpus = cActiveCpus;
582 }
583 }
584 break;
585 } /* Retry loop for stable active CPU count. */
586
587#undef MY_CHECK_RETURN
588
589 /*
590 * Special IPI fun for RTMpPokeCpu.
591 *
592 * On Vista and later the DPC method doesn't seem to reliably send IPIs,
593 * so we have to use alternative methods.
594 *
595 * On AMD64 We used to use the HalSendSoftwareInterrupt API (also x86 on
596 * W10+), it looks faster and more convenient to use, however we're either
597 * using it wrong or it doesn't reliably do what we want (see @bugref{8343}).
598 *
599 * The HalRequestIpip API is thus far the only alternative to KeInsertQueueDpc
600 * for doing targetted IPIs. Trouble with this API is that it changed
601 * fundamentally in Window 7 when they added support for lots of processors.
602 *
603 * If we really think we cannot use KeInsertQueueDpc, we use the broadcast IPI
604 * API KeIpiGenericCall.
605 */
606 if ( pOsVerInfo->uMajorVer > 6
607 || (pOsVerInfo->uMajorVer == 6 && pOsVerInfo->uMinorVer > 0))
608 g_pfnrtHalRequestIpiPreW7 = NULL;
609 else
610 g_pfnrtHalRequestIpiW7Plus = NULL;
611
612 if ( g_pfnrtHalRequestIpiW7Plus
613 && g_pfnrtKeInitializeAffinityEx
614 && g_pfnrtKeAddProcessorAffinityEx
615 && g_pfnrtKeGetProcessorIndexFromNumber)
616 {
617 /* Determine the real size of the KAFFINITY_EX structure. */
618 size_t const cbAffinity = _8K;
619 PKAFFINITY_EX pAffinity = (PKAFFINITY_EX)RTMemAllocZ(cbAffinity);
620 AssertReturn(pAffinity, VERR_NO_MEMORY);
621 size_t const cMaxEntries = (cbAffinity - RT_UOFFSETOF(KAFFINITY_EX, Bitmap[0])) / sizeof(pAffinity->Bitmap[0]);
622 g_pfnrtKeInitializeAffinityEx(pAffinity);
623 if (pAffinity->Size > 1 && pAffinity->Size <= cMaxEntries)
624 {
625 g_cRtMpNtKaffinityExEntries = pAffinity->Size;
626 g_cbRtMpNtKaffinityEx = pAffinity->Size * sizeof(pAffinity->Bitmap[0]) + RT_UOFFSETOF(KAFFINITY_EX, Bitmap[0]);
627 g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingHalRequestIpiW7Plus;
628 RTMemFree(pAffinity);
629 DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingHalRequestIpiW7Plus\n");
630 return VINF_SUCCESS;
631 }
632 DbgPrint("IPRT: RTMpPoke can't use rtMpPokeCpuUsingHalRequestIpiW7Plus! pAffinity->Size=%u\n", pAffinity->Size);
633 AssertReleaseMsg(pAffinity->Size <= cMaxEntries, ("%#x\n", pAffinity->Size)); /* stack is toast if larger (32768 CPUs). */
634 RTMemFree(pAffinity);
635 }
636
637 if (pOsVerInfo->uMajorVer >= 6 && g_pfnrtKeIpiGenericCall)
638 {
639 DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingBroadcastIpi\n");
640 g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingBroadcastIpi;
641 }
642 else if (g_pfnrtKeSetTargetProcessorDpc)
643 {
644 DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingDpc\n");
645 g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingDpc;
646 /* Windows XP should send always send an IPI -> VERIFY */
647 }
648 else
649 {
650 DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingFailureNotSupported\n");
651 Assert(pOsVerInfo->uMajorVer == 3 && pOsVerInfo->uMinorVer <= 50);
652 g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingFailureNotSupported;
653 }
654
655 return VINF_SUCCESS;
656}
657
658
659/**
660 * Called by rtR0TermNative.
661 */
662DECLHIDDEN(void) rtR0MpNtTerm(void)
663{
664 /*
665 * Deregister the processor change callback.
666 */
667 PVOID pvMpCpuChangeCallback = g_pvMpCpuChangeCallback;
668 g_pvMpCpuChangeCallback = NULL;
669 if (pvMpCpuChangeCallback)
670 {
671 AssertReturnVoid(g_pfnrtKeDeregisterProcessorChangeCallback);
672 g_pfnrtKeDeregisterProcessorChangeCallback(pvMpCpuChangeCallback);
673 }
674}
675
676
677DECLHIDDEN(int) rtR0MpNotificationNativeInit(void)
678{
679 return VINF_SUCCESS;
680}
681
682
683DECLHIDDEN(void) rtR0MpNotificationNativeTerm(void)
684{
685}
686
687
688/**
689 * Implements the NT PROCESSOR_CALLBACK_FUNCTION callback function.
690 *
691 * This maintains the g_rtMpNtCpuSet and works MP notification callbacks. When
692 * registered, it's called for each active CPU in the system, avoiding racing
693 * CPU hotplugging (as well as testing the callback).
694 *
695 * @param pvUser User context (not used).
696 * @param pChangeCtx Change context (in).
697 * @param prcOperationStatus Operation status (in/out).
698 *
699 * @remarks ASSUMES no concurrent execution of KeProcessorAddCompleteNotify
700 * notification callbacks. At least during callback registration
701 * callout, we're owning KiDynamicProcessorLock.
702 *
703 * @remarks When registering the handler, we first get KeProcessorAddStartNotify
704 * callbacks for all active CPUs, and after they all succeed we get the
705 * KeProcessorAddCompleteNotify callbacks.
706 */
707static VOID __stdcall rtR0NtMpProcessorChangeCallback(void *pvUser, PKE_PROCESSOR_CHANGE_NOTIFY_CONTEXT pChangeCtx,
708 PNTSTATUS prcOperationStatus)
709{
710 RT_NOREF(pvUser, prcOperationStatus);
711 switch (pChangeCtx->State)
712 {
713 /*
714 * Check whether we can deal with the CPU, failing the start operation if we
715 * can't. The checks we are doing here are to avoid complicated/impossible
716 * cases in KeProcessorAddCompleteNotify. They are really just verify specs.
717 */
718 case KeProcessorAddStartNotify:
719 {
720 NTSTATUS rcNt = STATUS_SUCCESS;
721 if (pChangeCtx->NtNumber < RTCPUSET_MAX_CPUS)
722 {
723 if (pChangeCtx->NtNumber >= g_cRtMpNtMaxCpus)
724 {
725 DbgPrint("IPRT: KeProcessorAddStartNotify failure: NtNumber=%u is higher than the max CPU count (%u)!\n",
726 pChangeCtx->NtNumber, g_cRtMpNtMaxCpus);
727 rcNt = STATUS_INTERNAL_ERROR;
728 }
729
730 /* The ProcessNumber field was introduced in Windows 7. */
731 PROCESSOR_NUMBER ProcNum;
732 if (g_pfnrtKeGetProcessorIndexFromNumber)
733 {
734 ProcNum = pChangeCtx->ProcNumber;
735 KEPROCESSORINDEX idxCpu = g_pfnrtKeGetProcessorIndexFromNumber(&ProcNum);
736 if (idxCpu != pChangeCtx->NtNumber)
737 {
738 DbgPrint("IPRT: KeProcessorAddStartNotify failure: g_pfnrtKeGetProcessorIndexFromNumber(%u.%u) -> %u, expected %u!\n",
739 ProcNum.Group, ProcNum.Number, idxCpu, pChangeCtx->NtNumber);
740 rcNt = STATUS_INTERNAL_ERROR;
741 }
742 }
743 else
744 {
745 ProcNum.Group = 0;
746 ProcNum.Number = pChangeCtx->NtNumber;
747 }
748
749 if ( ProcNum.Group < RT_ELEMENTS(g_aRtMpNtCpuGroups)
750 && ProcNum.Number < RT_ELEMENTS(g_aRtMpNtCpuGroups[0].aidxCpuSetMembers))
751 {
752 if (ProcNum.Group >= g_cRtMpNtMaxGroups)
753 {
754 DbgPrint("IPRT: KeProcessorAddStartNotify failure: %u.%u is out of range - max groups: %u!\n",
755 ProcNum.Group, ProcNum.Number, g_cRtMpNtMaxGroups);
756 rcNt = STATUS_INTERNAL_ERROR;
757 }
758
759 if (ProcNum.Number < g_aRtMpNtCpuGroups[ProcNum.Group].cMaxCpus)
760 {
761 Assert(g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] != -1);
762 if (g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] == -1)
763 {
764 DbgPrint("IPRT: KeProcessorAddStartNotify failure: Internal error! %u.%u was assigned -1 as set index!\n",
765 ProcNum.Group, ProcNum.Number);
766 rcNt = STATUS_INTERNAL_ERROR;
767 }
768
769 Assert(g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] != NIL_RTCPUID);
770 if (g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] == NIL_RTCPUID)
771 {
772 DbgPrint("IPRT: KeProcessorAddStartNotify failure: Internal error! %u (%u.%u) translates to NIL_RTCPUID!\n",
773 pChangeCtx->NtNumber, ProcNum.Group, ProcNum.Number);
774 rcNt = STATUS_INTERNAL_ERROR;
775 }
776 }
777 else
778 {
779 DbgPrint("IPRT: KeProcessorAddStartNotify failure: max processors in group %u is %u, cannot add %u to it!\n",
780 ProcNum.Group, g_aRtMpNtCpuGroups[ProcNum.Group].cMaxCpus, ProcNum.Group, ProcNum.Number);
781 rcNt = STATUS_INTERNAL_ERROR;
782 }
783 }
784 else
785 {
786 DbgPrint("IPRT: KeProcessorAddStartNotify failure: %u.%u is out of range (max %u.%u)!\n",
787 ProcNum.Group, ProcNum.Number, RT_ELEMENTS(g_aRtMpNtCpuGroups), RT_ELEMENTS(g_aRtMpNtCpuGroups[0].aidxCpuSetMembers));
788 rcNt = STATUS_INTERNAL_ERROR;
789 }
790 }
791 else
792 {
793 DbgPrint("IPRT: KeProcessorAddStartNotify failure: NtNumber=%u is outside RTCPUSET_MAX_CPUS (%u)!\n",
794 pChangeCtx->NtNumber, RTCPUSET_MAX_CPUS);
795 rcNt = STATUS_INTERNAL_ERROR;
796 }
797 if (!NT_SUCCESS(rcNt))
798 *prcOperationStatus = rcNt;
799 break;
800 }
801
802 /*
803 * Update the globals. Since we've checked out range limits and other
804 * limitations already we just AssertBreak here.
805 */
806 case KeProcessorAddCompleteNotify:
807 {
808 /*
809 * Calc the processor number and assert conditions checked in KeProcessorAddStartNotify.
810 */
811 AssertBreak(pChangeCtx->NtNumber < RTCPUSET_MAX_CPUS);
812 AssertBreak(pChangeCtx->NtNumber < g_cRtMpNtMaxCpus);
813 Assert(pChangeCtx->NtNumber == g_cRtMpNtActiveCpus); /* light assumption */
814 PROCESSOR_NUMBER ProcNum;
815 if (g_pfnrtKeGetProcessorIndexFromNumber)
816 {
817 ProcNum = pChangeCtx->ProcNumber;
818 AssertBreak(g_pfnrtKeGetProcessorIndexFromNumber(&ProcNum) == pChangeCtx->NtNumber);
819 AssertBreak(ProcNum.Group < RT_ELEMENTS(g_aRtMpNtCpuGroups));
820 AssertBreak(ProcNum.Group < g_cRtMpNtMaxGroups);
821 }
822 else
823 {
824 ProcNum.Group = 0;
825 ProcNum.Number = pChangeCtx->NtNumber;
826 }
827 AssertBreak(ProcNum.Number < RT_ELEMENTS(g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers));
828 AssertBreak(ProcNum.Number < g_aRtMpNtCpuGroups[ProcNum.Group].cMaxCpus);
829 AssertBreak(g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] != -1);
830 AssertBreak(g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] != NIL_RTCPUID);
831
832 /*
833 * Add ourselves to the online CPU set and update the active CPU count.
834 */
835 RTCpuSetAddByIndex(&g_rtMpNtCpuSet, pChangeCtx->NtNumber);
836 ASMAtomicIncU32(&g_cRtMpNtActiveCpus);
837
838 /*
839 * Update the group info.
840 *
841 * If the index prediction failed (real hotplugging callbacks only) we
842 * have to switch it around. This is particularly annoying when we
843 * use the index as the ID.
844 */
845#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
846 RTCPUID idCpu = RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number);
847 RTCPUID idOld = g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber];
848 if ((idOld & ~RTMPNT_ID_F_INACTIVE) != idCpu)
849 {
850 Assert(idOld & RTMPNT_ID_F_INACTIVE);
851 int idxDest = g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number];
852 g_aRtMpNtCpuGroups[rtMpCpuIdGetGroup(idOld)].aidxCpuSetMembers[rtMpCpuIdGetGroupMember(idOld)] = idxDest;
853 g_aidRtMpNtByCpuSetIdx[idxDest] = idOld;
854 }
855 g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] = idCpu;
856#else
857 Assert(g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] == pChangeCtx->NtNumber);
858 int idxDest = g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number];
859 if ((ULONG)idxDest != pChangeCtx->NtNumber)
860 {
861 bool fFound = false;
862 uint32_t idxOldGroup = g_cRtMpNtMaxGroups;
863 while (idxOldGroup-- > 0 && !fFound)
864 {
865 uint32_t idxMember = g_aRtMpNtCpuGroups[idxOldGroup].cMaxCpus;
866 while (idxMember-- > 0)
867 if (g_aRtMpNtCpuGroups[idxOldGroup].aidxCpuSetMembers[idxMember] == (int)pChangeCtx->NtNumber)
868 {
869 g_aRtMpNtCpuGroups[idxOldGroup].aidxCpuSetMembers[idxMember] = idxDest;
870 fFound = true;
871 break;
872 }
873 }
874 Assert(fFound);
875 }
876#endif
877 g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] = pChangeCtx->NtNumber;
878
879 /*
880 * Do MP notification callbacks.
881 */
882 rtMpNotificationDoCallbacks(RTMPEVENT_ONLINE, pChangeCtx->NtNumber);
883 break;
884 }
885
886 case KeProcessorAddFailureNotify:
887 /* ignore */
888 break;
889
890 default:
891 AssertMsgFailed(("State=%u\n", pChangeCtx->State));
892 }
893}
894
895
896/**
897 * Wrapper around KeQueryLogicalProcessorRelationship.
898 *
899 * @returns IPRT status code.
900 * @param ppInfo Where to return the info. Pass to RTMemFree when done.
901 */
902static int rtR0NtInitQueryGroupRelations(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX **ppInfo)
903{
904 ULONG cbInfo = sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX)
905 + g_cRtMpNtMaxGroups * sizeof(GROUP_RELATIONSHIP);
906 NTSTATUS rcNt;
907 do
908 {
909 SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *pInfo = (SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *)RTMemAlloc(cbInfo);
910 if (pInfo)
911 {
912 rcNt = g_pfnrtKeQueryLogicalProcessorRelationship(NULL /*pProcNumber*/, RelationGroup, pInfo, &cbInfo);
913 if (NT_SUCCESS(rcNt))
914 {
915 *ppInfo = pInfo;
916 return VINF_SUCCESS;
917 }
918
919 RTMemFree(pInfo);
920 pInfo = NULL;
921 }
922 else
923 rcNt = STATUS_NO_MEMORY;
924 } while (rcNt == STATUS_INFO_LENGTH_MISMATCH);
925 DbgPrint("IPRT: Fatal: KeQueryLogicalProcessorRelationship failed: %#x\n", rcNt);
926 AssertMsgFailed(("KeQueryLogicalProcessorRelationship failed: %#x\n", rcNt));
927 return RTErrConvertFromNtStatus(rcNt);
928}
929
930
931
932
933
934RTDECL(RTCPUID) RTMpCpuId(void)
935{
936 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
937
938#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
939 PROCESSOR_NUMBER ProcNum;
940 ProcNum.Group = 0;
941 if (g_pfnrtKeGetCurrentProcessorNumberEx)
942 {
943 ProcNum.Number = 0;
944 g_pfnrtKeGetCurrentProcessorNumberEx(&ProcNum);
945 }
946 else
947 ProcNum.Number = KeGetCurrentProcessorNumber(); /* Number is 8-bit, so we're not subject to BYTE -> WORD upgrade in WDK. */
948 return RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number);
949
950#else
951
952 if (g_pfnrtKeGetCurrentProcessorNumberEx)
953 {
954 KEPROCESSORINDEX idxCpu = g_pfnrtKeGetCurrentProcessorNumberEx(NULL);
955 Assert(idxCpu < RTCPUSET_MAX_CPUS);
956 return idxCpu;
957 }
958
959 return (uint8_t)KeGetCurrentProcessorNumber(); /* PCR->Number was changed from BYTE to WORD in the WDK, thus the cast. */
960#endif
961}
962
963
964RTDECL(int) RTMpCurSetIndex(void)
965{
966#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
967 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
968
969 if (g_pfnrtKeGetCurrentProcessorNumberEx)
970 {
971 KEPROCESSORINDEX idxCpu = g_pfnrtKeGetCurrentProcessorNumberEx(NULL);
972 Assert(idxCpu < RTCPUSET_MAX_CPUS);
973 return idxCpu;
974 }
975 return (uint8_t)KeGetCurrentProcessorNumber(); /* PCR->Number was changed from BYTE to WORD in the WDK, thus the cast. */
976#else
977 return (int)RTMpCpuId();
978#endif
979}
980
981
982RTDECL(int) RTMpCurSetIndexAndId(PRTCPUID pidCpu)
983{
984#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
985 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
986
987 PROCESSOR_NUMBER ProcNum = { 0 , 0, 0 };
988 KEPROCESSORINDEX idxCpu = g_pfnrtKeGetCurrentProcessorNumberEx(&ProcNum);
989 Assert(idxCpu < RTCPUSET_MAX_CPUS);
990 *pidCpu = RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number);
991 return idxCpu;
992#else
993 return *pidCpu = RTMpCpuId();
994#endif
995}
996
997
998RTDECL(int) RTMpCpuIdToSetIndex(RTCPUID idCpu)
999{
1000#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
1001 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1002
1003 if (idCpu != NIL_RTCPUID)
1004 {
1005 if (g_pfnrtKeGetProcessorIndexFromNumber)
1006 {
1007 PROCESSOR_NUMBER ProcNum;
1008 ProcNum.Group = rtMpCpuIdGetGroup(idCpu);
1009 ProcNum.Number = rtMpCpuIdGetGroupMember(idCpu);
1010 ProcNum.Reserved = 0;
1011 KEPROCESSORINDEX idxCpu = g_pfnrtKeGetProcessorIndexFromNumber(&ProcNum);
1012 if (idxCpu != INVALID_PROCESSOR_INDEX)
1013 {
1014 Assert(idxCpu < g_cRtMpNtMaxCpus);
1015 Assert((ULONG)g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] == idxCpu);
1016 return idxCpu;
1017 }
1018
1019 /* Since NT assigned indexes as the CPUs come online, we cannot produce an ID <-> index
1020 mapping for not-yet-onlined CPUS that is consistent. We just have to do our best... */
1021 if ( ProcNum.Group < g_cRtMpNtMaxGroups
1022 && ProcNum.Number < g_aRtMpNtCpuGroups[ProcNum.Group].cMaxCpus)
1023 return g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number];
1024 }
1025 else if (rtMpCpuIdGetGroup(idCpu) == 0)
1026 return rtMpCpuIdGetGroupMember(idCpu);
1027 }
1028 return -1;
1029#else
1030 /* 1:1 mapping, just do range checks. */
1031 return idCpu < RTCPUSET_MAX_CPUS ? (int)idCpu : -1;
1032#endif
1033}
1034
1035
1036RTDECL(RTCPUID) RTMpCpuIdFromSetIndex(int iCpu)
1037{
1038#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
1039 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1040
1041 if ((unsigned)iCpu < g_cRtMpNtMaxCpus)
1042 {
1043 if (g_pfnrtKeGetProcessorIndexFromNumber)
1044 {
1045 PROCESSOR_NUMBER ProcNum = { 0, 0, 0 };
1046 NTSTATUS rcNt = g_pfnrtKeGetProcessorNumberFromIndex(iCpu, &ProcNum);
1047 if (NT_SUCCESS(rcNt))
1048 {
1049 Assert(ProcNum.Group <= g_cRtMpNtMaxGroups);
1050 Assert( (g_aidRtMpNtByCpuSetIdx[iCpu] & ~RTMPNT_ID_F_INACTIVE)
1051 == RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number));
1052 return RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number);
1053 }
1054 }
1055 return g_aidRtMpNtByCpuSetIdx[iCpu];
1056 }
1057 return NIL_RTCPUID;
1058#else
1059 /* 1:1 mapping, just do range checks. */
1060 return (unsigned)iCpu < RTCPUSET_MAX_CPUS ? iCpu : NIL_RTCPUID;
1061#endif
1062}
1063
1064
1065RTDECL(int) RTMpSetIndexFromCpuGroupMember(uint32_t idxGroup, uint32_t idxMember)
1066{
1067 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1068
1069 if (idxGroup < g_cRtMpNtMaxGroups)
1070 if (idxMember < g_aRtMpNtCpuGroups[idxGroup].cMaxCpus)
1071 return g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember];
1072 return -1;
1073}
1074
1075
1076RTDECL(uint32_t) RTMpGetCpuGroupCounts(uint32_t idxGroup, uint32_t *pcActive)
1077{
1078 if (idxGroup < g_cRtMpNtMaxGroups)
1079 {
1080 if (pcActive)
1081 *pcActive = g_aRtMpNtCpuGroups[idxGroup].cActiveCpus;
1082 return g_aRtMpNtCpuGroups[idxGroup].cMaxCpus;
1083 }
1084 if (pcActive)
1085 *pcActive = 0;
1086 return 0;
1087}
1088
1089
1090RTDECL(uint32_t) RTMpGetMaxCpuGroupCount(void)
1091{
1092 return g_cRtMpNtMaxGroups;
1093}
1094
1095
1096RTDECL(RTCPUID) RTMpGetMaxCpuId(void)
1097{
1098 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1099
1100#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
1101 return RTMPCPUID_FROM_GROUP_AND_NUMBER(g_cRtMpNtMaxGroups - 1, g_aRtMpNtCpuGroups[g_cRtMpNtMaxGroups - 1].cMaxCpus - 1);
1102#else
1103 /* According to MSDN the processor indexes goes from 0 to the maximum
1104 number of CPUs in the system. We've check this in initterm-r0drv-nt.cpp. */
1105 return g_cRtMpNtMaxCpus - 1;
1106#endif
1107}
1108
1109
1110RTDECL(bool) RTMpIsCpuOnline(RTCPUID idCpu)
1111{
1112 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1113 return RTCpuSetIsMember(&g_rtMpNtCpuSet, idCpu);
1114}
1115
1116
1117RTDECL(bool) RTMpIsCpuPossible(RTCPUID idCpu)
1118{
1119 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1120
1121#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
1122 if (idCpu != NIL_RTCPUID)
1123 {
1124 unsigned idxGroup = rtMpCpuIdGetGroup(idCpu);
1125 if (idxGroup < g_cRtMpNtMaxGroups)
1126 return rtMpCpuIdGetGroupMember(idCpu) < g_aRtMpNtCpuGroups[idxGroup].cMaxCpus;
1127 }
1128 return false;
1129
1130#else
1131 /* A possible CPU ID is one with a value lower than g_cRtMpNtMaxCpus (see
1132 comment in RTMpGetMaxCpuId). */
1133 return idCpu < g_cRtMpNtMaxCpus;
1134#endif
1135}
1136
1137
1138
1139RTDECL(PRTCPUSET) RTMpGetSet(PRTCPUSET pSet)
1140{
1141 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1142
1143 /* The set of possible CPU IDs(/indexes) are from 0 up to
1144 g_cRtMpNtMaxCpus (see comment in RTMpGetMaxCpuId). */
1145 RTCpuSetEmpty(pSet);
1146 int idxCpu = g_cRtMpNtMaxCpus;
1147 while (idxCpu-- > 0)
1148 RTCpuSetAddByIndex(pSet, idxCpu);
1149 return pSet;
1150}
1151
1152
1153RTDECL(RTCPUID) RTMpGetCount(void)
1154{
1155 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1156 return g_cRtMpNtMaxCpus;
1157}
1158
1159
1160RTDECL(PRTCPUSET) RTMpGetOnlineSet(PRTCPUSET pSet)
1161{
1162 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1163
1164 *pSet = g_rtMpNtCpuSet;
1165 return pSet;
1166}
1167
1168
1169RTDECL(RTCPUID) RTMpGetOnlineCount(void)
1170{
1171 RTCPUSET Set;
1172 RTMpGetOnlineSet(&Set);
1173 return RTCpuSetCount(&Set);
1174}
1175
1176
1177RTDECL(RTCPUID) RTMpGetOnlineCoreCount(void)
1178{
1179 /** @todo fix me */
1180 return RTMpGetOnlineCount();
1181}
1182
1183
1184
1185#if 0
1186/* Experiment with checking the undocumented KPRCB structure
1187 * 'dt nt!_kprcb 0xaddress' shows the layout
1188 */
1189typedef struct
1190{
1191 LIST_ENTRY DpcListHead;
1192 ULONG_PTR DpcLock;
1193 volatile ULONG DpcQueueDepth;
1194 ULONG DpcQueueCount;
1195} KDPC_DATA, *PKDPC_DATA;
1196
1197RTDECL(bool) RTMpIsCpuWorkPending(void)
1198{
1199 uint8_t *pkprcb;
1200 PKDPC_DATA pDpcData;
1201
1202 _asm {
1203 mov eax, fs:0x20
1204 mov pkprcb, eax
1205 }
1206 pDpcData = (PKDPC_DATA)(pkprcb + 0x19e0);
1207 if (pDpcData->DpcQueueDepth)
1208 return true;
1209
1210 pDpcData++;
1211 if (pDpcData->DpcQueueDepth)
1212 return true;
1213 return false;
1214}
1215#else
1216RTDECL(bool) RTMpIsCpuWorkPending(void)
1217{
1218 /** @todo not implemented */
1219 return false;
1220}
1221#endif
1222
1223
1224/**
1225 * Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for
1226 * the RTMpOnAll case.
1227 *
1228 * @param uUserCtx The user context argument (PRTMPARGS).
1229 */
1230static ULONG_PTR rtmpNtOnAllBroadcastIpiWrapper(ULONG_PTR uUserCtx)
1231{
1232 PRTMPARGS pArgs = (PRTMPARGS)uUserCtx;
1233 /*ASMAtomicIncU32(&pArgs->cHits); - not needed */
1234 pArgs->pfnWorker(RTMpCpuId(), pArgs->pvUser1, pArgs->pvUser2);
1235 return 0;
1236}
1237
1238
1239/**
1240 * Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for
1241 * the RTMpOnOthers case.
1242 *
1243 * @param uUserCtx The user context argument (PRTMPARGS).
1244 */
1245static ULONG_PTR rtmpNtOnOthersBroadcastIpiWrapper(ULONG_PTR uUserCtx)
1246{
1247 PRTMPARGS pArgs = (PRTMPARGS)uUserCtx;
1248 RTCPUID idCpu = RTMpCpuId();
1249 if (pArgs->idCpu != idCpu)
1250 {
1251 /*ASMAtomicIncU32(&pArgs->cHits); - not needed */
1252 pArgs->pfnWorker(idCpu, pArgs->pvUser1, pArgs->pvUser2);
1253 }
1254 return 0;
1255}
1256
1257
1258/**
1259 * Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for
1260 * the RTMpOnPair case.
1261 *
1262 * @param uUserCtx The user context argument (PRTMPARGS).
1263 */
1264static ULONG_PTR rtmpNtOnPairBroadcastIpiWrapper(ULONG_PTR uUserCtx)
1265{
1266 PRTMPARGS pArgs = (PRTMPARGS)uUserCtx;
1267 RTCPUID idCpu = RTMpCpuId();
1268 if ( pArgs->idCpu == idCpu
1269 || pArgs->idCpu2 == idCpu)
1270 {
1271 ASMAtomicIncU32(&pArgs->cHits);
1272 pArgs->pfnWorker(idCpu, pArgs->pvUser1, pArgs->pvUser2);
1273 }
1274 return 0;
1275}
1276
1277
1278/**
1279 * Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for
1280 * the RTMpOnSpecific case.
1281 *
1282 * @param uUserCtx The user context argument (PRTMPARGS).
1283 */
1284static ULONG_PTR rtmpNtOnSpecificBroadcastIpiWrapper(ULONG_PTR uUserCtx)
1285{
1286 PRTMPARGS pArgs = (PRTMPARGS)uUserCtx;
1287 RTCPUID idCpu = RTMpCpuId();
1288 if (pArgs->idCpu == idCpu)
1289 {
1290 ASMAtomicIncU32(&pArgs->cHits);
1291 pArgs->pfnWorker(idCpu, pArgs->pvUser1, pArgs->pvUser2);
1292 }
1293 return 0;
1294}
1295
1296
1297/**
1298 * Internal worker for the RTMpOn* APIs using KeIpiGenericCall.
1299 *
1300 * @returns VINF_SUCCESS.
1301 * @param pfnWorker The callback.
1302 * @param pvUser1 User argument 1.
1303 * @param pvUser2 User argument 2.
1304 * @param pfnNativeWrapper The wrapper between the NT and IPRT callbacks.
1305 * @param idCpu First CPU to match, ultimately specific to the
1306 * pfnNativeWrapper used.
1307 * @param idCpu2 Second CPU to match, ultimately specific to the
1308 * pfnNativeWrapper used.
1309 * @param pcHits Where to return the number of this. Optional.
1310 */
1311static int rtMpCallUsingBroadcastIpi(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2,
1312 PKIPI_BROADCAST_WORKER pfnNativeWrapper, RTCPUID idCpu, RTCPUID idCpu2,
1313 uint32_t *pcHits)
1314{
1315 RTMPARGS Args;
1316 Args.pfnWorker = pfnWorker;
1317 Args.pvUser1 = pvUser1;
1318 Args.pvUser2 = pvUser2;
1319 Args.idCpu = idCpu;
1320 Args.idCpu2 = idCpu2;
1321 Args.cRefs = 0;
1322 Args.cHits = 0;
1323
1324 AssertPtr(g_pfnrtKeIpiGenericCall);
1325 g_pfnrtKeIpiGenericCall(pfnNativeWrapper, (uintptr_t)&Args);
1326 if (pcHits)
1327 *pcHits = Args.cHits;
1328 return VINF_SUCCESS;
1329}
1330
1331
1332/**
1333 * Wrapper between the native nt per-cpu callbacks and PFNRTWORKER
1334 *
1335 * @param Dpc DPC object
1336 * @param DeferredContext Context argument specified by KeInitializeDpc
1337 * @param SystemArgument1 Argument specified by KeInsertQueueDpc
1338 * @param SystemArgument2 Argument specified by KeInsertQueueDpc
1339 */
1340static VOID rtmpNtDPCWrapper(IN PKDPC Dpc, IN PVOID DeferredContext, IN PVOID SystemArgument1, IN PVOID SystemArgument2)
1341{
1342 PRTMPARGS pArgs = (PRTMPARGS)DeferredContext;
1343 RT_NOREF3(Dpc, SystemArgument1, SystemArgument2);
1344
1345 ASMAtomicIncU32(&pArgs->cHits);
1346 pArgs->pfnWorker(RTMpCpuId(), pArgs->pvUser1, pArgs->pvUser2);
1347
1348 /* Dereference the argument structure. */
1349 int32_t cRefs = ASMAtomicDecS32(&pArgs->cRefs);
1350 Assert(cRefs >= 0);
1351 if (cRefs == 0)
1352 RTMemFree(pArgs);
1353}
1354
1355
1356/**
1357 * Wrapper around KeSetTargetProcessorDpcEx / KeSetTargetProcessorDpc.
1358 *
1359 * This is shared with the timer code.
1360 *
1361 * @returns IPRT status code (errors are asserted).
1362 * @retval VERR_CPU_NOT_FOUND if impossible CPU. Not asserted.
1363 * @param pDpc The DPC.
1364 * @param idCpu The ID of the new target CPU.
1365 * @note Callable at any IRQL.
1366 */
1367DECLHIDDEN(int) rtMpNtSetTargetProcessorDpc(KDPC *pDpc, RTCPUID idCpu)
1368{
1369 if (g_pfnrtKeSetTargetProcessorDpcEx)
1370 {
1371 /* Convert to stupid process number (bet KeSetTargetProcessorDpcEx does
1372 the reverse conversion internally). */
1373 PROCESSOR_NUMBER ProcNum;
1374 NTSTATUS rcNt = g_pfnrtKeGetProcessorNumberFromIndex(RTMpCpuIdToSetIndex(idCpu), &ProcNum);
1375 if (NT_SUCCESS(rcNt))
1376 {
1377 rcNt = g_pfnrtKeSetTargetProcessorDpcEx(pDpc, &ProcNum);
1378 AssertLogRelMsgReturn(NT_SUCCESS(rcNt),
1379 ("KeSetTargetProcessorDpcEx(,%u(%u/%u)) -> %#x\n", idCpu, ProcNum.Group, ProcNum.Number, rcNt),
1380 RTErrConvertFromNtStatus(rcNt));
1381 }
1382 else if (rcNt == STATUS_INVALID_PARAMETER)
1383 return VERR_CPU_NOT_FOUND;
1384 else
1385 AssertLogRelMsgReturn(NT_SUCCESS(rcNt), ("KeGetProcessorNumberFromIndex(%u) -> %#x\n", idCpu, rcNt),
1386 RTErrConvertFromNtStatus(rcNt));
1387
1388 }
1389 else if (g_pfnrtKeSetTargetProcessorDpc)
1390 g_pfnrtKeSetTargetProcessorDpc(pDpc, RTMpCpuIdToSetIndex(idCpu));
1391 else
1392 return VERR_NOT_SUPPORTED;
1393 return VINF_SUCCESS;
1394}
1395
1396
1397/**
1398 * Internal worker for the RTMpOn* APIs.
1399 *
1400 * @returns IPRT status code.
1401 * @param pfnWorker The callback.
1402 * @param pvUser1 User argument 1.
1403 * @param pvUser2 User argument 2.
1404 * @param enmCpuid What to do / is idCpu valid.
1405 * @param idCpu Used if enmCpuid is RT_NT_CPUID_SPECIFIC or
1406 * RT_NT_CPUID_PAIR, otherwise ignored.
1407 * @param idCpu2 Used if enmCpuid is RT_NT_CPUID_PAIR, otherwise ignored.
1408 * @param pcHits Where to return the number of this. Optional.
1409 */
1410static int rtMpCallUsingDpcs(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2,
1411 RT_NT_CPUID enmCpuid, RTCPUID idCpu, RTCPUID idCpu2, uint32_t *pcHits)
1412{
1413#if 0
1414 /* KeFlushQueuedDpcs must be run at IRQL PASSIVE_LEVEL according to MSDN, but the
1415 * driver verifier doesn't complain...
1416 */
1417 AssertMsg(KeGetCurrentIrql() == PASSIVE_LEVEL, ("%d != %d (PASSIVE_LEVEL)\n", KeGetCurrentIrql(), PASSIVE_LEVEL));
1418#endif
1419 /* KeFlushQueuedDpcs is not present in Windows 2000; import it dynamically so we can just fail this call. */
1420 if (!g_pfnrtNtKeFlushQueuedDpcs)
1421 return VERR_NOT_SUPPORTED;
1422
1423 /*
1424 * Make a copy of the active CPU set and figure out how many KDPCs we really need.
1425 * We must not try setup DPCs for CPUs which aren't there, because that may fail.
1426 */
1427 RTCPUSET OnlineSet = g_rtMpNtCpuSet;
1428 uint32_t cDpcsNeeded;
1429 switch (enmCpuid)
1430 {
1431 case RT_NT_CPUID_SPECIFIC:
1432 cDpcsNeeded = 1;
1433 break;
1434 case RT_NT_CPUID_PAIR:
1435 cDpcsNeeded = 2;
1436 break;
1437 default:
1438 do
1439 {
1440 cDpcsNeeded = g_cRtMpNtActiveCpus;
1441 OnlineSet = g_rtMpNtCpuSet;
1442 } while (cDpcsNeeded != g_cRtMpNtActiveCpus);
1443 break;
1444 }
1445
1446 /*
1447 * Allocate an RTMPARGS structure followed by cDpcsNeeded KDPCs
1448 * and initialize them.
1449 */
1450 PRTMPARGS pArgs = (PRTMPARGS)RTMemAllocZ(sizeof(RTMPARGS) + cDpcsNeeded * sizeof(KDPC));
1451 if (!pArgs)
1452 return VERR_NO_MEMORY;
1453
1454 pArgs->pfnWorker = pfnWorker;
1455 pArgs->pvUser1 = pvUser1;
1456 pArgs->pvUser2 = pvUser2;
1457 pArgs->idCpu = NIL_RTCPUID;
1458 pArgs->idCpu2 = NIL_RTCPUID;
1459 pArgs->cHits = 0;
1460 pArgs->cRefs = 1;
1461
1462 int rc;
1463 KDPC *paExecCpuDpcs = (KDPC *)(pArgs + 1);
1464 if (enmCpuid == RT_NT_CPUID_SPECIFIC)
1465 {
1466 KeInitializeDpc(&paExecCpuDpcs[0], rtmpNtDPCWrapper, pArgs);
1467 if (g_pfnrtKeSetImportanceDpc)
1468 g_pfnrtKeSetImportanceDpc(&paExecCpuDpcs[0], HighImportance);
1469 rc = rtMpNtSetTargetProcessorDpc(&paExecCpuDpcs[0], idCpu);
1470 pArgs->idCpu = idCpu;
1471 }
1472 else if (enmCpuid == RT_NT_CPUID_PAIR)
1473 {
1474 KeInitializeDpc(&paExecCpuDpcs[0], rtmpNtDPCWrapper, pArgs);
1475 if (g_pfnrtKeSetImportanceDpc)
1476 g_pfnrtKeSetImportanceDpc(&paExecCpuDpcs[0], HighImportance);
1477 rc = rtMpNtSetTargetProcessorDpc(&paExecCpuDpcs[0], idCpu);
1478 pArgs->idCpu = idCpu;
1479
1480 KeInitializeDpc(&paExecCpuDpcs[1], rtmpNtDPCWrapper, pArgs);
1481 if (g_pfnrtKeSetImportanceDpc)
1482 g_pfnrtKeSetImportanceDpc(&paExecCpuDpcs[1], HighImportance);
1483 if (RT_SUCCESS(rc))
1484 rc = rtMpNtSetTargetProcessorDpc(&paExecCpuDpcs[1], (int)idCpu2);
1485 pArgs->idCpu2 = idCpu2;
1486 }
1487 else
1488 {
1489 rc = VINF_SUCCESS;
1490 for (uint32_t i = 0; i < cDpcsNeeded && RT_SUCCESS(rc); i++)
1491 if (RTCpuSetIsMemberByIndex(&OnlineSet, i))
1492 {
1493 KeInitializeDpc(&paExecCpuDpcs[i], rtmpNtDPCWrapper, pArgs);
1494 if (g_pfnrtKeSetImportanceDpc)
1495 g_pfnrtKeSetImportanceDpc(&paExecCpuDpcs[i], HighImportance);
1496 rc = rtMpNtSetTargetProcessorDpc(&paExecCpuDpcs[i], RTMpCpuIdFromSetIndex(i));
1497 }
1498 }
1499 if (RT_FAILURE(rc))
1500 {
1501 RTMemFree(pArgs);
1502 return rc;
1503 }
1504
1505 /*
1506 * Raise the IRQL to DISPATCH_LEVEL so we can't be rescheduled to another cpu.
1507 * KeInsertQueueDpc must also be executed at IRQL >= DISPATCH_LEVEL.
1508 */
1509 KIRQL oldIrql;
1510 KeRaiseIrql(DISPATCH_LEVEL, &oldIrql);
1511
1512 /*
1513 * We cannot do other than assume a 1:1 relationship between the
1514 * affinity mask and the process despite the warnings in the docs.
1515 * If someone knows a better way to get this done, please let bird know.
1516 */
1517 ASMCompilerBarrier(); /* paranoia */
1518 if (enmCpuid == RT_NT_CPUID_SPECIFIC)
1519 {
1520 ASMAtomicIncS32(&pArgs->cRefs);
1521 BOOLEAN fRc = KeInsertQueueDpc(&paExecCpuDpcs[0], 0, 0);
1522 Assert(fRc); NOREF(fRc);
1523 }
1524 else if (enmCpuid == RT_NT_CPUID_PAIR)
1525 {
1526 ASMAtomicIncS32(&pArgs->cRefs);
1527 BOOLEAN fRc = KeInsertQueueDpc(&paExecCpuDpcs[0], 0, 0);
1528 Assert(fRc); NOREF(fRc);
1529
1530 ASMAtomicIncS32(&pArgs->cRefs);
1531 fRc = KeInsertQueueDpc(&paExecCpuDpcs[1], 0, 0);
1532 Assert(fRc); NOREF(fRc);
1533 }
1534 else
1535 {
1536 uint32_t iSelf = RTMpCurSetIndex();
1537 for (uint32_t i = 0; i < cDpcsNeeded; i++)
1538 {
1539 if ( (i != iSelf)
1540 && RTCpuSetIsMemberByIndex(&OnlineSet, i))
1541 {
1542 ASMAtomicIncS32(&pArgs->cRefs);
1543 BOOLEAN fRc = KeInsertQueueDpc(&paExecCpuDpcs[i], 0, 0);
1544 Assert(fRc); NOREF(fRc);
1545 }
1546 }
1547 if (enmCpuid != RT_NT_CPUID_OTHERS)
1548 pfnWorker(iSelf, pvUser1, pvUser2);
1549 }
1550
1551 KeLowerIrql(oldIrql);
1552
1553 /*
1554 * Flush all DPCs and wait for completion. (can take long!)
1555 */
1556 /** @todo Consider changing this to an active wait using some atomic inc/dec
1557 * stuff (and check for the current cpu above in the specific case). */
1558 /** @todo Seems KeFlushQueuedDpcs doesn't wait for the DPCs to be completely
1559 * executed. Seen pArgs being freed while some CPU was using it before
1560 * cRefs was added. */
1561 if (g_pfnrtNtKeFlushQueuedDpcs)
1562 g_pfnrtNtKeFlushQueuedDpcs();
1563
1564 if (pcHits)
1565 *pcHits = pArgs->cHits;
1566
1567 /* Dereference the argument structure. */
1568 int32_t cRefs = ASMAtomicDecS32(&pArgs->cRefs);
1569 Assert(cRefs >= 0);
1570 if (cRefs == 0)
1571 RTMemFree(pArgs);
1572
1573 return VINF_SUCCESS;
1574}
1575
1576
1577RTDECL(int) RTMpOnAll(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
1578{
1579 if (g_pfnrtKeIpiGenericCall)
1580 return rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnAllBroadcastIpiWrapper,
1581 NIL_RTCPUID, NIL_RTCPUID, NULL);
1582 return rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_ALL, NIL_RTCPUID, NIL_RTCPUID, NULL);
1583}
1584
1585
1586RTDECL(int) RTMpOnOthers(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
1587{
1588 if (g_pfnrtKeIpiGenericCall)
1589 return rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnOthersBroadcastIpiWrapper,
1590 NIL_RTCPUID, NIL_RTCPUID, NULL);
1591 return rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_OTHERS, NIL_RTCPUID, NIL_RTCPUID, NULL);
1592}
1593
1594
1595RTDECL(int) RTMpOnPair(RTCPUID idCpu1, RTCPUID idCpu2, uint32_t fFlags, PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
1596{
1597 int rc;
1598 AssertReturn(idCpu1 != idCpu2, VERR_INVALID_PARAMETER);
1599 AssertReturn(!(fFlags & RTMPON_F_VALID_MASK), VERR_INVALID_FLAGS);
1600 if ((fFlags & RTMPON_F_CONCURRENT_EXEC) && !g_pfnrtKeIpiGenericCall)
1601 return VERR_NOT_SUPPORTED;
1602
1603 /*
1604 * Check that both CPUs are online before doing the broadcast call.
1605 */
1606 if ( RTMpIsCpuOnline(idCpu1)
1607 && RTMpIsCpuOnline(idCpu2))
1608 {
1609 /*
1610 * The broadcast IPI isn't quite as bad as it could have been, because
1611 * it looks like windows doesn't synchronize CPUs on the way out, they
1612 * seems to get back to normal work while the pair is still busy.
1613 */
1614 uint32_t cHits = 0;
1615 if (g_pfnrtKeIpiGenericCall)
1616 rc = rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnPairBroadcastIpiWrapper, idCpu1, idCpu2, &cHits);
1617 else
1618 rc = rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_PAIR, idCpu1, idCpu2, &cHits);
1619 if (RT_SUCCESS(rc))
1620 {
1621 Assert(cHits <= 2);
1622 if (cHits == 2)
1623 rc = VINF_SUCCESS;
1624 else if (cHits == 1)
1625 rc = VERR_NOT_ALL_CPUS_SHOWED;
1626 else if (cHits == 0)
1627 rc = VERR_CPU_OFFLINE;
1628 else
1629 rc = VERR_CPU_IPE_1;
1630 }
1631 }
1632 /*
1633 * A CPU must be present to be considered just offline.
1634 */
1635 else if ( RTMpIsCpuPresent(idCpu1)
1636 && RTMpIsCpuPresent(idCpu2))
1637 rc = VERR_CPU_OFFLINE;
1638 else
1639 rc = VERR_CPU_NOT_FOUND;
1640 return rc;
1641}
1642
1643
1644RTDECL(bool) RTMpOnPairIsConcurrentExecSupported(void)
1645{
1646 return g_pfnrtKeIpiGenericCall != NULL;
1647}
1648
1649
1650/**
1651 * Releases a reference to a RTMPNTONSPECIFICARGS heap allocation, freeing it
1652 * when the last reference is released.
1653 */
1654DECLINLINE(void) rtMpNtOnSpecificRelease(PRTMPNTONSPECIFICARGS pArgs)
1655{
1656 uint32_t cRefs = ASMAtomicDecU32(&pArgs->cRefs);
1657 AssertMsg(cRefs <= 1, ("cRefs=%#x\n", cRefs));
1658 if (cRefs == 0)
1659 RTMemFree(pArgs);
1660}
1661
1662
1663/**
1664 * Wrapper between the native nt per-cpu callbacks and PFNRTWORKER
1665 *
1666 * @param Dpc DPC object
1667 * @param DeferredContext Context argument specified by KeInitializeDpc
1668 * @param SystemArgument1 Argument specified by KeInsertQueueDpc
1669 * @param SystemArgument2 Argument specified by KeInsertQueueDpc
1670 */
1671static VOID rtMpNtOnSpecificDpcWrapper(IN PKDPC Dpc, IN PVOID DeferredContext,
1672 IN PVOID SystemArgument1, IN PVOID SystemArgument2)
1673{
1674 PRTMPNTONSPECIFICARGS pArgs = (PRTMPNTONSPECIFICARGS)DeferredContext;
1675 RT_NOREF3(Dpc, SystemArgument1, SystemArgument2);
1676
1677 ASMAtomicWriteBool(&pArgs->fExecuting, true);
1678
1679 pArgs->CallbackArgs.pfnWorker(RTMpCpuId(), pArgs->CallbackArgs.pvUser1, pArgs->CallbackArgs.pvUser2);
1680
1681 ASMAtomicWriteBool(&pArgs->fDone, true);
1682 KeSetEvent(&pArgs->DoneEvt, 1 /*PriorityIncrement*/, FALSE /*Wait*/);
1683
1684 rtMpNtOnSpecificRelease(pArgs);
1685}
1686
1687
1688RTDECL(int) RTMpOnSpecific(RTCPUID idCpu, PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
1689{
1690 /*
1691 * Don't try mess with an offline CPU.
1692 */
1693 if (!RTMpIsCpuOnline(idCpu))
1694 return !RTMpIsCpuPossible(idCpu)
1695 ? VERR_CPU_NOT_FOUND
1696 : VERR_CPU_OFFLINE;
1697
1698 /*
1699 * Use the broadcast IPI routine if there are no more than two CPUs online,
1700 * or if the current IRQL is unsuitable for KeWaitForSingleObject.
1701 */
1702 int rc;
1703 uint32_t cHits = 0;
1704 if ( g_pfnrtKeIpiGenericCall
1705 && ( RTMpGetOnlineCount() <= 2
1706 || KeGetCurrentIrql() > APC_LEVEL)
1707 )
1708 {
1709 rc = rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnSpecificBroadcastIpiWrapper,
1710 idCpu, NIL_RTCPUID, &cHits);
1711 if (RT_SUCCESS(rc))
1712 {
1713 if (cHits == 1)
1714 return VINF_SUCCESS;
1715 rc = cHits == 0 ? VERR_CPU_OFFLINE : VERR_CPU_IPE_1;
1716 }
1717 return rc;
1718 }
1719
1720#if 0
1721 rc = rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_SPECIFIC, idCpu, NIL_RTCPUID, &cHits);
1722 if (RT_SUCCESS(rc))
1723 {
1724 if (cHits == 1)
1725 return VINF_SUCCESS;
1726 rc = cHits == 0 ? VERR_CPU_OFFLINE : VERR_CPU_IPE_1;
1727 }
1728 return rc;
1729
1730#else
1731 /*
1732 * Initialize the argument package and the objects within it.
1733 * The package is referenced counted to avoid unnecessary spinning to
1734 * synchronize cleanup and prevent stack corruption.
1735 */
1736 PRTMPNTONSPECIFICARGS pArgs = (PRTMPNTONSPECIFICARGS)RTMemAllocZ(sizeof(*pArgs));
1737 if (!pArgs)
1738 return VERR_NO_MEMORY;
1739 pArgs->cRefs = 2;
1740 pArgs->fExecuting = false;
1741 pArgs->fDone = false;
1742 pArgs->CallbackArgs.pfnWorker = pfnWorker;
1743 pArgs->CallbackArgs.pvUser1 = pvUser1;
1744 pArgs->CallbackArgs.pvUser2 = pvUser2;
1745 pArgs->CallbackArgs.idCpu = idCpu;
1746 pArgs->CallbackArgs.cHits = 0;
1747 pArgs->CallbackArgs.cRefs = 2;
1748 KeInitializeEvent(&pArgs->DoneEvt, SynchronizationEvent, FALSE /* not signalled */);
1749 KeInitializeDpc(&pArgs->Dpc, rtMpNtOnSpecificDpcWrapper, pArgs);
1750 if (g_pfnrtKeSetImportanceDpc)
1751 g_pfnrtKeSetImportanceDpc(&pArgs->Dpc, HighImportance);
1752 rc = rtMpNtSetTargetProcessorDpc(&pArgs->Dpc, idCpu);
1753 if (RT_FAILURE(rc))
1754 {
1755 RTMemFree(pArgs);
1756 return rc;
1757 }
1758
1759 /*
1760 * Disable preemption while we check the current processor and inserts the DPC.
1761 */
1762 KIRQL bOldIrql;
1763 KeRaiseIrql(DISPATCH_LEVEL, &bOldIrql);
1764 ASMCompilerBarrier(); /* paranoia */
1765
1766 if (RTMpCpuId() == idCpu)
1767 {
1768 /* Just execute the callback on the current CPU. */
1769 pfnWorker(idCpu, pvUser1, pvUser2);
1770 KeLowerIrql(bOldIrql);
1771
1772 RTMemFree(pArgs);
1773 return VINF_SUCCESS;
1774 }
1775
1776 /* Different CPU, so queue it if the CPU is still online. */
1777 if (RTMpIsCpuOnline(idCpu))
1778 {
1779 BOOLEAN fRc = KeInsertQueueDpc(&pArgs->Dpc, 0, 0);
1780 Assert(fRc); NOREF(fRc);
1781 KeLowerIrql(bOldIrql);
1782
1783 uint64_t const nsRealWaitTS = RTTimeNanoTS();
1784
1785 /*
1786 * Wait actively for a while in case the CPU/thread responds quickly.
1787 */
1788 uint32_t cLoopsLeft = 0x20000;
1789 while (cLoopsLeft-- > 0)
1790 {
1791 if (pArgs->fDone)
1792 {
1793 rtMpNtOnSpecificRelease(pArgs);
1794 return VINF_SUCCESS;
1795 }
1796 ASMNopPause();
1797 }
1798
1799 /*
1800 * It didn't respond, so wait on the event object, poking the CPU if it's slow.
1801 */
1802 LARGE_INTEGER Timeout;
1803 Timeout.QuadPart = -10000; /* 1ms */
1804 NTSTATUS rcNt = KeWaitForSingleObject(&pArgs->DoneEvt, Executive, KernelMode, FALSE /* Alertable */, &Timeout);
1805 if (rcNt == STATUS_SUCCESS)
1806 {
1807 rtMpNtOnSpecificRelease(pArgs);
1808 return VINF_SUCCESS;
1809 }
1810
1811 /* If it hasn't respondend yet, maybe poke it and wait some more. */
1812 if (rcNt == STATUS_TIMEOUT)
1813 {
1814 if ( !pArgs->fExecuting
1815 && ( g_pfnrtMpPokeCpuWorker == rtMpPokeCpuUsingHalRequestIpiW7Plus
1816 || g_pfnrtMpPokeCpuWorker == rtMpPokeCpuUsingHalRequestIpiPreW7))
1817 RTMpPokeCpu(idCpu);
1818
1819 Timeout.QuadPart = -1280000; /* 128ms */
1820 rcNt = KeWaitForSingleObject(&pArgs->DoneEvt, Executive, KernelMode, FALSE /* Alertable */, &Timeout);
1821 if (rcNt == STATUS_SUCCESS)
1822 {
1823 rtMpNtOnSpecificRelease(pArgs);
1824 return VINF_SUCCESS;
1825 }
1826 }
1827
1828 /*
1829 * Something weird is happening, try bail out.
1830 */
1831 if (KeRemoveQueueDpc(&pArgs->Dpc))
1832 {
1833 RTMemFree(pArgs); /* DPC was still queued, so we can return without further ado. */
1834 LogRel(("RTMpOnSpecific(%#x): Not processed after %llu ns: rcNt=%#x\n", idCpu, RTTimeNanoTS() - nsRealWaitTS, rcNt));
1835 }
1836 else
1837 {
1838 /* DPC is running, wait a good while for it to complete. */
1839 LogRel(("RTMpOnSpecific(%#x): Still running after %llu ns: rcNt=%#x\n", idCpu, RTTimeNanoTS() - nsRealWaitTS, rcNt));
1840
1841 Timeout.QuadPart = -30*1000*1000*10; /* 30 seconds */
1842 rcNt = KeWaitForSingleObject(&pArgs->DoneEvt, Executive, KernelMode, FALSE /* Alertable */, &Timeout);
1843 if (rcNt != STATUS_SUCCESS)
1844 LogRel(("RTMpOnSpecific(%#x): Giving up on running worker after %llu ns: rcNt=%#x\n", idCpu, RTTimeNanoTS() - nsRealWaitTS, rcNt));
1845 }
1846 rc = RTErrConvertFromNtStatus(rcNt);
1847 }
1848 else
1849 {
1850 /* CPU is offline.*/
1851 KeLowerIrql(bOldIrql);
1852 rc = !RTMpIsCpuPossible(idCpu) ? VERR_CPU_NOT_FOUND : VERR_CPU_OFFLINE;
1853 }
1854
1855 rtMpNtOnSpecificRelease(pArgs);
1856 return rc;
1857#endif
1858}
1859
1860
1861
1862
1863static VOID rtMpNtPokeCpuDummy(IN PKDPC Dpc, IN PVOID DeferredContext, IN PVOID SystemArgument1, IN PVOID SystemArgument2)
1864{
1865 NOREF(Dpc);
1866 NOREF(DeferredContext);
1867 NOREF(SystemArgument1);
1868 NOREF(SystemArgument2);
1869}
1870
1871
1872/** Callback used by rtMpPokeCpuUsingBroadcastIpi. */
1873static ULONG_PTR rtMpIpiGenericCall(ULONG_PTR Argument)
1874{
1875 NOREF(Argument);
1876 return 0;
1877}
1878
1879
1880/**
1881 * RTMpPokeCpu worker that uses broadcast IPIs for doing the work.
1882 *
1883 * @returns VINF_SUCCESS
1884 * @param idCpu The CPU identifier.
1885 */
1886int rtMpPokeCpuUsingBroadcastIpi(RTCPUID idCpu)
1887{
1888 NOREF(idCpu);
1889 g_pfnrtKeIpiGenericCall(rtMpIpiGenericCall, 0);
1890 return VINF_SUCCESS;
1891}
1892
1893
1894/**
1895 * RTMpPokeCpu worker that uses the Windows 7 and later version of
1896 * HalRequestIpip to get the job done.
1897 *
1898 * @returns VINF_SUCCESS
1899 * @param idCpu The CPU identifier.
1900 */
1901int rtMpPokeCpuUsingHalRequestIpiW7Plus(RTCPUID idCpu)
1902{
1903 /* idCpu is an HAL processor index, so we can use it directly. */
1904 PKAFFINITY_EX pTarget = (PKAFFINITY_EX)alloca(g_cbRtMpNtKaffinityEx);
1905 pTarget->Size = g_cRtMpNtKaffinityExEntries; /* (just in case KeInitializeAffinityEx starts using it) */
1906 g_pfnrtKeInitializeAffinityEx(pTarget);
1907 g_pfnrtKeAddProcessorAffinityEx(pTarget, idCpu);
1908
1909 g_pfnrtHalRequestIpiW7Plus(0, pTarget);
1910 return VINF_SUCCESS;
1911}
1912
1913
1914/**
1915 * RTMpPokeCpu worker that uses the Vista and earlier version of HalRequestIpip
1916 * to get the job done.
1917 *
1918 * @returns VINF_SUCCESS
1919 * @param idCpu The CPU identifier.
1920 */
1921int rtMpPokeCpuUsingHalRequestIpiPreW7(RTCPUID idCpu)
1922{
1923 __debugbreak(); /** @todo this code needs testing!! */
1924 KAFFINITY Target = 1;
1925 Target <<= idCpu;
1926 g_pfnrtHalRequestIpiPreW7(Target);
1927 return VINF_SUCCESS;
1928}
1929
1930
1931int rtMpPokeCpuUsingFailureNotSupported(RTCPUID idCpu)
1932{
1933 NOREF(idCpu);
1934 return VERR_NOT_SUPPORTED;
1935}
1936
1937
1938int rtMpPokeCpuUsingDpc(RTCPUID idCpu)
1939{
1940 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1941
1942 /*
1943 * APC fallback.
1944 */
1945 static KDPC s_aPokeDpcs[RTCPUSET_MAX_CPUS] = {{0}};
1946 static bool s_fPokeDPCsInitialized = false;
1947
1948 if (!s_fPokeDPCsInitialized)
1949 {
1950 for (unsigned i = 0; i < g_cRtMpNtMaxCpus; i++)
1951 {
1952 KeInitializeDpc(&s_aPokeDpcs[i], rtMpNtPokeCpuDummy, NULL);
1953 if (g_pfnrtKeSetImportanceDpc)
1954 g_pfnrtKeSetImportanceDpc(&s_aPokeDpcs[i], HighImportance);
1955 int rc = rtMpNtSetTargetProcessorDpc(&s_aPokeDpcs[i], idCpu);
1956 if (RT_FAILURE(rc) && rc != VERR_CPU_NOT_FOUND)
1957 return rc;
1958 }
1959
1960 s_fPokeDPCsInitialized = true;
1961 }
1962
1963 /* Raise the IRQL to DISPATCH_LEVEL so we can't be rescheduled to another cpu.
1964 KeInsertQueueDpc must also be executed at IRQL >= DISPATCH_LEVEL. */
1965 KIRQL oldIrql;
1966 KeRaiseIrql(DISPATCH_LEVEL, &oldIrql);
1967
1968 if (g_pfnrtKeSetImportanceDpc)
1969 g_pfnrtKeSetImportanceDpc(&s_aPokeDpcs[idCpu], HighImportance);
1970 g_pfnrtKeSetTargetProcessorDpc(&s_aPokeDpcs[idCpu], (int)idCpu);
1971
1972 /* Assuming here that high importance DPCs will be delivered immediately; or at least an IPI will be sent immediately.
1973 Note! Not true on at least Vista & Windows 7 */
1974 BOOLEAN fRet = KeInsertQueueDpc(&s_aPokeDpcs[idCpu], 0, 0);
1975
1976 KeLowerIrql(oldIrql);
1977 return fRet == TRUE ? VINF_SUCCESS : VERR_ACCESS_DENIED /* already queued */;
1978}
1979
1980
1981RTDECL(int) RTMpPokeCpu(RTCPUID idCpu)
1982{
1983 if (!RTMpIsCpuOnline(idCpu))
1984 return !RTMpIsCpuPossible(idCpu)
1985 ? VERR_CPU_NOT_FOUND
1986 : VERR_CPU_OFFLINE;
1987 /* Calls rtMpPokeCpuUsingDpc, rtMpPokeCpuUsingHalRequestIpiW7Plus or rtMpPokeCpuUsingBroadcastIpi. */
1988 return g_pfnrtMpPokeCpuWorker(idCpu);
1989}
1990
1991
1992RTDECL(bool) RTMpOnAllIsConcurrentSafe(void)
1993{
1994 return false;
1995}
1996
Note: See TracBrowser for help on using the repository browser.

© 2023 Oracle
Contact – Privacy policy – Terms of Use