1use std::arch::x86_64::__cpuid;
6use std::arch::x86_64::__cpuid_count;
7use std::arch::x86_64::CpuidResult;
8use std::cmp;
9use std::result;
10
11use devices::Apic;
12use devices::IrqChipCap;
13use devices::IrqChipX86_64;
14use hypervisor::CpuConfigX86_64;
15use hypervisor::CpuHybridType;
16use hypervisor::CpuIdEntry;
17use hypervisor::HypervisorCap;
18use hypervisor::HypervisorX86_64;
19use hypervisor::NestedMode;
20use hypervisor::VcpuX86_64;
21use remain::sorted;
22use thiserror::Error;
23
24use crate::CpuManufacturer;
25
26#[sorted]
27#[derive(Error, Debug, PartialEq, Eq)]
28pub enum Error {
29 #[error("GetSupportedCpus ioctl failed: {0}")]
30 GetSupportedCpusFailed(base::Error),
31 #[error("nested virtualization requested, but not supported by the host")]
32 NestedVirtUnsupported,
33 #[error("SetSupportedCpus ioctl failed: {0}")]
34 SetSupportedCpusFailed(base::Error),
35}
36
37pub type Result<T> = result::Result<T, Error>;
38
39pub const EBX_CLFLUSH_CACHELINE: u32 = 8; pub const EBX_CLFLUSH_SIZE_SHIFT: u32 = 8; pub const EBX_CPU_COUNT_SHIFT: u32 = 16; pub const EBX_CPUID_SHIFT: u32 = 24; pub const ECX_EPB_SHIFT: u32 = 3; pub const ECX_VMX_SHIFT: u32 = 5; pub const ECX_X2APIC_SHIFT: u32 = 21; pub const ECX_TSC_DEADLINE_TIMER_SHIFT: u32 = 24; pub const ECX_HYPERVISOR_SHIFT: u32 = 31; pub const EDX_HTT_SHIFT: u32 = 28; pub const ECX_TOPO_TYPE_SHIFT: u32 = 8; pub const ECX_TOPO_SMT_TYPE: u32 = 1; pub const ECX_TOPO_CORE_TYPE: u32 = 2; pub const ECX_HCFC_PERF_SHIFT: u32 = 0; pub const ECX_SVM_SHIFT: u32 = 2; pub const EAX_CPU_CORES_SHIFT: u32 = 26; pub const EDX_HYBRID_CPU_SHIFT: u32 = 15; pub const EAX_HWP_SHIFT: u32 = 7; pub const EAX_HWP_NOTIFICATION_SHIFT: u32 = 8; pub const EAX_HWP_EPP_SHIFT: u32 = 10; pub const EAX_ITMT_SHIFT: u32 = 14; pub const EAX_CORE_TEMP: u32 = 0; pub const EAX_PKG_TEMP: u32 = 6; pub const EAX_CORE_TYPE_SHIFT: u32 = 24; const EAX_CORE_TYPE_ATOM: u32 = 0x20; const EAX_CORE_TYPE_CORE: u32 = 0x40; #[derive(Clone, Debug, PartialEq, Eq)]
70pub struct CpuIdContext {
71 vcpu_id: usize,
73 vcpu_count: usize,
75 x2apic: bool,
77 tsc_deadline_timer: bool,
79 apic_frequency: u32,
81 tsc_frequency: Option<u64>,
83 cpu_config: CpuConfigX86_64,
85 cpuid_count: unsafe fn(u32, u32) -> CpuidResult,
87 cpuid: unsafe fn(u32) -> CpuidResult,
89}
90
91impl CpuIdContext {
92 pub fn new(
93 vcpu_id: usize,
94 vcpu_count: usize,
95 irq_chip: Option<&dyn IrqChipX86_64>,
96 cpu_config: CpuConfigX86_64,
97 calibrated_tsc_leaf_required: bool,
98 cpuid_count: unsafe fn(u32, u32) -> CpuidResult,
99 cpuid: unsafe fn(u32) -> CpuidResult,
100 ) -> CpuIdContext {
101 CpuIdContext {
102 vcpu_id,
103 vcpu_count,
104 x2apic: irq_chip.is_some_and(|chip| chip.check_capability(IrqChipCap::X2Apic)),
105 tsc_deadline_timer: irq_chip
106 .is_some_and(|chip| chip.check_capability(IrqChipCap::TscDeadlineTimer)),
107 apic_frequency: irq_chip.map_or(Apic::frequency(), |chip| chip.lapic_frequency()),
108 tsc_frequency: if calibrated_tsc_leaf_required || cpu_config.force_calibrated_tsc_leaf {
109 devices::tsc::tsc_frequency().ok()
110 } else {
111 None
112 },
113 cpu_config,
114 cpuid_count,
115 cpuid,
116 }
117 }
118}
119
120fn nested_feature_available(entries: &[CpuIdEntry], manufacturer: CpuManufacturer) -> bool {
123 match manufacturer {
124 CpuManufacturer::Intel => cpuid_ecx_bit_set(entries, 1, ECX_VMX_SHIFT),
125 CpuManufacturer::Amd => cpuid_ecx_bit_set(entries, 0x80000001, ECX_SVM_SHIFT),
126 CpuManufacturer::Unknown => {
127 cpuid_ecx_bit_set(entries, 1, ECX_VMX_SHIFT)
128 || cpuid_ecx_bit_set(entries, 0x80000001, ECX_SVM_SHIFT)
129 }
130 }
131}
132
133fn cpuid_ecx_bit_set(entries: &[CpuIdEntry], leaf: u32, shift: u32) -> bool {
135 entries
136 .iter()
137 .any(|e| e.function == leaf && e.index == 0 && e.cpuid.ecx & (1 << shift) != 0)
138}
139
140pub fn adjust_cpuid(entry: &mut CpuIdEntry, ctx: &CpuIdContext) {
147 match entry.function {
148 0 => {
149 if ctx.tsc_frequency.is_some() {
150 entry.cpuid.eax = cmp::max(0x15, entry.cpuid.eax);
152 }
153 }
154 1 => {
155 if entry.index == 0 {
157 entry.cpuid.ecx |= 1 << ECX_HYPERVISOR_SHIFT;
158 }
159 if ctx.x2apic {
160 entry.cpuid.ecx |= 1 << ECX_X2APIC_SHIFT;
161 } else {
162 entry.cpuid.ecx &= !(1 << ECX_X2APIC_SHIFT);
163 }
164 if ctx.tsc_deadline_timer {
165 entry.cpuid.ecx |= 1 << ECX_TSC_DEADLINE_TIMER_SHIFT;
166 }
167
168 if ctx.cpu_config.nested == NestedMode::Off {
170 entry.cpuid.ecx &= !(1 << ECX_VMX_SHIFT);
171 }
172
173 if ctx.cpu_config.host_cpu_topology {
174 entry.cpuid.ebx |= EBX_CLFLUSH_CACHELINE << EBX_CLFLUSH_SIZE_SHIFT;
175
176 let result = unsafe { (ctx.cpuid)(entry.function) };
179 entry.cpuid.edx |= result.edx & (1 << EDX_HTT_SHIFT);
180 return;
181 }
182
183 entry.cpuid.ebx = (ctx.vcpu_id << EBX_CPUID_SHIFT) as u32
184 | (EBX_CLFLUSH_CACHELINE << EBX_CLFLUSH_SIZE_SHIFT);
185 if ctx.vcpu_count > 1 {
186 entry.cpuid.ebx |= (ctx.vcpu_count as u32) << EBX_CPU_COUNT_SHIFT;
188 entry.cpuid.edx |= 1 << EDX_HTT_SHIFT;
192 }
193 }
194 2 | 0x80000002 | 0x80000003 | 0x80000004 | 0x80000005 | 0x80000006 => entry.cpuid = {
198 unsafe { (ctx.cpuid)(entry.function) }},
200 4 => {
201 entry.cpuid = {
202 unsafe { (ctx.cpuid_count)(entry.function, entry.index) }};
204
205 if ctx.cpu_config.host_cpu_topology {
206 return;
207 }
208
209 entry.cpuid.eax &= !0xFC000000;
210 if ctx.vcpu_count > 1 {
211 let cpu_cores = if ctx.cpu_config.no_smt {
212 ctx.vcpu_count as u32
213 } else if ctx.vcpu_count % 2 == 0 {
214 (ctx.vcpu_count >> 1) as u32
215 } else {
216 1
217 };
218 entry.cpuid.eax |= (cpu_cores - 1) << EAX_CPU_CORES_SHIFT;
219 }
220 }
221 6 => {
222 let result = {
223 unsafe { (ctx.cpuid)(entry.function) }};
227
228 if ctx.cpu_config.enable_hwp {
229 entry.cpuid.eax |= result.eax & (1 << EAX_HWP_SHIFT);
230 entry.cpuid.eax |= result.eax & (1 << EAX_HWP_NOTIFICATION_SHIFT);
231 entry.cpuid.eax |= result.eax & (1 << EAX_HWP_EPP_SHIFT);
232 entry.cpuid.ecx |= result.ecx & (1 << ECX_EPB_SHIFT);
233
234 if ctx.cpu_config.itmt {
235 entry.cpuid.eax |= result.eax & (1 << EAX_ITMT_SHIFT);
236 }
237 }
238 }
239 7 => {
240 if ctx.cpu_config.host_cpu_topology && entry.index == 0 {
241 let result = unsafe { (ctx.cpuid_count)(entry.function, entry.index) };
245 entry.cpuid.edx |= result.edx & (1 << EDX_HYBRID_CPU_SHIFT);
246 }
247 if ctx.cpu_config.hybrid_type.is_some() && entry.index == 0 {
248 entry.cpuid.edx |= 1 << EDX_HYBRID_CPU_SHIFT;
249 }
250 }
251 0x15 => {
252 if let Some(tsc_freq) = ctx.tsc_frequency {
253 entry.cpuid = devices::tsc::fake_tsc_frequency_cpuid(tsc_freq, ctx.apic_frequency);
255 }
256 }
257 0x1A => {
258 if ctx.cpu_config.host_cpu_topology {
260 entry.cpuid = unsafe { (ctx.cpuid)(entry.function) };
264 }
265 if let Some(hybrid) = &ctx.cpu_config.hybrid_type {
266 match hybrid {
267 CpuHybridType::Atom => {
268 entry.cpuid.eax |= EAX_CORE_TYPE_ATOM << EAX_CORE_TYPE_SHIFT;
269 }
270 CpuHybridType::Core => {
271 entry.cpuid.eax |= EAX_CORE_TYPE_CORE << EAX_CORE_TYPE_SHIFT;
272 }
273 }
274 }
275 }
276 0xB | 0x1F => {
277 if ctx.cpu_config.host_cpu_topology {
278 return;
279 }
280 entry.cpuid.edx = ctx.vcpu_id as u32; if entry.index == 0 {
286 if ctx.cpu_config.no_smt || (ctx.vcpu_count == 1) {
287 entry.cpuid.eax = 0; entry.cpuid.ebx = 1; } else if ctx.vcpu_count % 2 == 0 {
292 entry.cpuid.eax = 1; entry.cpuid.ebx = 2; } else {
296 let cpu_bits: u32 = 32 - ((ctx.vcpu_count - 1) as u32).leading_zeros();
298 entry.cpuid.eax = cpu_bits; entry.cpuid.ebx = ctx.vcpu_count as u32; }
301 entry.cpuid.ecx = (ECX_TOPO_SMT_TYPE << ECX_TOPO_TYPE_SHIFT) | entry.index;
302 } else if entry.index == 1 {
303 let cpu_bits: u32 = 32 - ((ctx.vcpu_count - 1) as u32).leading_zeros();
304 entry.cpuid.eax = cpu_bits;
305 entry.cpuid.ebx = (ctx.vcpu_count as u32) & 0xffff;
307 entry.cpuid.ecx = (ECX_TOPO_CORE_TYPE << ECX_TOPO_TYPE_SHIFT) | entry.index;
308 } else {
309 entry.cpuid.eax = 0;
310 entry.cpuid.ebx = 0;
311 entry.cpuid.ecx = 0;
312 }
313 }
314 0x80000001 => {
315 if ctx.cpu_config.nested == NestedMode::Off {
317 entry.cpuid.ecx &= !(1 << ECX_SVM_SHIFT);
318 }
319 }
320 _ => (),
321 }
322}
323
324fn filter_cpuid(cpuid: &mut hypervisor::CpuId, ctx: &CpuIdContext) {
327 if ctx.tsc_frequency.is_some()
330 && !cpuid
331 .cpu_id_entries
332 .iter()
333 .any(|entry| entry.function == 0x15)
334 {
335 cpuid.cpu_id_entries.push(CpuIdEntry {
336 function: 0x15,
337 index: 0,
338 flags: 0,
339 cpuid: CpuidResult {
340 eax: 0,
341 ebx: 0,
342 ecx: 0,
343 edx: 0,
344 },
345 })
346 }
347
348 let entries = &mut cpuid.cpu_id_entries;
349 for entry in entries.iter_mut() {
350 adjust_cpuid(entry, ctx);
351 }
352}
353
354pub(crate) fn setup_cpuid(
365 hypervisor: &dyn HypervisorX86_64,
366 irq_chip: &dyn IrqChipX86_64,
367 vcpu: &dyn VcpuX86_64,
368 vcpu_id: usize,
369 vcpu_count: usize,
370 cpu_config: CpuConfigX86_64,
371) -> Result<()> {
372 let mut cpuid = hypervisor
373 .get_supported_cpuid()
374 .map_err(Error::GetSupportedCpusFailed)?;
375
376 let ctx = CpuIdContext::new(
377 vcpu_id,
378 vcpu_count,
379 Some(irq_chip),
380 cpu_config,
381 hypervisor.check_capability(HypervisorCap::CalibratedTscLeafRequired),
382 __cpuid_count,
383 __cpuid,
384 );
385 filter_cpuid(&mut cpuid, &ctx);
386
387 if ctx.cpu_config.nested == NestedMode::On
388 && !nested_feature_available(&cpuid.cpu_id_entries, cpu_manufacturer())
389 {
390 return Err(Error::NestedVirtUnsupported);
391 }
392
393 vcpu.set_cpuid(&cpuid)
394 .map_err(Error::SetSupportedCpusFailed)
395}
396
397const MANUFACTURER_ID_FUNCTION: u32 = 0x00000000;
398const AMD_EBX: u32 = u32::from_le_bytes([b'A', b'u', b't', b'h']);
399const AMD_EDX: u32 = u32::from_le_bytes([b'e', b'n', b't', b'i']);
400const AMD_ECX: u32 = u32::from_le_bytes([b'c', b'A', b'M', b'D']);
401const INTEL_EBX: u32 = u32::from_le_bytes([b'G', b'e', b'n', b'u']);
402const INTEL_EDX: u32 = u32::from_le_bytes([b'i', b'n', b'e', b'I']);
403const INTEL_ECX: u32 = u32::from_le_bytes([b'n', b't', b'e', b'l']);
404
405pub fn cpu_manufacturer() -> CpuManufacturer {
406 let result = unsafe { __cpuid(MANUFACTURER_ID_FUNCTION) };
410 if result.ebx == AMD_EBX && result.edx == AMD_EDX && result.ecx == AMD_ECX {
411 return CpuManufacturer::Amd;
412 } else if result.ebx == INTEL_EBX && result.edx == INTEL_EDX && result.ecx == INTEL_ECX {
413 return CpuManufacturer::Intel;
414 }
415 CpuManufacturer::Unknown
416}
417
418#[cfg(test)]
419mod tests {
420 use super::*;
421
422 const VMX_LEAF: u32 = 1;
423 const SVM_LEAF: u32 = 0x80000001;
424
425 #[test]
426 fn cpu_manufacturer_test() {
427 let manufacturer = cpu_manufacturer();
429 assert_ne!(manufacturer, CpuManufacturer::Unknown);
430 }
431
432 fn fake_cpuid_count(_function: u32, _index: u32) -> CpuidResult {
433 CpuidResult {
434 eax: 27,
435 ebx: 18,
436 ecx: 28,
437 edx: 18,
438 }
439 }
440
441 fn fake_cpuid(function: u32) -> CpuidResult {
442 fake_cpuid_count(function, 0)
443 }
444
445 fn test_ctx(nested: NestedMode) -> CpuIdContext {
446 CpuIdContext {
447 vcpu_id: 0,
448 vcpu_count: 1,
449 x2apic: false,
450 tsc_deadline_timer: false,
451 apic_frequency: 0,
452 tsc_frequency: None,
453 cpu_config: CpuConfigX86_64 {
454 force_calibrated_tsc_leaf: false,
455 host_cpu_topology: true,
456 enable_hwp: false,
457 no_smt: false,
458 itmt: false,
459 hybrid_type: None,
460 nested,
461 },
462 cpuid_count: fake_cpuid_count,
463 cpuid: fake_cpuid,
464 }
465 }
466
467 fn ecx_entry(leaf: u32, ecx: u32) -> CpuIdEntry {
468 CpuIdEntry {
469 function: leaf,
470 index: 0,
471 flags: 0,
472 cpuid: CpuidResult {
473 eax: 0,
474 ebx: 0,
475 ecx,
476 edx: 0,
477 },
478 }
479 }
480
481 fn adjusted_ecx(mode: NestedMode, leaf: u32, ecx: u32) -> u32 {
482 let ctx = test_ctx(mode);
483 let mut entry = ecx_entry(leaf, ecx);
484 adjust_cpuid(&mut entry, &ctx);
485 entry.cpuid.ecx
486 }
487
488 #[test]
489 fn cpuid_copies_register() {
490 let ctx = test_ctx(NestedMode::Auto);
491 let mut cpu_id_entry = CpuIdEntry {
492 function: 0x4,
493 index: 0,
494 flags: 0,
495 cpuid: CpuidResult {
496 eax: 31,
497 ebx: 41,
498 ecx: 59,
499 edx: 26,
500 },
501 };
502 adjust_cpuid(&mut cpu_id_entry, &ctx);
503 assert_eq!(cpu_id_entry.cpuid.eax, 27)
504 }
505
506 #[test]
507 fn nested_off_clears_vmx_and_svm() {
508 let vmx = 1u32 << ECX_VMX_SHIFT;
509 assert_eq!(adjusted_ecx(NestedMode::Off, VMX_LEAF, vmx) & vmx, 0);
510 let svm = 1u32 << ECX_SVM_SHIFT;
511 assert_eq!(adjusted_ecx(NestedMode::Off, SVM_LEAF, svm) & svm, 0);
512 }
513
514 #[test]
515 fn nested_on_keeps_host_vmx_and_svm() {
516 let vmx = 1u32 << ECX_VMX_SHIFT;
517 assert_eq!(adjusted_ecx(NestedMode::On, VMX_LEAF, vmx) & vmx, vmx);
518 assert_eq!(adjusted_ecx(NestedMode::On, VMX_LEAF, 0) & vmx, 0);
519 let svm = 1u32 << ECX_SVM_SHIFT;
520 assert_eq!(adjusted_ecx(NestedMode::On, SVM_LEAF, svm) & svm, svm);
521 assert_eq!(adjusted_ecx(NestedMode::On, SVM_LEAF, 0) & svm, 0);
522 }
523
524 #[test]
525 fn nested_auto_keeps_host_vmx_and_svm() {
526 let vmx = 1u32 << ECX_VMX_SHIFT;
527 assert_eq!(adjusted_ecx(NestedMode::Auto, VMX_LEAF, vmx) & vmx, vmx);
528 assert_eq!(adjusted_ecx(NestedMode::Auto, VMX_LEAF, 0) & vmx, 0);
529 let svm = 1u32 << ECX_SVM_SHIFT;
530 assert_eq!(adjusted_ecx(NestedMode::Auto, SVM_LEAF, svm) & svm, svm);
531 assert_eq!(adjusted_ecx(NestedMode::Auto, SVM_LEAF, 0) & svm, 0);
532 }
533
534 #[test]
535 fn nested_required_intel_needs_vmx() {
536 let with_vmx = [ecx_entry(VMX_LEAF, 1 << ECX_VMX_SHIFT)];
537 assert!(nested_feature_available(&with_vmx, CpuManufacturer::Intel));
538 let with_svm = [ecx_entry(SVM_LEAF, 1 << ECX_SVM_SHIFT)];
539 assert!(!nested_feature_available(&with_svm, CpuManufacturer::Intel));
540 }
541
542 #[test]
543 fn nested_required_amd_needs_svm() {
544 let with_svm = [ecx_entry(SVM_LEAF, 1 << ECX_SVM_SHIFT)];
545 assert!(nested_feature_available(&with_svm, CpuManufacturer::Amd));
546 let with_vmx = [ecx_entry(VMX_LEAF, 1 << ECX_VMX_SHIFT)];
547 assert!(!nested_feature_available(&with_vmx, CpuManufacturer::Amd));
548 }
549
550 #[test]
551 fn nested_required_unknown_vendor_accepts_either() {
552 let with_vmx = [ecx_entry(VMX_LEAF, 1 << ECX_VMX_SHIFT)];
553 assert!(nested_feature_available(
554 &with_vmx,
555 CpuManufacturer::Unknown
556 ));
557 let with_svm = [ecx_entry(SVM_LEAF, 1 << ECX_SVM_SHIFT)];
558 assert!(nested_feature_available(
559 &with_svm,
560 CpuManufacturer::Unknown
561 ));
562 let none = [ecx_entry(VMX_LEAF, 0), ecx_entry(SVM_LEAF, 0)];
563 assert!(!nested_feature_available(&none, CpuManufacturer::Unknown));
564 }
565
566 #[test]
567 fn nested_required_absent_when_host_lacks_it() {
568 let none = [ecx_entry(VMX_LEAF, 0), ecx_entry(SVM_LEAF, 0)];
569 assert!(!nested_feature_available(&none, CpuManufacturer::Intel));
570 assert!(!nested_feature_available(&none, CpuManufacturer::Amd));
571 }
572}