devices/irqchip/kvm/
x86_64.rs

1// Copyright 2020 The ChromiumOS Authors
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use std::any::Any;
6use std::sync::Arc;
7
8use anyhow::anyhow;
9use anyhow::Context;
10use base::error;
11#[cfg(not(test))]
12use base::Clock;
13use base::Error;
14use base::Event;
15#[cfg(test)]
16use base::FakeClock as Clock;
17use base::Result;
18use base::Tube;
19use hypervisor::kvm::KvmCap;
20use hypervisor::kvm::KvmVcpu;
21use hypervisor::kvm::KvmVm;
22use hypervisor::IoapicState;
23use hypervisor::IrqRoute;
24use hypervisor::IrqSource;
25use hypervisor::IrqSourceChip;
26use hypervisor::LapicState;
27use hypervisor::MPState;
28use hypervisor::PicSelect;
29use hypervisor::PicState;
30use hypervisor::PitState;
31use hypervisor::Vcpu;
32use hypervisor::VcpuArch;
33use kvm_sys::*;
34use resources::SystemAllocator;
35use serde::Deserialize;
36use serde::Serialize;
37use snapshot::AnySnapshot;
38use sync::Mutex;
39
40use crate::irqchip::Ioapic;
41use crate::irqchip::IrqEvent;
42use crate::irqchip::IrqEventIndex;
43use crate::irqchip::Pic;
44use crate::irqchip::VcpuRunState;
45use crate::irqchip::IOAPIC_BASE_ADDRESS;
46use crate::irqchip::IOAPIC_MEM_LENGTH_BYTES;
47use crate::Bus;
48use crate::IrqChip;
49use crate::IrqChipCap;
50use crate::IrqChipX86_64;
51use crate::IrqEdgeEvent;
52use crate::IrqEventSource;
53use crate::IrqLevelEvent;
54use crate::Pit;
55use crate::PitError;
56
57/// PIT tube 0 timer is connected to IRQ 0
58const PIT_CHANNEL0_IRQ: u32 = 0;
59
60/// Default x86 routing table.  Pins 0-7 go to primary pic and ioapic, pins 8-15 go to secondary
61/// pic and ioapic, and pins 16-23 go only to the ioapic.
62fn kvm_default_irq_routing_table(ioapic_pins: usize) -> Vec<IrqRoute> {
63    let mut routes: Vec<IrqRoute> = Vec::new();
64
65    for i in 0..8 {
66        routes.push(IrqRoute::pic_irq_route(IrqSourceChip::PicPrimary, i));
67        routes.push(IrqRoute::ioapic_irq_route(i));
68    }
69    for i in 8..16 {
70        routes.push(IrqRoute::pic_irq_route(IrqSourceChip::PicSecondary, i));
71        routes.push(IrqRoute::ioapic_irq_route(i));
72    }
73    for i in 16..ioapic_pins as u32 {
74        routes.push(IrqRoute::ioapic_irq_route(i));
75    }
76
77    routes
78}
79
80/// Restores the Local APIC state and re-signals any pending IRR interrupt vectors via MSI.
81///
82/// On Intel hosts with APICv enabled running older Linux kernels (v4.11 through 6.5.11),
83/// KVM_SET_LAPIC clears the hardware Posted Interrupt Request (PIR) descriptor.
84/// Re-signaling pending IRR vectors via MSI ensures they are properly populated in both
85/// the LAPIC IRR and the hardware PIR descriptor so they are delivered upon guest resume.
86/// This function is idempotent on newer kernels. The already set interrupts get set again,
87/// which is an idempotent function.
88fn set_lapic_and_restore_irr(vm: &KvmVm, vcpu: &KvmVcpu, state: &LapicState) -> Result<()> {
89    vcpu.set_lapic(&kvm_lapic_state::from(state))?;
90
91    let pending_vectors = state.get_pending_irr_vectors();
92    if !pending_vectors.is_empty() {
93        let apic_id = state.get_apic_id();
94        for vec in pending_vectors {
95            if vec >= 32 {
96                vm.signal_msi_to_lapic(apic_id, vec)?;
97            }
98        }
99    }
100    Ok(())
101}
102
103/// IrqChip implementation where the entire IrqChip is emulated by KVM.
104///
105/// This implementation will use the KVM API to create and configure the in-kernel irqchip.
106pub struct KvmKernelIrqChip {
107    pub(super) vm: Arc<KvmVm>,
108    pub(super) vcpus: Mutex<Vec<Option<Arc<KvmVcpu>>>>,
109    pub(super) routes: Mutex<Vec<IrqRoute>>,
110}
111
112#[derive(Serialize, Deserialize)]
113struct KvmKernelIrqChipSnapshot {
114    routes: Vec<IrqRoute>,
115    // apic_base and interrupt_bitmap are part of the IrqChip, despite the
116    // fact that we get the values from the Vcpu ioctl "KVM_GET_SREGS".
117    // Contains 1 entry per Vcpu.
118    apic_base: Vec<u64>,
119    interrupt_bitmap: Vec<[u64; 4usize]>,
120}
121
122impl KvmKernelIrqChip {
123    /// Construct a new KvmKernelIrqchip.
124    pub fn new(vm: Arc<KvmVm>, num_vcpus: usize) -> Result<KvmKernelIrqChip> {
125        vm.create_irq_chip()?;
126        vm.create_pit()?;
127        let ioapic_pins = vm.get_ioapic_num_pins()?;
128
129        Ok(KvmKernelIrqChip {
130            vm,
131            vcpus: Mutex::new((0..num_vcpus).map(|_| None).collect()),
132            routes: Mutex::new(kvm_default_irq_routing_table(ioapic_pins)),
133        })
134    }
135}
136
137impl IrqChipX86_64 for KvmKernelIrqChip {
138    /// Get the current state of the PIC
139    fn get_pic_state(&self, select: PicSelect) -> Result<PicState> {
140        Ok(PicState::from(&self.vm.get_pic_state(select)?))
141    }
142
143    /// Set the current state of the PIC
144    fn set_pic_state(&self, select: PicSelect, state: &PicState) -> Result<()> {
145        self.vm.set_pic_state(select, &kvm_pic_state::from(state))
146    }
147
148    /// Get the current state of the IOAPIC
149    fn get_ioapic_state(&self) -> Result<IoapicState> {
150        Ok(IoapicState::from(&self.vm.get_ioapic_state()?))
151    }
152
153    /// Set the current state of the IOAPIC
154    fn set_ioapic_state(&self, state: &IoapicState) -> Result<()> {
155        self.vm.set_ioapic_state(&kvm_ioapic_state::from(state))
156    }
157
158    /// Get the current state of the specified VCPU's local APIC
159    fn get_lapic_state(&self, vcpu_id: usize) -> Result<LapicState> {
160        match self.vcpus.lock().get(vcpu_id) {
161            Some(Some(vcpu)) => Ok(LapicState::from(&vcpu.get_lapic()?)),
162            _ => Err(Error::new(libc::ENOENT)),
163        }
164    }
165
166    /// Set the current state of the specified VCPU's local APIC
167    fn set_lapic_state(&self, vcpu_id: usize, state: &LapicState) -> Result<()> {
168        match self.vcpus.lock().get(vcpu_id) {
169            Some(Some(vcpu)) => set_lapic_and_restore_irr(&self.vm, vcpu, state),
170            _ => Err(Error::new(libc::ENOENT)),
171        }
172    }
173
174    /// Get the lapic frequency in Hz
175    fn lapic_frequency(&self) -> u32 {
176        // KVM emulates the lapic to have a bus frequency of 1GHz
177        1_000_000_000
178    }
179
180    /// Retrieves the state of the PIT. Gets the pit state via the KVM API.
181    fn get_pit(&self) -> Result<PitState> {
182        Ok(PitState::from(&self.vm.get_pit_state()?))
183    }
184
185    /// Sets the state of the PIT. Sets the pit state via the KVM API.
186    fn set_pit(&self, state: &PitState) -> Result<()> {
187        self.vm.set_pit_state(&kvm_pit_state2::from(state))
188    }
189
190    /// Returns true if the PIT uses port 0x61 for the PC speaker, false if 0x61 is unused.
191    /// KVM's kernel PIT doesn't use 0x61.
192    fn pit_uses_speaker_port(&self) -> bool {
193        false
194    }
195
196    fn snapshot_chip_specific(&self) -> anyhow::Result<AnySnapshot> {
197        let mut apics: Vec<u64> = Vec::new();
198        let mut interrupt_bitmaps: Vec<[u64; 4usize]> = Vec::new();
199        {
200            let vcpus_lock = self.vcpus.lock();
201            for vcpu in (*vcpus_lock).iter().flatten() {
202                apics.push(vcpu.get_apic_base()?);
203                interrupt_bitmaps.push(vcpu.get_interrupt_bitmap()?);
204            }
205        }
206        AnySnapshot::to_any(KvmKernelIrqChipSnapshot {
207            routes: self.routes.lock().clone(),
208            apic_base: apics,
209            interrupt_bitmap: interrupt_bitmaps,
210        })
211        .context("failed to serialize KvmKernelIrqChip")
212    }
213
214    fn restore_chip_specific(&self, data: AnySnapshot) -> anyhow::Result<()> {
215        let deser: KvmKernelIrqChipSnapshot =
216            AnySnapshot::from_any(data).context("failed to deserialize data")?;
217        self.set_irq_routes(&deser.routes)?;
218        let vcpus_lock = self.vcpus.lock();
219        assert_eq!(deser.interrupt_bitmap.len(), vcpus_lock.len());
220        assert_eq!(deser.apic_base.len(), vcpus_lock.len());
221        for (i, vcpu) in vcpus_lock.iter().enumerate() {
222            if let Some(vcpu) = vcpu {
223                vcpu.set_apic_base(*deser.apic_base.get(i).unwrap())?;
224                vcpu.set_interrupt_bitmap(*deser.interrupt_bitmap.get(i).unwrap())?;
225            } else {
226                return Err(anyhow!(
227                    "Received None instead of Vcpu while restoring apic_base and interrupt_bitmap"
228                ));
229            }
230        }
231        Ok(())
232    }
233}
234
235/// The KvmSplitIrqsChip supports KVM's SPLIT_IRQCHIP feature, where the PIC and IOAPIC
236/// are emulated in userspace, while the local APICs are emulated in the kernel.
237/// The SPLIT_IRQCHIP feature only supports x86/x86_64 so we only define this IrqChip in crosvm
238/// for x86/x86_64.
239pub struct KvmSplitIrqChip {
240    vm: Arc<KvmVm>,
241    vcpus: Arc<Mutex<Vec<Option<Arc<KvmVcpu>>>>>,
242    routes: Arc<Mutex<Vec<IrqRoute>>>,
243    pit: Arc<Mutex<Pit>>,
244    pic: Arc<Mutex<Pic>>,
245    ioapic: Arc<Mutex<Ioapic>>,
246    ioapic_pins: usize,
247    /// Vec of ioapic irq events that have been delayed because the ioapic was locked when
248    /// service_irq was called on the irqchip. This prevents deadlocks when a Vcpu thread has
249    /// locked the ioapic and the ioapic sends a AddMsiRoute signal to the main thread (which
250    /// itself may be busy trying to call service_irq).
251    delayed_ioapic_irq_events: Arc<Mutex<Vec<usize>>>,
252    /// Event which is meant to trigger process of any irqs events that were delayed.
253    delayed_ioapic_irq_trigger: Event,
254    /// Array of Events that devices will use to assert ioapic pins.
255    irq_events: Arc<Mutex<Vec<Option<IrqEvent>>>>,
256}
257
258fn kvm_dummy_msi_routes(ioapic_pins: usize) -> Vec<IrqRoute> {
259    let mut routes: Vec<IrqRoute> = Vec::new();
260    for i in 0..ioapic_pins {
261        routes.push(
262            // Add dummy MSI routes to replace the default IRQChip routes.
263            IrqRoute {
264                gsi: i as u32,
265                source: IrqSource::Msi {
266                    address: 0,
267                    data: 0,
268                },
269            },
270        );
271    }
272    routes
273}
274
275impl KvmSplitIrqChip {
276    /// Construct a new KvmSplitIrqChip.
277    pub fn new(
278        vm: Arc<KvmVm>,
279        num_vcpus: usize,
280        irq_tube: Tube,
281        ioapic_pins: Option<usize>,
282    ) -> Result<Self> {
283        let ioapic_pins = ioapic_pins.unwrap_or(vm.get_ioapic_num_pins()?);
284        vm.enable_split_irqchip(ioapic_pins)?;
285        let pit_evt = IrqEdgeEvent::new()?;
286        let pit = Pit::new(pit_evt.try_clone()?, Arc::new(Mutex::new(Clock::new()))).map_err(
287            |e| match e {
288                PitError::CloneEvent(err) => err,
289                PitError::CreateEvent(err) => err,
290                PitError::CreateWaitContext(err) => err,
291                PitError::WaitError(err) => err,
292                PitError::TimerCreateError(err) => err,
293                PitError::SpawnThread(_) => Error::new(libc::EIO),
294            },
295        )?;
296
297        let pit_event_source = IrqEventSource::from_device(&pit);
298
299        let chip = KvmSplitIrqChip {
300            vm,
301            vcpus: Arc::new(Mutex::new((0..num_vcpus).map(|_| None).collect())),
302            routes: Arc::new(Mutex::new(Vec::new())),
303            pit: Arc::new(Mutex::new(pit)),
304            pic: Arc::new(Mutex::new(Pic::new())),
305            ioapic: Arc::new(Mutex::new(Ioapic::new(irq_tube, ioapic_pins)?)),
306            ioapic_pins,
307            delayed_ioapic_irq_events: Arc::new(Mutex::new(Vec::new())),
308            delayed_ioapic_irq_trigger: Event::new()?,
309            irq_events: Arc::new(Mutex::new(Default::default())),
310        };
311
312        // Setup standard x86 irq routes
313        let mut routes = kvm_default_irq_routing_table(ioapic_pins);
314        // Add dummy MSI routes for the first ioapic_pins GSIs
315        routes.append(&mut kvm_dummy_msi_routes(ioapic_pins));
316
317        // Set the routes so they get sent to KVM
318        chip.set_irq_routes(&routes)?;
319
320        chip.register_edge_irq_event(PIT_CHANNEL0_IRQ, &pit_evt, pit_event_source)?;
321        Ok(chip)
322    }
323}
324
325impl KvmSplitIrqChip {
326    /// Convenience function for determining which chips the supplied irq routes to.
327    fn routes_to_chips(&self, irq: u32) -> Vec<(IrqSourceChip, u32)> {
328        let mut chips = Vec::new();
329        for route in self.routes.lock().iter() {
330            match route {
331                IrqRoute {
332                    gsi,
333                    source: IrqSource::Irqchip { chip, pin },
334                } if *gsi == irq => match chip {
335                    IrqSourceChip::PicPrimary
336                    | IrqSourceChip::PicSecondary
337                    | IrqSourceChip::Ioapic => chips.push((*chip, *pin)),
338                    IrqSourceChip::Gic => {
339                        error!("gic irq should not be possible on a KvmSplitIrqChip")
340                    }
341                    IrqSourceChip::Aia => {
342                        error!("Aia irq should not be possible on x86_64")
343                    }
344                },
345                // Ignore MSIs and other routes
346                _ => {}
347            }
348        }
349        chips
350    }
351
352    /// Return true if there is a pending interrupt for the specified vcpu. For KvmSplitIrqChip
353    /// this calls interrupt_requested on the pic.
354    pub fn interrupt_requested(&self, vcpu_id: usize) -> bool {
355        // Pic interrupts for the split irqchip only go to vcpu 0
356        if vcpu_id != 0 {
357            return false;
358        }
359        self.pic.lock().interrupt_requested()
360    }
361
362    /// Check if the specified vcpu has any pending interrupts. Returns [`None`] for no interrupts,
363    /// otherwise [`Some::<u8>`] should be the injected interrupt vector. For [`KvmSplitIrqChip`]
364    /// this calls `get_external_interrupt` on the pic.
365    pub fn get_external_interrupt(&self, vcpu_id: usize) -> Option<u8> {
366        // Pic interrupts for the split irqchip only go to vcpu 0
367        if vcpu_id != 0 {
368            return None;
369        }
370        self.pic.lock().get_external_interrupt()
371    }
372
373    /// Register an event that can trigger an interrupt for a particular GSI.
374    fn register_irq_event(
375        &self,
376        irq: u32,
377        irq_event: &Event,
378        resample_event: Option<&Event>,
379        source: IrqEventSource,
380    ) -> Result<Option<IrqEventIndex>> {
381        if irq < self.ioapic_pins as u32 {
382            let mut evt = IrqEvent {
383                gsi: irq,
384                event: irq_event.try_clone()?,
385                resample_event: None,
386                source,
387            };
388
389            if let Some(resample_event) = resample_event {
390                evt.resample_event = Some(resample_event.try_clone()?);
391            }
392
393            let mut irq_events = self.irq_events.lock();
394            let index = irq_events.len();
395            irq_events.push(Some(evt));
396            Ok(Some(index))
397        } else {
398            self.vm.register_irqfd(irq, irq_event, resample_event)?;
399            Ok(None)
400        }
401    }
402
403    /// Unregister an event for a particular GSI.
404    fn unregister_irq_event(&self, irq: u32, irq_event: &Event) -> Result<()> {
405        if irq < self.ioapic_pins as u32 {
406            let mut irq_events = self.irq_events.lock();
407            for (index, evt) in irq_events.iter().enumerate() {
408                if let Some(evt) = evt {
409                    if evt.gsi == irq && irq_event.eq(&evt.event) {
410                        irq_events[index] = None;
411                        break;
412                    }
413                }
414            }
415            Ok(())
416        } else {
417            self.vm.unregister_irqfd(irq, irq_event)
418        }
419    }
420}
421
422/// Convenience function for determining whether or not two irq routes conflict.
423/// Returns true if they conflict.
424fn routes_conflict(route: &IrqRoute, other: &IrqRoute) -> bool {
425    // They don't conflict if they have different GSIs.
426    if route.gsi != other.gsi {
427        return false;
428    }
429
430    // If they're both MSI with the same GSI then they conflict.
431    if let (IrqSource::Msi { .. }, IrqSource::Msi { .. }) = (route.source, other.source) {
432        return true;
433    }
434
435    // If the route chips match and they have the same GSI then they conflict.
436    if let (
437        IrqSource::Irqchip {
438            chip: route_chip, ..
439        },
440        IrqSource::Irqchip {
441            chip: other_chip, ..
442        },
443    ) = (route.source, other.source)
444    {
445        return route_chip == other_chip;
446    }
447
448    // Otherwise they do not conflict.
449    false
450}
451
452/// This IrqChip only works with Kvm so we only implement it for KvmVcpu.
453impl IrqChip for KvmSplitIrqChip {
454    /// Add a vcpu to the irq chip.
455    fn add_vcpu(&self, vcpu_id: usize, vcpu: Arc<dyn VcpuArch>) -> Result<()> {
456        let vcpu = Arc::downcast(vcpu)
457            .map_err(|_| ())
458            .expect("KvmSplitIrqChip::add_vcpu called with non-KvmVcpu");
459        self.vcpus.lock()[vcpu_id] = Some(vcpu);
460        Ok(())
461    }
462
463    /// Register an event that can trigger an interrupt for a particular GSI.
464    fn register_edge_irq_event(
465        &self,
466        irq: u32,
467        irq_event: &IrqEdgeEvent,
468        source: IrqEventSource,
469    ) -> Result<Option<IrqEventIndex>> {
470        self.register_irq_event(irq, irq_event.get_trigger(), None, source)
471    }
472
473    fn unregister_edge_irq_event(&self, irq: u32, irq_event: &IrqEdgeEvent) -> Result<()> {
474        self.unregister_irq_event(irq, irq_event.get_trigger())
475    }
476
477    fn register_level_irq_event(
478        &self,
479        irq: u32,
480        irq_event: &IrqLevelEvent,
481        source: IrqEventSource,
482    ) -> Result<Option<IrqEventIndex>> {
483        self.register_irq_event(
484            irq,
485            irq_event.get_trigger(),
486            Some(irq_event.get_resample()),
487            source,
488        )
489    }
490
491    fn unregister_level_irq_event(&self, irq: u32, irq_event: &IrqLevelEvent) -> Result<()> {
492        self.unregister_irq_event(irq, irq_event.get_trigger())
493    }
494
495    /// Route an IRQ line to an interrupt controller, or to a particular MSI vector.
496    fn route_irq(&self, route: IrqRoute) -> Result<()> {
497        let mut routes = self.routes.lock();
498        routes.retain(|r| !routes_conflict(r, &route));
499
500        routes.push(route);
501
502        // We only call set_gsi_routing with the msi routes
503        let mut msi_routes = routes.clone();
504        msi_routes.retain(|r| matches!(r.source, IrqSource::Msi { .. }));
505
506        self.vm.set_gsi_routing(&msi_routes)
507    }
508
509    /// Replace all irq routes with the supplied routes
510    fn set_irq_routes(&self, routes: &[IrqRoute]) -> Result<()> {
511        let mut current_routes = self.routes.lock();
512        *current_routes = routes.to_vec();
513
514        // We only call set_gsi_routing with the msi routes
515        let mut msi_routes = routes.to_vec();
516        msi_routes.retain(|r| matches!(r.source, IrqSource::Msi { .. }));
517
518        self.vm.set_gsi_routing(&msi_routes)
519    }
520
521    /// Return a vector of all registered irq numbers and their associated events and event
522    /// indices. These should be used by the main thread to wait for irq events.
523    fn irq_event_tokens(&self) -> Result<Vec<(IrqEventIndex, IrqEventSource, Event)>> {
524        let mut tokens = vec![];
525        for (index, evt) in self.irq_events.lock().iter().enumerate() {
526            if let Some(evt) = evt {
527                tokens.push((index, evt.source.clone(), evt.event.try_clone()?));
528            }
529        }
530        Ok(tokens)
531    }
532
533    /// Either assert or deassert an IRQ line.  Sends to either an interrupt controller, or does
534    /// a send_msi if the irq is associated with an MSI.
535    fn service_irq(&self, irq: u32, level: bool) -> Result<()> {
536        let chips = self.routes_to_chips(irq);
537        for (chip, pin) in chips {
538            match chip {
539                IrqSourceChip::PicPrimary | IrqSourceChip::PicSecondary => {
540                    self.pic.lock().service_irq(pin as u8, level);
541                }
542                IrqSourceChip::Ioapic => {
543                    self.ioapic.lock().service_irq(pin as usize, level);
544                }
545                _ => {}
546            }
547        }
548        Ok(())
549    }
550
551    /// Service an IRQ event by asserting then deasserting an IRQ line. The associated Event
552    /// that triggered the irq event will be read from. If the irq is associated with a resample
553    /// Event, then the deassert will only happen after an EOI is broadcast for a vector
554    /// associated with the irq line.
555    /// For the KvmSplitIrqChip, this function identifies which chips the irq routes to, then
556    /// attempts to call service_irq on those chips. If the ioapic is unable to be immediately
557    /// locked, we add the irq to the delayed_ioapic_irq_events Vec (though we still read
558    /// from the Event that triggered the irq event).
559    fn service_irq_event(&self, event_index: IrqEventIndex) -> Result<()> {
560        if let Some(evt) = &self.irq_events.lock()[event_index] {
561            evt.event.wait()?;
562            let chips = self.routes_to_chips(evt.gsi);
563
564            for (chip, pin) in chips {
565                match chip {
566                    IrqSourceChip::PicPrimary | IrqSourceChip::PicSecondary => {
567                        let mut pic = self.pic.lock();
568                        pic.service_irq(pin as u8, true);
569                        if evt.resample_event.is_none() {
570                            pic.service_irq(pin as u8, false);
571                        }
572                    }
573                    IrqSourceChip::Ioapic => {
574                        if let Ok(mut ioapic) = self.ioapic.try_lock() {
575                            ioapic.service_irq(pin as usize, true);
576                            if evt.resample_event.is_none() {
577                                ioapic.service_irq(pin as usize, false);
578                            }
579                        } else {
580                            self.delayed_ioapic_irq_events.lock().push(event_index);
581                            self.delayed_ioapic_irq_trigger.signal().unwrap();
582                        }
583                    }
584                    _ => {}
585                }
586            }
587        }
588
589        Ok(())
590    }
591
592    /// Broadcast an end of interrupt. For KvmSplitIrqChip this sends the EOI to the ioapic
593    fn broadcast_eoi(&self, vector: u8) -> Result<()> {
594        self.ioapic.lock().end_of_interrupt(vector);
595        Ok(())
596    }
597
598    /// Injects any pending interrupts for `vcpu`.
599    /// For KvmSplitIrqChip this injects any PIC interrupts on vcpu_id 0.
600    fn inject_interrupts(&self, vcpu: &dyn VcpuArch) -> Result<()> {
601        let vcpu: &KvmVcpu = <dyn Any>::downcast_ref(vcpu)
602            .expect("KvmSplitIrqChip::add_vcpu called with non-KvmVcpu");
603
604        let vcpu_id = vcpu.id();
605        if !self.interrupt_requested(vcpu_id) || !vcpu.ready_for_interrupt() {
606            return Ok(());
607        }
608
609        if let Some(vector) = self.get_external_interrupt(vcpu_id) {
610            vcpu.interrupt(vector)?;
611        }
612
613        // The second interrupt request should be handled immediately, so ask vCPU to exit as soon
614        // as possible.
615        if self.interrupt_requested(vcpu_id) {
616            vcpu.set_interrupt_window_requested(true);
617        }
618        Ok(())
619    }
620
621    /// Notifies the irq chip that the specified VCPU has executed a halt instruction.
622    /// For KvmSplitIrqChip this is a no-op because KVM handles VCPU blocking.
623    fn halted(&self, _vcpu_id: usize) {}
624
625    /// Blocks until `vcpu` is in a runnable state or until interrupted by
626    /// `IrqChip::kick_halted_vcpus`.  Returns `VcpuRunState::Runnable if vcpu is runnable, or
627    /// `VcpuRunState::Interrupted` if the wait was interrupted.
628    /// For KvmSplitIrqChip this is a no-op and always returns Runnable because KVM handles VCPU
629    /// blocking.
630    fn wait_until_runnable(&self, _vcpu: &dyn VcpuArch) -> Result<VcpuRunState> {
631        Ok(VcpuRunState::Runnable)
632    }
633
634    /// Makes unrunnable VCPUs return immediately from `wait_until_runnable`.
635    /// For KvmSplitIrqChip this is a no-op because KVM handles VCPU blocking.
636    fn kick_halted_vcpus(&self) {}
637
638    /// Get the current MP state of the specified VCPU.
639    fn get_mp_state(&self, vcpu_id: usize) -> Result<MPState> {
640        match self.vcpus.lock().get(vcpu_id) {
641            Some(Some(vcpu)) => Ok(MPState::from(&vcpu.get_mp_state()?)),
642            _ => Err(Error::new(libc::ENOENT)),
643        }
644    }
645
646    /// Set the current MP state of the specified VCPU.
647    fn set_mp_state(&self, vcpu_id: usize, state: &MPState) -> Result<()> {
648        match self.vcpus.lock().get(vcpu_id) {
649            Some(Some(vcpu)) => vcpu.set_mp_state(&kvm_mp_state::from(state)),
650            _ => Err(Error::new(libc::ENOENT)),
651        }
652    }
653
654    /// Finalize irqchip setup. Should be called once all devices have registered irq events and
655    /// been added to the io_bus and mmio_bus.
656    fn finalize_devices(
657        self: Arc<Self>,
658        resources: &mut SystemAllocator,
659        io_bus: &Bus,
660        mmio_bus: &Bus,
661    ) -> Result<()> {
662        // Insert pit into io_bus
663        io_bus.insert(self.pit.clone(), 0x040, 0x8).unwrap();
664        io_bus.insert(self.pit.clone(), 0x061, 0x1).unwrap();
665
666        // Insert pic into io_bus
667        io_bus.insert(self.pic.clone(), 0x20, 0x2).unwrap();
668        io_bus.insert(self.pic.clone(), 0xa0, 0x2).unwrap();
669        io_bus.insert(self.pic.clone(), 0x4d0, 0x2).unwrap();
670
671        // Insert ioapic into mmio_bus
672        mmio_bus
673            .insert(
674                self.ioapic.clone(),
675                IOAPIC_BASE_ADDRESS,
676                IOAPIC_MEM_LENGTH_BYTES,
677            )
678            .unwrap();
679
680        // At this point, all of our devices have been created and they have registered their
681        // irq events, so we can clone our resample events
682        let mut ioapic_resample_events: Vec<Vec<Event>> =
683            (0..self.ioapic_pins).map(|_| Vec::new()).collect();
684        let mut pic_resample_events: Vec<Vec<Event>> =
685            (0..self.ioapic_pins).map(|_| Vec::new()).collect();
686
687        for evt in self.irq_events.lock().iter().flatten() {
688            if (evt.gsi as usize) >= self.ioapic_pins {
689                continue;
690            }
691            if let Some(resample_evt) = &evt.resample_event {
692                ioapic_resample_events[evt.gsi as usize].push(resample_evt.try_clone()?);
693                pic_resample_events[evt.gsi as usize].push(resample_evt.try_clone()?);
694            }
695        }
696
697        // Register resample events with the ioapic
698        self.ioapic
699            .lock()
700            .register_resample_events(ioapic_resample_events);
701        // Register resample events with the pic
702        self.pic
703            .lock()
704            .register_resample_events(pic_resample_events);
705
706        // Make sure all future irq numbers are beyond IO-APIC range.
707        let mut irq_num = resources.allocate_irq().unwrap();
708        while irq_num < self.ioapic_pins as u32 {
709            irq_num = resources.allocate_irq().unwrap();
710        }
711
712        Ok(())
713    }
714
715    /// The KvmSplitIrqChip's ioapic may be locked because a vcpu thread is currently writing to
716    /// the ioapic, and the ioapic may be blocking on adding MSI routes, which requires blocking
717    /// socket communication back to the main thread.  Thus, we do not want the main thread to
718    /// block on a locked ioapic, so any irqs that could not be serviced because the ioapic could
719    /// not be immediately locked are added to the delayed_ioapic_irq_events Vec. This function
720    /// processes each delayed event in the vec each time it's called. If the ioapic is still
721    /// locked, we keep the queued irqs for the next time this function is called.
722    fn process_delayed_irq_events(&self) -> Result<()> {
723        self.delayed_ioapic_irq_events
724            .lock()
725            .retain(|&event_index| {
726                if let Some(evt) = &self.irq_events.lock()[event_index] {
727                    if let Ok(mut ioapic) = self.ioapic.try_lock() {
728                        ioapic.service_irq(evt.gsi as usize, true);
729                        if evt.resample_event.is_none() {
730                            ioapic.service_irq(evt.gsi as usize, false);
731                        }
732
733                        false
734                    } else {
735                        true
736                    }
737                } else {
738                    true
739                }
740            });
741
742        if self.delayed_ioapic_irq_events.lock().is_empty() {
743            self.delayed_ioapic_irq_trigger.wait()?;
744        }
745
746        Ok(())
747    }
748
749    fn irq_delayed_event_token(&self) -> Result<Option<Event>> {
750        Ok(Some(self.delayed_ioapic_irq_trigger.try_clone()?))
751    }
752
753    fn check_capability(&self, c: IrqChipCap) -> bool {
754        match c {
755            IrqChipCap::TscDeadlineTimer => self.vm.check_raw_capability(KvmCap::TscDeadlineTimer),
756            IrqChipCap::X2Apic => true,
757            IrqChipCap::MpStateGetSet => true,
758        }
759    }
760}
761
762#[derive(Serialize, Deserialize)]
763struct KvmSplitIrqChipSnapshot {
764    routes: Vec<IrqRoute>,
765}
766
767impl IrqChipX86_64 for KvmSplitIrqChip {
768    /// Get the current state of the PIC
769    fn get_pic_state(&self, select: PicSelect) -> Result<PicState> {
770        Ok(self.pic.lock().get_pic_state(select))
771    }
772
773    /// Set the current state of the PIC
774    fn set_pic_state(&self, select: PicSelect, state: &PicState) -> Result<()> {
775        self.pic.lock().set_pic_state(select, state);
776        Ok(())
777    }
778
779    /// Get the current state of the IOAPIC
780    fn get_ioapic_state(&self) -> Result<IoapicState> {
781        Ok(self.ioapic.lock().get_ioapic_state())
782    }
783
784    /// Set the current state of the IOAPIC
785    fn set_ioapic_state(&self, state: &IoapicState) -> Result<()> {
786        self.ioapic.lock().set_ioapic_state(state);
787        Ok(())
788    }
789
790    /// Get the current state of the specified VCPU's local APIC
791    fn get_lapic_state(&self, vcpu_id: usize) -> Result<LapicState> {
792        match self.vcpus.lock().get(vcpu_id) {
793            Some(Some(vcpu)) => Ok(LapicState::from(&vcpu.get_lapic()?)),
794            _ => Err(Error::new(libc::ENOENT)),
795        }
796    }
797
798    /// Set the current state of the specified VCPU's local APIC
799    fn set_lapic_state(&self, vcpu_id: usize, state: &LapicState) -> Result<()> {
800        match self.vcpus.lock().get(vcpu_id) {
801            Some(Some(vcpu)) => set_lapic_and_restore_irr(&self.vm, vcpu, state),
802            _ => Err(Error::new(libc::ENOENT)),
803        }
804    }
805
806    /// Get the lapic frequency in Hz
807    fn lapic_frequency(&self) -> u32 {
808        // KVM emulates the lapic to have a bus frequency of 1GHz
809        1_000_000_000
810    }
811
812    /// Retrieves the state of the PIT. Gets the pit state via the KVM API.
813    fn get_pit(&self) -> Result<PitState> {
814        Ok(self.pit.lock().get_pit_state())
815    }
816
817    /// Sets the state of the PIT. Sets the pit state via the KVM API.
818    fn set_pit(&self, state: &PitState) -> Result<()> {
819        self.pit.lock().set_pit_state(state);
820        Ok(())
821    }
822
823    /// Returns true if the PIT uses port 0x61 for the PC speaker, false if 0x61 is unused.
824    /// devices::Pit uses 0x61.
825    fn pit_uses_speaker_port(&self) -> bool {
826        true
827    }
828
829    fn snapshot_chip_specific(&self) -> anyhow::Result<AnySnapshot> {
830        AnySnapshot::to_any(KvmSplitIrqChipSnapshot {
831            routes: self.routes.lock().clone(),
832        })
833        .context("failed to serialize KvmSplitIrqChip")
834    }
835
836    fn restore_chip_specific(&self, data: AnySnapshot) -> anyhow::Result<()> {
837        let deser: KvmSplitIrqChipSnapshot =
838            AnySnapshot::from_any(data).context("failed to deserialize KvmSplitIrqChip")?;
839        self.set_irq_routes(&deser.routes)?;
840        Ok(())
841    }
842}
devices/irqchip/kvm/x86_64.rs

devices/irqchip/kvm/
x86_64.rs
