devices/irqchip/kvm/
x86_64.rs

1// Copyright 2020 The ChromiumOS Authors
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use std::any::Any;
6use std::sync::Arc;
7
8use anyhow::anyhow;
9use anyhow::Context;
10use base::error;
11#[cfg(not(test))]
12use base::Clock;
13use base::Error;
14use base::Event;
15#[cfg(test)]
16use base::FakeClock as Clock;
17use base::Result;
18use base::Tube;
19use hypervisor::kvm::KvmCap;
20use hypervisor::kvm::KvmVcpu;
21use hypervisor::kvm::KvmVm;
22use hypervisor::IoapicState;
23use hypervisor::IrqRoute;
24use hypervisor::IrqSource;
25use hypervisor::IrqSourceChip;
26use hypervisor::LapicState;
27use hypervisor::MPState;
28use hypervisor::PicSelect;
29use hypervisor::PicState;
30use hypervisor::PitState;
31use hypervisor::Vcpu;
32use hypervisor::VcpuArch;
33use kvm_sys::*;
34use resources::SystemAllocator;
35use serde::Deserialize;
36use serde::Serialize;
37use snapshot::AnySnapshot;
38use sync::Mutex;
39
40use crate::irqchip::Ioapic;
41use crate::irqchip::IrqEvent;
42use crate::irqchip::IrqEventIndex;
43use crate::irqchip::Pic;
44use crate::irqchip::VcpuRunState;
45use crate::irqchip::IOAPIC_BASE_ADDRESS;
46use crate::irqchip::IOAPIC_MEM_LENGTH_BYTES;
47use crate::Bus;
48use crate::IrqChip;
49use crate::IrqChipCap;
50use crate::IrqChipX86_64;
51use crate::IrqEdgeEvent;
52use crate::IrqEventSource;
53use crate::IrqLevelEvent;
54use crate::Pit;
55use crate::PitError;
56
57/// PIT tube 0 timer is connected to IRQ 0
58const PIT_CHANNEL0_IRQ: u32 = 0;
59
60/// Default x86 routing table.  Pins 0-7 go to primary pic and ioapic, pins 8-15 go to secondary
61/// pic and ioapic, and pins 16-23 go only to the ioapic.
62fn kvm_default_irq_routing_table(ioapic_pins: usize) -> Vec<IrqRoute> {
63    let mut routes: Vec<IrqRoute> = Vec::new();
64
65    for i in 0..8 {
66        routes.push(IrqRoute::pic_irq_route(IrqSourceChip::PicPrimary, i));
67        routes.push(IrqRoute::ioapic_irq_route(i));
68    }
69    for i in 8..16 {
70        routes.push(IrqRoute::pic_irq_route(IrqSourceChip::PicSecondary, i));
71        routes.push(IrqRoute::ioapic_irq_route(i));
72    }
73    for i in 16..ioapic_pins as u32 {
74        routes.push(IrqRoute::ioapic_irq_route(i));
75    }
76
77    routes
78}
79
80/// IrqChip implementation where the entire IrqChip is emulated by KVM.
81///
82/// This implementation will use the KVM API to create and configure the in-kernel irqchip.
83pub struct KvmKernelIrqChip {
84    pub(super) vm: Arc<KvmVm>,
85    pub(super) vcpus: Mutex<Vec<Option<Arc<KvmVcpu>>>>,
86    pub(super) routes: Mutex<Vec<IrqRoute>>,
87}
88
89#[derive(Serialize, Deserialize)]
90struct KvmKernelIrqChipSnapshot {
91    routes: Vec<IrqRoute>,
92    // apic_base and interrupt_bitmap are part of the IrqChip, despite the
93    // fact that we get the values from the Vcpu ioctl "KVM_GET_SREGS".
94    // Contains 1 entry per Vcpu.
95    apic_base: Vec<u64>,
96    interrupt_bitmap: Vec<[u64; 4usize]>,
97}
98
99impl KvmKernelIrqChip {
100    /// Construct a new KvmKernelIrqchip.
101    pub fn new(vm: Arc<KvmVm>, num_vcpus: usize) -> Result<KvmKernelIrqChip> {
102        vm.create_irq_chip()?;
103        vm.create_pit()?;
104        let ioapic_pins = vm.get_ioapic_num_pins()?;
105
106        Ok(KvmKernelIrqChip {
107            vm,
108            vcpus: Mutex::new((0..num_vcpus).map(|_| None).collect()),
109            routes: Mutex::new(kvm_default_irq_routing_table(ioapic_pins)),
110        })
111    }
112}
113
114impl IrqChipX86_64 for KvmKernelIrqChip {
115    /// Get the current state of the PIC
116    fn get_pic_state(&self, select: PicSelect) -> Result<PicState> {
117        Ok(PicState::from(&self.vm.get_pic_state(select)?))
118    }
119
120    /// Set the current state of the PIC
121    fn set_pic_state(&self, select: PicSelect, state: &PicState) -> Result<()> {
122        self.vm.set_pic_state(select, &kvm_pic_state::from(state))
123    }
124
125    /// Get the current state of the IOAPIC
126    fn get_ioapic_state(&self) -> Result<IoapicState> {
127        Ok(IoapicState::from(&self.vm.get_ioapic_state()?))
128    }
129
130    /// Set the current state of the IOAPIC
131    fn set_ioapic_state(&self, state: &IoapicState) -> Result<()> {
132        self.vm.set_ioapic_state(&kvm_ioapic_state::from(state))
133    }
134
135    /// Get the current state of the specified VCPU's local APIC
136    fn get_lapic_state(&self, vcpu_id: usize) -> Result<LapicState> {
137        match self.vcpus.lock().get(vcpu_id) {
138            Some(Some(vcpu)) => Ok(LapicState::from(&vcpu.get_lapic()?)),
139            _ => Err(Error::new(libc::ENOENT)),
140        }
141    }
142
143    /// Set the current state of the specified VCPU's local APIC
144    fn set_lapic_state(&self, vcpu_id: usize, state: &LapicState) -> Result<()> {
145        match self.vcpus.lock().get(vcpu_id) {
146            Some(Some(vcpu)) => vcpu.set_lapic(&kvm_lapic_state::from(state)),
147            _ => Err(Error::new(libc::ENOENT)),
148        }
149    }
150
151    /// Get the lapic frequency in Hz
152    fn lapic_frequency(&self) -> u32 {
153        // KVM emulates the lapic to have a bus frequency of 1GHz
154        1_000_000_000
155    }
156
157    /// Retrieves the state of the PIT. Gets the pit state via the KVM API.
158    fn get_pit(&self) -> Result<PitState> {
159        Ok(PitState::from(&self.vm.get_pit_state()?))
160    }
161
162    /// Sets the state of the PIT. Sets the pit state via the KVM API.
163    fn set_pit(&self, state: &PitState) -> Result<()> {
164        self.vm.set_pit_state(&kvm_pit_state2::from(state))
165    }
166
167    /// Returns true if the PIT uses port 0x61 for the PC speaker, false if 0x61 is unused.
168    /// KVM's kernel PIT doesn't use 0x61.
169    fn pit_uses_speaker_port(&self) -> bool {
170        false
171    }
172
173    fn snapshot_chip_specific(&self) -> anyhow::Result<AnySnapshot> {
174        let mut apics: Vec<u64> = Vec::new();
175        let mut interrupt_bitmaps: Vec<[u64; 4usize]> = Vec::new();
176        {
177            let vcpus_lock = self.vcpus.lock();
178            for vcpu in (*vcpus_lock).iter().flatten() {
179                apics.push(vcpu.get_apic_base()?);
180                interrupt_bitmaps.push(vcpu.get_interrupt_bitmap()?);
181            }
182        }
183        AnySnapshot::to_any(KvmKernelIrqChipSnapshot {
184            routes: self.routes.lock().clone(),
185            apic_base: apics,
186            interrupt_bitmap: interrupt_bitmaps,
187        })
188        .context("failed to serialize KvmKernelIrqChip")
189    }
190
191    fn restore_chip_specific(&self, data: AnySnapshot) -> anyhow::Result<()> {
192        let deser: KvmKernelIrqChipSnapshot =
193            AnySnapshot::from_any(data).context("failed to deserialize data")?;
194        self.set_irq_routes(&deser.routes)?;
195        let vcpus_lock = self.vcpus.lock();
196        assert_eq!(deser.interrupt_bitmap.len(), vcpus_lock.len());
197        assert_eq!(deser.apic_base.len(), vcpus_lock.len());
198        for (i, vcpu) in vcpus_lock.iter().enumerate() {
199            if let Some(vcpu) = vcpu {
200                vcpu.set_apic_base(*deser.apic_base.get(i).unwrap())?;
201                vcpu.set_interrupt_bitmap(*deser.interrupt_bitmap.get(i).unwrap())?;
202            } else {
203                return Err(anyhow!(
204                    "Received None instead of Vcpu while restoring apic_base and interrupt_bitmap"
205                ));
206            }
207        }
208        Ok(())
209    }
210}
211
212/// The KvmSplitIrqsChip supports KVM's SPLIT_IRQCHIP feature, where the PIC and IOAPIC
213/// are emulated in userspace, while the local APICs are emulated in the kernel.
214/// The SPLIT_IRQCHIP feature only supports x86/x86_64 so we only define this IrqChip in crosvm
215/// for x86/x86_64.
216pub struct KvmSplitIrqChip {
217    vm: Arc<KvmVm>,
218    vcpus: Arc<Mutex<Vec<Option<Arc<KvmVcpu>>>>>,
219    routes: Arc<Mutex<Vec<IrqRoute>>>,
220    pit: Arc<Mutex<Pit>>,
221    pic: Arc<Mutex<Pic>>,
222    ioapic: Arc<Mutex<Ioapic>>,
223    ioapic_pins: usize,
224    /// Vec of ioapic irq events that have been delayed because the ioapic was locked when
225    /// service_irq was called on the irqchip. This prevents deadlocks when a Vcpu thread has
226    /// locked the ioapic and the ioapic sends a AddMsiRoute signal to the main thread (which
227    /// itself may be busy trying to call service_irq).
228    delayed_ioapic_irq_events: Arc<Mutex<Vec<usize>>>,
229    /// Event which is meant to trigger process of any irqs events that were delayed.
230    delayed_ioapic_irq_trigger: Event,
231    /// Array of Events that devices will use to assert ioapic pins.
232    irq_events: Arc<Mutex<Vec<Option<IrqEvent>>>>,
233}
234
235fn kvm_dummy_msi_routes(ioapic_pins: usize) -> Vec<IrqRoute> {
236    let mut routes: Vec<IrqRoute> = Vec::new();
237    for i in 0..ioapic_pins {
238        routes.push(
239            // Add dummy MSI routes to replace the default IRQChip routes.
240            IrqRoute {
241                gsi: i as u32,
242                source: IrqSource::Msi {
243                    address: 0,
244                    data: 0,
245                },
246            },
247        );
248    }
249    routes
250}
251
252impl KvmSplitIrqChip {
253    /// Construct a new KvmSplitIrqChip.
254    pub fn new(
255        vm: Arc<KvmVm>,
256        num_vcpus: usize,
257        irq_tube: Tube,
258        ioapic_pins: Option<usize>,
259    ) -> Result<Self> {
260        let ioapic_pins = ioapic_pins.unwrap_or(vm.get_ioapic_num_pins()?);
261        vm.enable_split_irqchip(ioapic_pins)?;
262        let pit_evt = IrqEdgeEvent::new()?;
263        let pit = Pit::new(pit_evt.try_clone()?, Arc::new(Mutex::new(Clock::new()))).map_err(
264            |e| match e {
265                PitError::CloneEvent(err) => err,
266                PitError::CreateEvent(err) => err,
267                PitError::CreateWaitContext(err) => err,
268                PitError::WaitError(err) => err,
269                PitError::TimerCreateError(err) => err,
270                PitError::SpawnThread(_) => Error::new(libc::EIO),
271            },
272        )?;
273
274        let pit_event_source = IrqEventSource::from_device(&pit);
275
276        let chip = KvmSplitIrqChip {
277            vm,
278            vcpus: Arc::new(Mutex::new((0..num_vcpus).map(|_| None).collect())),
279            routes: Arc::new(Mutex::new(Vec::new())),
280            pit: Arc::new(Mutex::new(pit)),
281            pic: Arc::new(Mutex::new(Pic::new())),
282            ioapic: Arc::new(Mutex::new(Ioapic::new(irq_tube, ioapic_pins)?)),
283            ioapic_pins,
284            delayed_ioapic_irq_events: Arc::new(Mutex::new(Vec::new())),
285            delayed_ioapic_irq_trigger: Event::new()?,
286            irq_events: Arc::new(Mutex::new(Default::default())),
287        };
288
289        // Setup standard x86 irq routes
290        let mut routes = kvm_default_irq_routing_table(ioapic_pins);
291        // Add dummy MSI routes for the first ioapic_pins GSIs
292        routes.append(&mut kvm_dummy_msi_routes(ioapic_pins));
293
294        // Set the routes so they get sent to KVM
295        chip.set_irq_routes(&routes)?;
296
297        chip.register_edge_irq_event(PIT_CHANNEL0_IRQ, &pit_evt, pit_event_source)?;
298        Ok(chip)
299    }
300}
301
302impl KvmSplitIrqChip {
303    /// Convenience function for determining which chips the supplied irq routes to.
304    fn routes_to_chips(&self, irq: u32) -> Vec<(IrqSourceChip, u32)> {
305        let mut chips = Vec::new();
306        for route in self.routes.lock().iter() {
307            match route {
308                IrqRoute {
309                    gsi,
310                    source: IrqSource::Irqchip { chip, pin },
311                } if *gsi == irq => match chip {
312                    IrqSourceChip::PicPrimary
313                    | IrqSourceChip::PicSecondary
314                    | IrqSourceChip::Ioapic => chips.push((*chip, *pin)),
315                    IrqSourceChip::Gic => {
316                        error!("gic irq should not be possible on a KvmSplitIrqChip")
317                    }
318                    IrqSourceChip::Aia => {
319                        error!("Aia irq should not be possible on x86_64")
320                    }
321                },
322                // Ignore MSIs and other routes
323                _ => {}
324            }
325        }
326        chips
327    }
328
329    /// Return true if there is a pending interrupt for the specified vcpu. For KvmSplitIrqChip
330    /// this calls interrupt_requested on the pic.
331    pub fn interrupt_requested(&self, vcpu_id: usize) -> bool {
332        // Pic interrupts for the split irqchip only go to vcpu 0
333        if vcpu_id != 0 {
334            return false;
335        }
336        self.pic.lock().interrupt_requested()
337    }
338
339    /// Check if the specified vcpu has any pending interrupts. Returns [`None`] for no interrupts,
340    /// otherwise [`Some::<u8>`] should be the injected interrupt vector. For [`KvmSplitIrqChip`]
341    /// this calls `get_external_interrupt` on the pic.
342    pub fn get_external_interrupt(&self, vcpu_id: usize) -> Option<u8> {
343        // Pic interrupts for the split irqchip only go to vcpu 0
344        if vcpu_id != 0 {
345            return None;
346        }
347        self.pic.lock().get_external_interrupt()
348    }
349
350    /// Register an event that can trigger an interrupt for a particular GSI.
351    fn register_irq_event(
352        &self,
353        irq: u32,
354        irq_event: &Event,
355        resample_event: Option<&Event>,
356        source: IrqEventSource,
357    ) -> Result<Option<IrqEventIndex>> {
358        if irq < self.ioapic_pins as u32 {
359            let mut evt = IrqEvent {
360                gsi: irq,
361                event: irq_event.try_clone()?,
362                resample_event: None,
363                source,
364            };
365
366            if let Some(resample_event) = resample_event {
367                evt.resample_event = Some(resample_event.try_clone()?);
368            }
369
370            let mut irq_events = self.irq_events.lock();
371            let index = irq_events.len();
372            irq_events.push(Some(evt));
373            Ok(Some(index))
374        } else {
375            self.vm.register_irqfd(irq, irq_event, resample_event)?;
376            Ok(None)
377        }
378    }
379
380    /// Unregister an event for a particular GSI.
381    fn unregister_irq_event(&self, irq: u32, irq_event: &Event) -> Result<()> {
382        if irq < self.ioapic_pins as u32 {
383            let mut irq_events = self.irq_events.lock();
384            for (index, evt) in irq_events.iter().enumerate() {
385                if let Some(evt) = evt {
386                    if evt.gsi == irq && irq_event.eq(&evt.event) {
387                        irq_events[index] = None;
388                        break;
389                    }
390                }
391            }
392            Ok(())
393        } else {
394            self.vm.unregister_irqfd(irq, irq_event)
395        }
396    }
397}
398
399/// Convenience function for determining whether or not two irq routes conflict.
400/// Returns true if they conflict.
401fn routes_conflict(route: &IrqRoute, other: &IrqRoute) -> bool {
402    // They don't conflict if they have different GSIs.
403    if route.gsi != other.gsi {
404        return false;
405    }
406
407    // If they're both MSI with the same GSI then they conflict.
408    if let (IrqSource::Msi { .. }, IrqSource::Msi { .. }) = (route.source, other.source) {
409        return true;
410    }
411
412    // If the route chips match and they have the same GSI then they conflict.
413    if let (
414        IrqSource::Irqchip {
415            chip: route_chip, ..
416        },
417        IrqSource::Irqchip {
418            chip: other_chip, ..
419        },
420    ) = (route.source, other.source)
421    {
422        return route_chip == other_chip;
423    }
424
425    // Otherwise they do not conflict.
426    false
427}
428
429/// This IrqChip only works with Kvm so we only implement it for KvmVcpu.
430impl IrqChip for KvmSplitIrqChip {
431    /// Add a vcpu to the irq chip.
432    fn add_vcpu(&self, vcpu_id: usize, vcpu: Arc<dyn VcpuArch>) -> Result<()> {
433        let vcpu = Arc::downcast(vcpu)
434            .map_err(|_| ())
435            .expect("KvmSplitIrqChip::add_vcpu called with non-KvmVcpu");
436        self.vcpus.lock()[vcpu_id] = Some(vcpu);
437        Ok(())
438    }
439
440    /// Register an event that can trigger an interrupt for a particular GSI.
441    fn register_edge_irq_event(
442        &self,
443        irq: u32,
444        irq_event: &IrqEdgeEvent,
445        source: IrqEventSource,
446    ) -> Result<Option<IrqEventIndex>> {
447        self.register_irq_event(irq, irq_event.get_trigger(), None, source)
448    }
449
450    fn unregister_edge_irq_event(&self, irq: u32, irq_event: &IrqEdgeEvent) -> Result<()> {
451        self.unregister_irq_event(irq, irq_event.get_trigger())
452    }
453
454    fn register_level_irq_event(
455        &self,
456        irq: u32,
457        irq_event: &IrqLevelEvent,
458        source: IrqEventSource,
459    ) -> Result<Option<IrqEventIndex>> {
460        self.register_irq_event(
461            irq,
462            irq_event.get_trigger(),
463            Some(irq_event.get_resample()),
464            source,
465        )
466    }
467
468    fn unregister_level_irq_event(&self, irq: u32, irq_event: &IrqLevelEvent) -> Result<()> {
469        self.unregister_irq_event(irq, irq_event.get_trigger())
470    }
471
472    /// Route an IRQ line to an interrupt controller, or to a particular MSI vector.
473    fn route_irq(&self, route: IrqRoute) -> Result<()> {
474        let mut routes = self.routes.lock();
475        routes.retain(|r| !routes_conflict(r, &route));
476
477        routes.push(route);
478
479        // We only call set_gsi_routing with the msi routes
480        let mut msi_routes = routes.clone();
481        msi_routes.retain(|r| matches!(r.source, IrqSource::Msi { .. }));
482
483        self.vm.set_gsi_routing(&msi_routes)
484    }
485
486    /// Replace all irq routes with the supplied routes
487    fn set_irq_routes(&self, routes: &[IrqRoute]) -> Result<()> {
488        let mut current_routes = self.routes.lock();
489        *current_routes = routes.to_vec();
490
491        // We only call set_gsi_routing with the msi routes
492        let mut msi_routes = routes.to_vec();
493        msi_routes.retain(|r| matches!(r.source, IrqSource::Msi { .. }));
494
495        self.vm.set_gsi_routing(&msi_routes)
496    }
497
498    /// Return a vector of all registered irq numbers and their associated events and event
499    /// indices. These should be used by the main thread to wait for irq events.
500    fn irq_event_tokens(&self) -> Result<Vec<(IrqEventIndex, IrqEventSource, Event)>> {
501        let mut tokens = vec![];
502        for (index, evt) in self.irq_events.lock().iter().enumerate() {
503            if let Some(evt) = evt {
504                tokens.push((index, evt.source.clone(), evt.event.try_clone()?));
505            }
506        }
507        Ok(tokens)
508    }
509
510    /// Either assert or deassert an IRQ line.  Sends to either an interrupt controller, or does
511    /// a send_msi if the irq is associated with an MSI.
512    fn service_irq(&self, irq: u32, level: bool) -> Result<()> {
513        let chips = self.routes_to_chips(irq);
514        for (chip, pin) in chips {
515            match chip {
516                IrqSourceChip::PicPrimary | IrqSourceChip::PicSecondary => {
517                    self.pic.lock().service_irq(pin as u8, level);
518                }
519                IrqSourceChip::Ioapic => {
520                    self.ioapic.lock().service_irq(pin as usize, level);
521                }
522                _ => {}
523            }
524        }
525        Ok(())
526    }
527
528    /// Service an IRQ event by asserting then deasserting an IRQ line. The associated Event
529    /// that triggered the irq event will be read from. If the irq is associated with a resample
530    /// Event, then the deassert will only happen after an EOI is broadcast for a vector
531    /// associated with the irq line.
532    /// For the KvmSplitIrqChip, this function identifies which chips the irq routes to, then
533    /// attempts to call service_irq on those chips. If the ioapic is unable to be immediately
534    /// locked, we add the irq to the delayed_ioapic_irq_events Vec (though we still read
535    /// from the Event that triggered the irq event).
536    fn service_irq_event(&self, event_index: IrqEventIndex) -> Result<()> {
537        if let Some(evt) = &self.irq_events.lock()[event_index] {
538            evt.event.wait()?;
539            let chips = self.routes_to_chips(evt.gsi);
540
541            for (chip, pin) in chips {
542                match chip {
543                    IrqSourceChip::PicPrimary | IrqSourceChip::PicSecondary => {
544                        let mut pic = self.pic.lock();
545                        pic.service_irq(pin as u8, true);
546                        if evt.resample_event.is_none() {
547                            pic.service_irq(pin as u8, false);
548                        }
549                    }
550                    IrqSourceChip::Ioapic => {
551                        if let Ok(mut ioapic) = self.ioapic.try_lock() {
552                            ioapic.service_irq(pin as usize, true);
553                            if evt.resample_event.is_none() {
554                                ioapic.service_irq(pin as usize, false);
555                            }
556                        } else {
557                            self.delayed_ioapic_irq_events.lock().push(event_index);
558                            self.delayed_ioapic_irq_trigger.signal().unwrap();
559                        }
560                    }
561                    _ => {}
562                }
563            }
564        }
565
566        Ok(())
567    }
568
569    /// Broadcast an end of interrupt. For KvmSplitIrqChip this sends the EOI to the ioapic
570    fn broadcast_eoi(&self, vector: u8) -> Result<()> {
571        self.ioapic.lock().end_of_interrupt(vector);
572        Ok(())
573    }
574
575    /// Injects any pending interrupts for `vcpu`.
576    /// For KvmSplitIrqChip this injects any PIC interrupts on vcpu_id 0.
577    fn inject_interrupts(&self, vcpu: &dyn VcpuArch) -> Result<()> {
578        let vcpu: &KvmVcpu = <dyn Any>::downcast_ref(vcpu)
579            .expect("KvmSplitIrqChip::add_vcpu called with non-KvmVcpu");
580
581        let vcpu_id = vcpu.id();
582        if !self.interrupt_requested(vcpu_id) || !vcpu.ready_for_interrupt() {
583            return Ok(());
584        }
585
586        if let Some(vector) = self.get_external_interrupt(vcpu_id) {
587            vcpu.interrupt(vector)?;
588        }
589
590        // The second interrupt request should be handled immediately, so ask vCPU to exit as soon
591        // as possible.
592        if self.interrupt_requested(vcpu_id) {
593            vcpu.set_interrupt_window_requested(true);
594        }
595        Ok(())
596    }
597
598    /// Notifies the irq chip that the specified VCPU has executed a halt instruction.
599    /// For KvmSplitIrqChip this is a no-op because KVM handles VCPU blocking.
600    fn halted(&self, _vcpu_id: usize) {}
601
602    /// Blocks until `vcpu` is in a runnable state or until interrupted by
603    /// `IrqChip::kick_halted_vcpus`.  Returns `VcpuRunState::Runnable if vcpu is runnable, or
604    /// `VcpuRunState::Interrupted` if the wait was interrupted.
605    /// For KvmSplitIrqChip this is a no-op and always returns Runnable because KVM handles VCPU
606    /// blocking.
607    fn wait_until_runnable(&self, _vcpu: &dyn VcpuArch) -> Result<VcpuRunState> {
608        Ok(VcpuRunState::Runnable)
609    }
610
611    /// Makes unrunnable VCPUs return immediately from `wait_until_runnable`.
612    /// For KvmSplitIrqChip this is a no-op because KVM handles VCPU blocking.
613    fn kick_halted_vcpus(&self) {}
614
615    /// Get the current MP state of the specified VCPU.
616    fn get_mp_state(&self, vcpu_id: usize) -> Result<MPState> {
617        match self.vcpus.lock().get(vcpu_id) {
618            Some(Some(vcpu)) => Ok(MPState::from(&vcpu.get_mp_state()?)),
619            _ => Err(Error::new(libc::ENOENT)),
620        }
621    }
622
623    /// Set the current MP state of the specified VCPU.
624    fn set_mp_state(&self, vcpu_id: usize, state: &MPState) -> Result<()> {
625        match self.vcpus.lock().get(vcpu_id) {
626            Some(Some(vcpu)) => vcpu.set_mp_state(&kvm_mp_state::from(state)),
627            _ => Err(Error::new(libc::ENOENT)),
628        }
629    }
630
631    /// Finalize irqchip setup. Should be called once all devices have registered irq events and
632    /// been added to the io_bus and mmio_bus.
633    fn finalize_devices(
634        self: Arc<Self>,
635        resources: &mut SystemAllocator,
636        io_bus: &Bus,
637        mmio_bus: &Bus,
638    ) -> Result<()> {
639        // Insert pit into io_bus
640        io_bus.insert(self.pit.clone(), 0x040, 0x8).unwrap();
641        io_bus.insert(self.pit.clone(), 0x061, 0x1).unwrap();
642
643        // Insert pic into io_bus
644        io_bus.insert(self.pic.clone(), 0x20, 0x2).unwrap();
645        io_bus.insert(self.pic.clone(), 0xa0, 0x2).unwrap();
646        io_bus.insert(self.pic.clone(), 0x4d0, 0x2).unwrap();
647
648        // Insert ioapic into mmio_bus
649        mmio_bus
650            .insert(
651                self.ioapic.clone(),
652                IOAPIC_BASE_ADDRESS,
653                IOAPIC_MEM_LENGTH_BYTES,
654            )
655            .unwrap();
656
657        // At this point, all of our devices have been created and they have registered their
658        // irq events, so we can clone our resample events
659        let mut ioapic_resample_events: Vec<Vec<Event>> =
660            (0..self.ioapic_pins).map(|_| Vec::new()).collect();
661        let mut pic_resample_events: Vec<Vec<Event>> =
662            (0..self.ioapic_pins).map(|_| Vec::new()).collect();
663
664        for evt in self.irq_events.lock().iter().flatten() {
665            if (evt.gsi as usize) >= self.ioapic_pins {
666                continue;
667            }
668            if let Some(resample_evt) = &evt.resample_event {
669                ioapic_resample_events[evt.gsi as usize].push(resample_evt.try_clone()?);
670                pic_resample_events[evt.gsi as usize].push(resample_evt.try_clone()?);
671            }
672        }
673
674        // Register resample events with the ioapic
675        self.ioapic
676            .lock()
677            .register_resample_events(ioapic_resample_events);
678        // Register resample events with the pic
679        self.pic
680            .lock()
681            .register_resample_events(pic_resample_events);
682
683        // Make sure all future irq numbers are beyond IO-APIC range.
684        let mut irq_num = resources.allocate_irq().unwrap();
685        while irq_num < self.ioapic_pins as u32 {
686            irq_num = resources.allocate_irq().unwrap();
687        }
688
689        Ok(())
690    }
691
692    /// The KvmSplitIrqChip's ioapic may be locked because a vcpu thread is currently writing to
693    /// the ioapic, and the ioapic may be blocking on adding MSI routes, which requires blocking
694    /// socket communication back to the main thread.  Thus, we do not want the main thread to
695    /// block on a locked ioapic, so any irqs that could not be serviced because the ioapic could
696    /// not be immediately locked are added to the delayed_ioapic_irq_events Vec. This function
697    /// processes each delayed event in the vec each time it's called. If the ioapic is still
698    /// locked, we keep the queued irqs for the next time this function is called.
699    fn process_delayed_irq_events(&self) -> Result<()> {
700        self.delayed_ioapic_irq_events
701            .lock()
702            .retain(|&event_index| {
703                if let Some(evt) = &self.irq_events.lock()[event_index] {
704                    if let Ok(mut ioapic) = self.ioapic.try_lock() {
705                        ioapic.service_irq(evt.gsi as usize, true);
706                        if evt.resample_event.is_none() {
707                            ioapic.service_irq(evt.gsi as usize, false);
708                        }
709
710                        false
711                    } else {
712                        true
713                    }
714                } else {
715                    true
716                }
717            });
718
719        if self.delayed_ioapic_irq_events.lock().is_empty() {
720            self.delayed_ioapic_irq_trigger.wait()?;
721        }
722
723        Ok(())
724    }
725
726    fn irq_delayed_event_token(&self) -> Result<Option<Event>> {
727        Ok(Some(self.delayed_ioapic_irq_trigger.try_clone()?))
728    }
729
730    fn check_capability(&self, c: IrqChipCap) -> bool {
731        match c {
732            IrqChipCap::TscDeadlineTimer => self.vm.check_raw_capability(KvmCap::TscDeadlineTimer),
733            IrqChipCap::X2Apic => true,
734            IrqChipCap::MpStateGetSet => true,
735        }
736    }
737}
738
739#[derive(Serialize, Deserialize)]
740struct KvmSplitIrqChipSnapshot {
741    routes: Vec<IrqRoute>,
742}
743
744impl IrqChipX86_64 for KvmSplitIrqChip {
745    /// Get the current state of the PIC
746    fn get_pic_state(&self, select: PicSelect) -> Result<PicState> {
747        Ok(self.pic.lock().get_pic_state(select))
748    }
749
750    /// Set the current state of the PIC
751    fn set_pic_state(&self, select: PicSelect, state: &PicState) -> Result<()> {
752        self.pic.lock().set_pic_state(select, state);
753        Ok(())
754    }
755
756    /// Get the current state of the IOAPIC
757    fn get_ioapic_state(&self) -> Result<IoapicState> {
758        Ok(self.ioapic.lock().get_ioapic_state())
759    }
760
761    /// Set the current state of the IOAPIC
762    fn set_ioapic_state(&self, state: &IoapicState) -> Result<()> {
763        self.ioapic.lock().set_ioapic_state(state);
764        Ok(())
765    }
766
767    /// Get the current state of the specified VCPU's local APIC
768    fn get_lapic_state(&self, vcpu_id: usize) -> Result<LapicState> {
769        match self.vcpus.lock().get(vcpu_id) {
770            Some(Some(vcpu)) => Ok(LapicState::from(&vcpu.get_lapic()?)),
771            _ => Err(Error::new(libc::ENOENT)),
772        }
773    }
774
775    /// Set the current state of the specified VCPU's local APIC
776    fn set_lapic_state(&self, vcpu_id: usize, state: &LapicState) -> Result<()> {
777        match self.vcpus.lock().get(vcpu_id) {
778            Some(Some(vcpu)) => vcpu.set_lapic(&kvm_lapic_state::from(state)),
779            _ => Err(Error::new(libc::ENOENT)),
780        }
781    }
782
783    /// Get the lapic frequency in Hz
784    fn lapic_frequency(&self) -> u32 {
785        // KVM emulates the lapic to have a bus frequency of 1GHz
786        1_000_000_000
787    }
788
789    /// Retrieves the state of the PIT. Gets the pit state via the KVM API.
790    fn get_pit(&self) -> Result<PitState> {
791        Ok(self.pit.lock().get_pit_state())
792    }
793
794    /// Sets the state of the PIT. Sets the pit state via the KVM API.
795    fn set_pit(&self, state: &PitState) -> Result<()> {
796        self.pit.lock().set_pit_state(state);
797        Ok(())
798    }
799
800    /// Returns true if the PIT uses port 0x61 for the PC speaker, false if 0x61 is unused.
801    /// devices::Pit uses 0x61.
802    fn pit_uses_speaker_port(&self) -> bool {
803        true
804    }
805
806    fn snapshot_chip_specific(&self) -> anyhow::Result<AnySnapshot> {
807        AnySnapshot::to_any(KvmSplitIrqChipSnapshot {
808            routes: self.routes.lock().clone(),
809        })
810        .context("failed to serialize KvmSplitIrqChip")
811    }
812
813    fn restore_chip_specific(&self, data: AnySnapshot) -> anyhow::Result<()> {
814        let deser: KvmSplitIrqChipSnapshot =
815            AnySnapshot::from_any(data).context("failed to deserialize KvmSplitIrqChip")?;
816        self.set_irq_routes(&deser.routes)?;
817        Ok(())
818    }
819}
devices/irqchip/kvm/x86_64.rs

devices/irqchip/kvm/
x86_64.rs
