devices/platform/
vfio_platform.rs

1// Copyright 2022 The ChromiumOS Authors
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use std::fs::File;
6use std::sync::Arc;
7
8use anyhow::bail;
9use anyhow::Context;
10use anyhow::Result;
11use base::error;
12use base::pagesize;
13use base::AsRawDescriptor;
14use base::AsRawDescriptors;
15use base::Event;
16use base::MappedRegion;
17use base::MemoryMapping;
18use base::MemoryMappingBuilder;
19use base::Protection;
20use base::RawDescriptor;
21use hypervisor::MemCacheType;
22use hypervisor::Vm;
23use resources::SystemAllocator;
24use vfio_sys::*;
25use vm_control::api::VmMemoryClient;
26use vm_control::DeviceId;
27use vm_control::PlatformDeviceId;
28use vm_control::VmMemoryDestination;
29use vm_control::VmMemorySource;
30use vm_memory::GuestAddress;
31
32use crate::vfio::VfioDevice;
33use crate::vfio::VfioError;
34use crate::vfio::VfioIrq;
35use crate::BusAccessInfo;
36use crate::BusDevice;
37use crate::BusDeviceObj;
38use crate::IommuDevType;
39use crate::IrqEdgeEvent;
40use crate::IrqLevelEvent;
41use crate::Suspendable;
42
43struct MmioInfo {
44    index: usize,
45    start: u64,
46    length: u64,
47}
48
49pub struct VfioPlatformDevice {
50    device: Arc<VfioDevice>,
51    interrupt_edge_evt: Vec<IrqEdgeEvent>,
52    interrupt_level_evt: Vec<IrqLevelEvent>,
53    mmio_regions: Vec<MmioInfo>,
54    vm_memory_client: VmMemoryClient,
55    // scratch MemoryMapping to avoid unmap beform vm exit
56    mem: Vec<MemoryMapping>,
57}
58
59impl BusDevice for VfioPlatformDevice {
60    fn device_id(&self) -> DeviceId {
61        PlatformDeviceId::VfioPlatformDevice.into()
62    }
63
64    fn debug_label(&self) -> String {
65        format!("vfio {} device", self.device.device_name())
66    }
67
68    fn read(&mut self, info: BusAccessInfo, data: &mut [u8]) {
69        self.read_mmio(info.address, data)
70    }
71
72    fn write(&mut self, info: BusAccessInfo, data: &[u8]) {
73        self.write_mmio(info.address, data)
74    }
75
76    fn supports_power_management(&self) -> anyhow::Result<bool> {
77        Ok(self.device.supports_pm_low_power())
78    }
79
80    fn power_on(&mut self) -> anyhow::Result<()> {
81        self.device.pm_low_power_exit()?;
82        Ok(())
83    }
84
85    fn power_off(&mut self) -> anyhow::Result<()> {
86        self.device.pm_low_power_enter()?;
87        Ok(())
88    }
89}
90
91impl Suspendable for VfioPlatformDevice {}
92
93impl BusDeviceObj for VfioPlatformDevice {
94    fn as_platform_device(&self) -> Option<&VfioPlatformDevice> {
95        Some(self)
96    }
97    fn as_platform_device_mut(&mut self) -> Option<&mut VfioPlatformDevice> {
98        Some(self)
99    }
100    fn into_platform_device(self: Box<Self>) -> Option<Box<VfioPlatformDevice>> {
101        Some(self)
102    }
103}
104
105impl VfioPlatformDevice {
106    /// Constructs a new Vfio Platform device for the given Vfio device
107    pub fn new(device: VfioDevice, vm_memory_client: VmMemoryClient) -> Self {
108        let dev = Arc::new(device);
109        VfioPlatformDevice {
110            device: dev,
111            interrupt_edge_evt: Vec::new(),
112            interrupt_level_evt: Vec::new(),
113            mmio_regions: Vec::new(),
114            vm_memory_client,
115            mem: Vec::new(),
116        }
117    }
118
119    pub fn get_platform_irqs(&self) -> Result<Vec<VfioIrq>, VfioError> {
120        self.device.get_irqs()
121    }
122
123    pub fn irq_is_automask(&self, irq: &VfioIrq) -> bool {
124        irq.flags & VFIO_IRQ_INFO_AUTOMASKED != 0
125    }
126
127    fn setup_irq_resample(&mut self, resample_evt: &Event, index: u32) -> Result<()> {
128        self.device.irq_mask(index).context("Intx mask failed")?;
129        self.device
130            .resample_virq_enable(resample_evt, index)
131            .context("resample enable failed")?;
132        self.device
133            .irq_unmask(index)
134            .context("Intx unmask failed")?;
135        Ok(())
136    }
137
138    pub fn assign_edge_platform_irq(&mut self, irq_evt: &IrqEdgeEvent, index: u32) -> Result<()> {
139        let interrupt_evt = irq_evt.try_clone().context("failed to clone irq event")?;
140        self.device
141            .irq_enable(&[Some(interrupt_evt.get_trigger())], index, 0)
142            .context("platform irq enable failed")?;
143        self.interrupt_edge_evt.push(interrupt_evt);
144        Ok(())
145    }
146
147    pub fn assign_level_platform_irq(&mut self, irq_evt: &IrqLevelEvent, index: u32) -> Result<()> {
148        let interrupt_evt = irq_evt.try_clone().context("failed to clone irq event")?;
149        self.device
150            .irq_enable(&[Some(interrupt_evt.get_trigger())], index, 0)
151            .context("platform irq enable failed")?;
152        if let Err(e) = self.setup_irq_resample(interrupt_evt.get_resample(), index) {
153            self.disable_irqs(index);
154            bail!("failed to set up irq resampling: {}", e);
155        }
156        self.interrupt_level_evt.push(interrupt_evt);
157        Ok(())
158    }
159
160    fn find_region(&self, addr: u64) -> Option<MmioInfo> {
161        for mmio_info in self.mmio_regions.iter() {
162            if addr >= mmio_info.start && addr < mmio_info.start + mmio_info.length {
163                return Some(MmioInfo {
164                    index: mmio_info.index,
165                    start: mmio_info.start,
166                    length: mmio_info.length,
167                });
168            }
169        }
170        None
171    }
172
173    pub fn allocate_regions(
174        &mut self,
175        resources: &mut SystemAllocator,
176    ) -> Result<Vec<(u64, u64)>, resources::Error> {
177        let mut ranges = Vec::new();
178        for i in 0..self.device.get_region_count() {
179            let size = self.device.get_region_size(i);
180            let alloc_id = resources.get_anon_alloc();
181            let allocator = resources
182                .mmio_platform_allocator()
183                .ok_or(resources::Error::MissingPlatformMMIOAddresses)?;
184            let start_addr = allocator.allocate_with_align(
185                size,
186                alloc_id,
187                "vfio_mmio".to_string(),
188                pagesize() as u64,
189            )?;
190            ranges.push((start_addr, size));
191
192            self.mmio_regions.push(MmioInfo {
193                index: i,
194                start: start_addr,
195                length: size,
196            });
197        }
198        Ok(ranges)
199    }
200
201    fn region_mmap_early(&self, vm: &mut impl Vm, index: usize, start_addr: u64) {
202        if self.device.get_region_flags(index) & VFIO_REGION_INFO_FLAG_MMAP == 0 {
203            return;
204        }
205
206        for mmap in &self.device.get_region_mmap(index) {
207            let mmap_offset = mmap.offset;
208            let mmap_size = mmap.size;
209            let guest_map_start = start_addr + mmap_offset;
210            let region_offset = self.device.get_region_offset(index);
211            let offset = region_offset + mmap_offset;
212
213            let mmap = match MemoryMappingBuilder::new(mmap_size as usize)
214                .from_file(self.device.device_file())
215                .offset(offset)
216                .build()
217            {
218                Ok(v) => v,
219                Err(e) => {
220                    error!("{e}, index: {index}, start_addr:{start_addr:#x}, offset:{offset:#x}");
221                    break;
222                }
223            };
224
225            let host = mmap.as_ptr();
226            let guest_addr = GuestAddress(guest_map_start);
227            if let Err(e) = vm.add_memory_region(
228                guest_addr,
229                Box::new(mmap),
230                false,
231                false,
232                MemCacheType::CacheCoherent,
233            ) {
234                error!("{e}, index: {index}, guest_addr:{guest_addr}, host:{host:?}");
235                break;
236            }
237        }
238    }
239
240    /// Force adding the MMIO regions to the guest memory space.
241    ///
242    /// By default, MMIO regions are mapped lazily when the guest first accesses them. Instead,
243    /// this function maps them, even if the guest might end up not accessing them. It only runs in
244    /// the current thread and can therefore be called before the VM is started.
245    pub fn regions_mmap_early(&mut self, vm: &mut impl Vm) {
246        for mmio_info in self.mmio_regions.iter() {
247            self.region_mmap_early(vm, mmio_info.index, mmio_info.start);
248        }
249    }
250
251    fn region_mmap(&self, index: usize, start_addr: u64) -> Vec<MemoryMapping> {
252        let mut mem_map: Vec<MemoryMapping> = Vec::new();
253        if self.device.get_region_flags(index) & VFIO_REGION_INFO_FLAG_MMAP != 0 {
254            let mmaps = self.device.get_region_mmap(index);
255            if mmaps.is_empty() {
256                return mem_map;
257            }
258
259            for mmap in mmaps.iter() {
260                let mmap_offset = mmap.offset;
261                let mmap_size = mmap.size;
262                let guest_map_start = start_addr + mmap_offset;
263                let region_offset = self.device.get_region_offset(index);
264                let offset = region_offset + mmap_offset;
265                let descriptor = match self.device.device_file().try_clone() {
266                    Ok(device_file) => device_file.into(),
267                    Err(_) => break,
268                };
269                match self.vm_memory_client.register_memory(
270                    VmMemorySource::Descriptor {
271                        descriptor,
272                        offset,
273                        size: mmap_size,
274                    },
275                    VmMemoryDestination::GuestPhysicalAddress(guest_map_start),
276                    Protection::read_write(),
277                    MemCacheType::CacheCoherent,
278                ) {
279                    Ok(_region) => {
280                        // Even if vm has mapped this region, but it is in vm main process,
281                        // device process doesn't has this mapping, but vfio_dma_map() need it
282                        // in device process, so here map it again.
283                        let mmap = match MemoryMappingBuilder::new(mmap_size as usize)
284                            .from_file(self.device.device_file())
285                            .offset(offset)
286                            .build()
287                        {
288                            Ok(v) => v,
289                            Err(_e) => break,
290                        };
291                        let host = mmap.as_ptr() as u64;
292                        // SAFETY:
293                        // Safe because the given guest_map_start is valid guest bar address. and
294                        // the host pointer is correct and valid guaranteed by MemoryMapping
295                        // interface.
296                        match unsafe {
297                            self.device
298                                .vfio_dma_map(guest_map_start, mmap_size, host, true)
299                        } {
300                            Ok(_) => mem_map.push(mmap),
301                            Err(e) => {
302                                error!(
303                                    "{}, index: {}, start_addr:0x{:x}, host:0x{:x}",
304                                    e, index, start_addr, host
305                                );
306                                break;
307                            }
308                        }
309                    }
310                    Err(e) => {
311                        error!("register_memory failed: {}", e);
312                        break;
313                    }
314                }
315            }
316        }
317
318        mem_map
319    }
320
321    fn regions_mmap(&mut self) {
322        for mmio_info in self.mmio_regions.iter() {
323            let mut mem_map = self.region_mmap(mmio_info.index, mmio_info.start);
324            self.mem.append(&mut mem_map);
325        }
326    }
327
328    fn disable_irqs(&mut self, index: u32) {
329        if let Err(e) = self.device.irq_disable(index) {
330            error!("Platform irq disable failed: {}", e);
331        }
332    }
333
334    fn read_mmio(&mut self, addr: u64, data: &mut [u8]) {
335        if let Some(mmio_info) = self.find_region(addr) {
336            let offset = addr - mmio_info.start;
337            let index = mmio_info.index;
338            self.device.region_read(index, data, offset);
339        }
340        // We have no other way than wait for 1st access and then do the mmap,
341        // so that next accesses are dual-stage MMU accelerated.
342        self.regions_mmap();
343    }
344
345    fn write_mmio(&mut self, addr: u64, data: &[u8]) {
346        if let Some(mmio_info) = self.find_region(addr) {
347            let offset = addr - mmio_info.start;
348            let index = mmio_info.index;
349            self.device.region_write(index, data, offset);
350        }
351        // We have no other way than wait for 1st access and then do the mmap,
352        // so that next accesses are dual-stage MMU accelerated.
353        self.regions_mmap();
354    }
355
356    pub fn keep_rds(&self) -> Vec<RawDescriptor> {
357        let mut rds = self.device.keep_rds();
358
359        for irq_evt in self.interrupt_edge_evt.iter() {
360            rds.extend(irq_evt.as_raw_descriptors());
361        }
362
363        for irq_evt in self.interrupt_level_evt.iter() {
364            rds.extend(irq_evt.as_raw_descriptors());
365        }
366
367        rds.push(self.vm_memory_client.as_raw_descriptor());
368        rds
369    }
370
371    /// Gets the vfio device backing `File`.
372    pub fn device_file(&self) -> &File {
373        self.device.device_file()
374    }
375
376    /// Returns the DT symbol (node label) of the VFIO device.
377    pub fn dt_symbol(&self) -> Option<&str> {
378        self.device.dt_symbol()
379    }
380
381    /// Returns the type and indentifier (if applicable) of the IOMMU used by this VFIO device and
382    /// its master IDs.
383    pub fn iommu(&self) -> Option<(IommuDevType, Option<u32>, &[u32])> {
384        self.device.iommu()
385    }
386}
devices/platform/vfio_platform.rs

devices/platform/
vfio_platform.rs
