base/sys/linux/
mmap.rs

1// Copyright 2017 The ChromiumOS Authors
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5//! The mmap module provides a safe interface to mmap memory and ensures unmap is called when the
6//! mmap object leaves scope.
7
8use std::ptr::null_mut;
9
10use libc::c_int;
11use libc::PROT_READ;
12use libc::PROT_WRITE;
13use log::warn;
14
15use super::Error as ErrnoError;
16use crate::pagesize;
17use crate::AsRawDescriptor;
18use crate::Descriptor;
19use crate::MappedRegion;
20use crate::MemoryMapping as CrateMemoryMapping;
21use crate::MemoryMappingBuilder;
22use crate::MmapError as Error;
23use crate::MmapResult as Result;
24use crate::Protection;
25use crate::RawDescriptor;
26use crate::SafeDescriptor;
27
28impl From<Protection> for c_int {
29    #[inline(always)]
30    fn from(p: Protection) -> Self {
31        let mut value = 0;
32        if p.read {
33            value |= PROT_READ
34        }
35        if p.write {
36            value |= PROT_WRITE;
37        }
38        value
39    }
40}
41
42/// Validates that `offset`..`offset+range_size` lies within the bounds of a memory mapping of
43/// `mmap_size` bytes.  Also checks for any overflow.
44fn validate_includes_range(mmap_size: usize, offset: usize, range_size: usize) -> Result<()> {
45    // Ensure offset + size doesn't overflow
46    let end_offset = offset
47        .checked_add(range_size)
48        .ok_or(Error::InvalidAddress)?;
49    // Ensure offset + size are within the mapping bounds
50    if end_offset <= mmap_size {
51        Ok(())
52    } else {
53        Err(Error::InvalidAddress)
54    }
55}
56
57impl dyn MappedRegion {
58    /// Calls msync with MS_SYNC on a mapping of `size` bytes starting at `offset` from the start of
59    /// the region.  `offset`..`offset+size` must be contained within the `MappedRegion`.
60    pub fn msync(&self, offset: usize, size: usize) -> Result<()> {
61        validate_includes_range(self.size(), offset, size)?;
62
63        // SAFETY:
64        // Safe because the MemoryMapping/MemoryMappingArena interface ensures our pointer and size
65        // are correct, and we've validated that `offset`..`offset+size` is in the range owned by
66        // this `MappedRegion`.
67        let ret = unsafe {
68            libc::msync(
69                (self.as_ptr() as usize + offset) as *mut libc::c_void,
70                size,
71                libc::MS_SYNC,
72            )
73        };
74        if ret != -1 {
75            Ok(())
76        } else {
77            Err(Error::SystemCallFailed(ErrnoError::last()))
78        }
79    }
80
81    /// Calls madvise on a mapping of `size` bytes starting at `offset` from the start of
82    /// the region.  `offset`..`offset+size` must be contained within the `MappedRegion`.
83    pub fn madvise(&self, offset: usize, size: usize, advice: libc::c_int) -> Result<()> {
84        validate_includes_range(self.size(), offset, size)?;
85
86        // SAFETY:
87        // Safe because the MemoryMapping/MemoryMappingArena interface ensures our pointer and size
88        // are correct, and we've validated that `offset`..`offset+size` is in the range owned by
89        // this `MappedRegion`.
90        let ret = unsafe {
91            libc::madvise(
92                (self.as_ptr() as usize + offset) as *mut libc::c_void,
93                size,
94                advice,
95            )
96        };
97        if ret != -1 {
98            Ok(())
99        } else {
100            Err(Error::SystemCallFailed(ErrnoError::last()))
101        }
102    }
103}
104
105/// Wraps an anonymous shared memory mapping in the current process. Provides
106/// RAII semantics including munmap when no longer needed.
107#[derive(Debug)]
108pub struct MemoryMapping {
109    addr: *mut u8,
110    size: usize,
111}
112
113// SAFETY:
114// Send and Sync aren't automatically inherited for the raw address pointer.
115// Accessing that pointer is only done through the stateless interface which
116// allows the object to be shared by multiple threads without a decrease in
117// safety.
118unsafe impl Send for MemoryMapping {}
119// SAFETY: See safety comments for impl Send
120unsafe impl Sync for MemoryMapping {}
121
122impl MemoryMapping {
123    /// Creates an anonymous shared, read/write mapping of `size` bytes.
124    ///
125    /// # Arguments
126    /// * `size` - Size of memory region in bytes.
127    pub fn new(size: usize) -> Result<MemoryMapping> {
128        MemoryMapping::new_protection(size, None, Protection::read_write())
129    }
130
131    /// Creates an anonymous shared mapping of `size` bytes with `prot` protection.
132    ///
133    /// # Arguments
134    /// * `size` - Size of memory region in bytes.
135    /// * `align` - Optional alignment for MemoryMapping::addr.
136    /// * `prot` - Protection (e.g. readable/writable) of the memory region.
137    pub fn new_protection(
138        size: usize,
139        align: Option<u64>,
140        prot: Protection,
141    ) -> Result<MemoryMapping> {
142        // SAFETY:
143        // This is safe because we are creating an anonymous mapping in a place not already used by
144        // any other area in this process.
145        unsafe { MemoryMapping::try_mmap(None, size, align, prot.into(), None) }
146    }
147
148    /// Maps the first `size` bytes of the given `fd` as read/write.
149    ///
150    /// # Arguments
151    /// * `fd` - File descriptor to mmap from.
152    /// * `size` - Size of memory region in bytes.
153    pub fn from_fd(fd: &dyn AsRawDescriptor, size: usize) -> Result<MemoryMapping> {
154        MemoryMapping::from_fd_offset(fd, size, 0)
155    }
156
157    pub fn from_fd_offset(
158        fd: &dyn AsRawDescriptor,
159        size: usize,
160        offset: u64,
161    ) -> Result<MemoryMapping> {
162        MemoryMapping::from_fd_offset_protection(fd, size, offset, Protection::read_write())
163    }
164
165    /// Maps the `size` bytes starting at `offset` bytes of the given `fd` as read/write.
166    ///
167    /// # Arguments
168    /// * `fd` - File descriptor to mmap from.
169    /// * `size` - Size of memory region in bytes.
170    /// * `offset` - Offset in bytes from the beginning of `fd` to start the mmap.
171    /// * `prot` - Protection (e.g. readable/writable) of the memory region.
172    pub fn from_fd_offset_protection(
173        fd: &dyn AsRawDescriptor,
174        size: usize,
175        offset: u64,
176        prot: Protection,
177    ) -> Result<MemoryMapping> {
178        MemoryMapping::from_fd_offset_protection_populate(fd, size, offset, 0, prot, false)
179    }
180
181    /// Maps `size` bytes starting at `offset` from the given `fd` as read/write, and requests
182    /// that the pages are pre-populated.
183    /// # Arguments
184    /// * `fd` - File descriptor to mmap from.
185    /// * `size` - Size of memory region in bytes.
186    /// * `offset` - Offset in bytes from the beginning of `fd` to start the mmap.
187    /// * `align` - Alignment for MemoryMapping::addr.
188    /// * `prot` - Protection (e.g. readable/writable) of the memory region.
189    /// * `populate` - Populate (prefault) page tables for a mapping.
190    pub fn from_fd_offset_protection_populate(
191        fd: &dyn AsRawDescriptor,
192        size: usize,
193        offset: u64,
194        align: u64,
195        prot: Protection,
196        populate: bool,
197    ) -> Result<MemoryMapping> {
198        // SAFETY:
199        // This is safe because we are creating an anonymous mapping in a place not already used
200        // by any other area in this process.
201        unsafe {
202            MemoryMapping::try_mmap_populate(
203                None,
204                size,
205                Some(align),
206                prot.into(),
207                Some((fd, offset)),
208                populate,
209            )
210        }
211    }
212
213    /// Creates an anonymous shared mapping of `size` bytes with `prot` protection.
214    ///
215    /// # Arguments
216    ///
217    /// * `addr` - Memory address to mmap at.
218    /// * `size` - Size of memory region in bytes.
219    /// * `prot` - Protection (e.g. readable/writable) of the memory region.
220    ///
221    /// # Safety
222    ///
223    /// This function should not be called before the caller unmaps any mmap'd regions already
224    /// present at `(addr..addr+size)`.
225    pub unsafe fn new_protection_fixed(
226        addr: *mut u8,
227        size: usize,
228        prot: Protection,
229    ) -> Result<MemoryMapping> {
230        MemoryMapping::try_mmap(Some(addr), size, None, prot.into(), None)
231    }
232
233    /// Maps the `size` bytes starting at `offset` bytes of the given `fd` with
234    /// `prot` protections.
235    ///
236    /// # Arguments
237    ///
238    /// * `addr` - Memory address to mmap at.
239    /// * `fd` - File descriptor to mmap from.
240    /// * `size` - Size of memory region in bytes.
241    /// * `offset` - Offset in bytes from the beginning of `fd` to start the mmap.
242    /// * `prot` - Protection (e.g. readable/writable) of the memory region.
243    ///
244    /// # Safety
245    ///
246    /// This function should not be called before the caller unmaps any mmap'd regions already
247    /// present at `(addr..addr+size)`.
248    pub unsafe fn from_descriptor_offset_protection_fixed(
249        addr: *mut u8,
250        fd: &dyn AsRawDescriptor,
251        size: usize,
252        offset: u64,
253        prot: Protection,
254    ) -> Result<MemoryMapping> {
255        MemoryMapping::try_mmap(Some(addr), size, None, prot.into(), Some((fd, offset)))
256    }
257
258    /// Helper wrapper around try_mmap_populate when without MAP_POPULATE
259    unsafe fn try_mmap(
260        addr: Option<*mut u8>,
261        size: usize,
262        align: Option<u64>,
263        prot: c_int,
264        fd: Option<(&dyn AsRawDescriptor, u64)>,
265    ) -> Result<MemoryMapping> {
266        MemoryMapping::try_mmap_populate(addr, size, align, prot, fd, false)
267    }
268
269    /// Helper wrapper around libc::mmap that does some basic validation, and calls
270    /// madvise with MADV_DONTDUMP on the created mmap
271    unsafe fn try_mmap_populate(
272        addr: Option<*mut u8>,
273        size: usize,
274        align: Option<u64>,
275        prot: c_int,
276        fd: Option<(&dyn AsRawDescriptor, u64)>,
277        populate: bool,
278    ) -> Result<MemoryMapping> {
279        let mut flags = libc::MAP_SHARED;
280        if populate {
281            flags |= libc::MAP_POPULATE;
282        }
283        // If addr is provided, set the (FIXED | NORESERVE) flag, and validate addr alignment.
284        let addr = match addr {
285            Some(addr) => {
286                if (addr as usize) % pagesize() != 0 {
287                    return Err(Error::NotPageAligned);
288                }
289                flags |= libc::MAP_FIXED | libc::MAP_NORESERVE;
290                addr as *mut libc::c_void
291            }
292            None => null_mut(),
293        };
294
295        // mmap already PAGE_SIZE align the returned address.
296        let align = if align.unwrap_or(0) == pagesize() as u64 {
297            Some(0)
298        } else {
299            align
300        };
301
302        // Add an address if an alignment is requested.
303        let (addr, orig_addr, orig_size) = match align {
304            None | Some(0) => (addr, None, None),
305            Some(align) => {
306                if !addr.is_null() || !align.is_power_of_two() {
307                    return Err(Error::InvalidAlignment);
308                }
309                let orig_size = size + align as usize;
310                let orig_addr = libc::mmap64(
311                    null_mut(),
312                    orig_size,
313                    prot,
314                    libc::MAP_PRIVATE | libc::MAP_NORESERVE | libc::MAP_ANONYMOUS,
315                    -1,
316                    0,
317                );
318                if orig_addr == libc::MAP_FAILED {
319                    return Err(Error::SystemCallFailed(ErrnoError::last()));
320                }
321
322                flags |= libc::MAP_FIXED;
323
324                let mask = align - 1;
325                (
326                    (orig_addr.wrapping_add(mask as usize) as u64 & !mask) as *mut libc::c_void,
327                    Some(orig_addr),
328                    Some(orig_size),
329                )
330            }
331        };
332
333        // If fd is provided, validate fd offset is within bounds. If not, it's anonymous mapping
334        // and set the (ANONYMOUS | NORESERVE) flag.
335        let (fd, offset) = match fd {
336            Some((fd, offset)) => {
337                if offset > libc::off64_t::MAX as u64 {
338                    return Err(Error::InvalidOffset);
339                }
340                // Map private for read-only seal. See below for upstream relax of the restriction.
341                // - https://lore.kernel.org/bpf/20231013103208.kdffpyerufr4ygnw@quack3/T/
342                // SAFETY:
343                // Safe because no third parameter is expected and we check the return result.
344                let seals = unsafe { libc::fcntl(fd.as_raw_descriptor(), libc::F_GET_SEALS) };
345                if (seals >= 0) && (seals & libc::F_SEAL_WRITE != 0) {
346                    flags &= !libc::MAP_SHARED;
347                    flags |= libc::MAP_PRIVATE;
348                }
349                (fd.as_raw_descriptor(), offset as libc::off64_t)
350            }
351            None => {
352                flags |= libc::MAP_ANONYMOUS | libc::MAP_NORESERVE;
353                (-1, 0)
354            }
355        };
356        let addr = libc::mmap64(addr, size, prot, flags, fd, offset);
357        if addr == libc::MAP_FAILED {
358            return Err(Error::SystemCallFailed(ErrnoError::last()));
359        }
360
361        // If an original mmap exists, we can now remove the unused regions
362        if let Some(orig_addr) = orig_addr {
363            let mut unmap_start = orig_addr as usize;
364            let mut unmap_end = addr as usize;
365            let mut unmap_size = unmap_end - unmap_start;
366
367            if unmap_size > 0 {
368                libc::munmap(orig_addr, unmap_size);
369            }
370
371            unmap_start = addr as usize + size;
372            unmap_end = orig_addr as usize + orig_size.unwrap();
373            unmap_size = unmap_end - unmap_start;
374
375            if unmap_size > 0 {
376                libc::munmap(unmap_start as *mut libc::c_void, unmap_size);
377            }
378        }
379
380        // This is safe because we call madvise with a valid address and size.
381        let _ = libc::madvise(addr, size, libc::MADV_DONTDUMP);
382
383        // This is safe because KSM's only userspace visible effects are timing
384        // and memory consumption; it doesn't affect rust safety semantics.
385        // KSM is also disabled by default, and this flag is only a hint.
386        let _ = libc::madvise(addr, size, libc::MADV_MERGEABLE);
387
388        Ok(MemoryMapping {
389            addr: addr as *mut u8,
390            size,
391        })
392    }
393
394    /// Madvise the kernel to unmap on fork.
395    pub fn use_dontfork(&self) -> Result<()> {
396        // SAFETY:
397        // This is safe because we call madvise with a valid address and size, and we check the
398        // return value.
399        let ret = unsafe {
400            libc::madvise(
401                self.as_ptr() as *mut libc::c_void,
402                self.size(),
403                libc::MADV_DONTFORK,
404            )
405        };
406        if ret == -1 {
407            Err(Error::SystemCallFailed(ErrnoError::last()))
408        } else {
409            Ok(())
410        }
411    }
412
413    /// Madvise the kernel to use Huge Pages for this mapping.
414    pub fn use_hugepages(&self) -> Result<()> {
415        const SZ_2M: usize = 2 * 1024 * 1024;
416
417        // THP uses 2M pages, so use THP only on mappings that are at least
418        // 2M in size.
419        if self.size() < SZ_2M {
420            return Ok(());
421        }
422
423        // SAFETY:
424        // This is safe because we call madvise with a valid address and size, and we check the
425        // return value.
426        let ret = unsafe {
427            libc::madvise(
428                self.as_ptr() as *mut libc::c_void,
429                self.size(),
430                libc::MADV_HUGEPAGE,
431            )
432        };
433        if ret == -1 {
434            Err(Error::SystemCallFailed(ErrnoError::last()))
435        } else {
436            Ok(())
437        }
438    }
439
440    /// Calls msync with MS_SYNC on the mapping.
441    pub fn msync(&self) -> Result<()> {
442        // SAFETY:
443        // This is safe since we use the exact address and length of a known
444        // good memory mapping.
445        let ret = unsafe {
446            libc::msync(
447                self.as_ptr() as *mut libc::c_void,
448                self.size(),
449                libc::MS_SYNC,
450            )
451        };
452        if ret == -1 {
453            return Err(Error::SystemCallFailed(ErrnoError::last()));
454        }
455        Ok(())
456    }
457
458    /// Uses madvise to tell the kernel to remove the specified range.  Subsequent reads
459    /// to the pages in the range will return zero bytes.
460    pub fn remove_range(&self, mem_offset: usize, count: usize) -> Result<()> {
461        self.range_end(mem_offset, count)
462            .map_err(|_| Error::InvalidRange(mem_offset, count, self.size()))?;
463        // SAFETY: Safe because all the args to madvise are valid and the return
464        // value is checked.
465        let ret = unsafe {
466            // madvising away the region is the same as the guest changing it.
467            // Next time it is read, it may return zero pages.
468            libc::madvise(
469                (self.addr as usize + mem_offset) as *mut _,
470                count,
471                libc::MADV_REMOVE,
472            )
473        };
474        if ret < 0 {
475            Err(Error::SystemCallFailed(super::Error::last()))
476        } else {
477            Ok(())
478        }
479    }
480
481    /// Tell the kernel to readahead the range.
482    ///
483    /// This does not block the thread by I/O wait from reading the backed file. This does not
484    /// guarantee that the pages are surely present unless the pages are mlock(2)ed by
485    /// `lock_on_fault_unchecked()`.
486    ///
487    /// The `mem_offset` and `count` must be validated by caller.
488    ///
489    /// # Arguments
490    ///
491    /// * `mem_offset` - The offset of the head of the range.
492    /// * `count` - The size in bytes of the range.
493    pub fn async_prefetch(&self, mem_offset: usize, count: usize) -> Result<()> {
494        // Validation
495        self.range_end(mem_offset, count)
496            .map_err(|_| Error::InvalidRange(mem_offset, count, self.size()))?;
497        // SAFETY:
498        // Safe because populating the pages from the backed file does not affect the Rust memory
499        // safety.
500        let ret = unsafe {
501            libc::madvise(
502                (self.addr as usize + mem_offset) as *mut _,
503                count,
504                libc::MADV_WILLNEED,
505            )
506        };
507        if ret < 0 {
508            Err(Error::SystemCallFailed(super::Error::last()))
509        } else {
510            Ok(())
511        }
512    }
513
514    /// Tell the kernel to drop the page cache.
515    ///
516    /// This cannot be applied to locked pages.
517    ///
518    /// The `mem_offset` and `count` must be validated by caller.
519    ///
520    /// NOTE: This function has destructive semantics. It throws away data in the page cache without
521    /// writing it to the backing file. If the data is important, the caller should ensure it is
522    /// written to disk before calling this function or should use MADV_PAGEOUT instead.
523    ///
524    /// # Arguments
525    ///
526    /// * `mem_offset` - The offset of the head of the range.
527    /// * `count` - The size in bytes of the range.
528    pub fn drop_page_cache(&self, mem_offset: usize, count: usize) -> Result<()> {
529        // Validation
530        self.range_end(mem_offset, count)
531            .map_err(|_| Error::InvalidRange(mem_offset, count, self.size()))?;
532        // SAFETY:
533        // Safe because dropping the page cache does not affect the Rust memory safety.
534        let ret = unsafe {
535            libc::madvise(
536                (self.addr as usize + mem_offset) as *mut _,
537                count,
538                libc::MADV_DONTNEED,
539            )
540        };
541        if ret < 0 {
542            Err(Error::SystemCallFailed(super::Error::last()))
543        } else {
544            Ok(())
545        }
546    }
547
548    /// Lock the resident pages in the range not to be swapped out.
549    ///
550    /// The remaining nonresident page are locked when they are populated.
551    ///
552    /// The `mem_offset` and `count` must be validated by caller.
553    ///
554    /// # Arguments
555    ///
556    /// * `mem_offset` - The offset of the head of the range.
557    /// * `count` - The size in bytes of the range.
558    pub fn lock_on_fault(&self, mem_offset: usize, count: usize) -> Result<()> {
559        // Validation
560        self.range_end(mem_offset, count)
561            .map_err(|_| Error::InvalidRange(mem_offset, count, self.size()))?;
562        let addr = self.addr as usize + mem_offset;
563        // SAFETY:
564        // Safe because MLOCK_ONFAULT only affects the swap behavior of the kernel, so it has no
565        // impact on rust semantics.
566        let ret = unsafe { libc::mlock2(addr as *mut _, count, libc::MLOCK_ONFAULT) };
567        if ret < 0 {
568            let errno = super::Error::last();
569            warn!(
570                "failed to mlock at {:#x} with length {}: {}",
571                addr as u64,
572                self.size(),
573                errno,
574            );
575            Err(Error::SystemCallFailed(errno))
576        } else {
577            Ok(())
578        }
579    }
580
581    /// Unlock the range of pages.
582    ///
583    /// Unlocking non-locked pages does not fail.
584    ///
585    /// The `mem_offset` and `count` must be validated by caller.
586    ///
587    /// # Arguments
588    ///
589    /// * `mem_offset` - The offset of the head of the range.
590    /// * `count` - The size in bytes of the range.
591    pub fn unlock(&self, mem_offset: usize, count: usize) -> Result<()> {
592        // Validation
593        self.range_end(mem_offset, count)
594            .map_err(|_| Error::InvalidRange(mem_offset, count, self.size()))?;
595        // SAFETY:
596        // Safe because munlock(2) does not affect the Rust memory safety.
597        let ret = unsafe { libc::munlock((self.addr as usize + mem_offset) as *mut _, count) };
598        if ret < 0 {
599            Err(Error::SystemCallFailed(super::Error::last()))
600        } else {
601            Ok(())
602        }
603    }
604
605    // Check that offset+count is valid and return the sum.
606    pub(crate) fn range_end(&self, offset: usize, count: usize) -> Result<usize> {
607        let mem_end = offset.checked_add(count).ok_or(Error::InvalidAddress)?;
608        if mem_end > self.size() {
609            return Err(Error::InvalidAddress);
610        }
611        Ok(mem_end)
612    }
613}
614
615// SAFETY:
616// Safe because the pointer and size point to a memory range owned by this MemoryMapping that won't
617// be unmapped until it's Dropped.
618unsafe impl MappedRegion for MemoryMapping {
619    fn as_ptr(&self) -> *mut u8 {
620        self.addr
621    }
622
623    fn size(&self) -> usize {
624        self.size
625    }
626}
627
628impl Drop for MemoryMapping {
629    fn drop(&mut self) {
630        // SAFETY:
631        // This is safe because we mmap the area at addr ourselves, and nobody
632        // else is holding a reference to it.
633        unsafe {
634            libc::munmap(self.addr as *mut libc::c_void, self.size);
635        }
636    }
637}
638
639/// Tracks Fixed Memory Maps within an anonymous memory-mapped fixed-sized arena
640/// in the current process.
641pub struct MemoryMappingArena {
642    addr: *mut u8,
643    size: usize,
644}
645
646// SAFETY:
647// Send and Sync aren't automatically inherited for the raw address pointer.
648// Accessing that pointer is only done through the stateless interface which
649// allows the object to be shared by multiple threads without a decrease in
650// safety.
651unsafe impl Send for MemoryMappingArena {}
652// SAFETY: See safety comments for impl Send
653unsafe impl Sync for MemoryMappingArena {}
654
655impl MemoryMappingArena {
656    /// Creates an mmap arena of `size` bytes.
657    ///
658    /// # Arguments
659    /// * `size` - Size of memory region in bytes.
660    pub fn new(size: usize) -> Result<MemoryMappingArena> {
661        // Reserve the arena's memory using an anonymous read-only mmap.
662        MemoryMapping::new_protection(size, None, Protection::read()).map(From::from)
663    }
664
665    /// Creates an mmap arena of `size` bytes with custom initial memory protections.
666    pub fn new_protection(size: usize, prot: Protection) -> Result<MemoryMappingArena> {
667        MemoryMapping::new_protection(size, None, prot).map(From::from)
668    }
669
670    /// Anonymously maps `size` bytes at `offset` bytes from the start of the arena
671    /// with `prot` protections. `offset` must be page aligned.
672    ///
673    /// # Arguments
674    /// * `offset` - Page aligned offset into the arena in bytes.
675    /// * `size` - Size of memory region in bytes.
676    /// * `prot` - Protection (e.g. readable/writable) of the memory region.
677    pub fn add_anon_protection(
678        &mut self,
679        offset: usize,
680        size: usize,
681        prot: Protection,
682    ) -> Result<()> {
683        self.try_add(offset, size, prot, None)
684    }
685
686    /// Anonymously maps `size` bytes at `offset` bytes from the start of the arena.
687    /// `offset` must be page aligned.
688    ///
689    /// # Arguments
690    /// * `offset` - Page aligned offset into the arena in bytes.
691    /// * `size` - Size of memory region in bytes.
692    pub fn add_anon(&mut self, offset: usize, size: usize) -> Result<()> {
693        self.add_anon_protection(offset, size, Protection::read_write())
694    }
695
696    /// Maps `size` bytes from the start of the given `fd` at `offset` bytes from
697    /// the start of the arena. `offset` must be page aligned.
698    ///
699    /// # Arguments
700    /// * `offset` - Page aligned offset into the arena in bytes.
701    /// * `size` - Size of memory region in bytes.
702    /// * `fd` - File descriptor to mmap from.
703    pub fn add_fd(&mut self, offset: usize, size: usize, fd: &dyn AsRawDescriptor) -> Result<()> {
704        self.add_fd_offset(offset, size, fd, 0)
705    }
706
707    /// Maps `size` bytes starting at `fs_offset` bytes from within the given `fd`
708    /// at `offset` bytes from the start of the arena. `offset` must be page aligned.
709    ///
710    /// # Arguments
711    /// * `offset` - Page aligned offset into the arena in bytes.
712    /// * `size` - Size of memory region in bytes.
713    /// * `fd` - File descriptor to mmap from.
714    /// * `fd_offset` - Offset in bytes from the beginning of `fd` to start the mmap.
715    pub fn add_fd_offset(
716        &mut self,
717        offset: usize,
718        size: usize,
719        fd: &dyn AsRawDescriptor,
720        fd_offset: u64,
721    ) -> Result<()> {
722        self.add_fd_offset_protection(offset, size, fd, fd_offset, Protection::read_write())
723    }
724
725    /// Maps `size` bytes starting at `fs_offset` bytes from within the given `fd`
726    /// at `offset` bytes from the start of the arena with `prot` protections.
727    /// `offset` must be page aligned.
728    ///
729    /// # Arguments
730    /// * `offset` - Page aligned offset into the arena in bytes.
731    /// * `size` - Size of memory region in bytes.
732    /// * `fd` - File descriptor to mmap from.
733    /// * `fd_offset` - Offset in bytes from the beginning of `fd` to start the mmap.
734    /// * `prot` - Protection (e.g. readable/writable) of the memory region.
735    pub fn add_fd_offset_protection(
736        &mut self,
737        offset: usize,
738        size: usize,
739        fd: &dyn AsRawDescriptor,
740        fd_offset: u64,
741        prot: Protection,
742    ) -> Result<()> {
743        self.try_add(offset, size, prot, Some((fd, fd_offset)))
744    }
745
746    /// Helper method that calls appropriate MemoryMapping constructor and adds
747    /// the resulting map into the arena.
748    fn try_add(
749        &mut self,
750        offset: usize,
751        size: usize,
752        prot: Protection,
753        fd: Option<(&dyn AsRawDescriptor, u64)>,
754    ) -> Result<()> {
755        // Ensure offset is page-aligned
756        if offset % pagesize() != 0 {
757            return Err(Error::NotPageAligned);
758        }
759        validate_includes_range(self.size(), offset, size)?;
760
761        // SAFETY:
762        // This is safe since the range has been validated.
763        let mmap = unsafe {
764            match fd {
765                Some((fd, fd_offset)) => MemoryMapping::from_descriptor_offset_protection_fixed(
766                    self.addr.add(offset),
767                    fd,
768                    size,
769                    fd_offset,
770                    prot,
771                )?,
772                None => MemoryMapping::new_protection_fixed(self.addr.add(offset), size, prot)?,
773            }
774        };
775
776        // This mapping will get automatically removed when we drop the whole arena.
777        std::mem::forget(mmap);
778        Ok(())
779    }
780
781    /// Removes `size` bytes at `offset` bytes from the start of the arena. `offset` must be page
782    /// aligned.
783    ///
784    /// # Arguments
785    /// * `offset` - Page aligned offset into the arena in bytes.
786    /// * `size` - Size of memory region in bytes.
787    pub fn remove(&mut self, offset: usize, size: usize) -> Result<()> {
788        self.try_add(offset, size, Protection::read(), None)
789    }
790}
791
792// SAFETY:
793// Safe because the pointer and size point to a memory range owned by this MemoryMappingArena that
794// won't be unmapped until it's Dropped.
795unsafe impl MappedRegion for MemoryMappingArena {
796    fn as_ptr(&self) -> *mut u8 {
797        self.addr
798    }
799
800    fn size(&self) -> usize {
801        self.size
802    }
803
804    fn add_fd_mapping(
805        &mut self,
806        offset: usize,
807        size: usize,
808        fd: &dyn AsRawDescriptor,
809        fd_offset: u64,
810        prot: Protection,
811    ) -> Result<()> {
812        self.add_fd_offset_protection(offset, size, fd, fd_offset, prot)
813    }
814
815    fn remove_mapping(&mut self, offset: usize, size: usize) -> Result<()> {
816        self.remove(offset, size)
817    }
818}
819
820impl From<MemoryMapping> for MemoryMappingArena {
821    fn from(mmap: MemoryMapping) -> Self {
822        let addr = mmap.as_ptr();
823        let size = mmap.size();
824
825        // Forget the original mapping because the `MemoryMappingArena` will take care of calling
826        // `munmap` when it is dropped.
827        std::mem::forget(mmap);
828        MemoryMappingArena { addr, size }
829    }
830}
831
832impl From<CrateMemoryMapping> for MemoryMappingArena {
833    fn from(mmap: CrateMemoryMapping) -> Self {
834        MemoryMappingArena::from(mmap.mapping)
835    }
836}
837
838impl Drop for MemoryMappingArena {
839    fn drop(&mut self) {
840        // SAFETY:
841        // This is safe because we own this memory range, and nobody else is holding a reference to
842        // it.
843        unsafe {
844            libc::munmap(self.addr as *mut libc::c_void, self.size);
845        }
846    }
847}
848
849impl CrateMemoryMapping {
850    pub fn use_dontfork(&self) -> Result<()> {
851        self.mapping.use_dontfork()
852    }
853
854    pub fn use_hugepages(&self) -> Result<()> {
855        self.mapping.use_hugepages()
856    }
857
858    pub fn from_raw_ptr(addr: RawDescriptor, size: usize) -> Result<CrateMemoryMapping> {
859        MemoryMapping::from_fd_offset(&Descriptor(addr), size, 0).map(|mapping| {
860            CrateMemoryMapping {
861                mapping,
862                _file_descriptor: None,
863            }
864        })
865    }
866}
867
868pub trait MemoryMappingUnix {
869    /// Remove the specified range from the mapping.
870    fn remove_range(&self, mem_offset: usize, count: usize) -> Result<()>;
871    /// Tell the kernel to readahead the range.
872    fn async_prefetch(&self, mem_offset: usize, count: usize) -> Result<()>;
873    /// Tell the kernel to drop the page cache.
874    fn drop_page_cache(&self, mem_offset: usize, count: usize) -> Result<()>;
875    /// Lock the resident pages in the range not to be swapped out.
876    fn lock_on_fault(&self, mem_offset: usize, count: usize) -> Result<()>;
877    /// Unlock the range of pages.
878    fn unlock(&self, mem_offset: usize, count: usize) -> Result<()>;
879    /// Disable host swap for this mapping.
880    fn lock_all(&self) -> Result<()>;
881}
882
883impl MemoryMappingUnix for CrateMemoryMapping {
884    fn remove_range(&self, mem_offset: usize, count: usize) -> Result<()> {
885        self.mapping.remove_range(mem_offset, count)
886    }
887    fn async_prefetch(&self, mem_offset: usize, count: usize) -> Result<()> {
888        self.mapping.async_prefetch(mem_offset, count)
889    }
890    fn drop_page_cache(&self, mem_offset: usize, count: usize) -> Result<()> {
891        self.mapping.drop_page_cache(mem_offset, count)
892    }
893    fn lock_on_fault(&self, mem_offset: usize, count: usize) -> Result<()> {
894        self.mapping.lock_on_fault(mem_offset, count)
895    }
896    fn unlock(&self, mem_offset: usize, count: usize) -> Result<()> {
897        self.mapping.unlock(mem_offset, count)
898    }
899    fn lock_all(&self) -> Result<()> {
900        self.mapping.lock_on_fault(0, self.mapping.size())
901    }
902}
903
904pub trait MemoryMappingBuilderUnix<'a> {
905    #[allow(clippy::wrong_self_convention)]
906    fn from_descriptor(self, descriptor: &'a dyn AsRawDescriptor) -> MemoryMappingBuilder<'a>;
907}
908
909impl<'a> MemoryMappingBuilderUnix<'a> for MemoryMappingBuilder<'a> {
910    /// Build the memory mapping given the specified descriptor to mapped memory
911    ///
912    /// Default: Create a new memory mapping.
913    #[allow(clippy::wrong_self_convention)]
914    fn from_descriptor(mut self, descriptor: &'a dyn AsRawDescriptor) -> MemoryMappingBuilder<'a> {
915        self.descriptor = Some(descriptor);
916        self
917    }
918}
919
920impl<'a> MemoryMappingBuilder<'a> {
921    /// Request that the mapped pages are pre-populated
922    ///
923    /// Default: Do not populate
924    pub fn populate(mut self) -> MemoryMappingBuilder<'a> {
925        self.populate = true;
926        self
927    }
928
929    /// Build a MemoryMapping from the provided options.
930    pub fn build(self) -> Result<CrateMemoryMapping> {
931        match self.descriptor {
932            None => {
933                if self.populate {
934                    // Population not supported for new mmaps
935                    return Err(Error::InvalidArgument);
936                }
937                MemoryMappingBuilder::wrap(
938                    MemoryMapping::new_protection(
939                        self.size,
940                        self.align,
941                        self.protection.unwrap_or_else(Protection::read_write),
942                    )?,
943                    None,
944                )
945            }
946            Some(descriptor) => MemoryMappingBuilder::wrap(
947                MemoryMapping::from_fd_offset_protection_populate(
948                    descriptor,
949                    self.size,
950                    self.offset.unwrap_or(0),
951                    self.align.unwrap_or(0),
952                    self.protection.unwrap_or_else(Protection::read_write),
953                    self.populate,
954                )?,
955                None,
956            ),
957        }
958    }
959
960    pub(crate) fn wrap(
961        mapping: MemoryMapping,
962        file_descriptor: Option<&'a dyn AsRawDescriptor>,
963    ) -> Result<CrateMemoryMapping> {
964        let file_descriptor = match file_descriptor {
965            Some(descriptor) => Some(
966                SafeDescriptor::try_from(descriptor)
967                    .map_err(|_| Error::SystemCallFailed(ErrnoError::last()))?,
968            ),
969            None => None,
970        };
971        Ok(CrateMemoryMapping {
972            mapping,
973            _file_descriptor: file_descriptor,
974        })
975    }
976}
977
978#[cfg(test)]
979mod tests {
980    use tempfile::tempfile;
981
982    use super::*;
983    use crate::descriptor::Descriptor;
984    use crate::VolatileMemory;
985    use crate::VolatileMemoryError;
986
987    #[test]
988    fn basic_map() {
989        let m = MemoryMappingBuilder::new(1024).build().unwrap();
990        assert_eq!(1024, m.size());
991    }
992
993    #[test]
994    fn map_invalid_size() {
995        let res = MemoryMappingBuilder::new(0).build().unwrap_err();
996        if let Error::SystemCallFailed(e) = res {
997            assert_eq!(e.errno(), libc::EINVAL);
998        } else {
999            panic!("unexpected error: {res}");
1000        }
1001    }
1002
1003    #[test]
1004    fn map_invalid_fd() {
1005        let fd = Descriptor(-1);
1006        let res = MemoryMapping::from_fd(&fd, 1024).unwrap_err();
1007        if let Error::SystemCallFailed(e) = res {
1008            assert_eq!(e.errno(), libc::EBADF);
1009        } else {
1010            panic!("unexpected error: {res}");
1011        }
1012    }
1013
1014    #[test]
1015    fn test_write_past_end() {
1016        let m = MemoryMappingBuilder::new(5).build().unwrap();
1017        let res = m.write_slice(&[1, 2, 3, 4, 5, 6], 0);
1018        assert!(res.is_ok());
1019        assert_eq!(res.unwrap(), 5);
1020    }
1021
1022    #[test]
1023    fn slice_size() {
1024        let m = MemoryMappingBuilder::new(5).build().unwrap();
1025        let s = m.get_slice(2, 3).unwrap();
1026        assert_eq!(s.size(), 3);
1027    }
1028
1029    #[test]
1030    fn slice_addr() {
1031        let m = MemoryMappingBuilder::new(5).build().unwrap();
1032        let s = m.get_slice(2, 3).unwrap();
1033        // SAFETY: all addresses are known to exist.
1034        assert_eq!(s.as_ptr(), unsafe { m.as_ptr().offset(2) });
1035    }
1036
1037    #[test]
1038    fn slice_overflow_error() {
1039        let m = MemoryMappingBuilder::new(5).build().unwrap();
1040        let res = m.get_slice(usize::MAX, 3).unwrap_err();
1041        assert_eq!(
1042            res,
1043            VolatileMemoryError::Overflow {
1044                base: usize::MAX,
1045                offset: 3,
1046            }
1047        );
1048    }
1049    #[test]
1050    fn slice_oob_error() {
1051        let m = MemoryMappingBuilder::new(5).build().unwrap();
1052        let res = m.get_slice(3, 3).unwrap_err();
1053        assert_eq!(res, VolatileMemoryError::OutOfBounds { addr: 6 });
1054    }
1055
1056    #[test]
1057    fn from_fd_offset_invalid() {
1058        let fd = tempfile().unwrap();
1059        let res =
1060            MemoryMapping::from_fd_offset(&fd, 4096, (libc::off64_t::MAX as u64) + 1).unwrap_err();
1061        match res {
1062            Error::InvalidOffset => {}
1063            e => panic!("unexpected error: {e}"),
1064        }
1065    }
1066
1067    #[test]
1068    fn arena_new() {
1069        let m = MemoryMappingArena::new(0x40000).unwrap();
1070        assert_eq!(m.size(), 0x40000);
1071    }
1072
1073    #[test]
1074    fn arena_add() {
1075        let mut m = MemoryMappingArena::new(0x40000).unwrap();
1076        assert!(m.add_anon(0, pagesize() * 4).is_ok());
1077    }
1078
1079    #[test]
1080    fn arena_remove() {
1081        let mut m = MemoryMappingArena::new(0x40000).unwrap();
1082        assert!(m.add_anon(0, pagesize() * 4).is_ok());
1083        assert!(m.remove(0, pagesize()).is_ok());
1084        assert!(m.remove(0, pagesize() * 2).is_ok());
1085    }
1086
1087    #[test]
1088    fn arena_add_alignment_error() {
1089        let mut m = MemoryMappingArena::new(pagesize() * 2).unwrap();
1090        assert!(m.add_anon(0, 0x100).is_ok());
1091        let res = m.add_anon(pagesize() + 1, 0x100).unwrap_err();
1092        match res {
1093            Error::NotPageAligned => {}
1094            e => panic!("unexpected error: {e}"),
1095        }
1096    }
1097
1098    #[test]
1099    fn arena_add_oob_error() {
1100        let mut m = MemoryMappingArena::new(pagesize()).unwrap();
1101        let res = m.add_anon(0, pagesize() + 1).unwrap_err();
1102        match res {
1103            Error::InvalidAddress => {}
1104            e => panic!("unexpected error: {e}"),
1105        }
1106    }
1107
1108    #[test]
1109    fn arena_add_overlapping() {
1110        let ps = pagesize();
1111        let mut m =
1112            MemoryMappingArena::new(12 * ps).expect("failed to create `MemoryMappingArena`");
1113        m.add_anon(ps * 4, ps * 4)
1114            .expect("failed to add sub-mapping");
1115
1116        // Overlap in the front.
1117        m.add_anon(ps * 2, ps * 3)
1118            .expect("failed to add front overlapping sub-mapping");
1119
1120        // Overlap in the back.
1121        m.add_anon(ps * 7, ps * 3)
1122            .expect("failed to add back overlapping sub-mapping");
1123
1124        // Overlap the back of the first mapping, all of the middle mapping, and the front of the
1125        // last mapping.
1126        m.add_anon(ps * 3, ps * 6)
1127            .expect("failed to add mapping that overlaps several mappings");
1128    }
1129
1130    #[test]
1131    fn arena_remove_overlapping() {
1132        let ps = pagesize();
1133        let mut m =
1134            MemoryMappingArena::new(12 * ps).expect("failed to create `MemoryMappingArena`");
1135        m.add_anon(ps * 4, ps * 4)
1136            .expect("failed to add sub-mapping");
1137        m.add_anon(ps * 2, ps * 2)
1138            .expect("failed to add front overlapping sub-mapping");
1139        m.add_anon(ps * 8, ps * 2)
1140            .expect("failed to add back overlapping sub-mapping");
1141
1142        // Remove the back of the first mapping and the front of the second.
1143        m.remove(ps * 3, ps * 2)
1144            .expect("failed to remove front overlapping mapping");
1145
1146        // Remove the back of the second mapping and the front of the third.
1147        m.remove(ps * 7, ps * 2)
1148            .expect("failed to remove back overlapping mapping");
1149
1150        // Remove a mapping that completely overlaps the middle mapping.
1151        m.remove(ps * 5, ps * 2)
1152            .expect("failed to remove fully overlapping mapping");
1153    }
1154
1155    #[test]
1156    fn arena_remove_unaligned() {
1157        let ps = pagesize();
1158        let mut m =
1159            MemoryMappingArena::new(12 * ps).expect("failed to create `MemoryMappingArena`");
1160
1161        m.add_anon(0, ps).expect("failed to add mapping");
1162        m.remove(0, ps - 1)
1163            .expect("failed to remove unaligned mapping");
1164    }
1165
1166    #[test]
1167    fn arena_msync() {
1168        let size = 0x40000;
1169        let m = MemoryMappingArena::new(size).unwrap();
1170        let ps = pagesize();
1171        <dyn MappedRegion>::msync(&m, 0, ps).unwrap();
1172        <dyn MappedRegion>::msync(&m, 0, size).unwrap();
1173        <dyn MappedRegion>::msync(&m, ps, size - ps).unwrap();
1174        let res = <dyn MappedRegion>::msync(&m, ps, size).unwrap_err();
1175        match res {
1176            Error::InvalidAddress => {}
1177            e => panic!("unexpected error: {e}"),
1178        }
1179    }
1180
1181    #[test]
1182    fn arena_madvise() {
1183        let size = 0x40000;
1184        let mut m = MemoryMappingArena::new(size).unwrap();
1185        m.add_anon_protection(0, size, Protection::read_write())
1186            .expect("failed to add writable protection for madvise MADV_REMOVE");
1187        let ps = pagesize();
1188        <dyn MappedRegion>::madvise(&m, 0, ps, libc::MADV_PAGEOUT).unwrap();
1189        <dyn MappedRegion>::madvise(&m, 0, size, libc::MADV_PAGEOUT).unwrap();
1190        <dyn MappedRegion>::madvise(&m, ps, size - ps, libc::MADV_REMOVE).unwrap();
1191        let res = <dyn MappedRegion>::madvise(&m, ps, size, libc::MADV_PAGEOUT).unwrap_err();
1192        match res {
1193            Error::InvalidAddress => {}
1194            e => panic!("unexpected error: {e}"),
1195        }
1196    }
1197}
base/sys/linux/mmap.rs

base/sys/linux/
mmap.rs
