base/sys/linux/
mmap.rs

1// Copyright 2017 The ChromiumOS Authors
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5//! The mmap module provides a safe interface to mmap memory and ensures unmap is called when the
6//! mmap object leaves scope.
7
8use std::ptr::null_mut;
9
10use libc::c_int;
11use libc::PROT_READ;
12use libc::PROT_WRITE;
13use log::warn;
14
15use super::Error as ErrnoError;
16use crate::pagesize;
17use crate::AsRawDescriptor;
18use crate::Descriptor;
19use crate::MappedRegion;
20use crate::MemoryMapping as CrateMemoryMapping;
21use crate::MemoryMappingBuilder;
22use crate::MmapError as Error;
23use crate::MmapResult as Result;
24use crate::Protection;
25use crate::RawDescriptor;
26use crate::SafeDescriptor;
27
28impl From<Protection> for c_int {
29    #[inline(always)]
30    fn from(p: Protection) -> Self {
31        let mut value = 0;
32        if p.read {
33            value |= PROT_READ
34        }
35        if p.write {
36            value |= PROT_WRITE;
37        }
38        value
39    }
40}
41
42/// Validates that `offset`..`offset+range_size` lies within the bounds of a memory mapping of
43/// `mmap_size` bytes.  Also checks for any overflow.
44fn validate_includes_range(mmap_size: usize, offset: usize, range_size: usize) -> Result<()> {
45    // Ensure offset + size doesn't overflow
46    let end_offset = offset
47        .checked_add(range_size)
48        .ok_or(Error::InvalidAddress)?;
49    // Ensure offset + size are within the mapping bounds
50    if end_offset <= mmap_size {
51        Ok(())
52    } else {
53        Err(Error::InvalidAddress)
54    }
55}
56
57impl dyn MappedRegion {
58    /// Calls msync with MS_SYNC on a mapping of `size` bytes starting at `offset` from the start of
59    /// the region.  `offset`..`offset+size` must be contained within the `MappedRegion`.
60    pub fn msync(&self, offset: usize, size: usize) -> Result<()> {
61        validate_includes_range(self.size(), offset, size)?;
62
63        // SAFETY:
64        // Safe because the MemoryMapping/MemoryMappingArena interface ensures our pointer and size
65        // are correct, and we've validated that `offset`..`offset+size` is in the range owned by
66        // this `MappedRegion`.
67        let ret = unsafe {
68            libc::msync(
69                (self.as_ptr() as usize + offset) as *mut libc::c_void,
70                size,
71                libc::MS_SYNC,
72            )
73        };
74        if ret != -1 {
75            Ok(())
76        } else {
77            Err(Error::SystemCallFailed(ErrnoError::last()))
78        }
79    }
80
81    /// Calls madvise on a mapping of `size` bytes starting at `offset` from the start of
82    /// the region.  `offset`..`offset+size` must be contained within the `MappedRegion`.
83    pub fn madvise(&self, offset: usize, size: usize, advice: libc::c_int) -> Result<()> {
84        validate_includes_range(self.size(), offset, size)?;
85
86        // SAFETY:
87        // Safe because the MemoryMapping/MemoryMappingArena interface ensures our pointer and size
88        // are correct, and we've validated that `offset`..`offset+size` is in the range owned by
89        // this `MappedRegion`.
90        let ret = unsafe {
91            libc::madvise(
92                (self.as_ptr() as usize + offset) as *mut libc::c_void,
93                size,
94                advice,
95            )
96        };
97        if ret != -1 {
98            Ok(())
99        } else {
100            Err(Error::SystemCallFailed(ErrnoError::last()))
101        }
102    }
103}
104
105/// Wraps an anonymous shared memory mapping in the current process. Provides
106/// RAII semantics including munmap when no longer needed.
107#[derive(Debug)]
108pub struct MemoryMapping {
109    addr: *mut u8,
110    size: usize,
111}
112
113// SAFETY:
114// Send and Sync aren't automatically inherited for the raw address pointer.
115// Accessing that pointer is only done through the stateless interface which
116// allows the object to be shared by multiple threads without a decrease in
117// safety.
118unsafe impl Send for MemoryMapping {}
119// SAFETY: See safety comments for impl Send
120unsafe impl Sync for MemoryMapping {}
121
122impl MemoryMapping {
123    /// Creates an anonymous shared, read/write mapping of `size` bytes.
124    ///
125    /// # Arguments
126    /// * `size` - Size of memory region in bytes.
127    pub fn new(size: usize) -> Result<MemoryMapping> {
128        MemoryMapping::new_protection(size, None, Protection::read_write())
129    }
130
131    /// Creates an anonymous shared mapping of `size` bytes with `prot` protection.
132    ///
133    /// # Arguments
134    /// * `size` - Size of memory region in bytes.
135    /// * `align` - Optional alignment for MemoryMapping::addr.
136    /// * `prot` - Protection (e.g. readable/writable) of the memory region.
137    pub fn new_protection(
138        size: usize,
139        align: Option<u64>,
140        prot: Protection,
141    ) -> Result<MemoryMapping> {
142        // SAFETY:
143        // This is safe because we are creating an anonymous mapping in a place not already used by
144        // any other area in this process.
145        unsafe { MemoryMapping::try_mmap(None, size, align, prot.into(), None) }
146    }
147
148    /// Maps the first `size` bytes of the given `fd` as read/write.
149    ///
150    /// # Arguments
151    /// * `fd` - File descriptor to mmap from.
152    /// * `size` - Size of memory region in bytes.
153    pub fn from_fd(fd: &dyn AsRawDescriptor, size: usize) -> Result<MemoryMapping> {
154        MemoryMapping::from_fd_offset(fd, size, 0)
155    }
156
157    pub fn from_fd_offset(
158        fd: &dyn AsRawDescriptor,
159        size: usize,
160        offset: u64,
161    ) -> Result<MemoryMapping> {
162        MemoryMapping::from_fd_offset_protection(fd, size, offset, Protection::read_write())
163    }
164
165    /// Maps the `size` bytes starting at `offset` bytes of the given `fd` as read/write.
166    ///
167    /// # Arguments
168    /// * `fd` - File descriptor to mmap from.
169    /// * `size` - Size of memory region in bytes.
170    /// * `offset` - Offset in bytes from the beginning of `fd` to start the mmap.
171    /// * `prot` - Protection (e.g. readable/writable) of the memory region.
172    pub fn from_fd_offset_protection(
173        fd: &dyn AsRawDescriptor,
174        size: usize,
175        offset: u64,
176        prot: Protection,
177    ) -> Result<MemoryMapping> {
178        MemoryMapping::from_fd_offset_protection_populate(fd, size, offset, 0, prot, false)
179    }
180
181    /// Maps `size` bytes starting at `offset` from the given `fd` as read/write, and requests
182    /// that the pages are pre-populated.
183    /// # Arguments
184    /// * `fd` - File descriptor to mmap from.
185    /// * `size` - Size of memory region in bytes.
186    /// * `offset` - Offset in bytes from the beginning of `fd` to start the mmap.
187    /// * `align` - Alignment for MemoryMapping::addr.
188    /// * `prot` - Protection (e.g. readable/writable) of the memory region.
189    /// * `populate` - Populate (prefault) page tables for a mapping.
190    pub fn from_fd_offset_protection_populate(
191        fd: &dyn AsRawDescriptor,
192        size: usize,
193        offset: u64,
194        align: u64,
195        prot: Protection,
196        populate: bool,
197    ) -> Result<MemoryMapping> {
198        // SAFETY:
199        // This is safe because we are creating an anonymous mapping in a place not already used
200        // by any other area in this process.
201        unsafe {
202            MemoryMapping::try_mmap_populate(
203                None,
204                size,
205                Some(align),
206                prot.into(),
207                Some((fd, offset)),
208                populate,
209            )
210        }
211    }
212
213    /// Creates an anonymous shared mapping of `size` bytes with `prot` protection.
214    ///
215    /// # Arguments
216    ///
217    /// * `addr` - Memory address to mmap at.
218    /// * `size` - Size of memory region in bytes.
219    /// * `prot` - Protection (e.g. readable/writable) of the memory region.
220    ///
221    /// # Safety
222    ///
223    /// This function should not be called before the caller unmaps any mmap'd regions already
224    /// present at `(addr..addr+size)`.
225    pub unsafe fn new_protection_fixed(
226        addr: *mut u8,
227        size: usize,
228        prot: Protection,
229    ) -> Result<MemoryMapping> {
230        MemoryMapping::try_mmap(Some(addr), size, None, prot.into(), None)
231    }
232
233    /// Maps the `size` bytes starting at `offset` bytes of the given `fd` with
234    /// `prot` protections.
235    ///
236    /// # Arguments
237    ///
238    /// * `addr` - Memory address to mmap at.
239    /// * `fd` - File descriptor to mmap from.
240    /// * `size` - Size of memory region in bytes.
241    /// * `offset` - Offset in bytes from the beginning of `fd` to start the mmap.
242    /// * `prot` - Protection (e.g. readable/writable) of the memory region.
243    ///
244    /// # Safety
245    ///
246    /// This function should not be called before the caller unmaps any mmap'd regions already
247    /// present at `(addr..addr+size)`.
248    pub unsafe fn from_descriptor_offset_protection_fixed(
249        addr: *mut u8,
250        fd: &dyn AsRawDescriptor,
251        size: usize,
252        offset: u64,
253        prot: Protection,
254    ) -> Result<MemoryMapping> {
255        MemoryMapping::try_mmap(Some(addr), size, None, prot.into(), Some((fd, offset)))
256    }
257
258    /// Helper wrapper around try_mmap_populate when without MAP_POPULATE
259    unsafe fn try_mmap(
260        addr: Option<*mut u8>,
261        size: usize,
262        align: Option<u64>,
263        prot: c_int,
264        fd: Option<(&dyn AsRawDescriptor, u64)>,
265    ) -> Result<MemoryMapping> {
266        MemoryMapping::try_mmap_populate(addr, size, align, prot, fd, false)
267    }
268
269    /// Helper wrapper around libc::mmap that does some basic validation, and calls
270    /// madvise with MADV_DONTDUMP on the created mmap
271    unsafe fn try_mmap_populate(
272        addr: Option<*mut u8>,
273        size: usize,
274        align: Option<u64>,
275        prot: c_int,
276        fd: Option<(&dyn AsRawDescriptor, u64)>,
277        populate: bool,
278    ) -> Result<MemoryMapping> {
279        let mut flags = libc::MAP_SHARED;
280        if populate {
281            flags |= libc::MAP_POPULATE;
282        }
283        // If addr is provided, set the (FIXED | NORESERVE) flag, and validate addr alignment.
284        let addr = match addr {
285            Some(addr) => {
286                if (addr as usize) % pagesize() != 0 {
287                    return Err(Error::NotPageAligned);
288                }
289                flags |= libc::MAP_FIXED | libc::MAP_NORESERVE;
290                addr as *mut libc::c_void
291            }
292            None => null_mut(),
293        };
294
295        // mmap already PAGE_SIZE align the returned address.
296        let align = if align.unwrap_or(0) == pagesize() as u64 {
297            Some(0)
298        } else {
299            align
300        };
301
302        // Add an address if an alignment is requested.
303        let (addr, orig_addr, orig_size) = match align {
304            None | Some(0) => (addr, None, None),
305            Some(align) => {
306                if !addr.is_null() || !align.is_power_of_two() {
307                    return Err(Error::InvalidAlignment);
308                }
309                let orig_size = size
310                    .checked_add(align as usize)
311                    .ok_or(Error::InvalidAlignment)?;
312                let orig_addr = libc::mmap64(
313                    null_mut(),
314                    orig_size,
315                    prot,
316                    libc::MAP_PRIVATE | libc::MAP_NORESERVE | libc::MAP_ANONYMOUS,
317                    -1,
318                    0,
319                );
320                if orig_addr == libc::MAP_FAILED {
321                    return Err(Error::SystemCallFailed(ErrnoError::last()));
322                }
323
324                flags |= libc::MAP_FIXED;
325
326                let mask = align - 1;
327                (
328                    (orig_addr.wrapping_add(mask as usize) as u64 & !mask) as *mut libc::c_void,
329                    Some(orig_addr),
330                    Some(orig_size),
331                )
332            }
333        };
334
335        // If fd is provided, validate fd offset is within bounds. If not, it's anonymous mapping
336        // and set the (ANONYMOUS | NORESERVE) flag.
337        let (fd, offset) = match fd {
338            Some((fd, offset)) => {
339                if offset > libc::off64_t::MAX as u64 {
340                    return Err(Error::InvalidOffset);
341                }
342                // Map private for read-only seal. See below for upstream relax of the restriction.
343                // - https://lore.kernel.org/bpf/20231013103208.kdffpyerufr4ygnw@quack3/T/
344                // SAFETY:
345                // Safe because no third parameter is expected and we check the return result.
346                let seals = unsafe { libc::fcntl(fd.as_raw_descriptor(), libc::F_GET_SEALS) };
347                if (seals >= 0) && (seals & libc::F_SEAL_WRITE != 0) {
348                    flags &= !libc::MAP_SHARED;
349                    flags |= libc::MAP_PRIVATE;
350                }
351                (fd.as_raw_descriptor(), offset as libc::off64_t)
352            }
353            None => {
354                flags |= libc::MAP_ANONYMOUS | libc::MAP_NORESERVE;
355                (-1, 0)
356            }
357        };
358        let addr = libc::mmap64(addr, size, prot, flags, fd, offset);
359        if addr == libc::MAP_FAILED {
360            return Err(Error::SystemCallFailed(ErrnoError::last()));
361        }
362
363        // If an original mmap exists, we can now remove the unused regions
364        if let Some(orig_addr) = orig_addr {
365            let mut unmap_start = orig_addr as usize;
366            let mut unmap_end = addr as usize;
367            let mut unmap_size = unmap_end - unmap_start;
368
369            if unmap_size > 0 {
370                libc::munmap(orig_addr, unmap_size);
371            }
372
373            unmap_start = addr as usize + size;
374            unmap_end = orig_addr as usize + orig_size.unwrap();
375            unmap_size = unmap_end - unmap_start;
376
377            if unmap_size > 0 {
378                libc::munmap(unmap_start as *mut libc::c_void, unmap_size);
379            }
380        }
381
382        // This is safe because we call madvise with a valid address and size.
383        let _ = libc::madvise(addr, size, libc::MADV_DONTDUMP);
384
385        // This is safe because KSM's only userspace visible effects are timing
386        // and memory consumption; it doesn't affect rust safety semantics.
387        // KSM is also disabled by default, and this flag is only a hint.
388        let _ = libc::madvise(addr, size, libc::MADV_MERGEABLE);
389
390        Ok(MemoryMapping {
391            addr: addr as *mut u8,
392            size,
393        })
394    }
395
396    /// Madvise the kernel to unmap on fork.
397    pub fn use_dontfork(&self) -> Result<()> {
398        // SAFETY:
399        // This is safe because we call madvise with a valid address and size, and we check the
400        // return value.
401        let ret = unsafe {
402            libc::madvise(
403                self.as_ptr() as *mut libc::c_void,
404                self.size(),
405                libc::MADV_DONTFORK,
406            )
407        };
408        if ret == -1 {
409            Err(Error::SystemCallFailed(ErrnoError::last()))
410        } else {
411            Ok(())
412        }
413    }
414
415    /// Madvise the kernel to use Huge Pages for this mapping.
416    pub fn use_hugepages(&self) -> Result<()> {
417        const SZ_2M: usize = 2 * 1024 * 1024;
418
419        // THP uses 2M pages, so use THP only on mappings that are at least
420        // 2M in size.
421        if self.size() < SZ_2M {
422            return Ok(());
423        }
424
425        // SAFETY:
426        // This is safe because we call madvise with a valid address and size, and we check the
427        // return value.
428        let ret = unsafe {
429            libc::madvise(
430                self.as_ptr() as *mut libc::c_void,
431                self.size(),
432                libc::MADV_HUGEPAGE,
433            )
434        };
435        if ret == -1 {
436            Err(Error::SystemCallFailed(ErrnoError::last()))
437        } else {
438            Ok(())
439        }
440    }
441
442    /// Calls msync with MS_SYNC on the mapping.
443    pub fn msync(&self) -> Result<()> {
444        // SAFETY:
445        // This is safe since we use the exact address and length of a known
446        // good memory mapping.
447        let ret = unsafe {
448            libc::msync(
449                self.as_ptr() as *mut libc::c_void,
450                self.size(),
451                libc::MS_SYNC,
452            )
453        };
454        if ret == -1 {
455            return Err(Error::SystemCallFailed(ErrnoError::last()));
456        }
457        Ok(())
458    }
459
460    /// Uses madvise to tell the kernel to remove the specified range.  Subsequent reads
461    /// to the pages in the range will return zero bytes.
462    pub fn remove_range(&self, mem_offset: usize, count: usize) -> Result<()> {
463        self.range_end(mem_offset, count)
464            .map_err(|_| Error::InvalidRange(mem_offset, count, self.size()))?;
465        // SAFETY: Safe because all the args to madvise are valid and the return
466        // value is checked.
467        let ret = unsafe {
468            // madvising away the region is the same as the guest changing it.
469            // Next time it is read, it may return zero pages.
470            libc::madvise(
471                (self.addr as usize + mem_offset) as *mut _,
472                count,
473                libc::MADV_REMOVE,
474            )
475        };
476        if ret < 0 {
477            Err(Error::SystemCallFailed(super::Error::last()))
478        } else {
479            Ok(())
480        }
481    }
482
483    /// Tell the kernel to readahead the range.
484    ///
485    /// This does not block the thread by I/O wait from reading the backed file. This does not
486    /// guarantee that the pages are surely present unless the pages are mlock(2)ed by
487    /// `lock_on_fault_unchecked()`.
488    ///
489    /// The `mem_offset` and `count` must be validated by caller.
490    ///
491    /// # Arguments
492    ///
493    /// * `mem_offset` - The offset of the head of the range.
494    /// * `count` - The size in bytes of the range.
495    pub fn async_prefetch(&self, mem_offset: usize, count: usize) -> Result<()> {
496        // Validation
497        self.range_end(mem_offset, count)
498            .map_err(|_| Error::InvalidRange(mem_offset, count, self.size()))?;
499        // SAFETY:
500        // Safe because populating the pages from the backed file does not affect the Rust memory
501        // safety.
502        let ret = unsafe {
503            libc::madvise(
504                (self.addr as usize + mem_offset) as *mut _,
505                count,
506                libc::MADV_WILLNEED,
507            )
508        };
509        if ret < 0 {
510            Err(Error::SystemCallFailed(super::Error::last()))
511        } else {
512            Ok(())
513        }
514    }
515
516    /// Tell the kernel to drop the page cache.
517    ///
518    /// This cannot be applied to locked pages.
519    ///
520    /// The `mem_offset` and `count` must be validated by caller.
521    ///
522    /// NOTE: This function has destructive semantics. It throws away data in the page cache without
523    /// writing it to the backing file. If the data is important, the caller should ensure it is
524    /// written to disk before calling this function or should use MADV_PAGEOUT instead.
525    ///
526    /// # Arguments
527    ///
528    /// * `mem_offset` - The offset of the head of the range.
529    /// * `count` - The size in bytes of the range.
530    pub fn drop_page_cache(&self, mem_offset: usize, count: usize) -> Result<()> {
531        // Validation
532        self.range_end(mem_offset, count)
533            .map_err(|_| Error::InvalidRange(mem_offset, count, self.size()))?;
534        // SAFETY:
535        // Safe because dropping the page cache does not affect the Rust memory safety.
536        let ret = unsafe {
537            libc::madvise(
538                (self.addr as usize + mem_offset) as *mut _,
539                count,
540                libc::MADV_DONTNEED,
541            )
542        };
543        if ret < 0 {
544            Err(Error::SystemCallFailed(super::Error::last()))
545        } else {
546            Ok(())
547        }
548    }
549
550    /// Lock the resident pages in the range not to be swapped out.
551    ///
552    /// The remaining nonresident page are locked when they are populated.
553    ///
554    /// The `mem_offset` and `count` must be validated by caller.
555    ///
556    /// # Arguments
557    ///
558    /// * `mem_offset` - The offset of the head of the range.
559    /// * `count` - The size in bytes of the range.
560    pub fn lock_on_fault(&self, mem_offset: usize, count: usize) -> Result<()> {
561        // Validation
562        self.range_end(mem_offset, count)
563            .map_err(|_| Error::InvalidRange(mem_offset, count, self.size()))?;
564        let addr = self.addr as usize + mem_offset;
565        // SAFETY:
566        // Safe because MLOCK_ONFAULT only affects the swap behavior of the kernel, so it has no
567        // impact on rust semantics.
568        let ret = unsafe { libc::mlock2(addr as *mut _, count, libc::MLOCK_ONFAULT) };
569        if ret < 0 {
570            let errno = super::Error::last();
571            warn!(
572                "failed to mlock at {:#x} with length {}: {}",
573                addr as u64,
574                self.size(),
575                errno,
576            );
577            Err(Error::SystemCallFailed(errno))
578        } else {
579            Ok(())
580        }
581    }
582
583    /// Unlock the range of pages.
584    ///
585    /// Unlocking non-locked pages does not fail.
586    ///
587    /// The `mem_offset` and `count` must be validated by caller.
588    ///
589    /// # Arguments
590    ///
591    /// * `mem_offset` - The offset of the head of the range.
592    /// * `count` - The size in bytes of the range.
593    pub fn unlock(&self, mem_offset: usize, count: usize) -> Result<()> {
594        // Validation
595        self.range_end(mem_offset, count)
596            .map_err(|_| Error::InvalidRange(mem_offset, count, self.size()))?;
597        // SAFETY:
598        // Safe because munlock(2) does not affect the Rust memory safety.
599        let ret = unsafe { libc::munlock((self.addr as usize + mem_offset) as *mut _, count) };
600        if ret < 0 {
601            Err(Error::SystemCallFailed(super::Error::last()))
602        } else {
603            Ok(())
604        }
605    }
606
607    // Check that offset+count is valid and return the sum.
608    pub(crate) fn range_end(&self, offset: usize, count: usize) -> Result<usize> {
609        let mem_end = offset.checked_add(count).ok_or(Error::InvalidAddress)?;
610        if mem_end > self.size() {
611            return Err(Error::InvalidAddress);
612        }
613        Ok(mem_end)
614    }
615}
616
617// SAFETY:
618// Safe because the pointer and size point to a memory range owned by this MemoryMapping that won't
619// be unmapped until it's Dropped.
620unsafe impl MappedRegion for MemoryMapping {
621    fn as_ptr(&self) -> *mut u8 {
622        self.addr
623    }
624
625    fn size(&self) -> usize {
626        self.size
627    }
628}
629
630impl Drop for MemoryMapping {
631    fn drop(&mut self) {
632        // SAFETY:
633        // This is safe because we mmap the area at addr ourselves, and nobody
634        // else is holding a reference to it.
635        unsafe {
636            libc::munmap(self.addr as *mut libc::c_void, self.size);
637        }
638    }
639}
640
641/// Tracks Fixed Memory Maps within an anonymous memory-mapped fixed-sized arena
642/// in the current process.
643pub struct MemoryMappingArena {
644    addr: *mut u8,
645    size: usize,
646}
647
648// SAFETY:
649// Send and Sync aren't automatically inherited for the raw address pointer.
650// Accessing that pointer is only done through the stateless interface which
651// allows the object to be shared by multiple threads without a decrease in
652// safety.
653unsafe impl Send for MemoryMappingArena {}
654// SAFETY: See safety comments for impl Send
655unsafe impl Sync for MemoryMappingArena {}
656
657impl MemoryMappingArena {
658    /// Creates an mmap arena of `size` bytes.
659    ///
660    /// # Arguments
661    /// * `size` - Size of memory region in bytes.
662    pub fn new(size: usize) -> Result<MemoryMappingArena> {
663        // Reserve the arena's memory using an anonymous read-only mmap.
664        MemoryMapping::new_protection(size, None, Protection::read()).map(From::from)
665    }
666
667    /// Creates an mmap arena of `size` bytes with custom initial memory protections.
668    pub fn new_protection(size: usize, prot: Protection) -> Result<MemoryMappingArena> {
669        MemoryMapping::new_protection(size, None, prot).map(From::from)
670    }
671
672    /// Anonymously maps `size` bytes at `offset` bytes from the start of the arena
673    /// with `prot` protections. `offset` must be page aligned.
674    ///
675    /// # Arguments
676    /// * `offset` - Page aligned offset into the arena in bytes.
677    /// * `size` - Size of memory region in bytes.
678    /// * `prot` - Protection (e.g. readable/writable) of the memory region.
679    pub fn add_anon_protection(
680        &mut self,
681        offset: usize,
682        size: usize,
683        prot: Protection,
684    ) -> Result<()> {
685        self.try_add(offset, size, prot, None)
686    }
687
688    /// Anonymously maps `size` bytes at `offset` bytes from the start of the arena.
689    /// `offset` must be page aligned.
690    ///
691    /// # Arguments
692    /// * `offset` - Page aligned offset into the arena in bytes.
693    /// * `size` - Size of memory region in bytes.
694    pub fn add_anon(&mut self, offset: usize, size: usize) -> Result<()> {
695        self.add_anon_protection(offset, size, Protection::read_write())
696    }
697
698    /// Maps `size` bytes from the start of the given `fd` at `offset` bytes from
699    /// the start of the arena. `offset` must be page aligned.
700    ///
701    /// # Arguments
702    /// * `offset` - Page aligned offset into the arena in bytes.
703    /// * `size` - Size of memory region in bytes.
704    /// * `fd` - File descriptor to mmap from.
705    pub fn add_fd(&mut self, offset: usize, size: usize, fd: &dyn AsRawDescriptor) -> Result<()> {
706        self.add_fd_offset(offset, size, fd, 0)
707    }
708
709    /// Maps `size` bytes starting at `fs_offset` bytes from within the given `fd`
710    /// at `offset` bytes from the start of the arena. `offset` must be page aligned.
711    ///
712    /// # Arguments
713    /// * `offset` - Page aligned offset into the arena in bytes.
714    /// * `size` - Size of memory region in bytes.
715    /// * `fd` - File descriptor to mmap from.
716    /// * `fd_offset` - Offset in bytes from the beginning of `fd` to start the mmap.
717    pub fn add_fd_offset(
718        &mut self,
719        offset: usize,
720        size: usize,
721        fd: &dyn AsRawDescriptor,
722        fd_offset: u64,
723    ) -> Result<()> {
724        self.add_fd_offset_protection(offset, size, fd, fd_offset, Protection::read_write())
725    }
726
727    /// Maps `size` bytes starting at `fs_offset` bytes from within the given `fd`
728    /// at `offset` bytes from the start of the arena with `prot` protections.
729    /// `offset` must be page aligned.
730    ///
731    /// # Arguments
732    /// * `offset` - Page aligned offset into the arena in bytes.
733    /// * `size` - Size of memory region in bytes.
734    /// * `fd` - File descriptor to mmap from.
735    /// * `fd_offset` - Offset in bytes from the beginning of `fd` to start the mmap.
736    /// * `prot` - Protection (e.g. readable/writable) of the memory region.
737    pub fn add_fd_offset_protection(
738        &mut self,
739        offset: usize,
740        size: usize,
741        fd: &dyn AsRawDescriptor,
742        fd_offset: u64,
743        prot: Protection,
744    ) -> Result<()> {
745        self.try_add(offset, size, prot, Some((fd, fd_offset)))
746    }
747
748    /// Helper method that calls appropriate MemoryMapping constructor and adds
749    /// the resulting map into the arena.
750    fn try_add(
751        &mut self,
752        offset: usize,
753        size: usize,
754        prot: Protection,
755        fd: Option<(&dyn AsRawDescriptor, u64)>,
756    ) -> Result<()> {
757        // Ensure offset is page-aligned
758        if offset % pagesize() != 0 {
759            return Err(Error::NotPageAligned);
760        }
761        validate_includes_range(self.size(), offset, size)?;
762
763        // SAFETY:
764        // This is safe since the range has been validated.
765        let mmap = unsafe {
766            match fd {
767                Some((fd, fd_offset)) => MemoryMapping::from_descriptor_offset_protection_fixed(
768                    self.addr.add(offset),
769                    fd,
770                    size,
771                    fd_offset,
772                    prot,
773                )?,
774                None => MemoryMapping::new_protection_fixed(self.addr.add(offset), size, prot)?,
775            }
776        };
777
778        // This mapping will get automatically removed when we drop the whole arena.
779        std::mem::forget(mmap);
780        Ok(())
781    }
782
783    /// Removes `size` bytes at `offset` bytes from the start of the arena. `offset` must be page
784    /// aligned.
785    ///
786    /// # Arguments
787    /// * `offset` - Page aligned offset into the arena in bytes.
788    /// * `size` - Size of memory region in bytes.
789    pub fn remove(&mut self, offset: usize, size: usize) -> Result<()> {
790        self.try_add(offset, size, Protection::read(), None)
791    }
792}
793
794// SAFETY:
795// Safe because the pointer and size point to a memory range owned by this MemoryMappingArena that
796// won't be unmapped until it's Dropped.
797unsafe impl MappedRegion for MemoryMappingArena {
798    fn as_ptr(&self) -> *mut u8 {
799        self.addr
800    }
801
802    fn size(&self) -> usize {
803        self.size
804    }
805
806    fn add_fd_mapping(
807        &mut self,
808        offset: usize,
809        size: usize,
810        fd: &dyn AsRawDescriptor,
811        fd_offset: u64,
812        prot: Protection,
813    ) -> Result<()> {
814        self.add_fd_offset_protection(offset, size, fd, fd_offset, prot)
815    }
816
817    fn remove_mapping(&mut self, offset: usize, size: usize) -> Result<()> {
818        self.remove(offset, size)
819    }
820}
821
822impl From<MemoryMapping> for MemoryMappingArena {
823    fn from(mmap: MemoryMapping) -> Self {
824        let addr = mmap.as_ptr();
825        let size = mmap.size();
826
827        // Forget the original mapping because the `MemoryMappingArena` will take care of calling
828        // `munmap` when it is dropped.
829        std::mem::forget(mmap);
830        MemoryMappingArena { addr, size }
831    }
832}
833
834impl From<CrateMemoryMapping> for MemoryMappingArena {
835    fn from(mmap: CrateMemoryMapping) -> Self {
836        MemoryMappingArena::from(mmap.mapping)
837    }
838}
839
840impl Drop for MemoryMappingArena {
841    fn drop(&mut self) {
842        // SAFETY:
843        // This is safe because we own this memory range, and nobody else is holding a reference to
844        // it.
845        unsafe {
846            libc::munmap(self.addr as *mut libc::c_void, self.size);
847        }
848    }
849}
850
851impl CrateMemoryMapping {
852    pub fn use_dontfork(&self) -> Result<()> {
853        self.mapping.use_dontfork()
854    }
855
856    pub fn use_hugepages(&self) -> Result<()> {
857        self.mapping.use_hugepages()
858    }
859
860    pub fn from_raw_ptr(addr: RawDescriptor, size: usize) -> Result<CrateMemoryMapping> {
861        MemoryMapping::from_fd_offset(&Descriptor(addr), size, 0).map(|mapping| {
862            CrateMemoryMapping {
863                mapping,
864                _file_descriptor: None,
865            }
866        })
867    }
868}
869
870pub trait MemoryMappingUnix {
871    /// Remove the specified range from the mapping.
872    fn remove_range(&self, mem_offset: usize, count: usize) -> Result<()>;
873    /// Tell the kernel to readahead the range.
874    fn async_prefetch(&self, mem_offset: usize, count: usize) -> Result<()>;
875    /// Tell the kernel to drop the page cache.
876    fn drop_page_cache(&self, mem_offset: usize, count: usize) -> Result<()>;
877    /// Lock the resident pages in the range not to be swapped out.
878    fn lock_on_fault(&self, mem_offset: usize, count: usize) -> Result<()>;
879    /// Unlock the range of pages.
880    fn unlock(&self, mem_offset: usize, count: usize) -> Result<()>;
881    /// Disable host swap for this mapping.
882    fn lock_all(&self) -> Result<()>;
883}
884
885impl MemoryMappingUnix for CrateMemoryMapping {
886    fn remove_range(&self, mem_offset: usize, count: usize) -> Result<()> {
887        self.mapping.remove_range(mem_offset, count)
888    }
889    fn async_prefetch(&self, mem_offset: usize, count: usize) -> Result<()> {
890        self.mapping.async_prefetch(mem_offset, count)
891    }
892    fn drop_page_cache(&self, mem_offset: usize, count: usize) -> Result<()> {
893        self.mapping.drop_page_cache(mem_offset, count)
894    }
895    fn lock_on_fault(&self, mem_offset: usize, count: usize) -> Result<()> {
896        self.mapping.lock_on_fault(mem_offset, count)
897    }
898    fn unlock(&self, mem_offset: usize, count: usize) -> Result<()> {
899        self.mapping.unlock(mem_offset, count)
900    }
901    fn lock_all(&self) -> Result<()> {
902        self.mapping.lock_on_fault(0, self.mapping.size())
903    }
904}
905
906pub trait MemoryMappingBuilderUnix<'a> {
907    #[allow(clippy::wrong_self_convention)]
908    fn from_descriptor(self, descriptor: &'a dyn AsRawDescriptor) -> MemoryMappingBuilder<'a>;
909}
910
911impl<'a> MemoryMappingBuilderUnix<'a> for MemoryMappingBuilder<'a> {
912    /// Build the memory mapping given the specified descriptor to mapped memory
913    ///
914    /// Default: Create a new memory mapping.
915    #[allow(clippy::wrong_self_convention)]
916    fn from_descriptor(mut self, descriptor: &'a dyn AsRawDescriptor) -> MemoryMappingBuilder<'a> {
917        self.descriptor = Some(descriptor);
918        self
919    }
920}
921
922impl<'a> MemoryMappingBuilder<'a> {
923    /// Request that the mapped pages are pre-populated
924    ///
925    /// Default: Do not populate
926    pub fn populate(mut self) -> MemoryMappingBuilder<'a> {
927        self.populate = true;
928        self
929    }
930
931    /// Build a MemoryMapping from the provided options.
932    pub fn build(self) -> Result<CrateMemoryMapping> {
933        match self.descriptor {
934            None => {
935                if self.populate {
936                    // Population not supported for new mmaps
937                    return Err(Error::InvalidArgument);
938                }
939                MemoryMappingBuilder::wrap(
940                    MemoryMapping::new_protection(
941                        self.size,
942                        self.align,
943                        self.protection.unwrap_or_else(Protection::read_write),
944                    )?,
945                    None,
946                )
947            }
948            Some(descriptor) => MemoryMappingBuilder::wrap(
949                MemoryMapping::from_fd_offset_protection_populate(
950                    descriptor,
951                    self.size,
952                    self.offset.unwrap_or(0),
953                    self.align.unwrap_or(0),
954                    self.protection.unwrap_or_else(Protection::read_write),
955                    self.populate,
956                )?,
957                None,
958            ),
959        }
960    }
961
962    pub(crate) fn wrap(
963        mapping: MemoryMapping,
964        file_descriptor: Option<&'a dyn AsRawDescriptor>,
965    ) -> Result<CrateMemoryMapping> {
966        let file_descriptor = match file_descriptor {
967            Some(descriptor) => Some(
968                SafeDescriptor::try_from(descriptor)
969                    .map_err(|_| Error::SystemCallFailed(ErrnoError::last()))?,
970            ),
971            None => None,
972        };
973        Ok(CrateMemoryMapping {
974            mapping,
975            _file_descriptor: file_descriptor,
976        })
977    }
978}
979
980#[cfg(test)]
981mod tests {
982    use tempfile::tempfile;
983
984    use super::*;
985    use crate::descriptor::Descriptor;
986    use crate::VolatileMemory;
987    use crate::VolatileMemoryError;
988
989    #[test]
990    fn basic_map() {
991        let m = MemoryMappingBuilder::new(1024).build().unwrap();
992        assert_eq!(1024, m.size());
993    }
994
995    #[test]
996    fn map_invalid_size() {
997        let res = MemoryMappingBuilder::new(0).build().unwrap_err();
998        if let Error::SystemCallFailed(e) = res {
999            assert_eq!(e.errno(), libc::EINVAL);
1000        } else {
1001            panic!("unexpected error: {res}");
1002        }
1003    }
1004
1005    #[test]
1006    fn map_invalid_fd() {
1007        let fd = Descriptor(-1);
1008        let res = MemoryMapping::from_fd(&fd, 1024).unwrap_err();
1009        if let Error::SystemCallFailed(e) = res {
1010            assert_eq!(e.errno(), libc::EBADF);
1011        } else {
1012            panic!("unexpected error: {res}");
1013        }
1014    }
1015
1016    #[test]
1017    fn test_write_past_end() {
1018        let m = MemoryMappingBuilder::new(5).build().unwrap();
1019        let res = m.write_slice(&[1, 2, 3, 4, 5, 6], 0);
1020        assert!(res.is_ok());
1021        assert_eq!(res.unwrap(), 5);
1022    }
1023
1024    #[test]
1025    fn slice_size() {
1026        let m = MemoryMappingBuilder::new(5).build().unwrap();
1027        let s = m.get_slice(2, 3).unwrap();
1028        assert_eq!(s.size(), 3);
1029    }
1030
1031    #[test]
1032    fn slice_addr() {
1033        let m = MemoryMappingBuilder::new(5).build().unwrap();
1034        let s = m.get_slice(2, 3).unwrap();
1035        // SAFETY: all addresses are known to exist.
1036        assert_eq!(s.as_ptr(), unsafe { m.as_ptr().offset(2) });
1037    }
1038
1039    #[test]
1040    fn slice_overflow_error() {
1041        let m = MemoryMappingBuilder::new(5).build().unwrap();
1042        let res = m.get_slice(usize::MAX, 3).unwrap_err();
1043        assert_eq!(
1044            res,
1045            VolatileMemoryError::Overflow {
1046                base: usize::MAX,
1047                offset: 3,
1048            }
1049        );
1050    }
1051    #[test]
1052    fn slice_oob_error() {
1053        let m = MemoryMappingBuilder::new(5).build().unwrap();
1054        let res = m.get_slice(3, 3).unwrap_err();
1055        assert_eq!(res, VolatileMemoryError::OutOfBounds { addr: 6 });
1056    }
1057
1058    #[test]
1059    fn from_fd_offset_invalid() {
1060        let fd = tempfile().unwrap();
1061        let res =
1062            MemoryMapping::from_fd_offset(&fd, 4096, (libc::off64_t::MAX as u64) + 1).unwrap_err();
1063        match res {
1064            Error::InvalidOffset => {}
1065            e => panic!("unexpected error: {e}"),
1066        }
1067    }
1068
1069    #[test]
1070    fn map_align_overflow() {
1071        let res = MemoryMappingBuilder::new(usize::MAX)
1072            .align(pagesize() as u64 * 2)
1073            .build()
1074            .unwrap_err();
1075        match res {
1076            Error::InvalidAlignment => {}
1077            e => panic!("unexpected error: {e}"),
1078        }
1079    }
1080
1081    #[test]
1082    fn arena_new() {
1083        let m = MemoryMappingArena::new(0x40000).unwrap();
1084        assert_eq!(m.size(), 0x40000);
1085    }
1086
1087    #[test]
1088    fn arena_add() {
1089        let mut m = MemoryMappingArena::new(0x40000).unwrap();
1090        assert!(m.add_anon(0, pagesize() * 4).is_ok());
1091    }
1092
1093    #[test]
1094    fn arena_remove() {
1095        let mut m = MemoryMappingArena::new(0x40000).unwrap();
1096        assert!(m.add_anon(0, pagesize() * 4).is_ok());
1097        assert!(m.remove(0, pagesize()).is_ok());
1098        assert!(m.remove(0, pagesize() * 2).is_ok());
1099    }
1100
1101    #[test]
1102    fn arena_add_alignment_error() {
1103        let mut m = MemoryMappingArena::new(pagesize() * 2).unwrap();
1104        assert!(m.add_anon(0, 0x100).is_ok());
1105        let res = m.add_anon(pagesize() + 1, 0x100).unwrap_err();
1106        match res {
1107            Error::NotPageAligned => {}
1108            e => panic!("unexpected error: {e}"),
1109        }
1110    }
1111
1112    #[test]
1113    fn arena_add_oob_error() {
1114        let mut m = MemoryMappingArena::new(pagesize()).unwrap();
1115        let res = m.add_anon(0, pagesize() + 1).unwrap_err();
1116        match res {
1117            Error::InvalidAddress => {}
1118            e => panic!("unexpected error: {e}"),
1119        }
1120    }
1121
1122    #[test]
1123    fn arena_add_overlapping() {
1124        let ps = pagesize();
1125        let mut m =
1126            MemoryMappingArena::new(12 * ps).expect("failed to create `MemoryMappingArena`");
1127        m.add_anon(ps * 4, ps * 4)
1128            .expect("failed to add sub-mapping");
1129
1130        // Overlap in the front.
1131        m.add_anon(ps * 2, ps * 3)
1132            .expect("failed to add front overlapping sub-mapping");
1133
1134        // Overlap in the back.
1135        m.add_anon(ps * 7, ps * 3)
1136            .expect("failed to add back overlapping sub-mapping");
1137
1138        // Overlap the back of the first mapping, all of the middle mapping, and the front of the
1139        // last mapping.
1140        m.add_anon(ps * 3, ps * 6)
1141            .expect("failed to add mapping that overlaps several mappings");
1142    }
1143
1144    #[test]
1145    fn arena_remove_overlapping() {
1146        let ps = pagesize();
1147        let mut m =
1148            MemoryMappingArena::new(12 * ps).expect("failed to create `MemoryMappingArena`");
1149        m.add_anon(ps * 4, ps * 4)
1150            .expect("failed to add sub-mapping");
1151        m.add_anon(ps * 2, ps * 2)
1152            .expect("failed to add front overlapping sub-mapping");
1153        m.add_anon(ps * 8, ps * 2)
1154            .expect("failed to add back overlapping sub-mapping");
1155
1156        // Remove the back of the first mapping and the front of the second.
1157        m.remove(ps * 3, ps * 2)
1158            .expect("failed to remove front overlapping mapping");
1159
1160        // Remove the back of the second mapping and the front of the third.
1161        m.remove(ps * 7, ps * 2)
1162            .expect("failed to remove back overlapping mapping");
1163
1164        // Remove a mapping that completely overlaps the middle mapping.
1165        m.remove(ps * 5, ps * 2)
1166            .expect("failed to remove fully overlapping mapping");
1167    }
1168
1169    #[test]
1170    fn arena_remove_unaligned() {
1171        let ps = pagesize();
1172        let mut m =
1173            MemoryMappingArena::new(12 * ps).expect("failed to create `MemoryMappingArena`");
1174
1175        m.add_anon(0, ps).expect("failed to add mapping");
1176        m.remove(0, ps - 1)
1177            .expect("failed to remove unaligned mapping");
1178    }
1179
1180    #[test]
1181    fn arena_msync() {
1182        let size = 0x40000;
1183        let m = MemoryMappingArena::new(size).unwrap();
1184        let ps = pagesize();
1185        <dyn MappedRegion>::msync(&m, 0, ps).unwrap();
1186        <dyn MappedRegion>::msync(&m, 0, size).unwrap();
1187        <dyn MappedRegion>::msync(&m, ps, size - ps).unwrap();
1188        let res = <dyn MappedRegion>::msync(&m, ps, size).unwrap_err();
1189        match res {
1190            Error::InvalidAddress => {}
1191            e => panic!("unexpected error: {e}"),
1192        }
1193    }
1194
1195    #[test]
1196    fn arena_madvise() {
1197        let size = 0x40000;
1198        let mut m = MemoryMappingArena::new(size).unwrap();
1199        m.add_anon_protection(0, size, Protection::read_write())
1200            .expect("failed to add writable protection for madvise MADV_REMOVE");
1201        let ps = pagesize();
1202        <dyn MappedRegion>::madvise(&m, 0, ps, libc::MADV_PAGEOUT).unwrap();
1203        <dyn MappedRegion>::madvise(&m, 0, size, libc::MADV_PAGEOUT).unwrap();
1204        <dyn MappedRegion>::madvise(&m, ps, size - ps, libc::MADV_REMOVE).unwrap();
1205        let res = <dyn MappedRegion>::madvise(&m, ps, size, libc::MADV_PAGEOUT).unwrap_err();
1206        match res {
1207            Error::InvalidAddress => {}
1208            e => panic!("unexpected error: {e}"),
1209        }
1210    }
1211}
base/sys/linux/mmap.rs

base/sys/linux/
mmap.rs
