From 8f7a05072ffafb994bd92012d0dc69b451f0a678 Mon Sep 17 00:00:00 2001 From: Tom Dohrmann Date: Fri, 4 Jun 2021 20:26:48 +0200 Subject: [PATCH 01/18] add range functions to Mapper --- src/structures/paging/mapper/mod.rs | 284 +++++++++++++++++++++++++++- 1 file changed, 283 insertions(+), 1 deletion(-) diff --git a/src/structures/paging/mapper/mod.rs b/src/structures/paging/mapper/mod.rs index f971b9ebc..3cd5ff124 100644 --- a/src/structures/paging/mapper/mod.rs +++ b/src/structures/paging/mapper/mod.rs @@ -7,8 +7,9 @@ pub use self::offset_page_table::OffsetPageTable; pub use self::recursive_page_table::{InvalidPageTable, RecursivePageTable}; use crate::structures::paging::{ + frame::PhysFrameRange, frame_alloc::{FrameAllocator, FrameDeallocator}, - page::PageRangeInclusive, + page::{PageRange, PageRangeInclusive}, page_table::PageTableFlags, Page, PageSize, PhysFrame, Size1GiB, Size2MiB, Size4KiB, }; @@ -197,6 +198,52 @@ pub trait Mapper { } } + /// Maps the given range of frames to the range of virtual pages. + /// + /// ## Safety + /// + /// This is a convencience function that invokes [`Mapper::map_to`] internally, so + /// all safety requirements of it also apply for this function. + /// + /// ## Panics + /// + /// This function panics if the amount of pages does not equal the amount of frames. + /// + /// ## Errors + /// + /// If an error occurs half-way through a [`MapperFlushRange`] is returned that contains the frames that were successfully mapped. + #[inline] + unsafe fn map_to_range( + &mut self, + pages: PageRange, + frames: PhysFrameRange, + flags: PageTableFlags, + frame_allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + Self: Sized, + A: FrameAllocator + ?Sized, + { + assert_eq!(pages.count(), frames.count()); + + pages + .zip(frames) + .try_for_each(|(page, frame)| { + unsafe { self.map_to(page, frame, flags, frame_allocator) } + .map(|_| ()) + .map_err(|e| { + ( + e, + MapperFlushRange::new(PageRange { + start: pages.start, + end: page, + }), + ) + }) + }) + .map(|_| MapperFlushRange::new(pages)) + } + /// Creates a new mapping in the page table. /// /// This function might need additional physical frames to create new page tables. These @@ -279,11 +326,151 @@ pub trait Mapper { Self: Sized, A: FrameAllocator + ?Sized; + /// Maps the given range of frames to the range of virtual pages. + /// + /// ## Safety + /// + /// This is a convencience function that invokes [`Mapper::map_to_with_table_flags`] internally, so + /// all safety requirements of it also apply for this function. + /// + /// ## Panics + /// + /// This function panics if the amount of pages does not equal the amount of frames. + /// + /// ## Errors + /// + /// If an error occurs half-way through a [`MapperFlushRange`] is returned that contains the frames that were successfully mapped. + unsafe fn map_to_range_with_table_flags( + &mut self, + pages: PageRange, + frames: PhysFrameRange, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + frame_allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + Self: Sized, + A: FrameAllocator + ?Sized, + { + assert_eq!(pages.count(), frames.count()); + + pages + .zip(frames) + .try_for_each(|(page, frame)| { + unsafe { + self.map_to_with_table_flags( + page, + frame, + flags, + parent_table_flags, + frame_allocator, + ) + } + .map(|_| ()) + .map_err(|e| { + ( + e, + MapperFlushRange::new(PageRange { + start: pages.start, + end: page, + }), + ) + }) + }) + .map(|_| MapperFlushRange::new(pages)) + } + + /// Maps frames from the allocator to the given range of virtual pages. + /// + /// ## Safety + /// + /// This is a convencience function that invokes [`Mapper::map_to_with_table_flags`] internally, so + /// all safety requirements of it also apply for this function. + /// + /// ## Errors + /// + /// If an error occurs half-way through a [`MapperFlushRange`] is returned that contains the frames that were successfully mapped. + unsafe fn map_range_with_table_flags( + &mut self, + mut pages: PageRange, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + frame_allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + Self: Sized, + A: FrameAllocator + FrameAllocator + ?Sized, + { + pages + .try_for_each(|page| { + let frame = frame_allocator + .allocate_frame() + .ok_or((MapToError::FrameAllocationFailed, page))?; + + unsafe { + self.map_to_with_table_flags( + page, + frame, + flags, + parent_table_flags, + frame_allocator, + ) + } + .map(|_| ()) + .map_err(|e| (e, page)) + }) + .map(|_| MapperFlushRange::new(pages)) + .map_err(|(e, page)| { + ( + e, + MapperFlushRange::new(PageRange { + start: pages.start, + end: page, + }), + ) + }) + } + /// Removes a mapping from the page table and returns the frame that used to be mapped. /// /// Note that no page tables or pages are deallocated. fn unmap(&mut self, page: Page) -> Result<(PhysFrame, MapperFlush), UnmapError>; + /// Removes a range of mapping from the page table and deallocate the frames that used to be mapped. + /// + /// Note that no page tables or pages are deallocated. + /// + /// ## Errors + /// If an error occurs half-way through a [`MapperFlushRange`] is returned that contains the pages that were successfully unmapped. + fn unmap_range( + &mut self, + pages: PageRange, + deallocator: &mut D, + ) -> Result, (UnmapError, MapperFlushRange)> + where + D: FrameDeallocator, + { + pages + .clone() + .try_for_each(|page| { + let (frame, _) = self.unmap(page).map_err(|e| { + ( + e, + MapperFlushRange::new(PageRange { + start: pages.start, + end: page, + }), + ) + })?; + unsafe { + // SAFETY: the page has been unmapped so the frame is unused + deallocator.deallocate_frame(frame); + } + Ok(()) + }) + .map(|_| MapperFlushRange::new(pages)) + } + /// Updates the flags of an existing mapping. /// /// ## Safety @@ -299,6 +486,41 @@ pub trait Mapper { flags: PageTableFlags, ) -> Result, FlagUpdateError>; + /// Updates the flags of a range of existing mappings. + /// + /// ## Safety + /// + /// This method is unsafe because changing the flags of a mapping + /// might result in undefined behavior. For example, setting the + /// `GLOBAL` and `WRITABLE` flags for a page might result in the corruption + /// of values stored in that page from processes running in other address + /// spaces. + /// + /// ## Errors + /// If an error occurs half-way through a [`MapperFlushRange`] is returned that contains the pages that were successfully updated. + unsafe fn update_flags_range( + &mut self, + pages: PageRange, + flags: PageTableFlags, + ) -> Result, (FlagUpdateError, MapperFlushRange)> { + pages + .clone() + .try_for_each(|page| { + unsafe { self.update_flags(page, flags) } + .map(|_| ()) + .map_err(|e| { + ( + e, + MapperFlushRange::new(PageRange { + start: pages.start, + end: page, + }), + ) + }) + }) + .map(|_| MapperFlushRange::new(pages)) + } + /// Set the flags of an existing page level 4 table entry /// /// ## Safety @@ -372,6 +594,31 @@ pub trait Mapper { let page = Page::containing_address(VirtAddr::new(frame.start_address().as_u64())); unsafe { self.map_to(page, frame, flags, frame_allocator) } } + + /// Maps the given range of frames to the range of virtual pages with the same address. + /// + /// ## Safety + /// + /// This is a convencience function that invokes [`Mapper::map_to_range`] internally, so + /// all safety requirements of it also apply for this function. + #[inline] + unsafe fn identity_map_range( + &mut self, + frames: PhysFrameRange, + flags: PageTableFlags, + frame_allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + Self: Sized, + A: FrameAllocator + ?Sized, + S: PageSize, + Self: Mapper, + { + let start = Page::containing_address(VirtAddr::new(frames.start.start_address().as_u64())); + let end = Page::containing_address(VirtAddr::new(frames.end.start_address().as_u64())); + let pages = PageRange { start, end }; + unsafe { self.map_to_range(pages, frames, flags, frame_allocator) } + } } /// This type represents a page whose mapping has changed in the page table. @@ -405,6 +652,41 @@ impl MapperFlush { pub fn ignore(self) {} } +/// This type represents a range of pages whose mappings have changed in the page table. +/// +/// The old mappings might be still cached in the translation lookaside buffer (TLB), so they need +/// to be flushed from the TLB before they're accessed. This type is returned from a function that +/// changed the mappings of a range of pages to ensure that the TLB flush is not forgotten. +#[derive(Debug)] +#[must_use = "Page Table changes must be flushed or ignored."] +pub struct MapperFlushRange(PageRange); + +impl MapperFlushRange { + /// Create a new flush promise + #[inline] + fn new(pages: PageRange) -> Self { + MapperFlushRange(pages) + } + + /// Flush the page from the TLB to ensure that the newest mapping is used. + #[cfg(feature = "instructions")] + #[inline] + pub fn flush(self) { + for page in self.0 { + crate::instructions::tlb::flush(page.start_address()) + } + } + + /// Don't flush the TLB and silence the “must be used” warning. + #[inline] + pub fn ignore(self) {} + + /// Get the range of changed pages. + pub fn pages(&self) -> PageRange { + self.0 + } +} + /// This type represents a change of a page table requiring a complete TLB flush /// /// The old mapping might be still cached in the translation lookaside buffer (TLB), so it needs From 73a051ef6edc8a3184929232e352feeb71d6f4a1 Mon Sep 17 00:00:00 2001 From: Tom Dohrmann Date: Sat, 5 Jun 2021 17:31:42 +0200 Subject: [PATCH 02/18] specialize *_range methods for MappedPageTable --- .../paging/mapper/mapped_page_table.rs | 696 ++++++++++++++++++ src/structures/paging/mapper/mod.rs | 9 + .../paging/mapper/offset_page_table.rs | 159 ++++ 3 files changed, 864 insertions(+) diff --git a/src/structures/paging/mapper/mapped_page_table.rs b/src/structures/paging/mapper/mapped_page_table.rs index 6a3aa9f4b..1e2919691 100644 --- a/src/structures/paging/mapper/mapped_page_table.rs +++ b/src/structures/paging/mapper/mapped_page_table.rs @@ -4,6 +4,7 @@ use crate::structures::paging::{ mapper::*, page::{AddressNotAligned, Page, PageRangeInclusive, Size1GiB, Size2MiB, Size4KiB}, page_table::{FrameError, PageTable, PageTableEntry, PageTableFlags, PageTableLevel}, + PageTableIndex, }; /// A Mapper implementation that relies on a PhysAddr to VirtAddr conversion function. @@ -140,6 +141,418 @@ impl<'a, P: PageTableFrameMapping> MappedPageTable<'a, P> { Ok(MapperFlush::new(page)) } + + #[inline] + fn next_table_fn_create_next_table<'b, A>( + (flags, allocator): &mut (PageTableFlags, &mut A), + entry: &'b mut PageTableEntry, + walker: &PageTableWalker

, + ) -> Result<&'b mut PageTable, PageTableCreateError> + where + A: FrameAllocator + ?Sized, + { + walker + .create_next_table(entry, *flags, *allocator) + .map_err(Into::into) + } + + #[inline] + fn next_table_fn_next_table_mut<'b, T>( + _: &mut T, + entry: &'b mut PageTableEntry, + walker: &PageTableWalker

, + ) -> Result<&'b mut PageTable, PageTableWalkError> { + walker.next_table_mut(entry) + } + + fn modify_range_1gib( + &mut self, + pages: PageRange, + modify: ModifyFn, + mut info: ModifyInfo, + next_table: NextTableFn, + ) -> Result, (Err, MapperFlushRange)> + where + ModifyFn: Fn(&mut PageTableEntry, Page, &mut ModifyInfo) -> Result<(), Err>, + NextTableFn: for<'b> Fn( + &mut ModifyInfo, + &'b mut PageTableEntry, + &PageTableWalker

, + ) -> Result<&'b mut PageTable, NextTableFnErr>, + NextTableFnErr: Into, + { + if pages.is_empty() { + return Ok(MapperFlushRange::empty()); + } + + let p4 = &mut self.level_4_table; + let page_table_walker = &mut self.page_table_walker; + + (pages.start.p4_index().into()..=pages.end.p4_index().into()) + .map(PageTableIndex::new) + .try_for_each(|p4_index| { + let p4_start = Page::from_page_table_indices_1gib(p4_index, PageTableIndex::new(0)); + let p4_start = p4_start.max(pages.start); + let p4_end = Page::from_page_table_indices_1gib(p4_index, PageTableIndex::new(511)); + let p4_end = p4_end.min(pages.end); + + if p4_start == p4_end { + return Ok(()); + } + + let p3 = next_table(&mut info, &mut p4[p4_index], page_table_walker) + .map_err(|e| (e.into(), p4_start))?; + + let start_p3_index = p4_start.p3_index().into(); + let mut end_p3_index = p4_end.p3_index().into(); + + if p4_end != pages.end { + end_p3_index += 1; + } + + (start_p3_index..end_p3_index) + .map(PageTableIndex::new) + .map(move |p3_index| Page::from_page_table_indices_1gib(p4_index, p3_index)) + .try_for_each(|page| { + let entry = &mut p3[page.p3_index()]; + modify(entry, page, &mut info).map_err(|e| (e, page)) + }) + }) + .map(|_| MapperFlushRange::new(pages)) + .map_err(|(e, page)| { + ( + e, + MapperFlushRange::new(PageRange { + start: pages.start, + end: page, + }), + ) + }) + } + + #[inline] + fn map_to_range_1gib( + &mut self, + pages: PageRange, + frames: F, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + F: Fn(Page, &mut A) -> Option>, + A: FrameAllocator + ?Sized, + { + self.modify_range_1gib( + pages, + |entry, page, (_, allocator)| { + let frame = frames(page, allocator).ok_or(MapToError::FrameAllocationFailed)?; + if !entry.is_unused() { + return Err(MapToError::PageAlreadyMapped(frame)); + } + entry.set_addr(frame.start_address(), flags | PageTableFlags::HUGE_PAGE); + Ok(()) + }, + (parent_table_flags, allocator), + Self::next_table_fn_create_next_table, + ) + } + + fn modify_range_2mib( + &mut self, + pages: PageRange, + modify: ModifyFn, + mut info: ModifyInfo, + next_table: NextTableFn, + ) -> Result, (Err, MapperFlushRange)> + where + ModifyFn: Fn(&mut PageTableEntry, Page, &mut ModifyInfo) -> Result<(), Err>, + NextTableFn: for<'b> Fn( + &mut ModifyInfo, + &'b mut PageTableEntry, + &PageTableWalker

, + ) -> Result<&'b mut PageTable, NextTableFnErr>, + NextTableFnErr: Into, + { + if pages.is_empty() { + return Ok(MapperFlushRange::empty()); + } + + let p4 = &mut self.level_4_table; + let page_table_walker = &mut self.page_table_walker; + + (pages.start.p4_index().into()..=pages.end.p4_index().into()) + .map(PageTableIndex::new) + .try_for_each(|p4_index| { + let p4_start = Page::from_page_table_indices_2mib( + p4_index, + PageTableIndex::new(0), + PageTableIndex::new(0), + ); + let p4_start = p4_start.max(pages.start); + let p4_end = Page::from_page_table_indices_2mib( + p4_index, + PageTableIndex::new(511), + PageTableIndex::new(511), + ); + let p4_end = p4_end.min(pages.end); + + if p4_start == p4_end { + return Ok(()); + } + + let p3 = next_table(&mut info, &mut p4[p4_index], page_table_walker) + .map_err(|e| (e.into(), p4_start))?; + + let start_p3_index = p4_start.p3_index(); + let end_p3_index = p4_end.p3_index(); + + (start_p3_index.into()..=end_p3_index.into()) + .map(PageTableIndex::new) + .try_for_each(|p3_index| { + let p3_start = Page::from_page_table_indices_2mib( + p4_index, + p3_index, + PageTableIndex::new(0), + ); + let p3_start = p3_start.max(p4_start); + let p3_end = Page::from_page_table_indices_2mib( + p4_index, + p3_index, + PageTableIndex::new(511), + ); + let p3_end = p3_end.min(p4_end); + + if p3_start == p3_end { + return Ok(()); + } + + let p2 = next_table(&mut info, &mut p3[p3_index], page_table_walker) + .map_err(|e| (e.into(), p3_start))?; + + let start_p2_index = p3_start.p2_index().into(); + let mut end_p2_index = p3_end.p2_index().into(); + + if p3_end != pages.end { + end_p2_index += 1; + } + + (start_p2_index..end_p2_index) + .map(PageTableIndex::new) + .map(move |p2_index| { + Page::from_page_table_indices_2mib(p4_index, p3_index, p2_index) + }) + .try_for_each(|page| { + let entry = &mut p2[page.p2_index()]; + modify(entry, page, &mut info).map_err(|e| (e, page)) + }) + }) + }) + .map(|_| MapperFlushRange::new(pages)) + .map_err(|(e, page)| { + ( + e, + MapperFlushRange::new(PageRange { + start: pages.start, + end: page, + }), + ) + }) + } + + #[inline] + fn map_range_2mib( + &mut self, + pages: PageRange, + frames: F, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + F: Fn(Page, &mut A) -> Option>, + A: FrameAllocator + ?Sized, + { + self.modify_range_2mib( + pages, + |entry, page, (_, allocator)| { + let frame = frames(page, allocator).ok_or(MapToError::FrameAllocationFailed)?; + if !entry.is_unused() { + return Err(MapToError::PageAlreadyMapped(frame)); + } + entry.set_addr(frame.start_address(), flags | PageTableFlags::HUGE_PAGE); + Ok(()) + }, + (parent_table_flags, allocator), + Self::next_table_fn_create_next_table, + ) + } + + fn modify_range_4kib( + &mut self, + pages: PageRange, + modify: ModifyFn, + mut info: ModifyInfo, + next_table: NextTableFn, + ) -> Result, (Err, MapperFlushRange)> + where + ModifyFn: Fn(&mut PageTableEntry, Page, &mut ModifyInfo) -> Result<(), Err>, + NextTableFn: for<'b> Fn( + &mut ModifyInfo, + &'b mut PageTableEntry, + &PageTableWalker

, + ) -> Result<&'b mut PageTable, NextTableFnErr>, + NextTableFnErr: Into, + { + if pages.is_empty() { + return Ok(MapperFlushRange::empty()); + } + + let p4 = &mut self.level_4_table; + let page_table_walker = &mut self.page_table_walker; + + (pages.start.p4_index().into()..=pages.end.p4_index().into()) + .map(PageTableIndex::new) + .try_for_each(|p4_index| { + let p4_start = Page::from_page_table_indices( + p4_index, + PageTableIndex::new(0), + PageTableIndex::new(0), + PageTableIndex::new(0), + ); + let p4_start = p4_start.max(pages.start); + let p4_end = Page::from_page_table_indices( + p4_index, + PageTableIndex::new(511), + PageTableIndex::new(511), + PageTableIndex::new(511), + ); + let p4_end = p4_end.min(pages.end); + + if p4_start == p4_end { + return Ok(()); + } + + let p3 = next_table(&mut info, &mut p4[p4_index], page_table_walker) + .map_err(|e| (e.into(), p4_start))?; + + let start_p3_index = p4_start.p3_index(); + let end_p3_index = p4_end.p3_index(); + + (start_p3_index.into()..=end_p3_index.into()) + .map(PageTableIndex::new) + .try_for_each(|p3_index| { + let p3_start = Page::from_page_table_indices( + p4_index, + p3_index, + PageTableIndex::new(0), + PageTableIndex::new(0), + ); + let p3_start = p3_start.max(p4_start); + let p3_end = Page::from_page_table_indices( + p4_index, + p3_index, + PageTableIndex::new(511), + PageTableIndex::new(511), + ); + let p3_end = p3_end.min(p4_end); + + if p3_start == p3_end { + return Ok(()); + } + + let p2 = next_table(&mut info, &mut p3[p3_index], page_table_walker) + .map_err(|e| (e.into(), p3_start))?; + + let start_p2_index = p3_start.p2_index(); + let end_p2_index = p3_end.p2_index(); + + (start_p2_index.into()..=end_p2_index.into()) + .map(PageTableIndex::new) + .try_for_each(|p2_index| { + let p2_start = Page::from_page_table_indices( + p4_index, + p3_index, + p2_index, + PageTableIndex::new(0), + ); + let p2_start = p2_start.max(p4_start); + let p2_end = Page::from_page_table_indices( + p4_index, + p3_index, + p2_index, + PageTableIndex::new(511), + ); + let p2_end = p2_end.min(p4_end); + + if p2_start == p2_end { + return Ok(()); + } + + let p1 = + next_table(&mut info, &mut p2[p2_index], page_table_walker) + .map_err(|e| (e.into(), p2_start))?; + + let start_p1_index = p2_start.p1_index().into(); + let mut end_p1_index = p2_end.p1_index().into(); + + if p2_end != pages.end { + end_p1_index += 1; + } + + (start_p1_index..end_p1_index) + .map(PageTableIndex::new) + .map(move |p1_index| { + Page::from_page_table_indices( + p4_index, p3_index, p2_index, p1_index, + ) + }) + .try_for_each(|page| { + let entry = &mut p1[page.p1_index()]; + modify(entry, page, &mut info).map_err(|e| (e, page)) + }) + }) + }) + }) + .map(|_| MapperFlushRange::new(pages)) + .map_err(|(e, page)| { + ( + e, + MapperFlushRange::new(PageRange { + start: pages.start, + end: page, + }), + ) + }) + } + + #[inline] + fn map_to_range_4kib( + &mut self, + pages: PageRange, + frames: F, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + F: Fn(Page, &mut A) -> Option>, + A: FrameAllocator + ?Sized, + { + self.modify_range_4kib( + pages, + |entry, page, (_, allocator)| { + let frame = frames(page, allocator).ok_or(MapToError::FrameAllocationFailed)?; + if !entry.is_unused() { + return Err(MapToError::PageAlreadyMapped(frame)); + } + entry.set_addr(frame.start_address(), flags | PageTableFlags::HUGE_PAGE); + Ok(()) + }, + (parent_table_flags, allocator), + Self::next_table_fn_create_next_table, + ) + } } impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { @@ -158,6 +571,53 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { self.map_to_1gib(page, frame, flags, parent_table_flags, allocator) } + #[inline] + unsafe fn map_to_range_with_table_flags( + &mut self, + pages: PageRange, + frames: PhysFrameRange, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + Self: Sized, + A: FrameAllocator + ?Sized, + { + assert_eq!(pages.count(), frames.count()); + self.map_to_range_1gib( + pages, + |page, _| { + let offset = pages.start - page; + Some(frames.start + (offset / Size1GiB::SIZE)) + }, + flags, + parent_table_flags, + allocator, + ) + } + + #[inline] + unsafe fn map_range_with_table_flags( + &mut self, + pages: PageRange, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + Self: Sized, + A: FrameAllocator + FrameAllocator + ?Sized, + { + self.map_to_range_1gib( + pages, + |_, allocator| allocator.allocate_frame(), + flags, + parent_table_flags, + allocator, + ) + } + fn unmap( &mut self, page: Page, @@ -184,6 +644,32 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { Ok((frame, MapperFlush::new(page))) } + #[inline] + fn unmap_range( + &mut self, + pages: PageRange, + deallocator: &mut D, + ) -> Result, (UnmapError, MapperFlushRange)> + where + D: FrameDeallocator, + { + self.modify_range_1gib( + pages, + |entry, _, deallocator| { + let frame = PhysFrame::from_start_address(entry.addr()) + .map_err(|AddressNotAligned| UnmapError::InvalidFrameAddress(entry.addr()))?; + unsafe { + deallocator.deallocate_frame(frame); + } + + entry.set_unused(); + Ok(()) + }, + deallocator, + Self::next_table_fn_next_table_mut, + ) + } + unsafe fn update_flags( &mut self, page: Page, @@ -202,6 +688,27 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { Ok(MapperFlush::new(page)) } + #[inline] + unsafe fn update_flags_range( + &mut self, + pages: PageRange, + flags: PageTableFlags, + ) -> Result, (FlagUpdateError, MapperFlushRange)> { + self.modify_range_1gib( + pages, + |entry, _, _| { + if entry.is_unused() { + return Err(FlagUpdateError::PageNotMapped); + } + + entry.set_flags(flags); + Ok(()) + }, + (), + Self::next_table_fn_next_table_mut, + ) + } + unsafe fn set_flags_p4_entry( &mut self, page: Page, @@ -266,6 +773,53 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { self.map_to_2mib(page, frame, flags, parent_table_flags, allocator) } + #[inline] + unsafe fn map_to_range_with_table_flags( + &mut self, + pages: PageRange, + frames: PhysFrameRange, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + Self: Sized, + A: FrameAllocator + ?Sized, + { + assert_eq!(pages.count(), frames.count()); + self.map_range_2mib( + pages, + |page, _| { + let offset = pages.start - page; + Some(frames.start + (offset / Size2MiB::SIZE)) + }, + flags, + parent_table_flags, + allocator, + ) + } + + #[inline] + unsafe fn map_range_with_table_flags( + &mut self, + pages: PageRange, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + Self: Sized, + A: FrameAllocator + FrameAllocator + ?Sized, + { + self.map_range_2mib( + pages, + |_, allocator| allocator.allocate_frame(), + flags, + parent_table_flags, + allocator, + ) + } + fn unmap( &mut self, page: Page, @@ -295,6 +849,32 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { Ok((frame, MapperFlush::new(page))) } + #[inline] + fn unmap_range( + &mut self, + pages: PageRange, + deallocator: &mut D, + ) -> Result, (UnmapError, MapperFlushRange)> + where + D: FrameDeallocator, + { + self.modify_range_2mib( + pages, + |entry, _, deallocator| { + let frame = PhysFrame::from_start_address(entry.addr()) + .map_err(|AddressNotAligned| UnmapError::InvalidFrameAddress(entry.addr()))?; + unsafe { + deallocator.deallocate_frame(frame); + } + + entry.set_unused(); + Ok(()) + }, + deallocator, + Self::next_table_fn_next_table_mut, + ) + } + unsafe fn update_flags( &mut self, page: Page, @@ -317,6 +897,27 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { Ok(MapperFlush::new(page)) } + #[inline] + unsafe fn update_flags_range( + &mut self, + pages: PageRange, + flags: PageTableFlags, + ) -> Result, (FlagUpdateError, MapperFlushRange)> { + self.modify_range_2mib( + pages, + |entry, _, _| { + if entry.is_unused() { + return Err(FlagUpdateError::PageNotMapped); + } + + entry.set_flags(flags); + Ok(()) + }, + (), + Self::next_table_fn_next_table_mut, + ) + } + unsafe fn set_flags_p4_entry( &mut self, page: Page, @@ -394,6 +995,52 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { self.map_to_4kib(page, frame, flags, parent_table_flags, allocator) } + #[inline] + unsafe fn map_to_range_with_table_flags( + &mut self, + pages: PageRange, + frames: PhysFrameRange, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + Self: Sized, + A: FrameAllocator + ?Sized, + { + assert_eq!(pages.count(), frames.count()); + self.map_to_range_4kib( + pages, + |page, _| { + let offset = pages.start - page; + Some(frames.start + (offset / Size4KiB::SIZE)) + }, + flags, + parent_table_flags, + allocator, + ) + } + + #[inline] + unsafe fn map_range_with_table_flags( + &mut self, + pages: PageRange, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + A: FrameAllocator + ?Sized, + { + self.map_to_range_4kib( + pages, + |_, allocator| allocator.allocate_frame(), + flags, + parent_table_flags, + allocator, + ) + } + fn unmap( &mut self, page: Page, @@ -420,6 +1067,34 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { Ok((frame, MapperFlush::new(page))) } + #[inline] + fn unmap_range( + &mut self, + pages: PageRange, + deallocator: &mut D, + ) -> Result, (UnmapError, MapperFlushRange)> + where + D: FrameDeallocator, + { + self.modify_range_4kib( + pages, + |entry, _, deallocator| { + let frame = entry.frame().map_err(|err| match err { + FrameError::FrameNotPresent => UnmapError::PageNotMapped, + FrameError::HugeFrame => UnmapError::ParentEntryHugePage, + })?; + unsafe { + deallocator.deallocate_frame(frame); + } + + entry.set_unused(); + Ok(()) + }, + deallocator, + Self::next_table_fn_next_table_mut, + ) + } + unsafe fn update_flags( &mut self, page: Page, @@ -445,6 +1120,27 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { Ok(MapperFlush::new(page)) } + #[inline] + unsafe fn update_flags_range( + &mut self, + pages: PageRange, + flags: PageTableFlags, + ) -> Result, (FlagUpdateError, MapperFlushRange)> { + self.modify_range_4kib( + pages, + |entry, _, _| { + if entry.is_unused() { + return Err(FlagUpdateError::PageNotMapped); + } + + entry.set_flags(flags); + Ok(()) + }, + (), + Self::next_table_fn_next_table_mut, + ) + } + unsafe fn set_flags_p4_entry( &mut self, page: Page, diff --git a/src/structures/paging/mapper/mod.rs b/src/structures/paging/mapper/mod.rs index 3cd5ff124..dbde9e681 100644 --- a/src/structures/paging/mapper/mod.rs +++ b/src/structures/paging/mapper/mod.rs @@ -668,6 +668,15 @@ impl MapperFlushRange { MapperFlushRange(pages) } + /// Create a new empty flush promise + #[inline] + fn empty() -> Self { + MapperFlushRange::new(PageRange { + start: Page::containing_address(VirtAddr::zero()), + end: Page::containing_address(VirtAddr::zero()), + }) + } + /// Flush the page from the TLB to ensure that the newest mapping is used. #[cfg(feature = "instructions")] #[inline] diff --git a/src/structures/paging/mapper/offset_page_table.rs b/src/structures/paging/mapper/offset_page_table.rs index 580924fe9..fec972d06 100644 --- a/src/structures/paging/mapper/offset_page_table.rs +++ b/src/structures/paging/mapper/offset_page_table.rs @@ -77,6 +77,47 @@ impl<'a> Mapper for OffsetPageTable<'a> { } } + #[inline] + unsafe fn map_to_range_with_table_flags( + &mut self, + pages: PageRange, + frames: PhysFrameRange, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + Self: Sized, + A: FrameAllocator + ?Sized, + { + unsafe { + self.inner.map_to_range_with_table_flags( + pages, + frames, + flags, + parent_table_flags, + allocator, + ) + } + } + + #[inline] + unsafe fn map_range_with_table_flags( + &mut self, + pages: PageRange, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + A: FrameAllocator + FrameAllocator + ?Sized, + { + unsafe { + self.inner + .map_range_with_table_flags(pages, flags, parent_table_flags, allocator) + } + } + #[inline] fn unmap( &mut self, @@ -85,6 +126,18 @@ impl<'a> Mapper for OffsetPageTable<'a> { self.inner.unmap(page) } + #[inline] + fn unmap_range( + &mut self, + pages: PageRange, + deallocator: &mut D, + ) -> Result, (UnmapError, MapperFlushRange)> + where + D: FrameDeallocator, + { + self.inner.unmap_range(pages, deallocator) + } + #[inline] unsafe fn update_flags( &mut self, @@ -146,6 +199,47 @@ impl<'a> Mapper for OffsetPageTable<'a> { } } + #[inline] + unsafe fn map_to_range_with_table_flags( + &mut self, + pages: PageRange, + frames: PhysFrameRange, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + Self: Sized, + A: FrameAllocator + ?Sized, + { + unsafe { + self.inner.map_to_range_with_table_flags( + pages, + frames, + flags, + parent_table_flags, + allocator, + ) + } + } + + #[inline] + unsafe fn map_range_with_table_flags( + &mut self, + pages: PageRange, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + A: FrameAllocator + FrameAllocator + ?Sized, + { + unsafe { + self.inner + .map_range_with_table_flags(pages, flags, parent_table_flags, allocator) + } + } + #[inline] fn unmap( &mut self, @@ -154,6 +248,18 @@ impl<'a> Mapper for OffsetPageTable<'a> { self.inner.unmap(page) } + #[inline] + fn unmap_range( + &mut self, + pages: PageRange, + deallocator: &mut D, + ) -> Result, (UnmapError, MapperFlushRange)> + where + D: FrameDeallocator, + { + self.inner.unmap_range(pages, deallocator) + } + #[inline] unsafe fn update_flags( &mut self, @@ -215,6 +321,47 @@ impl<'a> Mapper for OffsetPageTable<'a> { } } + #[inline] + unsafe fn map_to_range_with_table_flags( + &mut self, + pages: PageRange, + frames: PhysFrameRange, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + Self: Sized, + A: FrameAllocator + ?Sized, + { + unsafe { + self.inner.map_to_range_with_table_flags( + pages, + frames, + flags, + parent_table_flags, + allocator, + ) + } + } + + #[inline] + unsafe fn map_range_with_table_flags( + &mut self, + pages: PageRange, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + A: FrameAllocator + ?Sized, + { + unsafe { + self.inner + .map_range_with_table_flags(pages, flags, parent_table_flags, allocator) + } + } + #[inline] fn unmap( &mut self, @@ -223,6 +370,18 @@ impl<'a> Mapper for OffsetPageTable<'a> { self.inner.unmap(page) } + #[inline] + fn unmap_range( + &mut self, + pages: PageRange, + deallocator: &mut D, + ) -> Result, (UnmapError, MapperFlushRange)> + where + D: FrameDeallocator, + { + self.inner.unmap_range(pages, deallocator) + } + #[inline] unsafe fn update_flags( &mut self, From 4fb47e412d7bb618991f45cd7ff1085a0337bbfd Mon Sep 17 00:00:00 2001 From: Tom Dohrmann Date: Sat, 5 Jun 2021 17:32:13 +0200 Subject: [PATCH 03/18] specialize count for PageRange and PhysFrameRange --- src/structures/paging/frame.rs | 9 +++++++++ src/structures/paging/page.rs | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/src/structures/paging/frame.rs b/src/structures/paging/frame.rs index 64935caee..57f58d38d 100644 --- a/src/structures/paging/frame.rs +++ b/src/structures/paging/frame.rs @@ -161,6 +161,15 @@ impl Iterator for PhysFrameRange { None } } + + #[inline] + fn count(self) -> usize { + if !self.is_empty() { + ((self.end.start_address() - self.start.start_address()) / S::SIZE) as usize + } else { + 0 + } + } } impl fmt::Debug for PhysFrameRange { diff --git a/src/structures/paging/page.rs b/src/structures/paging/page.rs index 3e2878bfe..d26e70fcd 100644 --- a/src/structures/paging/page.rs +++ b/src/structures/paging/page.rs @@ -324,6 +324,15 @@ impl Iterator for PageRange { None } } + + #[inline] + fn count(self) -> usize { + if !self.is_empty() { + ((self.end.start_address() - self.start.start_address()) / S::SIZE) as usize + } else { + 0 + } + } } impl PageRange { From 6ffad4a3fa7473a4418a2d14b9c8185f987e063c Mon Sep 17 00:00:00 2001 From: Tom Dohrmann Date: Mon, 7 Jun 2021 20:21:55 +0200 Subject: [PATCH 04/18] specialize *_range methods for RecursivePageTable --- src/structures/paging/mapper/mod.rs | 14 + .../paging/mapper/recursive_page_table.rs | 727 +++++++++++++++++- 2 files changed, 740 insertions(+), 1 deletion(-) diff --git a/src/structures/paging/mapper/mod.rs b/src/structures/paging/mapper/mod.rs index dbde9e681..c208933fe 100644 --- a/src/structures/paging/mapper/mod.rs +++ b/src/structures/paging/mapper/mod.rs @@ -6,6 +6,8 @@ pub use self::offset_page_table::OffsetPageTable; #[cfg(feature = "instructions")] pub use self::recursive_page_table::{InvalidPageTable, RecursivePageTable}; +use core::convert::Infallible; + use crate::structures::paging::{ frame::PhysFrameRange, frame_alloc::{FrameAllocator, FrameDeallocator}, @@ -752,6 +754,12 @@ pub enum UnmapError { InvalidFrameAddress(PhysAddr), } +impl From for UnmapError { + fn from(i: Infallible) -> Self { + match i {} + } +} + /// An error indicating that an `update_flags` call failed. #[derive(Debug)] pub enum FlagUpdateError { @@ -762,6 +770,12 @@ pub enum FlagUpdateError { ParentEntryHugePage, } +impl From for FlagUpdateError { + fn from(i: Infallible) -> Self { + match i {} + } +} + /// An error indicating that an `translate` call failed. #[derive(Debug)] pub enum TranslateError { diff --git a/src/structures/paging/mapper/recursive_page_table.rs b/src/structures/paging/mapper/recursive_page_table.rs index 7dc1f6398..de8943cbb 100644 --- a/src/structures/paging/mapper/recursive_page_table.rs +++ b/src/structures/paging/mapper/recursive_page_table.rs @@ -1,6 +1,6 @@ //! Access the page tables through a recursively mapped level 4 table. -use core::fmt; +use core::{convert::Infallible, fmt}; use super::*; use crate::registers::control::Cr3; @@ -286,6 +286,442 @@ impl<'a> RecursivePageTable<'a> { Ok(MapperFlush::new(page)) } + + unsafe fn next_table_fn_create_next_table<'b, A, S>( + (insert_flags, allocator): &mut (PageTableFlags, &mut A), + entry: &'b mut PageTableEntry, + page: Page, + ) -> Result<&'b mut PageTable, MapToError> + where + A: FrameAllocator + ?Sized, + S: PageSize, + { + unsafe { Self::create_next_table(entry, page, *insert_flags, *allocator) } + } + + unsafe fn next_table_fn_next_table_mut<'b, I>( + _: &mut I, + _: &'b mut PageTableEntry, + page: Page, + ) -> Result<&'b mut PageTable, Infallible> { + Ok(unsafe { &mut *page.start_address().as_mut_ptr() }) + } + + fn modify_range_1gib( + &mut self, + pages: PageRange, + modify: ModifyFn, + mut info: ModifyInfo, + next_table: for<'b> unsafe fn( + &mut ModifyInfo, + &'b mut PageTableEntry, + Page, + ) -> Result<&'b mut PageTable, NextTableFnErr>, + ) -> Result, (Err, MapperFlushRange)> + where + ModifyFn: Fn(&mut PageTableEntry, Page, &mut ModifyInfo) -> Result<(), Err>, + NextTableFnErr: Into, + { + if pages.is_empty() { + return Ok(MapperFlushRange::empty()); + } + + let recursive_index = self.recursive_index; + let p4 = self.level_4_table(); + + (pages.start.p4_index().into()..=pages.end.p4_index().into()) + .map(PageTableIndex::new) + .try_for_each(|p4_index| { + let p4_start = Page::from_page_table_indices_1gib(p4_index, PageTableIndex::new(0)); + let p4_start = p4_start.max(pages.start); + let p4_end = Page::from_page_table_indices_1gib(p4_index, PageTableIndex::new(511)); + let p4_end = p4_end.min(pages.end); + + if p4_start == p4_end { + return Ok(()); + } + + let p3_page = p3_page(p4_start, recursive_index); + let p3 = unsafe { next_table(&mut info, &mut p4[p4_index], p3_page) } + .map_err(|e| (e.into(), p4_start))?; + + let start_p3_index = p4_start.p3_index().into(); + let mut end_p3_index = p4_end.p3_index().into(); + + if p4_end != pages.end { + end_p3_index += 1; + } + + (start_p3_index..end_p3_index) + .map(PageTableIndex::new) + .map(move |p3_index| Page::from_page_table_indices_1gib(p4_index, p3_index)) + .try_for_each(|page| { + let entry = &mut p3[page.p3_index()]; + modify(entry, page, &mut info).map_err(|e| (e, page)) + }) + }) + .map(|_| MapperFlushRange::new(pages)) + .map_err(|(e, page)| { + ( + e, + MapperFlushRange::new(PageRange { + start: pages.start, + end: page, + }), + ) + }) + } + + #[inline] + fn map_to_range_1gib( + &mut self, + pages: PageRange, + frames: F, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + F: Fn(Page, &mut A) -> Option>, + A: FrameAllocator + ?Sized, + { + self.modify_range_1gib( + pages, + |entry, page, (_, allocator)| { + let frame = frames(page, allocator).ok_or(MapToError::FrameAllocationFailed)?; + if !entry.is_unused() { + return Err(MapToError::PageAlreadyMapped(frame)); + } + entry.set_addr(frame.start_address(), flags | PageTableFlags::HUGE_PAGE); + Ok(()) + }, + (parent_table_flags, allocator), + Self::next_table_fn_create_next_table, + ) + } + + fn modify_range_2mib( + &mut self, + pages: PageRange, + modify: ModifyFn, + mut info: ModifyInfo, + next_table: for<'b> unsafe fn( + &mut ModifyInfo, + &'b mut PageTableEntry, + Page, + ) -> Result<&'b mut PageTable, NextTableFnErr>, + ) -> Result, (Err, MapperFlushRange)> + where + ModifyFn: Fn(&mut PageTableEntry, Page, &mut ModifyInfo) -> Result<(), Err>, + NextTableFnErr: Into, + { + if pages.is_empty() { + return Ok(MapperFlushRange::empty()); + } + + let recursive_index = self.recursive_index; + let p4 = self.level_4_table(); + + (pages.start.p4_index().into()..=pages.end.p4_index().into()) + .map(PageTableIndex::new) + .try_for_each(|p4_index| { + let p4_start = Page::from_page_table_indices_2mib( + p4_index, + PageTableIndex::new(0), + PageTableIndex::new(0), + ); + let p4_start = p4_start.max(pages.start); + let p4_end = Page::from_page_table_indices_2mib( + p4_index, + PageTableIndex::new(511), + PageTableIndex::new(511), + ); + let p4_end = p4_end.min(pages.end); + + if p4_start == p4_end { + return Ok(()); + } + + let p3 = unsafe { + next_table( + &mut info, + &mut p4[p4_index], + p3_page(p4_start, recursive_index), + ) + } + .map_err(|e| (e.into(), p4_start))?; + + let start_p3_index = p4_start.p3_index(); + let end_p3_index = p4_end.p3_index(); + + (start_p3_index.into()..=end_p3_index.into()) + .map(PageTableIndex::new) + .try_for_each(|p3_index| { + let p3_start = Page::from_page_table_indices_2mib( + p4_index, + p3_index, + PageTableIndex::new(0), + ); + let p3_start = p3_start.max(p4_start); + let p3_end = Page::from_page_table_indices_2mib( + p4_index, + p3_index, + PageTableIndex::new(511), + ); + let p3_end = p3_end.min(p4_end); + + if p3_start == p3_end { + return Ok(()); + } + + let p2 = unsafe { + next_table( + &mut info, + &mut p3[p3_index], + p2_page(p3_start, recursive_index), + ) + } + .map_err(|e| (e.into(), p3_start))?; + + let start_p2_index = p3_start.p2_index().into(); + let mut end_p2_index = p3_end.p2_index().into(); + + if p3_end != pages.end { + end_p2_index += 1; + } + + (start_p2_index..end_p2_index) + .map(PageTableIndex::new) + .map(move |p2_index| { + Page::from_page_table_indices_2mib(p4_index, p3_index, p2_index) + }) + .try_for_each(|page| { + let entry = &mut p2[page.p2_index()]; + modify(entry, page, &mut info).map_err(|e| (e, page)) + }) + }) + }) + .map(|_| MapperFlushRange::new(pages)) + .map_err(|(e, page)| { + ( + e, + MapperFlushRange::new(PageRange { + start: pages.start, + end: page, + }), + ) + }) + } + + #[inline] + fn map_range_2mib( + &mut self, + pages: PageRange, + frames: F, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + F: Fn(Page, &mut A) -> Option>, + A: FrameAllocator + ?Sized, + { + self.modify_range_2mib( + pages, + |entry, page, (_, allocator)| { + let frame = frames(page, allocator).ok_or(MapToError::FrameAllocationFailed)?; + if !entry.is_unused() { + return Err(MapToError::PageAlreadyMapped(frame)); + } + entry.set_addr(frame.start_address(), flags | PageTableFlags::HUGE_PAGE); + Ok(()) + }, + (parent_table_flags, allocator), + Self::next_table_fn_create_next_table, + ) + } + + fn modify_range_4kib( + &mut self, + pages: PageRange, + modify: ModifyFn, + mut info: ModifyInfo, + next_table: for<'b> unsafe fn( + &mut ModifyInfo, + &'b mut PageTableEntry, + Page, + ) -> Result<&'b mut PageTable, NextTableFnErr>, + ) -> Result, (Err, MapperFlushRange)> + where + ModifyFn: Fn(&mut PageTableEntry, Page, &mut ModifyInfo) -> Result<(), Err>, + NextTableFnErr: Into, + { + if pages.is_empty() { + return Ok(MapperFlushRange::empty()); + } + + let recursive_index = self.recursive_index; + let p4 = self.level_4_table(); + + (pages.start.p4_index().into()..=pages.end.p4_index().into()) + .map(PageTableIndex::new) + .try_for_each(|p4_index| { + let p4_start = Page::from_page_table_indices( + p4_index, + PageTableIndex::new(0), + PageTableIndex::new(0), + PageTableIndex::new(0), + ); + let p4_start = p4_start.max(pages.start); + let p4_end = Page::from_page_table_indices( + p4_index, + PageTableIndex::new(511), + PageTableIndex::new(511), + PageTableIndex::new(511), + ); + let p4_end = p4_end.min(pages.end); + + if p4_start == p4_end { + return Ok(()); + } + + let p3 = unsafe { + next_table( + &mut info, + &mut p4[p4_index], + p3_page(p4_start, recursive_index), + ) + } + .map_err(|e| (e.into(), p4_start))?; + + let start_p3_index = p4_start.p3_index(); + let end_p3_index = p4_end.p3_index(); + + (start_p3_index.into()..=end_p3_index.into()) + .map(PageTableIndex::new) + .try_for_each(|p3_index| { + let p3_start = Page::from_page_table_indices( + p4_index, + p3_index, + PageTableIndex::new(0), + PageTableIndex::new(0), + ); + let p3_start = p3_start.max(p4_start); + let p3_end = Page::from_page_table_indices( + p4_index, + p3_index, + PageTableIndex::new(511), + PageTableIndex::new(511), + ); + let p3_end = p3_end.min(p4_end); + + if p3_start == p3_end { + return Ok(()); + } + + let p2 = unsafe { + next_table( + &mut info, + &mut p3[p3_index], + p2_page(p3_start, recursive_index), + ) + } + .map_err(|e| (e.into(), p3_start))?; + + let start_p2_index = p3_start.p2_index(); + let end_p2_index = p3_end.p2_index(); + + (start_p2_index.into()..=end_p2_index.into()) + .map(PageTableIndex::new) + .try_for_each(|p2_index| { + let p2_start = Page::from_page_table_indices( + p4_index, + p3_index, + p2_index, + PageTableIndex::new(0), + ); + let p2_start = p2_start.max(p3_start); + let p2_end = Page::from_page_table_indices( + p4_index, + p3_index, + p2_index, + PageTableIndex::new(511), + ); + let p2_end = p2_end.min(p4_end); + + if p2_start == p2_end { + return Ok(()); + } + + let p1 = unsafe { + next_table( + &mut info, + &mut p2[p2_index], + p1_page(p2_start, recursive_index), + ) + } + .map_err(|e| (e.into(), p2_start))?; + + let start_p1_index = p2_start.p1_index().into(); + let mut end_p1_index = p2_end.p1_index().into(); + + if p2_end != pages.end { + end_p1_index += 1; + } + + (start_p1_index..end_p1_index) + .map(PageTableIndex::new) + .map(move |p1_index| { + Page::from_page_table_indices( + p4_index, p3_index, p2_index, p1_index, + ) + }) + .try_for_each(|page| { + let entry = &mut p1[page.p1_index()]; + modify(entry, page, &mut info).map_err(|e| (e, page)) + }) + }) + }) + }) + .map(|_| MapperFlushRange::new(pages)) + .map_err(|(e, page)| { + ( + e, + MapperFlushRange::new(PageRange { + start: pages.start, + end: page, + }), + ) + }) + } + + #[inline] + fn map_to_range_4kib( + &mut self, + pages: PageRange, + frames: F, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + F: Fn(Page, &mut A) -> Option>, + A: FrameAllocator + ?Sized, + { + self.modify_range_4kib( + pages, + |entry, page, (_, allocator)| { + let frame = frames(page, allocator).ok_or(MapToError::FrameAllocationFailed)?; + if !entry.is_unused() { + return Err(MapToError::PageAlreadyMapped(frame)); + } + entry.set_addr(frame.start_address(), flags | PageTableFlags::HUGE_PAGE); + Ok(()) + }, + (parent_table_flags, allocator), + Self::next_table_fn_create_next_table, + ) + } } impl<'a> Mapper for RecursivePageTable<'a> { @@ -304,6 +740,53 @@ impl<'a> Mapper for RecursivePageTable<'a> { self.map_to_1gib(page, frame, flags, parent_table_flags, allocator) } + #[inline] + unsafe fn map_to_range_with_table_flags( + &mut self, + pages: PageRange, + frames: PhysFrameRange, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + Self: Sized, + A: FrameAllocator + ?Sized, + { + assert_eq!(pages.count(), frames.count()); + self.map_to_range_1gib( + pages, + |page, _| { + let offset = pages.start - page; + Some(frames.start + (offset / Size1GiB::SIZE)) + }, + flags, + parent_table_flags, + allocator, + ) + } + + #[inline] + unsafe fn map_range_with_table_flags( + &mut self, + pages: PageRange, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + Self: Sized, + A: FrameAllocator + FrameAllocator + ?Sized, + { + self.map_to_range_1gib( + pages, + |_, allocator| allocator.allocate_frame(), + flags, + parent_table_flags, + allocator, + ) + } + fn unmap( &mut self, page: Page, @@ -334,6 +817,34 @@ impl<'a> Mapper for RecursivePageTable<'a> { Ok((frame, MapperFlush::new(page))) } + #[inline] + fn unmap_range( + &mut self, + pages: PageRange, + deallocator: &mut D, + ) -> Result, (UnmapError, MapperFlushRange)> + where + D: FrameDeallocator, + { + self.modify_range_1gib( + pages, + |entry, _, deallocator| { + let frame = PhysFrame::from_start_address(entry.addr()) + .map_err(|AddressNotAligned| UnmapError::InvalidFrameAddress(entry.addr()))?; + unsafe { + deallocator.deallocate_frame(frame); + } + + entry.set_unused(); + Ok(()) + }, + deallocator, + Self::next_table_fn_next_table_mut, + ) + } + + // allow unused_unsafe until https://github.com/rust-lang/rfcs/pull/2585 lands + #[allow(unused_unsafe)] unsafe fn update_flags( &mut self, page: Page, @@ -356,6 +867,27 @@ impl<'a> Mapper for RecursivePageTable<'a> { Ok(MapperFlush::new(page)) } + #[inline] + unsafe fn update_flags_range( + &mut self, + pages: PageRange, + flags: PageTableFlags, + ) -> Result, (FlagUpdateError, MapperFlushRange)> { + self.modify_range_1gib( + pages, + |entry, _, _| { + if entry.is_unused() { + return Err(FlagUpdateError::PageNotMapped); + } + + entry.set_flags(flags); + Ok(()) + }, + (), + Self::next_table_fn_next_table_mut, + ) + } + unsafe fn set_flags_p4_entry( &mut self, page: Page, @@ -424,6 +956,53 @@ impl<'a> Mapper for RecursivePageTable<'a> { self.map_to_2mib(page, frame, flags, parent_table_flags, allocator) } + #[inline] + unsafe fn map_to_range_with_table_flags( + &mut self, + pages: PageRange, + frames: PhysFrameRange, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + Self: Sized, + A: FrameAllocator + ?Sized, + { + assert_eq!(pages.count(), frames.count()); + self.map_range_2mib( + pages, + |page, _| { + let offset = pages.start - page; + Some(frames.start + (offset / Size2MiB::SIZE)) + }, + flags, + parent_table_flags, + allocator, + ) + } + + #[inline] + unsafe fn map_range_with_table_flags( + &mut self, + pages: PageRange, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + Self: Sized, + A: FrameAllocator + FrameAllocator + ?Sized, + { + self.map_range_2mib( + pages, + |_, allocator| allocator.allocate_frame(), + flags, + parent_table_flags, + allocator, + ) + } + fn unmap( &mut self, page: Page, @@ -460,6 +1039,34 @@ impl<'a> Mapper for RecursivePageTable<'a> { Ok((frame, MapperFlush::new(page))) } + #[inline] + fn unmap_range( + &mut self, + pages: PageRange, + deallocator: &mut D, + ) -> Result, (UnmapError, MapperFlushRange)> + where + D: FrameDeallocator, + { + self.modify_range_2mib( + pages, + |entry, _, deallocator| { + let frame = PhysFrame::from_start_address(entry.addr()) + .map_err(|AddressNotAligned| UnmapError::InvalidFrameAddress(entry.addr()))?; + unsafe { + deallocator.deallocate_frame(frame); + } + + entry.set_unused(); + Ok(()) + }, + deallocator, + Self::next_table_fn_next_table_mut, + ) + } + + // allow unused_unsafe until https://github.com/rust-lang/rfcs/pull/2585 lands + #[allow(unused_unsafe)] unsafe fn update_flags( &mut self, page: Page, @@ -489,6 +1096,27 @@ impl<'a> Mapper for RecursivePageTable<'a> { Ok(MapperFlush::new(page)) } + #[inline] + unsafe fn update_flags_range( + &mut self, + pages: PageRange, + flags: PageTableFlags, + ) -> Result, (FlagUpdateError, MapperFlushRange)> { + self.modify_range_2mib( + pages, + |entry, _, _| { + if entry.is_unused() { + return Err(FlagUpdateError::PageNotMapped); + } + + entry.set_flags(flags); + Ok(()) + }, + (), + Self::next_table_fn_next_table_mut, + ) + } + unsafe fn set_flags_p4_entry( &mut self, page: Page, @@ -579,6 +1207,52 @@ impl<'a> Mapper for RecursivePageTable<'a> { self.map_to_4kib(page, frame, flags, parent_table_flags, allocator) } + #[inline] + unsafe fn map_to_range_with_table_flags( + &mut self, + pages: PageRange, + frames: PhysFrameRange, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + Self: Sized, + A: FrameAllocator + ?Sized, + { + assert_eq!(pages.count(), frames.count()); + self.map_to_range_4kib( + pages, + |page, _| { + let offset = pages.start - page; + Some(frames.start + (offset / Size4KiB::SIZE)) + }, + flags, + parent_table_flags, + allocator, + ) + } + + #[inline] + unsafe fn map_range_with_table_flags( + &mut self, + pages: PageRange, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + A: FrameAllocator + ?Sized, + { + self.map_to_range_4kib( + pages, + |_, allocator| allocator.allocate_frame(), + flags, + parent_table_flags, + allocator, + ) + } + fn unmap( &mut self, page: Page, @@ -616,6 +1290,36 @@ impl<'a> Mapper for RecursivePageTable<'a> { Ok((frame, MapperFlush::new(page))) } + #[inline] + fn unmap_range( + &mut self, + pages: PageRange, + deallocator: &mut D, + ) -> Result, (UnmapError, MapperFlushRange)> + where + D: FrameDeallocator, + { + self.modify_range_4kib( + pages, + |entry, _, deallocator| { + let frame = entry.frame().map_err(|err| match err { + FrameError::FrameNotPresent => UnmapError::PageNotMapped, + FrameError::HugeFrame => UnmapError::ParentEntryHugePage, + })?; + unsafe { + deallocator.deallocate_frame(frame); + } + + entry.set_unused(); + Ok(()) + }, + deallocator, + Self::next_table_fn_next_table_mut, + ) + } + + // allow unused_unsafe until https://github.com/rust-lang/rfcs/pull/2585 lands + #[allow(unused_unsafe)] unsafe fn update_flags( &mut self, page: Page, @@ -650,6 +1354,27 @@ impl<'a> Mapper for RecursivePageTable<'a> { Ok(MapperFlush::new(page)) } + #[inline] + unsafe fn update_flags_range( + &mut self, + pages: PageRange, + flags: PageTableFlags, + ) -> Result, (FlagUpdateError, MapperFlushRange)> { + self.modify_range_4kib( + pages, + |entry, _, _| { + if entry.is_unused() { + return Err(FlagUpdateError::PageNotMapped); + } + + entry.set_flags(flags); + Ok(()) + }, + (), + Self::next_table_fn_next_table_mut, + ) + } + unsafe fn set_flags_p4_entry( &mut self, page: Page, From 1f6ce05e009f24511776fcef324d4f0480cb3113 Mon Sep 17 00:00:00 2001 From: Tom Dohrmann Date: Mon, 7 Jun 2021 20:28:25 +0200 Subject: [PATCH 05/18] make unmap_range unsafe --- src/structures/paging/mapper/mapped_page_table.rs | 6 +++--- src/structures/paging/mapper/mod.rs | 6 +++++- src/structures/paging/mapper/offset_page_table.rs | 12 ++++++------ src/structures/paging/mapper/recursive_page_table.rs | 6 +++--- 4 files changed, 17 insertions(+), 13 deletions(-) diff --git a/src/structures/paging/mapper/mapped_page_table.rs b/src/structures/paging/mapper/mapped_page_table.rs index 1e2919691..8a3faff91 100644 --- a/src/structures/paging/mapper/mapped_page_table.rs +++ b/src/structures/paging/mapper/mapped_page_table.rs @@ -645,7 +645,7 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { } #[inline] - fn unmap_range( + unsafe fn unmap_range( &mut self, pages: PageRange, deallocator: &mut D, @@ -850,7 +850,7 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { } #[inline] - fn unmap_range( + unsafe fn unmap_range( &mut self, pages: PageRange, deallocator: &mut D, @@ -1068,7 +1068,7 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { } #[inline] - fn unmap_range( + unsafe fn unmap_range( &mut self, pages: PageRange, deallocator: &mut D, diff --git a/src/structures/paging/mapper/mod.rs b/src/structures/paging/mapper/mod.rs index c208933fe..152818138 100644 --- a/src/structures/paging/mapper/mod.rs +++ b/src/structures/paging/mapper/mod.rs @@ -444,7 +444,11 @@ pub trait Mapper { /// /// ## Errors /// If an error occurs half-way through a [`MapperFlushRange`] is returned that contains the pages that were successfully unmapped. - fn unmap_range( + /// + /// ## Safety + /// The caller has to guarantee that it's safe to deallocate frames: + /// All unmapped frames must only be only in this page table. + unsafe fn unmap_range( &mut self, pages: PageRange, deallocator: &mut D, diff --git a/src/structures/paging/mapper/offset_page_table.rs b/src/structures/paging/mapper/offset_page_table.rs index fec972d06..6d714d4e3 100644 --- a/src/structures/paging/mapper/offset_page_table.rs +++ b/src/structures/paging/mapper/offset_page_table.rs @@ -127,7 +127,7 @@ impl<'a> Mapper for OffsetPageTable<'a> { } #[inline] - fn unmap_range( + unsafe fn unmap_range( &mut self, pages: PageRange, deallocator: &mut D, @@ -135,7 +135,7 @@ impl<'a> Mapper for OffsetPageTable<'a> { where D: FrameDeallocator, { - self.inner.unmap_range(pages, deallocator) + unsafe { self.inner.unmap_range(pages, deallocator) } } #[inline] @@ -249,7 +249,7 @@ impl<'a> Mapper for OffsetPageTable<'a> { } #[inline] - fn unmap_range( + unsafe fn unmap_range( &mut self, pages: PageRange, deallocator: &mut D, @@ -257,7 +257,7 @@ impl<'a> Mapper for OffsetPageTable<'a> { where D: FrameDeallocator, { - self.inner.unmap_range(pages, deallocator) + unsafe { self.inner.unmap_range(pages, deallocator) } } #[inline] @@ -371,7 +371,7 @@ impl<'a> Mapper for OffsetPageTable<'a> { } #[inline] - fn unmap_range( + unsafe fn unmap_range( &mut self, pages: PageRange, deallocator: &mut D, @@ -379,7 +379,7 @@ impl<'a> Mapper for OffsetPageTable<'a> { where D: FrameDeallocator, { - self.inner.unmap_range(pages, deallocator) + unsafe { self.inner.unmap_range(pages, deallocator) } } #[inline] diff --git a/src/structures/paging/mapper/recursive_page_table.rs b/src/structures/paging/mapper/recursive_page_table.rs index de8943cbb..fd606fa48 100644 --- a/src/structures/paging/mapper/recursive_page_table.rs +++ b/src/structures/paging/mapper/recursive_page_table.rs @@ -818,7 +818,7 @@ impl<'a> Mapper for RecursivePageTable<'a> { } #[inline] - fn unmap_range( + unsafe fn unmap_range( &mut self, pages: PageRange, deallocator: &mut D, @@ -1040,7 +1040,7 @@ impl<'a> Mapper for RecursivePageTable<'a> { } #[inline] - fn unmap_range( + unsafe fn unmap_range( &mut self, pages: PageRange, deallocator: &mut D, @@ -1291,7 +1291,7 @@ impl<'a> Mapper for RecursivePageTable<'a> { } #[inline] - fn unmap_range( + unsafe fn unmap_range( &mut self, pages: PageRange, deallocator: &mut D, From d3a8468bf2ccf5aa680d12d198704a5d7095a010 Mon Sep 17 00:00:00 2001 From: Tom Dohrmann Date: Mon, 7 Jun 2021 20:43:29 +0200 Subject: [PATCH 06/18] add map_range --- src/structures/paging/mapper/mod.rs | 30 +++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/structures/paging/mapper/mod.rs b/src/structures/paging/mapper/mod.rs index 152818138..435801b59 100644 --- a/src/structures/paging/mapper/mod.rs +++ b/src/structures/paging/mapper/mod.rs @@ -433,6 +433,36 @@ pub trait Mapper { }) } + /// Maps frames from the allocator to the given range of virtual pages. + /// + /// ## Safety + /// + /// This is a convencience function that invokes [`Mapper::map_range_with_table_flags`] internally, so + /// all safety requirements of it also apply for this function. + /// + /// ## Errors + /// + /// If an error occurs half-way through a [`MapperFlushRange`] is returned that contains the frames that were successfully mapped. + #[inline] + unsafe fn map_range( + &mut self, + pages: PageRange, + flags: PageTableFlags, + frame_allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + Self: Sized, + A: FrameAllocator + FrameAllocator + ?Sized, + { + let parent_table_flags = flags + & (PageTableFlags::PRESENT + | PageTableFlags::WRITABLE + | PageTableFlags::USER_ACCESSIBLE); + unsafe { + self.map_range_with_table_flags(pages, flags, parent_table_flags, frame_allocator) + } + } + /// Removes a mapping from the page table and returns the frame that used to be mapped. /// /// Note that no page tables or pages are deallocated. From 96a28924a708704cd44435fdb47f132c52b5372e Mon Sep 17 00:00:00 2001 From: Tom Dohrmann Date: Tue, 8 Jun 2021 17:44:06 +0200 Subject: [PATCH 07/18] make map_range safe --- .../paging/mapper/mapped_page_table.rs | 6 ++--- src/structures/paging/mapper/mod.rs | 20 ++++------------ .../paging/mapper/offset_page_table.rs | 24 +++++++------------ .../paging/mapper/recursive_page_table.rs | 6 ++--- 4 files changed, 20 insertions(+), 36 deletions(-) diff --git a/src/structures/paging/mapper/mapped_page_table.rs b/src/structures/paging/mapper/mapped_page_table.rs index 8a3faff91..563109045 100644 --- a/src/structures/paging/mapper/mapped_page_table.rs +++ b/src/structures/paging/mapper/mapped_page_table.rs @@ -598,7 +598,7 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { } #[inline] - unsafe fn map_range_with_table_flags( + fn map_range_with_table_flags( &mut self, pages: PageRange, flags: PageTableFlags, @@ -800,7 +800,7 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { } #[inline] - unsafe fn map_range_with_table_flags( + fn map_range_with_table_flags( &mut self, pages: PageRange, flags: PageTableFlags, @@ -1022,7 +1022,7 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { } #[inline] - unsafe fn map_range_with_table_flags( + fn map_range_with_table_flags( &mut self, pages: PageRange, flags: PageTableFlags, diff --git a/src/structures/paging/mapper/mod.rs b/src/structures/paging/mapper/mod.rs index 435801b59..3ec9616c5 100644 --- a/src/structures/paging/mapper/mod.rs +++ b/src/structures/paging/mapper/mod.rs @@ -384,15 +384,10 @@ pub trait Mapper { /// Maps frames from the allocator to the given range of virtual pages. /// - /// ## Safety - /// - /// This is a convencience function that invokes [`Mapper::map_to_with_table_flags`] internally, so - /// all safety requirements of it also apply for this function. - /// /// ## Errors /// /// If an error occurs half-way through a [`MapperFlushRange`] is returned that contains the frames that were successfully mapped. - unsafe fn map_range_with_table_flags( + fn map_range_with_table_flags( &mut self, mut pages: PageRange, flags: PageTableFlags, @@ -410,6 +405,7 @@ pub trait Mapper { .ok_or((MapToError::FrameAllocationFailed, page))?; unsafe { + // SAFETY: frame was just freshly allocated and therefore can't cause aliasing issues self.map_to_with_table_flags( page, frame, @@ -435,16 +431,11 @@ pub trait Mapper { /// Maps frames from the allocator to the given range of virtual pages. /// - /// ## Safety - /// - /// This is a convencience function that invokes [`Mapper::map_range_with_table_flags`] internally, so - /// all safety requirements of it also apply for this function. - /// /// ## Errors /// /// If an error occurs half-way through a [`MapperFlushRange`] is returned that contains the frames that were successfully mapped. #[inline] - unsafe fn map_range( + fn map_range( &mut self, pages: PageRange, flags: PageTableFlags, @@ -458,9 +449,8 @@ pub trait Mapper { & (PageTableFlags::PRESENT | PageTableFlags::WRITABLE | PageTableFlags::USER_ACCESSIBLE); - unsafe { - self.map_range_with_table_flags(pages, flags, parent_table_flags, frame_allocator) - } + + self.map_range_with_table_flags(pages, flags, parent_table_flags, frame_allocator) } /// Removes a mapping from the page table and returns the frame that used to be mapped. diff --git a/src/structures/paging/mapper/offset_page_table.rs b/src/structures/paging/mapper/offset_page_table.rs index 6d714d4e3..a1925ed18 100644 --- a/src/structures/paging/mapper/offset_page_table.rs +++ b/src/structures/paging/mapper/offset_page_table.rs @@ -102,7 +102,7 @@ impl<'a> Mapper for OffsetPageTable<'a> { } #[inline] - unsafe fn map_range_with_table_flags( + fn map_range_with_table_flags( &mut self, pages: PageRange, flags: PageTableFlags, @@ -112,10 +112,8 @@ impl<'a> Mapper for OffsetPageTable<'a> { where A: FrameAllocator + FrameAllocator + ?Sized, { - unsafe { - self.inner - .map_range_with_table_flags(pages, flags, parent_table_flags, allocator) - } + self.inner + .map_range_with_table_flags(pages, flags, parent_table_flags, allocator) } #[inline] @@ -224,7 +222,7 @@ impl<'a> Mapper for OffsetPageTable<'a> { } #[inline] - unsafe fn map_range_with_table_flags( + fn map_range_with_table_flags( &mut self, pages: PageRange, flags: PageTableFlags, @@ -234,10 +232,8 @@ impl<'a> Mapper for OffsetPageTable<'a> { where A: FrameAllocator + FrameAllocator + ?Sized, { - unsafe { - self.inner - .map_range_with_table_flags(pages, flags, parent_table_flags, allocator) - } + self.inner + .map_range_with_table_flags(pages, flags, parent_table_flags, allocator) } #[inline] @@ -346,7 +342,7 @@ impl<'a> Mapper for OffsetPageTable<'a> { } #[inline] - unsafe fn map_range_with_table_flags( + fn map_range_with_table_flags( &mut self, pages: PageRange, flags: PageTableFlags, @@ -356,10 +352,8 @@ impl<'a> Mapper for OffsetPageTable<'a> { where A: FrameAllocator + ?Sized, { - unsafe { - self.inner - .map_range_with_table_flags(pages, flags, parent_table_flags, allocator) - } + self.inner + .map_range_with_table_flags(pages, flags, parent_table_flags, allocator) } #[inline] diff --git a/src/structures/paging/mapper/recursive_page_table.rs b/src/structures/paging/mapper/recursive_page_table.rs index fd606fa48..a5a43cbaf 100644 --- a/src/structures/paging/mapper/recursive_page_table.rs +++ b/src/structures/paging/mapper/recursive_page_table.rs @@ -767,7 +767,7 @@ impl<'a> Mapper for RecursivePageTable<'a> { } #[inline] - unsafe fn map_range_with_table_flags( + fn map_range_with_table_flags( &mut self, pages: PageRange, flags: PageTableFlags, @@ -983,7 +983,7 @@ impl<'a> Mapper for RecursivePageTable<'a> { } #[inline] - unsafe fn map_range_with_table_flags( + fn map_range_with_table_flags( &mut self, pages: PageRange, flags: PageTableFlags, @@ -1234,7 +1234,7 @@ impl<'a> Mapper for RecursivePageTable<'a> { } #[inline] - unsafe fn map_range_with_table_flags( + fn map_range_with_table_flags( &mut self, pages: PageRange, flags: PageTableFlags, From caecf8053ce9731f0531c9832ae8fe46d1ddd0e7 Mon Sep 17 00:00:00 2001 From: Tom Dohrmann Date: Mon, 14 Jun 2021 12:09:08 +0200 Subject: [PATCH 08/18] fix map_to_range_4kib --- src/structures/paging/mapper/mapped_page_table.rs | 2 +- src/structures/paging/mapper/recursive_page_table.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/structures/paging/mapper/mapped_page_table.rs b/src/structures/paging/mapper/mapped_page_table.rs index 563109045..86c38ec90 100644 --- a/src/structures/paging/mapper/mapped_page_table.rs +++ b/src/structures/paging/mapper/mapped_page_table.rs @@ -546,7 +546,7 @@ impl<'a, P: PageTableFrameMapping> MappedPageTable<'a, P> { if !entry.is_unused() { return Err(MapToError::PageAlreadyMapped(frame)); } - entry.set_addr(frame.start_address(), flags | PageTableFlags::HUGE_PAGE); + entry.set_addr(frame.start_address(), flags); Ok(()) }, (parent_table_flags, allocator), diff --git a/src/structures/paging/mapper/recursive_page_table.rs b/src/structures/paging/mapper/recursive_page_table.rs index a5a43cbaf..be5f4846c 100644 --- a/src/structures/paging/mapper/recursive_page_table.rs +++ b/src/structures/paging/mapper/recursive_page_table.rs @@ -715,7 +715,7 @@ impl<'a> RecursivePageTable<'a> { if !entry.is_unused() { return Err(MapToError::PageAlreadyMapped(frame)); } - entry.set_addr(frame.start_address(), flags | PageTableFlags::HUGE_PAGE); + entry.set_addr(frame.start_address(), flags); Ok(()) }, (parent_table_flags, allocator), From fc941f9af9933261722b4dbdc2eadc97f2c7e60e Mon Sep 17 00:00:00 2001 From: Tom Dohrmann Date: Sun, 20 Jun 2021 15:45:06 +0200 Subject: [PATCH 09/18] fix offsets --- src/structures/paging/mapper/mapped_page_table.rs | 12 ++++++------ src/structures/paging/mapper/recursive_page_table.rs | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/structures/paging/mapper/mapped_page_table.rs b/src/structures/paging/mapper/mapped_page_table.rs index 86c38ec90..59e3f8625 100644 --- a/src/structures/paging/mapper/mapped_page_table.rs +++ b/src/structures/paging/mapper/mapped_page_table.rs @@ -588,8 +588,8 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { self.map_to_range_1gib( pages, |page, _| { - let offset = pages.start - page; - Some(frames.start + (offset / Size1GiB::SIZE)) + let offset = page - pages.start; + Some(frames.start + offset) }, flags, parent_table_flags, @@ -790,8 +790,8 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { self.map_range_2mib( pages, |page, _| { - let offset = pages.start - page; - Some(frames.start + (offset / Size2MiB::SIZE)) + let offset = page - pages.start; + Some(frames.start + offset) }, flags, parent_table_flags, @@ -1012,8 +1012,8 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { self.map_to_range_4kib( pages, |page, _| { - let offset = pages.start - page; - Some(frames.start + (offset / Size4KiB::SIZE)) + let offset = page - pages.start; + Some(frames.start + offset) }, flags, parent_table_flags, diff --git a/src/structures/paging/mapper/recursive_page_table.rs b/src/structures/paging/mapper/recursive_page_table.rs index be5f4846c..e29453a3d 100644 --- a/src/structures/paging/mapper/recursive_page_table.rs +++ b/src/structures/paging/mapper/recursive_page_table.rs @@ -757,8 +757,8 @@ impl<'a> Mapper for RecursivePageTable<'a> { self.map_to_range_1gib( pages, |page, _| { - let offset = pages.start - page; - Some(frames.start + (offset / Size1GiB::SIZE)) + let offset = page - pages.start; + Some(frames.start + offset) }, flags, parent_table_flags, @@ -973,8 +973,8 @@ impl<'a> Mapper for RecursivePageTable<'a> { self.map_range_2mib( pages, |page, _| { - let offset = pages.start - page; - Some(frames.start + (offset / Size2MiB::SIZE)) + let offset = page - pages.start; + Some(frames.start + offset) }, flags, parent_table_flags, @@ -1224,8 +1224,8 @@ impl<'a> Mapper for RecursivePageTable<'a> { self.map_to_range_4kib( pages, |page, _| { - let offset = pages.start - page; - Some(frames.start + (offset / Size4KiB::SIZE)) + let offset = page - pages.start; + Some(frames.start + offset) }, flags, parent_table_flags, From e232d1fb6fb1d3d9136289f15e0b3f1c7b04dcaa Mon Sep 17 00:00:00 2001 From: Tom Dohrmann Date: Sun, 20 Jun 2021 15:45:36 +0200 Subject: [PATCH 10/18] forward map_to_range to map_to_range_with_table_flags by default --- src/structures/paging/mapper/mod.rs | 30 +++++++++++++---------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/src/structures/paging/mapper/mod.rs b/src/structures/paging/mapper/mod.rs index 3ec9616c5..45e560b67 100644 --- a/src/structures/paging/mapper/mod.rs +++ b/src/structures/paging/mapper/mod.rs @@ -226,24 +226,20 @@ pub trait Mapper { Self: Sized, A: FrameAllocator + ?Sized, { - assert_eq!(pages.count(), frames.count()); + let parent_table_flags = flags + & (PageTableFlags::PRESENT + | PageTableFlags::WRITABLE + | PageTableFlags::USER_ACCESSIBLE); - pages - .zip(frames) - .try_for_each(|(page, frame)| { - unsafe { self.map_to(page, frame, flags, frame_allocator) } - .map(|_| ()) - .map_err(|e| { - ( - e, - MapperFlushRange::new(PageRange { - start: pages.start, - end: page, - }), - ) - }) - }) - .map(|_| MapperFlushRange::new(pages)) + unsafe { + self.map_to_range_with_table_flags( + pages, + frames, + flags, + parent_table_flags, + frame_allocator, + ) + } } /// Creates a new mapping in the page table. From 6f197c75d8b4fb0acbb31bf93a090e5805fb61a4 Mon Sep 17 00:00:00 2001 From: Tom Dohrmann Date: Sat, 4 Sep 2021 16:50:47 +0200 Subject: [PATCH 11/18] fix bounds on unmap_range to make sure Mapper is object safe --- src/structures/paging/mapper/mapped_page_table.rs | 9 ++++++--- src/structures/paging/mapper/mod.rs | 3 ++- src/structures/paging/mapper/offset_page_table.rs | 9 ++++++--- src/structures/paging/mapper/recursive_page_table.rs | 9 ++++++--- 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/src/structures/paging/mapper/mapped_page_table.rs b/src/structures/paging/mapper/mapped_page_table.rs index 59e3f8625..77a9fc59f 100644 --- a/src/structures/paging/mapper/mapped_page_table.rs +++ b/src/structures/paging/mapper/mapped_page_table.rs @@ -651,7 +651,8 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { deallocator: &mut D, ) -> Result, (UnmapError, MapperFlushRange)> where - D: FrameDeallocator, + Self: Sized, + D: FrameDeallocator + ?Sized, { self.modify_range_1gib( pages, @@ -856,7 +857,8 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { deallocator: &mut D, ) -> Result, (UnmapError, MapperFlushRange)> where - D: FrameDeallocator, + Self: Sized, + D: FrameDeallocator + ?Sized, { self.modify_range_2mib( pages, @@ -1074,7 +1076,8 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { deallocator: &mut D, ) -> Result, (UnmapError, MapperFlushRange)> where - D: FrameDeallocator, + Self: Sized, + D: FrameDeallocator + ?Sized, { self.modify_range_4kib( pages, diff --git a/src/structures/paging/mapper/mod.rs b/src/structures/paging/mapper/mod.rs index 45e560b67..dbdfd920b 100644 --- a/src/structures/paging/mapper/mod.rs +++ b/src/structures/paging/mapper/mod.rs @@ -470,7 +470,8 @@ pub trait Mapper { deallocator: &mut D, ) -> Result, (UnmapError, MapperFlushRange)> where - D: FrameDeallocator, + Self: Sized, + D: FrameDeallocator + ?Sized, { pages .clone() diff --git a/src/structures/paging/mapper/offset_page_table.rs b/src/structures/paging/mapper/offset_page_table.rs index a1925ed18..b3232c251 100644 --- a/src/structures/paging/mapper/offset_page_table.rs +++ b/src/structures/paging/mapper/offset_page_table.rs @@ -131,7 +131,8 @@ impl<'a> Mapper for OffsetPageTable<'a> { deallocator: &mut D, ) -> Result, (UnmapError, MapperFlushRange)> where - D: FrameDeallocator, + Self: Sized, + D: FrameDeallocator + ?Sized, { unsafe { self.inner.unmap_range(pages, deallocator) } } @@ -251,7 +252,8 @@ impl<'a> Mapper for OffsetPageTable<'a> { deallocator: &mut D, ) -> Result, (UnmapError, MapperFlushRange)> where - D: FrameDeallocator, + Self: Sized, + D: FrameDeallocator + ?Sized, { unsafe { self.inner.unmap_range(pages, deallocator) } } @@ -371,7 +373,8 @@ impl<'a> Mapper for OffsetPageTable<'a> { deallocator: &mut D, ) -> Result, (UnmapError, MapperFlushRange)> where - D: FrameDeallocator, + Self: Sized, + D: FrameDeallocator + ?Sized, { unsafe { self.inner.unmap_range(pages, deallocator) } } diff --git a/src/structures/paging/mapper/recursive_page_table.rs b/src/structures/paging/mapper/recursive_page_table.rs index e29453a3d..72257c830 100644 --- a/src/structures/paging/mapper/recursive_page_table.rs +++ b/src/structures/paging/mapper/recursive_page_table.rs @@ -824,7 +824,8 @@ impl<'a> Mapper for RecursivePageTable<'a> { deallocator: &mut D, ) -> Result, (UnmapError, MapperFlushRange)> where - D: FrameDeallocator, + Self: Sized, + D: FrameDeallocator + ?Sized, { self.modify_range_1gib( pages, @@ -1046,7 +1047,8 @@ impl<'a> Mapper for RecursivePageTable<'a> { deallocator: &mut D, ) -> Result, (UnmapError, MapperFlushRange)> where - D: FrameDeallocator, + Self: Sized, + D: FrameDeallocator + ?Sized, { self.modify_range_2mib( pages, @@ -1297,7 +1299,8 @@ impl<'a> Mapper for RecursivePageTable<'a> { deallocator: &mut D, ) -> Result, (UnmapError, MapperFlushRange)> where - D: FrameDeallocator, + Self: Sized, + D: FrameDeallocator + ?Sized, { self.modify_range_4kib( pages, From 82acf51eee329ca63ff06d024ca46991636bc930 Mon Sep 17 00:00:00 2001 From: Tom Dohrmann Date: Sat, 4 Sep 2021 16:56:44 +0200 Subject: [PATCH 12/18] rename MapperFlushRange::flush and add MapperFlushRange::flush_all --- src/structures/paging/mapper/mod.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/structures/paging/mapper/mod.rs b/src/structures/paging/mapper/mod.rs index dbdfd920b..17b7390e1 100644 --- a/src/structures/paging/mapper/mod.rs +++ b/src/structures/paging/mapper/mod.rs @@ -700,15 +700,22 @@ impl MapperFlushRange { }) } - /// Flush the page from the TLB to ensure that the newest mapping is used. + /// Flush the pages from the TLB to ensure that the newest mapping is used. #[cfg(feature = "instructions")] #[inline] - pub fn flush(self) { + pub fn flush_range(self) { for page in self.0 { crate::instructions::tlb::flush(page.start_address()) } } + /// Flush all pages from the TLB to ensure that the newest mapping is used. + #[cfg(feature = "instructions")] + #[inline] + pub fn flush_all(self) { + crate::instructions::tlb::flush_all() + } + /// Don't flush the TLB and silence the “must be used” warning. #[inline] pub fn ignore(self) {} From e38d84ecab1b5b60ebf51e3c7727a349145eb292 Mon Sep 17 00:00:00 2001 From: Tom Dohrmann Date: Sat, 4 Sep 2021 17:06:35 +0200 Subject: [PATCH 13/18] rename map_range_2mib to map_to_range_2mib --- src/structures/paging/mapper/mapped_page_table.rs | 6 +++--- src/structures/paging/mapper/recursive_page_table.rs | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/structures/paging/mapper/mapped_page_table.rs b/src/structures/paging/mapper/mapped_page_table.rs index 77a9fc59f..cb58bb12c 100644 --- a/src/structures/paging/mapper/mapped_page_table.rs +++ b/src/structures/paging/mapper/mapped_page_table.rs @@ -361,7 +361,7 @@ impl<'a, P: PageTableFrameMapping> MappedPageTable<'a, P> { } #[inline] - fn map_range_2mib( + fn map_to_range_2mib( &mut self, pages: PageRange, frames: F, @@ -788,7 +788,7 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { A: FrameAllocator + ?Sized, { assert_eq!(pages.count(), frames.count()); - self.map_range_2mib( + self.map_to_range_2mib( pages, |page, _| { let offset = page - pages.start; @@ -812,7 +812,7 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { Self: Sized, A: FrameAllocator + FrameAllocator + ?Sized, { - self.map_range_2mib( + self.map_to_range_2mib( pages, |_, allocator| allocator.allocate_frame(), flags, diff --git a/src/structures/paging/mapper/recursive_page_table.rs b/src/structures/paging/mapper/recursive_page_table.rs index 72257c830..93dc9a1f3 100644 --- a/src/structures/paging/mapper/recursive_page_table.rs +++ b/src/structures/paging/mapper/recursive_page_table.rs @@ -514,7 +514,7 @@ impl<'a> RecursivePageTable<'a> { } #[inline] - fn map_range_2mib( + fn map_to_range_2mib( &mut self, pages: PageRange, frames: F, @@ -971,7 +971,7 @@ impl<'a> Mapper for RecursivePageTable<'a> { A: FrameAllocator + ?Sized, { assert_eq!(pages.count(), frames.count()); - self.map_range_2mib( + self.map_to_range_2mib( pages, |page, _| { let offset = page - pages.start; @@ -995,7 +995,7 @@ impl<'a> Mapper for RecursivePageTable<'a> { Self: Sized, A: FrameAllocator + FrameAllocator + ?Sized, { - self.map_range_2mib( + self.map_to_range_2mib( pages, |_, allocator| allocator.allocate_frame(), flags, From 87c8e4c33cb05bbf345d9c98fba0462ec4305214 Mon Sep 17 00:00:00 2001 From: Tom Dohrmann Date: Sun, 5 Sep 2021 13:51:26 +0200 Subject: [PATCH 14/18] make map_range_with_table_flags and map_range unsafe --- .../paging/mapper/mapped_page_table.rs | 6 ++--- src/structures/paging/mapper/mod.rs | 18 +++++++++++--- .../paging/mapper/offset_page_table.rs | 24 ++++++++++++------- .../paging/mapper/recursive_page_table.rs | 6 ++--- 4 files changed, 36 insertions(+), 18 deletions(-) diff --git a/src/structures/paging/mapper/mapped_page_table.rs b/src/structures/paging/mapper/mapped_page_table.rs index cb58bb12c..4d2a270b6 100644 --- a/src/structures/paging/mapper/mapped_page_table.rs +++ b/src/structures/paging/mapper/mapped_page_table.rs @@ -598,7 +598,7 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { } #[inline] - fn map_range_with_table_flags( + unsafe fn map_range_with_table_flags( &mut self, pages: PageRange, flags: PageTableFlags, @@ -801,7 +801,7 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { } #[inline] - fn map_range_with_table_flags( + unsafe fn map_range_with_table_flags( &mut self, pages: PageRange, flags: PageTableFlags, @@ -1024,7 +1024,7 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { } #[inline] - fn map_range_with_table_flags( + unsafe fn map_range_with_table_flags( &mut self, pages: PageRange, flags: PageTableFlags, diff --git a/src/structures/paging/mapper/mod.rs b/src/structures/paging/mapper/mod.rs index 17b7390e1..6bb2cc14a 100644 --- a/src/structures/paging/mapper/mod.rs +++ b/src/structures/paging/mapper/mod.rs @@ -380,10 +380,15 @@ pub trait Mapper { /// Maps frames from the allocator to the given range of virtual pages. /// + /// ## Safety + /// + /// This function invokes [`Mapper::map_to_with_table_flags`] internally, so + /// all safety requirements of it also apply for this function. + /// /// ## Errors /// /// If an error occurs half-way through a [`MapperFlushRange`] is returned that contains the frames that were successfully mapped. - fn map_range_with_table_flags( + unsafe fn map_range_with_table_flags( &mut self, mut pages: PageRange, flags: PageTableFlags, @@ -427,11 +432,16 @@ pub trait Mapper { /// Maps frames from the allocator to the given range of virtual pages. /// + /// ## Safety + /// + /// This function invokes [`Mapper::map_to_with_table_flags`] internally, so + /// all safety requirements of it also apply for this function. + /// /// ## Errors /// /// If an error occurs half-way through a [`MapperFlushRange`] is returned that contains the frames that were successfully mapped. #[inline] - fn map_range( + unsafe fn map_range( &mut self, pages: PageRange, flags: PageTableFlags, @@ -446,7 +456,9 @@ pub trait Mapper { | PageTableFlags::WRITABLE | PageTableFlags::USER_ACCESSIBLE); - self.map_range_with_table_flags(pages, flags, parent_table_flags, frame_allocator) + unsafe { + self.map_range_with_table_flags(pages, flags, parent_table_flags, frame_allocator) + } } /// Removes a mapping from the page table and returns the frame that used to be mapped. diff --git a/src/structures/paging/mapper/offset_page_table.rs b/src/structures/paging/mapper/offset_page_table.rs index b3232c251..ba7dd382a 100644 --- a/src/structures/paging/mapper/offset_page_table.rs +++ b/src/structures/paging/mapper/offset_page_table.rs @@ -102,7 +102,7 @@ impl<'a> Mapper for OffsetPageTable<'a> { } #[inline] - fn map_range_with_table_flags( + unsafe fn map_range_with_table_flags( &mut self, pages: PageRange, flags: PageTableFlags, @@ -112,8 +112,10 @@ impl<'a> Mapper for OffsetPageTable<'a> { where A: FrameAllocator + FrameAllocator + ?Sized, { - self.inner - .map_range_with_table_flags(pages, flags, parent_table_flags, allocator) + unsafe { + self.inner + .map_range_with_table_flags(pages, flags, parent_table_flags, allocator) + } } #[inline] @@ -223,7 +225,7 @@ impl<'a> Mapper for OffsetPageTable<'a> { } #[inline] - fn map_range_with_table_flags( + unsafe fn map_range_with_table_flags( &mut self, pages: PageRange, flags: PageTableFlags, @@ -233,8 +235,10 @@ impl<'a> Mapper for OffsetPageTable<'a> { where A: FrameAllocator + FrameAllocator + ?Sized, { - self.inner - .map_range_with_table_flags(pages, flags, parent_table_flags, allocator) + unsafe { + self.inner + .map_range_with_table_flags(pages, flags, parent_table_flags, allocator) + } } #[inline] @@ -344,7 +348,7 @@ impl<'a> Mapper for OffsetPageTable<'a> { } #[inline] - fn map_range_with_table_flags( + unsafe fn map_range_with_table_flags( &mut self, pages: PageRange, flags: PageTableFlags, @@ -354,8 +358,10 @@ impl<'a> Mapper for OffsetPageTable<'a> { where A: FrameAllocator + ?Sized, { - self.inner - .map_range_with_table_flags(pages, flags, parent_table_flags, allocator) + unsafe { + self.inner + .map_range_with_table_flags(pages, flags, parent_table_flags, allocator) + } } #[inline] diff --git a/src/structures/paging/mapper/recursive_page_table.rs b/src/structures/paging/mapper/recursive_page_table.rs index 93dc9a1f3..8c2d7dccd 100644 --- a/src/structures/paging/mapper/recursive_page_table.rs +++ b/src/structures/paging/mapper/recursive_page_table.rs @@ -767,7 +767,7 @@ impl<'a> Mapper for RecursivePageTable<'a> { } #[inline] - fn map_range_with_table_flags( + unsafe fn map_range_with_table_flags( &mut self, pages: PageRange, flags: PageTableFlags, @@ -984,7 +984,7 @@ impl<'a> Mapper for RecursivePageTable<'a> { } #[inline] - fn map_range_with_table_flags( + unsafe fn map_range_with_table_flags( &mut self, pages: PageRange, flags: PageTableFlags, @@ -1236,7 +1236,7 @@ impl<'a> Mapper for RecursivePageTable<'a> { } #[inline] - fn map_range_with_table_flags( + unsafe fn map_range_with_table_flags( &mut self, pages: PageRange, flags: PageTableFlags, From 3fc76875f3615f199b06d4bbb8a9bb6515beea1b Mon Sep 17 00:00:00 2001 From: Tom Dohrmann Date: Sun, 5 Sep 2021 14:02:27 +0200 Subject: [PATCH 15/18] fix typo --- src/structures/paging/mapper/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/structures/paging/mapper/mod.rs b/src/structures/paging/mapper/mod.rs index 6bb2cc14a..f49469524 100644 --- a/src/structures/paging/mapper/mod.rs +++ b/src/structures/paging/mapper/mod.rs @@ -475,7 +475,7 @@ pub trait Mapper { /// /// ## Safety /// The caller has to guarantee that it's safe to deallocate frames: - /// All unmapped frames must only be only in this page table. + /// All unmapped frames must only be in this page table. unsafe fn unmap_range( &mut self, pages: PageRange, From 6995024fb641448927bc681e4d82f9a4c1214bc3 Mon Sep 17 00:00:00 2001 From: Tom Dohrmann Date: Mon, 6 Sep 2021 14:22:14 +0200 Subject: [PATCH 16/18] add experimental feature flag --- Cargo.toml | 1 + .../paging/mapper/mapped_page_table.rs | 23 +++++++++++++++++- src/structures/paging/mapper/mod.rs | 14 +++++++++-- .../paging/mapper/offset_page_table.rs | 9 +++++++ .../paging/mapper/recursive_page_table.rs | 24 ++++++++++++++++++- 5 files changed, 67 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c261ada52..716047c14 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,6 +39,7 @@ nightly = [ "const_fn", "step_trait", "abi_x86_interrupt", "doc_cfg" ] abi_x86_interrupt = [] const_fn = [] step_trait = [] +experimental = [] doc_cfg = [] # These features are no longer used and only there for backwards compatibility. diff --git a/src/structures/paging/mapper/mapped_page_table.rs b/src/structures/paging/mapper/mapped_page_table.rs index 4d2a270b6..394e112d6 100644 --- a/src/structures/paging/mapper/mapped_page_table.rs +++ b/src/structures/paging/mapper/mapped_page_table.rs @@ -1,10 +1,11 @@ +#[cfg(feature = "experimental")] +use crate::structures::paging::PageTableIndex; use crate::structures::paging::{ frame::PhysFrame, frame_alloc::{FrameAllocator, FrameDeallocator}, mapper::*, page::{AddressNotAligned, Page, PageRangeInclusive, Size1GiB, Size2MiB, Size4KiB}, page_table::{FrameError, PageTable, PageTableEntry, PageTableFlags, PageTableLevel}, - PageTableIndex, }; /// A Mapper implementation that relies on a PhysAddr to VirtAddr conversion function. @@ -142,6 +143,7 @@ impl<'a, P: PageTableFrameMapping> MappedPageTable<'a, P> { Ok(MapperFlush::new(page)) } + #[cfg(feature = "experimental")] #[inline] fn next_table_fn_create_next_table<'b, A>( (flags, allocator): &mut (PageTableFlags, &mut A), @@ -156,6 +158,7 @@ impl<'a, P: PageTableFrameMapping> MappedPageTable<'a, P> { .map_err(Into::into) } + #[cfg(feature = "experimental")] #[inline] fn next_table_fn_next_table_mut<'b, T>( _: &mut T, @@ -165,6 +168,7 @@ impl<'a, P: PageTableFrameMapping> MappedPageTable<'a, P> { walker.next_table_mut(entry) } + #[cfg(feature = "experimental")] fn modify_range_1gib( &mut self, pages: PageRange, @@ -230,6 +234,7 @@ impl<'a, P: PageTableFrameMapping> MappedPageTable<'a, P> { }) } + #[cfg(feature = "experimental")] #[inline] fn map_to_range_1gib( &mut self, @@ -258,6 +263,7 @@ impl<'a, P: PageTableFrameMapping> MappedPageTable<'a, P> { ) } + #[cfg(feature = "experimental")] fn modify_range_2mib( &mut self, pages: PageRange, @@ -360,6 +366,7 @@ impl<'a, P: PageTableFrameMapping> MappedPageTable<'a, P> { }) } + #[cfg(feature = "experimental")] #[inline] fn map_to_range_2mib( &mut self, @@ -388,6 +395,7 @@ impl<'a, P: PageTableFrameMapping> MappedPageTable<'a, P> { ) } + #[cfg(feature = "experimental")] fn modify_range_4kib( &mut self, pages: PageRange, @@ -526,6 +534,7 @@ impl<'a, P: PageTableFrameMapping> MappedPageTable<'a, P> { }) } + #[cfg(feature = "experimental")] #[inline] fn map_to_range_4kib( &mut self, @@ -571,6 +580,7 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { self.map_to_1gib(page, frame, flags, parent_table_flags, allocator) } + #[cfg(feature = "experimental")] #[inline] unsafe fn map_to_range_with_table_flags( &mut self, @@ -597,6 +607,7 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { ) } + #[cfg(feature = "experimental")] #[inline] unsafe fn map_range_with_table_flags( &mut self, @@ -644,6 +655,7 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { Ok((frame, MapperFlush::new(page))) } + #[cfg(feature = "experimental")] #[inline] unsafe fn unmap_range( &mut self, @@ -689,6 +701,7 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { Ok(MapperFlush::new(page)) } + #[cfg(feature = "experimental")] #[inline] unsafe fn update_flags_range( &mut self, @@ -774,6 +787,7 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { self.map_to_2mib(page, frame, flags, parent_table_flags, allocator) } + #[cfg(feature = "experimental")] #[inline] unsafe fn map_to_range_with_table_flags( &mut self, @@ -800,6 +814,7 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { ) } + #[cfg(feature = "experimental")] #[inline] unsafe fn map_range_with_table_flags( &mut self, @@ -850,6 +865,7 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { Ok((frame, MapperFlush::new(page))) } + #[cfg(feature = "experimental")] #[inline] unsafe fn unmap_range( &mut self, @@ -899,6 +915,7 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { Ok(MapperFlush::new(page)) } + #[cfg(feature = "experimental")] #[inline] unsafe fn update_flags_range( &mut self, @@ -997,6 +1014,7 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { self.map_to_4kib(page, frame, flags, parent_table_flags, allocator) } + #[cfg(feature = "experimental")] #[inline] unsafe fn map_to_range_with_table_flags( &mut self, @@ -1023,6 +1041,7 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { ) } + #[cfg(feature = "experimental")] #[inline] unsafe fn map_range_with_table_flags( &mut self, @@ -1069,6 +1088,7 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { Ok((frame, MapperFlush::new(page))) } + #[cfg(feature = "experimental")] #[inline] unsafe fn unmap_range( &mut self, @@ -1123,6 +1143,7 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { Ok(MapperFlush::new(page)) } + #[cfg(feature = "experimental")] #[inline] unsafe fn update_flags_range( &mut self, diff --git a/src/structures/paging/mapper/mod.rs b/src/structures/paging/mapper/mod.rs index f49469524..382ee375a 100644 --- a/src/structures/paging/mapper/mod.rs +++ b/src/structures/paging/mapper/mod.rs @@ -8,10 +8,11 @@ pub use self::recursive_page_table::{InvalidPageTable, RecursivePageTable}; use core::convert::Infallible; +#[cfg(feature = "experimental")] +use crate::structures::paging::{frame::PhysFrameRange, page::PageRange}; use crate::structures::paging::{ - frame::PhysFrameRange, frame_alloc::{FrameAllocator, FrameDeallocator}, - page::{PageRange, PageRangeInclusive}, + page::PageRangeInclusive, page_table::PageTableFlags, Page, PageSize, PhysFrame, Size1GiB, Size2MiB, Size4KiB, }; @@ -200,6 +201,7 @@ pub trait Mapper { } } + #[cfg(feature = "experimental")] /// Maps the given range of frames to the range of virtual pages. /// /// ## Safety @@ -324,6 +326,7 @@ pub trait Mapper { Self: Sized, A: FrameAllocator + ?Sized; + #[cfg(feature = "experimental")] /// Maps the given range of frames to the range of virtual pages. /// /// ## Safety @@ -378,6 +381,7 @@ pub trait Mapper { .map(|_| MapperFlushRange::new(pages)) } + #[cfg(feature = "experimental")] /// Maps frames from the allocator to the given range of virtual pages. /// /// ## Safety @@ -430,6 +434,7 @@ pub trait Mapper { }) } + #[cfg(feature = "experimental")] /// Maps frames from the allocator to the given range of virtual pages. /// /// ## Safety @@ -466,6 +471,7 @@ pub trait Mapper { /// Note that no page tables or pages are deallocated. fn unmap(&mut self, page: Page) -> Result<(PhysFrame, MapperFlush), UnmapError>; + #[cfg(feature = "experimental")] /// Removes a range of mapping from the page table and deallocate the frames that used to be mapped. /// /// Note that no page tables or pages are deallocated. @@ -521,6 +527,7 @@ pub trait Mapper { flags: PageTableFlags, ) -> Result, FlagUpdateError>; + #[cfg(feature = "experimental")] /// Updates the flags of a range of existing mappings. /// /// ## Safety @@ -630,6 +637,7 @@ pub trait Mapper { unsafe { self.map_to(page, frame, flags, frame_allocator) } } + #[cfg(feature = "experimental")] /// Maps the given range of frames to the range of virtual pages with the same address. /// /// ## Safety @@ -687,6 +695,7 @@ impl MapperFlush { pub fn ignore(self) {} } +#[cfg(feature = "experimental")] /// This type represents a range of pages whose mappings have changed in the page table. /// /// The old mappings might be still cached in the translation lookaside buffer (TLB), so they need @@ -696,6 +705,7 @@ impl MapperFlush { #[must_use = "Page Table changes must be flushed or ignored."] pub struct MapperFlushRange(PageRange); +#[cfg(feature = "experimental")] impl MapperFlushRange { /// Create a new flush promise #[inline] diff --git a/src/structures/paging/mapper/offset_page_table.rs b/src/structures/paging/mapper/offset_page_table.rs index ba7dd382a..91748fe9c 100644 --- a/src/structures/paging/mapper/offset_page_table.rs +++ b/src/structures/paging/mapper/offset_page_table.rs @@ -77,6 +77,7 @@ impl<'a> Mapper for OffsetPageTable<'a> { } } + #[cfg(feature = "experimental")] #[inline] unsafe fn map_to_range_with_table_flags( &mut self, @@ -101,6 +102,7 @@ impl<'a> Mapper for OffsetPageTable<'a> { } } + #[cfg(feature = "experimental")] #[inline] unsafe fn map_range_with_table_flags( &mut self, @@ -126,6 +128,7 @@ impl<'a> Mapper for OffsetPageTable<'a> { self.inner.unmap(page) } + #[cfg(feature = "experimental")] #[inline] unsafe fn unmap_range( &mut self, @@ -200,6 +203,7 @@ impl<'a> Mapper for OffsetPageTable<'a> { } } + #[cfg(feature = "experimental")] #[inline] unsafe fn map_to_range_with_table_flags( &mut self, @@ -224,6 +228,7 @@ impl<'a> Mapper for OffsetPageTable<'a> { } } + #[cfg(feature = "experimental")] #[inline] unsafe fn map_range_with_table_flags( &mut self, @@ -249,6 +254,7 @@ impl<'a> Mapper for OffsetPageTable<'a> { self.inner.unmap(page) } + #[cfg(feature = "experimental")] #[inline] unsafe fn unmap_range( &mut self, @@ -323,6 +329,7 @@ impl<'a> Mapper for OffsetPageTable<'a> { } } + #[cfg(feature = "experimental")] #[inline] unsafe fn map_to_range_with_table_flags( &mut self, @@ -347,6 +354,7 @@ impl<'a> Mapper for OffsetPageTable<'a> { } } + #[cfg(feature = "experimental")] #[inline] unsafe fn map_range_with_table_flags( &mut self, @@ -372,6 +380,7 @@ impl<'a> Mapper for OffsetPageTable<'a> { self.inner.unmap(page) } + #[cfg(feature = "experimental")] #[inline] unsafe fn unmap_range( &mut self, diff --git a/src/structures/paging/mapper/recursive_page_table.rs b/src/structures/paging/mapper/recursive_page_table.rs index 8c2d7dccd..96e4823bb 100644 --- a/src/structures/paging/mapper/recursive_page_table.rs +++ b/src/structures/paging/mapper/recursive_page_table.rs @@ -1,6 +1,8 @@ //! Access the page tables through a recursively mapped level 4 table. -use core::{convert::Infallible, fmt}; +#[cfg(feature = "experimental")] +use core::convert::Infallible; +use core::fmt; use super::*; use crate::registers::control::Cr3; @@ -287,6 +289,7 @@ impl<'a> RecursivePageTable<'a> { Ok(MapperFlush::new(page)) } + #[cfg(feature = "experimental")] unsafe fn next_table_fn_create_next_table<'b, A, S>( (insert_flags, allocator): &mut (PageTableFlags, &mut A), entry: &'b mut PageTableEntry, @@ -299,6 +302,7 @@ impl<'a> RecursivePageTable<'a> { unsafe { Self::create_next_table(entry, page, *insert_flags, *allocator) } } + #[cfg(feature = "experimental")] unsafe fn next_table_fn_next_table_mut<'b, I>( _: &mut I, _: &'b mut PageTableEntry, @@ -307,6 +311,7 @@ impl<'a> RecursivePageTable<'a> { Ok(unsafe { &mut *page.start_address().as_mut_ptr() }) } + #[cfg(feature = "experimental")] fn modify_range_1gib( &mut self, pages: PageRange, @@ -372,6 +377,7 @@ impl<'a> RecursivePageTable<'a> { }) } + #[cfg(feature = "experimental")] #[inline] fn map_to_range_1gib( &mut self, @@ -400,6 +406,7 @@ impl<'a> RecursivePageTable<'a> { ) } + #[cfg(feature = "experimental")] fn modify_range_2mib( &mut self, pages: PageRange, @@ -513,6 +520,7 @@ impl<'a> RecursivePageTable<'a> { }) } + #[cfg(feature = "experimental")] #[inline] fn map_to_range_2mib( &mut self, @@ -541,6 +549,7 @@ impl<'a> RecursivePageTable<'a> { ) } + #[cfg(feature = "experimental")] fn modify_range_4kib( &mut self, pages: PageRange, @@ -695,6 +704,7 @@ impl<'a> RecursivePageTable<'a> { }) } + #[cfg(feature = "experimental")] #[inline] fn map_to_range_4kib( &mut self, @@ -740,6 +750,7 @@ impl<'a> Mapper for RecursivePageTable<'a> { self.map_to_1gib(page, frame, flags, parent_table_flags, allocator) } + #[cfg(feature = "experimental")] #[inline] unsafe fn map_to_range_with_table_flags( &mut self, @@ -766,6 +777,7 @@ impl<'a> Mapper for RecursivePageTable<'a> { ) } + #[cfg(feature = "experimental")] #[inline] unsafe fn map_range_with_table_flags( &mut self, @@ -817,6 +829,7 @@ impl<'a> Mapper for RecursivePageTable<'a> { Ok((frame, MapperFlush::new(page))) } + #[cfg(feature = "experimental")] #[inline] unsafe fn unmap_range( &mut self, @@ -868,6 +881,7 @@ impl<'a> Mapper for RecursivePageTable<'a> { Ok(MapperFlush::new(page)) } + #[cfg(feature = "experimental")] #[inline] unsafe fn update_flags_range( &mut self, @@ -957,6 +971,7 @@ impl<'a> Mapper for RecursivePageTable<'a> { self.map_to_2mib(page, frame, flags, parent_table_flags, allocator) } + #[cfg(feature = "experimental")] #[inline] unsafe fn map_to_range_with_table_flags( &mut self, @@ -983,6 +998,7 @@ impl<'a> Mapper for RecursivePageTable<'a> { ) } + #[cfg(feature = "experimental")] #[inline] unsafe fn map_range_with_table_flags( &mut self, @@ -1040,6 +1056,7 @@ impl<'a> Mapper for RecursivePageTable<'a> { Ok((frame, MapperFlush::new(page))) } + #[cfg(feature = "experimental")] #[inline] unsafe fn unmap_range( &mut self, @@ -1098,6 +1115,7 @@ impl<'a> Mapper for RecursivePageTable<'a> { Ok(MapperFlush::new(page)) } + #[cfg(feature = "experimental")] #[inline] unsafe fn update_flags_range( &mut self, @@ -1209,6 +1227,7 @@ impl<'a> Mapper for RecursivePageTable<'a> { self.map_to_4kib(page, frame, flags, parent_table_flags, allocator) } + #[cfg(feature = "experimental")] #[inline] unsafe fn map_to_range_with_table_flags( &mut self, @@ -1235,6 +1254,7 @@ impl<'a> Mapper for RecursivePageTable<'a> { ) } + #[cfg(feature = "experimental")] #[inline] unsafe fn map_range_with_table_flags( &mut self, @@ -1292,6 +1312,7 @@ impl<'a> Mapper for RecursivePageTable<'a> { Ok((frame, MapperFlush::new(page))) } + #[cfg(feature = "experimental")] #[inline] unsafe fn unmap_range( &mut self, @@ -1357,6 +1378,7 @@ impl<'a> Mapper for RecursivePageTable<'a> { Ok(MapperFlush::new(page)) } + #[cfg(feature = "experimental")] #[inline] unsafe fn update_flags_range( &mut self, From 532244eca00241ccda9ad30425ac7229433a471b Mon Sep 17 00:00:00 2001 From: Tom Dohrmann Date: Wed, 20 Oct 2021 16:37:24 +0200 Subject: [PATCH 17/18] check if entry contains a frame --- src/structures/paging/mapper/mod.rs | 20 +++++++++++++++++++ .../paging/mapper/recursive_page_table.rs | 5 +++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/structures/paging/mapper/mod.rs b/src/structures/paging/mapper/mod.rs index 382ee375a..286b422db 100644 --- a/src/structures/paging/mapper/mod.rs +++ b/src/structures/paging/mapper/mod.rs @@ -18,6 +18,8 @@ use crate::structures::paging::{ }; use crate::{PhysAddr, VirtAddr}; +use super::page_table::FrameError; + mod mapped_page_table; mod offset_page_table; #[cfg(feature = "instructions")] @@ -804,6 +806,15 @@ pub enum UnmapError { InvalidFrameAddress(PhysAddr), } +impl From for UnmapError { + fn from(e: FrameError) -> Self { + match e { + FrameError::FrameNotPresent => Self::PageNotMapped, + FrameError::HugeFrame => Self::ParentEntryHugePage, + } + } +} + impl From for UnmapError { fn from(i: Infallible) -> Self { match i {} @@ -820,6 +831,15 @@ pub enum FlagUpdateError { ParentEntryHugePage, } +impl From for FlagUpdateError { + fn from(e: FrameError) -> Self { + match e { + FrameError::FrameNotPresent => Self::PageNotMapped, + FrameError::HugeFrame => Self::ParentEntryHugePage, + } + } +} + impl From for FlagUpdateError { fn from(i: Infallible) -> Self { match i {} diff --git a/src/structures/paging/mapper/recursive_page_table.rs b/src/structures/paging/mapper/recursive_page_table.rs index 96e4823bb..246b7e2eb 100644 --- a/src/structures/paging/mapper/recursive_page_table.rs +++ b/src/structures/paging/mapper/recursive_page_table.rs @@ -305,9 +305,10 @@ impl<'a> RecursivePageTable<'a> { #[cfg(feature = "experimental")] unsafe fn next_table_fn_next_table_mut<'b, I>( _: &mut I, - _: &'b mut PageTableEntry, + e: &'b mut PageTableEntry, page: Page, - ) -> Result<&'b mut PageTable, Infallible> { + ) -> Result<&'b mut PageTable, FrameError> { + e.frame()?; Ok(unsafe { &mut *page.start_address().as_mut_ptr() }) } From e46d8d833d376a633b19f3fcb5ea4af1d68a7785 Mon Sep 17 00:00:00 2001 From: Tom Dohrmann Date: Wed, 20 Oct 2021 18:09:27 +0200 Subject: [PATCH 18/18] remove unused import --- src/structures/paging/mapper/recursive_page_table.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/structures/paging/mapper/recursive_page_table.rs b/src/structures/paging/mapper/recursive_page_table.rs index 246b7e2eb..e7acafbbb 100644 --- a/src/structures/paging/mapper/recursive_page_table.rs +++ b/src/structures/paging/mapper/recursive_page_table.rs @@ -1,7 +1,5 @@ //! Access the page tables through a recursively mapped level 4 table. -#[cfg(feature = "experimental")] -use core::convert::Infallible; use core::fmt; use super::*;