diff --git a/Cargo.toml b/Cargo.toml index c261ada52..716047c14 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,6 +39,7 @@ nightly = [ "const_fn", "step_trait", "abi_x86_interrupt", "doc_cfg" ] abi_x86_interrupt = [] const_fn = [] step_trait = [] +experimental = [] doc_cfg = [] # These features are no longer used and only there for backwards compatibility. diff --git a/src/structures/paging/frame.rs b/src/structures/paging/frame.rs index 64935caee..57f58d38d 100644 --- a/src/structures/paging/frame.rs +++ b/src/structures/paging/frame.rs @@ -161,6 +161,15 @@ impl Iterator for PhysFrameRange { None } } + + #[inline] + fn count(self) -> usize { + if !self.is_empty() { + ((self.end.start_address() - self.start.start_address()) / S::SIZE) as usize + } else { + 0 + } + } } impl fmt::Debug for PhysFrameRange { diff --git a/src/structures/paging/mapper/mapped_page_table.rs b/src/structures/paging/mapper/mapped_page_table.rs index 6a3aa9f4b..394e112d6 100644 --- a/src/structures/paging/mapper/mapped_page_table.rs +++ b/src/structures/paging/mapper/mapped_page_table.rs @@ -1,3 +1,5 @@ +#[cfg(feature = "experimental")] +use crate::structures::paging::PageTableIndex; use crate::structures::paging::{ frame::PhysFrame, frame_alloc::{FrameAllocator, FrameDeallocator}, @@ -140,6 +142,426 @@ impl<'a, P: PageTableFrameMapping> MappedPageTable<'a, P> { Ok(MapperFlush::new(page)) } + + #[cfg(feature = "experimental")] + #[inline] + fn next_table_fn_create_next_table<'b, A>( + (flags, allocator): &mut (PageTableFlags, &mut A), + entry: &'b mut PageTableEntry, + walker: &PageTableWalker

, + ) -> Result<&'b mut PageTable, PageTableCreateError> + where + A: FrameAllocator + ?Sized, + { + walker + .create_next_table(entry, *flags, *allocator) + .map_err(Into::into) + } + + #[cfg(feature = "experimental")] + #[inline] + fn next_table_fn_next_table_mut<'b, T>( + _: &mut T, + entry: &'b mut PageTableEntry, + walker: &PageTableWalker

, + ) -> Result<&'b mut PageTable, PageTableWalkError> { + walker.next_table_mut(entry) + } + + #[cfg(feature = "experimental")] + fn modify_range_1gib( + &mut self, + pages: PageRange, + modify: ModifyFn, + mut info: ModifyInfo, + next_table: NextTableFn, + ) -> Result, (Err, MapperFlushRange)> + where + ModifyFn: Fn(&mut PageTableEntry, Page, &mut ModifyInfo) -> Result<(), Err>, + NextTableFn: for<'b> Fn( + &mut ModifyInfo, + &'b mut PageTableEntry, + &PageTableWalker

, + ) -> Result<&'b mut PageTable, NextTableFnErr>, + NextTableFnErr: Into, + { + if pages.is_empty() { + return Ok(MapperFlushRange::empty()); + } + + let p4 = &mut self.level_4_table; + let page_table_walker = &mut self.page_table_walker; + + (pages.start.p4_index().into()..=pages.end.p4_index().into()) + .map(PageTableIndex::new) + .try_for_each(|p4_index| { + let p4_start = Page::from_page_table_indices_1gib(p4_index, PageTableIndex::new(0)); + let p4_start = p4_start.max(pages.start); + let p4_end = Page::from_page_table_indices_1gib(p4_index, PageTableIndex::new(511)); + let p4_end = p4_end.min(pages.end); + + if p4_start == p4_end { + return Ok(()); + } + + let p3 = next_table(&mut info, &mut p4[p4_index], page_table_walker) + .map_err(|e| (e.into(), p4_start))?; + + let start_p3_index = p4_start.p3_index().into(); + let mut end_p3_index = p4_end.p3_index().into(); + + if p4_end != pages.end { + end_p3_index += 1; + } + + (start_p3_index..end_p3_index) + .map(PageTableIndex::new) + .map(move |p3_index| Page::from_page_table_indices_1gib(p4_index, p3_index)) + .try_for_each(|page| { + let entry = &mut p3[page.p3_index()]; + modify(entry, page, &mut info).map_err(|e| (e, page)) + }) + }) + .map(|_| MapperFlushRange::new(pages)) + .map_err(|(e, page)| { + ( + e, + MapperFlushRange::new(PageRange { + start: pages.start, + end: page, + }), + ) + }) + } + + #[cfg(feature = "experimental")] + #[inline] + fn map_to_range_1gib( + &mut self, + pages: PageRange, + frames: F, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + F: Fn(Page, &mut A) -> Option>, + A: FrameAllocator + ?Sized, + { + self.modify_range_1gib( + pages, + |entry, page, (_, allocator)| { + let frame = frames(page, allocator).ok_or(MapToError::FrameAllocationFailed)?; + if !entry.is_unused() { + return Err(MapToError::PageAlreadyMapped(frame)); + } + entry.set_addr(frame.start_address(), flags | PageTableFlags::HUGE_PAGE); + Ok(()) + }, + (parent_table_flags, allocator), + Self::next_table_fn_create_next_table, + ) + } + + #[cfg(feature = "experimental")] + fn modify_range_2mib( + &mut self, + pages: PageRange, + modify: ModifyFn, + mut info: ModifyInfo, + next_table: NextTableFn, + ) -> Result, (Err, MapperFlushRange)> + where + ModifyFn: Fn(&mut PageTableEntry, Page, &mut ModifyInfo) -> Result<(), Err>, + NextTableFn: for<'b> Fn( + &mut ModifyInfo, + &'b mut PageTableEntry, + &PageTableWalker

, + ) -> Result<&'b mut PageTable, NextTableFnErr>, + NextTableFnErr: Into, + { + if pages.is_empty() { + return Ok(MapperFlushRange::empty()); + } + + let p4 = &mut self.level_4_table; + let page_table_walker = &mut self.page_table_walker; + + (pages.start.p4_index().into()..=pages.end.p4_index().into()) + .map(PageTableIndex::new) + .try_for_each(|p4_index| { + let p4_start = Page::from_page_table_indices_2mib( + p4_index, + PageTableIndex::new(0), + PageTableIndex::new(0), + ); + let p4_start = p4_start.max(pages.start); + let p4_end = Page::from_page_table_indices_2mib( + p4_index, + PageTableIndex::new(511), + PageTableIndex::new(511), + ); + let p4_end = p4_end.min(pages.end); + + if p4_start == p4_end { + return Ok(()); + } + + let p3 = next_table(&mut info, &mut p4[p4_index], page_table_walker) + .map_err(|e| (e.into(), p4_start))?; + + let start_p3_index = p4_start.p3_index(); + let end_p3_index = p4_end.p3_index(); + + (start_p3_index.into()..=end_p3_index.into()) + .map(PageTableIndex::new) + .try_for_each(|p3_index| { + let p3_start = Page::from_page_table_indices_2mib( + p4_index, + p3_index, + PageTableIndex::new(0), + ); + let p3_start = p3_start.max(p4_start); + let p3_end = Page::from_page_table_indices_2mib( + p4_index, + p3_index, + PageTableIndex::new(511), + ); + let p3_end = p3_end.min(p4_end); + + if p3_start == p3_end { + return Ok(()); + } + + let p2 = next_table(&mut info, &mut p3[p3_index], page_table_walker) + .map_err(|e| (e.into(), p3_start))?; + + let start_p2_index = p3_start.p2_index().into(); + let mut end_p2_index = p3_end.p2_index().into(); + + if p3_end != pages.end { + end_p2_index += 1; + } + + (start_p2_index..end_p2_index) + .map(PageTableIndex::new) + .map(move |p2_index| { + Page::from_page_table_indices_2mib(p4_index, p3_index, p2_index) + }) + .try_for_each(|page| { + let entry = &mut p2[page.p2_index()]; + modify(entry, page, &mut info).map_err(|e| (e, page)) + }) + }) + }) + .map(|_| MapperFlushRange::new(pages)) + .map_err(|(e, page)| { + ( + e, + MapperFlushRange::new(PageRange { + start: pages.start, + end: page, + }), + ) + }) + } + + #[cfg(feature = "experimental")] + #[inline] + fn map_to_range_2mib( + &mut self, + pages: PageRange, + frames: F, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + F: Fn(Page, &mut A) -> Option>, + A: FrameAllocator + ?Sized, + { + self.modify_range_2mib( + pages, + |entry, page, (_, allocator)| { + let frame = frames(page, allocator).ok_or(MapToError::FrameAllocationFailed)?; + if !entry.is_unused() { + return Err(MapToError::PageAlreadyMapped(frame)); + } + entry.set_addr(frame.start_address(), flags | PageTableFlags::HUGE_PAGE); + Ok(()) + }, + (parent_table_flags, allocator), + Self::next_table_fn_create_next_table, + ) + } + + #[cfg(feature = "experimental")] + fn modify_range_4kib( + &mut self, + pages: PageRange, + modify: ModifyFn, + mut info: ModifyInfo, + next_table: NextTableFn, + ) -> Result, (Err, MapperFlushRange)> + where + ModifyFn: Fn(&mut PageTableEntry, Page, &mut ModifyInfo) -> Result<(), Err>, + NextTableFn: for<'b> Fn( + &mut ModifyInfo, + &'b mut PageTableEntry, + &PageTableWalker

, + ) -> Result<&'b mut PageTable, NextTableFnErr>, + NextTableFnErr: Into, + { + if pages.is_empty() { + return Ok(MapperFlushRange::empty()); + } + + let p4 = &mut self.level_4_table; + let page_table_walker = &mut self.page_table_walker; + + (pages.start.p4_index().into()..=pages.end.p4_index().into()) + .map(PageTableIndex::new) + .try_for_each(|p4_index| { + let p4_start = Page::from_page_table_indices( + p4_index, + PageTableIndex::new(0), + PageTableIndex::new(0), + PageTableIndex::new(0), + ); + let p4_start = p4_start.max(pages.start); + let p4_end = Page::from_page_table_indices( + p4_index, + PageTableIndex::new(511), + PageTableIndex::new(511), + PageTableIndex::new(511), + ); + let p4_end = p4_end.min(pages.end); + + if p4_start == p4_end { + return Ok(()); + } + + let p3 = next_table(&mut info, &mut p4[p4_index], page_table_walker) + .map_err(|e| (e.into(), p4_start))?; + + let start_p3_index = p4_start.p3_index(); + let end_p3_index = p4_end.p3_index(); + + (start_p3_index.into()..=end_p3_index.into()) + .map(PageTableIndex::new) + .try_for_each(|p3_index| { + let p3_start = Page::from_page_table_indices( + p4_index, + p3_index, + PageTableIndex::new(0), + PageTableIndex::new(0), + ); + let p3_start = p3_start.max(p4_start); + let p3_end = Page::from_page_table_indices( + p4_index, + p3_index, + PageTableIndex::new(511), + PageTableIndex::new(511), + ); + let p3_end = p3_end.min(p4_end); + + if p3_start == p3_end { + return Ok(()); + } + + let p2 = next_table(&mut info, &mut p3[p3_index], page_table_walker) + .map_err(|e| (e.into(), p3_start))?; + + let start_p2_index = p3_start.p2_index(); + let end_p2_index = p3_end.p2_index(); + + (start_p2_index.into()..=end_p2_index.into()) + .map(PageTableIndex::new) + .try_for_each(|p2_index| { + let p2_start = Page::from_page_table_indices( + p4_index, + p3_index, + p2_index, + PageTableIndex::new(0), + ); + let p2_start = p2_start.max(p4_start); + let p2_end = Page::from_page_table_indices( + p4_index, + p3_index, + p2_index, + PageTableIndex::new(511), + ); + let p2_end = p2_end.min(p4_end); + + if p2_start == p2_end { + return Ok(()); + } + + let p1 = + next_table(&mut info, &mut p2[p2_index], page_table_walker) + .map_err(|e| (e.into(), p2_start))?; + + let start_p1_index = p2_start.p1_index().into(); + let mut end_p1_index = p2_end.p1_index().into(); + + if p2_end != pages.end { + end_p1_index += 1; + } + + (start_p1_index..end_p1_index) + .map(PageTableIndex::new) + .map(move |p1_index| { + Page::from_page_table_indices( + p4_index, p3_index, p2_index, p1_index, + ) + }) + .try_for_each(|page| { + let entry = &mut p1[page.p1_index()]; + modify(entry, page, &mut info).map_err(|e| (e, page)) + }) + }) + }) + }) + .map(|_| MapperFlushRange::new(pages)) + .map_err(|(e, page)| { + ( + e, + MapperFlushRange::new(PageRange { + start: pages.start, + end: page, + }), + ) + }) + } + + #[cfg(feature = "experimental")] + #[inline] + fn map_to_range_4kib( + &mut self, + pages: PageRange, + frames: F, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + F: Fn(Page, &mut A) -> Option>, + A: FrameAllocator + ?Sized, + { + self.modify_range_4kib( + pages, + |entry, page, (_, allocator)| { + let frame = frames(page, allocator).ok_or(MapToError::FrameAllocationFailed)?; + if !entry.is_unused() { + return Err(MapToError::PageAlreadyMapped(frame)); + } + entry.set_addr(frame.start_address(), flags); + Ok(()) + }, + (parent_table_flags, allocator), + Self::next_table_fn_create_next_table, + ) + } } impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { @@ -158,6 +580,55 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { self.map_to_1gib(page, frame, flags, parent_table_flags, allocator) } + #[cfg(feature = "experimental")] + #[inline] + unsafe fn map_to_range_with_table_flags( + &mut self, + pages: PageRange, + frames: PhysFrameRange, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + Self: Sized, + A: FrameAllocator + ?Sized, + { + assert_eq!(pages.count(), frames.count()); + self.map_to_range_1gib( + pages, + |page, _| { + let offset = page - pages.start; + Some(frames.start + offset) + }, + flags, + parent_table_flags, + allocator, + ) + } + + #[cfg(feature = "experimental")] + #[inline] + unsafe fn map_range_with_table_flags( + &mut self, + pages: PageRange, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + Self: Sized, + A: FrameAllocator + FrameAllocator + ?Sized, + { + self.map_to_range_1gib( + pages, + |_, allocator| allocator.allocate_frame(), + flags, + parent_table_flags, + allocator, + ) + } + fn unmap( &mut self, page: Page, @@ -184,6 +655,34 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { Ok((frame, MapperFlush::new(page))) } + #[cfg(feature = "experimental")] + #[inline] + unsafe fn unmap_range( + &mut self, + pages: PageRange, + deallocator: &mut D, + ) -> Result, (UnmapError, MapperFlushRange)> + where + Self: Sized, + D: FrameDeallocator + ?Sized, + { + self.modify_range_1gib( + pages, + |entry, _, deallocator| { + let frame = PhysFrame::from_start_address(entry.addr()) + .map_err(|AddressNotAligned| UnmapError::InvalidFrameAddress(entry.addr()))?; + unsafe { + deallocator.deallocate_frame(frame); + } + + entry.set_unused(); + Ok(()) + }, + deallocator, + Self::next_table_fn_next_table_mut, + ) + } + unsafe fn update_flags( &mut self, page: Page, @@ -202,6 +701,28 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { Ok(MapperFlush::new(page)) } + #[cfg(feature = "experimental")] + #[inline] + unsafe fn update_flags_range( + &mut self, + pages: PageRange, + flags: PageTableFlags, + ) -> Result, (FlagUpdateError, MapperFlushRange)> { + self.modify_range_1gib( + pages, + |entry, _, _| { + if entry.is_unused() { + return Err(FlagUpdateError::PageNotMapped); + } + + entry.set_flags(flags); + Ok(()) + }, + (), + Self::next_table_fn_next_table_mut, + ) + } + unsafe fn set_flags_p4_entry( &mut self, page: Page, @@ -266,6 +787,55 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { self.map_to_2mib(page, frame, flags, parent_table_flags, allocator) } + #[cfg(feature = "experimental")] + #[inline] + unsafe fn map_to_range_with_table_flags( + &mut self, + pages: PageRange, + frames: PhysFrameRange, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + Self: Sized, + A: FrameAllocator + ?Sized, + { + assert_eq!(pages.count(), frames.count()); + self.map_to_range_2mib( + pages, + |page, _| { + let offset = page - pages.start; + Some(frames.start + offset) + }, + flags, + parent_table_flags, + allocator, + ) + } + + #[cfg(feature = "experimental")] + #[inline] + unsafe fn map_range_with_table_flags( + &mut self, + pages: PageRange, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + Self: Sized, + A: FrameAllocator + FrameAllocator + ?Sized, + { + self.map_to_range_2mib( + pages, + |_, allocator| allocator.allocate_frame(), + flags, + parent_table_flags, + allocator, + ) + } + fn unmap( &mut self, page: Page, @@ -295,6 +865,34 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { Ok((frame, MapperFlush::new(page))) } + #[cfg(feature = "experimental")] + #[inline] + unsafe fn unmap_range( + &mut self, + pages: PageRange, + deallocator: &mut D, + ) -> Result, (UnmapError, MapperFlushRange)> + where + Self: Sized, + D: FrameDeallocator + ?Sized, + { + self.modify_range_2mib( + pages, + |entry, _, deallocator| { + let frame = PhysFrame::from_start_address(entry.addr()) + .map_err(|AddressNotAligned| UnmapError::InvalidFrameAddress(entry.addr()))?; + unsafe { + deallocator.deallocate_frame(frame); + } + + entry.set_unused(); + Ok(()) + }, + deallocator, + Self::next_table_fn_next_table_mut, + ) + } + unsafe fn update_flags( &mut self, page: Page, @@ -317,6 +915,28 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { Ok(MapperFlush::new(page)) } + #[cfg(feature = "experimental")] + #[inline] + unsafe fn update_flags_range( + &mut self, + pages: PageRange, + flags: PageTableFlags, + ) -> Result, (FlagUpdateError, MapperFlushRange)> { + self.modify_range_2mib( + pages, + |entry, _, _| { + if entry.is_unused() { + return Err(FlagUpdateError::PageNotMapped); + } + + entry.set_flags(flags); + Ok(()) + }, + (), + Self::next_table_fn_next_table_mut, + ) + } + unsafe fn set_flags_p4_entry( &mut self, page: Page, @@ -394,6 +1014,54 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { self.map_to_4kib(page, frame, flags, parent_table_flags, allocator) } + #[cfg(feature = "experimental")] + #[inline] + unsafe fn map_to_range_with_table_flags( + &mut self, + pages: PageRange, + frames: PhysFrameRange, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + Self: Sized, + A: FrameAllocator + ?Sized, + { + assert_eq!(pages.count(), frames.count()); + self.map_to_range_4kib( + pages, + |page, _| { + let offset = page - pages.start; + Some(frames.start + offset) + }, + flags, + parent_table_flags, + allocator, + ) + } + + #[cfg(feature = "experimental")] + #[inline] + unsafe fn map_range_with_table_flags( + &mut self, + pages: PageRange, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + A: FrameAllocator + ?Sized, + { + self.map_to_range_4kib( + pages, + |_, allocator| allocator.allocate_frame(), + flags, + parent_table_flags, + allocator, + ) + } + fn unmap( &mut self, page: Page, @@ -420,6 +1088,36 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { Ok((frame, MapperFlush::new(page))) } + #[cfg(feature = "experimental")] + #[inline] + unsafe fn unmap_range( + &mut self, + pages: PageRange, + deallocator: &mut D, + ) -> Result, (UnmapError, MapperFlushRange)> + where + Self: Sized, + D: FrameDeallocator + ?Sized, + { + self.modify_range_4kib( + pages, + |entry, _, deallocator| { + let frame = entry.frame().map_err(|err| match err { + FrameError::FrameNotPresent => UnmapError::PageNotMapped, + FrameError::HugeFrame => UnmapError::ParentEntryHugePage, + })?; + unsafe { + deallocator.deallocate_frame(frame); + } + + entry.set_unused(); + Ok(()) + }, + deallocator, + Self::next_table_fn_next_table_mut, + ) + } + unsafe fn update_flags( &mut self, page: Page, @@ -445,6 +1143,28 @@ impl<'a, P: PageTableFrameMapping> Mapper for MappedPageTable<'a, P> { Ok(MapperFlush::new(page)) } + #[cfg(feature = "experimental")] + #[inline] + unsafe fn update_flags_range( + &mut self, + pages: PageRange, + flags: PageTableFlags, + ) -> Result, (FlagUpdateError, MapperFlushRange)> { + self.modify_range_4kib( + pages, + |entry, _, _| { + if entry.is_unused() { + return Err(FlagUpdateError::PageNotMapped); + } + + entry.set_flags(flags); + Ok(()) + }, + (), + Self::next_table_fn_next_table_mut, + ) + } + unsafe fn set_flags_p4_entry( &mut self, page: Page, diff --git a/src/structures/paging/mapper/mod.rs b/src/structures/paging/mapper/mod.rs index f971b9ebc..286b422db 100644 --- a/src/structures/paging/mapper/mod.rs +++ b/src/structures/paging/mapper/mod.rs @@ -6,6 +6,10 @@ pub use self::offset_page_table::OffsetPageTable; #[cfg(feature = "instructions")] pub use self::recursive_page_table::{InvalidPageTable, RecursivePageTable}; +use core::convert::Infallible; + +#[cfg(feature = "experimental")] +use crate::structures::paging::{frame::PhysFrameRange, page::PageRange}; use crate::structures::paging::{ frame_alloc::{FrameAllocator, FrameDeallocator}, page::PageRangeInclusive, @@ -14,6 +18,8 @@ use crate::structures::paging::{ }; use crate::{PhysAddr, VirtAddr}; +use super::page_table::FrameError; + mod mapped_page_table; mod offset_page_table; #[cfg(feature = "instructions")] @@ -197,6 +203,49 @@ pub trait Mapper { } } + #[cfg(feature = "experimental")] + /// Maps the given range of frames to the range of virtual pages. + /// + /// ## Safety + /// + /// This is a convencience function that invokes [`Mapper::map_to`] internally, so + /// all safety requirements of it also apply for this function. + /// + /// ## Panics + /// + /// This function panics if the amount of pages does not equal the amount of frames. + /// + /// ## Errors + /// + /// If an error occurs half-way through a [`MapperFlushRange`] is returned that contains the frames that were successfully mapped. + #[inline] + unsafe fn map_to_range( + &mut self, + pages: PageRange, + frames: PhysFrameRange, + flags: PageTableFlags, + frame_allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + Self: Sized, + A: FrameAllocator + ?Sized, + { + let parent_table_flags = flags + & (PageTableFlags::PRESENT + | PageTableFlags::WRITABLE + | PageTableFlags::USER_ACCESSIBLE); + + unsafe { + self.map_to_range_with_table_flags( + pages, + frames, + flags, + parent_table_flags, + frame_allocator, + ) + } + } + /// Creates a new mapping in the page table. /// /// This function might need additional physical frames to create new page tables. These @@ -279,11 +328,192 @@ pub trait Mapper { Self: Sized, A: FrameAllocator + ?Sized; + #[cfg(feature = "experimental")] + /// Maps the given range of frames to the range of virtual pages. + /// + /// ## Safety + /// + /// This is a convencience function that invokes [`Mapper::map_to_with_table_flags`] internally, so + /// all safety requirements of it also apply for this function. + /// + /// ## Panics + /// + /// This function panics if the amount of pages does not equal the amount of frames. + /// + /// ## Errors + /// + /// If an error occurs half-way through a [`MapperFlushRange`] is returned that contains the frames that were successfully mapped. + unsafe fn map_to_range_with_table_flags( + &mut self, + pages: PageRange, + frames: PhysFrameRange, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + frame_allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + Self: Sized, + A: FrameAllocator + ?Sized, + { + assert_eq!(pages.count(), frames.count()); + + pages + .zip(frames) + .try_for_each(|(page, frame)| { + unsafe { + self.map_to_with_table_flags( + page, + frame, + flags, + parent_table_flags, + frame_allocator, + ) + } + .map(|_| ()) + .map_err(|e| { + ( + e, + MapperFlushRange::new(PageRange { + start: pages.start, + end: page, + }), + ) + }) + }) + .map(|_| MapperFlushRange::new(pages)) + } + + #[cfg(feature = "experimental")] + /// Maps frames from the allocator to the given range of virtual pages. + /// + /// ## Safety + /// + /// This function invokes [`Mapper::map_to_with_table_flags`] internally, so + /// all safety requirements of it also apply for this function. + /// + /// ## Errors + /// + /// If an error occurs half-way through a [`MapperFlushRange`] is returned that contains the frames that were successfully mapped. + unsafe fn map_range_with_table_flags( + &mut self, + mut pages: PageRange, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + frame_allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + Self: Sized, + A: FrameAllocator + FrameAllocator + ?Sized, + { + pages + .try_for_each(|page| { + let frame = frame_allocator + .allocate_frame() + .ok_or((MapToError::FrameAllocationFailed, page))?; + + unsafe { + // SAFETY: frame was just freshly allocated and therefore can't cause aliasing issues + self.map_to_with_table_flags( + page, + frame, + flags, + parent_table_flags, + frame_allocator, + ) + } + .map(|_| ()) + .map_err(|e| (e, page)) + }) + .map(|_| MapperFlushRange::new(pages)) + .map_err(|(e, page)| { + ( + e, + MapperFlushRange::new(PageRange { + start: pages.start, + end: page, + }), + ) + }) + } + + #[cfg(feature = "experimental")] + /// Maps frames from the allocator to the given range of virtual pages. + /// + /// ## Safety + /// + /// This function invokes [`Mapper::map_to_with_table_flags`] internally, so + /// all safety requirements of it also apply for this function. + /// + /// ## Errors + /// + /// If an error occurs half-way through a [`MapperFlushRange`] is returned that contains the frames that were successfully mapped. + #[inline] + unsafe fn map_range( + &mut self, + pages: PageRange, + flags: PageTableFlags, + frame_allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + Self: Sized, + A: FrameAllocator + FrameAllocator + ?Sized, + { + let parent_table_flags = flags + & (PageTableFlags::PRESENT + | PageTableFlags::WRITABLE + | PageTableFlags::USER_ACCESSIBLE); + + unsafe { + self.map_range_with_table_flags(pages, flags, parent_table_flags, frame_allocator) + } + } + /// Removes a mapping from the page table and returns the frame that used to be mapped. /// /// Note that no page tables or pages are deallocated. fn unmap(&mut self, page: Page) -> Result<(PhysFrame, MapperFlush), UnmapError>; + #[cfg(feature = "experimental")] + /// Removes a range of mapping from the page table and deallocate the frames that used to be mapped. + /// + /// Note that no page tables or pages are deallocated. + /// + /// ## Errors + /// If an error occurs half-way through a [`MapperFlushRange`] is returned that contains the pages that were successfully unmapped. + /// + /// ## Safety + /// The caller has to guarantee that it's safe to deallocate frames: + /// All unmapped frames must only be in this page table. + unsafe fn unmap_range( + &mut self, + pages: PageRange, + deallocator: &mut D, + ) -> Result, (UnmapError, MapperFlushRange)> + where + Self: Sized, + D: FrameDeallocator + ?Sized, + { + pages + .clone() + .try_for_each(|page| { + let (frame, _) = self.unmap(page).map_err(|e| { + ( + e, + MapperFlushRange::new(PageRange { + start: pages.start, + end: page, + }), + ) + })?; + unsafe { + // SAFETY: the page has been unmapped so the frame is unused + deallocator.deallocate_frame(frame); + } + Ok(()) + }) + .map(|_| MapperFlushRange::new(pages)) + } + /// Updates the flags of an existing mapping. /// /// ## Safety @@ -299,6 +529,42 @@ pub trait Mapper { flags: PageTableFlags, ) -> Result, FlagUpdateError>; + #[cfg(feature = "experimental")] + /// Updates the flags of a range of existing mappings. + /// + /// ## Safety + /// + /// This method is unsafe because changing the flags of a mapping + /// might result in undefined behavior. For example, setting the + /// `GLOBAL` and `WRITABLE` flags for a page might result in the corruption + /// of values stored in that page from processes running in other address + /// spaces. + /// + /// ## Errors + /// If an error occurs half-way through a [`MapperFlushRange`] is returned that contains the pages that were successfully updated. + unsafe fn update_flags_range( + &mut self, + pages: PageRange, + flags: PageTableFlags, + ) -> Result, (FlagUpdateError, MapperFlushRange)> { + pages + .clone() + .try_for_each(|page| { + unsafe { self.update_flags(page, flags) } + .map(|_| ()) + .map_err(|e| { + ( + e, + MapperFlushRange::new(PageRange { + start: pages.start, + end: page, + }), + ) + }) + }) + .map(|_| MapperFlushRange::new(pages)) + } + /// Set the flags of an existing page level 4 table entry /// /// ## Safety @@ -372,6 +638,32 @@ pub trait Mapper { let page = Page::containing_address(VirtAddr::new(frame.start_address().as_u64())); unsafe { self.map_to(page, frame, flags, frame_allocator) } } + + #[cfg(feature = "experimental")] + /// Maps the given range of frames to the range of virtual pages with the same address. + /// + /// ## Safety + /// + /// This is a convencience function that invokes [`Mapper::map_to_range`] internally, so + /// all safety requirements of it also apply for this function. + #[inline] + unsafe fn identity_map_range( + &mut self, + frames: PhysFrameRange, + flags: PageTableFlags, + frame_allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + Self: Sized, + A: FrameAllocator + ?Sized, + S: PageSize, + Self: Mapper, + { + let start = Page::containing_address(VirtAddr::new(frames.start.start_address().as_u64())); + let end = Page::containing_address(VirtAddr::new(frames.end.start_address().as_u64())); + let pages = PageRange { start, end }; + unsafe { self.map_to_range(pages, frames, flags, frame_allocator) } + } } /// This type represents a page whose mapping has changed in the page table. @@ -405,6 +697,59 @@ impl MapperFlush { pub fn ignore(self) {} } +#[cfg(feature = "experimental")] +/// This type represents a range of pages whose mappings have changed in the page table. +/// +/// The old mappings might be still cached in the translation lookaside buffer (TLB), so they need +/// to be flushed from the TLB before they're accessed. This type is returned from a function that +/// changed the mappings of a range of pages to ensure that the TLB flush is not forgotten. +#[derive(Debug)] +#[must_use = "Page Table changes must be flushed or ignored."] +pub struct MapperFlushRange(PageRange); + +#[cfg(feature = "experimental")] +impl MapperFlushRange { + /// Create a new flush promise + #[inline] + fn new(pages: PageRange) -> Self { + MapperFlushRange(pages) + } + + /// Create a new empty flush promise + #[inline] + fn empty() -> Self { + MapperFlushRange::new(PageRange { + start: Page::containing_address(VirtAddr::zero()), + end: Page::containing_address(VirtAddr::zero()), + }) + } + + /// Flush the pages from the TLB to ensure that the newest mapping is used. + #[cfg(feature = "instructions")] + #[inline] + pub fn flush_range(self) { + for page in self.0 { + crate::instructions::tlb::flush(page.start_address()) + } + } + + /// Flush all pages from the TLB to ensure that the newest mapping is used. + #[cfg(feature = "instructions")] + #[inline] + pub fn flush_all(self) { + crate::instructions::tlb::flush_all() + } + + /// Don't flush the TLB and silence the “must be used” warning. + #[inline] + pub fn ignore(self) {} + + /// Get the range of changed pages. + pub fn pages(&self) -> PageRange { + self.0 + } +} + /// This type represents a change of a page table requiring a complete TLB flush /// /// The old mapping might be still cached in the translation lookaside buffer (TLB), so it needs @@ -461,6 +806,21 @@ pub enum UnmapError { InvalidFrameAddress(PhysAddr), } +impl From for UnmapError { + fn from(e: FrameError) -> Self { + match e { + FrameError::FrameNotPresent => Self::PageNotMapped, + FrameError::HugeFrame => Self::ParentEntryHugePage, + } + } +} + +impl From for UnmapError { + fn from(i: Infallible) -> Self { + match i {} + } +} + /// An error indicating that an `update_flags` call failed. #[derive(Debug)] pub enum FlagUpdateError { @@ -471,6 +831,21 @@ pub enum FlagUpdateError { ParentEntryHugePage, } +impl From for FlagUpdateError { + fn from(e: FrameError) -> Self { + match e { + FrameError::FrameNotPresent => Self::PageNotMapped, + FrameError::HugeFrame => Self::ParentEntryHugePage, + } + } +} + +impl From for FlagUpdateError { + fn from(i: Infallible) -> Self { + match i {} + } +} + /// An error indicating that an `translate` call failed. #[derive(Debug)] pub enum TranslateError { diff --git a/src/structures/paging/mapper/offset_page_table.rs b/src/structures/paging/mapper/offset_page_table.rs index 580924fe9..91748fe9c 100644 --- a/src/structures/paging/mapper/offset_page_table.rs +++ b/src/structures/paging/mapper/offset_page_table.rs @@ -77,6 +77,49 @@ impl<'a> Mapper for OffsetPageTable<'a> { } } + #[cfg(feature = "experimental")] + #[inline] + unsafe fn map_to_range_with_table_flags( + &mut self, + pages: PageRange, + frames: PhysFrameRange, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + Self: Sized, + A: FrameAllocator + ?Sized, + { + unsafe { + self.inner.map_to_range_with_table_flags( + pages, + frames, + flags, + parent_table_flags, + allocator, + ) + } + } + + #[cfg(feature = "experimental")] + #[inline] + unsafe fn map_range_with_table_flags( + &mut self, + pages: PageRange, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + A: FrameAllocator + FrameAllocator + ?Sized, + { + unsafe { + self.inner + .map_range_with_table_flags(pages, flags, parent_table_flags, allocator) + } + } + #[inline] fn unmap( &mut self, @@ -85,6 +128,20 @@ impl<'a> Mapper for OffsetPageTable<'a> { self.inner.unmap(page) } + #[cfg(feature = "experimental")] + #[inline] + unsafe fn unmap_range( + &mut self, + pages: PageRange, + deallocator: &mut D, + ) -> Result, (UnmapError, MapperFlushRange)> + where + Self: Sized, + D: FrameDeallocator + ?Sized, + { + unsafe { self.inner.unmap_range(pages, deallocator) } + } + #[inline] unsafe fn update_flags( &mut self, @@ -146,6 +203,49 @@ impl<'a> Mapper for OffsetPageTable<'a> { } } + #[cfg(feature = "experimental")] + #[inline] + unsafe fn map_to_range_with_table_flags( + &mut self, + pages: PageRange, + frames: PhysFrameRange, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + Self: Sized, + A: FrameAllocator + ?Sized, + { + unsafe { + self.inner.map_to_range_with_table_flags( + pages, + frames, + flags, + parent_table_flags, + allocator, + ) + } + } + + #[cfg(feature = "experimental")] + #[inline] + unsafe fn map_range_with_table_flags( + &mut self, + pages: PageRange, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + A: FrameAllocator + FrameAllocator + ?Sized, + { + unsafe { + self.inner + .map_range_with_table_flags(pages, flags, parent_table_flags, allocator) + } + } + #[inline] fn unmap( &mut self, @@ -154,6 +254,20 @@ impl<'a> Mapper for OffsetPageTable<'a> { self.inner.unmap(page) } + #[cfg(feature = "experimental")] + #[inline] + unsafe fn unmap_range( + &mut self, + pages: PageRange, + deallocator: &mut D, + ) -> Result, (UnmapError, MapperFlushRange)> + where + Self: Sized, + D: FrameDeallocator + ?Sized, + { + unsafe { self.inner.unmap_range(pages, deallocator) } + } + #[inline] unsafe fn update_flags( &mut self, @@ -215,6 +329,49 @@ impl<'a> Mapper for OffsetPageTable<'a> { } } + #[cfg(feature = "experimental")] + #[inline] + unsafe fn map_to_range_with_table_flags( + &mut self, + pages: PageRange, + frames: PhysFrameRange, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + Self: Sized, + A: FrameAllocator + ?Sized, + { + unsafe { + self.inner.map_to_range_with_table_flags( + pages, + frames, + flags, + parent_table_flags, + allocator, + ) + } + } + + #[cfg(feature = "experimental")] + #[inline] + unsafe fn map_range_with_table_flags( + &mut self, + pages: PageRange, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + A: FrameAllocator + ?Sized, + { + unsafe { + self.inner + .map_range_with_table_flags(pages, flags, parent_table_flags, allocator) + } + } + #[inline] fn unmap( &mut self, @@ -223,6 +380,20 @@ impl<'a> Mapper for OffsetPageTable<'a> { self.inner.unmap(page) } + #[cfg(feature = "experimental")] + #[inline] + unsafe fn unmap_range( + &mut self, + pages: PageRange, + deallocator: &mut D, + ) -> Result, (UnmapError, MapperFlushRange)> + where + Self: Sized, + D: FrameDeallocator + ?Sized, + { + unsafe { self.inner.unmap_range(pages, deallocator) } + } + #[inline] unsafe fn update_flags( &mut self, diff --git a/src/structures/paging/mapper/recursive_page_table.rs b/src/structures/paging/mapper/recursive_page_table.rs index 7dc1f6398..e7acafbbb 100644 --- a/src/structures/paging/mapper/recursive_page_table.rs +++ b/src/structures/paging/mapper/recursive_page_table.rs @@ -286,6 +286,451 @@ impl<'a> RecursivePageTable<'a> { Ok(MapperFlush::new(page)) } + + #[cfg(feature = "experimental")] + unsafe fn next_table_fn_create_next_table<'b, A, S>( + (insert_flags, allocator): &mut (PageTableFlags, &mut A), + entry: &'b mut PageTableEntry, + page: Page, + ) -> Result<&'b mut PageTable, MapToError> + where + A: FrameAllocator + ?Sized, + S: PageSize, + { + unsafe { Self::create_next_table(entry, page, *insert_flags, *allocator) } + } + + #[cfg(feature = "experimental")] + unsafe fn next_table_fn_next_table_mut<'b, I>( + _: &mut I, + e: &'b mut PageTableEntry, + page: Page, + ) -> Result<&'b mut PageTable, FrameError> { + e.frame()?; + Ok(unsafe { &mut *page.start_address().as_mut_ptr() }) + } + + #[cfg(feature = "experimental")] + fn modify_range_1gib( + &mut self, + pages: PageRange, + modify: ModifyFn, + mut info: ModifyInfo, + next_table: for<'b> unsafe fn( + &mut ModifyInfo, + &'b mut PageTableEntry, + Page, + ) -> Result<&'b mut PageTable, NextTableFnErr>, + ) -> Result, (Err, MapperFlushRange)> + where + ModifyFn: Fn(&mut PageTableEntry, Page, &mut ModifyInfo) -> Result<(), Err>, + NextTableFnErr: Into, + { + if pages.is_empty() { + return Ok(MapperFlushRange::empty()); + } + + let recursive_index = self.recursive_index; + let p4 = self.level_4_table(); + + (pages.start.p4_index().into()..=pages.end.p4_index().into()) + .map(PageTableIndex::new) + .try_for_each(|p4_index| { + let p4_start = Page::from_page_table_indices_1gib(p4_index, PageTableIndex::new(0)); + let p4_start = p4_start.max(pages.start); + let p4_end = Page::from_page_table_indices_1gib(p4_index, PageTableIndex::new(511)); + let p4_end = p4_end.min(pages.end); + + if p4_start == p4_end { + return Ok(()); + } + + let p3_page = p3_page(p4_start, recursive_index); + let p3 = unsafe { next_table(&mut info, &mut p4[p4_index], p3_page) } + .map_err(|e| (e.into(), p4_start))?; + + let start_p3_index = p4_start.p3_index().into(); + let mut end_p3_index = p4_end.p3_index().into(); + + if p4_end != pages.end { + end_p3_index += 1; + } + + (start_p3_index..end_p3_index) + .map(PageTableIndex::new) + .map(move |p3_index| Page::from_page_table_indices_1gib(p4_index, p3_index)) + .try_for_each(|page| { + let entry = &mut p3[page.p3_index()]; + modify(entry, page, &mut info).map_err(|e| (e, page)) + }) + }) + .map(|_| MapperFlushRange::new(pages)) + .map_err(|(e, page)| { + ( + e, + MapperFlushRange::new(PageRange { + start: pages.start, + end: page, + }), + ) + }) + } + + #[cfg(feature = "experimental")] + #[inline] + fn map_to_range_1gib( + &mut self, + pages: PageRange, + frames: F, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + F: Fn(Page, &mut A) -> Option>, + A: FrameAllocator + ?Sized, + { + self.modify_range_1gib( + pages, + |entry, page, (_, allocator)| { + let frame = frames(page, allocator).ok_or(MapToError::FrameAllocationFailed)?; + if !entry.is_unused() { + return Err(MapToError::PageAlreadyMapped(frame)); + } + entry.set_addr(frame.start_address(), flags | PageTableFlags::HUGE_PAGE); + Ok(()) + }, + (parent_table_flags, allocator), + Self::next_table_fn_create_next_table, + ) + } + + #[cfg(feature = "experimental")] + fn modify_range_2mib( + &mut self, + pages: PageRange, + modify: ModifyFn, + mut info: ModifyInfo, + next_table: for<'b> unsafe fn( + &mut ModifyInfo, + &'b mut PageTableEntry, + Page, + ) -> Result<&'b mut PageTable, NextTableFnErr>, + ) -> Result, (Err, MapperFlushRange)> + where + ModifyFn: Fn(&mut PageTableEntry, Page, &mut ModifyInfo) -> Result<(), Err>, + NextTableFnErr: Into, + { + if pages.is_empty() { + return Ok(MapperFlushRange::empty()); + } + + let recursive_index = self.recursive_index; + let p4 = self.level_4_table(); + + (pages.start.p4_index().into()..=pages.end.p4_index().into()) + .map(PageTableIndex::new) + .try_for_each(|p4_index| { + let p4_start = Page::from_page_table_indices_2mib( + p4_index, + PageTableIndex::new(0), + PageTableIndex::new(0), + ); + let p4_start = p4_start.max(pages.start); + let p4_end = Page::from_page_table_indices_2mib( + p4_index, + PageTableIndex::new(511), + PageTableIndex::new(511), + ); + let p4_end = p4_end.min(pages.end); + + if p4_start == p4_end { + return Ok(()); + } + + let p3 = unsafe { + next_table( + &mut info, + &mut p4[p4_index], + p3_page(p4_start, recursive_index), + ) + } + .map_err(|e| (e.into(), p4_start))?; + + let start_p3_index = p4_start.p3_index(); + let end_p3_index = p4_end.p3_index(); + + (start_p3_index.into()..=end_p3_index.into()) + .map(PageTableIndex::new) + .try_for_each(|p3_index| { + let p3_start = Page::from_page_table_indices_2mib( + p4_index, + p3_index, + PageTableIndex::new(0), + ); + let p3_start = p3_start.max(p4_start); + let p3_end = Page::from_page_table_indices_2mib( + p4_index, + p3_index, + PageTableIndex::new(511), + ); + let p3_end = p3_end.min(p4_end); + + if p3_start == p3_end { + return Ok(()); + } + + let p2 = unsafe { + next_table( + &mut info, + &mut p3[p3_index], + p2_page(p3_start, recursive_index), + ) + } + .map_err(|e| (e.into(), p3_start))?; + + let start_p2_index = p3_start.p2_index().into(); + let mut end_p2_index = p3_end.p2_index().into(); + + if p3_end != pages.end { + end_p2_index += 1; + } + + (start_p2_index..end_p2_index) + .map(PageTableIndex::new) + .map(move |p2_index| { + Page::from_page_table_indices_2mib(p4_index, p3_index, p2_index) + }) + .try_for_each(|page| { + let entry = &mut p2[page.p2_index()]; + modify(entry, page, &mut info).map_err(|e| (e, page)) + }) + }) + }) + .map(|_| MapperFlushRange::new(pages)) + .map_err(|(e, page)| { + ( + e, + MapperFlushRange::new(PageRange { + start: pages.start, + end: page, + }), + ) + }) + } + + #[cfg(feature = "experimental")] + #[inline] + fn map_to_range_2mib( + &mut self, + pages: PageRange, + frames: F, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + F: Fn(Page, &mut A) -> Option>, + A: FrameAllocator + ?Sized, + { + self.modify_range_2mib( + pages, + |entry, page, (_, allocator)| { + let frame = frames(page, allocator).ok_or(MapToError::FrameAllocationFailed)?; + if !entry.is_unused() { + return Err(MapToError::PageAlreadyMapped(frame)); + } + entry.set_addr(frame.start_address(), flags | PageTableFlags::HUGE_PAGE); + Ok(()) + }, + (parent_table_flags, allocator), + Self::next_table_fn_create_next_table, + ) + } + + #[cfg(feature = "experimental")] + fn modify_range_4kib( + &mut self, + pages: PageRange, + modify: ModifyFn, + mut info: ModifyInfo, + next_table: for<'b> unsafe fn( + &mut ModifyInfo, + &'b mut PageTableEntry, + Page, + ) -> Result<&'b mut PageTable, NextTableFnErr>, + ) -> Result, (Err, MapperFlushRange)> + where + ModifyFn: Fn(&mut PageTableEntry, Page, &mut ModifyInfo) -> Result<(), Err>, + NextTableFnErr: Into, + { + if pages.is_empty() { + return Ok(MapperFlushRange::empty()); + } + + let recursive_index = self.recursive_index; + let p4 = self.level_4_table(); + + (pages.start.p4_index().into()..=pages.end.p4_index().into()) + .map(PageTableIndex::new) + .try_for_each(|p4_index| { + let p4_start = Page::from_page_table_indices( + p4_index, + PageTableIndex::new(0), + PageTableIndex::new(0), + PageTableIndex::new(0), + ); + let p4_start = p4_start.max(pages.start); + let p4_end = Page::from_page_table_indices( + p4_index, + PageTableIndex::new(511), + PageTableIndex::new(511), + PageTableIndex::new(511), + ); + let p4_end = p4_end.min(pages.end); + + if p4_start == p4_end { + return Ok(()); + } + + let p3 = unsafe { + next_table( + &mut info, + &mut p4[p4_index], + p3_page(p4_start, recursive_index), + ) + } + .map_err(|e| (e.into(), p4_start))?; + + let start_p3_index = p4_start.p3_index(); + let end_p3_index = p4_end.p3_index(); + + (start_p3_index.into()..=end_p3_index.into()) + .map(PageTableIndex::new) + .try_for_each(|p3_index| { + let p3_start = Page::from_page_table_indices( + p4_index, + p3_index, + PageTableIndex::new(0), + PageTableIndex::new(0), + ); + let p3_start = p3_start.max(p4_start); + let p3_end = Page::from_page_table_indices( + p4_index, + p3_index, + PageTableIndex::new(511), + PageTableIndex::new(511), + ); + let p3_end = p3_end.min(p4_end); + + if p3_start == p3_end { + return Ok(()); + } + + let p2 = unsafe { + next_table( + &mut info, + &mut p3[p3_index], + p2_page(p3_start, recursive_index), + ) + } + .map_err(|e| (e.into(), p3_start))?; + + let start_p2_index = p3_start.p2_index(); + let end_p2_index = p3_end.p2_index(); + + (start_p2_index.into()..=end_p2_index.into()) + .map(PageTableIndex::new) + .try_for_each(|p2_index| { + let p2_start = Page::from_page_table_indices( + p4_index, + p3_index, + p2_index, + PageTableIndex::new(0), + ); + let p2_start = p2_start.max(p3_start); + let p2_end = Page::from_page_table_indices( + p4_index, + p3_index, + p2_index, + PageTableIndex::new(511), + ); + let p2_end = p2_end.min(p4_end); + + if p2_start == p2_end { + return Ok(()); + } + + let p1 = unsafe { + next_table( + &mut info, + &mut p2[p2_index], + p1_page(p2_start, recursive_index), + ) + } + .map_err(|e| (e.into(), p2_start))?; + + let start_p1_index = p2_start.p1_index().into(); + let mut end_p1_index = p2_end.p1_index().into(); + + if p2_end != pages.end { + end_p1_index += 1; + } + + (start_p1_index..end_p1_index) + .map(PageTableIndex::new) + .map(move |p1_index| { + Page::from_page_table_indices( + p4_index, p3_index, p2_index, p1_index, + ) + }) + .try_for_each(|page| { + let entry = &mut p1[page.p1_index()]; + modify(entry, page, &mut info).map_err(|e| (e, page)) + }) + }) + }) + }) + .map(|_| MapperFlushRange::new(pages)) + .map_err(|(e, page)| { + ( + e, + MapperFlushRange::new(PageRange { + start: pages.start, + end: page, + }), + ) + }) + } + + #[cfg(feature = "experimental")] + #[inline] + fn map_to_range_4kib( + &mut self, + pages: PageRange, + frames: F, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + F: Fn(Page, &mut A) -> Option>, + A: FrameAllocator + ?Sized, + { + self.modify_range_4kib( + pages, + |entry, page, (_, allocator)| { + let frame = frames(page, allocator).ok_or(MapToError::FrameAllocationFailed)?; + if !entry.is_unused() { + return Err(MapToError::PageAlreadyMapped(frame)); + } + entry.set_addr(frame.start_address(), flags); + Ok(()) + }, + (parent_table_flags, allocator), + Self::next_table_fn_create_next_table, + ) + } } impl<'a> Mapper for RecursivePageTable<'a> { @@ -304,6 +749,55 @@ impl<'a> Mapper for RecursivePageTable<'a> { self.map_to_1gib(page, frame, flags, parent_table_flags, allocator) } + #[cfg(feature = "experimental")] + #[inline] + unsafe fn map_to_range_with_table_flags( + &mut self, + pages: PageRange, + frames: PhysFrameRange, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + Self: Sized, + A: FrameAllocator + ?Sized, + { + assert_eq!(pages.count(), frames.count()); + self.map_to_range_1gib( + pages, + |page, _| { + let offset = page - pages.start; + Some(frames.start + offset) + }, + flags, + parent_table_flags, + allocator, + ) + } + + #[cfg(feature = "experimental")] + #[inline] + unsafe fn map_range_with_table_flags( + &mut self, + pages: PageRange, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + Self: Sized, + A: FrameAllocator + FrameAllocator + ?Sized, + { + self.map_to_range_1gib( + pages, + |_, allocator| allocator.allocate_frame(), + flags, + parent_table_flags, + allocator, + ) + } + fn unmap( &mut self, page: Page, @@ -334,6 +828,36 @@ impl<'a> Mapper for RecursivePageTable<'a> { Ok((frame, MapperFlush::new(page))) } + #[cfg(feature = "experimental")] + #[inline] + unsafe fn unmap_range( + &mut self, + pages: PageRange, + deallocator: &mut D, + ) -> Result, (UnmapError, MapperFlushRange)> + where + Self: Sized, + D: FrameDeallocator + ?Sized, + { + self.modify_range_1gib( + pages, + |entry, _, deallocator| { + let frame = PhysFrame::from_start_address(entry.addr()) + .map_err(|AddressNotAligned| UnmapError::InvalidFrameAddress(entry.addr()))?; + unsafe { + deallocator.deallocate_frame(frame); + } + + entry.set_unused(); + Ok(()) + }, + deallocator, + Self::next_table_fn_next_table_mut, + ) + } + + // allow unused_unsafe until https://github.com/rust-lang/rfcs/pull/2585 lands + #[allow(unused_unsafe)] unsafe fn update_flags( &mut self, page: Page, @@ -356,6 +880,28 @@ impl<'a> Mapper for RecursivePageTable<'a> { Ok(MapperFlush::new(page)) } + #[cfg(feature = "experimental")] + #[inline] + unsafe fn update_flags_range( + &mut self, + pages: PageRange, + flags: PageTableFlags, + ) -> Result, (FlagUpdateError, MapperFlushRange)> { + self.modify_range_1gib( + pages, + |entry, _, _| { + if entry.is_unused() { + return Err(FlagUpdateError::PageNotMapped); + } + + entry.set_flags(flags); + Ok(()) + }, + (), + Self::next_table_fn_next_table_mut, + ) + } + unsafe fn set_flags_p4_entry( &mut self, page: Page, @@ -424,6 +970,55 @@ impl<'a> Mapper for RecursivePageTable<'a> { self.map_to_2mib(page, frame, flags, parent_table_flags, allocator) } + #[cfg(feature = "experimental")] + #[inline] + unsafe fn map_to_range_with_table_flags( + &mut self, + pages: PageRange, + frames: PhysFrameRange, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + Self: Sized, + A: FrameAllocator + ?Sized, + { + assert_eq!(pages.count(), frames.count()); + self.map_to_range_2mib( + pages, + |page, _| { + let offset = page - pages.start; + Some(frames.start + offset) + }, + flags, + parent_table_flags, + allocator, + ) + } + + #[cfg(feature = "experimental")] + #[inline] + unsafe fn map_range_with_table_flags( + &mut self, + pages: PageRange, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + Self: Sized, + A: FrameAllocator + FrameAllocator + ?Sized, + { + self.map_to_range_2mib( + pages, + |_, allocator| allocator.allocate_frame(), + flags, + parent_table_flags, + allocator, + ) + } + fn unmap( &mut self, page: Page, @@ -460,6 +1055,36 @@ impl<'a> Mapper for RecursivePageTable<'a> { Ok((frame, MapperFlush::new(page))) } + #[cfg(feature = "experimental")] + #[inline] + unsafe fn unmap_range( + &mut self, + pages: PageRange, + deallocator: &mut D, + ) -> Result, (UnmapError, MapperFlushRange)> + where + Self: Sized, + D: FrameDeallocator + ?Sized, + { + self.modify_range_2mib( + pages, + |entry, _, deallocator| { + let frame = PhysFrame::from_start_address(entry.addr()) + .map_err(|AddressNotAligned| UnmapError::InvalidFrameAddress(entry.addr()))?; + unsafe { + deallocator.deallocate_frame(frame); + } + + entry.set_unused(); + Ok(()) + }, + deallocator, + Self::next_table_fn_next_table_mut, + ) + } + + // allow unused_unsafe until https://github.com/rust-lang/rfcs/pull/2585 lands + #[allow(unused_unsafe)] unsafe fn update_flags( &mut self, page: Page, @@ -489,6 +1114,28 @@ impl<'a> Mapper for RecursivePageTable<'a> { Ok(MapperFlush::new(page)) } + #[cfg(feature = "experimental")] + #[inline] + unsafe fn update_flags_range( + &mut self, + pages: PageRange, + flags: PageTableFlags, + ) -> Result, (FlagUpdateError, MapperFlushRange)> { + self.modify_range_2mib( + pages, + |entry, _, _| { + if entry.is_unused() { + return Err(FlagUpdateError::PageNotMapped); + } + + entry.set_flags(flags); + Ok(()) + }, + (), + Self::next_table_fn_next_table_mut, + ) + } + unsafe fn set_flags_p4_entry( &mut self, page: Page, @@ -579,6 +1226,54 @@ impl<'a> Mapper for RecursivePageTable<'a> { self.map_to_4kib(page, frame, flags, parent_table_flags, allocator) } + #[cfg(feature = "experimental")] + #[inline] + unsafe fn map_to_range_with_table_flags( + &mut self, + pages: PageRange, + frames: PhysFrameRange, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + Self: Sized, + A: FrameAllocator + ?Sized, + { + assert_eq!(pages.count(), frames.count()); + self.map_to_range_4kib( + pages, + |page, _| { + let offset = page - pages.start; + Some(frames.start + offset) + }, + flags, + parent_table_flags, + allocator, + ) + } + + #[cfg(feature = "experimental")] + #[inline] + unsafe fn map_range_with_table_flags( + &mut self, + pages: PageRange, + flags: PageTableFlags, + parent_table_flags: PageTableFlags, + allocator: &mut A, + ) -> Result, (MapToError, MapperFlushRange)> + where + A: FrameAllocator + ?Sized, + { + self.map_to_range_4kib( + pages, + |_, allocator| allocator.allocate_frame(), + flags, + parent_table_flags, + allocator, + ) + } + fn unmap( &mut self, page: Page, @@ -616,6 +1311,38 @@ impl<'a> Mapper for RecursivePageTable<'a> { Ok((frame, MapperFlush::new(page))) } + #[cfg(feature = "experimental")] + #[inline] + unsafe fn unmap_range( + &mut self, + pages: PageRange, + deallocator: &mut D, + ) -> Result, (UnmapError, MapperFlushRange)> + where + Self: Sized, + D: FrameDeallocator + ?Sized, + { + self.modify_range_4kib( + pages, + |entry, _, deallocator| { + let frame = entry.frame().map_err(|err| match err { + FrameError::FrameNotPresent => UnmapError::PageNotMapped, + FrameError::HugeFrame => UnmapError::ParentEntryHugePage, + })?; + unsafe { + deallocator.deallocate_frame(frame); + } + + entry.set_unused(); + Ok(()) + }, + deallocator, + Self::next_table_fn_next_table_mut, + ) + } + + // allow unused_unsafe until https://github.com/rust-lang/rfcs/pull/2585 lands + #[allow(unused_unsafe)] unsafe fn update_flags( &mut self, page: Page, @@ -650,6 +1377,28 @@ impl<'a> Mapper for RecursivePageTable<'a> { Ok(MapperFlush::new(page)) } + #[cfg(feature = "experimental")] + #[inline] + unsafe fn update_flags_range( + &mut self, + pages: PageRange, + flags: PageTableFlags, + ) -> Result, (FlagUpdateError, MapperFlushRange)> { + self.modify_range_4kib( + pages, + |entry, _, _| { + if entry.is_unused() { + return Err(FlagUpdateError::PageNotMapped); + } + + entry.set_flags(flags); + Ok(()) + }, + (), + Self::next_table_fn_next_table_mut, + ) + } + unsafe fn set_flags_p4_entry( &mut self, page: Page, diff --git a/src/structures/paging/page.rs b/src/structures/paging/page.rs index 3e2878bfe..d26e70fcd 100644 --- a/src/structures/paging/page.rs +++ b/src/structures/paging/page.rs @@ -324,6 +324,15 @@ impl Iterator for PageRange { None } } + + #[inline] + fn count(self) -> usize { + if !self.is_empty() { + ((self.end.start_address() - self.start.start_address()) / S::SIZE) as usize + } else { + 0 + } + } } impl PageRange {