+24 -22

drivers/gpu/nova-core/gsp.rs

reviewed

··· 47 47 unsafe impl<const NUM_ENTRIES: usize> AsBytes for PteArray<NUM_ENTRIES> {} 48 48 49 49 impl<const NUM_PAGES: usize> PteArray<NUM_PAGES> { 50 50 - /// Creates a new page table array mapping `NUM_PAGES` GSP pages starting at address `start`. 51 51 - fn new(start: DmaAddress) -> Result<Self> { 52 52 - let mut ptes = [0u64; NUM_PAGES]; 53 53 - for (i, pte) in ptes.iter_mut().enumerate() { 54 54 - *pte = start 55 55 - .checked_add(num::usize_as_u64(i) << GSP_PAGE_SHIFT) 56 56 - .ok_or(EOVERFLOW)?; 57 57 - } 58 58 - 59 59 - Ok(Self(ptes)) 50 50 + /// Returns the page table entry for `index`, for a mapping starting at `start`. 51 51 + // TODO: Replace with `IoView` projection once available. 52 52 + fn entry(start: DmaAddress, index: usize) -> Result<u64> { 53 53 + start 54 54 + .checked_add(num::usize_as_u64(index) << GSP_PAGE_SHIFT) 55 55 + .ok_or(EOVERFLOW) 60 56 } 61 57 } 62 58 ··· 82 86 NUM_PAGES * GSP_PAGE_SIZE, 83 87 GFP_KERNEL | __GFP_ZERO, 84 88 )?); 85 85 - let ptes = PteArray::<NUM_PAGES>::new(obj.0.dma_handle())?; 89 89 + 90 90 + let start_addr = obj.0.dma_handle(); 86 91 87 92 // SAFETY: `obj` has just been created and we are its sole user. 88 88 - unsafe { 89 89 - // Copy the self-mapping PTE at the expected location. 93 93 + let pte_region = unsafe { 90 94 obj.0 91 91 - .as_slice_mut(size_of::<u64>(), size_of_val(&ptes))? 92 92 - .copy_from_slice(ptes.as_bytes()) 95 95 + .as_slice_mut(size_of::<u64>(), NUM_PAGES * size_of::<u64>())? 93 96 }; 97 97 + 98 98 + // Write values one by one to avoid an on-stack instance of `PteArray`. 99 99 + for (i, chunk) in pte_region.chunks_exact_mut(size_of::<u64>()).enumerate() { 100 100 + let pte_value = PteArray::<0>::entry(start_addr, i)?; 101 101 + 102 102 + chunk.copy_from_slice(&pte_value.to_ne_bytes()); 103 103 + } 94 104 95 105 Ok(obj) 96 106 } ··· 145 143 // _kgspInitLibosLoggingStructures (allocates memory for buffers) 146 144 // kgspSetupLibosInitArgs_IMPL (creates pLibosInitArgs[] array) 147 145 dma_write!( 148 148 - libos[0] = LibosMemoryRegionInitArgument::new("LOGINIT", &loginit.0) 149 149 - )?; 146 146 + libos, [0]?, LibosMemoryRegionInitArgument::new("LOGINIT", &loginit.0) 147 147 + ); 150 148 dma_write!( 151 151 - libos[1] = LibosMemoryRegionInitArgument::new("LOGINTR", &logintr.0) 152 152 - )?; 153 153 - dma_write!(libos[2] = LibosMemoryRegionInitArgument::new("LOGRM", &logrm.0))?; 154 154 - dma_write!(rmargs[0].inner = fw::GspArgumentsCached::new(cmdq))?; 155 155 - dma_write!(libos[3] = LibosMemoryRegionInitArgument::new("RMARGS", rmargs))?; 149 149 + libos, [1]?, LibosMemoryRegionInitArgument::new("LOGINTR", &logintr.0) 150 150 + ); 151 151 + dma_write!(libos, [2]?, LibosMemoryRegionInitArgument::new("LOGRM", &logrm.0)); 152 152 + dma_write!(rmargs, [0]?.inner, fw::GspArgumentsCached::new(cmdq)); 153 153 + dma_write!(libos, [3]?, LibosMemoryRegionInitArgument::new("RMARGS", rmargs)); 156 154 }, 157 155 })) 158 156 })

+1 -1

drivers/gpu/nova-core/gsp/boot.rs

reviewed

··· 157 157 158 158 let wpr_meta = 159 159 CoherentAllocation::<GspFwWprMeta>::alloc_coherent(dev, 1, GFP_KERNEL | __GFP_ZERO)?; 160 160 - dma_write!(wpr_meta[0] = GspFwWprMeta::new(&gsp_fw, &fb_layout))?; 160 160 + dma_write!(wpr_meta, [0]?, GspFwWprMeta::new(&gsp_fw, &fb_layout)); 161 161 162 162 self.cmdq 163 163 .send_command(bar, commands::SetSystemInfo::new(pdev))?;

+33 -60

drivers/gpu/nova-core/gsp/cmdq.rs

reviewed

··· 2 2 3 3 use core::{ 4 4 cmp, 5 5 - mem, 6 6 - sync::atomic::{ 7 7 - fence, 8 8 - Ordering, // 9 9 - }, // 5 5 + mem, // 10 6 }; 11 7 12 8 use kernel::{ ··· 142 146 #[repr(C)] 143 147 // There is no struct defined for this in the open-gpu-kernel-source headers. 144 148 // Instead it is defined by code in `GspMsgQueuesInit()`. 145 145 - struct Msgq { 149 149 + // TODO: Revert to private once `IoView` projections replace the `gsp_mem` module. 150 150 + pub(super) struct Msgq { 146 151 /// Header for sending messages, including the write pointer. 147 147 - tx: MsgqTxHeader, 152 152 + pub(super) tx: MsgqTxHeader, 148 153 /// Header for receiving messages, including the read pointer. 149 149 - rx: MsgqRxHeader, 154 154 + pub(super) rx: MsgqRxHeader, 150 155 /// The message queue proper. 151 156 msgq: MsgqData, 152 157 } 153 158 154 159 /// Structure shared between the driver and the GSP and containing the command and message queues. 155 160 #[repr(C)] 156 156 - struct GspMem { 161 161 + // TODO: Revert to private once `IoView` projections replace the `gsp_mem` module. 162 162 + pub(super) struct GspMem { 157 163 /// Self-mapping page table entries. 158 158 - ptes: PteArray<{ GSP_PAGE_SIZE / size_of::<u64>() }>, 164 164 + ptes: PteArray<{ Self::PTE_ARRAY_SIZE }>, 159 165 /// CPU queue: the driver writes commands here, and the GSP reads them. It also contains the 160 166 /// write and read pointers that the CPU updates. 161 167 /// 162 168 /// This member is read-only for the GSP. 163 163 - cpuq: Msgq, 169 169 + pub(super) cpuq: Msgq, 164 170 /// GSP queue: the GSP writes messages here, and the driver reads them. It also contains the 165 171 /// write and read pointers that the GSP updates. 166 172 /// 167 173 /// This member is read-only for the driver. 168 168 - gspq: Msgq, 174 174 + pub(super) gspq: Msgq, 175 175 + } 176 176 + 177 177 + impl GspMem { 178 178 + const PTE_ARRAY_SIZE: usize = GSP_PAGE_SIZE / size_of::<u64>(); 169 179 } 170 180 171 181 // SAFETY: These structs don't meet the no-padding requirements of AsBytes but ··· 203 201 204 202 let gsp_mem = 205 203 CoherentAllocation::<GspMem>::alloc_coherent(dev, 1, GFP_KERNEL | __GFP_ZERO)?; 206 206 - dma_write!(gsp_mem[0].ptes = PteArray::new(gsp_mem.dma_handle())?)?; 207 207 - dma_write!(gsp_mem[0].cpuq.tx = MsgqTxHeader::new(MSGQ_SIZE, RX_HDR_OFF, MSGQ_NUM_PAGES))?; 208 208 - dma_write!(gsp_mem[0].cpuq.rx = MsgqRxHeader::new())?; 204 204 + 205 205 + let start = gsp_mem.dma_handle(); 206 206 + // Write values one by one to avoid an on-stack instance of `PteArray`. 207 207 + for i in 0..GspMem::PTE_ARRAY_SIZE { 208 208 + dma_write!(gsp_mem, [0]?.ptes.0[i], PteArray::<0>::entry(start, i)?); 209 209 + } 210 210 + 211 211 + dma_write!( 212 212 + gsp_mem, 213 213 + [0]?.cpuq.tx, 214 214 + MsgqTxHeader::new(MSGQ_SIZE, RX_HDR_OFF, MSGQ_NUM_PAGES) 215 215 + ); 216 216 + dma_write!(gsp_mem, [0]?.cpuq.rx, MsgqRxHeader::new()); 209 217 210 218 Ok(Self(gsp_mem)) 211 219 } ··· 329 317 // 330 318 // - The returned value is between `0` and `MSGQ_NUM_PAGES`. 331 319 fn gsp_write_ptr(&self) -> u32 { 332 332 - let gsp_mem = self.0.start_ptr(); 333 333 - 334 334 - // SAFETY: 335 335 - // - The 'CoherentAllocation' contains at least one object. 336 336 - // - By the invariants of `CoherentAllocation` the pointer is valid. 337 337 - (unsafe { (*gsp_mem).gspq.tx.write_ptr() } % MSGQ_NUM_PAGES) 320 320 + super::fw::gsp_mem::gsp_write_ptr(&self.0) 338 321 } 339 322 340 323 // Returns the index of the memory page the GSP will read the next command from. ··· 338 331 // 339 332 // - The returned value is between `0` and `MSGQ_NUM_PAGES`. 340 333 fn gsp_read_ptr(&self) -> u32 { 341 341 - let gsp_mem = self.0.start_ptr(); 342 342 - 343 343 - // SAFETY: 344 344 - // - The 'CoherentAllocation' contains at least one object. 345 345 - // - By the invariants of `CoherentAllocation` the pointer is valid. 346 346 - (unsafe { (*gsp_mem).gspq.rx.read_ptr() } % MSGQ_NUM_PAGES) 334 334 + super::fw::gsp_mem::gsp_read_ptr(&self.0) 347 335 } 348 336 349 337 // Returns the index of the memory page the CPU can read the next message from. ··· 347 345 // 348 346 // - The returned value is between `0` and `MSGQ_NUM_PAGES`. 349 347 fn cpu_read_ptr(&self) -> u32 { 350 350 - let gsp_mem = self.0.start_ptr(); 351 351 - 352 352 - // SAFETY: 353 353 - // - The ['CoherentAllocation'] contains at least one object. 354 354 - // - By the invariants of CoherentAllocation the pointer is valid. 355 355 - (unsafe { (*gsp_mem).cpuq.rx.read_ptr() } % MSGQ_NUM_PAGES) 348 348 + super::fw::gsp_mem::cpu_read_ptr(&self.0) 356 349 } 357 350 358 351 // Informs the GSP that it can send `elem_count` new pages into the message queue. 359 352 fn advance_cpu_read_ptr(&mut self, elem_count: u32) { 360 360 - let rptr = self.cpu_read_ptr().wrapping_add(elem_count) % MSGQ_NUM_PAGES; 361 361 - 362 362 - // Ensure read pointer is properly ordered. 363 363 - fence(Ordering::SeqCst); 364 364 - 365 365 - let gsp_mem = self.0.start_ptr_mut(); 366 366 - 367 367 - // SAFETY: 368 368 - // - The 'CoherentAllocation' contains at least one object. 369 369 - // - By the invariants of `CoherentAllocation` the pointer is valid. 370 370 - unsafe { (*gsp_mem).cpuq.rx.set_read_ptr(rptr) }; 353 353 + super::fw::gsp_mem::advance_cpu_read_ptr(&self.0, elem_count) 371 354 } 372 355 373 356 // Returns the index of the memory page the CPU can write the next command to. ··· 361 374 // 362 375 // - The returned value is between `0` and `MSGQ_NUM_PAGES`. 363 376 fn cpu_write_ptr(&self) -> u32 { 364 364 - let gsp_mem = self.0.start_ptr(); 365 365 - 366 366 - // SAFETY: 367 367 - // - The 'CoherentAllocation' contains at least one object. 368 368 - // - By the invariants of `CoherentAllocation` the pointer is valid. 369 369 - (unsafe { (*gsp_mem).cpuq.tx.write_ptr() } % MSGQ_NUM_PAGES) 377 377 + super::fw::gsp_mem::cpu_write_ptr(&self.0) 370 378 } 371 379 372 380 // Informs the GSP that it can process `elem_count` new pages from the command queue. 373 381 fn advance_cpu_write_ptr(&mut self, elem_count: u32) { 374 374 - let wptr = self.cpu_write_ptr().wrapping_add(elem_count) & MSGQ_NUM_PAGES; 375 375 - let gsp_mem = self.0.start_ptr_mut(); 376 376 - 377 377 - // SAFETY: 378 378 - // - The 'CoherentAllocation' contains at least one object. 379 379 - // - By the invariants of `CoherentAllocation` the pointer is valid. 380 380 - unsafe { (*gsp_mem).cpuq.tx.set_write_ptr(wptr) }; 381 381 - 382 382 - // Ensure all command data is visible before triggering the GSP read. 383 383 - fence(Ordering::SeqCst); 382 382 + super::fw::gsp_mem::advance_cpu_write_ptr(&self.0, elem_count) 384 383 } 385 384 } 386 385

+69 -32

drivers/gpu/nova-core/gsp/fw.rs

reviewed

··· 40 40 }, 41 41 }; 42 42 43 43 + // TODO: Replace with `IoView` projections once available; the `unwrap()` calls go away once we 44 44 + // switch to the new `dma::Coherent` API. 45 45 + pub(super) mod gsp_mem { 46 46 + use core::sync::atomic::{ 47 47 + fence, 48 48 + Ordering, // 49 49 + }; 50 50 + 51 51 + use kernel::{ 52 52 + dma::CoherentAllocation, 53 53 + dma_read, 54 54 + dma_write, 55 55 + prelude::*, // 56 56 + }; 57 57 + 58 58 + use crate::gsp::cmdq::{ 59 59 + GspMem, 60 60 + MSGQ_NUM_PAGES, // 61 61 + }; 62 62 + 63 63 + pub(in crate::gsp) fn gsp_write_ptr(qs: &CoherentAllocation<GspMem>) -> u32 { 64 64 + // PANIC: A `dma::CoherentAllocation` always contains at least one element. 65 65 + || -> Result<u32> { Ok(dma_read!(qs, [0]?.gspq.tx.0.writePtr) % MSGQ_NUM_PAGES) }().unwrap() 66 66 + } 67 67 + 68 68 + pub(in crate::gsp) fn gsp_read_ptr(qs: &CoherentAllocation<GspMem>) -> u32 { 69 69 + // PANIC: A `dma::CoherentAllocation` always contains at least one element. 70 70 + || -> Result<u32> { Ok(dma_read!(qs, [0]?.gspq.rx.0.readPtr) % MSGQ_NUM_PAGES) }().unwrap() 71 71 + } 72 72 + 73 73 + pub(in crate::gsp) fn cpu_read_ptr(qs: &CoherentAllocation<GspMem>) -> u32 { 74 74 + // PANIC: A `dma::CoherentAllocation` always contains at least one element. 75 75 + || -> Result<u32> { Ok(dma_read!(qs, [0]?.cpuq.rx.0.readPtr) % MSGQ_NUM_PAGES) }().unwrap() 76 76 + } 77 77 + 78 78 + pub(in crate::gsp) fn advance_cpu_read_ptr(qs: &CoherentAllocation<GspMem>, count: u32) { 79 79 + let rptr = cpu_read_ptr(qs).wrapping_add(count) % MSGQ_NUM_PAGES; 80 80 + 81 81 + // Ensure read pointer is properly ordered. 82 82 + fence(Ordering::SeqCst); 83 83 + 84 84 + // PANIC: A `dma::CoherentAllocation` always contains at least one element. 85 85 + || -> Result { 86 86 + dma_write!(qs, [0]?.cpuq.rx.0.readPtr, rptr); 87 87 + Ok(()) 88 88 + }() 89 89 + .unwrap() 90 90 + } 91 91 + 92 92 + pub(in crate::gsp) fn cpu_write_ptr(qs: &CoherentAllocation<GspMem>) -> u32 { 93 93 + // PANIC: A `dma::CoherentAllocation` always contains at least one element. 94 94 + || -> Result<u32> { Ok(dma_read!(qs, [0]?.cpuq.tx.0.writePtr) % MSGQ_NUM_PAGES) }().unwrap() 95 95 + } 96 96 + 97 97 + pub(in crate::gsp) fn advance_cpu_write_ptr(qs: &CoherentAllocation<GspMem>, count: u32) { 98 98 + let wptr = cpu_write_ptr(qs).wrapping_add(count) % MSGQ_NUM_PAGES; 99 99 + 100 100 + // PANIC: A `dma::CoherentAllocation` always contains at least one element. 101 101 + || -> Result { 102 102 + dma_write!(qs, [0]?.cpuq.tx.0.writePtr, wptr); 103 103 + Ok(()) 104 104 + }() 105 105 + .unwrap(); 106 106 + 107 107 + // Ensure all command data is visible before triggering the GSP read. 108 108 + fence(Ordering::SeqCst); 109 109 + } 110 110 + } 111 111 + 43 112 /// Empty type to group methods related to heap parameters for running the GSP firmware. 44 113 enum GspFwHeapParams {} 45 114 ··· 777 708 entryOff: num::usize_into_u32::<GSP_PAGE_SIZE>(), 778 709 }) 779 710 } 780 780 - 781 781 - /// Returns the value of the write pointer for this queue. 782 782 - pub(crate) fn write_ptr(&self) -> u32 { 783 783 - let ptr = core::ptr::from_ref(&self.0.writePtr); 784 784 - 785 785 - // SAFETY: `ptr` is a valid pointer to a `u32`. 786 786 - unsafe { ptr.read_volatile() } 787 787 - } 788 788 - 789 789 - /// Sets the value of the write pointer for this queue. 790 790 - pub(crate) fn set_write_ptr(&mut self, val: u32) { 791 791 - let ptr = core::ptr::from_mut(&mut self.0.writePtr); 792 792 - 793 793 - // SAFETY: `ptr` is a valid pointer to a `u32`. 794 794 - unsafe { ptr.write_volatile(val) } 795 795 - } 796 711 } 797 712 798 713 // SAFETY: Padding is explicit and does not contain uninitialized data. ··· 791 738 /// Creates a new RX queue header. 792 739 pub(crate) fn new() -> Self { 793 740 Self(Default::default()) 794 794 - } 795 795 - 796 796 - /// Returns the value of the read pointer for this queue. 797 797 - pub(crate) fn read_ptr(&self) -> u32 { 798 798 - let ptr = core::ptr::from_ref(&self.0.readPtr); 799 799 - 800 800 - // SAFETY: `ptr` is a valid pointer to a `u32`. 801 801 - unsafe { ptr.read_volatile() } 802 802 - } 803 803 - 804 804 - /// Sets the value of the read pointer for this queue. 805 805 - pub(crate) fn set_read_ptr(&mut self, val: u32) { 806 806 - let ptr = core::ptr::from_mut(&mut self.0.readPtr); 807 807 - 808 808 - // SAFETY: `ptr` is a valid pointer to a `u32`. 809 809 - unsafe { ptr.write_volatile(val) } 810 741 } 811 742 } 812 743

+50 -64

rust/kernel/dma.rs

reviewed

··· 461 461 self.count * core::mem::size_of::<T>() 462 462 } 463 463 464 464 + /// Returns the raw pointer to the allocated region in the CPU's virtual address space. 465 465 + #[inline] 466 466 + pub fn as_ptr(&self) -> *const [T] { 467 467 + core::ptr::slice_from_raw_parts(self.cpu_addr.as_ptr(), self.count) 468 468 + } 469 469 + 470 470 + /// Returns the raw pointer to the allocated region in the CPU's virtual address space as 471 471 + /// a mutable pointer. 472 472 + #[inline] 473 473 + pub fn as_mut_ptr(&self) -> *mut [T] { 474 474 + core::ptr::slice_from_raw_parts_mut(self.cpu_addr.as_ptr(), self.count) 475 475 + } 476 476 + 464 477 /// Returns the base address to the allocated region in the CPU's virtual address space. 465 478 pub fn start_ptr(&self) -> *const T { 466 479 self.cpu_addr.as_ptr() ··· 594 581 Ok(()) 595 582 } 596 583 597 597 - /// Returns a pointer to an element from the region with bounds checking. `offset` is in 598 598 - /// units of `T`, not the number of bytes. 599 599 - /// 600 600 - /// Public but hidden since it should only be used from [`dma_read`] and [`dma_write`] macros. 601 601 - #[doc(hidden)] 602 602 - pub fn item_from_index(&self, offset: usize) -> Result<*mut T> { 603 603 - if offset >= self.count { 604 604 - return Err(EINVAL); 605 605 - } 606 606 - // SAFETY: 607 607 - // - The pointer is valid due to type invariant on `CoherentAllocation` 608 608 - // and we've just checked that the range and index is within bounds. 609 609 - // - `offset` can't overflow since it is smaller than `self.count` and we've checked 610 610 - // that `self.count` won't overflow early in the constructor. 611 611 - Ok(unsafe { self.cpu_addr.as_ptr().add(offset) }) 612 612 - } 613 613 - 614 584 /// Reads the value of `field` and ensures that its type is [`FromBytes`]. 615 585 /// 616 586 /// # Safety ··· 666 670 667 671 /// Reads a field of an item from an allocated region of structs. 668 672 /// 673 673 + /// The syntax is of the form `kernel::dma_read!(dma, proj)` where `dma` is an expression evaluating 674 674 + /// to a [`CoherentAllocation`] and `proj` is a [projection specification](kernel::ptr::project!). 675 675 + /// 669 676 /// # Examples 670 677 /// 671 678 /// ``` ··· 683 684 /// unsafe impl kernel::transmute::AsBytes for MyStruct{}; 684 685 /// 685 686 /// # fn test(alloc: &kernel::dma::CoherentAllocation<MyStruct>) -> Result { 686 686 - /// let whole = kernel::dma_read!(alloc[2]); 687 687 - /// let field = kernel::dma_read!(alloc[1].field); 687 687 + /// let whole = kernel::dma_read!(alloc, [2]?); 688 688 + /// let field = kernel::dma_read!(alloc, [1]?.field); 688 689 /// # Ok::<(), Error>(()) } 689 690 /// ``` 690 691 #[macro_export] 691 692 macro_rules! dma_read { 692 692 - ($dma:expr, $idx: expr, $($field:tt)*) => {{ 693 693 - (|| -> ::core::result::Result<_, $crate::error::Error> { 694 694 - let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?; 695 695 - // SAFETY: `item_from_index` ensures that `item` is always a valid pointer and can be 696 696 - // dereferenced. The compiler also further validates the expression on whether `field` 697 697 - // is a member of `item` when expanded by the macro. 698 698 - unsafe { 699 699 - let ptr_field = ::core::ptr::addr_of!((*item) $($field)*); 700 700 - ::core::result::Result::Ok( 701 701 - $crate::dma::CoherentAllocation::field_read(&$dma, ptr_field) 702 702 - ) 703 703 - } 704 704 - })() 693 693 + ($dma:expr, $($proj:tt)*) => {{ 694 694 + let dma = &$dma; 695 695 + let ptr = $crate::ptr::project!( 696 696 + $crate::dma::CoherentAllocation::as_ptr(dma), $($proj)* 697 697 + ); 698 698 + // SAFETY: The pointer created by the projection is within the DMA region. 699 699 + unsafe { $crate::dma::CoherentAllocation::field_read(dma, ptr) } 705 700 }}; 706 706 - ($dma:ident [ $idx:expr ] $($field:tt)* ) => { 707 707 - $crate::dma_read!($dma, $idx, $($field)*) 708 708 - }; 709 709 - ($($dma:ident).* [ $idx:expr ] $($field:tt)* ) => { 710 710 - $crate::dma_read!($($dma).*, $idx, $($field)*) 711 711 - }; 712 701 } 713 702 714 703 /// Writes to a field of an item from an allocated region of structs. 704 704 + /// 705 705 + /// The syntax is of the form `kernel::dma_write!(dma, proj, val)` where `dma` is an expression 706 706 + /// evaluating to a [`CoherentAllocation`], `proj` is a 707 707 + /// [projection specification](kernel::ptr::project!), and `val` is the value to be written to the 708 708 + /// projected location. 715 709 /// 716 710 /// # Examples 717 711 /// ··· 720 728 /// unsafe impl kernel::transmute::AsBytes for MyStruct{}; 721 729 /// 722 730 /// # fn test(alloc: &kernel::dma::CoherentAllocation<MyStruct>) -> Result { 723 723 - /// kernel::dma_write!(alloc[2].member = 0xf); 724 724 - /// kernel::dma_write!(alloc[1] = MyStruct { member: 0xf }); 731 731 + /// kernel::dma_write!(alloc, [2]?.member, 0xf); 732 732 + /// kernel::dma_write!(alloc, [1]?, MyStruct { member: 0xf }); 725 733 /// # Ok::<(), Error>(()) } 726 734 /// ``` 727 735 #[macro_export] 728 736 macro_rules! dma_write { 729 729 - ($dma:ident [ $idx:expr ] $($field:tt)*) => {{ 730 730 - $crate::dma_write!($dma, $idx, $($field)*) 737 737 + (@parse [$dma:expr] [$($proj:tt)*] [, $val:expr]) => {{ 738 738 + let dma = &$dma; 739 739 + let ptr = $crate::ptr::project!( 740 740 + mut $crate::dma::CoherentAllocation::as_mut_ptr(dma), $($proj)* 741 741 + ); 742 742 + let val = $val; 743 743 + // SAFETY: The pointer created by the projection is within the DMA region. 744 744 + unsafe { $crate::dma::CoherentAllocation::field_write(dma, ptr, val) } 731 745 }}; 732 732 - ($($dma:ident).* [ $idx:expr ] $($field:tt)* ) => {{ 733 733 - $crate::dma_write!($($dma).*, $idx, $($field)*) 734 734 - }}; 735 735 - ($dma:expr, $idx: expr, = $val:expr) => { 736 736 - (|| -> ::core::result::Result<_, $crate::error::Error> { 737 737 - let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?; 738 738 - // SAFETY: `item_from_index` ensures that `item` is always a valid item. 739 739 - unsafe { $crate::dma::CoherentAllocation::field_write(&$dma, item, $val) } 740 740 - ::core::result::Result::Ok(()) 741 741 - })() 746 746 + (@parse [$dma:expr] [$($proj:tt)*] [.$field:tt $($rest:tt)*]) => { 747 747 + $crate::dma_write!(@parse [$dma] [$($proj)* .$field] [$($rest)*]) 742 748 }; 743 743 - ($dma:expr, $idx: expr, $(.$field:ident)* = $val:expr) => { 744 744 - (|| -> ::core::result::Result<_, $crate::error::Error> { 745 745 - let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?; 746 746 - // SAFETY: `item_from_index` ensures that `item` is always a valid pointer and can be 747 747 - // dereferenced. The compiler also further validates the expression on whether `field` 748 748 - // is a member of `item` when expanded by the macro. 749 749 - unsafe { 750 750 - let ptr_field = ::core::ptr::addr_of_mut!((*item) $(.$field)*); 751 751 - $crate::dma::CoherentAllocation::field_write(&$dma, ptr_field, $val) 752 752 - } 753 753 - ::core::result::Result::Ok(()) 754 754 - })() 749 749 + (@parse [$dma:expr] [$($proj:tt)*] [[$index:expr]? $($rest:tt)*]) => { 750 750 + $crate::dma_write!(@parse [$dma] [$($proj)* [$index]?] [$($rest)*]) 751 751 + }; 752 752 + (@parse [$dma:expr] [$($proj:tt)*] [[$index:expr] $($rest:tt)*]) => { 753 753 + $crate::dma_write!(@parse [$dma] [$($proj)* [$index]] [$($rest)*]) 754 754 + }; 755 755 + ($dma:expr, $($rest:tt)*) => { 756 756 + $crate::dma_write!(@parse [$dma] [] [$($rest)*]) 755 757 }; 756 758 }

+4

rust/kernel/lib.rs

reviewed

··· 20 20 #![feature(generic_nonzero)] 21 21 #![feature(inline_const)] 22 22 #![feature(pointer_is_aligned)] 23 23 + #![feature(slice_ptr_len)] 23 24 // 24 25 // Stable since Rust 1.80.0. 25 26 #![feature(slice_flatten)] ··· 37 36 #![feature(const_option)] 38 37 #![feature(const_ptr_write)] 39 38 #![feature(const_refs_to_cell)] 39 39 + // 40 40 + // Stable since Rust 1.84.0. 41 41 + #![feature(strict_provenance)] 40 42 // 41 43 // Expected to become stable. 42 44 #![feature(arbitrary_self_types)]

+29 -1

rust/kernel/ptr.rs

reviewed

··· 2 2 3 3 //! Types and functions to work with pointers and addresses. 4 4 5 5 - use core::mem::align_of; 5 5 + pub mod projection; 6 6 + pub use crate::project_pointer as project; 7 7 + 8 8 + use core::mem::{ 9 9 + align_of, 10 10 + size_of, // 11 11 + }; 6 12 use core::num::NonZero; 7 13 8 14 /// Type representing an alignment, which is always a power of two. ··· 231 225 } 232 226 233 227 impl_alignable_uint!(u8, u16, u32, u64, usize); 228 228 + 229 229 + /// Trait to represent compile-time known size information. 230 230 + /// 231 231 + /// This is a generalization of [`size_of`] that works for dynamically sized types. 232 232 + pub trait KnownSize { 233 233 + /// Get the size of an object of this type in bytes, with the metadata of the given pointer. 234 234 + fn size(p: *const Self) -> usize; 235 235 + } 236 236 + 237 237 + impl<T> KnownSize for T { 238 238 + #[inline(always)] 239 239 + fn size(_: *const Self) -> usize { 240 240 + size_of::<T>() 241 241 + } 242 242 + } 243 243 + 244 244 + impl<T> KnownSize for [T] { 245 245 + #[inline(always)] 246 246 + fn size(p: *const Self) -> usize { 247 247 + p.len() * size_of::<T>() 248 248 + } 249 249 + }

+305

rust/kernel/ptr/projection.rs

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + 3 3 + //! Infrastructure for handling projections. 4 4 + 5 5 + use core::{ 6 6 + mem::MaybeUninit, 7 7 + ops::Deref, // 8 8 + }; 9 9 + 10 10 + use crate::prelude::*; 11 11 + 12 12 + /// Error raised when a projection is attempted on an array or slice out of bounds. 13 13 + pub struct OutOfBound; 14 14 + 15 15 + impl From<OutOfBound> for Error { 16 16 + #[inline(always)] 17 17 + fn from(_: OutOfBound) -> Self { 18 18 + ERANGE 19 19 + } 20 20 + } 21 21 + 22 22 + /// A helper trait to perform index projection. 23 23 + /// 24 24 + /// This is similar to [`core::slice::SliceIndex`], but operates on raw pointers safely and 25 25 + /// fallibly. 26 26 + /// 27 27 + /// # Safety 28 28 + /// 29 29 + /// The implementation of `index` and `get` (if [`Some`] is returned) must ensure that, if provided 30 30 + /// input pointer `slice` and returned pointer `output`, then: 31 31 + /// - `output` has the same provenance as `slice`; 32 32 + /// - `output.byte_offset_from(slice)` is between 0 to 33 33 + /// `KnownSize::size(slice) - KnownSize::size(output)`. 34 34 + /// 35 35 + /// This means that if the input pointer is valid, then pointer returned by `get` or `index` is 36 36 + /// also valid. 37 37 + #[diagnostic::on_unimplemented(message = "`{Self}` cannot be used to index `{T}`")] 38 38 + #[doc(hidden)] 39 39 + pub unsafe trait ProjectIndex<T: ?Sized>: Sized { 40 40 + type Output: ?Sized; 41 41 + 42 42 + /// Returns an index-projected pointer, if in bounds. 43 43 + fn get(self, slice: *mut T) -> Option<*mut Self::Output>; 44 44 + 45 45 + /// Returns an index-projected pointer; fail the build if it cannot be proved to be in bounds. 46 46 + #[inline(always)] 47 47 + fn index(self, slice: *mut T) -> *mut Self::Output { 48 48 + Self::get(self, slice).unwrap_or_else(|| build_error!()) 49 49 + } 50 50 + } 51 51 + 52 52 + // Forward array impl to slice impl. 53 53 + // 54 54 + // SAFETY: Safety requirement guaranteed by the forwarded impl. 55 55 + unsafe impl<T, I, const N: usize> ProjectIndex<[T; N]> for I 56 56 + where 57 57 + I: ProjectIndex<[T]>, 58 58 + { 59 59 + type Output = <I as ProjectIndex<[T]>>::Output; 60 60 + 61 61 + #[inline(always)] 62 62 + fn get(self, slice: *mut [T; N]) -> Option<*mut Self::Output> { 63 63 + <I as ProjectIndex<[T]>>::get(self, slice) 64 64 + } 65 65 + 66 66 + #[inline(always)] 67 67 + fn index(self, slice: *mut [T; N]) -> *mut Self::Output { 68 68 + <I as ProjectIndex<[T]>>::index(self, slice) 69 69 + } 70 70 + } 71 71 + 72 72 + // SAFETY: `get`-returned pointer has the same provenance as `slice` and the offset is checked to 73 73 + // not exceed the required bound. 74 74 + unsafe impl<T> ProjectIndex<[T]> for usize { 75 75 + type Output = T; 76 76 + 77 77 + #[inline(always)] 78 78 + fn get(self, slice: *mut [T]) -> Option<*mut T> { 79 79 + if self >= slice.len() { 80 80 + None 81 81 + } else { 82 82 + Some(slice.cast::<T>().wrapping_add(self)) 83 83 + } 84 84 + } 85 85 + } 86 86 + 87 87 + // SAFETY: `get`-returned pointer has the same provenance as `slice` and the offset is checked to 88 88 + // not exceed the required bound. 89 89 + unsafe impl<T> ProjectIndex<[T]> for core::ops::Range<usize> { 90 90 + type Output = [T]; 91 91 + 92 92 + #[inline(always)] 93 93 + fn get(self, slice: *mut [T]) -> Option<*mut [T]> { 94 94 + let new_len = self.end.checked_sub(self.start)?; 95 95 + if self.end > slice.len() { 96 96 + return None; 97 97 + } 98 98 + Some(core::ptr::slice_from_raw_parts_mut( 99 99 + slice.cast::<T>().wrapping_add(self.start), 100 100 + new_len, 101 101 + )) 102 102 + } 103 103 + } 104 104 + 105 105 + // SAFETY: Safety requirement guaranteed by the forwarded impl. 106 106 + unsafe impl<T> ProjectIndex<[T]> for core::ops::RangeTo<usize> { 107 107 + type Output = [T]; 108 108 + 109 109 + #[inline(always)] 110 110 + fn get(self, slice: *mut [T]) -> Option<*mut [T]> { 111 111 + (0..self.end).get(slice) 112 112 + } 113 113 + } 114 114 + 115 115 + // SAFETY: Safety requirement guaranteed by the forwarded impl. 116 116 + unsafe impl<T> ProjectIndex<[T]> for core::ops::RangeFrom<usize> { 117 117 + type Output = [T]; 118 118 + 119 119 + #[inline(always)] 120 120 + fn get(self, slice: *mut [T]) -> Option<*mut [T]> { 121 121 + (self.start..slice.len()).get(slice) 122 122 + } 123 123 + } 124 124 + 125 125 + // SAFETY: `get` returned the pointer as is, so it always has the same provenance and offset of 0. 126 126 + unsafe impl<T> ProjectIndex<[T]> for core::ops::RangeFull { 127 127 + type Output = [T]; 128 128 + 129 129 + #[inline(always)] 130 130 + fn get(self, slice: *mut [T]) -> Option<*mut [T]> { 131 131 + Some(slice) 132 132 + } 133 133 + } 134 134 + 135 135 + /// A helper trait to perform field projection. 136 136 + /// 137 137 + /// This trait has a `DEREF` generic parameter so it can be implemented twice for types that 138 138 + /// implement [`Deref`]. This will cause an ambiguity error and thus block [`Deref`] types being 139 139 + /// used as base of projection, as they can inject unsoundness. Users therefore must not specify 140 140 + /// `DEREF` and should always leave it to be inferred. 141 141 + /// 142 142 + /// # Safety 143 143 + /// 144 144 + /// `proj` may only invoke `f` with a valid allocation, as the documentation of [`Self::proj`] 145 145 + /// describes. 146 146 + #[doc(hidden)] 147 147 + pub unsafe trait ProjectField<const DEREF: bool> { 148 148 + /// Project a pointer to a type to a pointer of a field. 149 149 + /// 150 150 + /// `f` may only be invoked with a valid allocation so it can safely obtain raw pointers to 151 151 + /// fields using `&raw mut`. 152 152 + /// 153 153 + /// This is needed because `base` might not point to a valid allocation, while `&raw mut` 154 154 + /// requires pointers to be in bounds of a valid allocation. 155 155 + /// 156 156 + /// # Safety 157 157 + /// 158 158 + /// `f` must return a pointer in bounds of the provided pointer. 159 159 + unsafe fn proj<F>(base: *mut Self, f: impl FnOnce(*mut Self) -> *mut F) -> *mut F; 160 160 + } 161 161 + 162 162 + // NOTE: in theory, this API should work for `T: ?Sized` and `F: ?Sized`, too. However, we cannot 163 163 + // currently support that as we need to obtain a valid allocation that `&raw const` can operate on. 164 164 + // 165 165 + // SAFETY: `proj` invokes `f` with valid allocation. 166 166 + unsafe impl<T> ProjectField<false> for T { 167 167 + #[inline(always)] 168 168 + unsafe fn proj<F>(base: *mut Self, f: impl FnOnce(*mut Self) -> *mut F) -> *mut F { 169 169 + // Create a valid allocation to start projection, as `base` is not necessarily so. The 170 170 + // memory is never actually used so it will be optimized out, so it should work even for 171 171 + // very large `T` (`memoffset` crate also relies on this). To be extra certain, we also 172 172 + // annotate `f` closure with `#[inline(always)]` in the macro. 173 173 + let mut place = MaybeUninit::uninit(); 174 174 + let place_base = place.as_mut_ptr(); 175 175 + let field = f(place_base); 176 176 + // SAFETY: `field` is in bounds from `base` per safety requirement. 177 177 + let offset = unsafe { field.byte_offset_from(place_base) }; 178 178 + // Use `wrapping_byte_offset` as `base` does not need to be of valid allocation. 179 179 + base.wrapping_byte_offset(offset).cast() 180 180 + } 181 181 + } 182 182 + 183 183 + // SAFETY: Vacuously satisfied. 184 184 + unsafe impl<T: Deref> ProjectField<true> for T { 185 185 + #[inline(always)] 186 186 + unsafe fn proj<F>(_: *mut Self, _: impl FnOnce(*mut Self) -> *mut F) -> *mut F { 187 187 + build_error!("this function is a guard against `Deref` impl and is never invoked"); 188 188 + } 189 189 + } 190 190 + 191 191 + /// Create a projection from a raw pointer. 192 192 + /// 193 193 + /// The projected pointer is within the memory region marked by the input pointer. There is no 194 194 + /// requirement that the input raw pointer needs to be valid, so this macro may be used for 195 195 + /// projecting pointers outside normal address space, e.g. I/O pointers. However, if the input 196 196 + /// pointer is valid, the projected pointer is also valid. 197 197 + /// 198 198 + /// Supported projections include field projections and index projections. 199 199 + /// It is not allowed to project into types that implement custom [`Deref`] or 200 200 + /// [`Index`](core::ops::Index). 201 201 + /// 202 202 + /// The macro has basic syntax of `kernel::ptr::project!(ptr, projection)`, where `ptr` is an 203 203 + /// expression that evaluates to a raw pointer which serves as the base of projection. `projection` 204 204 + /// can be a projection expression of form `.field` (normally identifier, or numeral in case of 205 205 + /// tuple structs) or of form `[index]`. 206 206 + /// 207 207 + /// If a mutable pointer is needed, the macro input can be prefixed with the `mut` keyword, i.e. 208 208 + /// `kernel::ptr::project!(mut ptr, projection)`. By default, a const pointer is created. 209 209 + /// 210 210 + /// `ptr::project!` macro can perform both fallible indexing and build-time checked indexing. 211 211 + /// `[index]` form performs build-time bounds checking; if compiler fails to prove `[index]` is in 212 212 + /// bounds, compilation will fail. `[index]?` can be used to perform runtime bounds checking; 213 213 + /// `OutOfBound` error is raised via `?` if the index is out of bounds. 214 214 + /// 215 215 + /// # Examples 216 216 + /// 217 217 + /// Field projections are performed with `.field_name`: 218 218 + /// 219 219 + /// ``` 220 220 + /// struct MyStruct { field: u32, } 221 221 + /// let ptr: *const MyStruct = core::ptr::dangling(); 222 222 + /// let field_ptr: *const u32 = kernel::ptr::project!(ptr, .field); 223 223 + /// 224 224 + /// struct MyTupleStruct(u32, u32); 225 225 + /// 226 226 + /// fn proj(ptr: *const MyTupleStruct) { 227 227 + /// let field_ptr: *const u32 = kernel::ptr::project!(ptr, .1); 228 228 + /// } 229 229 + /// ``` 230 230 + /// 231 231 + /// Index projections are performed with `[index]`: 232 232 + /// 233 233 + /// ``` 234 234 + /// fn proj(ptr: *const [u8; 32]) -> Result { 235 235 + /// let field_ptr: *const u8 = kernel::ptr::project!(ptr, [1]); 236 236 + /// // The following invocation, if uncommented, would fail the build. 237 237 + /// // 238 238 + /// // kernel::ptr::project!(ptr, [128]); 239 239 + /// 240 240 + /// // This will raise an `OutOfBound` error (which is convertible to `ERANGE`). 241 241 + /// kernel::ptr::project!(ptr, [128]?); 242 242 + /// Ok(()) 243 243 + /// } 244 244 + /// ``` 245 245 + /// 246 246 + /// If you need to match on the error instead of propagate, put the invocation inside a closure: 247 247 + /// 248 248 + /// ``` 249 249 + /// let ptr: *const [u8; 32] = core::ptr::dangling(); 250 250 + /// let field_ptr: Result<*const u8> = (|| -> Result<_> { 251 251 + /// Ok(kernel::ptr::project!(ptr, [128]?)) 252 252 + /// })(); 253 253 + /// assert!(field_ptr.is_err()); 254 254 + /// ``` 255 255 + /// 256 256 + /// For mutable pointers, put `mut` as the first token in macro invocation. 257 257 + /// 258 258 + /// ``` 259 259 + /// let ptr: *mut [(u8, u16); 32] = core::ptr::dangling_mut(); 260 260 + /// let field_ptr: *mut u16 = kernel::ptr::project!(mut ptr, [1].1); 261 261 + /// ``` 262 262 + #[macro_export] 263 263 + macro_rules! project_pointer { 264 264 + (@gen $ptr:ident, ) => {}; 265 265 + // Field projection. `$field` needs to be `tt` to support tuple index like `.0`. 266 266 + (@gen $ptr:ident, .$field:tt $($rest:tt)*) => { 267 267 + // SAFETY: The provided closure always returns an in-bounds pointer. 268 268 + let $ptr = unsafe { 269 269 + $crate::ptr::projection::ProjectField::proj($ptr, #[inline(always)] |ptr| { 270 270 + // Check unaligned field. Not all users (e.g. DMA) can handle unaligned 271 271 + // projections. 272 272 + if false { 273 273 + let _ = &(*ptr).$field; 274 274 + } 275 275 + // SAFETY: `$field` is in bounds, and no implicit `Deref` is possible (if the 276 276 + // type implements `Deref`, Rust cannot infer the generic parameter `DEREF`). 277 277 + &raw mut (*ptr).$field 278 278 + }) 279 279 + }; 280 280 + $crate::ptr::project!(@gen $ptr, $($rest)*) 281 281 + }; 282 282 + // Fallible index projection. 283 283 + (@gen $ptr:ident, [$index:expr]? $($rest:tt)*) => { 284 284 + let $ptr = $crate::ptr::projection::ProjectIndex::get($index, $ptr) 285 285 + .ok_or($crate::ptr::projection::OutOfBound)?; 286 286 + $crate::ptr::project!(@gen $ptr, $($rest)*) 287 287 + }; 288 288 + // Build-time checked index projection. 289 289 + (@gen $ptr:ident, [$index:expr] $($rest:tt)*) => { 290 290 + let $ptr = $crate::ptr::projection::ProjectIndex::index($index, $ptr); 291 291 + $crate::ptr::project!(@gen $ptr, $($rest)*) 292 292 + }; 293 293 + (mut $ptr:expr, $($proj:tt)*) => {{ 294 294 + let ptr: *mut _ = $ptr; 295 295 + $crate::ptr::project!(@gen ptr, $($proj)*); 296 296 + ptr 297 297 + }}; 298 298 + ($ptr:expr, $($proj:tt)*) => {{ 299 299 + let ptr = <*const _>::cast_mut($ptr); 300 300 + // We currently always project using mutable pointer, as it is not decided whether `&raw 301 301 + // const` allows the resulting pointer to be mutated (see documentation of `addr_of!`). 302 302 + $crate::ptr::project!(@gen ptr, $($proj)*); 303 303 + ptr.cast_const() 304 304 + }}; 305 305 + }

+16 -14

samples/rust/rust_dma.rs

reviewed

··· 68 68 CoherentAllocation::alloc_coherent(pdev.as_ref(), TEST_VALUES.len(), GFP_KERNEL)?; 69 69 70 70 for (i, value) in TEST_VALUES.into_iter().enumerate() { 71 71 - kernel::dma_write!(ca[i] = MyStruct::new(value.0, value.1))?; 71 71 + kernel::dma_write!(ca, [i]?, MyStruct::new(value.0, value.1)); 72 72 } 73 73 74 74 let size = 4 * page::PAGE_SIZE; ··· 85 85 } 86 86 } 87 87 88 88 + impl DmaSampleDriver { 89 89 + fn check_dma(&self) -> Result { 90 90 + for (i, value) in TEST_VALUES.into_iter().enumerate() { 91 91 + let val0 = kernel::dma_read!(self.ca, [i]?.h); 92 92 + let val1 = kernel::dma_read!(self.ca, [i]?.b); 93 93 + 94 94 + assert_eq!(val0, value.0); 95 95 + assert_eq!(val1, value.1); 96 96 + } 97 97 + 98 98 + Ok(()) 99 99 + } 100 100 + } 101 101 + 88 102 #[pinned_drop] 89 103 impl PinnedDrop for DmaSampleDriver { 90 104 fn drop(self: Pin<&mut Self>) { 91 105 dev_info!(self.pdev, "Unload DMA test driver.\n"); 92 106 93 93 - for (i, value) in TEST_VALUES.into_iter().enumerate() { 94 94 - let val0 = kernel::dma_read!(self.ca[i].h); 95 95 - let val1 = kernel::dma_read!(self.ca[i].b); 96 96 - assert!(val0.is_ok()); 97 97 - assert!(val1.is_ok()); 98 98 - 99 99 - if let Ok(val0) = val0 { 100 100 - assert_eq!(val0, value.0); 101 101 - } 102 102 - if let Ok(val1) = val1 { 103 103 - assert_eq!(val1, value.1); 104 104 - } 105 105 - } 107 107 + assert!(self.check_dma().is_ok()); 106 108 107 109 for (i, entry) in self.sgt.iter().enumerate() { 108 110 dev_info!(

+3 -1

scripts/Makefile.build

reviewed

··· 310 310 311 311 # The features in this list are the ones allowed for non-`rust/` code. 312 312 # 313 313 + # - Stable since Rust 1.79.0: `feature(slice_ptr_len)`. 313 314 # - Stable since Rust 1.81.0: `feature(lint_reasons)`. 314 315 # - Stable since Rust 1.82.0: `feature(asm_const)`, 315 316 # `feature(offset_of_nested)`, `feature(raw_ref_op)`. 317 317 + # - Stable since Rust 1.84.0: `feature(strict_provenance)`. 316 318 # - Stable since Rust 1.87.0: `feature(asm_goto)`. 317 319 # - Expected to become stable: `feature(arbitrary_self_types)`. 318 320 # - To be determined: `feature(used_with_arg)`. 319 321 # 320 322 # Please see https://github.com/Rust-for-Linux/linux/issues/2 for details on 321 323 # the unstable features in use. 322 322 - rust_allowed_features := asm_const,asm_goto,arbitrary_self_types,lint_reasons,offset_of_nested,raw_ref_op,used_with_arg 324 324 + rust_allowed_features := asm_const,asm_goto,arbitrary_self_types,lint_reasons,offset_of_nested,raw_ref_op,slice_ptr_len,strict_provenance,used_with_arg 323 325 324 326 # `--out-dir` is required to avoid temporaries being created by `rustc` in the 325 327 # current working directory, which may be not accessible in the out-of-tree