Merge tag 'drm-rust-next-2026-01-26' of https://gitlab.freedesktop.org/drm/rust/kernel into drm-next

+15 -44

Documentation/gpu/nova/core/todo.rst

··· 41 41 Having this generalization also helps with implementing a generic macro that 42 42 automatically generates the corresponding mappings between a value and a number. 43 43 44 + FromPrimitive support has been worked on in the past, but hasn't been followed 45 + since then [1]. 46 + 47 + There also have been considerations of ToPrimitive [2]. 48 + 44 49 | Complexity: Beginner 45 50 | Link: https://docs.rs/num/latest/num/trait.FromPrimitive.html 51 + | Link: https://lore.kernel.org/all/cover.1750689857.git.y.j3ms.n@gmail.com/ [1] 52 + | Link: https://rust-for-linux.zulipchat.com/#narrow/channel/288089-General/topic/Implement.20.60FromPrimitive.60.20trait.20.2B.20derive.20macro.20for.20nova-core/with/541971854 [2] 46 53 47 54 Generic register abstraction [REGA] 48 55 ----------------------------------- ··· 141 134 | Complexity: Intermediate 142 135 | Contact: Alexandre Courbot 143 136 144 - IRQ abstractions 145 - ---------------- 146 - 147 - Rust abstractions for IRQ handling. 148 - 149 - There is active ongoing work from Daniel Almeida [1] for the "core" abstractions 150 - to request IRQs. 151 - 152 - Besides optional review and testing work, the required ``pci::Device`` code 153 - around those core abstractions needs to be worked out. 154 - 155 - | Complexity: Intermediate 156 - | Link: https://lore.kernel.org/lkml/20250122163932.46697-1-daniel.almeida@collabora.com/ [1] 157 - | Contact: Daniel Almeida 158 - 159 137 Page abstraction for foreign pages 160 138 ---------------------------------- 161 139 ··· 153 161 | Link: https://lore.kernel.org/linux-mm/20241119112408.779243-1-abdiel.janulgue@gmail.com/ [1] 154 162 | Link: https://lore.kernel.org/rust-for-linux/20250202-rust-page-v1-0-e3170d7fe55e@asahilina.net/ [2] 155 163 156 - Scatterlist / sg_table abstractions 157 - ----------------------------------- 158 - 159 - Rust abstractions for scatterlist / sg_table. 160 - 161 - There is preceding work from Abdiel Janulgue, which hasn't made it to the 162 - mailing list yet. 163 - 164 - | Complexity: Intermediate 165 - | Contact: Abdiel Janulgue 166 - 167 164 PCI MISC APIs 168 165 ------------- 169 166 170 - Extend the existing PCI device / driver abstractions by SR-IOV, config space, 171 - capability, MSI API abstractions. 167 + Extend the existing PCI device / driver abstractions by SR-IOV, capability, MSI 168 + API abstractions. 169 + 170 + SR-IOV [1] is work in progress. 172 171 173 172 | Complexity: Beginner 174 - 175 - XArray bindings [XARR] 176 - ---------------------- 177 - 178 - We need bindings for `xa_alloc`/`xa_alloc_cyclic` in order to generate the 179 - auxiliary device IDs. 180 - 181 - | Complexity: Intermediate 182 - 183 - Debugfs abstractions 184 - -------------------- 185 - 186 - Rust abstraction for debugfs APIs. 187 - 188 - | Reference: Export GSP log buffers 189 - | Complexity: Intermediate 173 + | Link: https://lore.kernel.org/all/20251119-rust-pci-sriov-v1-0-883a94599a97@redhat.com/ [1] 190 174 191 175 GPU (general) 192 176 ============= ··· 201 233 - maple_tree 202 234 - native Rust collections 203 235 236 + There is work in progress for using drm_buddy [1]. 237 + 204 238 | Complexity: Advanced 239 + | Link: https://lore.kernel.org/all/20251219203805.1246586-4-joelagnelf@nvidia.com/ [1] 205 240 206 241 Instance Memory 207 242 ---------------

+13 -5

drivers/gpu/drm/nova/driver.rs

··· 1 1 // SPDX-License-Identifier: GPL-2.0 2 2 3 3 use kernel::{ 4 - auxiliary, c_str, device::Core, drm, drm::gem, drm::ioctl, prelude::*, sync::aref::ARef, 4 + auxiliary, 5 + device::Core, 6 + drm::{ 7 + self, 8 + gem, 9 + ioctl, // 10 + }, 11 + prelude::*, 12 + sync::aref::ARef, // 5 13 }; 6 14 7 15 use crate::file::File; ··· 32 24 major: 0, 33 25 minor: 0, 34 26 patchlevel: 0, 35 - name: c_str!("nova"), 36 - desc: c_str!("Nvidia Graphics"), 27 + name: c"nova", 28 + desc: c"Nvidia Graphics", 37 29 }; 38 30 39 - const NOVA_CORE_MODULE_NAME: &CStr = c_str!("NovaCore"); 40 - const AUXILIARY_NAME: &CStr = c_str!("nova-drm"); 31 + const NOVA_CORE_MODULE_NAME: &CStr = c"NovaCore"; 32 + const AUXILIARY_NAME: &CStr = c"nova-drm"; 41 33 42 34 kernel::auxiliary_device_table!( 43 35 AUX_TABLE,

+3 -3

drivers/gpu/drm/nova/gem.rs

··· 3 3 use kernel::{ 4 4 drm, 5 5 drm::{gem, gem::BaseObject}, 6 + page, 6 7 prelude::*, 7 8 sync::aref::ARef, 8 9 }; ··· 28 27 impl NovaObject { 29 28 /// Create a new DRM GEM object. 30 29 pub(crate) fn new(dev: &NovaDevice, size: usize) -> Result<ARef<gem::Object<Self>>> { 31 - let aligned_size = size.next_multiple_of(1 << 12); 32 - 33 - if size == 0 || size > aligned_size { 30 + if size == 0 { 34 31 return Err(EINVAL); 35 32 } 33 + let aligned_size = page::page_align(size).ok_or(EINVAL)?; 36 34 37 35 gem::Object::new(dev, aligned_size) 38 36 }

+24 -31

drivers/gpu/drm/tyr/driver.rs

··· 1 1 // SPDX-License-Identifier: GPL-2.0 or MIT 2 2 3 - use kernel::c_str; 4 3 use kernel::clk::Clk; 5 4 use kernel::clk::OptionalClk; 6 5 use kernel::device::Bound; ··· 8 9 use kernel::devres::Devres; 9 10 use kernel::drm; 10 11 use kernel::drm::ioctl; 12 + use kernel::io::poll; 11 13 use kernel::new_mutex; 12 14 use kernel::of; 13 15 use kernel::platform; ··· 16 16 use kernel::regulator; 17 17 use kernel::regulator::Regulator; 18 18 use kernel::sizes::SZ_2M; 19 + use kernel::sync::aref::ARef; 19 20 use kernel::sync::Arc; 20 21 use kernel::sync::Mutex; 21 22 use kernel::time; 22 - use kernel::types::ARef; 23 23 24 24 use crate::file::File; 25 25 use crate::gem::TyrObject; ··· 34 34 35 35 #[pin_data(PinnedDrop)] 36 36 pub(crate) struct TyrDriver { 37 - device: ARef<TyrDevice>, 37 + _device: ARef<TyrDevice>, 38 38 } 39 39 40 40 #[pin_data(PinnedDrop)] ··· 68 68 fn issue_soft_reset(dev: &Device<Bound>, iomem: &Devres<IoMem>) -> Result { 69 69 regs::GPU_CMD.write(dev, iomem, regs::GPU_CMD_SOFT_RESET)?; 70 70 71 - // TODO: We cannot poll, as there is no support in Rust currently, so we 72 - // sleep. Change this when read_poll_timeout() is implemented in Rust. 73 - kernel::time::delay::fsleep(time::Delta::from_millis(100)); 74 - 75 - if regs::GPU_IRQ_RAWSTAT.read(dev, iomem)? & regs::GPU_IRQ_RAWSTAT_RESET_COMPLETED == 0 { 76 - dev_err!(dev, "GPU reset failed with errno\n"); 77 - dev_err!( 78 - dev, 79 - "GPU_INT_RAWSTAT is {}\n", 80 - regs::GPU_IRQ_RAWSTAT.read(dev, iomem)? 81 - ); 82 - 83 - return Err(EIO); 84 - } 71 + poll::read_poll_timeout( 72 + || regs::GPU_IRQ_RAWSTAT.read(dev, iomem), 73 + |status| *status & regs::GPU_IRQ_RAWSTAT_RESET_COMPLETED != 0, 74 + time::Delta::from_millis(1), 75 + time::Delta::from_millis(100), 76 + ) 77 + .inspect_err(|_| dev_err!(dev, "GPU reset failed."))?; 85 78 86 79 Ok(()) 87 80 } ··· 84 91 MODULE_OF_TABLE, 85 92 <TyrDriver as platform::Driver>::IdInfo, 86 93 [ 87 - (of::DeviceId::new(c_str!("rockchip,rk3588-mali")), ()), 88 - (of::DeviceId::new(c_str!("arm,mali-valhall-csf")), ()) 94 + (of::DeviceId::new(c"rockchip,rk3588-mali"), ()), 95 + (of::DeviceId::new(c"arm,mali-valhall-csf"), ()) 89 96 ] 90 97 ); 91 98 ··· 97 104 pdev: &platform::Device<Core>, 98 105 _info: Option<&Self::IdInfo>, 99 106 ) -> impl PinInit<Self, Error> { 100 - let core_clk = Clk::get(pdev.as_ref(), Some(c_str!("core")))?; 101 - let stacks_clk = OptionalClk::get(pdev.as_ref(), Some(c_str!("stacks")))?; 102 - let coregroup_clk = OptionalClk::get(pdev.as_ref(), Some(c_str!("coregroup")))?; 107 + let core_clk = Clk::get(pdev.as_ref(), Some(c"core"))?; 108 + let stacks_clk = OptionalClk::get(pdev.as_ref(), Some(c"stacks"))?; 109 + let coregroup_clk = OptionalClk::get(pdev.as_ref(), Some(c"coregroup"))?; 103 110 104 111 core_clk.prepare_enable()?; 105 112 stacks_clk.prepare_enable()?; 106 113 coregroup_clk.prepare_enable()?; 107 114 108 - let mali_regulator = Regulator::<regulator::Enabled>::get(pdev.as_ref(), c_str!("mali"))?; 109 - let sram_regulator = Regulator::<regulator::Enabled>::get(pdev.as_ref(), c_str!("sram"))?; 115 + let mali_regulator = Regulator::<regulator::Enabled>::get(pdev.as_ref(), c"mali")?; 116 + let sram_regulator = Regulator::<regulator::Enabled>::get(pdev.as_ref(), c"sram")?; 110 117 111 118 let request = pdev.io_request_by_index(0).ok_or(ENODEV)?; 112 119 let iomem = Arc::pin_init(request.iomap_sized::<SZ_2M>(), GFP_KERNEL)?; ··· 127 134 coregroup: coregroup_clk, 128 135 }), 129 136 regulators <- new_mutex!(Regulators { 130 - mali: mali_regulator, 131 - sram: sram_regulator, 137 + _mali: mali_regulator, 138 + _sram: sram_regulator, 132 139 }), 133 140 gpu_info, 134 141 }); ··· 136 143 let tdev: ARef<TyrDevice> = drm::Device::new(pdev.as_ref(), data)?; 137 144 drm::driver::Registration::new_foreign_owned(&tdev, pdev.as_ref(), 0)?; 138 145 139 - let driver = TyrDriver { device: tdev }; 146 + let driver = TyrDriver { _device: tdev }; 140 147 141 148 // We need this to be dev_info!() because dev_dbg!() does not work at 142 149 // all in Rust for now, and we need to see whether probe succeeded. ··· 167 174 major: 1, 168 175 minor: 5, 169 176 patchlevel: 0, 170 - name: c_str!("panthor"), 171 - desc: c_str!("ARM Mali Tyr DRM driver"), 177 + name: c"panthor", 178 + desc: c"ARM Mali Tyr DRM driver", 172 179 }; 173 180 174 181 #[vtable] ··· 193 200 194 201 #[pin_data] 195 202 struct Regulators { 196 - mali: Regulator<regulator::Enabled>, 197 - sram: Regulator<regulator::Enabled>, 203 + _mali: Regulator<regulator::Enabled>, 204 + _sram: Regulator<regulator::Enabled>, 198 205 }

+31 -34

drivers/gpu/drm/tyr/gpu.rs

··· 1 1 // SPDX-License-Identifier: GPL-2.0 or MIT 2 2 3 + use core::ops::Deref; 4 + use core::ops::DerefMut; 3 5 use kernel::bits::genmask_u32; 4 6 use kernel::device::Bound; 5 7 use kernel::device::Device; 6 8 use kernel::devres::Devres; 9 + use kernel::io::poll; 7 10 use kernel::platform; 8 11 use kernel::prelude::*; 9 - use kernel::time; 12 + use kernel::time::Delta; 10 13 use kernel::transmute::AsBytes; 11 14 use kernel::uapi; 12 15 ··· 22 19 /// # Invariants 23 20 /// 24 21 /// - The layout of this struct identical to the C `struct drm_panthor_gpu_info`. 25 - #[repr(C)] 26 - pub(crate) struct GpuInfo { 27 - pub(crate) gpu_id: u32, 28 - pub(crate) gpu_rev: u32, 29 - pub(crate) csf_id: u32, 30 - pub(crate) l2_features: u32, 31 - pub(crate) tiler_features: u32, 32 - pub(crate) mem_features: u32, 33 - pub(crate) mmu_features: u32, 34 - pub(crate) thread_features: u32, 35 - pub(crate) max_threads: u32, 36 - pub(crate) thread_max_workgroup_size: u32, 37 - pub(crate) thread_max_barrier_size: u32, 38 - pub(crate) coherency_features: u32, 39 - pub(crate) texture_features: [u32; 4], 40 - pub(crate) as_present: u32, 41 - pub(crate) selected_coherency: u32, 42 - pub(crate) shader_present: u64, 43 - pub(crate) l2_present: u64, 44 - pub(crate) tiler_present: u64, 45 - pub(crate) core_features: u32, 46 - pub(crate) pad: u32, 47 - } 22 + #[repr(transparent)] 23 + #[derive(Clone, Copy)] 24 + pub(crate) struct GpuInfo(pub(crate) uapi::drm_panthor_gpu_info); 48 25 49 26 impl GpuInfo { 50 27 pub(crate) fn new(dev: &Device<Bound>, iomem: &Devres<IoMem>) -> Result<Self> { ··· 57 74 let l2_present = u64::from(regs::GPU_L2_PRESENT_LO.read(dev, iomem)?); 58 75 let l2_present = l2_present | u64::from(regs::GPU_L2_PRESENT_HI.read(dev, iomem)?) << 32; 59 76 60 - Ok(Self { 77 + Ok(Self(uapi::drm_panthor_gpu_info { 61 78 gpu_id, 62 79 gpu_rev, 63 80 csf_id, ··· 79 96 tiler_present, 80 97 core_features, 81 98 pad: 0, 82 - }) 99 + gpu_features: 0, 100 + })) 83 101 } 84 102 85 103 pub(crate) fn log(&self, pdev: &platform::Device) { ··· 139 155 } 140 156 } 141 157 158 + impl Deref for GpuInfo { 159 + type Target = uapi::drm_panthor_gpu_info; 160 + 161 + fn deref(&self) -> &Self::Target { 162 + &self.0 163 + } 164 + } 165 + 166 + impl DerefMut for GpuInfo { 167 + fn deref_mut(&mut self) -> &mut Self::Target { 168 + &mut self.0 169 + } 170 + } 171 + 142 172 // SAFETY: `GpuInfo`'s invariant guarantees that it is the same type that is 143 173 // already exposed to userspace by the C driver. This implies that it fulfills 144 174 // the requirements for `AsBytes`. ··· 205 207 pub(crate) fn l2_power_on(dev: &Device<Bound>, iomem: &Devres<IoMem>) -> Result { 206 208 regs::L2_PWRON_LO.write(dev, iomem, 1)?; 207 209 208 - // TODO: We cannot poll, as there is no support in Rust currently, so we 209 - // sleep. Change this when read_poll_timeout() is implemented in Rust. 210 - kernel::time::delay::fsleep(time::Delta::from_millis(100)); 211 - 212 - if regs::L2_READY_LO.read(dev, iomem)? != 1 { 213 - dev_err!(dev, "Failed to power on the GPU\n"); 214 - return Err(EIO); 215 - } 210 + poll::read_poll_timeout( 211 + || regs::L2_READY_LO.read(dev, iomem), 212 + |status| *status == 1, 213 + Delta::from_millis(1), 214 + Delta::from_millis(100), 215 + ) 216 + .inspect_err(|_| dev_err!(dev, "Failed to power on the GPU."))?; 216 217 217 218 Ok(()) 218 219 }

+2 -3

drivers/gpu/nova-core/driver.rs

··· 2 2 3 3 use kernel::{ 4 4 auxiliary, 5 - c_str, 6 5 device::Core, 7 6 devres::Devres, 8 7 dma::Device, ··· 81 82 unsafe { pdev.dma_set_mask_and_coherent(DmaMask::new::<GPU_DMA_BITS>())? }; 82 83 83 84 let bar = Arc::pin_init( 84 - pdev.iomap_region_sized::<BAR0_SIZE>(0, c_str!("nova-core/bar0")), 85 + pdev.iomap_region_sized::<BAR0_SIZE>(0, c"nova-core/bar0"), 85 86 GFP_KERNEL, 86 87 )?; 87 88 ··· 89 90 gpu <- Gpu::new(pdev, bar.clone(), bar.access(pdev.as_ref())?), 90 91 _reg <- auxiliary::Registration::new( 91 92 pdev.as_ref(), 92 - c_str!("nova-drm"), 93 + c"nova-drm", 93 94 0, // TODO[XARR]: Once it lands, use XArray; for now we don't use the ID. 94 95 crate::MODULE_NAME 95 96 ),

+50 -57

drivers/gpu/nova-core/falcon.rs

··· 8 8 9 9 use kernel::{ 10 10 device, 11 - dma::DmaAddress, 11 + dma::{ 12 + DmaAddress, 13 + DmaMask, // 14 + }, 12 15 io::poll::read_poll_timeout, 13 16 prelude::*, 14 17 sync::aref::ARef, 15 18 time::{ 16 - delay::fsleep, 17 19 Delta, // 18 20 }, 19 21 }; ··· 23 21 use crate::{ 24 22 dma::DmaObject, 25 23 driver::Bar0, 24 + falcon::hal::LoadMethod, 26 25 gpu::Chipset, 27 26 num::{ 28 27 FromSafeCast, ··· 240 237 /// Different types of memory present in a falcon core. 241 238 #[derive(Debug, Clone, Copy, PartialEq, Eq)] 242 239 pub(crate) enum FalconMem { 243 - /// Instruction Memory. 244 - Imem, 240 + /// Secure Instruction Memory. 241 + ImemSecure, 242 + /// Non-Secure Instruction Memory. 243 + #[expect(unused)] 244 + ImemNonSecure, 245 245 /// Data Memory. 246 246 Dmem, 247 247 } ··· 351 345 352 346 /// Trait for providing load parameters of falcon firmwares. 353 347 pub(crate) trait FalconLoadParams { 354 - /// Returns the load parameters for `IMEM`. 355 - fn imem_load_params(&self) -> FalconLoadTarget; 348 + /// Returns the load parameters for Secure `IMEM`. 349 + fn imem_sec_load_params(&self) -> FalconLoadTarget; 350 + 351 + /// Returns the load parameters for Non-Secure `IMEM`, 352 + /// used only on Turing and GA100. 353 + fn imem_ns_load_params(&self) -> Option<FalconLoadTarget>; 356 354 357 355 /// Returns the load parameters for `DMEM`. 358 356 fn dmem_load_params(&self) -> FalconLoadTarget; ··· 398 388 regs::NV_PFALCON_FALCON_DMACTL::default().write(bar, &E::ID); 399 389 } 400 390 401 - /// Wait for memory scrubbing to complete. 402 - fn reset_wait_mem_scrubbing(&self, bar: &Bar0) -> Result { 403 - // TIMEOUT: memory scrubbing should complete in less than 20ms. 404 - read_poll_timeout( 405 - || Ok(regs::NV_PFALCON_FALCON_HWCFG2::read(bar, &E::ID)), 406 - |r| r.mem_scrubbing_done(), 407 - Delta::ZERO, 408 - Delta::from_millis(20), 409 - ) 410 - .map(|_| ()) 411 - } 412 - 413 - /// Reset the falcon engine. 414 - fn reset_eng(&self, bar: &Bar0) -> Result { 415 - let _ = regs::NV_PFALCON_FALCON_HWCFG2::read(bar, &E::ID); 416 - 417 - // According to OpenRM's `kflcnPreResetWait_GA102` documentation, HW sometimes does not set 418 - // RESET_READY so a non-failing timeout is used. 419 - let _ = read_poll_timeout( 420 - || Ok(regs::NV_PFALCON_FALCON_HWCFG2::read(bar, &E::ID)), 421 - |r| r.reset_ready(), 422 - Delta::ZERO, 423 - Delta::from_micros(150), 424 - ); 425 - 426 - regs::NV_PFALCON_FALCON_ENGINE::update(bar, &E::ID, |v| v.set_reset(true)); 427 - 428 - // TIMEOUT: falcon engine should not take more than 10us to reset. 429 - fsleep(Delta::from_micros(10)); 430 - 431 - regs::NV_PFALCON_FALCON_ENGINE::update(bar, &E::ID, |v| v.set_reset(false)); 432 - 433 - self.reset_wait_mem_scrubbing(bar)?; 434 - 435 - Ok(()) 436 - } 437 - 438 391 /// Reset the controller, select the falcon core, and wait for memory scrubbing to complete. 439 392 pub(crate) fn reset(&self, bar: &Bar0) -> Result { 440 - self.reset_eng(bar)?; 393 + self.hal.reset_eng(bar)?; 441 394 self.hal.select_core(self, bar)?; 442 - self.reset_wait_mem_scrubbing(bar)?; 395 + self.hal.reset_wait_mem_scrubbing(bar)?; 443 396 444 397 regs::NV_PFALCON_FALCON_RM::default() 445 398 .set_value(regs::NV_PMC_BOOT_0::read(bar).into()) ··· 421 448 fw: &F, 422 449 target_mem: FalconMem, 423 450 load_offsets: FalconLoadTarget, 424 - sec: bool, 425 451 ) -> Result { 426 452 const DMA_LEN: u32 = 256; 427 453 ··· 429 457 // 430 458 // For DMEM we can fold the start offset into the DMA handle. 431 459 let (src_start, dma_start) = match target_mem { 432 - FalconMem::Imem => (load_offsets.src_start, fw.dma_handle()), 460 + FalconMem::ImemSecure | FalconMem::ImemNonSecure => { 461 + (load_offsets.src_start, fw.dma_handle()) 462 + } 433 463 FalconMem::Dmem => ( 434 464 0, 435 465 fw.dma_handle_with_offset(load_offsets.src_start.into_safe_cast())?, ··· 440 466 if dma_start % DmaAddress::from(DMA_LEN) > 0 { 441 467 dev_err!( 442 468 self.dev, 443 - "DMA transfer start addresses must be a multiple of {}", 469 + "DMA transfer start addresses must be a multiple of {}\n", 444 470 DMA_LEN 445 471 ); 446 472 return Err(EINVAL); 473 + } 474 + 475 + // The DMATRFBASE/1 register pair only supports a 49-bit address. 476 + if dma_start > DmaMask::new::<49>().value() { 477 + dev_err!(self.dev, "DMA address {:#x} exceeds 49 bits\n", dma_start); 478 + return Err(ERANGE); 447 479 } 448 480 449 481 // DMA transfers can only be done in units of 256 bytes. Compute how many such transfers we ··· 463 483 .and_then(|size| size.checked_add(load_offsets.src_start)) 464 484 { 465 485 None => { 466 - dev_err!(self.dev, "DMA transfer length overflow"); 486 + dev_err!(self.dev, "DMA transfer length overflow\n"); 467 487 return Err(EOVERFLOW); 468 488 } 469 489 Some(upper_bound) if usize::from_safe_cast(upper_bound) > fw.size() => { 470 - dev_err!(self.dev, "DMA transfer goes beyond range of DMA object"); 490 + dev_err!(self.dev, "DMA transfer goes beyond range of DMA object\n"); 471 491 return Err(EINVAL); 472 492 } 473 493 Some(_) => (), ··· 488 508 489 509 let cmd = regs::NV_PFALCON_FALCON_DMATRFCMD::default() 490 510 .set_size(DmaTrfCmdSize::Size256B) 491 - .set_imem(target_mem == FalconMem::Imem) 492 - .set_sec(if sec { 1 } else { 0 }); 511 + .with_falcon_mem(target_mem); 493 512 494 513 for pos in (0..num_transfers).map(|i| i * DMA_LEN) { 495 514 // Perform a transfer of size `DMA_LEN`. ··· 515 536 } 516 537 517 538 /// Perform a DMA load into `IMEM` and `DMEM` of `fw`, and prepare the falcon to run it. 518 - pub(crate) fn dma_load<F: FalconFirmware<Target = E>>(&self, bar: &Bar0, fw: &F) -> Result { 539 + fn dma_load<F: FalconFirmware<Target = E>>(&self, bar: &Bar0, fw: &F) -> Result { 540 + // The Non-Secure section only exists on firmware used by Turing and GA100, and 541 + // those platforms do not use DMA. 542 + if fw.imem_ns_load_params().is_some() { 543 + debug_assert!(false); 544 + return Err(EINVAL); 545 + } 546 + 519 547 self.dma_reset(bar); 520 548 regs::NV_PFALCON_FBIF_TRANSCFG::update(bar, &E::ID, 0, |v| { 521 549 v.set_target(FalconFbifTarget::CoherentSysmem) 522 550 .set_mem_type(FalconFbifMemType::Physical) 523 551 }); 524 552 525 - self.dma_wr(bar, fw, FalconMem::Imem, fw.imem_load_params(), true)?; 526 - self.dma_wr(bar, fw, FalconMem::Dmem, fw.dmem_load_params(), true)?; 553 + self.dma_wr(bar, fw, FalconMem::ImemSecure, fw.imem_sec_load_params())?; 554 + self.dma_wr(bar, fw, FalconMem::Dmem, fw.dmem_load_params())?; 527 555 528 556 self.hal.program_brom(self, bar, &fw.brom_params())?; 529 557 ··· 637 651 /// 638 652 /// Returns `true` if the RISC-V core is active, `false` otherwise. 639 653 pub(crate) fn is_riscv_active(&self, bar: &Bar0) -> bool { 640 - let cpuctl = regs::NV_PRISCV_RISCV_CPUCTL::read(bar, &E::ID); 641 - cpuctl.active_stat() 654 + self.hal.is_riscv_active(bar) 655 + } 656 + 657 + // Load a firmware image into Falcon memory 658 + pub(crate) fn load<F: FalconFirmware<Target = E>>(&self, bar: &Bar0, fw: &F) -> Result { 659 + match self.hal.load_method() { 660 + LoadMethod::Dma => self.dma_load(bar, fw), 661 + LoadMethod::Pio => Err(ENOTSUPP), 662 + } 642 663 } 643 664 644 665 /// Write the application version to the OS register.

+26

drivers/gpu/nova-core/falcon/hal.rs

··· 13 13 }; 14 14 15 15 mod ga102; 16 + mod tu102; 17 + 18 + /// Method used to load data into falcon memory. Some GPU architectures need 19 + /// PIO and others can use DMA. 20 + pub(crate) enum LoadMethod { 21 + /// Programmed I/O 22 + Pio, 23 + /// Direct Memory Access 24 + Dma, 25 + } 16 26 17 27 /// Hardware Abstraction Layer for Falcon cores. 18 28 /// ··· 47 37 48 38 /// Program the boot ROM registers prior to starting a secure firmware. 49 39 fn program_brom(&self, falcon: &Falcon<E>, bar: &Bar0, params: &FalconBromParams) -> Result; 40 + 41 + /// Check if the RISC-V core is active. 42 + /// Returns `true` if the RISC-V core is active, `false` otherwise. 43 + fn is_riscv_active(&self, bar: &Bar0) -> bool; 44 + 45 + /// Wait for memory scrubbing to complete. 46 + fn reset_wait_mem_scrubbing(&self, bar: &Bar0) -> Result; 47 + 48 + /// Reset the falcon engine. 49 + fn reset_eng(&self, bar: &Bar0) -> Result; 50 + 51 + /// returns the method needed to load data into Falcon memory 52 + fn load_method(&self) -> LoadMethod; 50 53 } 51 54 52 55 /// Returns a boxed falcon HAL adequate for `chipset`. ··· 73 50 use Chipset::*; 74 51 75 52 let hal = match chipset { 53 + TU102 | TU104 | TU106 | TU116 | TU117 => { 54 + KBox::new(tu102::Tu102::<E>::new(), GFP_KERNEL)? as KBox<dyn FalconHal<E>> 55 + } 76 56 GA102 | GA103 | GA104 | GA106 | GA107 | AD102 | AD103 | AD104 | AD106 | AD107 => { 77 57 KBox::new(ga102::Ga102::<E>::new(), GFP_KERNEL)? as KBox<dyn FalconHal<E>> 78 58 }

+41 -2

drivers/gpu/nova-core/falcon/hal/ga102.rs

··· 12 12 use crate::{ 13 13 driver::Bar0, 14 14 falcon::{ 15 + hal::LoadMethod, 15 16 Falcon, 16 17 FalconBromParams, 17 18 FalconEngine, ··· 53 52 let ucode_idx = match usize::from(ucode_id) { 54 53 ucode_id @ 1..=regs::NV_FUSE_OPT_FPF_SIZE => ucode_id - 1, 55 54 _ => { 56 - dev_err!(dev, "invalid ucode id {:#x}", ucode_id); 55 + dev_err!(dev, "invalid ucode id {:#x}\n", ucode_id); 57 56 return Err(EINVAL); 58 57 } 59 58 }; ··· 67 66 } else if engine_id_mask & 0x0400 != 0 { 68 67 regs::NV_FUSE_OPT_FPF_GSP_UCODE1_VERSION::read(bar, ucode_idx).data() 69 68 } else { 70 - dev_err!(dev, "unexpected engine_id_mask {:#x}", engine_id_mask); 69 + dev_err!(dev, "unexpected engine_id_mask {:#x}\n", engine_id_mask); 71 70 return Err(EINVAL); 72 71 }; 73 72 ··· 117 116 118 117 fn program_brom(&self, _falcon: &Falcon<E>, bar: &Bar0, params: &FalconBromParams) -> Result { 119 118 program_brom_ga102::<E>(bar, params) 119 + } 120 + 121 + fn is_riscv_active(&self, bar: &Bar0) -> bool { 122 + let cpuctl = regs::NV_PRISCV_RISCV_CPUCTL::read(bar, &E::ID); 123 + cpuctl.active_stat() 124 + } 125 + 126 + fn reset_wait_mem_scrubbing(&self, bar: &Bar0) -> Result { 127 + // TIMEOUT: memory scrubbing should complete in less than 20ms. 128 + read_poll_timeout( 129 + || Ok(regs::NV_PFALCON_FALCON_HWCFG2::read(bar, &E::ID)), 130 + |r| r.mem_scrubbing_done(), 131 + Delta::ZERO, 132 + Delta::from_millis(20), 133 + ) 134 + .map(|_| ()) 135 + } 136 + 137 + fn reset_eng(&self, bar: &Bar0) -> Result { 138 + let _ = regs::NV_PFALCON_FALCON_HWCFG2::read(bar, &E::ID); 139 + 140 + // According to OpenRM's `kflcnPreResetWait_GA102` documentation, HW sometimes does not set 141 + // RESET_READY so a non-failing timeout is used. 142 + let _ = read_poll_timeout( 143 + || Ok(regs::NV_PFALCON_FALCON_HWCFG2::read(bar, &E::ID)), 144 + |r| r.reset_ready(), 145 + Delta::ZERO, 146 + Delta::from_micros(150), 147 + ); 148 + 149 + regs::NV_PFALCON_FALCON_ENGINE::reset_engine::<E>(bar); 150 + self.reset_wait_mem_scrubbing(bar)?; 151 + 152 + Ok(()) 153 + } 154 + 155 + fn load_method(&self) -> LoadMethod { 156 + LoadMethod::Dma 120 157 } 121 158 }

+77

drivers/gpu/nova-core/falcon/hal/tu102.rs

··· 1 + // SPDX-License-Identifier: GPL-2.0 2 + 3 + use core::marker::PhantomData; 4 + 5 + use kernel::{ 6 + io::poll::read_poll_timeout, 7 + prelude::*, 8 + time::Delta, // 9 + }; 10 + 11 + use crate::{ 12 + driver::Bar0, 13 + falcon::{ 14 + hal::LoadMethod, 15 + Falcon, 16 + FalconBromParams, 17 + FalconEngine, // 18 + }, 19 + regs, // 20 + }; 21 + 22 + use super::FalconHal; 23 + 24 + pub(super) struct Tu102<E: FalconEngine>(PhantomData<E>); 25 + 26 + impl<E: FalconEngine> Tu102<E> { 27 + pub(super) fn new() -> Self { 28 + Self(PhantomData) 29 + } 30 + } 31 + 32 + impl<E: FalconEngine> FalconHal<E> for Tu102<E> { 33 + fn select_core(&self, _falcon: &Falcon<E>, _bar: &Bar0) -> Result { 34 + Ok(()) 35 + } 36 + 37 + fn signature_reg_fuse_version( 38 + &self, 39 + _falcon: &Falcon<E>, 40 + _bar: &Bar0, 41 + _engine_id_mask: u16, 42 + _ucode_id: u8, 43 + ) -> Result<u32> { 44 + Ok(0) 45 + } 46 + 47 + fn program_brom(&self, _falcon: &Falcon<E>, _bar: &Bar0, _params: &FalconBromParams) -> Result { 48 + Ok(()) 49 + } 50 + 51 + fn is_riscv_active(&self, bar: &Bar0) -> bool { 52 + let cpuctl = regs::NV_PRISCV_RISCV_CORE_SWITCH_RISCV_STATUS::read(bar, &E::ID); 53 + cpuctl.active_stat() 54 + } 55 + 56 + fn reset_wait_mem_scrubbing(&self, bar: &Bar0) -> Result { 57 + // TIMEOUT: memory scrubbing should complete in less than 10ms. 58 + read_poll_timeout( 59 + || Ok(regs::NV_PFALCON_FALCON_DMACTL::read(bar, &E::ID)), 60 + |r| r.mem_scrubbing_done(), 61 + Delta::ZERO, 62 + Delta::from_millis(10), 63 + ) 64 + .map(|_| ()) 65 + } 66 + 67 + fn reset_eng(&self, bar: &Bar0) -> Result { 68 + regs::NV_PFALCON_FALCON_ENGINE::reset_engine::<E>(bar); 69 + self.reset_wait_mem_scrubbing(bar)?; 70 + 71 + Ok(()) 72 + } 73 + 74 + fn load_method(&self) -> LoadMethod { 75 + LoadMethod::Pio 76 + } 77 + }

+1 -1

drivers/gpu/nova-core/fb.rs

··· 80 80 let _ = hal.write_sysmem_flush_page(bar, 0).inspect_err(|e| { 81 81 dev_warn!( 82 82 &self.device, 83 - "failed to unregister sysmem flush page: {:?}", 83 + "failed to unregister sysmem flush page: {:?}\n", 84 84 e 85 85 ) 86 86 });

+199 -4

drivers/gpu/nova-core/firmware.rs

··· 4 4 //! to be loaded into a given execution unit. 5 5 6 6 use core::marker::PhantomData; 7 + use core::ops::Deref; 7 8 8 9 use kernel::{ 9 10 device, ··· 16 15 17 16 use crate::{ 18 17 dma::DmaObject, 19 - falcon::FalconFirmware, 18 + falcon::{ 19 + FalconFirmware, 20 + FalconLoadTarget, // 21 + }, 20 22 gpu, 21 23 num::{ 22 24 FromSafeCast, ··· 46 42 CString::try_from_fmt(fmt!("nvidia/{chip_name}/gsp/{name}-{ver}.bin")) 47 43 .and_then(|path| firmware::Firmware::request(&path, dev)) 48 44 } 45 + 46 + /// Structure used to describe some firmwares, notably FWSEC-FRTS. 47 + #[repr(C)] 48 + #[derive(Debug, Clone)] 49 + pub(crate) struct FalconUCodeDescV2 { 50 + /// Header defined by 'NV_BIT_FALCON_UCODE_DESC_HEADER_VDESC*' in OpenRM. 51 + hdr: u32, 52 + /// Stored size of the ucode after the header, compressed or uncompressed 53 + stored_size: u32, 54 + /// Uncompressed size of the ucode. If store_size == uncompressed_size, then the ucode 55 + /// is not compressed. 56 + pub(crate) uncompressed_size: u32, 57 + /// Code entry point 58 + pub(crate) virtual_entry: u32, 59 + /// Offset after the code segment at which the Application Interface Table headers are located. 60 + pub(crate) interface_offset: u32, 61 + /// Base address at which to load the code segment into 'IMEM'. 62 + pub(crate) imem_phys_base: u32, 63 + /// Size in bytes of the code to copy into 'IMEM'. 64 + pub(crate) imem_load_size: u32, 65 + /// Virtual 'IMEM' address (i.e. 'tag') at which the code should start. 66 + pub(crate) imem_virt_base: u32, 67 + /// Virtual address of secure IMEM segment. 68 + pub(crate) imem_sec_base: u32, 69 + /// Size of secure IMEM segment. 70 + pub(crate) imem_sec_size: u32, 71 + /// Offset into stored (uncompressed) image at which DMEM begins. 72 + pub(crate) dmem_offset: u32, 73 + /// Base address at which to load the data segment into 'DMEM'. 74 + pub(crate) dmem_phys_base: u32, 75 + /// Size in bytes of the data to copy into 'DMEM'. 76 + pub(crate) dmem_load_size: u32, 77 + /// "Alternate" Size of data to load into IMEM. 78 + pub(crate) alt_imem_load_size: u32, 79 + /// "Alternate" Size of data to load into DMEM. 80 + pub(crate) alt_dmem_load_size: u32, 81 + } 82 + 83 + // SAFETY: all bit patterns are valid for this type, and it doesn't use interior mutability. 84 + unsafe impl FromBytes for FalconUCodeDescV2 {} 49 85 50 86 /// Structure used to describe some firmwares, notably FWSEC-FRTS. 51 87 #[repr(C)] ··· 120 76 _reserved: u16, 121 77 } 122 78 123 - impl FalconUCodeDescV3 { 79 + // SAFETY: all bit patterns are valid for this type, and it doesn't use 80 + // interior mutability. 81 + unsafe impl FromBytes for FalconUCodeDescV3 {} 82 + 83 + /// Enum wrapping the different versions of Falcon microcode descriptors. 84 + /// 85 + /// This allows handling both V2 and V3 descriptor formats through a 86 + /// unified type, providing version-agnostic access to firmware metadata 87 + /// via the [`FalconUCodeDescriptor`] trait. 88 + #[derive(Debug, Clone)] 89 + pub(crate) enum FalconUCodeDesc { 90 + V2(FalconUCodeDescV2), 91 + V3(FalconUCodeDescV3), 92 + } 93 + 94 + impl Deref for FalconUCodeDesc { 95 + type Target = dyn FalconUCodeDescriptor; 96 + 97 + fn deref(&self) -> &Self::Target { 98 + match self { 99 + FalconUCodeDesc::V2(v2) => v2, 100 + FalconUCodeDesc::V3(v3) => v3, 101 + } 102 + } 103 + } 104 + 105 + /// Trait providing a common interface for accessing Falcon microcode descriptor fields. 106 + /// 107 + /// This trait abstracts over the different descriptor versions ([`FalconUCodeDescV2`] and 108 + /// [`FalconUCodeDescV3`]), allowing code to work with firmware metadata without needing to 109 + /// know the specific descriptor version. Fields not present return zero. 110 + pub(crate) trait FalconUCodeDescriptor { 111 + fn hdr(&self) -> u32; 112 + fn imem_load_size(&self) -> u32; 113 + fn interface_offset(&self) -> u32; 114 + fn dmem_load_size(&self) -> u32; 115 + fn pkc_data_offset(&self) -> u32; 116 + fn engine_id_mask(&self) -> u16; 117 + fn ucode_id(&self) -> u8; 118 + fn signature_count(&self) -> u8; 119 + fn signature_versions(&self) -> u16; 120 + 124 121 /// Returns the size in bytes of the header. 125 - pub(crate) fn size(&self) -> usize { 122 + fn size(&self) -> usize { 123 + let hdr = self.hdr(); 124 + 126 125 const HDR_SIZE_SHIFT: u32 = 16; 127 126 const HDR_SIZE_MASK: u32 = 0xffff0000; 127 + ((hdr & HDR_SIZE_MASK) >> HDR_SIZE_SHIFT).into_safe_cast() 128 + } 128 129 129 - ((self.hdr & HDR_SIZE_MASK) >> HDR_SIZE_SHIFT).into_safe_cast() 130 + fn imem_sec_load_params(&self) -> FalconLoadTarget; 131 + fn imem_ns_load_params(&self) -> Option<FalconLoadTarget>; 132 + fn dmem_load_params(&self) -> FalconLoadTarget; 133 + } 134 + 135 + impl FalconUCodeDescriptor for FalconUCodeDescV2 { 136 + fn hdr(&self) -> u32 { 137 + self.hdr 138 + } 139 + fn imem_load_size(&self) -> u32 { 140 + self.imem_load_size 141 + } 142 + fn interface_offset(&self) -> u32 { 143 + self.interface_offset 144 + } 145 + fn dmem_load_size(&self) -> u32 { 146 + self.dmem_load_size 147 + } 148 + fn pkc_data_offset(&self) -> u32 { 149 + 0 150 + } 151 + fn engine_id_mask(&self) -> u16 { 152 + 0 153 + } 154 + fn ucode_id(&self) -> u8 { 155 + 0 156 + } 157 + fn signature_count(&self) -> u8 { 158 + 0 159 + } 160 + fn signature_versions(&self) -> u16 { 161 + 0 162 + } 163 + 164 + fn imem_sec_load_params(&self) -> FalconLoadTarget { 165 + FalconLoadTarget { 166 + src_start: 0, 167 + dst_start: self.imem_sec_base, 168 + len: self.imem_sec_size, 169 + } 170 + } 171 + 172 + fn imem_ns_load_params(&self) -> Option<FalconLoadTarget> { 173 + Some(FalconLoadTarget { 174 + src_start: 0, 175 + dst_start: self.imem_phys_base, 176 + len: self.imem_load_size.checked_sub(self.imem_sec_size)?, 177 + }) 178 + } 179 + 180 + fn dmem_load_params(&self) -> FalconLoadTarget { 181 + FalconLoadTarget { 182 + src_start: self.dmem_offset, 183 + dst_start: self.dmem_phys_base, 184 + len: self.dmem_load_size, 185 + } 186 + } 187 + } 188 + 189 + impl FalconUCodeDescriptor for FalconUCodeDescV3 { 190 + fn hdr(&self) -> u32 { 191 + self.hdr 192 + } 193 + fn imem_load_size(&self) -> u32 { 194 + self.imem_load_size 195 + } 196 + fn interface_offset(&self) -> u32 { 197 + self.interface_offset 198 + } 199 + fn dmem_load_size(&self) -> u32 { 200 + self.dmem_load_size 201 + } 202 + fn pkc_data_offset(&self) -> u32 { 203 + self.pkc_data_offset 204 + } 205 + fn engine_id_mask(&self) -> u16 { 206 + self.engine_id_mask 207 + } 208 + fn ucode_id(&self) -> u8 { 209 + self.ucode_id 210 + } 211 + fn signature_count(&self) -> u8 { 212 + self.signature_count 213 + } 214 + fn signature_versions(&self) -> u16 { 215 + self.signature_versions 216 + } 217 + 218 + fn imem_sec_load_params(&self) -> FalconLoadTarget { 219 + FalconLoadTarget { 220 + src_start: 0, 221 + dst_start: self.imem_phys_base, 222 + len: self.imem_load_size, 223 + } 224 + } 225 + 226 + fn imem_ns_load_params(&self) -> Option<FalconLoadTarget> { 227 + // Not used on V3 platforms 228 + None 229 + } 230 + 231 + fn dmem_load_params(&self) -> FalconLoadTarget { 232 + FalconLoadTarget { 233 + src_start: self.imem_load_size, 234 + dst_start: self.dmem_phys_base, 235 + len: self.dmem_load_size, 236 + } 130 237 } 131 238 } 132 239

+36 -7

drivers/gpu/nova-core/firmware/booter.rs

··· 251 251 252 252 /// The `Booter` loader firmware, responsible for loading the GSP. 253 253 pub(crate) struct BooterFirmware { 254 - // Load parameters for `IMEM` falcon memory. 255 - imem_load_target: FalconLoadTarget, 254 + // Load parameters for Secure `IMEM` falcon memory. 255 + imem_sec_load_target: FalconLoadTarget, 256 + // Load parameters for Non-Secure `IMEM` falcon memory, 257 + // used only on Turing and GA100 258 + imem_ns_load_target: Option<FalconLoadTarget>, 256 259 // Load parameters for `DMEM` falcon memory. 257 260 dmem_load_target: FalconLoadTarget, 258 261 // BROM falcon parameters. ··· 356 353 } 357 354 }; 358 355 356 + // There are two versions of Booter, one for Turing/GA100, and another for 357 + // GA102+. The extraction of the IMEM sections differs between the two 358 + // versions. Unfortunately, the file names are the same, and the headers 359 + // don't indicate the versions. The only way to differentiate is by the Chipset. 360 + let (imem_sec_dst_start, imem_ns_load_target) = if chipset <= Chipset::GA100 { 361 + ( 362 + app0.offset, 363 + Some(FalconLoadTarget { 364 + src_start: 0, 365 + dst_start: load_hdr.os_code_offset, 366 + len: load_hdr.os_code_size, 367 + }), 368 + ) 369 + } else { 370 + (0, None) 371 + }; 372 + 359 373 Ok(Self { 360 - imem_load_target: FalconLoadTarget { 374 + imem_sec_load_target: FalconLoadTarget { 361 375 src_start: app0.offset, 362 - dst_start: 0, 376 + dst_start: imem_sec_dst_start, 363 377 len: app0.len, 364 378 }, 379 + imem_ns_load_target, 365 380 dmem_load_target: FalconLoadTarget { 366 381 src_start: load_hdr.os_data_offset, 367 382 dst_start: 0, ··· 392 371 } 393 372 394 373 impl FalconLoadParams for BooterFirmware { 395 - fn imem_load_params(&self) -> FalconLoadTarget { 396 - self.imem_load_target.clone() 374 + fn imem_sec_load_params(&self) -> FalconLoadTarget { 375 + self.imem_sec_load_target.clone() 376 + } 377 + 378 + fn imem_ns_load_params(&self) -> Option<FalconLoadTarget> { 379 + self.imem_ns_load_target.clone() 397 380 } 398 381 399 382 fn dmem_load_params(&self) -> FalconLoadTarget { ··· 409 384 } 410 385 411 386 fn boot_addr(&self) -> u32 { 412 - self.imem_load_target.src_start 387 + if let Some(ns_target) = &self.imem_ns_load_target { 388 + ns_target.dst_start 389 + } else { 390 + self.imem_sec_load_target.src_start 391 + } 413 392 } 414 393 } 415 394

+24 -27

drivers/gpu/nova-core/firmware/fwsec.rs

··· 40 40 FalconLoadTarget, // 41 41 }, 42 42 firmware::{ 43 - FalconUCodeDescV3, 43 + FalconUCodeDesc, 44 44 FirmwareDmaObject, 45 45 FirmwareSignature, 46 46 Signed, ··· 218 218 /// It is responsible for e.g. carving out the WPR2 region as the first step of the GSP bootflow. 219 219 pub(crate) struct FwsecFirmware { 220 220 /// Descriptor of the firmware. 221 - desc: FalconUCodeDescV3, 221 + desc: FalconUCodeDesc, 222 222 /// GPU-accessible DMA object containing the firmware. 223 223 ucode: FirmwareDmaObject<Self, Signed>, 224 224 } 225 225 226 226 impl FalconLoadParams for FwsecFirmware { 227 - fn imem_load_params(&self) -> FalconLoadTarget { 228 - FalconLoadTarget { 229 - src_start: 0, 230 - dst_start: self.desc.imem_phys_base, 231 - len: self.desc.imem_load_size, 232 - } 227 + fn imem_sec_load_params(&self) -> FalconLoadTarget { 228 + self.desc.imem_sec_load_params() 229 + } 230 + 231 + fn imem_ns_load_params(&self) -> Option<FalconLoadTarget> { 232 + self.desc.imem_ns_load_params() 233 233 } 234 234 235 235 fn dmem_load_params(&self) -> FalconLoadTarget { 236 - FalconLoadTarget { 237 - src_start: self.desc.imem_load_size, 238 - dst_start: self.desc.dmem_phys_base, 239 - len: self.desc.dmem_load_size, 240 - } 236 + self.desc.dmem_load_params() 241 237 } 242 238 243 239 fn brom_params(&self) -> FalconBromParams { 244 240 FalconBromParams { 245 - pkc_data_offset: self.desc.pkc_data_offset, 246 - engine_id_mask: self.desc.engine_id_mask, 247 - ucode_id: self.desc.ucode_id, 241 + pkc_data_offset: self.desc.pkc_data_offset(), 242 + engine_id_mask: self.desc.engine_id_mask(), 243 + ucode_id: self.desc.ucode_id(), 248 244 } 249 245 } 250 246 ··· 264 268 impl FirmwareDmaObject<FwsecFirmware, Unsigned> { 265 269 fn new_fwsec(dev: &Device<device::Bound>, bios: &Vbios, cmd: FwsecCommand) -> Result<Self> { 266 270 let desc = bios.fwsec_image().header()?; 267 - let ucode = bios.fwsec_image().ucode(desc)?; 271 + let ucode = bios.fwsec_image().ucode(&desc)?; 268 272 let mut dma_object = DmaObject::from_data(dev, ucode)?; 269 273 270 - let hdr_offset = usize::from_safe_cast(desc.imem_load_size + desc.interface_offset); 274 + let hdr_offset = usize::from_safe_cast(desc.imem_load_size() + desc.interface_offset()); 271 275 // SAFETY: we have exclusive access to `dma_object`. 272 276 let hdr: &FalconAppifHdrV1 = unsafe { transmute(&dma_object, hdr_offset) }?; 273 277 ··· 294 298 let dmem_mapper: &mut FalconAppifDmemmapperV3 = unsafe { 295 299 transmute_mut( 296 300 &mut dma_object, 297 - (desc.imem_load_size + dmem_base).into_safe_cast(), 301 + (desc.imem_load_size() + dmem_base).into_safe_cast(), 298 302 ) 299 303 }?; 300 304 ··· 308 312 let frts_cmd: &mut FrtsCmd = unsafe { 309 313 transmute_mut( 310 314 &mut dma_object, 311 - (desc.imem_load_size + cmd_in_buffer_offset).into_safe_cast(), 315 + (desc.imem_load_size() + cmd_in_buffer_offset).into_safe_cast(), 312 316 ) 313 317 }?; 314 318 ··· 355 359 356 360 // Patch signature if needed. 357 361 let desc = bios.fwsec_image().header()?; 358 - let ucode_signed = if desc.signature_count != 0 { 359 - let sig_base_img = usize::from_safe_cast(desc.imem_load_size + desc.pkc_data_offset); 360 - let desc_sig_versions = u32::from(desc.signature_versions); 362 + let ucode_signed = if desc.signature_count() != 0 { 363 + let sig_base_img = 364 + usize::from_safe_cast(desc.imem_load_size() + desc.pkc_data_offset()); 365 + let desc_sig_versions = u32::from(desc.signature_versions()); 361 366 let reg_fuse_version = 362 - falcon.signature_reg_fuse_version(bar, desc.engine_id_mask, desc.ucode_id)?; 367 + falcon.signature_reg_fuse_version(bar, desc.engine_id_mask(), desc.ucode_id())?; 363 368 dev_dbg!( 364 369 dev, 365 370 "desc_sig_versions: {:#x}, reg_fuse_version: {}\n", ··· 394 397 dev_dbg!(dev, "patching signature with index {}\n", signature_idx); 395 398 let signature = bios 396 399 .fwsec_image() 397 - .sigs(desc) 400 + .sigs(&desc) 398 401 .and_then(|sigs| sigs.get(signature_idx).ok_or(EINVAL))?; 399 402 400 403 ucode_dma.patch_signature(signature, sig_base_img)? ··· 403 406 }; 404 407 405 408 Ok(FwsecFirmware { 406 - desc: desc.clone(), 409 + desc, 407 410 ucode: ucode_signed, 408 411 }) 409 412 } ··· 420 423 .reset(bar) 421 424 .inspect_err(|e| dev_err!(dev, "Failed to reset GSP falcon: {:?}\n", e))?; 422 425 falcon 423 - .dma_load(bar, self) 426 + .load(bar, self) 424 427 .inspect_err(|e| dev_err!(dev, "Failed to load FWSEC firmware: {:?}\n", e))?; 425 428 let (mbox0, _) = falcon 426 429 .boot(bar, Some(0), None)

+77 -69

drivers/gpu/nova-core/firmware/gsp.rs

··· 93 93 94 94 // Get the start of the name. 95 95 elf.get(name_idx..) 96 - // Stop at the first `0`. 97 - .and_then(|nstr| nstr.get(0..=nstr.iter().position(|b| *b == 0)?)) 98 - // Convert into CStr. This should never fail because of the line above. 99 - .and_then(|nstr| CStr::from_bytes_with_nul(nstr).ok()) 96 + .and_then(|nstr| CStr::from_bytes_until_nul(nstr).ok()) 100 97 // Convert into str. 101 98 .and_then(|c_str| c_str.to_str().ok()) 102 99 // Check that the name matches. ··· 150 153 impl GspFirmware { 151 154 /// Loads the GSP firmware binaries, map them into `dev`'s address-space, and creates the page 152 155 /// tables expected by the GSP bootloader to load it. 153 - pub(crate) fn new<'a, 'b>( 156 + pub(crate) fn new<'a>( 154 157 dev: &'a device::Device<device::Bound>, 155 158 chipset: Chipset, 156 - ver: &'b str, 157 - ) -> Result<impl PinInit<Self, Error> + 'a> { 158 - let fw = super::request_firmware(dev, chipset, "gsp", ver)?; 159 + ver: &'a str, 160 + ) -> impl PinInit<Self, Error> + 'a { 161 + pin_init::pin_init_scope(move || { 162 + let firmware = super::request_firmware(dev, chipset, "gsp", ver)?; 159 163 160 - let fw_section = elf::elf64_section(fw.data(), ".fwimage").ok_or(EINVAL)?; 164 + let fw_section = elf::elf64_section(firmware.data(), ".fwimage").ok_or(EINVAL)?; 161 165 162 - let sigs_section = match chipset.arch() { 163 - Architecture::Ampere => ".fwsignature_ga10x", 164 - Architecture::Ada => ".fwsignature_ad10x", 165 - _ => return Err(ENOTSUPP), 166 - }; 167 - let signatures = elf::elf64_section(fw.data(), sigs_section) 168 - .ok_or(EINVAL) 169 - .and_then(|data| DmaObject::from_data(dev, data))?; 166 + let size = fw_section.len(); 170 167 171 - let size = fw_section.len(); 168 + // Move the firmware into a vmalloc'd vector and map it into the device address 169 + // space. 170 + let fw_vvec = VVec::with_capacity(fw_section.len(), GFP_KERNEL) 171 + .and_then(|mut v| { 172 + v.extend_from_slice(fw_section, GFP_KERNEL)?; 173 + Ok(v) 174 + }) 175 + .map_err(|_| ENOMEM)?; 172 176 173 - // Move the firmware into a vmalloc'd vector and map it into the device address 174 - // space. 175 - let fw_vvec = VVec::with_capacity(fw_section.len(), GFP_KERNEL) 176 - .and_then(|mut v| { 177 - v.extend_from_slice(fw_section, GFP_KERNEL)?; 178 - Ok(v) 179 - }) 180 - .map_err(|_| ENOMEM)?; 177 + Ok(try_pin_init!(Self { 178 + fw <- SGTable::new(dev, fw_vvec, DataDirection::ToDevice, GFP_KERNEL), 179 + level2 <- { 180 + // Allocate the level 2 page table, map the firmware onto it, and map it into 181 + // the device address space. 182 + VVec::<u8>::with_capacity( 183 + fw.iter().count() * core::mem::size_of::<u64>(), 184 + GFP_KERNEL, 185 + ) 186 + .map_err(|_| ENOMEM) 187 + .and_then(|level2| map_into_lvl(&fw, level2)) 188 + .map(|level2| SGTable::new(dev, level2, DataDirection::ToDevice, GFP_KERNEL))? 189 + }, 190 + level1 <- { 191 + // Allocate the level 1 page table, map the level 2 page table onto it, and map 192 + // it into the device address space. 193 + VVec::<u8>::with_capacity( 194 + level2.iter().count() * core::mem::size_of::<u64>(), 195 + GFP_KERNEL, 196 + ) 197 + .map_err(|_| ENOMEM) 198 + .and_then(|level1| map_into_lvl(&level2, level1)) 199 + .map(|level1| SGTable::new(dev, level1, DataDirection::ToDevice, GFP_KERNEL))? 200 + }, 201 + level0: { 202 + // Allocate the level 0 page table as a device-visible DMA object, and map the 203 + // level 1 page table onto it. 181 204 182 - let bl = super::request_firmware(dev, chipset, "bootloader", ver)?; 183 - let bootloader = RiscvFirmware::new(dev, &bl)?; 205 + // Level 0 page table data. 206 + let mut level0_data = kvec![0u8; GSP_PAGE_SIZE]?; 184 207 185 - Ok(try_pin_init!(Self { 186 - fw <- SGTable::new(dev, fw_vvec, DataDirection::ToDevice, GFP_KERNEL), 187 - level2 <- { 188 - // Allocate the level 2 page table, map the firmware onto it, and map it into the 189 - // device address space. 190 - VVec::<u8>::with_capacity( 191 - fw.iter().count() * core::mem::size_of::<u64>(), 192 - GFP_KERNEL, 193 - ) 194 - .map_err(|_| ENOMEM) 195 - .and_then(|level2| map_into_lvl(&fw, level2)) 196 - .map(|level2| SGTable::new(dev, level2, DataDirection::ToDevice, GFP_KERNEL))? 197 - }, 198 - level1 <- { 199 - // Allocate the level 1 page table, map the level 2 page table onto it, and map it 200 - // into the device address space. 201 - VVec::<u8>::with_capacity( 202 - level2.iter().count() * core::mem::size_of::<u64>(), 203 - GFP_KERNEL, 204 - ) 205 - .map_err(|_| ENOMEM) 206 - .and_then(|level1| map_into_lvl(&level2, level1)) 207 - .map(|level1| SGTable::new(dev, level1, DataDirection::ToDevice, GFP_KERNEL))? 208 - }, 209 - level0: { 210 - // Allocate the level 0 page table as a device-visible DMA object, and map the 211 - // level 1 page table onto it. 208 + // Fill level 1 page entry. 209 + let level1_entry = level1.iter().next().ok_or(EINVAL)?; 210 + let level1_entry_addr = level1_entry.dma_address(); 211 + let dst = &mut level0_data[..size_of_val(&level1_entry_addr)]; 212 + dst.copy_from_slice(&level1_entry_addr.to_le_bytes()); 212 213 213 - // Level 0 page table data. 214 - let mut level0_data = kvec![0u8; GSP_PAGE_SIZE]?; 214 + // Turn the level0 page table into a [`DmaObject`]. 215 + DmaObject::from_data(dev, &level0_data)? 216 + }, 217 + size, 218 + signatures: { 219 + let sigs_section = match chipset.arch() { 220 + Architecture::Turing 221 + if matches!(chipset, Chipset::TU116 | Chipset::TU117) => 222 + { 223 + ".fwsignature_tu11x" 224 + } 225 + Architecture::Turing => ".fwsignature_tu10x", 226 + // GA100 uses the same firmware as Turing 227 + Architecture::Ampere if chipset == Chipset::GA100 => ".fwsignature_tu10x", 228 + Architecture::Ampere => ".fwsignature_ga10x", 229 + Architecture::Ada => ".fwsignature_ad10x", 230 + }; 215 231 216 - // Fill level 1 page entry. 217 - let level1_entry = level1.iter().next().ok_or(EINVAL)?; 218 - let level1_entry_addr = level1_entry.dma_address(); 219 - let dst = &mut level0_data[..size_of_val(&level1_entry_addr)]; 220 - dst.copy_from_slice(&level1_entry_addr.to_le_bytes()); 232 + elf::elf64_section(firmware.data(), sigs_section) 233 + .ok_or(EINVAL) 234 + .and_then(|data| DmaObject::from_data(dev, data))? 235 + }, 236 + bootloader: { 237 + let bl = super::request_firmware(dev, chipset, "bootloader", ver)?; 221 238 222 - // Turn the level0 page table into a [`DmaObject`]. 223 - DmaObject::from_data(dev, &level0_data)? 224 - }, 225 - size, 226 - signatures, 227 - bootloader, 228 - })) 239 + RiscvFirmware::new(dev, &bl)? 240 + }, 241 + })) 242 + }) 229 243 } 230 244 231 245 /// Returns the DMA handle of the radix3 level 0 page table.

+2 -2

drivers/gpu/nova-core/gpu.rs

··· 268 268 // We must wait for GFW_BOOT completion before doing any significant setup on the GPU. 269 269 _: { 270 270 gfw::wait_gfw_boot_completion(bar) 271 - .inspect_err(|_| dev_err!(pdev.as_ref(), "GFW boot did not complete"))?; 271 + .inspect_err(|_| dev_err!(pdev.as_ref(), "GFW boot did not complete\n"))?; 272 272 }, 273 273 274 274 sysmem_flush: SysmemFlush::register(pdev.as_ref(), bar, spec.chipset)?, ··· 281 281 282 282 sec2_falcon: Falcon::new(pdev.as_ref(), spec.chipset)?, 283 283 284 - gsp <- Gsp::new(pdev)?, 284 + gsp <- Gsp::new(pdev), 285 285 286 286 _: { gsp.boot(pdev, bar, spec.chipset, gsp_falcon, sec2_falcon)? }, 287 287

+36 -39

drivers/gpu/nova-core/gsp.rs

··· 27 27 use crate::{ 28 28 gsp::cmdq::Cmdq, 29 29 gsp::fw::{ 30 - GspArgumentsCached, 30 + GspArgumentsPadded, 31 31 LibosMemoryRegionInitArgument, // 32 32 }, 33 33 num, ··· 114 114 /// Command queue. 115 115 pub(crate) cmdq: Cmdq, 116 116 /// RM arguments. 117 - rmargs: CoherentAllocation<GspArgumentsCached>, 117 + rmargs: CoherentAllocation<GspArgumentsPadded>, 118 118 } 119 119 120 120 impl Gsp { 121 121 // Creates an in-place initializer for a `Gsp` manager for `pdev`. 122 - pub(crate) fn new(pdev: &pci::Device<device::Bound>) -> Result<impl PinInit<Self, Error>> { 123 - let dev = pdev.as_ref(); 124 - let libos = CoherentAllocation::<LibosMemoryRegionInitArgument>::alloc_coherent( 125 - dev, 126 - GSP_PAGE_SIZE / size_of::<LibosMemoryRegionInitArgument>(), 127 - GFP_KERNEL | __GFP_ZERO, 128 - )?; 122 + pub(crate) fn new(pdev: &pci::Device<device::Bound>) -> impl PinInit<Self, Error> + '_ { 123 + pin_init::pin_init_scope(move || { 124 + let dev = pdev.as_ref(); 129 125 130 - // Initialise the logging structures. The OpenRM equivalents are in: 131 - // _kgspInitLibosLoggingStructures (allocates memory for buffers) 132 - // kgspSetupLibosInitArgs_IMPL (creates pLibosInitArgs[] array) 133 - let loginit = LogBuffer::new(dev)?; 134 - dma_write!(libos[0] = LibosMemoryRegionInitArgument::new("LOGINIT", &loginit.0))?; 135 - 136 - let logintr = LogBuffer::new(dev)?; 137 - dma_write!(libos[1] = LibosMemoryRegionInitArgument::new("LOGINTR", &logintr.0))?; 138 - 139 - let logrm = LogBuffer::new(dev)?; 140 - dma_write!(libos[2] = LibosMemoryRegionInitArgument::new("LOGRM", &logrm.0))?; 141 - 142 - let cmdq = Cmdq::new(dev)?; 143 - 144 - let rmargs = CoherentAllocation::<GspArgumentsCached>::alloc_coherent( 145 - dev, 146 - 1, 147 - GFP_KERNEL | __GFP_ZERO, 148 - )?; 149 - dma_write!(rmargs[0] = fw::GspArgumentsCached::new(&cmdq))?; 150 - dma_write!(libos[3] = LibosMemoryRegionInitArgument::new("RMARGS", &rmargs))?; 151 - 152 - Ok(try_pin_init!(Self { 153 - libos, 154 - loginit, 155 - logintr, 156 - logrm, 157 - rmargs, 158 - cmdq, 159 - })) 126 + Ok(try_pin_init!(Self { 127 + libos: CoherentAllocation::<LibosMemoryRegionInitArgument>::alloc_coherent( 128 + dev, 129 + GSP_PAGE_SIZE / size_of::<LibosMemoryRegionInitArgument>(), 130 + GFP_KERNEL | __GFP_ZERO, 131 + )?, 132 + loginit: LogBuffer::new(dev)?, 133 + logintr: LogBuffer::new(dev)?, 134 + logrm: LogBuffer::new(dev)?, 135 + cmdq: Cmdq::new(dev)?, 136 + rmargs: CoherentAllocation::<GspArgumentsPadded>::alloc_coherent( 137 + dev, 138 + 1, 139 + GFP_KERNEL | __GFP_ZERO, 140 + )?, 141 + _: { 142 + // Initialise the logging structures. The OpenRM equivalents are in: 143 + // _kgspInitLibosLoggingStructures (allocates memory for buffers) 144 + // kgspSetupLibosInitArgs_IMPL (creates pLibosInitArgs[] array) 145 + dma_write!( 146 + libos[0] = LibosMemoryRegionInitArgument::new("LOGINIT", &loginit.0) 147 + )?; 148 + dma_write!( 149 + libos[1] = LibosMemoryRegionInitArgument::new("LOGINTR", &logintr.0) 150 + )?; 151 + dma_write!(libos[2] = LibosMemoryRegionInitArgument::new("LOGRM", &logrm.0))?; 152 + dma_write!(rmargs[0].inner = fw::GspArgumentsCached::new(cmdq))?; 153 + dma_write!(libos[3] = LibosMemoryRegionInitArgument::new("RMARGS", rmargs))?; 154 + }, 155 + })) 156 + }) 160 157 } 161 158 }

+7 -11

drivers/gpu/nova-core/gsp/boot.rs

··· 82 82 if frts_status != 0 { 83 83 dev_err!( 84 84 dev, 85 - "FWSEC-FRTS returned with error code {:#x}", 85 + "FWSEC-FRTS returned with error code {:#x}\n", 86 86 frts_status 87 87 ); 88 88 ··· 139 139 140 140 let bios = Vbios::new(dev, bar)?; 141 141 142 - let gsp_fw = KBox::pin_init( 143 - GspFirmware::new(dev, chipset, FIRMWARE_VERSION)?, 144 - GFP_KERNEL, 145 - )?; 142 + let gsp_fw = KBox::pin_init(GspFirmware::new(dev, chipset, FIRMWARE_VERSION), GFP_KERNEL)?; 146 143 147 144 let fb_layout = FbLayout::new(chipset, bar, &gsp_fw)?; 148 145 dev_dbg!(dev, "{:#x?}\n", fb_layout); ··· 183 186 ); 184 187 185 188 sec2_falcon.reset(bar)?; 186 - sec2_falcon.dma_load(bar, &booter_loader)?; 189 + sec2_falcon.load(bar, &booter_loader)?; 187 190 let wpr_handle = wpr_meta.dma_handle(); 188 191 let (mbox0, mbox1) = sec2_falcon.boot( 189 192 bar, ··· 238 241 239 242 // Obtain and display basic GPU information. 240 243 let info = commands::get_gsp_info(&mut self.cmdq, bar)?; 241 - dev_info!( 242 - pdev.as_ref(), 243 - "GPU name: {}\n", 244 - info.gpu_name().unwrap_or("invalid GPU name") 245 - ); 244 + match info.gpu_name() { 245 + Ok(name) => dev_info!(pdev.as_ref(), "GPU name: {}\n", name), 246 + Err(e) => dev_warn!(pdev.as_ref(), "GPU name unavailable: {:?}\n", e), 247 + } 246 248 247 249 Ok(()) 248 250 }

+1 -1

drivers/gpu/nova-core/gsp/cmdq.rs

··· 617 617 { 618 618 dev_err!( 619 619 self.dev, 620 - "GSP RPC: receive: Call {} - bad checksum", 620 + "GSP RPC: receive: Call {} - bad checksum\n", 621 621 header.sequence() 622 622 ); 623 623 return Err(EIO);

+23 -6

drivers/gpu/nova-core/gsp/commands.rs

··· 2 2 3 3 use core::{ 4 4 array, 5 - convert::Infallible, // 5 + convert::Infallible, 6 + ffi::FromBytesUntilNulError, 7 + str::Utf8Error, // 6 8 }; 7 9 8 10 use kernel::{ ··· 32 30 }, 33 31 }, 34 32 sbuffer::SBufferIter, 35 - util, 36 33 }; 37 34 38 35 /// The `GspSetSystemInfo` command. ··· 206 205 } 207 206 } 208 207 208 + /// Error type for [`GetGspStaticInfoReply::gpu_name`]. 209 + #[derive(Debug)] 210 + pub(crate) enum GpuNameError { 211 + /// The GPU name string does not contain a null terminator. 212 + NoNullTerminator(FromBytesUntilNulError), 213 + 214 + /// The GPU name string contains invalid UTF-8. 215 + #[expect(dead_code)] 216 + InvalidUtf8(Utf8Error), 217 + } 218 + 209 219 impl GetGspStaticInfoReply { 210 - /// Returns the name of the GPU as a string, or `None` if the string given by the GSP was 211 - /// invalid. 212 - pub(crate) fn gpu_name(&self) -> Option<&str> { 213 - util::str_from_null_terminated(&self.gpu_name) 220 + /// Returns the name of the GPU as a string. 221 + /// 222 + /// Returns an error if the string given by the GSP does not contain a null terminator or 223 + /// contains invalid UTF-8. 224 + pub(crate) fn gpu_name(&self) -> core::result::Result<&str, GpuNameError> { 225 + CStr::from_bytes_until_nul(&self.gpu_name) 226 + .map_err(GpuNameError::NoNullTerminator)? 227 + .to_str() 228 + .map_err(GpuNameError::InvalidUtf8) 214 229 } 215 230 } 216 231

+13 -1

drivers/gpu/nova-core/gsp/fw.rs

··· 904 904 // SAFETY: Padding is explicit and will not contain uninitialized data. 905 905 unsafe impl AsBytes for GspArgumentsCached {} 906 906 907 + /// On Turing and GA100, the entries in the `LibosMemoryRegionInitArgument` 908 + /// must all be a multiple of GSP_PAGE_SIZE in size, so add padding to force it 909 + /// to that size. 910 + #[repr(C)] 911 + pub(crate) struct GspArgumentsPadded { 912 + pub(crate) inner: GspArgumentsCached, 913 + _padding: [u8; GSP_PAGE_SIZE - core::mem::size_of::<bindings::GSP_ARGUMENTS_CACHED>()], 914 + } 915 + 916 + // SAFETY: Padding is explicit and will not contain uninitialized data. 917 + unsafe impl AsBytes for GspArgumentsPadded {} 918 + 907 919 // SAFETY: This struct only contains integer types for which all bit patterns 908 920 // are valid. 909 - unsafe impl FromBytes for GspArgumentsCached {} 921 + unsafe impl FromBytes for GspArgumentsPadded {} 910 922 911 923 /// Init arguments for the message queue. 912 924 #[repr(transparent)]

+7 -7

drivers/gpu/nova-core/gsp/sequencer.rs

··· 14 14 device, 15 15 io::poll::read_poll_timeout, 16 16 prelude::*, 17 + sync::aref::ARef, 17 18 time::{ 18 19 delay::fsleep, 19 20 Delta, // 20 21 }, 21 - transmute::FromBytes, 22 - types::ARef, // 22 + transmute::FromBytes, // 23 23 }; 24 24 25 25 use crate::{ ··· 121 121 }; 122 122 123 123 if data.len() < size { 124 - dev_err!(dev, "Data is not enough for command"); 124 + dev_err!(dev, "Data is not enough for command\n"); 125 125 return Err(EINVAL); 126 126 } 127 127 ··· 320 320 321 321 cmd_result.map_or_else( 322 322 |_err| { 323 - dev_err!(self.dev, "Error parsing command at offset {}", offset); 323 + dev_err!(self.dev, "Error parsing command at offset {}\n", offset); 324 324 None 325 325 }, 326 326 |(cmd, size)| { ··· 382 382 dev: params.dev, 383 383 }; 384 384 385 - dev_dbg!(sequencer.dev, "Running CPU Sequencer commands"); 385 + dev_dbg!(sequencer.dev, "Running CPU Sequencer commands\n"); 386 386 387 387 for cmd_result in sequencer.iter() { 388 388 match cmd_result { ··· 390 390 Err(e) => { 391 391 dev_err!( 392 392 sequencer.dev, 393 - "Error running command at index {}", 393 + "Error running command at index {}\n", 394 394 sequencer.seq_info.cmd_index 395 395 ); 396 396 return Err(e); ··· 400 400 401 401 dev_dbg!( 402 402 sequencer.dev, 403 - "CPU Sequencer commands completed successfully" 403 + "CPU Sequencer commands completed successfully\n" 404 404 ); 405 405 Ok(()) 406 406 }

-1

drivers/gpu/nova-core/nova_core.rs

··· 16 16 mod num; 17 17 mod regs; 18 18 mod sbuffer; 19 - mod util; 20 19 mod vbios; 21 20 22 21 pub(crate) const MODULE_NAME: &kernel::str::CStr = <LocalModule as kernel::ModuleMetadata>::NAME;

+41 -1

drivers/gpu/nova-core/regs.rs

··· 7 7 #[macro_use] 8 8 pub(crate) mod macros; 9 9 10 - use kernel::prelude::*; 10 + use kernel::{ 11 + prelude::*, 12 + time, // 13 + }; 11 14 12 15 use crate::{ 16 + driver::Bar0, 13 17 falcon::{ 14 18 DmaTrfCmdSize, 15 19 FalconCoreRev, 16 20 FalconCoreRevSubversion, 21 + FalconEngine, 17 22 FalconFbifMemType, 18 23 FalconFbifTarget, 24 + FalconMem, 19 25 FalconModSelAlgo, 20 26 FalconSecurityModel, 21 27 PFalcon2Base, ··· 312 306 7:7 secure_stat as bool; 313 307 }); 314 308 309 + impl NV_PFALCON_FALCON_DMACTL { 310 + /// Returns `true` if memory scrubbing is completed. 311 + pub(crate) fn mem_scrubbing_done(self) -> bool { 312 + !self.dmem_scrubbing() && !self.imem_scrubbing() 313 + } 314 + } 315 + 315 316 register!(NV_PFALCON_FALCON_DMATRFBASE @ PFalconBase[0x00000110] { 316 317 31:0 base as u32; 317 318 }); ··· 337 324 14:12 ctxdma as u8; 338 325 16:16 set_dmtag as u8; 339 326 }); 327 + 328 + impl NV_PFALCON_FALCON_DMATRFCMD { 329 + /// Programs the `imem` and `sec` fields for the given FalconMem 330 + pub(crate) fn with_falcon_mem(self, mem: FalconMem) -> Self { 331 + self.set_imem(mem != FalconMem::Dmem) 332 + .set_sec(if mem == FalconMem::ImemSecure { 1 } else { 0 }) 333 + } 334 + } 340 335 341 336 register!(NV_PFALCON_FALCON_DMATRFFBOFFS @ PFalconBase[0x0000011c] { 342 337 31:0 offs as u32; ··· 369 348 register!(NV_PFALCON_FALCON_ENGINE @ PFalconBase[0x000003c0] { 370 349 0:0 reset as bool; 371 350 }); 351 + 352 + impl NV_PFALCON_FALCON_ENGINE { 353 + /// Resets the falcon 354 + pub(crate) fn reset_engine<E: FalconEngine>(bar: &Bar0) { 355 + Self::read(bar, &E::ID).set_reset(true).write(bar, &E::ID); 356 + 357 + // TIMEOUT: falcon engine should not take more than 10us to reset. 358 + time::delay::fsleep(time::Delta::from_micros(10)); 359 + 360 + Self::read(bar, &E::ID).set_reset(false).write(bar, &E::ID); 361 + } 362 + } 372 363 373 364 register!(NV_PFALCON_FBIF_TRANSCFG @ PFalconBase[0x00000600[8]] { 374 365 1:0 target as u8 ?=> FalconFbifTarget; ··· 413 380 414 381 // PRISCV 415 382 383 + // RISC-V status register for debug (Turing and GA100 only). 384 + // Reflects current RISC-V core status. 385 + register!(NV_PRISCV_RISCV_CORE_SWITCH_RISCV_STATUS @ PFalcon2Base[0x00000240] { 386 + 0:0 active_stat as bool, "RISC-V core active/inactive status"; 387 + }); 388 + 389 + // GA102 and later 416 390 register!(NV_PRISCV_RISCV_CPUCTL @ PFalcon2Base[0x00000388] { 417 391 0:0 halted as bool; 418 392 7:7 active_stat as bool;

-16

drivers/gpu/nova-core/util.rs

··· 1 - // SPDX-License-Identifier: GPL-2.0 2 - 3 - /// Converts a null-terminated byte slice to a string, or `None` if the array does not 4 - /// contains any null byte or contains invalid characters. 5 - /// 6 - /// Contrary to [`kernel::str::CStr::from_bytes_with_nul`], the null byte can be anywhere in the 7 - /// slice, and not only in the last position. 8 - pub(crate) fn str_from_null_terminated(bytes: &[u8]) -> Option<&str> { 9 - use kernel::str::CStr; 10 - 11 - bytes 12 - .iter() 13 - .position(|&b| b == 0) 14 - .and_then(|null_pos| CStr::from_bytes_with_nul(&bytes[..=null_pos]).ok()) 15 - .and_then(|cstr| cstr.to_str().ok()) 16 - }

+33 -40

drivers/gpu/nova-core/vbios.rs

··· 11 11 Alignable, 12 12 Alignment, // 13 13 }, 14 + sync::aref::ARef, 14 15 transmute::FromBytes, 15 - types::ARef, 16 16 }; 17 17 18 18 use crate::{ 19 19 driver::Bar0, 20 20 firmware::{ 21 21 fwsec::Bcrt30Rsa3kSignature, 22 + FalconUCodeDesc, 23 + FalconUCodeDescV2, 22 24 FalconUCodeDescV3, // 23 25 }, 24 26 num::FromSafeCast, ··· 792 790 // read the 4 bytes at the offset specified in the token 793 791 let offset = usize::from(token.data_offset); 794 792 let bytes: [u8; 4] = self.base.data[offset..offset + 4].try_into().map_err(|_| { 795 - dev_err!(self.base.dev, "Failed to convert data slice to array"); 793 + dev_err!(self.base.dev, "Failed to convert data slice to array\n"); 796 794 EINVAL 797 795 })?; 798 796 ··· 888 886 ret.extend_from_slice(&data[header_len..required_bytes], GFP_KERNEL)?; 889 887 ret 890 888 }; 891 - 892 - // Debug logging of entries (dumps the table data to dmesg) 893 - for i in (header_len..required_bytes).step_by(entry_len) { 894 - dev_dbg!(dev, "PMU entry: {:02x?}\n", &data[i..][..entry_len]); 895 - } 896 889 897 890 Ok(PmuLookupTable { header, table_data }) 898 891 } ··· 1000 1003 } 1001 1004 1002 1005 impl FwSecBiosImage { 1003 - /// Get the FwSec header ([`FalconUCodeDescV3`]). 1004 - pub(crate) fn header(&self) -> Result<&FalconUCodeDescV3> { 1006 + /// Get the FwSec header ([`FalconUCodeDesc`]). 1007 + pub(crate) fn header(&self) -> Result<FalconUCodeDesc> { 1005 1008 // Get the falcon ucode offset that was found in setup_falcon_data. 1006 1009 let falcon_ucode_offset = self.falcon_ucode_offset; 1007 - 1008 - // Make sure the offset is within the data bounds. 1009 - if falcon_ucode_offset + core::mem::size_of::<FalconUCodeDescV3>() > self.base.data.len() { 1010 - dev_err!( 1011 - self.base.dev, 1012 - "fwsec-frts header not contained within BIOS bounds\n" 1013 - ); 1014 - return Err(ERANGE); 1015 - } 1016 1010 1017 1011 // Read the first 4 bytes to get the version. 1018 1012 let hdr_bytes: [u8; 4] = self.base.data[falcon_ucode_offset..falcon_ucode_offset + 4] ··· 1012 1024 let hdr = u32::from_le_bytes(hdr_bytes); 1013 1025 let ver = (hdr & 0xff00) >> 8; 1014 1026 1015 - if ver != 3 { 1016 - dev_err!(self.base.dev, "invalid fwsec firmware version: {:?}\n", ver); 1017 - return Err(EINVAL); 1027 + let data = self.base.data.get(falcon_ucode_offset..).ok_or(EINVAL)?; 1028 + match ver { 1029 + 2 => { 1030 + let v2 = FalconUCodeDescV2::from_bytes_copy_prefix(data) 1031 + .ok_or(EINVAL)? 1032 + .0; 1033 + Ok(FalconUCodeDesc::V2(v2)) 1034 + } 1035 + 3 => { 1036 + let v3 = FalconUCodeDescV3::from_bytes_copy_prefix(data) 1037 + .ok_or(EINVAL)? 1038 + .0; 1039 + Ok(FalconUCodeDesc::V3(v3)) 1040 + } 1041 + _ => { 1042 + dev_err!(self.base.dev, "invalid fwsec firmware version: {:?}\n", ver); 1043 + Err(EINVAL) 1044 + } 1018 1045 } 1019 - 1020 - // Return a reference to the FalconUCodeDescV3 structure. 1021 - // 1022 - // SAFETY: We have checked that `falcon_ucode_offset + size_of::<FalconUCodeDescV3>` is 1023 - // within the bounds of `data`. Also, this data vector is from ROM, and the `data` field 1024 - // in `BiosImageBase` is immutable after construction. 1025 - Ok(unsafe { 1026 - &*(self 1027 - .base 1028 - .data 1029 - .as_ptr() 1030 - .add(falcon_ucode_offset) 1031 - .cast::<FalconUCodeDescV3>()) 1032 - }) 1033 1046 } 1034 1047 1035 1048 /// Get the ucode data as a byte slice 1036 - pub(crate) fn ucode(&self, desc: &FalconUCodeDescV3) -> Result<&[u8]> { 1049 + pub(crate) fn ucode(&self, desc: &FalconUCodeDesc) -> Result<&[u8]> { 1037 1050 let falcon_ucode_offset = self.falcon_ucode_offset; 1038 1051 1039 1052 // The ucode data follows the descriptor. 1040 1053 let ucode_data_offset = falcon_ucode_offset + desc.size(); 1041 - let size = usize::from_safe_cast(desc.imem_load_size + desc.dmem_load_size); 1054 + let size = usize::from_safe_cast(desc.imem_load_size() + desc.dmem_load_size()); 1042 1055 1043 1056 // Get the data slice, checking bounds in a single operation. 1044 1057 self.base ··· 1055 1066 } 1056 1067 1057 1068 /// Get the signatures as a byte slice 1058 - pub(crate) fn sigs(&self, desc: &FalconUCodeDescV3) -> Result<&[Bcrt30Rsa3kSignature]> { 1069 + pub(crate) fn sigs(&self, desc: &FalconUCodeDesc) -> Result<&[Bcrt30Rsa3kSignature]> { 1070 + let hdr_size = match desc { 1071 + FalconUCodeDesc::V2(_v2) => core::mem::size_of::<FalconUCodeDescV2>(), 1072 + FalconUCodeDesc::V3(_v3) => core::mem::size_of::<FalconUCodeDescV3>(), 1073 + }; 1059 1074 // The signatures data follows the descriptor. 1060 - let sigs_data_offset = self.falcon_ucode_offset + core::mem::size_of::<FalconUCodeDescV3>(); 1061 - let sigs_count = usize::from(desc.signature_count); 1075 + let sigs_data_offset = self.falcon_ucode_offset + hdr_size; 1076 + let sigs_count = usize::from(desc.signature_count()); 1062 1077 let sigs_size = sigs_count * core::mem::size_of::<Bcrt30Rsa3kSignature>(); 1063 1078 1064 1079 // Make sure the data is within bounds.

+4 -3

rust/helpers/drm.c

··· 5 5 6 6 #ifdef CONFIG_DRM 7 7 8 - void rust_helper_drm_gem_object_get(struct drm_gem_object *obj) 8 + __rust_helper void rust_helper_drm_gem_object_get(struct drm_gem_object *obj) 9 9 { 10 10 drm_gem_object_get(obj); 11 11 } 12 12 13 - void rust_helper_drm_gem_object_put(struct drm_gem_object *obj) 13 + __rust_helper void rust_helper_drm_gem_object_put(struct drm_gem_object *obj) 14 14 { 15 15 drm_gem_object_put(obj); 16 16 } 17 17 18 - __u64 rust_helper_drm_vma_node_offset_addr(struct drm_vma_offset_node *node) 18 + __rust_helper __u64 19 + rust_helper_drm_vma_node_offset_addr(struct drm_vma_offset_node *node) 19 20 { 20 21 return drm_vma_node_offset_addr(node); 21 22 }

+3 -3

rust/kernel/drm/driver.rs

··· 121 121 pub struct Registration<T: Driver>(ARef<drm::Device<T>>); 122 122 123 123 impl<T: Driver> Registration<T> { 124 - /// Creates a new [`Registration`] and registers it. 125 124 fn new(drm: &drm::Device<T>, flags: usize) -> Result<Self> { 126 125 // SAFETY: `drm.as_raw()` is valid by the invariants of `drm::Device`. 127 126 to_result(unsafe { bindings::drm_dev_register(drm.as_raw(), flags) })?; ··· 128 129 Ok(Self(drm.into())) 129 130 } 130 131 131 - /// Same as [`Registration::new`}, but transfers ownership of the [`Registration`] to 132 - /// [`devres::register`]. 132 + /// Registers a new [`Device`](drm::Device) with userspace. 133 + /// 134 + /// Ownership of the [`Registration`] object is passed to [`devres::register`]. 133 135 pub fn new_foreign_owned( 134 136 drm: &drm::Device<T>, 135 137 dev: &device::Device<device::Bound>,

+3 -5

rust/kernel/drm/gem/mod.rs

··· 210 210 // SAFETY: The arguments are all valid per the type invariants. 211 211 to_result(unsafe { bindings::drm_gem_object_init(dev.as_raw(), obj.obj.get(), size) })?; 212 212 213 - // SAFETY: We never move out of `Self`. 213 + // SAFETY: We will never move out of `Self` as `ARef<Self>` is always treated as pinned. 214 214 let ptr = KBox::into_raw(unsafe { Pin::into_inner_unchecked(obj) }); 215 215 216 216 // SAFETY: `ptr` comes from `KBox::into_raw` and hence can't be NULL. ··· 253 253 } 254 254 255 255 // SAFETY: Instances of `Object<T>` are always reference-counted. 256 - unsafe impl<T: DriverObject> crate::types::AlwaysRefCounted for Object<T> { 256 + unsafe impl<T: DriverObject> crate::sync::aref::AlwaysRefCounted for Object<T> { 257 257 fn inc_ref(&self) { 258 258 // SAFETY: The existence of a shared reference guarantees that the refcount is non-zero. 259 259 unsafe { bindings::drm_gem_object_get(self.as_raw()) }; ··· 293 293 } 294 294 295 295 pub(super) const fn create_fops() -> bindings::file_operations { 296 - // SAFETY: As by the type invariant, it is safe to initialize `bindings::file_operations` 297 - // zeroed. 298 - let mut fops: bindings::file_operations = unsafe { core::mem::zeroed() }; 296 + let mut fops: bindings::file_operations = pin_init::zeroed(); 299 297 300 298 fops.owner = core::ptr::null_mut(); 301 299 fops.open = Some(bindings::drm_open);

+29 -7

rust/kernel/page.rs

··· 25 25 /// A bitmask that gives the page containing a given address. 26 26 pub const PAGE_MASK: usize = !(PAGE_SIZE - 1); 27 27 28 - /// Round up the given number to the next multiple of [`PAGE_SIZE`]. 28 + /// Rounds up to the next multiple of [`PAGE_SIZE`]. 29 29 /// 30 - /// It is incorrect to pass an address where the next multiple of [`PAGE_SIZE`] doesn't fit in a 31 - /// [`usize`]. 32 - pub const fn page_align(addr: usize) -> usize { 33 - // Parentheses around `PAGE_SIZE - 1` to avoid triggering overflow sanitizers in the wrong 34 - // cases. 35 - (addr + (PAGE_SIZE - 1)) & PAGE_MASK 30 + /// Returns [`None`] on integer overflow. 31 + /// 32 + /// # Examples 33 + /// 34 + /// ``` 35 + /// use kernel::page::{ 36 + /// page_align, 37 + /// PAGE_SIZE, 38 + /// }; 39 + /// 40 + /// // Requested address is already aligned. 41 + /// assert_eq!(page_align(0x0), Some(0x0)); 42 + /// assert_eq!(page_align(PAGE_SIZE), Some(PAGE_SIZE)); 43 + /// 44 + /// // Requested address needs alignment up. 45 + /// assert_eq!(page_align(0x1), Some(PAGE_SIZE)); 46 + /// assert_eq!(page_align(PAGE_SIZE + 1), Some(2 * PAGE_SIZE)); 47 + /// 48 + /// // Requested address causes overflow (returns `None`). 49 + /// let overflow_addr = usize::MAX - (PAGE_SIZE / 2); 50 + /// assert_eq!(page_align(overflow_addr), None); 51 + /// ``` 52 + #[inline(always)] 53 + pub const fn page_align(addr: usize) -> Option<usize> { 54 + let Some(sum) = addr.checked_add(PAGE_SIZE - 1) else { 55 + return None; 56 + }; 57 + Some(sum & PAGE_MASK) 36 58 } 37 59 38 60 /// Representation of a non-owning reference to a [`Page`].

Configure Feed

Configure Feed