Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1// SPDX-License-Identifier: GPL-2.0
2
3use kernel::{
4 device,
5 dma::{
6 Coherent,
7 CoherentBox,
8 DataDirection,
9 DmaAddress, //
10 },
11 prelude::*,
12 scatterlist::{
13 Owned,
14 SGTable, //
15 },
16};
17
18use crate::{
19 firmware::{
20 elf,
21 riscv::RiscvFirmware, //
22 },
23 gpu::{
24 Architecture,
25 Chipset, //
26 },
27 gsp::GSP_PAGE_SIZE,
28 num::FromSafeCast,
29};
30
31/// GSP firmware with 3-level radix page tables for the GSP bootloader.
32///
33/// The bootloader expects firmware to be mapped starting at address 0 in GSP's virtual address
34/// space:
35///
36/// ```text
37/// Level 0: 1 page, 1 entry -> points to first level 1 page
38/// Level 1: Multiple pages/entries -> each entry points to a level 2 page
39/// Level 2: Multiple pages/entries -> each entry points to a firmware page
40/// ```
41///
42/// Each page is 4KB, each entry is 8 bytes (64-bit DMA address).
43/// Also known as "Radix3" firmware.
44#[pin_data]
45pub(crate) struct GspFirmware {
46 /// The GSP firmware inside a [`VVec`], device-mapped via a SG table.
47 #[pin]
48 fw: SGTable<Owned<VVec<u8>>>,
49 /// Level 2 page table whose entries contain DMA addresses of firmware pages.
50 #[pin]
51 level2: SGTable<Owned<VVec<u8>>>,
52 /// Level 1 page table whose entries contain DMA addresses of level 2 pages.
53 #[pin]
54 level1: SGTable<Owned<VVec<u8>>>,
55 /// Level 0 page table (single 4KB page) with one entry: DMA address of first level 1 page.
56 level0: Coherent<[u64]>,
57 /// Size in bytes of the firmware contained in [`Self::fw`].
58 pub(crate) size: usize,
59 /// Device-mapped GSP signatures matching the GPU's [`Chipset`].
60 pub(crate) signatures: Coherent<[u8]>,
61 /// GSP bootloader, verifies the GSP firmware before loading and running it.
62 pub(crate) bootloader: RiscvFirmware,
63}
64
65impl GspFirmware {
66 /// Loads the GSP firmware binaries, map them into `dev`'s address-space, and creates the page
67 /// tables expected by the GSP bootloader to load it.
68 pub(crate) fn new<'a>(
69 dev: &'a device::Device<device::Bound>,
70 chipset: Chipset,
71 ver: &'a str,
72 ) -> impl PinInit<Self, Error> + 'a {
73 pin_init::pin_init_scope(move || {
74 let firmware = super::request_firmware(dev, chipset, "gsp", ver)?;
75
76 let fw_section = elf::elf64_section(firmware.data(), ".fwimage").ok_or(EINVAL)?;
77
78 let size = fw_section.len();
79
80 // Move the firmware into a vmalloc'd vector and map it into the device address
81 // space.
82 let fw_vvec = VVec::with_capacity(fw_section.len(), GFP_KERNEL)
83 .and_then(|mut v| {
84 v.extend_from_slice(fw_section, GFP_KERNEL)?;
85 Ok(v)
86 })
87 .map_err(|_| ENOMEM)?;
88
89 Ok(try_pin_init!(Self {
90 fw <- SGTable::new(dev, fw_vvec, DataDirection::ToDevice, GFP_KERNEL),
91 level2 <- {
92 // Allocate the level 2 page table, map the firmware onto it, and map it into
93 // the device address space.
94 VVec::<u8>::with_capacity(
95 fw.iter().count() * core::mem::size_of::<u64>(),
96 GFP_KERNEL,
97 )
98 .map_err(|_| ENOMEM)
99 .and_then(|level2| map_into_lvl(&fw, level2))
100 .map(|level2| SGTable::new(dev, level2, DataDirection::ToDevice, GFP_KERNEL))?
101 },
102 level1 <- {
103 // Allocate the level 1 page table, map the level 2 page table onto it, and map
104 // it into the device address space.
105 VVec::<u8>::with_capacity(
106 level2.iter().count() * core::mem::size_of::<u64>(),
107 GFP_KERNEL,
108 )
109 .map_err(|_| ENOMEM)
110 .and_then(|level1| map_into_lvl(&level2, level1))
111 .map(|level1| SGTable::new(dev, level1, DataDirection::ToDevice, GFP_KERNEL))?
112 },
113 level0: {
114 // Allocate the level 0 page table as a device-visible DMA object, and map the
115 // level 1 page table onto it.
116
117 // Fill level 1 page entry.
118 let level1_entry = level1.iter().next().ok_or(EINVAL)?;
119 let level1_entry_addr = level1_entry.dma_address();
120
121 // Create level 0 page table data and fill its first entry with the level 1
122 // table.
123 let mut level0 = CoherentBox::<[u64]>::zeroed_slice(
124 dev,
125 GSP_PAGE_SIZE / size_of::<u64>(),
126 GFP_KERNEL
127 )?;
128 level0[0] = level1_entry_addr.to_le();
129
130 level0.into()
131 },
132 size,
133 signatures: {
134 let sigs_section = match chipset.arch() {
135 Architecture::Turing
136 if matches!(chipset, Chipset::TU116 | Chipset::TU117) =>
137 {
138 ".fwsignature_tu11x"
139 }
140 Architecture::Turing => ".fwsignature_tu10x",
141 // GA100 uses the same firmware as Turing
142 Architecture::Ampere if chipset == Chipset::GA100 => ".fwsignature_tu10x",
143 Architecture::Ampere => ".fwsignature_ga10x",
144 Architecture::Ada => ".fwsignature_ad10x",
145 };
146
147 elf::elf64_section(firmware.data(), sigs_section)
148 .ok_or(EINVAL)
149 .and_then(|data| Coherent::from_slice(dev, data, GFP_KERNEL))?
150 },
151 bootloader: {
152 let bl = super::request_firmware(dev, chipset, "bootloader", ver)?;
153
154 RiscvFirmware::new(dev, &bl)?
155 },
156 }))
157 })
158 }
159
160 /// Returns the DMA handle of the radix3 level 0 page table.
161 pub(crate) fn radix3_dma_handle(&self) -> DmaAddress {
162 self.level0.dma_handle()
163 }
164}
165
166/// Build a page table from a scatter-gather list.
167///
168/// Takes each DMA-mapped region from `sg_table` and writes page table entries
169/// for all 4KB pages within that region. For example, a 16KB SG entry becomes
170/// 4 consecutive page table entries.
171fn map_into_lvl(sg_table: &SGTable<Owned<VVec<u8>>>, mut dst: VVec<u8>) -> Result<VVec<u8>> {
172 for sg_entry in sg_table.iter() {
173 // Number of pages we need to map.
174 let num_pages = usize::from_safe_cast(sg_entry.dma_len()).div_ceil(GSP_PAGE_SIZE);
175
176 for i in 0..num_pages {
177 let entry = sg_entry.dma_address()
178 + (u64::from_safe_cast(i) * u64::from_safe_cast(GSP_PAGE_SIZE));
179 dst.extend_from_slice(&entry.to_le_bytes(), GFP_KERNEL)?;
180 }
181 }
182
183 Ok(dst)
184}