John Hubbard
2025-Dec-03 05:59 UTC
[PATCH 23/31] gpu: nova-core: Hopper/Blackwell: add FSP Chain of Trust boot
Add the boot functions that construct FMC boot parameters and send the
Chain of Trust message to FSP. This completes the FSP communication
infrastructure needed to boot GSP firmware on Hopper/Blackwell GPUs.
Signed-off-by: John Hubbard <jhubbard at nvidia.com>
---
drivers/gpu/nova-core/fsp.rs | 156 +++++++++++++++++++++++++++++++++++
drivers/gpu/nova-core/gpu.rs | 1 -
2 files changed, 156 insertions(+), 1 deletion(-)
diff --git a/drivers/gpu/nova-core/fsp.rs b/drivers/gpu/nova-core/fsp.rs
index bb1e19c03c30..5840ab78e79f 100644
--- a/drivers/gpu/nova-core/fsp.rs
+++ b/drivers/gpu/nova-core/fsp.rs
@@ -13,6 +13,10 @@
device,
io::poll::read_poll_timeout,
prelude::*,
+ ptr::{
+ Alignable,
+ Alignment, //
+ },
time::Delta,
transmute::{
AsBytes,
@@ -22,6 +26,10 @@
use crate::regs::FSP_BOOT_COMPLETE_SUCCESS;
+/// FSP Chain of Trust (COT) version for Blackwell.
+/// GB202 uses version 2 (not 1 like GH100)
+const FSP_COT_VERSION: u16 = 2;
+
/// FSP message timeout in milliseconds.
const FSP_MSG_TIMEOUT_MS: i64 = 2000;
@@ -364,6 +372,154 @@ pub(crate) fn extract_fmc_signatures_static(
Ok(signatures)
}
+ /// Creates FMC boot parameters structure for FSP.
+ ///
+ /// This structure tells FSP how to boot GSP-RM with the correct memory
layout.
+ pub(crate) fn create_fmc_boot_params(
+ dev: &device::Device<device::Bound>,
+ wpr_meta_addr: u64,
+ wpr_meta_size: u32,
+ libos_addr: u64,
+ ) ->
Result<kernel::dma::CoherentAllocation<GspFmcBootParams>> {
+ use kernel::dma::CoherentAllocation;
+
+ const GSP_DMA_TARGET_COHERENT_SYSTEM: u32 = 1;
+ const GSP_DMA_TARGET_NONCOHERENT_SYSTEM: u32 = 2;
+
+ let fmc_boot_params =
CoherentAllocation::<GspFmcBootParams>::alloc_coherent(
+ dev,
+ 1,
+ GFP_KERNEL | __GFP_ZERO,
+ )?;
+
+ // Configure ACR boot parameters (WPR metadata location) using
dma_write! macro
+ kernel::dma_write!(
+ fmc_boot_params[0].boot_gsp_rm_params.target =
GSP_DMA_TARGET_COHERENT_SYSTEM
+ )?;
+ kernel::dma_write!(
+ fmc_boot_params[0].boot_gsp_rm_params.gsp_rm_desc_offset =
wpr_meta_addr
+ )?;
+
kernel::dma_write!(fmc_boot_params[0].boot_gsp_rm_params.gsp_rm_desc_size =
wpr_meta_size)?;
+
+ // Blackwell FSP expects wpr_carveout_offset and wpr_carveout_size to
be zero;
+ // it obtains WPR info from other sources.
+
+
kernel::dma_write!(fmc_boot_params[0].boot_gsp_rm_params.b_is_gsp_rm_boot = 1)?;
+
+ // Configure RM parameters (libos location) using dma_write! macro
+ kernel::dma_write!(
+ fmc_boot_params[0].gsp_rm_params.target =
GSP_DMA_TARGET_NONCOHERENT_SYSTEM
+ )?;
+ kernel::dma_write!(fmc_boot_params[0].gsp_rm_params.boot_args_offset =
libos_addr)?;
+
+ dev_dbg!(
+ dev,
+ "FMC Boot Params (addr={:#x}):\n target={}\n
desc_size={:#x}\n \
+ desc_offset={:#x}\n rm_target={}\n boot_args_offset={:#x} \
+ (libos_addr passed in: {:#x})\n",
+ fmc_boot_params.dma_handle(),
+ GSP_DMA_TARGET_COHERENT_SYSTEM,
+ wpr_meta_size,
+ wpr_meta_addr,
+ GSP_DMA_TARGET_NONCOHERENT_SYSTEM,
+ libos_addr,
+ libos_addr
+ );
+
+ Ok(fmc_boot_params)
+ }
+
+ /// Boot GSP FMC with pre-extracted signatures.
+ ///
+ /// This version takes pre-extracted signatures and FMC image data.
+ /// Used when signatures are extracted separately from the full ELF file.
+ #[allow(clippy::too_many_arguments)]
+ pub(crate) fn boot_gsp_fmc_with_signatures(
+ dev: &device::Device<device::Bound>,
+ bar: &crate::driver::Bar0,
+ chipset: crate::gpu::Chipset,
+ fmc_image_fw: &crate::dma::DmaObject, // Contains only the image
section
+ fmc_boot_params:
&kernel::dma::CoherentAllocation<GspFmcBootParams>,
+ total_reserved_size: u64,
+ resume: bool,
+ fsp_falcon: &crate::falcon::Falcon<crate::falcon::fsp::Fsp>,
+ signatures: &FmcSignatures,
+ ) -> Result<()> {
+ dev_dbg!(dev, "Starting FSP boot sequence for {}\n",
chipset);
+
+ // Build FSP Chain of Trust message
+ let fmc_addr = fmc_image_fw.dma_handle(); // Now points to image data
only
+ let fmc_boot_params_addr = fmc_boot_params.dma_handle();
+
+ // frts_offset is relative to FB end: FRTS_location = FB_END -
frts_offset
+ let frts_offset = if !resume {
+ let mut frts_reserved_size = if chipset.needs_large_reserved_mem()
{
+ 0x220000 // heap_size_non_wpr for Hopper/Blackwell+
+ } else {
+ total_reserved_size
+ };
+
+ // Add PMU reserved size
+ frts_reserved_size += u64::from(crate::fb::PMU_RESERVED_SIZE);
+
+ frts_reserved_size
+ .align_up(Alignment::new::<0x200000>())
+ .unwrap_or(frts_reserved_size)
+ } else {
+ 0
+ };
+ let frts_size = if !resume { 0x100000 } else { 0 }; // 1MB FRTS size
+
+ // Build the FSP message
+ let msg = KBox::new(
+ FspMessage {
+ mctp_header: (mctp::HEADER_SOM << 31)
+ | (mctp::HEADER_EOM << 30)
+ | (mctp::HEADER_SEID << 16)
+ | (mctp::HEADER_SEQ << 28),
+
+ nvdm_header: (mctp::MSG_TYPE_VENDOR_PCI)
+ | (mctp::VENDOR_ID_NV << 8)
+ | (mctp::NVDM_TYPE_COT << 24),
+
+ cot: NvdmPayloadCot {
+ version: FSP_COT_VERSION,
+ size: core::mem::size_of::<NvdmPayloadCot>() as u16,
+ gsp_fmc_sysmem_offset: fmc_addr,
+ frts_sysmem_offset: 0,
+ frts_sysmem_size: 0,
+ frts_vidmem_offset: frts_offset,
+ frts_vidmem_size: frts_size,
+ hash384: signatures.hash384,
+ public_key: signatures.public_key,
+ signature: signatures.signature,
+ gsp_boot_args_sysmem_offset: fmc_boot_params_addr,
+ },
+ },
+ GFP_KERNEL,
+ )?;
+
+ // Convert message to bytes for sending
+ let msg_bytes = msg.as_bytes();
+
+ dev_dbg!(
+ dev,
+ "FSP COT Message:\n size={} bytes\n fmc_addr={:#x}\n
boot_params={:#x}\n \
+ frts_offset={:#x}\n frts_size={:#x}\n",
+ msg_bytes.len(),
+ fmc_addr,
+ fmc_boot_params_addr,
+ frts_offset,
+ frts_size
+ );
+
+ // Send COT message to FSP and wait for response
+ Self::send_sync_fsp(dev, bar, fsp_falcon, mctp::NVDM_TYPE_COT,
msg_bytes)?;
+
+ dev_dbg!(dev, "FSP Chain of Trust completed successfully\n");
+ Ok(())
+ }
+
/// Send message to FSP and wait for response.
fn send_sync_fsp(
dev: &device::Device<device::Bound>,
diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs
index c0473ef8ac47..8fdce488612a 100644
--- a/drivers/gpu/nova-core/gpu.rs
+++ b/drivers/gpu/nova-core/gpu.rs
@@ -124,7 +124,6 @@ pub(crate) const fn arch(&self) -> Architecture {
}
}
- #[expect(dead_code)]
pub(crate) fn needs_large_reserved_mem(&self) -> bool {
matches!(self.arch(), Architecture::Hopper | Architecture::Blackwell)
}
--
2.52.0
Joel Fernandes
2025-Dec-05 17:15 UTC
[PATCH 23/31] gpu: nova-core: Hopper/Blackwell: add FSP Chain of Trust boot
Hi John, On 12/3/2025 12:59 AM, John Hubbard wrote:> + /// > + /// This version takes pre-extracted signatures and FMC image data. > + /// Used when signatures are extracted separately from the full ELF file. > + #[allow(clippy::too_many_arguments)] > + pub(crate) fn boot_gsp_fmc_with_signatures( > + dev: &device::Device<device::Bound>, > + bar: &crate::driver::Bar0, > + chipset: crate::gpu::Chipset, > + fmc_image_fw: &crate::dma::DmaObject, // Contains only the image section > + fmc_boot_params: &kernel::dma::CoherentAllocation<GspFmcBootParams>, > + total_reserved_size: u64, > + resume: bool, > + fsp_falcon: &crate::falcon::Falcon<crate::falcon::fsp::Fsp>, > + signatures: &FmcSignatures, > + ) -> Result<()> { > + dev_dbg!(dev, "Starting FSP boot sequence for {}\n", chipset);I see lots of dev_dbg left as you also pointed. I guess you will remove these as we discussed in the other thread.> + > + // Build FSP Chain of Trust message > + let fmc_addr = fmc_image_fw.dma_handle(); // Now points to image data only > + let fmc_boot_params_addr = fmc_boot_params.dma_handle(); > + > + // frts_offset is relative to FB end: FRTS_location = FB_END - frts_offset > + let frts_offset = if !resume { > + let mut frts_reserved_size = if chipset.needs_large_reserved_mem() { > + 0x220000 // heap_size_non_wpr for Hopper/Blackwell+Can we please use constants for the magic numbers? Also with comment headers clearly documenting the constants where they are defined.> + } else { > + total_reserved_size > + }; > + > + // Add PMU reserved size > + frts_reserved_size += u64::from(crate::fb::PMU_RESERVED_SIZE); > + > + frts_reserved_size > + .align_up(Alignment::new::<0x200000>()) > + .unwrap_or(frts_reserved_size) > + } else { > + 0 > + }; > + let frts_size = if !resume { 0x100000 } else { 0 }; // 1MB FRTS sizeShould use SZ_ constants, here and everywhere. Like SZ_1M etc [1]. [1] https://rust.docs.kernel.org/src/kernel/sizes.rs.html> + > + // Build the FSP message > + let msg = KBox::new( > + FspMessage { > + mctp_header: (mctp::HEADER_SOM << 31) > + | (mctp::HEADER_EOM << 30) > + | (mctp::HEADER_SEID << 16) > + | (mctp::HEADER_SEQ << 28), > + > + nvdm_header: (mctp::MSG_TYPE_VENDOR_PCI) > + | (mctp::VENDOR_ID_NV << 8) > + | (mctp::NVDM_TYPE_COT << 24), > + > + cot: NvdmPayloadCot { > + version: FSP_COT_VERSION, > + size: core::mem::size_of::<NvdmPayloadCot>() as u16, > + gsp_fmc_sysmem_offset: fmc_addr, > + frts_sysmem_offset: 0, > + frts_sysmem_size: 0, > + frts_vidmem_offset: frts_offset, > + frts_vidmem_size: frts_size, > + hash384: signatures.hash384, > + public_key: signatures.public_key, > + signature: signatures.signature, > + gsp_boot_args_sysmem_offset: fmc_boot_params_addr, > + }, > + }, > + GFP_KERNEL, > + )?; > + > + // Convert message to bytes for sending > + let msg_bytes = msg.as_bytes(); > + > + dev_dbg!( > + dev, > + "FSP COT Message:\n size={} bytes\n fmc_addr={:#x}\n boot_params={:#x}\n \ > + frts_offset={:#x}\n frts_size={:#x}\n", > + msg_bytes.len(), > + fmc_addr, > + fmc_boot_params_addr, > + frts_offset, > + frts_size > + ); > + > + // Send COT message to FSP and wait for response > + Self::send_sync_fsp(dev, bar, fsp_falcon, mctp::NVDM_TYPE_COT, msg_bytes)?;This seems to diverge from the pattern we use in GSP `cmdq`. Can we keep it consistent, like `send_sync_fsp<M>` ? pub(crate) fn send_command<M>(&mut self, bar: &Bar0, command: M) -> Result where M: CommandToGsp, Alistair/Alex may have more design suggestions since they came up with this pattern. Especially around safety (guarantees that the part of buffer being read will not be modified by HW etc). Over all, I feel we have 3 different mechanisms now (1 upstream, 2 in the works). The GSP cmdq, RM control and then this one. It would be good to get some consistency in the API designs for all these different mechanisms (and possibly share any common code). Thanks! - Joel
Joel Fernandes
2025-Dec-06 21:36 UTC
[PATCH 23/31] gpu: nova-core: Hopper/Blackwell: add FSP Chain of Trust boot
On Tue, Dec 02, 2025 at 09:59:15PM -0800, John Hubbard wrote:> Add the boot functions that construct FMC boot parameters and send the > Chain of Trust message to FSP. This completes the FSP communication > infrastructure needed to boot GSP firmware on Hopper/Blackwell GPUs. > > Signed-off-by: John Hubbard <jhubbard at nvidia.com> > --- > drivers/gpu/nova-core/fsp.rs | 156 +++++++++++++++++++++++++++++++++++ > drivers/gpu/nova-core/gpu.rs | 1 - > 2 files changed, 156 insertions(+), 1 deletion(-) > > diff --git a/drivers/gpu/nova-core/fsp.rs b/drivers/gpu/nova-core/fsp.rs > index bb1e19c03c30..5840ab78e79f 100644 > --- a/drivers/gpu/nova-core/fsp.rs > +++ b/drivers/gpu/nova-core/fsp.rs > @@ -13,6 +13,10 @@ > device, > io::poll::read_poll_timeout, > prelude::*, > + ptr::{ > + Alignable, > + Alignment, // > + }, > time::Delta, > transmute::{ > AsBytes, > @@ -22,6 +26,10 @@ > > use crate::regs::FSP_BOOT_COMPLETE_SUCCESS; > > +/// FSP Chain of Trust (COT) version for Blackwell. > +/// GB202 uses version 2 (not 1 like GH100) > +const FSP_COT_VERSION: u16 = 2;the comment says that Hopper needs this to be version 1 so how does this series boot on Hopper? maybe we need to add a new function to return a different version number for Hopper versus Blackwell? edit: I just noticed in the cover letter that you mentioned this is tested only on Blackwell. I think perhaps we should drop the claim that the series works on hopper in the patches / cover letter. thanks, - Joel> + > /// FSP message timeout in milliseconds. > const FSP_MSG_TIMEOUT_MS: i64 = 2000; > > @@ -364,6 +372,154 @@ pub(crate) fn extract_fmc_signatures_static( > Ok(signatures) > } > > + /// Creates FMC boot parameters structure for FSP. > + /// > + /// This structure tells FSP how to boot GSP-RM with the correct memory layout. > + pub(crate) fn create_fmc_boot_params( > + dev: &device::Device<device::Bound>, > + wpr_meta_addr: u64, > + wpr_meta_size: u32, > + libos_addr: u64, > + ) -> Result<kernel::dma::CoherentAllocation<GspFmcBootParams>> { > + use kernel::dma::CoherentAllocation; > + > + const GSP_DMA_TARGET_COHERENT_SYSTEM: u32 = 1; > + const GSP_DMA_TARGET_NONCOHERENT_SYSTEM: u32 = 2; > + > + let fmc_boot_params = CoherentAllocation::<GspFmcBootParams>::alloc_coherent( > + dev, > + 1, > + GFP_KERNEL | __GFP_ZERO, > + )?; > + > + // Configure ACR boot parameters (WPR metadata location) using dma_write! macro > + kernel::dma_write!( > + fmc_boot_params[0].boot_gsp_rm_params.target = GSP_DMA_TARGET_COHERENT_SYSTEM > + )?; > + kernel::dma_write!( > + fmc_boot_params[0].boot_gsp_rm_params.gsp_rm_desc_offset = wpr_meta_addr > + )?; > + kernel::dma_write!(fmc_boot_params[0].boot_gsp_rm_params.gsp_rm_desc_size = wpr_meta_size)?; > + > + // Blackwell FSP expects wpr_carveout_offset and wpr_carveout_size to be zero; > + // it obtains WPR info from other sources. > + > + kernel::dma_write!(fmc_boot_params[0].boot_gsp_rm_params.b_is_gsp_rm_boot = 1)?; > + > + // Configure RM parameters (libos location) using dma_write! macro > + kernel::dma_write!( > + fmc_boot_params[0].gsp_rm_params.target = GSP_DMA_TARGET_NONCOHERENT_SYSTEM > + )?; > + kernel::dma_write!(fmc_boot_params[0].gsp_rm_params.boot_args_offset = libos_addr)?; > + > + dev_dbg!( > + dev, > + "FMC Boot Params (addr={:#x}):\n target={}\n desc_size={:#x}\n \ > + desc_offset={:#x}\n rm_target={}\n boot_args_offset={:#x} \ > + (libos_addr passed in: {:#x})\n", > + fmc_boot_params.dma_handle(), > + GSP_DMA_TARGET_COHERENT_SYSTEM, > + wpr_meta_size, > + wpr_meta_addr, > + GSP_DMA_TARGET_NONCOHERENT_SYSTEM, > + libos_addr, > + libos_addr > + ); > + > + Ok(fmc_boot_params) > + } > + > + /// Boot GSP FMC with pre-extracted signatures. > + /// > + /// This version takes pre-extracted signatures and FMC image data. > + /// Used when signatures are extracted separately from the full ELF file. > + #[allow(clippy::too_many_arguments)] > + pub(crate) fn boot_gsp_fmc_with_signatures( > + dev: &device::Device<device::Bound>, > + bar: &crate::driver::Bar0, > + chipset: crate::gpu::Chipset, > + fmc_image_fw: &crate::dma::DmaObject, // Contains only the image section > + fmc_boot_params: &kernel::dma::CoherentAllocation<GspFmcBootParams>, > + total_reserved_size: u64, > + resume: bool, > + fsp_falcon: &crate::falcon::Falcon<crate::falcon::fsp::Fsp>, > + signatures: &FmcSignatures, > + ) -> Result<()> { > + dev_dbg!(dev, "Starting FSP boot sequence for {}\n", chipset); > + > + // Build FSP Chain of Trust message > + let fmc_addr = fmc_image_fw.dma_handle(); // Now points to image data only > + let fmc_boot_params_addr = fmc_boot_params.dma_handle(); > + > + // frts_offset is relative to FB end: FRTS_location = FB_END - frts_offset > + let frts_offset = if !resume { > + let mut frts_reserved_size = if chipset.needs_large_reserved_mem() { > + 0x220000 // heap_size_non_wpr for Hopper/Blackwell+ > + } else { > + total_reserved_size > + }; > + > + // Add PMU reserved size > + frts_reserved_size += u64::from(crate::fb::PMU_RESERVED_SIZE); > + > + frts_reserved_size > + .align_up(Alignment::new::<0x200000>()) > + .unwrap_or(frts_reserved_size) > + } else { > + 0 > + }; > + let frts_size = if !resume { 0x100000 } else { 0 }; // 1MB FRTS size > + > + // Build the FSP message > + let msg = KBox::new( > + FspMessage { > + mctp_header: (mctp::HEADER_SOM << 31) > + | (mctp::HEADER_EOM << 30) > + | (mctp::HEADER_SEID << 16) > + | (mctp::HEADER_SEQ << 28), > + > + nvdm_header: (mctp::MSG_TYPE_VENDOR_PCI) > + | (mctp::VENDOR_ID_NV << 8) > + | (mctp::NVDM_TYPE_COT << 24), > + > + cot: NvdmPayloadCot { > + version: FSP_COT_VERSION, > + size: core::mem::size_of::<NvdmPayloadCot>() as u16, > + gsp_fmc_sysmem_offset: fmc_addr, > + frts_sysmem_offset: 0, > + frts_sysmem_size: 0, > + frts_vidmem_offset: frts_offset, > + frts_vidmem_size: frts_size, > + hash384: signatures.hash384, > + public_key: signatures.public_key, > + signature: signatures.signature, > + gsp_boot_args_sysmem_offset: fmc_boot_params_addr, > + }, > + }, > + GFP_KERNEL, > + )?; > + > + // Convert message to bytes for sending > + let msg_bytes = msg.as_bytes(); > + > + dev_dbg!( > + dev, > + "FSP COT Message:\n size={} bytes\n fmc_addr={:#x}\n boot_params={:#x}\n \ > + frts_offset={:#x}\n frts_size={:#x}\n", > + msg_bytes.len(), > + fmc_addr, > + fmc_boot_params_addr, > + frts_offset, > + frts_size > + ); > + > + // Send COT message to FSP and wait for response > + Self::send_sync_fsp(dev, bar, fsp_falcon, mctp::NVDM_TYPE_COT, msg_bytes)?; > + > + dev_dbg!(dev, "FSP Chain of Trust completed successfully\n"); > + Ok(()) > + } > + > /// Send message to FSP and wait for response. > fn send_sync_fsp( > dev: &device::Device<device::Bound>, > diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs > index c0473ef8ac47..8fdce488612a 100644 > --- a/drivers/gpu/nova-core/gpu.rs > +++ b/drivers/gpu/nova-core/gpu.rs > @@ -124,7 +124,6 @@ pub(crate) const fn arch(&self) -> Architecture { > } > } > > - #[expect(dead_code)] > pub(crate) fn needs_large_reserved_mem(&self) -> bool { > matches!(self.arch(), Architecture::Hopper | Architecture::Blackwell) > } > -- > 2.52.0 >