Alexandre Courbot
2025-Apr-20  12:19 UTC
[PATCH 09/16] gpu: nova-core: register sysmem flush page
A page of system memory is reserved so sysmembar can perform a read on
it if a system write occurred since the last flush. Do this early as it
can be required to e.g. reset the GPU falcons.
Signed-off-by: Alexandre Courbot <acourbot at nvidia.com>
---
 drivers/gpu/nova-core/dma.rs       | 54 ++++++++++++++++++++++++++++++++++++++
 drivers/gpu/nova-core/gpu.rs       | 53 +++++++++++++++++++++++++++++++++++--
 drivers/gpu/nova-core/nova_core.rs |  1 +
 drivers/gpu/nova-core/regs.rs      | 10 +++++++
 4 files changed, 116 insertions(+), 2 deletions(-)
diff --git a/drivers/gpu/nova-core/dma.rs b/drivers/gpu/nova-core/dma.rs
new file mode 100644
index
0000000000000000000000000000000000000000..a4162bff597132a04e002b2b910a4537bbabc287
--- /dev/null
+++ b/drivers/gpu/nova-core/dma.rs
@@ -0,0 +1,54 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Simple DMA object wrapper.
+
+// To be removed when all code is used.
+#![allow(dead_code)]
+
+use kernel::device;
+use kernel::dma::CoherentAllocation;
+use kernel::page::PAGE_SIZE;
+use kernel::prelude::*;
+
+pub(crate) struct DmaObject {
+    pub dma: CoherentAllocation<u8>,
+    pub len: usize,
+    #[allow(dead_code)]
+    pub name: &'static str,
+}
+
+impl DmaObject {
+    pub(crate) fn new(
+        dev: &device::Device<device::Bound>,
+        len: usize,
+        name: &'static str,
+    ) -> Result<Self> {
+        let len = core::alloc::Layout::from_size_align(len, PAGE_SIZE)
+            .map_err(|_| EINVAL)?
+            .pad_to_align()
+            .size();
+        let dma = CoherentAllocation::alloc_coherent(dev, len, GFP_KERNEL |
__GFP_ZERO)?;
+
+        Ok(Self { dma, len, name })
+    }
+
+    pub(crate) fn from_data(
+        dev: &device::Device<device::Bound>,
+        data: &[u8],
+        name: &'static str,
+    ) -> Result<Self> {
+        Self::new(dev, data.len(), name).and_then(|mut dma_obj| {
+            // SAFETY:
+            // - The copied data fits within the size of the allocated object.
+            // - We have just created this object and there is no other user at
this stage.
+            unsafe {
+                core::ptr::copy_nonoverlapping(
+                    data.as_ptr(),
+                    dma_obj.dma.start_ptr_mut(),
+                    data.len(),
+                );
+            }
+            Ok(dma_obj)
+        })
+    }
+}
diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs
index
1f7799692a0ab042f2540e01414f5ca347ae9ecc..d43e710cc983d51f053dacbd77cbbfb79fa882c3
100644
--- a/drivers/gpu/nova-core/gpu.rs
+++ b/drivers/gpu/nova-core/gpu.rs
@@ -3,6 +3,7 @@
 use kernel::{device, devres::Devres, error::code::*, pci, prelude::*};
 
 use crate::devinit;
+use crate::dma::DmaObject;
 use crate::driver::Bar0;
 use crate::firmware::Firmware;
 use crate::regs;
@@ -145,12 +146,30 @@ fn new(bar: &Devres<Bar0>) ->
Result<Spec> {
 }
 
 /// Structure holding the resources required to operate the GPU.
-#[pin_data]
+#[pin_data(PinnedDrop)]
 pub(crate) struct Gpu {
     spec: Spec,
     /// MMIO mapping of PCI BAR 0
     bar: Devres<Bar0>,
     fw: Firmware,
+    sysmem_flush: DmaObject,
+}
+
+#[pinned_drop]
+impl PinnedDrop for Gpu {
+    fn drop(self: Pin<&mut Self>) {
+        // Unregister the sysmem flush page before we release it.
+        let _ = with_bar!(&self.bar, |b| {
+            regs::PfbNisoFlushSysmemAddr::default()
+                .set_adr_39_08(0)
+                .write(b);
+            if self.spec.chipset >= Chipset::GA102 {
+                regs::PfbNisoFlushSysmemAddrHi::default()
+                    .set_adr_63_40(0)
+                    .write(b);
+            }
+        });
+    }
 }
 
 impl Gpu {
@@ -173,6 +192,36 @@ pub(crate) fn new(
         devinit::wait_gfw_boot_completion(&bar)
             .inspect_err(|_| pr_err!("GFW boot did not complete"))?;
 
-        Ok(pin_init!(Self { spec, bar, fw }))
+        // System memory page required for sysmembar to properly flush into
system memory.
+        let sysmem_flush = {
+            let page = DmaObject::new(
+                pdev.as_ref(),
+                kernel::bindings::PAGE_SIZE,
+                "sysmem flush page",
+            )?;
+
+            // Register the sysmem flush page.
+            with_bar!(bar, |b| {
+                let handle = page.dma.dma_handle();
+
+                regs::PfbNisoFlushSysmemAddr::default()
+                    .set_adr_39_08((handle >> 8) as u32)
+                    .write(b);
+                if spec.chipset >= Chipset::GA102 {
+                    regs::PfbNisoFlushSysmemAddrHi::default()
+                        .set_adr_63_40((handle >> 40) as u32)
+                        .write(b);
+                }
+            })?;
+
+            page
+        };
+
+        Ok(pin_init!(Self {
+            spec,
+            bar,
+            fw,
+            sysmem_flush,
+        }))
     }
 }
diff --git a/drivers/gpu/nova-core/nova_core.rs
b/drivers/gpu/nova-core/nova_core.rs
index
878161e060f54da7738c656f6098936a62dcaa93..37c7eb0ea7a926bee4e3c661028847291bf07fa2
100644
--- a/drivers/gpu/nova-core/nova_core.rs
+++ b/drivers/gpu/nova-core/nova_core.rs
@@ -21,6 +21,7 @@ macro_rules! with_bar {
 }
 
 mod devinit;
+mod dma;
 mod driver;
 mod firmware;
 mod gpu;
diff --git a/drivers/gpu/nova-core/regs.rs b/drivers/gpu/nova-core/regs.rs
index
fd7096f0ddd4af90114dd1119d9715d2cd3aa2ac..1e24787c4b5f432ac25fe399c8cb38b7350e44ae
100644
--- a/drivers/gpu/nova-core/regs.rs
+++ b/drivers/gpu/nova-core/regs.rs
@@ -14,6 +14,16 @@
     28:20   chipset => try_into Chipset, "chipset model"
 );
 
+/* PFB */
+
+register!(PfbNisoFlushSysmemAddr at 0x00100c10;
+    31:0    adr_39_08 => as u32
+);
+
+register!(PfbNisoFlushSysmemAddrHi at 0x00100c40;
+    23:0    adr_63_40 => as u32
+);
+
 /* GC6 */
 
 register!(Pgc6AonSecureScratchGroup05PrivLevelMask at 0x00118128;
-- 
2.49.0
Danilo Krummrich
2025-Apr-22  11:45 UTC
[PATCH 09/16] gpu: nova-core: register sysmem flush page
On Sun, Apr 20, 2025 at 09:19:41PM +0900, Alexandre Courbot wrote:> A page of system memory is reserved so sysmembar can perform a read on > it if a system write occurred since the last flush. Do this early as it > can be required to e.g. reset the GPU falcons. > > Signed-off-by: Alexandre Courbot <acourbot at nvidia.com> > --- > drivers/gpu/nova-core/dma.rs | 54 ++++++++++++++++++++++++++++++++++++++ > drivers/gpu/nova-core/gpu.rs | 53 +++++++++++++++++++++++++++++++++++-- > drivers/gpu/nova-core/nova_core.rs | 1 + > drivers/gpu/nova-core/regs.rs | 10 +++++++ > 4 files changed, 116 insertions(+), 2 deletions(-) > > diff --git a/drivers/gpu/nova-core/dma.rs b/drivers/gpu/nova-core/dma.rs > new file mode 100644 > index 0000000000000000000000000000000000000000..a4162bff597132a04e002b2b910a4537bbabc287 > --- /dev/null > +++ b/drivers/gpu/nova-core/dma.rs > @@ -0,0 +1,54 @@ > +// SPDX-License-Identifier: GPL-2.0 > + > +//! Simple DMA object wrapper. > + > +// To be removed when all code is used. > +#![allow(dead_code)] > + > +use kernel::device; > +use kernel::dma::CoherentAllocation; > +use kernel::page::PAGE_SIZE; > +use kernel::prelude::*; > + > +pub(crate) struct DmaObject { > + pub dma: CoherentAllocation<u8>, > + pub len: usize,This should be covered by CoherentAllocation already, no? If it does not have a public accessor for its size, please add it for CoherentAllocation instead. I can take the corresponding patch through the nova tree.> + #[allow(dead_code)]Please prefer #[expect(dead_code)], such that we are forced to remove it once it's subsequently used.> + pub name: &'static str, > +} > + > +impl DmaObject { > + pub(crate) fn new( > + dev: &device::Device<device::Bound>, > + len: usize, > + name: &'static str, > + ) -> Result<Self> { > + let len = core::alloc::Layout::from_size_align(len, PAGE_SIZE) > + .map_err(|_| EINVAL)? > + .pad_to_align() > + .size(); > + let dma = CoherentAllocation::alloc_coherent(dev, len, GFP_KERNEL | __GFP_ZERO)?; > + > + Ok(Self { dma, len, name }) > + } > + > + pub(crate) fn from_data( > + dev: &device::Device<device::Bound>, > + data: &[u8], > + name: &'static str, > + ) -> Result<Self> { > + Self::new(dev, data.len(), name).and_then(|mut dma_obj| { > + // SAFETY: > + // - The copied data fits within the size of the allocated object. > + // - We have just created this object and there is no other user at this stage. > + unsafe { > + core::ptr::copy_nonoverlapping( > + data.as_ptr(), > + dma_obj.dma.start_ptr_mut(), > + data.len(), > + ); > + } > + Ok(dma_obj) > + }) > + } > +}The DMA wrapper should probably be added in a separate patch.> diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs > index 1f7799692a0ab042f2540e01414f5ca347ae9ecc..d43e710cc983d51f053dacbd77cbbfb79fa882c3 100644 > --- a/drivers/gpu/nova-core/gpu.rs > +++ b/drivers/gpu/nova-core/gpu.rs > @@ -3,6 +3,7 @@ > use kernel::{device, devres::Devres, error::code::*, pci, prelude::*}; > > use crate::devinit; > +use crate::dma::DmaObject; > use crate::driver::Bar0; > use crate::firmware::Firmware; > use crate::regs; > @@ -145,12 +146,30 @@ fn new(bar: &Devres<Bar0>) -> Result<Spec> { > } > > /// Structure holding the resources required to operate the GPU. > -#[pin_data] > +#[pin_data(PinnedDrop)] > pub(crate) struct Gpu { > spec: Spec, > /// MMIO mapping of PCI BAR 0 > bar: Devres<Bar0>, > fw: Firmware, > + sysmem_flush: DmaObject,Please add a doc-comment for this DmaObject explaining what it is used for by the driver and why it is needed.> +} > + > +#[pinned_drop] > +impl PinnedDrop for Gpu { > + fn drop(self: Pin<&mut Self>) { > + // Unregister the sysmem flush page before we release it. > + let _ = with_bar!(&self.bar, |b| { > + regs::PfbNisoFlushSysmemAddr::default() > + .set_adr_39_08(0) > + .write(b); > + if self.spec.chipset >= Chipset::GA102 { > + regs::PfbNisoFlushSysmemAddrHi::default() > + .set_adr_63_40(0) > + .write(b); > + } > + }); > + } > } > > impl Gpu { > @@ -173,6 +192,36 @@ pub(crate) fn new( > devinit::wait_gfw_boot_completion(&bar) > .inspect_err(|_| pr_err!("GFW boot did not complete"))?; > > - Ok(pin_init!(Self { spec, bar, fw })) > + // System memory page required for sysmembar to properly flush into system memory. > + let sysmem_flush = { > + let page = DmaObject::new( > + pdev.as_ref(), > + kernel::bindings::PAGE_SIZE, > + "sysmem flush page", > + )?; > + > + // Register the sysmem flush page. > + with_bar!(bar, |b| { > + let handle = page.dma.dma_handle(); > + > + regs::PfbNisoFlushSysmemAddr::default() > + .set_adr_39_08((handle >> 8) as u32) > + .write(b); > + if spec.chipset >= Chipset::GA102 { > + regs::PfbNisoFlushSysmemAddrHi::default() > + .set_adr_63_40((handle >> 40) as u32) > + .write(b); > + } > + })?; > + > + page > + }; > + > + Ok(pin_init!(Self { > + spec, > + bar, > + fw, > + sysmem_flush, > + })) > } > } > diff --git a/drivers/gpu/nova-core/nova_core.rs b/drivers/gpu/nova-core/nova_core.rs > index 878161e060f54da7738c656f6098936a62dcaa93..37c7eb0ea7a926bee4e3c661028847291bf07fa2 100644 > --- a/drivers/gpu/nova-core/nova_core.rs > +++ b/drivers/gpu/nova-core/nova_core.rs > @@ -21,6 +21,7 @@ macro_rules! with_bar { > } > > mod devinit; > +mod dma; > mod driver; > mod firmware; > mod gpu; > diff --git a/drivers/gpu/nova-core/regs.rs b/drivers/gpu/nova-core/regs.rs > index fd7096f0ddd4af90114dd1119d9715d2cd3aa2ac..1e24787c4b5f432ac25fe399c8cb38b7350e44ae 100644 > --- a/drivers/gpu/nova-core/regs.rs > +++ b/drivers/gpu/nova-core/regs.rs > @@ -14,6 +14,16 @@ > 28:20 chipset => try_into Chipset, "chipset model" > ); > > +/* PFB */ > + > +register!(PfbNisoFlushSysmemAddr at 0x00100c10; > + 31:0 adr_39_08 => as u32 > +); > + > +register!(PfbNisoFlushSysmemAddrHi at 0x00100c40; > + 23:0 adr_63_40 => as u32 > +);Please add some documentation for the register and its fields.
Joel Fernandes
2025-Apr-22  18:50 UTC
[PATCH 09/16] gpu: nova-core: register sysmem flush page
On 4/20/2025 8:19 AM, Alexandre Courbot wrote:> A page of system memory is reserved so sysmembar can perform a read on > it if a system write occurred since the last flush. Do this early as it > can be required to e.g. reset the GPU falcons. > > Signed-off-by: Alexandre Courbot <acourbot at nvidia.com> > --- > drivers/gpu/nova-core/dma.rs | 54 ++++++++++++++++++++++++++++++++++++++ > drivers/gpu/nova-core/gpu.rs | 53 +++++++++++++++++++++++++++++++++++-- > drivers/gpu/nova-core/nova_core.rs | 1 + > drivers/gpu/nova-core/regs.rs | 10 +++++++ > 4 files changed, 116 insertions(+), 2 deletions(-) > > diff --git a/drivers/gpu/nova-core/dma.rs b/drivers/gpu/nova-core/dma.rs > new file mode 100644 > index 0000000000000000000000000000000000000000..a4162bff597132a04e002b2b910a4537bbabc287 > --- /dev/null > +++ b/drivers/gpu/nova-core/dma.rs > @@ -0,0 +1,54 @@ > +// SPDX-License-Identifier: GPL-2.0 > + > +//! Simple DMA object wrapper. > + > +// To be removed when all code is used. > +#![allow(dead_code)] > + > +use kernel::device; > +use kernel::dma::CoherentAllocation; > +use kernel::page::PAGE_SIZE; > +use kernel::prelude::*; > + > +pub(crate) struct DmaObject { > + pub dma: CoherentAllocation<u8>, > + pub len: usize, > + #[allow(dead_code)] > + pub name: &'static str, > +} > + > +impl DmaObject { > + pub(crate) fn new( > + dev: &device::Device<device::Bound>, > + len: usize, > + name: &'static str, > + ) -> Result<Self> { > + let len = core::alloc::Layout::from_size_align(len, PAGE_SIZE) > + .map_err(|_| EINVAL)? > + .pad_to_align() > + .size(); > + let dma = CoherentAllocation::alloc_coherent(dev, len, GFP_KERNEL | __GFP_ZERO)?; > + > + Ok(Self { dma, len, name }) > + } > + > + pub(crate) fn from_data( > + dev: &device::Device<device::Bound>, > + data: &[u8], > + name: &'static str, > + ) -> Result<Self> { > + Self::new(dev, data.len(), name).and_then(|mut dma_obj| { > + // SAFETY: > + // - The copied data fits within the size of the allocated object. > + // - We have just created this object and there is no other user at this stage. > + unsafe { > + core::ptr::copy_nonoverlapping( > + data.as_ptr(), > + dma_obj.dma.start_ptr_mut(), > + data.len(), > + ); > + } > + Ok(dma_obj) > + }) > + } > +} > diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs > index 1f7799692a0ab042f2540e01414f5ca347ae9ecc..d43e710cc983d51f053dacbd77cbbfb79fa882c3 100644 > --- a/drivers/gpu/nova-core/gpu.rs > +++ b/drivers/gpu/nova-core/gpu.rs > @@ -3,6 +3,7 @@ > use kernel::{device, devres::Devres, error::code::*, pci, prelude::*}; > > use crate::devinit; > +use crate::dma::DmaObject; > use crate::driver::Bar0; > use crate::firmware::Firmware; > use crate::regs; > @@ -145,12 +146,30 @@ fn new(bar: &Devres<Bar0>) -> Result<Spec> { > } > > /// Structure holding the resources required to operate the GPU. > -#[pin_data] > +#[pin_data(PinnedDrop)] > pub(crate) struct Gpu { > spec: Spec, > /// MMIO mapping of PCI BAR 0 > bar: Devres<Bar0>, > fw: Firmware,Can add here: // System memory page required for sysmembar which is a GPU-initiated hardware // memory-barrier operation that flushes all pending GPU-side memory writes // that were done through PCIE, to system memory. Will add to my git tree as well (but feel free to squash as needed).> + sysmem_flush: DmaObject, > +} > + > +#[pinned_drop] > +impl PinnedDrop for Gpu { > + fn drop(self: Pin<&mut Self>) { > + // Unregister the sysmem flush page before we release it. > + let _ = with_bar!(&self.bar, |b| { > + regs::PfbNisoFlushSysmemAddr::default() > + .set_adr_39_08(0) > + .write(b); > + if self.spec.chipset >= Chipset::GA102 { > + regs::PfbNisoFlushSysmemAddrHi::default() > + .set_adr_63_40(0) > + .write(b); > + } > + }); > + } > } > > impl Gpu { > @@ -173,6 +192,36 @@ pub(crate) fn new( > devinit::wait_gfw_boot_completion(&bar) > .inspect_err(|_| pr_err!("GFW boot did not complete"))?; > > - Ok(pin_init!(Self { spec, bar, fw })) > + // System memory page required for sysmembar to properly flush into system memory.Can elaborate more here: // System memory page required for sysmembar which is a GPU-initiated hardware // memory-barrier operation that flushes all GPU-side memory writes that were // done through PCIE, to system memory. It is required for Falcon to be reset // as the reset operation involves a reset handshake. When the falcon acks the // reset, it writes its acknowledgement into system memory, but for this write // to be visible to the host, it needs to do sysmembar to flush the write and // prevent the driver from timing out.> + let sysmem_flush = { > + let page = DmaObject::new( > + pdev.as_ref(), > + kernel::bindings::PAGE_SIZE, > + "sysmem flush page", > + )?; > + > + // Register the sysmem flush page. > + with_bar!(bar, |b| { > + let handle = page.dma.dma_handle(); > + > + regs::PfbNisoFlushSysmemAddr::default() > + .set_adr_39_08((handle >> 8) as u32) > + .write(b); > + if spec.chipset >= Chipset::GA102 { > + regs::PfbNisoFlushSysmemAddrHi::default() > + .set_adr_63_40((handle >> 40) as u32) > + .write(b); > + } > + })?; > + > + page > + }; > + > + Ok(pin_init!(Self { > + spec, > + bar, > + fw, > + sysmem_flush, > + })) > } > } > diff --git a/drivers/gpu/nova-core/nova_core.rs b/drivers/gpu/nova-core/nova_core.rs > index 878161e060f54da7738c656f6098936a62dcaa93..37c7eb0ea7a926bee4e3c661028847291bf07fa2 100644 > --- a/drivers/gpu/nova-core/nova_core.rs > +++ b/drivers/gpu/nova-core/nova_core.rs > @@ -21,6 +21,7 @@ macro_rules! with_bar { > } > > mod devinit; > +mod dma; > mod driver; > mod firmware; > mod gpu; > diff --git a/drivers/gpu/nova-core/regs.rs b/drivers/gpu/nova-core/regs.rs > index fd7096f0ddd4af90114dd1119d9715d2cd3aa2ac..1e24787c4b5f432ac25fe399c8cb38b7350e44ae 100644 > --- a/drivers/gpu/nova-core/regs.rs > +++ b/drivers/gpu/nova-core/regs.rs > @@ -14,6 +14,16 @@ > 28:20 chipset => try_into Chipset, "chipset model" > ); > > +/* PFB */Also can add: /// These two registers together hold the physical system memory address /// that is used by the GPU for perform sysmembar operation (see gpu.rs).> + > +register!(PfbNisoFlushSysmemAddr at 0x00100c10; > + 31:0 adr_39_08 => as u32 > +); > + > +register!(PfbNisoFlushSysmemAddrHi at 0x00100c40; > + 23:0 adr_63_40 => as u32 > +); > +Thanks.