Zhi Wang
2024-Sep-22 12:49 UTC
[RFC 05/29] nvkm/vgpu: populate GSP_VF_INFO when NVIDIA vGPU is enabled
GSP firmware needs to know the VF BAR offsets to correctly calculate the
VF events.
The VF BAR information is stored in GSP_VF_INFO, which needs to be
initialized and uploaded together with the GSP_SYSTEM_INFO.
Populate GSP_VF_INFO when nvkm uploads the GSP_SYSTEM_INFO if NVIDIA
vGPU is enabled.
Cc: Surath Mitra <smitra at nvidia.com>
Signed-off-by: Zhi Wang <zhiw at nvidia.com>
---
.../nouveau/include/nvkm/vgpu_mgr/vgpu_mgr.h | 2 +
.../gpu/drm/nouveau/nvkm/subdev/gsp/r535.c | 3 ++
.../gpu/drm/nouveau/nvkm/vgpu_mgr/vgpu_mgr.c | 50 +++++++++++++++++++
3 files changed, 55 insertions(+)
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/vgpu_mgr/vgpu_mgr.h
b/drivers/gpu/drm/nouveau/include/nvkm/vgpu_mgr/vgpu_mgr.h
index 9e10e18306b0..6bc10fa40cde 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/vgpu_mgr/vgpu_mgr.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/vgpu_mgr/vgpu_mgr.h
@@ -21,5 +21,7 @@ bool nvkm_vgpu_mgr_is_supported(struct nvkm_device *device);
bool nvkm_vgpu_mgr_is_enabled(struct nvkm_device *device);
int nvkm_vgpu_mgr_init(struct nvkm_device *device);
void nvkm_vgpu_mgr_fini(struct nvkm_device *device);
+void nvkm_vgpu_mgr_populate_gsp_vf_info(struct nvkm_device *device,
+ void *info);
#endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/r535.c
b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/r535.c
index 14fc152d6859..49552d7df88f 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/r535.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/r535.c
@@ -1717,6 +1717,9 @@ r535_gsp_rpc_set_system_info(struct nvkm_gsp *gsp)
info->pciConfigMirrorSize = 0x001000;
r535_gsp_acpi_info(gsp, &info->acpiMethodData);
+ if (nvkm_vgpu_mgr_is_supported(device))
+ nvkm_vgpu_mgr_populate_gsp_vf_info(device, info);
+
return nvkm_gsp_rpc_wr(gsp, info, false);
}
diff --git a/drivers/gpu/drm/nouveau/nvkm/vgpu_mgr/vgpu_mgr.c
b/drivers/gpu/drm/nouveau/nvkm/vgpu_mgr/vgpu_mgr.c
index 0639596f8a96..d6ddb1f02275 100644
--- a/drivers/gpu/drm/nouveau/nvkm/vgpu_mgr/vgpu_mgr.c
+++ b/drivers/gpu/drm/nouveau/nvkm/vgpu_mgr/vgpu_mgr.c
@@ -3,6 +3,10 @@
#include <core/driver.h>
#include <nvif/driverif.h>
#include <core/pci.h>
+
+#include <nvrm/nvtypes.h>
+#include <nvrm/535.113.01/nvidia/inc/kernel/gpu/gsp/gsp_static_config.h>
+
#include <vgpu_mgr/vgpu_mgr.h>
static bool support_vgpu_mgr = false;
@@ -120,3 +124,49 @@ void nvkm_vgpu_mgr_fini(struct nvkm_device *device)
detach_nvkm(vgpu_mgr);
vgpu_mgr->enabled = false;
}
+
+/**
+ * nvkm_vgpu_mgr_populate_vf_info - populate GSP_VF_INFO when vGPU
+ * is enabled
+ * @device: the nvkm_device pointer
+ * @info: GSP_VF_INFO data structure
+ */
+void nvkm_vgpu_mgr_populate_gsp_vf_info(struct nvkm_device *device,
+ void *info)
+{
+ struct pci_dev *pdev = nvkm_to_pdev(device);
+ GspSystemInfo *gsp_info = info;
+ GSP_VF_INFO *vf_info = &gsp_info->gspVFInfo;
+ u32 lo, hi;
+ u16 v;
+ int pos;
+
+ pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV);
+
+ pci_read_config_word(pdev, pos + PCI_SRIOV_TOTAL_VF, &v);
+ vf_info->totalVFs = v;
+
+ pci_read_config_word(pdev, pos + PCI_SRIOV_VF_OFFSET, &v);
+ vf_info->firstVFOffset = v;
+
+ pci_read_config_dword(pdev, pos + PCI_SRIOV_BAR, &lo);
+ vf_info->FirstVFBar0Address = lo & 0xFFFFFFF0;
+
+ pci_read_config_dword(pdev, pos + PCI_SRIOV_BAR + 4, &lo);
+ pci_read_config_dword(pdev, pos + PCI_SRIOV_BAR + 8, &hi);
+
+ vf_info->FirstVFBar1Address = (((u64)hi) << 32) + (lo &
0xFFFFFFF0);
+
+ pci_read_config_dword(pdev, pos + PCI_SRIOV_BAR + 12, &lo);
+ pci_read_config_dword(pdev, pos + PCI_SRIOV_BAR + 16, &hi);
+
+ vf_info->FirstVFBar2Address = (((u64)hi) << 32) + (lo &
0xFFFFFFF0);
+
+#define IS_BAR_64(i) (((i) & 0x00000006) == 0x00000004)
+
+ v = nvkm_rd32(device, 0x88000 + 0xbf4);
+ vf_info->b64bitBar1 = IS_BAR_64(v);
+
+ v = nvkm_rd32(device, 0x88000 + 0xbfc);
+ vf_info->b64bitBar2 = IS_BAR_64(v);
+}
--
2.34.1
Jason Gunthorpe
2024-Sep-26 22:52 UTC
[RFC 05/29] nvkm/vgpu: populate GSP_VF_INFO when NVIDIA vGPU is enabled
On Sun, Sep 22, 2024 at 05:49:27AM -0700, Zhi Wang wrote:> +void nvkm_vgpu_mgr_populate_gsp_vf_info(struct nvkm_device *device, > + void *info) > +{ > + struct pci_dev *pdev = nvkm_to_pdev(device); > + GspSystemInfo *gsp_info = info; > + GSP_VF_INFO *vf_info = &gsp_info->gspVFInfo; > + u32 lo, hi; > + u16 v; > + int pos; > + > + pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV); > + > + pci_read_config_word(pdev, pos + PCI_SRIOV_TOTAL_VF, &v); > + vf_info->totalVFs = v; > + > + pci_read_config_word(pdev, pos + PCI_SRIOV_VF_OFFSET, &v); > + vf_info->firstVFOffset = v; > + > + pci_read_config_dword(pdev, pos + PCI_SRIOV_BAR, &lo); > + vf_info->FirstVFBar0Address = lo & 0xFFFFFFF0; > + > + pci_read_config_dword(pdev, pos + PCI_SRIOV_BAR + 4, &lo); > + pci_read_config_dword(pdev, pos + PCI_SRIOV_BAR + 8, &hi); > + > + vf_info->FirstVFBar1Address = (((u64)hi) << 32) + (lo & 0xFFFFFFF0); > + > + pci_read_config_dword(pdev, pos + PCI_SRIOV_BAR + 12, &lo); > + pci_read_config_dword(pdev, pos + PCI_SRIOV_BAR + 16, &hi); > + > + vf_info->FirstVFBar2Address = (((u64)hi) << 32) + (lo & 0xFFFFFFF0); > + > +#define IS_BAR_64(i) (((i) & 0x00000006) == 0x00000004)This should come from the PCI core not be re-read with pci_read_config and hand rolled macros.>From a mlx perspective this is really weird, I'd expect the FW to beable to read its own config space. Jason