Jason Wang
2023-Mar-23 05:14 UTC
[PATCH v3 virtio 5/8] pds_vdpa: add vdpa config client commands
On Thu, Mar 23, 2023 at 3:11?AM Shannon Nelson <shannon.nelson at amd.com> wrote:> > These are the adminq commands that will be needed for > setting up and using the vDPA device. There are a number > of commands defined in the FW's API, but by making use of > the FW's virtio BAR we only need a few of these commands > for vDPA support. > > Signed-off-by: Shannon Nelson <shannon.nelson at amd.com>Acked-by: Jason Wang <jasowang at redhat.com> Thanks> --- > drivers/vdpa/pds/Makefile | 1 + > drivers/vdpa/pds/cmds.c | 178 +++++++++++++++++++++++++++++++++++ > drivers/vdpa/pds/cmds.h | 16 ++++ > drivers/vdpa/pds/vdpa_dev.h | 33 ++++++- > include/linux/pds/pds_vdpa.h | 175 ++++++++++++++++++++++++++++++++++ > 5 files changed, 402 insertions(+), 1 deletion(-) > create mode 100644 drivers/vdpa/pds/cmds.c > create mode 100644 drivers/vdpa/pds/cmds.h > > diff --git a/drivers/vdpa/pds/Makefile b/drivers/vdpa/pds/Makefile > index 13b50394ec64..2e22418e3ab3 100644 > --- a/drivers/vdpa/pds/Makefile > +++ b/drivers/vdpa/pds/Makefile > @@ -4,6 +4,7 @@ > obj-$(CONFIG_PDS_VDPA) := pds_vdpa.o > > pds_vdpa-y := aux_drv.o \ > + cmds.o \ > vdpa_dev.o > > pds_vdpa-$(CONFIG_DEBUG_FS) += debugfs.o > diff --git a/drivers/vdpa/pds/cmds.c b/drivers/vdpa/pds/cmds.c > new file mode 100644 > index 000000000000..b847d708e4cc > --- /dev/null > +++ b/drivers/vdpa/pds/cmds.c > @@ -0,0 +1,178 @@ > +// SPDX-License-Identifier: GPL-2.0-only > +/* Copyright(c) 2023 Advanced Micro Devices, Inc */ > + > +#include <linux/vdpa.h> > +#include <linux/virtio_pci_modern.h> > + > +#include <linux/pds/pds_common.h> > +#include <linux/pds/pds_core_if.h> > +#include <linux/pds/pds_adminq.h> > +#include <linux/pds/pds_auxbus.h> > +#include <linux/pds/pds_vdpa.h> > + > +#include "vdpa_dev.h" > +#include "aux_drv.h" > +#include "cmds.h" > + > +int pds_vdpa_init_hw(struct pds_vdpa_device *pdsv) > +{ > + struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev; > + struct device *dev = &padev->aux_dev.dev; > + struct pds_vdpa_init_cmd init_cmd = { > + .opcode = PDS_VDPA_CMD_INIT, > + .vdpa_index = pdsv->vdpa_index, > + .vf_id = cpu_to_le16(pdsv->vdpa_aux->vf_id), > + }; > + struct pds_vdpa_comp init_comp = {0}; > + int err; > + > + /* Initialize the vdpa/virtio device */ > + err = padev->ops->adminq_cmd(padev, > + (union pds_core_adminq_cmd *)&init_cmd, > + sizeof(init_cmd), > + (union pds_core_adminq_comp *)&init_comp, > + 0); > + if (err) > + dev_dbg(dev, "Failed to init hw, status %d: %pe\n", > + init_comp.status, ERR_PTR(err)); > + > + return err; > +} > + > +int pds_vdpa_cmd_reset(struct pds_vdpa_device *pdsv) > +{ > + struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev; > + struct device *dev = &padev->aux_dev.dev; > + struct pds_vdpa_cmd cmd = { > + .opcode = PDS_VDPA_CMD_RESET, > + .vdpa_index = pdsv->vdpa_index, > + .vf_id = cpu_to_le16(pdsv->vdpa_aux->vf_id), > + }; > + struct pds_vdpa_comp comp = {0}; > + int err; > + > + err = padev->ops->adminq_cmd(padev, > + (union pds_core_adminq_cmd *)&cmd, > + sizeof(cmd), > + (union pds_core_adminq_comp *)&comp, > + 0); > + if (err) > + dev_dbg(dev, "Failed to reset hw, status %d: %pe\n", > + comp.status, ERR_PTR(err)); > + > + return err; > +} > + > +int pds_vdpa_cmd_set_mac(struct pds_vdpa_device *pdsv, u8 *mac) > +{ > + struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev; > + struct device *dev = &padev->aux_dev.dev; > + struct pds_vdpa_setattr_cmd cmd = { > + .opcode = PDS_VDPA_CMD_SET_ATTR, > + .vdpa_index = pdsv->vdpa_index, > + .vf_id = cpu_to_le16(pdsv->vdpa_aux->vf_id), > + .attr = PDS_VDPA_ATTR_MAC, > + }; > + struct pds_vdpa_comp comp = {0}; > + int err; > + > + ether_addr_copy(cmd.mac, mac); > + err = padev->ops->adminq_cmd(padev, > + (union pds_core_adminq_cmd *)&cmd, > + sizeof(cmd), > + (union pds_core_adminq_comp *)&comp, > + 0); > + if (err) > + dev_dbg(dev, "Failed to set mac address %pM, status %d: %pe\n", > + mac, comp.status, ERR_PTR(err)); > + > + return err; > +} > + > +int pds_vdpa_cmd_set_max_vq_pairs(struct pds_vdpa_device *pdsv, u16 max_vqp) > +{ > + struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev; > + struct device *dev = &padev->aux_dev.dev; > + struct pds_vdpa_setattr_cmd cmd = { > + .opcode = PDS_VDPA_CMD_SET_ATTR, > + .vdpa_index = pdsv->vdpa_index, > + .vf_id = cpu_to_le16(pdsv->vdpa_aux->vf_id), > + .attr = PDS_VDPA_ATTR_MAX_VQ_PAIRS, > + .max_vq_pairs = cpu_to_le16(max_vqp), > + }; > + struct pds_vdpa_comp comp = {0}; > + int err; > + > + err = padev->ops->adminq_cmd(padev, > + (union pds_core_adminq_cmd *)&cmd, > + sizeof(cmd), > + (union pds_core_adminq_comp *)&comp, > + 0); > + if (err) > + dev_dbg(dev, "Failed to set max vq pairs %u, status %d: %pe\n", > + max_vqp, comp.status, ERR_PTR(err)); > + > + return err; > +} > + > +int pds_vdpa_cmd_init_vq(struct pds_vdpa_device *pdsv, u16 qid, > + struct pds_vdpa_vq_info *vq_info) > +{ > + struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev; > + struct device *dev = &padev->aux_dev.dev; > + struct pds_vdpa_vq_init_comp comp = {0}; > + struct pds_vdpa_vq_init_cmd cmd = { > + .opcode = PDS_VDPA_CMD_VQ_INIT, > + .vdpa_index = pdsv->vdpa_index, > + .vf_id = cpu_to_le16(pdsv->vdpa_aux->vf_id), > + .qid = cpu_to_le16(qid), > + .len = cpu_to_le16(ilog2(vq_info->q_len)), > + .desc_addr = cpu_to_le64(vq_info->desc_addr), > + .avail_addr = cpu_to_le64(vq_info->avail_addr), > + .used_addr = cpu_to_le64(vq_info->used_addr), > + .intr_index = cpu_to_le16(qid), > + }; > + int err; > + > + dev_dbg(dev, "%s: qid %d len %d desc_addr %#llx avail_addr %#llx used_addr %#llx\n", > + __func__, qid, ilog2(vq_info->q_len), > + vq_info->desc_addr, vq_info->avail_addr, vq_info->used_addr); > + > + err = padev->ops->adminq_cmd(padev, > + (union pds_core_adminq_cmd *)&cmd, > + sizeof(cmd), > + (union pds_core_adminq_comp *)&comp, > + 0); > + if (err) { > + dev_dbg(dev, "Failed to init vq %d, status %d: %pe\n", > + qid, comp.status, ERR_PTR(err)); > + return err; > + } > + > + return 0; > +} > + > +int pds_vdpa_cmd_reset_vq(struct pds_vdpa_device *pdsv, u16 qid) > +{ > + struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev; > + struct device *dev = &padev->aux_dev.dev; > + struct pds_vdpa_vq_reset_cmd cmd = { > + .opcode = PDS_VDPA_CMD_VQ_RESET, > + .vdpa_index = pdsv->vdpa_index, > + .vf_id = cpu_to_le16(pdsv->vdpa_aux->vf_id), > + .qid = cpu_to_le16(qid), > + }; > + struct pds_vdpa_comp comp = {0}; > + int err; > + > + err = padev->ops->adminq_cmd(padev, > + (union pds_core_adminq_cmd *)&cmd, > + sizeof(cmd), > + (union pds_core_adminq_comp *)&comp, > + 0); > + if (err) > + dev_dbg(dev, "Failed to reset vq %d, status %d: %pe\n", > + qid, comp.status, ERR_PTR(err)); > + > + return err; > +} > diff --git a/drivers/vdpa/pds/cmds.h b/drivers/vdpa/pds/cmds.h > new file mode 100644 > index 000000000000..72e19f4efde6 > --- /dev/null > +++ b/drivers/vdpa/pds/cmds.h > @@ -0,0 +1,16 @@ > +/* SPDX-License-Identifier: GPL-2.0-only */ > +/* Copyright(c) 2023 Advanced Micro Devices, Inc */ > + > +#ifndef _VDPA_CMDS_H_ > +#define _VDPA_CMDS_H_ > + > +int pds_vdpa_init_hw(struct pds_vdpa_device *pdsv); > + > +int pds_vdpa_cmd_reset(struct pds_vdpa_device *pdsv); > +int pds_vdpa_cmd_set_mac(struct pds_vdpa_device *pdsv, u8 *mac); > +int pds_vdpa_cmd_set_max_vq_pairs(struct pds_vdpa_device *pdsv, u16 max_vqp); > +int pds_vdpa_cmd_init_vq(struct pds_vdpa_device *pdsv, u16 qid, > + struct pds_vdpa_vq_info *vq_info); > +int pds_vdpa_cmd_reset_vq(struct pds_vdpa_device *pdsv, u16 qid); > +int pds_vdpa_cmd_set_features(struct pds_vdpa_device *pdsv, u64 features); > +#endif /* _VDPA_CMDS_H_ */ > diff --git a/drivers/vdpa/pds/vdpa_dev.h b/drivers/vdpa/pds/vdpa_dev.h > index 97fab833a0aa..a21596f438c1 100644 > --- a/drivers/vdpa/pds/vdpa_dev.h > +++ b/drivers/vdpa/pds/vdpa_dev.h > @@ -4,11 +4,42 @@ > #ifndef _VDPA_DEV_H_ > #define _VDPA_DEV_H_ > > -#define PDS_VDPA_MAX_QUEUES 65 > +#include <linux/pci.h> > +#include <linux/vdpa.h> > + > +struct pds_vdpa_vq_info { > + bool ready; > + u64 desc_addr; > + u64 avail_addr; > + u64 used_addr; > + u32 q_len; > + u16 qid; > + int irq; > + char irq_name[32]; > + > + void __iomem *notify; > + dma_addr_t notify_pa; > + > + u64 doorbell; > + u16 avail_idx; > + u16 used_idx; > > + struct vdpa_callback event_cb; > + struct pds_vdpa_device *pdsv; > +}; > + > +#define PDS_VDPA_MAX_QUEUES 65 > +#define PDS_VDPA_MAX_QLEN 32768 > struct pds_vdpa_device { > struct vdpa_device vdpa_dev; > struct pds_vdpa_aux *vdpa_aux; > + > + struct pds_vdpa_vq_info vqs[PDS_VDPA_MAX_QUEUES]; > + u64 req_features; /* features requested by vdpa */ > + u64 actual_features; /* features negotiated and in use */ > + u8 vdpa_index; /* rsvd for future subdevice use */ > + u8 num_vqs; /* num vqs in use */ > + struct vdpa_callback config_cb; > }; > > int pds_vdpa_get_mgmt_info(struct pds_vdpa_aux *vdpa_aux); > diff --git a/include/linux/pds/pds_vdpa.h b/include/linux/pds/pds_vdpa.h > index c1d6a3fe2d61..785909a6daf6 100644 > --- a/include/linux/pds/pds_vdpa.h > +++ b/include/linux/pds/pds_vdpa.h > @@ -97,4 +97,179 @@ struct pds_vdpa_ident_cmd { > __le32 len; > __le64 ident_pa; > }; > + > +/** > + * struct pds_vdpa_status_cmd - STATUS_UPDATE command > + * @opcode: Opcode PDS_VDPA_CMD_STATUS_UPDATE > + * @vdpa_index: Index for vdpa subdevice > + * @vf_id: VF id > + * @status: new status bits > + */ > +struct pds_vdpa_status_cmd { > + u8 opcode; > + u8 vdpa_index; > + __le16 vf_id; > + u8 status; > +}; > + > +/** > + * enum pds_vdpa_attr - List of VDPA device attributes > + * @PDS_VDPA_ATTR_MAC: MAC address > + * @PDS_VDPA_ATTR_MAX_VQ_PAIRS: Max virtqueue pairs > + */ > +enum pds_vdpa_attr { > + PDS_VDPA_ATTR_MAC = 1, > + PDS_VDPA_ATTR_MAX_VQ_PAIRS = 2, > +}; > + > +/** > + * struct pds_vdpa_setattr_cmd - SET_ATTR command > + * @opcode: Opcode PDS_VDPA_CMD_SET_ATTR > + * @vdpa_index: Index for vdpa subdevice > + * @vf_id: VF id > + * @attr: attribute to be changed (enum pds_vdpa_attr) > + * @pad: Word boundary padding > + * @mac: new mac address to be assigned as vdpa device address > + * @max_vq_pairs: new limit of virtqueue pairs > + */ > +struct pds_vdpa_setattr_cmd { > + u8 opcode; > + u8 vdpa_index; > + __le16 vf_id; > + u8 attr; > + u8 pad[3]; > + union { > + u8 mac[6]; > + __le16 max_vq_pairs; > + } __packed; > +}; > + > +/** > + * struct pds_vdpa_vq_init_cmd - queue init command > + * @opcode: Opcode PDS_VDPA_CMD_VQ_INIT > + * @vdpa_index: Index for vdpa subdevice > + * @vf_id: VF id > + * @qid: Queue id (bit0 clear = rx, bit0 set = tx, qid=N is ctrlq) > + * @len: log(2) of max descriptor count > + * @desc_addr: DMA address of descriptor area > + * @avail_addr: DMA address of available descriptors (aka driver area) > + * @used_addr: DMA address of used descriptors (aka device area) > + * @intr_index: interrupt index > + */ > +struct pds_vdpa_vq_init_cmd { > + u8 opcode; > + u8 vdpa_index; > + __le16 vf_id; > + __le16 qid; > + __le16 len; > + __le64 desc_addr; > + __le64 avail_addr; > + __le64 used_addr; > + __le16 intr_index; > +}; > + > +/** > + * struct pds_vdpa_vq_init_comp - queue init completion > + * @status: Status of the command (enum pds_core_status_code) > + * @hw_qtype: HW queue type, used in doorbell selection > + * @hw_qindex: HW queue index, used in doorbell selection > + * @rsvd: Word boundary padding > + * @color: Color bit > + */ > +struct pds_vdpa_vq_init_comp { > + u8 status; > + u8 hw_qtype; > + __le16 hw_qindex; > + u8 rsvd[11]; > + u8 color; > +}; > + > +/** > + * struct pds_vdpa_vq_reset_cmd - queue reset command > + * @opcode: Opcode PDS_VDPA_CMD_VQ_RESET > + * @vdpa_index: Index for vdpa subdevice > + * @vf_id: VF id > + * @qid: Queue id > + */ > +struct pds_vdpa_vq_reset_cmd { > + u8 opcode; > + u8 vdpa_index; > + __le16 vf_id; > + __le16 qid; > +}; > + > +/** > + * struct pds_vdpa_set_features_cmd - set hw features > + * @opcode: Opcode PDS_VDPA_CMD_SET_FEATURES > + * @vdpa_index: Index for vdpa subdevice > + * @vf_id: VF id > + * @rsvd: Word boundary padding > + * @features: Feature bit mask > + */ > +struct pds_vdpa_set_features_cmd { > + u8 opcode; > + u8 vdpa_index; > + __le16 vf_id; > + __le32 rsvd; > + __le64 features; > +}; > + > +/** > + * struct pds_vdpa_vq_set_state_cmd - set vq state > + * @opcode: Opcode PDS_VDPA_CMD_VQ_SET_STATE > + * @vdpa_index: Index for vdpa subdevice > + * @vf_id: VF id > + * @qid: Queue id > + * @avail: Device avail index. > + * @used: Device used index. > + * > + * If the virtqueue uses packed descriptor format, then the avail and used > + * index must have a wrap count. The bits should be arranged like the upper > + * 16 bits in the device available notification data: 15 bit index, 1 bit wrap. > + */ > +struct pds_vdpa_vq_set_state_cmd { > + u8 opcode; > + u8 vdpa_index; > + __le16 vf_id; > + __le16 qid; > + __le16 avail; > + __le16 used; > +}; > + > +/** > + * struct pds_vdpa_vq_get_state_cmd - get vq state > + * @opcode: Opcode PDS_VDPA_CMD_VQ_GET_STATE > + * @vdpa_index: Index for vdpa subdevice > + * @vf_id: VF id > + * @qid: Queue id > + */ > +struct pds_vdpa_vq_get_state_cmd { > + u8 opcode; > + u8 vdpa_index; > + __le16 vf_id; > + __le16 qid; > +}; > + > +/** > + * struct pds_vdpa_vq_get_state_comp - get vq state completion > + * @status: Status of the command (enum pds_core_status_code) > + * @rsvd0: Word boundary padding > + * @avail: Device avail index. > + * @used: Device used index. > + * @rsvd: Word boundary padding > + * @color: Color bit > + * > + * If the virtqueue uses packed descriptor format, then the avail and used > + * index will have a wrap count. The bits will be arranged like the "next" > + * part of device available notification data: 15 bit index, 1 bit wrap. > + */ > +struct pds_vdpa_vq_get_state_comp { > + u8 status; > + u8 rsvd0; > + __le16 avail; > + __le16 used; > + u8 rsvd[9]; > + u8 color; > +}; > + > #endif /* _PDS_VDPA_H_ */ > -- > 2.17.1 >