Linux Virtualization - Oct 2023 - [PATCH vhost 13/16] vdpa/mlx5: Introduce mr for vq descriptor

On Wed, 2023-10-04 at 20:53 +0200, Eugenio Perez Martin
wrote:
> On Thu, Sep 28, 2023 at 6:50?PM Dragos Tatulea <dtatulea at
nvidia.com> wrote:
> > 
> > Introduce the vq descriptor group and ASID 1. Until now .set_map on
ASID
> 
> s/ASID/vq group/?
> 
Oh, indeed.
> > 1 was only updating the cvq iotlb. From now on it also creates a mkey
> > for it. The current patch doesn't use it but follow-up patches
will
> > add hardware support for mapping the vq descriptors.
> > 
> > Signed-off-by: Dragos Tatulea <dtatulea at nvidia.com>
> > ---
> > ?drivers/vdpa/mlx5/core/mlx5_vdpa.h |? 5 +++--
> > ?drivers/vdpa/mlx5/core/mr.c??????? | 14 +++++++++-----
> > ?drivers/vdpa/mlx5/net/mlx5_vnet.c? | 20 +++++++++++++-------
> > ?3 files changed, 25 insertions(+), 14 deletions(-)
> > 
> > diff --git a/drivers/vdpa/mlx5/core/mlx5_vdpa.h
> > b/drivers/vdpa/mlx5/core/mlx5_vdpa.h
> > index bbe4335106bd..ae09296f4270 100644
> > --- a/drivers/vdpa/mlx5/core/mlx5_vdpa.h
> > +++ b/drivers/vdpa/mlx5/core/mlx5_vdpa.h
> > @@ -70,11 +70,12 @@ struct mlx5_vdpa_wq_ent {
> > ?enum {
> > ??????? MLX5_VDPA_DATAVQ_GROUP,
> > ??????? MLX5_VDPA_CVQ_GROUP,
> > +?????? MLX5_VDPA_DATAVQ_DESC_GROUP,
> > ??????? MLX5_VDPA_NUMVQ_GROUPS
> > ?};
> > 
> > ?enum {
> > -?????? MLX5_VDPA_NUM_AS = MLX5_VDPA_NUMVQ_GROUPS
> > +?????? MLX5_VDPA_NUM_AS = 2
> > ?};
> > 
> > ?struct mlx5_vdpa_dev {
> > @@ -89,7 +90,7 @@ struct mlx5_vdpa_dev {
> > ??????? u16 max_idx;
> > ??????? u32 generation;
> > 
> > -?????? struct mlx5_vdpa_mr *mr;
> > +?????? struct mlx5_vdpa_mr *mr[MLX5_VDPA_NUM_AS];
> 
> I'm wondering if it makes sense to squash all of this patch with the
> previous one, as I think *mr[MLX5_VDPA_NUM_AS] makes way more sense
> than just *mr.
> 
I've been on the fence about this one. It seemed cleaner to have two
patches.
> Whatever you choose, for both patches:
> 
> Acked-by: Eugenio P?rez <eperezma at redhat.com>
> 
> > ??????? /* serialize mr access */
> > ??????? struct mutex mr_mtx;
> > ??????? struct mlx5_control_vq cvq;
> > diff --git a/drivers/vdpa/mlx5/core/mr.c b/drivers/vdpa/mlx5/core/mr.c
> > index 00eff5a07152..3dee6d9bed6b 100644
> > --- a/drivers/vdpa/mlx5/core/mr.c
> > +++ b/drivers/vdpa/mlx5/core/mr.c
> > @@ -511,8 +511,10 @@ void mlx5_vdpa_destroy_mr(struct mlx5_vdpa_dev
*mvdev,
> > 
> > ??????? _mlx5_vdpa_destroy_mr(mvdev, mr);
> > 
> > -?????? if (mvdev->mr == mr)
> > -?????????????? mvdev->mr = NULL;
> > +?????? for (int i = 0; i < MLX5_VDPA_NUM_AS; i++) {
> > +?????????????? if (mvdev->mr[i] == mr)
> > +?????????????????????? mvdev->mr[i] = NULL;
> > +?????? }
> > 
> > ??????? mutex_unlock(&mvdev->mr_mtx);
> > 
> > @@ -523,11 +525,11 @@ void mlx5_vdpa_update_mr(struct mlx5_vdpa_dev
*mvdev,
> > ???????????????????????? struct mlx5_vdpa_mr *new_mr,
> > ???????????????????????? unsigned int asid)
> > ?{
> > -?????? struct mlx5_vdpa_mr *old_mr = mvdev->mr;
> > +?????? struct mlx5_vdpa_mr *old_mr = mvdev->mr[asid];
> > 
> > ??????? mutex_lock(&mvdev->mr_mtx);
> > 
> > -?????? mvdev->mr = new_mr;
> > +?????? mvdev->mr[asid] = new_mr;
> > ??????? if (old_mr) {
> > ??????????????? _mlx5_vdpa_destroy_mr(mvdev, old_mr);
> > ??????????????? kfree(old_mr);
> > @@ -539,7 +541,9 @@ void mlx5_vdpa_update_mr(struct mlx5_vdpa_dev
*mvdev,
> > 
> > ?void mlx5_vdpa_destroy_mr_resources(struct mlx5_vdpa_dev *mvdev)
> > ?{
> > -?????? mlx5_vdpa_destroy_mr(mvdev, mvdev->mr);
> > +?????? for (int i = 0; i < MLX5_VDPA_NUM_AS; i++)
> > +?????????????? mlx5_vdpa_destroy_mr(mvdev, mvdev->mr[i]);
> > +
> > ??????? prune_iotlb(mvdev);
> > ?}
> > 
> > diff --git a/drivers/vdpa/mlx5/net/mlx5_vnet.c
> > b/drivers/vdpa/mlx5/net/mlx5_vnet.c
> > index 4a87f9119fca..25bd2c324f5b 100644
> > --- a/drivers/vdpa/mlx5/net/mlx5_vnet.c
> > +++ b/drivers/vdpa/mlx5/net/mlx5_vnet.c
> > @@ -821,6 +821,8 @@ static int create_virtqueue(struct mlx5_vdpa_net
*ndev,
> > struct mlx5_vdpa_virtque
> > ?{
> > ??????? int inlen = MLX5_ST_SZ_BYTES(create_virtio_net_q_in);
> > ??????? u32 out[MLX5_ST_SZ_DW(create_virtio_net_q_out)] = {};
> > +?????? struct mlx5_vdpa_dev *mvdev = &ndev->mvdev;
> > +?????? struct mlx5_vdpa_mr *vq_mr;
> > ??????? void *obj_context;
> > ??????? u16 mlx_features;
> > ??????? void *cmd_hdr;
> > @@ -873,7 +875,9 @@ static int create_virtqueue(struct mlx5_vdpa_net
*ndev,
> > struct mlx5_vdpa_virtque
> > ??????? MLX5_SET64(virtio_q, vq_ctx, desc_addr, mvq->desc_addr);
> > ??????? MLX5_SET64(virtio_q, vq_ctx, used_addr, mvq->device_addr);
> > ??????? MLX5_SET64(virtio_q, vq_ctx, available_addr,
mvq->driver_addr);
> > -?????? MLX5_SET(virtio_q, vq_ctx, virtio_q_mkey,
ndev->mvdev.mr->mkey);
> > +?????? vq_mr =
mvdev->mr[mvdev->group2asid[MLX5_VDPA_DATAVQ_GROUP]];
> > +?????? if (vq_mr)
> > +?????????????? MLX5_SET(virtio_q, vq_ctx, virtio_q_mkey,
vq_mr->mkey);
> > ??????? MLX5_SET(virtio_q, vq_ctx, umem_1_id, mvq->umem1.id);
> > ??????? MLX5_SET(virtio_q, vq_ctx, umem_1_size, mvq->umem1.size);
> > ??????? MLX5_SET(virtio_q, vq_ctx, umem_2_id, mvq->umem2.id);
> > @@ -2633,7 +2637,8 @@ static void restore_channels_info(struct
mlx5_vdpa_net
> > *ndev)
> > ?}
> > 
> > ?static int mlx5_vdpa_change_map(struct mlx5_vdpa_dev *mvdev,
> > -?????????????????????????????? struct mlx5_vdpa_mr *new_mr, unsigned
int
> > asid)
> > +?????????????????????????????? struct mlx5_vdpa_mr *new_mr,
> > +?????????????????????????????? unsigned int asid)
> > ?{
> > ??????? struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
> > ??????? int err;
> > @@ -2652,8 +2657,10 @@ static int mlx5_vdpa_change_map(struct
mlx5_vdpa_dev
> > *mvdev,
> > 
> > ??????? restore_channels_info(ndev);
> > ??????? err = setup_driver(mvdev);
> > +?????? if (err)
> > +?????????????? return err;
> > 
> > -?????? return err;
> > +?????? return 0;
> > ?}
> > 
> > ?/* reslock must be held for this function */
> > @@ -2869,8 +2876,8 @@ static int set_map_data(struct mlx5_vdpa_dev
*mvdev,
> > struct vhost_iotlb *iotlb,
> > ??????? struct mlx5_vdpa_mr *new_mr;
> > ??????? int err;
> > 
> > -?????? if (mvdev->group2asid[MLX5_VDPA_DATAVQ_GROUP] != asid)
> > -?????????????? goto end;
> > +?????? if (asid >= MLX5_VDPA_NUM_AS)
> > +?????????????? return -EINVAL;
> > 
> > ??????? new_mr = mlx5_vdpa_create_mr(mvdev, iotlb);
> > ??????? if (IS_ERR(new_mr)) {
> > @@ -2879,7 +2886,7 @@ static int set_map_data(struct mlx5_vdpa_dev
*mvdev,
> > struct vhost_iotlb *iotlb,
> > ??????????????? return err;
> > ??????? }
> > 
> > -?????? if (!mvdev->mr) {
> > +?????? if (!mvdev->mr[asid]) {
> > ??????????????? mlx5_vdpa_update_mr(mvdev, new_mr, asid);
> > ??????? } else {
> > ??????????????? err = mlx5_vdpa_change_map(mvdev, new_mr, asid);
> > @@ -2889,7 +2896,6 @@ static int set_map_data(struct mlx5_vdpa_dev
*mvdev,
> > struct vhost_iotlb *iotlb,
> > ??????????????? }
> > ??????? }
> > 
> > -end:
> > ??????? return mlx5_vdpa_update_cvq_iotlb(mvdev, iotlb, asid);
> > 
> > ?out_err:
> > --
> > 2.41.0
> > 
>