search for: mlx5_vdpa_destroy_mr

This patch series is based on Eugenio's fix for handling CVQs in a different ASID [0]. The first patch is the actual fix. The next 2 patches are fixing a possible issue that I found while implementing patch 1. The patches are ordered like this for clarity. [0] https://lore.kernel.org/lkml/20230112142218.725622-1-eperezma at redhat.com/ Dragos Tatulea (1): vdpa/mlx5: Fix

This patch series is based on Eugenio's fix for handling CVQs in a different ASID [0]. The first patch is the actual fix. The next 2 patches are fixing a possible issue that I found while implementing patch 1. The patches are ordered like this for clarity. [0] https://lore.kernel.org/lkml/20230112142218.725622-1-eperezma at redhat.com/ Dragos Tatulea (1): vdpa/mlx5: Fix

...lx5/core/mlx5_vdpa.h b/drivers/vdpa/mlx5/core/mlx5_vdpa.h index b53420e..5c9a25a 100644 --- a/drivers/vdpa/mlx5/core/mlx5_vdpa.h +++ b/drivers/vdpa/mlx5/core/mlx5_vdpa.h @@ -123,6 +123,7 @@ int mlx5_vdpa_create_mr(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *iotlb, unsigned int asid); void mlx5_vdpa_destroy_mr(struct mlx5_vdpa_dev *mvdev); void mlx5_vdpa_destroy_mr_asid(struct mlx5_vdpa_dev *mvdev, unsigned int asid); +int mlx5_vdpa_reset_mr(struct mlx5_vdpa_dev *mvdev, unsigned int asid); #define mlx5_vdpa_warn(__dev, format, ...) \ dev_warn(...

...ivers/vdpa/mlx5/core/mr.c > +++ b/drivers/vdpa/mlx5/core/mr.c > @@ -434,6 +434,7 @@ int mlx5_vdpa_create_mr(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *iotlb) > mutex_unlock(&mr->mkey_mtx); > return err; > } > +EXPORT_SYMBOL(mlx5_vdpa_create_mr); > > void mlx5_vdpa_destroy_mr(struct mlx5_vdpa_dev *mvdev) > { > @@ -456,6 +457,7 @@ void mlx5_vdpa_destroy_mr(struct mlx5_vdpa_dev *mvdev) > out: > mutex_unlock(&mr->mkey_mtx); > } > +EXPORT_SYMBOL(mlx5_vdpa_destroy_mr); > > static bool map_empty(struct vhost_iotlb *iotlb) > { > @@...

...> > @@ -434,6 +434,7 @@ int mlx5_vdpa_create_mr(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *iotlb) > > > mutex_unlock(&mr->mkey_mtx); > > > return err; > > > } > > > +EXPORT_SYMBOL(mlx5_vdpa_create_mr); > > > > > > void mlx5_vdpa_destroy_mr(struct mlx5_vdpa_dev *mvdev) > > > { > > > @@ -456,6 +457,7 @@ void mlx5_vdpa_destroy_mr(struct mlx5_vdpa_dev *mvdev) > > > out: > > > mutex_unlock(&mr->mkey_mtx); > > > } > > > +EXPORT_SYMBOL(mlx5_vdpa_destroy_mr); > > >...

...: warning: no previous prototype for 'mlx5_vdpa_create_mr' [-Wmissing-prototypes] 414 | int mlx5_vdpa_create_mr(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *iotlb) | ^~~~~~~~~~~~~~~~~~~ >> drivers/vdpa/mlx5/core/mr.c:425:6: warning: no previous prototype for 'mlx5_vdpa_destroy_mr' [-Wmissing-prototypes] 425 | void mlx5_vdpa_destroy_mr(struct mlx5_vdpa_dev *mvdev) | ^~~~~~~~~~~~~~~~~~~~ vim +/mlx5_vdpa_create_mr +414 drivers/vdpa/mlx5/core/mr.c 413 > 414 int mlx5_vdpa_create_mr(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *iotlb) 415 {...

...mlx5_vdpa_dev *mvdev, struct vhost_iotlb *iotlb) > > > > > mutex_unlock(&mr->mkey_mtx); > > > > > return err; > > > > > } > > > > > +EXPORT_SYMBOL(mlx5_vdpa_create_mr); > > > > > > > > > > void mlx5_vdpa_destroy_mr(struct mlx5_vdpa_dev *mvdev) > > > > > { > > > > > @@ -456,6 +457,7 @@ void mlx5_vdpa_destroy_mr(struct mlx5_vdpa_dev *mvdev) > > > > > out: > > > > > mutex_unlock(&mr->mkey_mtx); > > > > > } > > > >...

...> > > > > mutex_unlock(&mr->mkey_mtx); > > > > > > > return err; > > > > > > > } > > > > > > > +EXPORT_SYMBOL(mlx5_vdpa_create_mr); > > > > > > > > > > > > > > void mlx5_vdpa_destroy_mr(struct mlx5_vdpa_dev *mvdev) > > > > > > > { > > > > > > > @@ -456,6 +457,7 @@ void mlx5_vdpa_destroy_mr(struct mlx5_vdpa_dev *mvdev) > > > > > > > out: > > > > > > > mutex_unlock(&mr->mkey_mtx); > &gt...

...r_mr; - if (!(mvdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) + if (!(mvdev->status & VIRTIO_CONFIG_S_DRIVER_OK) || mvdev->suspended) goto err_mr; restore_channels_info(ndev); @@ -2579,6 +2579,7 @@ static int mlx5_vdpa_reset(struct vdpa_device *vdev) clear_vqs_ready(ndev); mlx5_vdpa_destroy_mr(&ndev->mvdev); ndev->mvdev.status = 0; + ndev->mvdev.suspended = false; ndev->cur_num_vqs = 0; ndev->mvdev.cvq.received_desc = 0; ndev->mvdev.cvq.completed_desc = 0; @@ -2815,6 +2816,8 @@ static int mlx5_vdpa_suspend(struct vdpa_device *vdev) struct mlx5_vdpa_virtque...

...#39;m not sure this is correct or at least we need restrict in in the vDPA core. Thanks > goto err_mr; > > restore_channels_info(ndev); > @@ -2579,6 +2579,7 @@ static int mlx5_vdpa_reset(struct vdpa_device *vdev) > clear_vqs_ready(ndev); > mlx5_vdpa_destroy_mr(&ndev->mvdev); > ndev->mvdev.status = 0; > + ndev->mvdev.suspended = false; > ndev->cur_num_vqs = 0; > ndev->mvdev.cvq.received_desc = 0; > ndev->mvdev.cvq.completed_desc = 0; > @@ -2815,6 +2816,8 @@ static int mlx5_vdpa_...

...t we need restrict in in the vDPA core. > > Thanks > >> goto err_mr; >> >> restore_channels_info(ndev); >> @@ -2579,6 +2579,7 @@ static int mlx5_vdpa_reset(struct vdpa_device *vdev) >> clear_vqs_ready(ndev); >> mlx5_vdpa_destroy_mr(&ndev->mvdev); >> ndev->mvdev.status = 0; >> + ndev->mvdev.suspended = false; >> ndev->cur_num_vqs = 0; >> ndev->mvdev.cvq.received_desc = 0; >> ndev->mvdev.cvq.completed_desc = 0; >> @@ -2815,6 +281...

In order to reduce needlessly high setup and teardown cost of iotlb mapping during live migration, it's crucial to decouple the vhost-vdpa iotlb abstraction from the virtio device life cycle, i.e. iotlb mappings should be left intact across virtio device reset [1]. For it to work, the on-chip IOMMU parent device should implement a separate .reset_map() operation callback to restore 1:1 DMA

In order to reduce needlessly high setup and teardown cost of iotlb mapping during live migration, it's crucial to decouple the vhost-vdpa iotlb abstraction from the virtio device life cycle, i.e. iotlb mappings should be left intact across virtio device reset [1]. For it to work, the on-chip IOMMU parent device should implement a separate .reset_map() operation callback to restore 1:1 DMA

...:414:5: warning: no previous prototype for 'mlx5_vdpa_create_mr' [-Wmissing-prototypes] 414 | int mlx5_vdpa_create_mr(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *iotlb) | ^~~~~~~~~~~~~~~~~~~ drivers/vdpa/mlx5/core/mr.c:425:6: warning: no previous prototype for 'mlx5_vdpa_destroy_mr' [-Wmissing-prototypes] 425 | void mlx5_vdpa_destroy_mr(struct mlx5_vdpa_dev *mvdev) | ^~~~~~~~~~~~~~~~~~~~ cc1: some warnings being treated as errors vim +254 drivers/vdpa/mlx5/core/mr.c 215 216 static int map_direct_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vd...

...t;cb; > + } > +} > + > +static int mlx5_vdpa_change_map(struct mlx5_vdpa_net *ndev, struct vhost_iotlb *iotlb) > +{ > + int err; > + > + suspend_vqs(ndev); > + err = save_channels_info(ndev); > + if (err) > + goto err_mr; > + > + teardown_driver(ndev); > + mlx5_vdpa_destroy_mr(&ndev->mvdev); > + err = mlx5_vdpa_create_mr(&ndev->mvdev, iotlb); > + if (err) > + goto err_mr; > + > + restore_channels_info(ndev); > + err = setup_driver(ndev); > + if (err) > + goto err_setup; > + > + return 0; > + > +err_setup: > + mlx5_vd...

...5_vdpa_info(mvdev, "performing device reset\n"); > >> > >> down_write(&ndev->reslock); > >> + unregister_link_notifier(ndev); > >> teardown_driver(ndev); > >> clear_vqs_ready(ndev); > >> mlx5_vdpa_destroy_mr(&ndev->mvdev); > >> @@ -2883,9 +2993,7 @@ static int mlx5_vdpa_suspend(struct vdpa_device *vdev) > >> mlx5_vdpa_info(mvdev, "suspending device\n"); > >> > >> down_write(&ndev->reslock); > >> - ndev->nb_r...