search for: mmu_interval_read_begin

...g invalidate_seq; +}; + #ifdef CONFIG_MMU_NOTIFIER #ifdef CONFIG_LOCKDEP @@ -263,6 +289,81 @@ extern int __mmu_notifier_register(struct mmu_notifier *mn, struct mm_struct *mm); extern void mmu_notifier_unregister(struct mmu_notifier *mn, struct mm_struct *mm); + +unsigned long mmu_interval_read_begin(struct mmu_interval_notifier *mni); +int mmu_interval_notifier_insert(struct mmu_interval_notifier *mni, + struct mm_struct *mm, unsigned long start, + unsigned long length, + const struct mmu_interval_notifier_ops *ops); +int mmu_interval_notifier_insert_locked( + struct mmu_interval_n...

...pfn_shift = NVIF_VMM_PFNMAP_V0_ADDR_SHIFT, > }; > - struct mm_struct *mm = notifier->notifier.mm; > + struct mm_struct *mm = svmm->mm; > long ret; > > while (true) { > if (time_after(jiffies, timeout)) > return -EBUSY; > > - range.notifier_seq = mmu_interval_read_begin(range.notifier); > - range.default_flags = 0; > - range.pfn_flags_mask = -1UL; > down_read(&mm->mmap_sem); mmap sem doesn't have to be held for the interval search, and again we have lifetime issues with the membership here. > + ret = nouveau_svmm_interval_find(svmm,...

...it isn't. put should be called 'remove_deferred' OK. > And then you also need a way to barrier this scheme on driver unload. Good point. I can add something like void mmu_interval_notifier_synchronize(struct mm_struct *mm) that waits for deferred operations to complete similar to mmu_interval_read_begin(). >> + } >> svmm->vmm = NULL; >> mutex_unlock(&svmm->mutex); >> - mmu_notifier_put(&svmm->notifier); > > While here it was actually a refcount. > >> +static void nouveau_svmm_do_unmap(struct mmu_interval_notifier *mni, >> +...

...n return the range_notifier may not be present in the interval tree yet. - * The caller must use the normal interval notifier read flow via + * Upon return, the mmu_interval_notifier may not be present in the interval + * tree yet. The caller must use the normal interval notifier read flow via * mmu_interval_read_begin() to establish SPTEs for this range. */ int mmu_interval_notifier_insert(struct mmu_interval_notifier *mni, @@ -969,6 +970,42 @@ int mmu_interval_notifier_insert_locked( } EXPORT_SYMBOL_GPL(mmu_interval_notifier_insert_locked); +/** + * mmu_interval_notifier_insert_safe - Insert an interval...

...freed at any moment. The existing locks do not prevent it. > > > + ret = nouveau_svmm_interval_find(svmm, &range); > > > + if (ret) { > > > + up_read(&mm->mmap_sem); > > > + return ret; > > > + } > > > + range.notifier_seq = mmu_interval_read_begin(range.notifier); > > > ret = hmm_range_fault(&range, 0); > > > up_read(&mm->mmap_sem); > > > if (ret <= 0) { > > > > I'm still not sure this is a better approach than what ODP does. It > > looks very expensive on the fault...

On Mon, Jan 13, 2020 at 02:47:01PM -0800, Ralph Campbell wrote: > diff --git a/mm/mmu_notifier.c b/mm/mmu_notifier.c > index 47ad9cc89aab..4efecc0f13cb 100644 > +++ b/mm/mmu_notifier.c > @@ -1171,6 +1171,39 @@ void mmu_interval_notifier_update(struct mmu_interval_notifier *mni, > } > EXPORT_SYMBOL_GPL(mmu_interval_notifier_update); > > +struct mmu_interval_notifier

From: Jason Gunthorpe <jgg at mellanox.com> 8 of the mmu_notifier using drivers (i915_gem, radeon_mn, umem_odp, hfi1, scif_dma, vhost, gntdev, hmm) drivers are using a common pattern where they only use invalidate_range_start/end and immediately check the invalidating range against some driver data structure to tell if the driver is interested. Half of them use an interval_tree, the others

...default_flags = 0, + .pfn_flags_mask = ~0UL, .pfn_shift = NVIF_VMM_PFNMAP_V0_ADDR_SHIFT, }; - struct mm_struct *mm = notifier->notifier.mm; + struct mm_struct *mm = svmm->mm; long ret; while (true) { if (time_after(jiffies, timeout)) return -EBUSY; - range.notifier_seq = mmu_interval_read_begin(range.notifier); - range.default_flags = 0; - range.pfn_flags_mask = -1UL; down_read(&mm->mmap_sem); + ret = nouveau_svmm_interval_find(svmm, &range); + if (ret) { + up_read(&mm->mmap_sem); + return ret; + } + range.notifier_seq = mmu_interval_read_begin(range.notifie...

...return value of hmm_range_fault() below under the - * mmap_sem to ascertain the validity of the range. - */ - hmm_range_wait_until_valid(&range, TIMEOUT_IN_MSEC); + if (!mmget_not_zero(mni->notifier.mm)) + return -EFAULT; again: + range.notifier_seq = mmu_interval_read_begin(&mni); down_read(&mm->mmap_sem); ret = hmm_range_fault(&range, HMM_RANGE_SNAPSHOT); if (ret) { up_read(&mm->mmap_sem); - if (ret == -EBUSY) { - /* - * No need to check hmm_range_wait_until_valid() return value -...

...eft in the read lock state > until the caller is done with the pointer. > > .. and this poses all sorts of questions about consistency with items > on the deferred list. Should find return an item undergoing deletion? I don't think so. The deferred operations are all complete when mmu_interval_read_begin() returns, and the sequence number check with mmu_interval_read_retry() guarantees there have been no changes while not holding the driver page table lock and calling hmm_range_fault(). > Should find return items using the old interval tree values or their > new updated values? > > Ja...

...ent. The existing locks do not prevent it. > >>>> + ret = nouveau_svmm_interval_find(svmm, &range); >>>> + if (ret) { >>>> + up_read(&mm->mmap_sem); >>>> + return ret; >>>> + } >>>> + range.notifier_seq = mmu_interval_read_begin(range.notifier); >>>> ret = hmm_range_fault(&range, 0); >>>> up_read(&mm->mmap_sem); >>>> if (ret <= 0) { >>> >>> I'm still not sure this is a better approach than what ODP does. It >>> looks very expens...

...mething like gp100_vmm_pgt_pfn() > + * > + * This is all just encoding the internal hmm reprensetation into a "representation" ... > @@ -542,12 +564,15 @@ static int nouveau_range_fault(struct nouveau_svmm *svmm, > return -EBUSY; > > range.notifier_seq = mmu_interval_read_begin(range.notifier); > - range.default_flags = 0; > - range.pfn_flags_mask = -1UL; > down_read(&mm->mmap_sem); > ret = hmm_range_fault(&range); > up_read(&mm->mmap_sem); > if (ret) { > + /* > + * FIXME: the input PFN_REQ flags are destroy...

On 11/13/19 8:46 AM, Jason Gunthorpe wrote: > On Wed, Nov 13, 2019 at 05:59:52AM -0800, Christoph Hellwig wrote: >>> +int mmu_interval_notifier_insert(struct mmu_interval_notifier *mni, >>> + struct mm_struct *mm, unsigned long start, >>> + unsigned long length, >>> + const struct mmu_interval_notifier_ops *ops); >>> +int

...include/linux/hmm.h +++ b/include/linux/hmm.h @@ -145,6 +145,9 @@ enum hmm_pfn_value_e { /* * struct hmm_range - track invalidation lock on virtual address range * + * @notifier: an optional mmu_interval_notifier + * @notifier_seq: when notifier is used this is the result of + * mmu_interval_read_begin() * @hmm: the core HMM structure this range is active against * @vma: the vm area struct for the range * @list: all range lock are on a list @@ -159,6 +162,8 @@ enum hmm_pfn_value_e { * @valid: pfns array did not change since it has been fill by an HMM function */ struct hmm_range { + st...

...the CPU has mapped that physical range with the same permissions so + * that a device MMU mapping of up to the size of the return value can be + * used without giving the device more access than the CPU process. + * + * This must be called under the caller 'user_lock' after a successful + * mmu_interval_read_begin(). The caller must have tested for HMM_PFN_VALID + * already. + */ +static inline unsigned int hmm_pfn_to_map_order(unsigned long hmm_pfn) +{ + return (hmm_pfn >> HMM_PFN_ORDER_SHIFT) & 0x1F; +} + /* * struct hmm_range - track invalidation lock on virtual address range * diff --git a...

...e) to monitor + * + * This function updates the range being monitored and is safe to call from + * the invalidate() callback function. + * Upon return, the mmu_interval_notifier range may not be updated in the + * interval tree yet. The caller must use the normal interval notifier read + * flow via mmu_interval_read_begin() to establish SPTEs for this range. + */ +void mmu_interval_notifier_update(struct mmu_interval_notifier *mni, + unsigned long start, unsigned long last) +{ + struct mm_struct *mm = mni->mm; + struct mmu_notifier_mm *mmn_mm = mm->mmu_notifier_mm; + unsigned long seq = 0; + + if (WARN_ON...

...-260,8 +246,6 @@ static int dmirror_range_fault(struct dmirror *dmirror, int ret; while (true) { - long count; - if (time_after(jiffies, timeout)) { ret = -EBUSY; goto out; @@ -269,12 +253,11 @@ static int dmirror_range_fault(struct dmirror *dmirror, range->notifier_seq = mmu_interval_read_begin(range->notifier); down_read(&mm->mmap_sem); - count = hmm_range_fault(range); + ret = hmm_range_fault(range); up_read(&mm->mmap_sem); - if (count <= 0) { - if (count == 0 || count == -EBUSY) + if (ret) { + if (ret == -EBUSY) continue; - ret = count; goto...

..._PFNMAP_V0_ADDR_SHIFT, + .pfn_flags_mask = HMM_PFN_REQ_FAULT | HMM_PFN_REQ_WRITE, + .hmm_pfns = hmm_pfns, }; struct mm_struct *mm = notifier->notifier.mm; int ret; @@ -542,12 +529,15 @@ static int nouveau_range_fault(struct nouveau_svmm *svmm, return -EBUSY; range.notifier_seq = mmu_interval_read_begin(range.notifier); - range.default_flags = 0; - range.pfn_flags_mask = -1UL; down_read(&mm->mmap_sem); ret = hmm_range_fault(&range); up_read(&mm->mmap_sem); if (ret) { + /* + * FIXME: the input PFN_REQ flags are destroyed on + * -EBUSY, we need to regenerate t...

This series adds new functions to the mmu interval notifier API to allow device drivers with MMUs to dynamically mirror a process' page tables based on device faults and invalidation callbacks. The Nouveau driver is updated to use the extended API and a set of stand alone self tests is added to help validate and maintain correctness. The patches are based on linux-5.5.0-rc6 and are for

..._PFNMAP_V0_ADDR_SHIFT, + .pfn_flags_mask = HMM_PFN_REQ_FAULT | HMM_PFN_REQ_WRITE, + .hmm_pfns = hmm_pfns, }; struct mm_struct *mm = notifier->notifier.mm; int ret; @@ -542,12 +564,15 @@ static int nouveau_range_fault(struct nouveau_svmm *svmm, return -EBUSY; range.notifier_seq = mmu_interval_read_begin(range.notifier); - range.default_flags = 0; - range.pfn_flags_mask = -1UL; down_read(&mm->mmap_sem); ret = hmm_range_fault(&range); up_read(&mm->mmap_sem); if (ret) { + /* + * FIXME: the input PFN_REQ flags are destroyed on + * -EBUSY, we need to regenerate t...