Dave Chinner
2023-Aug-08 02:44 UTC
[PATCH v4 46/48] mm: shrinker: make memcg slab shrink lockless
On Mon, Aug 07, 2023 at 07:09:34PM +0800, Qi Zheng wrote:> Like global slab shrink, this commit also uses refcount+RCU method to make > memcg slab shrink lockless.This patch does random code cleanups amongst the actual RCU changes. Can you please move the cleanups to a spearate patch to reduce the noise in this one?> diff --git a/mm/shrinker.c b/mm/shrinker.c > index d318f5621862..fee6f62904fb 100644 > --- a/mm/shrinker.c > +++ b/mm/shrinker.c > @@ -107,6 +107,12 @@ static struct shrinker_info *shrinker_info_protected(struct mem_cgroup *memcg, > lockdep_is_held(&shrinker_rwsem)); > } > > +static struct shrinker_info *shrinker_info_rcu(struct mem_cgroup *memcg, > + int nid) > +{ > + return rcu_dereference(memcg->nodeinfo[nid]->shrinker_info); > +}This helper doesn't add value. It doesn't tell me that rcu_read_lock() needs to be held when it is called, for one....> static int expand_one_shrinker_info(struct mem_cgroup *memcg, int new_size, > int old_size, int new_nr_max) > { > @@ -198,7 +204,7 @@ void set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id) > struct shrinker_info_unit *unit; > > rcu_read_lock(); > - info = rcu_dereference(memcg->nodeinfo[nid]->shrinker_info); > + info = shrinker_info_rcu(memcg, nid);... whilst the original code here was obviously correct.> unit = info->unit[shriner_id_to_index(shrinker_id)]; > if (!WARN_ON_ONCE(shrinker_id >= info->map_nr_max)) { > /* Pairs with smp mb in shrink_slab() */ > @@ -211,7 +217,7 @@ void set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id) > > static DEFINE_IDR(shrinker_idr); > > -static int prealloc_memcg_shrinker(struct shrinker *shrinker) > +static int shrinker_memcg_alloc(struct shrinker *shrinker)Cleanups in a separate patch.> @@ -253,10 +258,15 @@ static long xchg_nr_deferred_memcg(int nid, struct shrinker *shrinker, > { > struct shrinker_info *info; > struct shrinker_info_unit *unit; > + long nr_deferred; > > - info = shrinker_info_protected(memcg, nid); > + rcu_read_lock(); > + info = shrinker_info_rcu(memcg, nid); > unit = info->unit[shriner_id_to_index(shrinker->id)]; > - return atomic_long_xchg(&unit->nr_deferred[shriner_id_to_offset(shrinker->id)], 0); > + nr_deferred = atomic_long_xchg(&unit->nr_deferred[shriner_id_to_offset(shrinker->id)], 0); > + rcu_read_unlock(); > + > + return nr_deferred; > }This adds two rcu_read_lock() sections to every call to do_shrink_slab(). It's not at all clear ifrom any of the other code that do_shrink_slab() now has internal rcu_read_lock() sections....> @@ -464,18 +480,23 @@ static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid, > if (!mem_cgroup_online(memcg)) > return 0; > > - if (!down_read_trylock(&shrinker_rwsem)) > - return 0; > - > - info = shrinker_info_protected(memcg, nid); > +again: > + rcu_read_lock(); > + info = shrinker_info_rcu(memcg, nid); > if (unlikely(!info)) > goto unlock; > > - for (; index < shriner_id_to_index(info->map_nr_max); index++) { > + if (index < shriner_id_to_index(info->map_nr_max)) { > struct shrinker_info_unit *unit; > > unit = info->unit[index]; > > + /* > + * The shrinker_info_unit will not be freed, so we can > + * safely release the RCU lock here. > + */ > + rcu_read_unlock();Why - what guarantees that the shrinker_info_unit exists at this point? We hold no reference to it, we hold no reference to any shrinker, etc. What provides this existence guarantee?> + > for_each_set_bit(offset, unit->map, SHRINKER_UNIT_BITS) { > struct shrink_control sc = { > .gfp_mask = gfp_mask, > @@ -485,12 +506,14 @@ static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid, > struct shrinker *shrinker; > int shrinker_id = calc_shrinker_id(index, offset); > > + rcu_read_lock(); > shrinker = idr_find(&shrinker_idr, shrinker_id); > - if (unlikely(!shrinker || !(shrinker->flags & SHRINKER_REGISTERED))) { > - if (!shrinker) > - clear_bit(offset, unit->map); > + if (unlikely(!shrinker || !shrinker_try_get(shrinker))) { > + clear_bit(offset, unit->map); > + rcu_read_unlock(); > continue; > } > + rcu_read_unlock(); > > /* Call non-slab shrinkers even though kmem is disabled */ > if (!memcg_kmem_online() && > @@ -523,15 +546,20 @@ static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid, > set_shrinker_bit(memcg, nid, shrinker_id); > } > freed += ret; > - > - if (rwsem_is_contended(&shrinker_rwsem)) { > - freed = freed ? : 1; > - goto unlock; > - } > + shrinker_put(shrinker);Ok, so why is this safe to call without holding the rcu read lock? The global shrinker has to hold the rcu_read_lock() whilst calling shrinker_put() to guarantee the validity of the list next pointer, but we don't hold off RCU here so what guarantees a racing global shrinker walk doesn't trip over this shrinker_put() call dropping the refcount to zero and freeing occuring in a different context...> + /* > + * We have already exited the read-side of rcu critical section > + * before calling do_shrink_slab(), the shrinker_info may be > + * released in expand_one_shrinker_info(), so reacquire the > + * shrinker_info. > + */ > + index++; > + goto again;With that, what makes the use of shrinker_info in xchg_nr_deferred_memcg() in do_shrink_slab() coherent and valid? -Dave. -- Dave Chinner david at fromorbit.com