Hello, This series try to enable migration of non-LRU pages, such as driver's page. My ARM-based platform occured severe fragmentation problem after long-term (several days) test. Sometimes even order-3 page allocation failed. It has memory size 512MB ~ 1024MB. 30% ~ 40% memory is consumed for graphic processing and 20~30 memory is reserved for zram. I found that many pages of GPU driver and zram are non-movable pages. So I reported Minchan Kim, the maintainer of zram, and he made the internal compaction logic of zram. And I made the internal compaction of GPU driver. They reduced some fragmentation but they are not enough effective. They are activated by its own interface, /sys, so they are not cooperative with kernel compaction. If there is too much fragmentation and kernel starts to compaction, zram and GPU driver cannot work with the kernel compaction. The first this patch adds a generic isolate/migrate/putback callbacks for page address-space. The zram and GPU, and any other modules can register its own migration method. The kernel compaction can call the registered migration when it works. Therefore all page in the system can be migrated at once. The 2nd the generic migration callbacks are applied into balloon driver. My gpu driver code is not open so I apply generic migration into balloon to show how it works. I've tested it with qemu enabled by kvm like followings: - turn on Ubuntu 14.04 with 1G memory on qemu. - do kernel building - after several seconds check more than 512MB is used with free command - command "balloon 512" in qemu monitor - check hundreds MB of pages are migrated Next kernel compaction code can call generic migration callbacks instead of balloon driver interface. Finally calling migration of balloon driver is removed. This patch-set is based on v4.1 Gioh Kim (5): mm/compaction: enable driver page migration fs/anon_inode: get a new inode mm/balloon: apply driver page migratable into balloon driver mm/compaction: compaction calls generic migration mm: remove direct calling of migration drivers/virtio/virtio_balloon.c | 4 ++++ fs/anon_inodes.c | 6 ++++++ fs/proc/page.c | 3 +++ include/linux/anon_inodes.h | 1 + include/linux/balloon_compaction.h | 33 +++++++++++++++++++++------------ include/linux/compaction.h | 11 +++++++++++ include/linux/fs.h | 2 ++ include/linux/page-flags.h | 19 +++++++++++++++++++ include/linux/pagemap.h | 27 +++++++++++++++++++++++++++ include/uapi/linux/kernel-page-flags.h | 2 +- mm/balloon_compaction.c | 25 ++++++++++++++++--------- mm/compaction.c | 9 +++++---- mm/migrate.c | 24 ++++++------------------ 13 files changed, 122 insertions(+), 44 deletions(-) -- 1.9.1
Add framework to register callback functions and
check pages migratable.
There are some modes of page isolation so that isolate interface
has an arguments of page address and isolation mode.
Signed-off-by: Gioh Kim <gioh.kim at lge.com>
---
include/linux/compaction.h | 11 +++++++++++
include/linux/fs.h | 2 ++
include/linux/page-flags.h | 19 +++++++++++++++++++
include/linux/pagemap.h | 27 +++++++++++++++++++++++++++
4 files changed, 59 insertions(+)
diff --git a/include/linux/compaction.h b/include/linux/compaction.h
index aa8f61c..4e91a07 100644
--- a/include/linux/compaction.h
+++ b/include/linux/compaction.h
@@ -1,6 +1,9 @@
#ifndef _LINUX_COMPACTION_H
#define _LINUX_COMPACTION_H
+#include <linux/pagemap.h>
+#include <linux/mm.h>
+
/* Return values for compact_zone() and try_to_compact_pages() */
/* compaction didn't start as it was deferred due to past failures */
#define COMPACT_DEFERRED 0
@@ -51,6 +54,10 @@ extern void compaction_defer_reset(struct zone *zone, int
order,
bool alloc_success);
extern bool compaction_restarting(struct zone *zone, int order);
+static inline bool driver_page_migratable(struct page *page)
+{
+ return PageMigratable(page) && mapping_migratable(page->mapping);
+}
#else
static inline unsigned long try_to_compact_pages(gfp_t gfp_mask,
unsigned int order, int alloc_flags,
@@ -83,6 +90,10 @@ static inline bool compaction_deferred(struct zone *zone, int
order)
return true;
}
+static inline bool driver_page_migratable(struct page *page)
+{
+ return false
+}
#endif /* CONFIG_COMPACTION */
#if defined(CONFIG_COMPACTION) && defined(CONFIG_SYSFS) &&
defined(CONFIG_NUMA)
diff --git a/include/linux/fs.h b/include/linux/fs.h
index a0653e5..2cc4b24 100644
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -396,6 +396,8 @@ struct address_space_operations {
*/
int (*migratepage) (struct address_space *,
struct page *, struct page *, enum migrate_mode);
+ bool (*isolatepage) (struct page *, isolate_mode_t);
+ void (*putbackpage) (struct page *);
int (*launder_page) (struct page *);
int (*is_partially_uptodate) (struct page *, unsigned long,
unsigned long);
diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h
index 91b7f9b..c8a66de 100644
--- a/include/linux/page-flags.h
+++ b/include/linux/page-flags.h
@@ -649,6 +649,25 @@ static inline void __ClearPageBalloon(struct page *page)
atomic_set(&page->_mapcount, -1);
}
+#define PAGE_MIGRATABLE_MAPCOUNT_VALUE (-255)
+
+static inline int PageMigratable(struct page *page)
+{
+ return atomic_read(&page->_mapcount) == PAGE_MIGRATABLE_MAPCOUNT_VALUE;
+}
+
+static inline void __SetPageMigratable(struct page *page)
+{
+ VM_BUG_ON_PAGE(atomic_read(&page->_mapcount) != -1, page);
+ atomic_set(&page->_mapcount, PAGE_MIGRATABLE_MAPCOUNT_VALUE);
+}
+
+static inline void __ClearPageMigratable(struct page *page)
+{
+ VM_BUG_ON_PAGE(!PageMigratable(page), page);
+ atomic_set(&page->_mapcount, -1);
+}
+
/*
* If network-based swap is enabled, sl*b must keep track of whether pages
* were allocated from pfmemalloc reserves.
diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h
index 3e95fb6..a306798 100644
--- a/include/linux/pagemap.h
+++ b/include/linux/pagemap.h
@@ -25,8 +25,35 @@ enum mapping_flags {
AS_MM_ALL_LOCKS = __GFP_BITS_SHIFT + 2, /* under mm_take_all_locks() */
AS_UNEVICTABLE = __GFP_BITS_SHIFT + 3, /* e.g., ramdisk, SHM_LOCK */
AS_EXITING = __GFP_BITS_SHIFT + 4, /* final truncate in progress */
+ AS_MIGRATABLE = __GFP_BITS_SHIFT + 5,
};
+static inline void mapping_set_migratable(struct address_space *mapping)
+{
+ set_bit(AS_MIGRATABLE, &mapping->flags);
+}
+
+static inline void mapping_clear_migratable(struct address_space *mapping)
+{
+ clear_bit(AS_MIGRATABLE, &mapping->flags);
+}
+
+static inline int __mapping_ops(struct address_space *mapping)
+{
+ /* migrating page should define all following methods */
+ return mapping->a_ops &&
+ mapping->a_ops->migratepage &&
+ mapping->a_ops->isolatepage &&
+ mapping->a_ops->putbackpage;
+}
+
+static inline int mapping_migratable(struct address_space *mapping)
+{
+ if (mapping && __mapping_ops(mapping))
+ return test_bit(AS_MIGRATABLE, &mapping->flags);
+ return !!mapping;
+}
+
static inline void mapping_set_error(struct address_space *mapping, int error)
{
if (unlikely(error)) {
--
1.9.1
A inode is necessary for some drivers that needs special address_space
and address_space_operation for page migration. Each drivers can create
inode with the anon_inodefs.
Signed-off-by: Gioh Kim <gioh.kim at lge.com>
---
fs/anon_inodes.c | 6 ++++++
include/linux/anon_inodes.h | 1 +
2 files changed, 7 insertions(+)
diff --git a/fs/anon_inodes.c b/fs/anon_inodes.c
index 80ef38c..1d51f96 100644
--- a/fs/anon_inodes.c
+++ b/fs/anon_inodes.c
@@ -162,6 +162,12 @@ err_put_unused_fd:
}
EXPORT_SYMBOL_GPL(anon_inode_getfd);
+struct inode *anon_inode_new(void)
+{
+ return alloc_anon_inode(anon_inode_mnt->mnt_sb);
+}
+EXPORT_SYMBOL_GPL(anon_inode_new);
+
static int __init anon_inode_init(void)
{
anon_inode_mnt = kern_mount(&anon_inode_fs_type);
diff --git a/include/linux/anon_inodes.h b/include/linux/anon_inodes.h
index 8013a45..ddbd67f 100644
--- a/include/linux/anon_inodes.h
+++ b/include/linux/anon_inodes.h
@@ -15,6 +15,7 @@ struct file *anon_inode_getfile(const char *name,
void *priv, int flags);
int anon_inode_getfd(const char *name, const struct file_operations *fops,
void *priv, int flags);
+struct inode *anon_inode_new(void);
#endif /* _LINUX_ANON_INODES_H */
--
1.9.1
Gioh Kim
2015-Jun-26 09:58 UTC
[RFCv2 3/5] mm/balloon: apply driver page migratable into balloon
Apply driver page migration into balloon driver.
Signed-off-by: Gioh Kim <gioh.kim at lge.com>
---
drivers/virtio/virtio_balloon.c | 3 +++
fs/proc/page.c | 3 +++
include/linux/balloon_compaction.h | 33 +++++++++++++++++++++------------
include/uapi/linux/kernel-page-flags.h | 2 +-
mm/balloon_compaction.c | 19 +++++++++++++++++--
5 files changed, 45 insertions(+), 15 deletions(-)
diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c
index 82e80e0..c49b553 100644
--- a/drivers/virtio/virtio_balloon.c
+++ b/drivers/virtio/virtio_balloon.c
@@ -505,6 +505,9 @@ static int virtballoon_probe(struct virtio_device *vdev)
balloon_devinfo_init(&vb->vb_dev_info);
#ifdef CONFIG_BALLOON_COMPACTION
vb->vb_dev_info.migratepage = virtballoon_migratepage;
+ vb->vb_dev_info.inode = anon_inode_new();
+ vb->vb_dev_info.inode->i_mapping->a_ops = &balloon_aops;
+ mapping_set_migratable(vb->vb_dev_info.inode->i_mapping);
#endif
err = init_vqs(vb);
diff --git a/fs/proc/page.c b/fs/proc/page.c
index 7eee2d8..2dc3673 100644
--- a/fs/proc/page.c
+++ b/fs/proc/page.c
@@ -146,6 +146,9 @@ u64 stable_page_flags(struct page *page)
if (PageBalloon(page))
u |= 1 << KPF_BALLOON;
+ if (PageMigratable(page))
+ u |= 1 << KPF_MIGRATABLE;
+
u |= kpf_copy_bit(k, KPF_LOCKED, PG_locked);
u |= kpf_copy_bit(k, KPF_SLAB, PG_slab);
diff --git a/include/linux/balloon_compaction.h
b/include/linux/balloon_compaction.h
index 9b0a15d..e8a3670 100644
--- a/include/linux/balloon_compaction.h
+++ b/include/linux/balloon_compaction.h
@@ -48,6 +48,7 @@
#include <linux/migrate.h>
#include <linux/gfp.h>
#include <linux/err.h>
+#include <linux/fs.h>
/*
* Balloon device information descriptor.
@@ -62,6 +63,7 @@ struct balloon_dev_info {
struct list_head pages; /* Pages enqueued & handled to Host */
int (*migratepage)(struct balloon_dev_info *, struct page *newpage,
struct page *page, enum migrate_mode mode);
+ struct inode *inode;
};
extern struct page *balloon_page_enqueue(struct balloon_dev_info *b_dev_info);
@@ -73,24 +75,28 @@ static inline void balloon_devinfo_init(struct
balloon_dev_info *balloon)
spin_lock_init(&balloon->pages_lock);
INIT_LIST_HEAD(&balloon->pages);
balloon->migratepage = NULL;
+ balloon->inode = NULL;
}
#ifdef CONFIG_BALLOON_COMPACTION
-extern bool balloon_page_isolate(struct page *page);
+extern const struct address_space_operations balloon_aops;
+extern bool balloon_page_isolate(struct page *page,
+ isolate_mode_t mode);
extern void balloon_page_putback(struct page *page);
-extern int balloon_page_migrate(struct page *newpage,
+extern int balloon_page_migrate(struct address_space *mapping,
+ struct page *newpage,
struct page *page, enum migrate_mode mode);
/*
- * __is_movable_balloon_page - helper to perform @page PageBalloon tests
+ * __is_movable_balloon_page - helper to perform @page PageMigratable tests
*/
static inline bool __is_movable_balloon_page(struct page *page)
{
- return PageBalloon(page);
+ return PageMigratable(page);
}
/*
- * balloon_page_movable - test PageBalloon to identify balloon pages
+ * balloon_page_movable - test PageMigratable to identify balloon pages
* and PagePrivate to check that the page is not
* isolated and can be moved by compaction/migration.
*
@@ -99,7 +105,7 @@ static inline bool __is_movable_balloon_page(struct page
*page)
*/
static inline bool balloon_page_movable(struct page *page)
{
- return PageBalloon(page) && PagePrivate(page);
+ return PageMigratable(page) && PagePrivate(page);
}
/*
@@ -108,7 +114,7 @@ static inline bool balloon_page_movable(struct page *page)
*/
static inline bool isolated_balloon_page(struct page *page)
{
- return PageBalloon(page);
+ return PageMigratable(page);
}
/*
@@ -123,7 +129,8 @@ static inline bool isolated_balloon_page(struct page *page)
static inline void balloon_page_insert(struct balloon_dev_info *balloon,
struct page *page)
{
- __SetPageBalloon(page);
+ page->mapping = balloon->inode->i_mapping;
+ __SetPageMigratable(page);
SetPagePrivate(page);
set_page_private(page, (unsigned long)balloon);
list_add(&page->lru, &balloon->pages);
@@ -139,7 +146,8 @@ static inline void balloon_page_insert(struct
balloon_dev_info *balloon,
*/
static inline void balloon_page_delete(struct page *page)
{
- __ClearPageBalloon(page);
+ page->mapping = NULL;
+ __ClearPageMigratable(page);
set_page_private(page, 0);
if (PagePrivate(page)) {
ClearPagePrivate(page);
@@ -166,13 +174,13 @@ static inline gfp_t balloon_mapping_gfp_mask(void)
static inline void balloon_page_insert(struct balloon_dev_info *balloon,
struct page *page)
{
- __SetPageBalloon(page);
+ __SetPageMigratable(page);
list_add(&page->lru, &balloon->pages);
}
static inline void balloon_page_delete(struct page *page)
{
- __ClearPageBalloon(page);
+ __ClearPageMigratable(page);
list_del(&page->lru);
}
@@ -191,7 +199,8 @@ static inline bool isolated_balloon_page(struct page *page)
return false;
}
-static inline bool balloon_page_isolate(struct page *page)
+static inline bool balloon_page_isolate(struct page *page,
+ isolate_mode_t mode)
{
return false;
}
diff --git a/include/uapi/linux/kernel-page-flags.h
b/include/uapi/linux/kernel-page-flags.h
index a6c4962..65db3a6 100644
--- a/include/uapi/linux/kernel-page-flags.h
+++ b/include/uapi/linux/kernel-page-flags.h
@@ -33,6 +33,6 @@
#define KPF_THP 22
#define KPF_BALLOON 23
#define KPF_ZERO_PAGE 24
-
+#define KPF_MIGRATABLE 25
#endif /* _UAPILINUX_KERNEL_PAGE_FLAGS_H */
diff --git a/mm/balloon_compaction.c b/mm/balloon_compaction.c
index fcad832..df72846 100644
--- a/mm/balloon_compaction.c
+++ b/mm/balloon_compaction.c
@@ -131,7 +131,7 @@ static inline void __putback_balloon_page(struct page *page)
}
/* __isolate_lru_page() counterpart for a ballooned page */
-bool balloon_page_isolate(struct page *page)
+bool balloon_page_isolate(struct page *page, isolate_mode_t mode)
{
/*
* Avoid burning cycles with pages that are yet under __free_pages(),
@@ -175,6 +175,9 @@ bool balloon_page_isolate(struct page *page)
/* putback_lru_page() counterpart for a ballooned page */
void balloon_page_putback(struct page *page)
{
+ if (!isolated_balloon_page(page))
+ return;
+
/*
* 'lock_page()' stabilizes the page and prevents races against
* concurrent isolation threads attempting to re-isolate it.
@@ -193,12 +196,16 @@ void balloon_page_putback(struct page *page)
}
/* move_to_new_page() counterpart for a ballooned page */
-int balloon_page_migrate(struct page *newpage,
+int balloon_page_migrate(struct address_space *mapping,
+ struct page *newpage,
struct page *page, enum migrate_mode mode)
{
struct balloon_dev_info *balloon = balloon_page_device(page);
int rc = -EAGAIN;
+ if (!isolated_balloon_page(page))
+ return rc;
+
/*
* Block others from accessing the 'newpage' when we get around to
* establishing additional references. We should be the only one
@@ -218,4 +225,12 @@ int balloon_page_migrate(struct page *newpage,
unlock_page(newpage);
return rc;
}
+
+/* define the balloon_mapping->a_ops callback to allow balloon page
migration */
+const struct address_space_operations balloon_aops = {
+ .migratepage = balloon_page_migrate,
+ .isolatepage = balloon_page_isolate,
+ .putbackpage = balloon_page_putback,
+};
+EXPORT_SYMBOL_GPL(balloon_aops);
#endif /* CONFIG_BALLOON_COMPACTION */
--
1.9.1
Gioh Kim
2015-Jun-26 09:58 UTC
[RFCv2 4/5] mm/compaction: compaction calls generic migration
Compaction calls interfaces of driver page migration
instead of calling balloon migration directly.
Signed-off-by: Gioh Kim <gioh.kim at lge.com>
---
drivers/virtio/virtio_balloon.c | 1 +
mm/compaction.c | 9 +++++----
mm/migrate.c | 21 ++++++++++++---------
3 files changed, 18 insertions(+), 13 deletions(-)
diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c
index c49b553..5e5cbea 100644
--- a/drivers/virtio/virtio_balloon.c
+++ b/drivers/virtio/virtio_balloon.c
@@ -30,6 +30,7 @@
#include <linux/balloon_compaction.h>
#include <linux/oom.h>
#include <linux/wait.h>
+#include <linux/anon_inodes.h>
/*
* Balloon device works in 4K page units. So each page is pointed to by
diff --git a/mm/compaction.c b/mm/compaction.c
index 16e1b57..cc5ec81 100644
--- a/mm/compaction.c
+++ b/mm/compaction.c
@@ -14,7 +14,7 @@
#include <linux/backing-dev.h>
#include <linux/sysctl.h>
#include <linux/sysfs.h>
-#include <linux/balloon_compaction.h>
+#include <linux/compaction.h>
#include <linux/page-isolation.h>
#include <linux/kasan.h>
#include "internal.h"
@@ -714,12 +714,13 @@ isolate_migratepages_block(struct compact_control *cc,
unsigned long low_pfn,
/*
* Check may be lockless but that's ok as we recheck later.
- * It's possible to migrate LRU pages and balloon pages
+ * It's possible to migrate LRU pages and driver pages
* Skip any other type of page
*/
if (!PageLRU(page)) {
- if (unlikely(balloon_page_movable(page))) {
- if (balloon_page_isolate(page)) {
+ if (unlikely(driver_page_migratable(page))) {
+ if (page->mapping->a_ops->isolatepage(page,
+ isolate_mode)) {
/* Successfully isolated */
goto isolate_success;
}
diff --git a/mm/migrate.c b/mm/migrate.c
index 236ee25..a0bc1e4 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -35,7 +35,7 @@
#include <linux/hugetlb.h>
#include <linux/hugetlb_cgroup.h>
#include <linux/gfp.h>
-#include <linux/balloon_compaction.h>
+#include <linux/compaction.h>
#include <linux/mmu_notifier.h>
#include <asm/tlbflush.h>
@@ -76,7 +76,7 @@ int migrate_prep_local(void)
* from where they were once taken off for compaction/migration.
*
* This function shall be used whenever the isolated pageset has been
- * built from lru, balloon, hugetlbfs page. See isolate_migratepages_range()
+ * built from lru, driver, hugetlbfs page. See isolate_migratepages_range()
* and isolate_huge_page().
*/
void putback_movable_pages(struct list_head *l)
@@ -92,8 +92,8 @@ void putback_movable_pages(struct list_head *l)
list_del(&page->lru);
dec_zone_page_state(page, NR_ISOLATED_ANON +
page_is_file_cache(page));
- if (unlikely(isolated_balloon_page(page)))
- balloon_page_putback(page);
+ if (unlikely(driver_page_migratable(page)))
+ page->mapping->a_ops->putbackpage(page);
else
putback_lru_page(page);
}
@@ -844,15 +844,18 @@ static int __unmap_and_move(struct page *page, struct page
*newpage,
}
}
- if (unlikely(isolated_balloon_page(page))) {
+ if (unlikely(driver_page_migratable(page))) {
/*
- * A ballooned page does not need any special attention from
+ * A driver page does not need any special attention from
* physical to virtual reverse mapping procedures.
* Skip any attempt to unmap PTEs or to remap swap cache,
* in order to avoid burning cycles at rmap level, and perform
* the page migration right away (proteced by page lock).
*/
- rc = balloon_page_migrate(newpage, page, mode);
+ rc = page->mapping->a_ops->migratepage(page->mapping,
+ newpage,
+ page,
+ mode);
goto out_unlock;
}
@@ -962,8 +965,8 @@ out:
if (rc != MIGRATEPAGE_SUCCESS && put_new_page) {
ClearPageSwapBacked(newpage);
put_new_page(newpage, private);
- } else if (unlikely(__is_movable_balloon_page(newpage))) {
- /* drop our reference, page already in the balloon */
+ } else if (unlikely(driver_page_migratable(newpage))) {
+ /* drop our reference */
put_page(newpage);
} else
putback_lru_page(newpage);
--
1.9.1
Migration is completely generalized.
Signed-off-by: Gioh Kim <gioh.kim at lge.com>
---
mm/balloon_compaction.c | 8 --------
mm/migrate.c | 15 ---------------
2 files changed, 23 deletions(-)
diff --git a/mm/balloon_compaction.c b/mm/balloon_compaction.c
index df72846..a7b7c79 100644
--- a/mm/balloon_compaction.c
+++ b/mm/balloon_compaction.c
@@ -206,13 +206,6 @@ int balloon_page_migrate(struct address_space *mapping,
if (!isolated_balloon_page(page))
return rc;
- /*
- * Block others from accessing the 'newpage' when we get around to
- * establishing additional references. We should be the only one
- * holding a reference to the 'newpage' at this point.
- */
- BUG_ON(!trylock_page(newpage));
-
if (WARN_ON(!__is_movable_balloon_page(page))) {
dump_page(page, "not movable balloon page");
unlock_page(newpage);
@@ -222,7 +215,6 @@ int balloon_page_migrate(struct address_space *mapping,
if (balloon && balloon->migratepage)
rc = balloon->migratepage(balloon, newpage, page, mode);
- unlock_page(newpage);
return rc;
}
diff --git a/mm/migrate.c b/mm/migrate.c
index a0bc1e4..0b52fa4 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -844,21 +844,6 @@ static int __unmap_and_move(struct page *page, struct page
*newpage,
}
}
- if (unlikely(driver_page_migratable(page))) {
- /*
- * A driver page does not need any special attention from
- * physical to virtual reverse mapping procedures.
- * Skip any attempt to unmap PTEs or to remap swap cache,
- * in order to avoid burning cycles at rmap level, and perform
- * the page migration right away (proteced by page lock).
- */
- rc = page->mapping->a_ops->migratepage(page->mapping,
- newpage,
- page,
- mode);
- goto out_unlock;
- }
-
/*
* Corner case handling:
* 1. When a new swap-cache page is read into, it is added to the LRU
--
1.9.1
Konstantin Khlebnikov
2015-Jul-04 18:13 UTC
[RFCv2 4/5] mm/compaction: compaction calls generic migration
On Fri, Jun 26, 2015 at 12:58 PM, Gioh Kim <gioh.kim at lge.com> wrote:> Compaction calls interfaces of driver page migration > instead of calling balloon migration directly. > > Signed-off-by: Gioh Kim <gioh.kim at lge.com> > --- > drivers/virtio/virtio_balloon.c | 1 + > mm/compaction.c | 9 +++++---- > mm/migrate.c | 21 ++++++++++++--------- > 3 files changed, 18 insertions(+), 13 deletions(-) > > diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c > index c49b553..5e5cbea 100644 > --- a/drivers/virtio/virtio_balloon.c > +++ b/drivers/virtio/virtio_balloon.c > @@ -30,6 +30,7 @@ > #include <linux/balloon_compaction.h> > #include <linux/oom.h> > #include <linux/wait.h> > +#include <linux/anon_inodes.h> > > /* > * Balloon device works in 4K page units. So each page is pointed to by > diff --git a/mm/compaction.c b/mm/compaction.c > index 16e1b57..cc5ec81 100644 > --- a/mm/compaction.c > +++ b/mm/compaction.c > @@ -14,7 +14,7 @@ > #include <linux/backing-dev.h> > #include <linux/sysctl.h> > #include <linux/sysfs.h> > -#include <linux/balloon_compaction.h> > +#include <linux/compaction.h> > #include <linux/page-isolation.h> > #include <linux/kasan.h> > #include "internal.h" > @@ -714,12 +714,13 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, > > /* > * Check may be lockless but that's ok as we recheck later. > - * It's possible to migrate LRU pages and balloon pages > + * It's possible to migrate LRU pages and driver pages > * Skip any other type of page > */ > if (!PageLRU(page)) { > - if (unlikely(balloon_page_movable(page))) { > - if (balloon_page_isolate(page)) { > + if (unlikely(driver_page_migratable(page))) { > + if (page->mapping->a_ops->isolatepage(page, > + isolate_mode)) {Dereferencing page->mapping isn't safe here. Page could be "truncated" from mapping at any time. As you can see balloon_page_isolate() calls get_page_unless_zero, trylock_page and only after that checks balloon_page_movable again. Existing code already does similar unsafe dereference in __isolate_lru_page(): page->mapping->a_ops->migratepage> /* Successfully isolated */ > goto isolate_success; > } > diff --git a/mm/migrate.c b/mm/migrate.c > index 236ee25..a0bc1e4 100644 > --- a/mm/migrate.c > +++ b/mm/migrate.c > @@ -35,7 +35,7 @@ > #include <linux/hugetlb.h> > #include <linux/hugetlb_cgroup.h> > #include <linux/gfp.h> > -#include <linux/balloon_compaction.h> > +#include <linux/compaction.h> > #include <linux/mmu_notifier.h> > > #include <asm/tlbflush.h> > @@ -76,7 +76,7 @@ int migrate_prep_local(void) > * from where they were once taken off for compaction/migration. > * > * This function shall be used whenever the isolated pageset has been > - * built from lru, balloon, hugetlbfs page. See isolate_migratepages_range() > + * built from lru, driver, hugetlbfs page. See isolate_migratepages_range() > * and isolate_huge_page(). > */ > void putback_movable_pages(struct list_head *l) > @@ -92,8 +92,8 @@ void putback_movable_pages(struct list_head *l) > list_del(&page->lru); > dec_zone_page_state(page, NR_ISOLATED_ANON + > page_is_file_cache(page)); > - if (unlikely(isolated_balloon_page(page))) > - balloon_page_putback(page); > + if (unlikely(driver_page_migratable(page))) > + page->mapping->a_ops->putbackpage(page); > else > putback_lru_page(page); > } > @@ -844,15 +844,18 @@ static int __unmap_and_move(struct page *page, struct page *newpage, > } > } > > - if (unlikely(isolated_balloon_page(page))) { > + if (unlikely(driver_page_migratable(page))) { > /* > - * A ballooned page does not need any special attention from > + * A driver page does not need any special attention from > * physical to virtual reverse mapping procedures. > * Skip any attempt to unmap PTEs or to remap swap cache, > * in order to avoid burning cycles at rmap level, and perform > * the page migration right away (proteced by page lock). > */ > - rc = balloon_page_migrate(newpage, page, mode); > + rc = page->mapping->a_ops->migratepage(page->mapping, > + newpage, > + page, > + mode); > goto out_unlock; > } > > @@ -962,8 +965,8 @@ out: > if (rc != MIGRATEPAGE_SUCCESS && put_new_page) { > ClearPageSwapBacked(newpage); > put_new_page(newpage, private); > - } else if (unlikely(__is_movable_balloon_page(newpage))) { > - /* drop our reference, page already in the balloon */ > + } else if (unlikely(driver_page_migratable(newpage))) { > + /* drop our reference */ > put_page(newpage); > } else > putback_lru_page(newpage); > -- > 1.9.1 >
Konstantin Khlebnikov
2015-Jul-04 18:49 UTC
[RFCv2 1/5] mm/compaction: enable driver page migration
On Fri, Jun 26, 2015 at 12:58 PM, Gioh Kim <gioh.kim at lge.com> wrote:> Add framework to register callback functions and > check pages migratable. > There are some modes of page isolation so that isolate interface > has an arguments of page address and isolation mode. > > Signed-off-by: Gioh Kim <gioh.kim at lge.com> > --- > include/linux/compaction.h | 11 +++++++++++ > include/linux/fs.h | 2 ++ > include/linux/page-flags.h | 19 +++++++++++++++++++ > include/linux/pagemap.h | 27 +++++++++++++++++++++++++++ > 4 files changed, 59 insertions(+) > > diff --git a/include/linux/compaction.h b/include/linux/compaction.h > index aa8f61c..4e91a07 100644 > --- a/include/linux/compaction.h > +++ b/include/linux/compaction.h > @@ -1,6 +1,9 @@ > #ifndef _LINUX_COMPACTION_H > #define _LINUX_COMPACTION_H > > +#include <linux/pagemap.h> > +#include <linux/mm.h> > + > /* Return values for compact_zone() and try_to_compact_pages() */ > /* compaction didn't start as it was deferred due to past failures */ > #define COMPACT_DEFERRED 0 > @@ -51,6 +54,10 @@ extern void compaction_defer_reset(struct zone *zone, int order, > bool alloc_success); > extern bool compaction_restarting(struct zone *zone, int order); > > +static inline bool driver_page_migratable(struct page *page) > +{ > + return PageMigratable(page) && mapping_migratable(page->mapping); > +} > #else > static inline unsigned long try_to_compact_pages(gfp_t gfp_mask, > unsigned int order, int alloc_flags, > @@ -83,6 +90,10 @@ static inline bool compaction_deferred(struct zone *zone, int order) > return true; > } > > +static inline bool driver_page_migratable(struct page *page) > +{ > + return false > +} > #endif /* CONFIG_COMPACTION */ > > #if defined(CONFIG_COMPACTION) && defined(CONFIG_SYSFS) && defined(CONFIG_NUMA) > diff --git a/include/linux/fs.h b/include/linux/fs.h > index a0653e5..2cc4b24 100644 > --- a/include/linux/fs.h > +++ b/include/linux/fs.h > @@ -396,6 +396,8 @@ struct address_space_operations { > */ > int (*migratepage) (struct address_space *, > struct page *, struct page *, enum migrate_mode); > + bool (*isolatepage) (struct page *, isolate_mode_t); > + void (*putbackpage) (struct page *); > int (*launder_page) (struct page *); > int (*is_partially_uptodate) (struct page *, unsigned long, > unsigned long); > diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h > index 91b7f9b..c8a66de 100644 > --- a/include/linux/page-flags.h > +++ b/include/linux/page-flags.h > @@ -649,6 +649,25 @@ static inline void __ClearPageBalloon(struct page *page) > atomic_set(&page->_mapcount, -1); > } > > +#define PAGE_MIGRATABLE_MAPCOUNT_VALUE (-255) > + > +static inline int PageMigratable(struct page *page) > +{ > + return atomic_read(&page->_mapcount) == PAGE_MIGRATABLE_MAPCOUNT_VALUE; > +}I don't like the name. It's boring and overused. Let's call it "mobile" PageMobile() That will be fun.> + > +static inline void __SetPageMigratable(struct page *page) > +{ > + VM_BUG_ON_PAGE(atomic_read(&page->_mapcount) != -1, page); > + atomic_set(&page->_mapcount, PAGE_MIGRATABLE_MAPCOUNT_VALUE); > +} > + > +static inline void __ClearPageMigratable(struct page *page) > +{ > + VM_BUG_ON_PAGE(!PageMigratable(page), page); > + atomic_set(&page->_mapcount, -1); > +} > + > /* > * If network-based swap is enabled, sl*b must keep track of whether pages > * were allocated from pfmemalloc reserves. > diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h > index 3e95fb6..a306798 100644 > --- a/include/linux/pagemap.h > +++ b/include/linux/pagemap.h > @@ -25,8 +25,35 @@ enum mapping_flags { > AS_MM_ALL_LOCKS = __GFP_BITS_SHIFT + 2, /* under mm_take_all_locks() */ > AS_UNEVICTABLE = __GFP_BITS_SHIFT + 3, /* e.g., ramdisk, SHM_LOCK */ > AS_EXITING = __GFP_BITS_SHIFT + 4, /* final truncate in progress */ > + AS_MIGRATABLE = __GFP_BITS_SHIFT + 5,I think this is redudant. Mark at page should be enough. That inode should just provide way for calling migration methods, that's all.> }; > > +static inline void mapping_set_migratable(struct address_space *mapping) > +{ > + set_bit(AS_MIGRATABLE, &mapping->flags); > +} > + > +static inline void mapping_clear_migratable(struct address_space *mapping) > +{ > + clear_bit(AS_MIGRATABLE, &mapping->flags); > +} > + > +static inline int __mapping_ops(struct address_space *mapping) > +{ > + /* migrating page should define all following methods */ > + return mapping->a_ops && > + mapping->a_ops->migratepage && > + mapping->a_ops->isolatepage && > + mapping->a_ops->putbackpage; > +}This is sanity check or debug? You already have mark right at page. You could check them with VM_BUG_ON in some place.> + > +static inline int mapping_migratable(struct address_space *mapping) > +{ > + if (mapping && __mapping_ops(mapping)) > + return test_bit(AS_MIGRATABLE, &mapping->flags); > + return !!mapping; > +} > + > static inline void mapping_set_error(struct address_space *mapping, int error) > { > if (unlikely(error)) { > -- > 1.9.1 >
Konstantin Khlebnikov
2015-Jul-04 18:55 UTC
[RFCv2 3/5] mm/balloon: apply driver page migratable into balloon
On Fri, Jun 26, 2015 at 12:58 PM, Gioh Kim <gioh.kim at lge.com> wrote:> Apply driver page migration into balloon driver. > > Signed-off-by: Gioh Kim <gioh.kim at lge.com> > --- > drivers/virtio/virtio_balloon.c | 3 +++ > fs/proc/page.c | 3 +++ > include/linux/balloon_compaction.h | 33 +++++++++++++++++++++------------ > include/uapi/linux/kernel-page-flags.h | 2 +- > mm/balloon_compaction.c | 19 +++++++++++++++++-- > 5 files changed, 45 insertions(+), 15 deletions(-) > > diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c > index 82e80e0..c49b553 100644 > --- a/drivers/virtio/virtio_balloon.c > +++ b/drivers/virtio/virtio_balloon.c > @@ -505,6 +505,9 @@ static int virtballoon_probe(struct virtio_device *vdev) > balloon_devinfo_init(&vb->vb_dev_info); > #ifdef CONFIG_BALLOON_COMPACTION > vb->vb_dev_info.migratepage = virtballoon_migratepage; > + vb->vb_dev_info.inode = anon_inode_new(); > + vb->vb_dev_info.inode->i_mapping->a_ops = &balloon_aops; > + mapping_set_migratable(vb->vb_dev_info.inode->i_mapping); > #endif > > err = init_vqs(vb); > diff --git a/fs/proc/page.c b/fs/proc/page.c > index 7eee2d8..2dc3673 100644 > --- a/fs/proc/page.c > +++ b/fs/proc/page.c > @@ -146,6 +146,9 @@ u64 stable_page_flags(struct page *page) > if (PageBalloon(page)) > u |= 1 << KPF_BALLOON; > > + if (PageMigratable(page)) > + u |= 1 << KPF_MIGRATABLE; > + > u |= kpf_copy_bit(k, KPF_LOCKED, PG_locked); > > u |= kpf_copy_bit(k, KPF_SLAB, PG_slab); > diff --git a/include/linux/balloon_compaction.h b/include/linux/balloon_compaction.h > index 9b0a15d..e8a3670 100644 > --- a/include/linux/balloon_compaction.h > +++ b/include/linux/balloon_compaction.h > @@ -48,6 +48,7 @@ > #include <linux/migrate.h> > #include <linux/gfp.h> > #include <linux/err.h> > +#include <linux/fs.h> > > /* > * Balloon device information descriptor. > @@ -62,6 +63,7 @@ struct balloon_dev_info { > struct list_head pages; /* Pages enqueued & handled to Host */ > int (*migratepage)(struct balloon_dev_info *, struct page *newpage, > struct page *page, enum migrate_mode mode); > + struct inode *inode; > }; > > extern struct page *balloon_page_enqueue(struct balloon_dev_info *b_dev_info); > @@ -73,24 +75,28 @@ static inline void balloon_devinfo_init(struct balloon_dev_info *balloon) > spin_lock_init(&balloon->pages_lock); > INIT_LIST_HEAD(&balloon->pages); > balloon->migratepage = NULL; > + balloon->inode = NULL; > } > > #ifdef CONFIG_BALLOON_COMPACTION > -extern bool balloon_page_isolate(struct page *page); > +extern const struct address_space_operations balloon_aops; > +extern bool balloon_page_isolate(struct page *page, > + isolate_mode_t mode); > extern void balloon_page_putback(struct page *page); > -extern int balloon_page_migrate(struct page *newpage, > +extern int balloon_page_migrate(struct address_space *mapping, > + struct page *newpage, > struct page *page, enum migrate_mode mode); > > /* > - * __is_movable_balloon_page - helper to perform @page PageBalloon tests > + * __is_movable_balloon_page - helper to perform @page PageMigratable tests > */ > static inline bool __is_movable_balloon_page(struct page *page) > { > - return PageBalloon(page); > + return PageMigratable(page); > } > > /* > - * balloon_page_movable - test PageBalloon to identify balloon pages > + * balloon_page_movable - test PageMigratable to identify balloon pages > * and PagePrivate to check that the page is not > * isolated and can be moved by compaction/migration. > * > @@ -99,7 +105,7 @@ static inline bool __is_movable_balloon_page(struct page *page) > */ > static inline bool balloon_page_movable(struct page *page) > { > - return PageBalloon(page) && PagePrivate(page); > + return PageMigratable(page) && PagePrivate(page); > } > > /* > @@ -108,7 +114,7 @@ static inline bool balloon_page_movable(struct page *page) > */ > static inline bool isolated_balloon_page(struct page *page) > { > - return PageBalloon(page); > + return PageMigratable(page); > } > > /* > @@ -123,7 +129,8 @@ static inline bool isolated_balloon_page(struct page *page) > static inline void balloon_page_insert(struct balloon_dev_info *balloon, > struct page *page) > { > - __SetPageBalloon(page); > + page->mapping = balloon->inode->i_mapping; > + __SetPageMigratable(page); > SetPagePrivate(page); > set_page_private(page, (unsigned long)balloon); > list_add(&page->lru, &balloon->pages); > @@ -139,7 +146,8 @@ static inline void balloon_page_insert(struct balloon_dev_info *balloon, > */ > static inline void balloon_page_delete(struct page *page) > { > - __ClearPageBalloon(page); > + page->mapping = NULL; > + __ClearPageMigratable(page);Please leave balloon pages marked as balloon. Just check them in migration code like: if (PageBalloon(page) || PageMobile(page)) ...> set_page_private(page, 0); > if (PagePrivate(page)) { > ClearPagePrivate(page); > @@ -166,13 +174,13 @@ static inline gfp_t balloon_mapping_gfp_mask(void) > static inline void balloon_page_insert(struct balloon_dev_info *balloon, > struct page *page) > { > - __SetPageBalloon(page); > + __SetPageMigratable(page); > list_add(&page->lru, &balloon->pages); > } > > static inline void balloon_page_delete(struct page *page) > { > - __ClearPageBalloon(page); > + __ClearPageMigratable(page); > list_del(&page->lru); > } > > @@ -191,7 +199,8 @@ static inline bool isolated_balloon_page(struct page *page) > return false; > } > > -static inline bool balloon_page_isolate(struct page *page) > +static inline bool balloon_page_isolate(struct page *page, > + isolate_mode_t mode) > { > return false; > } > diff --git a/include/uapi/linux/kernel-page-flags.h b/include/uapi/linux/kernel-page-flags.h > index a6c4962..65db3a6 100644 > --- a/include/uapi/linux/kernel-page-flags.h > +++ b/include/uapi/linux/kernel-page-flags.h > @@ -33,6 +33,6 @@ > #define KPF_THP 22 > #define KPF_BALLOON 23 > #define KPF_ZERO_PAGE 24 > - > +#define KPF_MIGRATABLE 25 > > #endif /* _UAPILINUX_KERNEL_PAGE_FLAGS_H */ > diff --git a/mm/balloon_compaction.c b/mm/balloon_compaction.c > index fcad832..df72846 100644 > --- a/mm/balloon_compaction.c > +++ b/mm/balloon_compaction.c > @@ -131,7 +131,7 @@ static inline void __putback_balloon_page(struct page *page) > } > > /* __isolate_lru_page() counterpart for a ballooned page */ > -bool balloon_page_isolate(struct page *page) > +bool balloon_page_isolate(struct page *page, isolate_mode_t mode) > { > /* > * Avoid burning cycles with pages that are yet under __free_pages(), > @@ -175,6 +175,9 @@ bool balloon_page_isolate(struct page *page) > /* putback_lru_page() counterpart for a ballooned page */ > void balloon_page_putback(struct page *page) > { > + if (!isolated_balloon_page(page)) > + return; > + > /* > * 'lock_page()' stabilizes the page and prevents races against > * concurrent isolation threads attempting to re-isolate it. > @@ -193,12 +196,16 @@ void balloon_page_putback(struct page *page) > } > > /* move_to_new_page() counterpart for a ballooned page */ > -int balloon_page_migrate(struct page *newpage, > +int balloon_page_migrate(struct address_space *mapping, > + struct page *newpage, > struct page *page, enum migrate_mode mode) > { > struct balloon_dev_info *balloon = balloon_page_device(page); > int rc = -EAGAIN; > > + if (!isolated_balloon_page(page)) > + return rc; > + > /* > * Block others from accessing the 'newpage' when we get around to > * establishing additional references. We should be the only one > @@ -218,4 +225,12 @@ int balloon_page_migrate(struct page *newpage, > unlock_page(newpage); > return rc; > } > + > +/* define the balloon_mapping->a_ops callback to allow balloon page migration */ > +const struct address_space_operations balloon_aops = { > + .migratepage = balloon_page_migrate, > + .isolatepage = balloon_page_isolate, > + .putbackpage = balloon_page_putback, > +}; > +EXPORT_SYMBOL_GPL(balloon_aops); > #endif /* CONFIG_BALLOON_COMPACTION */ > -- > 1.9.1 >