#include <linux/mempolicy.h>
#include <linux/vmalloc.h>
#include <linux/security.h>
+#include <linux/memcontrol.h>
#include "internal.h"
if (!is_migration_entry(entry) || migration_entry_to_page(entry) != old)
goto out;
+ /*
+ * Yes, ignore the return value from a GFP_ATOMIC mem_cgroup_charge.
+ * Failure is not an option here: we're now expected to remove every
+ * migration pte, and will cause crashes otherwise. Normally this
+ * is not an issue: mem_cgroup_prepare_migration bumped up the old
+ * page_cgroup count for safety, that's now attached to the new page,
+ * so this charge should just be another incrementation of the count,
+ * to keep in balance with rmap.c's mem_cgroup_uncharging. But if
+ * there's been a force_empty, those reference counts may no longer
+ * be reliable, and this charge can actually fail: oh well, we don't
+ * make the situation any worse by proceeding as if it had succeeded.
+ */
+ mem_cgroup_charge(new, mm, GFP_ATOMIC);
+
get_page(new);
pte = pte_mkold(mk_pte(new, vma->vm_page_prot));
if (is_write_migration_entry(entry))
if (PageDirty(page)) {
clear_page_dirty_for_io(page);
- set_page_dirty(newpage);
+ /*
+ * Want to mark the page and the radix tree as dirty, and
+ * redo the accounting that clear_page_dirty_for_io undid,
+ * but we can't use set_page_dirty because that function
+ * is actually a signal that all of the page has become dirty.
+ * Wheras only part of our page may be dirty.
+ */
+ __set_page_dirty_nobuffers(newpage);
}
#ifdef CONFIG_SWAP
else
rc = fallback_migrate_page(mapping, newpage, page);
- if (!rc)
+ if (!rc) {
+ mem_cgroup_page_migration(page, newpage);
remove_migration_ptes(page, newpage);
- else
+ } else
newpage->mapping = NULL;
unlock_page(newpage);
int *result = NULL;
struct page *newpage = get_new_page(page, private, &result);
int rcu_locked = 0;
+ int charge = 0;
if (!newpage)
return -ENOMEM;
goto rcu_unlock;
}
+ charge = mem_cgroup_prepare_migration(page);
/* Establish migration ptes or remove ptes */
try_to_unmap(page, 1);
if (!page_mapped(page))
rc = move_to_new_page(newpage, page);
- if (rc)
+ if (rc) {
remove_migration_ptes(page, page);
+ if (charge)
+ mem_cgroup_end_migration(page);
+ } else if (charge)
+ mem_cgroup_end_migration(newpage);
rcu_unlock:
if (rcu_locked)
rcu_read_unlock();