GitHub-Mirror/linux-bl808
2
0
Fork 0
mirror of https://github.com/Fishwaldo/linux-bl808.git synced 2025-06-11 09:06:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

mm: hugetlbfs: move the put/get_page slab and hugetlbfs optimization in a faster path

Browse source 
We don't actually need a reference on the head page in the slab and
hugetlbfs paths, as long as we add a smp_rmb() which should be faster
than get_page_unless_zero.

[akpm@linux-foundation.org: fix typo in comment]
Signed-off-by: Andrea Arcangeli <aarcange@redhat.com>
Cc: Khalid Aziz <khalid.aziz@oracle.com>
Cc: Pravin Shelar <pshelar@nicira.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Ben Hutchings <bhutchings@solarflare.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Johannes Weiner <jweiner@redhat.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <andi@firstfloor.org>
Cc: Minchan Kim <minchan@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
This commit is contained in:
Andrea Arcangeli 2014-01-21 15:48:51 -08:00 • committed by Linus Torvalds
parent a0368d4e48
commit ebf360f9bb
1 changed files with 78 additions and 62 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

140
mm/swap.c
View file

@ -86,45 +86,61 @@ static void put_compound_page(struct page *page)
/* __split_huge_page_refcount can run under us */ /* __split_huge_page_refcount can run under us */
struct page *page_head = compound_trans_head(page); struct page *page_head = compound_trans_head(page);
/*
* THP can not break up slab pages so avoid taking
* compound_lock(). Slab performs non-atomic bit ops
* on page->flags for better performance. In
* particular slab_unlock() in slub used to be a hot
* path. It is still hot on arches that do not support
* this_cpu_cmpxchg_double().
*
* If "page" is part of a slab or hugetlbfs page it
* cannot be splitted and the head page cannot change
* from under us. And if "page" is part of a THP page
* under splitting, if the head page pointed by the
* THP tail isn't a THP head anymore, we'll find
* PageTail clear after smp_rmb() and we'll treat it
* as a single page.
*/
if (PageSlab(page_head) || PageHeadHuge(page_head)) {
/*
* If "page" is a THP tail, we must read the tail page
* flags after the head page flags. The
* split_huge_page side enforces write memory
* barriers between clearing PageTail and before the
* head page can be freed and reallocated.
*/
smp_rmb();
if (likely(PageTail(page))) {
/*
* __split_huge_page_refcount
* cannot race here.
*/
VM_BUG_ON(!PageHead(page_head));
VM_BUG_ON(page_mapcount(page) <= 0);
atomic_dec(&page->_mapcount);
if (put_page_testzero(page_head))
__put_compound_page(page_head);
return;
} else
/*
* __split_huge_page_refcount
* run before us, "page" was a
* THP tail. The split
* page_head has been freed
* and reallocated as slab or
* hugetlbfs page of smaller
* order (only possible if
* reallocated as slab on
* x86).
*/
goto out_put_single;
}
if (likely(page != page_head && if (likely(page != page_head &&
get_page_unless_zero(page_head))) { get_page_unless_zero(page_head))) {
unsigned long flags; unsigned long flags;
/*
* THP can not break up slab pages so avoid taking
* compound_lock(). Slab performs non-atomic bit ops
* on page->flags for better performance. In particular
* slab_unlock() in slub used to be a hot path. It is
* still hot on arches that do not support
* this_cpu_cmpxchg_double().
*/
if (PageSlab(page_head) || PageHeadHuge(page_head)) {
if (likely(PageTail(page))) {
/*
* __split_huge_page_refcount
* cannot race here.
*/
VM_BUG_ON(!PageHead(page_head));
atomic_dec(&page->_mapcount);
if (put_page_testzero(page_head))
VM_BUG_ON(1);
if (put_page_testzero(page_head))
__put_compound_page(page_head);
return;
} else
/*
* __split_huge_page_refcount
* run before us, "page" was a
* THP tail. The split
* page_head has been freed
* and reallocated as slab or
* hugetlbfs page of smaller
* order (only possible if
* reallocated as slab on
* x86).
*/
goto skip_lock;
}
/* /*
* page_head wasn't a dangling pointer but it * page_head wasn't a dangling pointer but it
* may not be a head page anymore by the time * may not be a head page anymore by the time
@ -135,7 +151,6 @@ static void put_compound_page(struct page *page)
if (unlikely(!PageTail(page))) { if (unlikely(!PageTail(page))) {
/* __split_huge_page_refcount run before us */ /* __split_huge_page_refcount run before us */
compound_unlock_irqrestore(page_head, flags); compound_unlock_irqrestore(page_head, flags);
skip_lock:
if (put_page_testzero(page_head)) { if (put_page_testzero(page_head)) {
/* /*
* The head page may have been * The head page may have been
@ -221,36 +236,37 @@ bool __get_page_tail(struct page *page)
* split_huge_page(). * split_huge_page().
*/ */
unsigned long flags; unsigned long flags;
bool got = false; bool got;
struct page *page_head = compound_trans_head(page); struct page *page_head = compound_trans_head(page);
if (likely(page != page_head && get_page_unless_zero(page_head))) { /* Ref to put_compound_page() comment. */
/* Ref to put_compound_page() comment. */ if (PageSlab(page_head) || PageHeadHuge(page_head)) {
if (PageSlab(page_head) || PageHeadHuge(page_head)) { smp_rmb();
if (likely(PageTail(page))) { if (likely(PageTail(page))) {
/* /*
* This is a hugetlbfs page or a slab * This is a hugetlbfs page or a slab
* page. __split_huge_page_refcount * page. __split_huge_page_refcount
* cannot race here. * cannot race here.
*/ */
VM_BUG_ON(!PageHead(page_head)); VM_BUG_ON(!PageHead(page_head));
__get_page_tail_foll(page, false); __get_page_tail_foll(page, true);
return true; return true;
} else { } else {
/* /*
* __split_huge_page_refcount run * __split_huge_page_refcount run
* before us, "page" was a THP * before us, "page" was a THP
* tail. The split page_head has been * tail. The split page_head has been
* freed and reallocated as slab or * freed and reallocated as slab or
* hugetlbfs page of smaller order * hugetlbfs page of smaller order
* (only possible if reallocated as * (only possible if reallocated as
* slab on x86). * slab on x86).
*/ */
put_page(page_head); return false;
return false;
}
} }
}
got = false;
if (likely(page != page_head && get_page_unless_zero(page_head))) {
/* /*
* page_head wasn't a dangling pointer but it * page_head wasn't a dangling pointer but it
* may not be a head page anymore by the time * may not be a head page anymore by the time