From ee67dd75e215f88e5632c05c7e7e7c7d30910359 Mon Sep 17 00:00:00 2001 From: Vitaly Vul Date: Thu, 24 Sep 2015 14:28:59 +0200 Subject: [PATCH] mmap: fix calculation of is_private_anonymous variable Currently is_private_anonymous is calculated as true if _either_ MAP_PRIVATE or MAP_ANONYMOUS is set, which is a mistake. According to Documentation/vm/ksm.txt, "KSM only merges anonymous (private) pages, never pagecache (file) pages". MAP_PRIVATE can still be set on file cache pages so in order to not redundantly set MADV_MERGEABLE on pages that are not fitted for it, both MAP_PRIVATE and MAP_ANONYMOUS should be set. Along with this fix, add an extra check that the mapped page is not a stack page before setting MADV_MERGEABLE for it. Stack pages change too quickly and always end up in KSM 'page_volatile' list. Change-Id: If4954142852f17cc61f02985ea1cb625a7f3dec6 --- libc/bionic/mmap.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/libc/bionic/mmap.cpp b/libc/bionic/mmap.cpp index 8301445e1..57a8cdf95 100644 --- a/libc/bionic/mmap.cpp +++ b/libc/bionic/mmap.cpp @@ -54,10 +54,14 @@ void* mmap64(void* addr, size_t size, int prot, int flags, int fd, off64_t offse return MAP_FAILED; } - bool is_private_anonymous = (flags & (MAP_PRIVATE | MAP_ANONYMOUS)) != 0; + bool is_private_anonymous = + (flags & (MAP_PRIVATE | MAP_ANONYMOUS)) == (MAP_PRIVATE | MAP_ANONYMOUS); + bool is_stack_or_grows_down = (flags & (MAP_STACK | MAP_GROWSDOWN)) != 0; + void* result = __mmap2(addr, size, prot, flags, fd, offset >> MMAP2_SHIFT); - if (result != MAP_FAILED && kernel_has_MADV_MERGEABLE && is_private_anonymous) { + if (result != MAP_FAILED && kernel_has_MADV_MERGEABLE && + is_private_anonymous && !is_stack_or_grows_down) { ErrnoRestorer errno_restorer; int rc = madvise(result, size, MADV_MERGEABLE); if (rc == -1 && errno == EINVAL) {