Introduces the notion of temporal and nontemporal stores for page clearing. The existing clear_page function does zeroing using temporal stores. A new function is added clear_cold(address, page-order) to zero pages using non-temporal stores. Temporal clearing (hot clearing) is advantageous because it populates the cache with the cache lines of the page cleared. This avoids the loading of cache lines when the page is written to later. Hot clearing is not useful for to zero larger areas of memory because doing so will cause the cpu caches to be filled with zeroed cachelines that cannot be used directly. Nontemporal cleaning (cold clearing) is advantageous because it does *not* populate the cache with zeroed cachelines and is faster than hot clearing. See David Mosberger's post at http://www.gelato.unsw.edu.au/linux-ia64/0409/11012.html. This is only beneficial if the page being cleared is not written to immediately because it preserves the existing cache. It is particularly useful to zero larger areas of memory without trashing the cpu caches. The patch does: 1. Provide clear_cold implementations for ia64,i386, x86_64 and sparc64. (the clear_cold implementations for x86_64 and i386 are really hot clearing at this point. Please fix this!) 2. Allows the specification of __GFP_COLD with __GFP_ZERO to specify that cold clearing is to used. 3. Changes ia64 to use hot zeroing for clear_page() 4. Use cold clearing for clearing huge pages. 5. Use cold clearing for shmem buffers allocated that are not immediately used. Signed-off-by: Christoph Lameter Index: linux-2.6.11/mm/hugetlb.c =================================================================== --- linux-2.6.11.orig/mm/hugetlb.c 2005-03-01 23:38:12.000000000 -0800 +++ linux-2.6.11/mm/hugetlb.c 2005-04-05 20:26:53.000000000 -0700 @@ -78,7 +78,6 @@ void free_huge_page(struct page *page) struct page *alloc_huge_page(void) { struct page *page; - int i; spin_lock(&hugetlb_lock); page = dequeue_huge_page(); @@ -89,8 +88,7 @@ struct page *alloc_huge_page(void) spin_unlock(&hugetlb_lock); set_page_count(page, 1); page[1].mapping = (void *)free_huge_page; - for (i = 0; i < (HPAGE_SIZE/PAGE_SIZE); ++i) - clear_highpage(&page[i]); + prep_zero_page(page, HUGETLB_PAGE_ORDER, GFP_HIGHUSER | __GFP_COLD); return page; } Index: linux-2.6.11/mm/page_alloc.c =================================================================== --- linux-2.6.11.orig/mm/page_alloc.c 2005-04-05 20:26:33.000000000 -0700 +++ linux-2.6.11/mm/page_alloc.c 2005-04-05 20:26:53.000000000 -0700 @@ -633,11 +633,16 @@ void fastcall free_cold_page(struct page free_hot_cold_page(page, 1); } -static inline void prep_zero_page(struct page *page, int order, unsigned int __nocast gfp_flags) +void prep_zero_page(struct page *page, unsigned int order, unsigned int __nocast gfp_flags) { int i; BUG_ON((gfp_flags & (__GFP_WAIT | __GFP_HIGHMEM)) == __GFP_HIGHMEM); +#ifdef CONFIG_CLEAR_COLD + if ((gfp_flags & __GFP_COLD) && !PageHighMem(page)) + clear_cold(page_address(page), order); + else +#endif for(i = 0; i < (1 << order); i++) clear_highpage(page + i); } Index: linux-2.6.11/include/linux/gfp.h =================================================================== --- linux-2.6.11.orig/include/linux/gfp.h 2005-04-05 20:26:14.000000000 -0700 +++ linux-2.6.11/include/linux/gfp.h 2005-04-05 20:26:53.000000000 -0700 @@ -131,4 +131,5 @@ extern void FASTCALL(free_cold_page(stru void page_alloc_init(void); +void prep_zero_page(struct page *, unsigned int order, unsigned int __nocast gfp_flags); #endif /* __LINUX_GFP_H */ Index: linux-2.6.11/arch/ia64/Kconfig =================================================================== --- linux-2.6.11.orig/arch/ia64/Kconfig 2005-04-05 20:23:55.000000000 -0700 +++ linux-2.6.11/arch/ia64/Kconfig 2005-04-05 20:26:55.000000000 -0700 @@ -46,6 +46,10 @@ config GENERIC_IOMAP bool default y +config CLEAR_COLD + bool + default y + choice prompt "System type" default IA64_GENERIC Index: linux-2.6.11/include/asm-ia64/page.h =================================================================== --- linux-2.6.11.orig/include/asm-ia64/page.h 2005-04-05 20:26:10.000000000 -0700 +++ linux-2.6.11/include/asm-ia64/page.h 2005-04-05 20:26:57.000000000 -0700 @@ -57,6 +57,8 @@ # define STRICT_MM_TYPECHECKS extern void clear_page (void *page); +/* Clear arbitrary order page using nontemporal writes */ +extern void clear_cold (void *page, unsigned int order); extern void copy_page (void *to, void *from); /* Index: linux-2.6.11/arch/ia64/kernel/ia64_ksyms.c =================================================================== --- linux-2.6.11.orig/arch/ia64/kernel/ia64_ksyms.c 2005-03-01 23:38:08.000000000 -0800 +++ linux-2.6.11/arch/ia64/kernel/ia64_ksyms.c 2005-04-05 20:26:57.000000000 -0700 @@ -39,6 +39,7 @@ EXPORT_SYMBOL(__up); #include EXPORT_SYMBOL(clear_page); +EXPORT_SYMBOL(clear_cold); #ifdef CONFIG_VIRTUAL_MEM_MAP #include Index: linux-2.6.11/arch/ia64/lib/clear_page.S =================================================================== --- linux-2.6.11.orig/arch/ia64/lib/clear_page.S 2005-03-01 23:37:47.000000000 -0800 +++ linux-2.6.11/arch/ia64/lib/clear_page.S 2005-04-05 20:26:57.000000000 -0700 @@ -7,6 +7,8 @@ * 1/06/01 davidm Tuned for Itanium. * 2/12/02 kchen Tuned for both Itanium and McKinley * 3/08/02 davidm Some more tweaking + * 24/3/04 clameter Make clear_page use temporal stores + add clear_cold using nontemporal stores */ #include @@ -53,6 +55,59 @@ GLOBAL_ENTRY(clear_page) ;; #ifdef CONFIG_ITANIUM // Optimized for Itanium +1: stf.spill [dst1] = f0, 64 + stf.spill [dst2] = f0, 64 + cmp.lt p8,p0=dst_fetch, dst_last + ;; +#else + // Optimized for McKinley +1: stf.spill [dst1] = f0, 64 + stf.spill [dst2] = f0, 64 + stf.spill [dst3] = f0, 64 + stf.spill [dst4] = f0, 128 + cmp.lt p8,p0=dst_fetch, dst_last + ;; + stf.spill [dst1] = f0, 64 + stf.spill [dst2] = f0, 64 +#endif + stf.spill [dst3] = f0, 64 +(p8) stf.spill [dst_fetch] = f0, L3_LINE_SIZE + br.cloop.sptk.few 1b + ;; + mov ar.lc = saved_lc // restore lc + br.ret.sptk.many rp +END(clear_page) + +#define totsize r14 + +GLOBAL_ENTRY(clear_cold) + .prologue + .regstk 2,0,0,0 + mov r16 = PAGE_SIZE/L3_LINE_SIZE // main loop count + mov totsize = PAGE_SIZE + .save ar.lc, saved_lc + mov saved_lc = ar.lc + ;; + .body + adds dst1 = 16, in0 + mov ar.lc = (PREFETCH_LINES - 1) + mov dst_fetch = in0 + adds dst2 = 32, in0 + shl r16 = r16, in1 + shl totsize = totsize, in1 + ;; +.fetc: stf.spill.nta [dst_fetch] = f0, L3_LINE_SIZE + adds dst3 = 48, in0 // executing this multiple times is harmless + br.cloop.sptk.few .fetc + add r16 = -1,r16 + add dst_last = totsize, dst_fetch + adds dst4 = 64, in0 + ;; + mov ar.lc = r16 // one L3 line per iteration + adds dst_last = -PREFETCH_LINES*L3_LINE_SIZE, dst_last + ;; +#ifdef CONFIG_ITANIUM + // Optimized for Itanium 1: stf.spill.nta [dst1] = f0, 64 stf.spill.nta [dst2] = f0, 64 cmp.lt p8,p0=dst_fetch, dst_last @@ -74,4 +129,4 @@ GLOBAL_ENTRY(clear_page) ;; mov ar.lc = saved_lc // restore lc br.ret.sptk.many rp -END(clear_page) +END(clear_cold) Index: linux-2.6.11/arch/i386/Kconfig =================================================================== --- linux-2.6.11.orig/arch/i386/Kconfig 2005-04-05 20:23:51.000000000 -0700 +++ linux-2.6.11/arch/i386/Kconfig 2005-04-05 20:26:57.000000000 -0700 @@ -33,6 +33,10 @@ config GENERIC_IOMAP bool default y +config CLEAR_COLD + bool + default y + source "init/Kconfig" menu "Processor type and features" Index: linux-2.6.11/include/asm-i386/page.h =================================================================== --- linux-2.6.11.orig/include/asm-i386/page.h 2005-03-01 23:37:49.000000000 -0800 +++ linux-2.6.11/include/asm-i386/page.h 2005-04-05 20:26:57.000000000 -0700 @@ -19,6 +19,7 @@ #include #define clear_page(page) mmx_clear_page((void *)(page)) +#define clear_cold(page, order) mmx_clear_cold((void *)(page), order) #define copy_page(to,from) mmx_copy_page(to,from) #else @@ -29,6 +30,8 @@ */ #define clear_page(page) memset((void *)(page), 0, PAGE_SIZE) +/* Clear arbitrary order page with nontemporal stores... is memset temporal?? */ +#define clear_cold(page, order) memset((void *)(page), 0, PAGE_SIZE << order) #define copy_page(to,from) memcpy((void *)(to), (void *)(from), PAGE_SIZE) #endif Index: linux-2.6.11/include/asm-i386/mmx.h =================================================================== --- linux-2.6.11.orig/include/asm-i386/mmx.h 2005-03-01 23:38:09.000000000 -0800 +++ linux-2.6.11/include/asm-i386/mmx.h 2005-04-05 20:26:57.000000000 -0700 @@ -9,6 +9,7 @@ extern void *_mmx_memcpy(void *to, const void *from, size_t size); extern void mmx_clear_page(void *page); +extern void mmx_clear_cold(void *page, unsigned int order); extern void mmx_copy_page(void *to, void *from); #endif Index: linux-2.6.11/arch/i386/lib/mmx.c =================================================================== --- linux-2.6.11.orig/arch/i386/lib/mmx.c 2005-03-01 23:38:09.000000000 -0800 +++ linux-2.6.11/arch/i386/lib/mmx.c 2005-04-05 20:26:58.000000000 -0700 @@ -397,3 +397,14 @@ void mmx_copy_page(void *to, void *from) else fast_copy_page(to, from); } + +/* FIXME: Make this a real cold zeroing function */ +void mmx_clear_cold(void *page, int order) +{ + int i; + + for(i=0; i < (1 << order); i++) { + mmx_clear_page(page); + page += PAGE_SIZE; + } +} Index: linux-2.6.11/arch/x86_64/Kconfig =================================================================== --- linux-2.6.11.orig/arch/x86_64/Kconfig 2005-04-05 20:24:33.000000000 -0700 +++ linux-2.6.11/arch/x86_64/Kconfig 2005-04-05 20:26:58.000000000 -0700 @@ -61,6 +61,10 @@ config GENERIC_IOMAP bool default y +config CLEAR_COLD + bool + default y + source "init/Kconfig" Index: linux-2.6.11/include/asm-x86_64/page.h =================================================================== --- linux-2.6.11.orig/include/asm-x86_64/page.h 2005-04-05 20:26:14.000000000 -0700 +++ linux-2.6.11/include/asm-x86_64/page.h 2005-04-05 20:26:58.000000000 -0700 @@ -33,6 +33,8 @@ #ifndef __ASSEMBLY__ void clear_page(void *); +/* Clear arbitrary order page using non-temporal writes */ +void clear_cold(void *, int order); void copy_page(void *, void *); #define clear_user_page(page, vaddr, pg) clear_page(page) Index: linux-2.6.11/arch/x86_64/kernel/x8664_ksyms.c =================================================================== --- linux-2.6.11.orig/arch/x86_64/kernel/x8664_ksyms.c 2005-04-05 20:24:34.000000000 -0700 +++ linux-2.6.11/arch/x86_64/kernel/x8664_ksyms.c 2005-04-05 20:26:58.000000000 -0700 @@ -108,6 +108,7 @@ EXPORT_SYMBOL(pci_mem_start); EXPORT_SYMBOL(copy_page); EXPORT_SYMBOL(clear_page); +EXPORT_SYMBOL(clear_cold); EXPORT_SYMBOL(cpu_pda); #ifdef CONFIG_SMP Index: linux-2.6.11/arch/x86_64/lib/clear_page.S =================================================================== --- linux-2.6.11.orig/arch/x86_64/lib/clear_page.S 2005-03-01 23:38:08.000000000 -0800 +++ linux-2.6.11/arch/x86_64/lib/clear_page.S 2005-04-05 20:26:58.000000000 -0700 @@ -48,3 +48,57 @@ clear_page_c: ret clear_page_c_end: .previous + + +/* + * Zero a page cold. + * rdi page + * rsi order + */ + .globl clear_cold + .p2align 4 +clear_cold: + movl $4096/64,%eax + movl %esi, %ecx + shll %cl, %eax + movl %eax, %ecx + xorl %eax,%eax + .p2align 4 +.Lcloop: + decl %ecx +#define PUTC(x) movq %rax,x*8(%rdi) + movq %rax,(%rdi) + PUTC(1) + PUTC(2) + PUTC(3) + PUTC(4) + PUTC(5) + PUTC(6) + PUTC(7) + leaq 64(%rdi),%rdi + jnz .Lcloop + nop + ret +clear_cold_end: + + .section .altinstructions,"a" + .align 8 + .quad clear_cold + .quad clear_cold_c + .byte X86_FEATURE_K8_C + .byte clear_cold_end-clear_cold + .byte clear_cold_c_end-clear_cold_c + .previous + + .section .altinstr_replacement,"ax" +clear_cold_c: + movl $4096/8,%eax + movl %esi, %ecx + shll %cl, %eax + movl %eax, %ecx + xorl %eax,%eax + rep + stosq + ret +clear_cold_c_end: + .previous Index: linux-2.6.11/arch/sparc64/lib/clear_page.S =================================================================== --- linux-2.6.11.orig/arch/sparc64/lib/clear_page.S 2005-03-01 23:38:17.000000000 -0800 +++ linux-2.6.11/arch/sparc64/lib/clear_page.S 2005-04-05 20:26:58.000000000 -0700 @@ -72,13 +72,127 @@ clear_user_page: /* %o0=dest, %o1=vaddr mov 1, %o4 clear_page_common: + membar #StoreLoad | #StoreStore | #LoadStore + sethi %hi(PAGE_SIZE/64), %o1 + mov %o0, %g1 ! remember vaddr for tlbflush + or %o1, %lo(PAGE_SIZE/64), %o1 + +#define PREFETCH(x, y) prefetch x, y +#define PREFETCH_CODE 2 + + PREFETCH([%o0 + 0x000], PREFETCH_CODE) + PREFETCH([%o0 + 0x040], PREFETCH_CODE) + PREFETCH([%o0 + 0x080], PREFETCH_CODE) + PREFETCH([%o0 + 0x0c0], PREFETCH_CODE) + PREFETCH([%o0 + 0x100], PREFETCH_CODE) + PREFETCH([%o0 + 0x140], PREFETCH_CODE) + PREFETCH([%o0 + 0x180], PREFETCH_CODE) +1: + stx %g0, [%o0 + 0x00] + stx %g0, [%o0 + 0x08] + stx %g0, [%o0 + 0x10] + stx %g0, [%o0 + 0x18] + stx %g0, [%o0 + 0x20] + stx %g0, [%o0 + 0x28] + stx %g0, [%o0 + 0x30] + stx %g0, [%o0 + 0x38] + PREFETCH([%o0 + 0x1c0], PREFETCH_CODE) + subcc %o1, 1, %o1 + bne,pt %icc, 1b + add %o0, 0x40, %o0 + + brz,pn %o4, out + nop + + stxa %g0, [%g1] ASI_DMMU_DEMAP + membar #Sync + stw %o2, [%g6 + TI_PRE_COUNT] + +out: retl + nop + + .globl clear_cold +clear_cold: /* %o0=dest, %o1=order */ + sethi %hi(PAGE_SIZE/64), %o2 + clr %o4 + or %o2, %lo(PAGE_SIZE/64), %o2 + sllx %o2, %o1, %o1 VISEntryHalf membar #StoreLoad | #StoreStore | #LoadStore fzero %f0 + fzero %f2 + faddd %f0, %f2, %f4 + fmuld %f0, %f2, %f6 + faddd %f0, %f2, %f8 + fmuld %f0, %f2, %f10 + + faddd %f0, %f2, %f12 + fmuld %f0, %f2, %f14 +2: stda %f0, [%o0 + %g0] ASI_BLK_P + subcc %o1, 1, %o1 + bne,pt %icc, 2b + add %o0, 0x40, %o0 + membar #Sync + VISExitHalf + + retl + nop + + .globl clear_cold +clear_cold: /* %o0=dest, %o1=order */ + sethi %hi(PAGE_SIZE/64), %o2 + clr %o4 + or %o2, %lo(PAGE_SIZE/64), %o2 + ba,pt %xcc, clear_cold_common + sllx %o2, %o1, %o1 + + /* This thing is pretty important, it shows up + * on the profiles via do_anonymous_page(). + */ + .align 32 + .globl clear_cold_page +clear_cold_user_page: /* %o0=dest, %o1=vaddr */ + lduw [%g6 + TI_PRE_COUNT], %o2 + sethi %uhi(PAGE_OFFSET), %g2 + sethi %hi(PAGE_SIZE), %o4 + + sllx %g2, 32, %g2 + sethi %uhi(TTE_BITS_TOP), %g3 + + sllx %g3, 32, %g3 + sub %o0, %g2, %g1 ! paddr + + or %g3, TTE_BITS_BOTTOM, %g3 + and %o1, %o4, %o0 ! vaddr D-cache alias bit + + or %g1, %g3, %g1 ! TTE data + sethi %hi(TLBTEMP_BASE), %o3 + + add %o2, 1, %o4 + add %o0, %o3, %o0 ! TTE vaddr + + /* Disable preemption. */ + mov TLB_TAG_ACCESS, %g3 + stw %o4, [%g6 + TI_PRE_COUNT] + + /* Load TLB entry. */ + rdpr %pstate, %o4 + wrpr %o4, PSTATE_IE, %pstate + stxa %o0, [%g3] ASI_DMMU + stxa %g1, [%g0] ASI_DTLB_DATA_IN + flush %g6 + wrpr %o4, 0x0, %pstate + sethi %hi(PAGE_SIZE/64), %o1 + mov 1, %o4 + or %o1, %lo(PAGE_SIZE/64), %o1 + +clear_cold_common: + VISEntryHalf + membar #StoreLoad | #StoreStore | #LoadStore + fzero %f0 mov %o0, %g1 ! remember vaddr for tlbflush fzero %f2 - or %o1, %lo(PAGE_SIZE/64), %o1 faddd %f0, %f2, %f4 fmuld %f0, %f2, %f6 faddd %f0, %f2, %f8 @@ -86,20 +200,20 @@ clear_page_common: faddd %f0, %f2, %f12 fmuld %f0, %f2, %f14 -1: stda %f0, [%o0 + %g0] ASI_BLK_P +2: stda %f0, [%o0 + %g0] ASI_BLK_P subcc %o1, 1, %o1 - bne,pt %icc, 1b + bne,pt %icc, 2b add %o0, 0x40, %o0 membar #Sync VISExitHalf - brz,pn %o4, out + brz,pn %o4, outcold nop stxa %g0, [%g1] ASI_DMMU_DEMAP membar #Sync stw %o2, [%g6 + TI_PRE_COUNT] -out: retl +outcold: retl nop Index: linux-2.6.11/include/asm-sparc64/page.h =================================================================== --- linux-2.6.11.orig/include/asm-sparc64/page.h 2005-04-05 20:26:13.000000000 -0700 +++ linux-2.6.11/include/asm-sparc64/page.h 2005-04-05 20:26:58.000000000 -0700 @@ -27,6 +27,8 @@ extern void _clear_page(void *page); #define clear_page(X) _clear_page((void *)(X)) +/* Non temporal clear an arbitrary order page */ +extern void clear_cold(void *page, unsigned int order); struct page; extern void clear_user_page(void *addr, unsigned long vaddr, struct page *page); #define copy_page(X,Y) memcpy((void *)(X), (void *)(Y), PAGE_SIZE) Index: linux-2.6.11/arch/sparc64/Kconfig =================================================================== --- linux-2.6.11.orig/arch/sparc64/Kconfig 2005-04-05 20:24:23.000000000 -0700 +++ linux-2.6.11/arch/sparc64/Kconfig 2005-04-05 20:26:58.000000000 -0700 @@ -43,6 +43,10 @@ config SPARC64_PAGE_SIZE_4MB endchoice +config CLEAR_COLD + bool + default y + source "init/Kconfig" config SYSVIPC_COMPAT Index: linux-2.6.11/drivers/block/pktcdvd.c =================================================================== --- linux-2.6.11.orig/drivers/block/pktcdvd.c 2005-04-05 20:24:38.000000000 -0700 +++ linux-2.6.11/drivers/block/pktcdvd.c 2005-04-05 20:26:58.000000000 -0700 @@ -135,7 +135,7 @@ static struct packet_data *pkt_alloc_pac goto no_bio; for (i = 0; i < PAGES_PER_PACKET; i++) { - pkt->pages[i] = alloc_page(GFP_KERNEL|__GFP_ZERO); + pkt->pages[i] = alloc_page(GFP_KERNEL|__GFP_ZERO|__GFP_COLD); if (!pkt->pages[i]) goto no_page; } Index: linux-2.6.11/kernel/profile.c =================================================================== --- linux-2.6.11.orig/kernel/profile.c 2005-03-01 23:38:26.000000000 -0800 +++ linux-2.6.11/kernel/profile.c 2005-04-05 20:26:58.000000000 -0700 @@ -326,13 +326,13 @@ static int __devinit profile_cpu_callbac node = cpu_to_node(cpu); per_cpu(cpu_profile_flip, cpu) = 0; if (!per_cpu(cpu_profile_hits, cpu)[1]) { - page = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0); + page = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO | __GFP_COLD, 0); if (!page) return NOTIFY_BAD; per_cpu(cpu_profile_hits, cpu)[1] = page_address(page); } if (!per_cpu(cpu_profile_hits, cpu)[0]) { - page = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0); + page = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO | __GFP_COLD, 0); if (!page) goto out_free; per_cpu(cpu_profile_hits, cpu)[0] = page_address(page); @@ -508,12 +508,12 @@ static int __init create_hash_tables(voi int node = cpu_to_node(cpu); struct page *page; - page = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0); + page = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO | __GFP_COLD, 0); if (!page) goto out_cleanup; per_cpu(cpu_profile_hits, cpu)[1] = (struct profile_hit *)page_address(page); - page = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0); + page = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO | __GFP_COLD, 0); if (!page) goto out_cleanup; per_cpu(cpu_profile_hits, cpu)[0] Index: linux-2.6.11/mm/shmem.c =================================================================== --- linux-2.6.11.orig/mm/shmem.c 2005-04-05 20:26:33.000000000 -0700 +++ linux-2.6.11/mm/shmem.c 2005-04-05 20:26:58.000000000 -0700 @@ -369,6 +369,13 @@ static swp_entry_t *shmem_swp_alloc(stru } spin_unlock(&info->lock); + + /* + * No use of __GFP_COLD here since the page will be immediately + * refernced after shmem_swp_alloc returns. However, the access + * typically only affects one cache line. So one could use + * __GFP_COLD if one is willing to refetch a cacheline + */ page = shmem_dir_alloc(mapping_gfp_mask(inode->i_mapping) | __GFP_ZERO); if (page) { page->nr_swapped = 0; @@ -909,7 +916,7 @@ shmem_alloc_page(unsigned long gfp, stru pvma.vm_policy = mpol_shared_policy_lookup(&info->policy, idx); pvma.vm_pgoff = idx; pvma.vm_end = PAGE_SIZE; - page = alloc_page_vma(gfp | __GFP_ZERO, &pvma, 0); + page = alloc_page_vma(gfp | __GFP_ZERO | __GFP_COLD, &pvma, 0); mpol_free(pvma.vm_policy); return page; } @@ -925,7 +932,7 @@ static inline struct page * shmem_alloc_page(unsigned int __nocast gfp,struct shmem_inode_info *info, unsigned long idx) { - return alloc_page(gfp | __GFP_ZERO); + return alloc_page(gfp | __GFP_ZERO | __GFP_COLD); } #endif