0ed361dec3
After running SetPageUptodate, preceeding stores to the page contents to actually bring it uptodate may not be ordered with the store to set the page uptodate. Therefore, another CPU which checks PageUptodate is true, then reads the page contents can get stale data. Fix this by having an smp_wmb before SetPageUptodate, and smp_rmb after PageUptodate. Many places that test PageUptodate, do so with the page locked, and this would be enough to ensure memory ordering in those places if SetPageUptodate were only called while the page is locked. Unfortunately that is not always the case for some filesystems, but it could be an idea for the future. Also bring the handling of anonymous page uptodateness in line with that of file backed page management, by marking anon pages as uptodate when they _are_ uptodate, rather than when our implementation requires that they be marked as such. Doing allows us to get rid of the smp_wmb's in the page copying functions, which were especially added for anonymous pages for an analogous memory ordering problem. Both file and anonymous pages are handled with the same barriers. FAQ: Q. Why not do this in flush_dcache_page? A. Firstly, flush_dcache_page handles only one side (the smb side) of the ordering protocol; we'd still need smp_rmb somewhere. Secondly, hiding away memory barriers in a completely unrelated function is nasty; at least in the PageUptodate macros, they are located together with (half) the operations involved in the ordering. Thirdly, the smp_wmb is only required when first bringing the page uptodate, wheras flush_dcache_page should be called each time it is written to through the kernel mapping. It is logically the wrong place to put it. Q. Why does this increase my text size / reduce my performance / etc. A. Because it is adding the necessary instructions to eliminate the data-race. Q. Can it be improved? A. Yes, eg. if you were to create a rule that all SetPageUptodate operations run under the page lock, we could avoid the smp_rmb places where PageUptodate is queried under the page lock. Requires audit of all filesystems and at least some would need reworking. That's great you're interested, I'm eagerly awaiting your patches. Signed-off-by: Nick Piggin <npiggin@suse.de> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
188 lines
4.7 KiB
C
188 lines
4.7 KiB
C
#ifndef _LINUX_HIGHMEM_H
|
|
#define _LINUX_HIGHMEM_H
|
|
|
|
#include <linux/fs.h>
|
|
#include <linux/mm.h>
|
|
#include <linux/uaccess.h>
|
|
|
|
#include <asm/cacheflush.h>
|
|
|
|
#ifndef ARCH_HAS_FLUSH_ANON_PAGE
|
|
static inline void flush_anon_page(struct vm_area_struct *vma, struct page *page, unsigned long vmaddr)
|
|
{
|
|
}
|
|
#endif
|
|
|
|
#ifndef ARCH_HAS_FLUSH_KERNEL_DCACHE_PAGE
|
|
static inline void flush_kernel_dcache_page(struct page *page)
|
|
{
|
|
}
|
|
#endif
|
|
|
|
#ifdef CONFIG_HIGHMEM
|
|
|
|
#include <asm/highmem.h>
|
|
|
|
/* declarations for linux/mm/highmem.c */
|
|
unsigned int nr_free_highpages(void);
|
|
extern unsigned long totalhigh_pages;
|
|
|
|
void kmap_flush_unused(void);
|
|
|
|
#else /* CONFIG_HIGHMEM */
|
|
|
|
static inline unsigned int nr_free_highpages(void) { return 0; }
|
|
|
|
#define totalhigh_pages 0
|
|
|
|
#ifndef ARCH_HAS_KMAP
|
|
static inline void *kmap(struct page *page)
|
|
{
|
|
might_sleep();
|
|
return page_address(page);
|
|
}
|
|
|
|
#define kunmap(page) do { (void) (page); } while (0)
|
|
|
|
#include <asm/kmap_types.h>
|
|
|
|
static inline void *kmap_atomic(struct page *page, enum km_type idx)
|
|
{
|
|
pagefault_disable();
|
|
return page_address(page);
|
|
}
|
|
#define kmap_atomic_prot(page, idx, prot) kmap_atomic(page, idx)
|
|
|
|
#define kunmap_atomic(addr, idx) do { pagefault_enable(); } while (0)
|
|
#define kmap_atomic_pfn(pfn, idx) kmap_atomic(pfn_to_page(pfn), (idx))
|
|
#define kmap_atomic_to_page(ptr) virt_to_page(ptr)
|
|
|
|
#define kmap_flush_unused() do {} while(0)
|
|
#endif
|
|
|
|
#endif /* CONFIG_HIGHMEM */
|
|
|
|
/* when CONFIG_HIGHMEM is not set these will be plain clear/copy_page */
|
|
static inline void clear_user_highpage(struct page *page, unsigned long vaddr)
|
|
{
|
|
void *addr = kmap_atomic(page, KM_USER0);
|
|
clear_user_page(addr, vaddr, page);
|
|
kunmap_atomic(addr, KM_USER0);
|
|
}
|
|
|
|
#ifndef __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE
|
|
/**
|
|
* __alloc_zeroed_user_highpage - Allocate a zeroed HIGHMEM page for a VMA with caller-specified movable GFP flags
|
|
* @movableflags: The GFP flags related to the pages future ability to move like __GFP_MOVABLE
|
|
* @vma: The VMA the page is to be allocated for
|
|
* @vaddr: The virtual address the page will be inserted into
|
|
*
|
|
* This function will allocate a page for a VMA but the caller is expected
|
|
* to specify via movableflags whether the page will be movable in the
|
|
* future or not
|
|
*
|
|
* An architecture may override this function by defining
|
|
* __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE and providing their own
|
|
* implementation.
|
|
*/
|
|
static inline struct page *
|
|
__alloc_zeroed_user_highpage(gfp_t movableflags,
|
|
struct vm_area_struct *vma,
|
|
unsigned long vaddr)
|
|
{
|
|
struct page *page = alloc_page_vma(GFP_HIGHUSER | movableflags,
|
|
vma, vaddr);
|
|
|
|
if (page)
|
|
clear_user_highpage(page, vaddr);
|
|
|
|
return page;
|
|
}
|
|
#endif
|
|
|
|
/**
|
|
* alloc_zeroed_user_highpage_movable - Allocate a zeroed HIGHMEM page for a VMA that the caller knows can move
|
|
* @vma: The VMA the page is to be allocated for
|
|
* @vaddr: The virtual address the page will be inserted into
|
|
*
|
|
* This function will allocate a page for a VMA that the caller knows will
|
|
* be able to migrate in the future using move_pages() or reclaimed
|
|
*/
|
|
static inline struct page *
|
|
alloc_zeroed_user_highpage_movable(struct vm_area_struct *vma,
|
|
unsigned long vaddr)
|
|
{
|
|
return __alloc_zeroed_user_highpage(__GFP_MOVABLE, vma, vaddr);
|
|
}
|
|
|
|
static inline void clear_highpage(struct page *page)
|
|
{
|
|
void *kaddr = kmap_atomic(page, KM_USER0);
|
|
clear_page(kaddr);
|
|
kunmap_atomic(kaddr, KM_USER0);
|
|
}
|
|
|
|
static inline void zero_user_segments(struct page *page,
|
|
unsigned start1, unsigned end1,
|
|
unsigned start2, unsigned end2)
|
|
{
|
|
void *kaddr = kmap_atomic(page, KM_USER0);
|
|
|
|
BUG_ON(end1 > PAGE_SIZE || end2 > PAGE_SIZE);
|
|
|
|
if (end1 > start1)
|
|
memset(kaddr + start1, 0, end1 - start1);
|
|
|
|
if (end2 > start2)
|
|
memset(kaddr + start2, 0, end2 - start2);
|
|
|
|
kunmap_atomic(kaddr, KM_USER0);
|
|
flush_dcache_page(page);
|
|
}
|
|
|
|
static inline void zero_user_segment(struct page *page,
|
|
unsigned start, unsigned end)
|
|
{
|
|
zero_user_segments(page, start, end, 0, 0);
|
|
}
|
|
|
|
static inline void zero_user(struct page *page,
|
|
unsigned start, unsigned size)
|
|
{
|
|
zero_user_segments(page, start, start + size, 0, 0);
|
|
}
|
|
|
|
static inline void __deprecated memclear_highpage_flush(struct page *page,
|
|
unsigned int offset, unsigned int size)
|
|
{
|
|
zero_user(page, offset, size);
|
|
}
|
|
|
|
#ifndef __HAVE_ARCH_COPY_USER_HIGHPAGE
|
|
|
|
static inline void copy_user_highpage(struct page *to, struct page *from,
|
|
unsigned long vaddr, struct vm_area_struct *vma)
|
|
{
|
|
char *vfrom, *vto;
|
|
|
|
vfrom = kmap_atomic(from, KM_USER0);
|
|
vto = kmap_atomic(to, KM_USER1);
|
|
copy_user_page(vto, vfrom, vaddr, to);
|
|
kunmap_atomic(vfrom, KM_USER0);
|
|
kunmap_atomic(vto, KM_USER1);
|
|
}
|
|
|
|
#endif
|
|
|
|
static inline void copy_highpage(struct page *to, struct page *from)
|
|
{
|
|
char *vfrom, *vto;
|
|
|
|
vfrom = kmap_atomic(from, KM_USER0);
|
|
vto = kmap_atomic(to, KM_USER1);
|
|
copy_page(vto, vfrom);
|
|
kunmap_atomic(vfrom, KM_USER0);
|
|
kunmap_atomic(vto, KM_USER1);
|
|
}
|
|
|
|
#endif /* _LINUX_HIGHMEM_H */
|