Do some intelligent pagetable preconstruction, in combination with a small bit of restoration of struct mmu_gather's opacity to the core VM. -- wli arch/i386/mm/Makefile | 2 arch/i386/mm/init.c | 2 arch/i386/mm/pgtable.c | 14 - arch/i386/mm/tlb.c | 351 ++++++++++++++++++++++++++++++++++++++++++++ include/asm-generic/tlb.h | 10 + include/asm-i386/pgalloc.h | 129 ++-------------- include/asm-i386/tlb.h | 45 ++++- include/asm-i386/tlbflush.h | 8 - include/linux/gfp.h | 5 mm/memory.c | 20 +- mm/mmap.c | 2 mm/page_alloc.c | 2 12 files changed, 434 insertions(+), 156 deletions(-) diff -prauN pgcl-2.5.70-bk9-1/arch/i386/mm/Makefile pgcl-2.5.70-bk9-2/arch/i386/mm/Makefile --- pgcl-2.5.70-bk9-1/arch/i386/mm/Makefile 2003-05-26 18:00:26.000000000 -0700 +++ pgcl-2.5.70-bk9-2/arch/i386/mm/Makefile 2003-06-04 14:25:26.000000000 -0700 @@ -2,7 +2,7 @@ # Makefile for the linux i386-specific parts of the memory manager. # -obj-y := init.o pgtable.o fault.o ioremap.o extable.o pageattr.o +obj-y := init.o pgtable.o fault.o ioremap.o extable.o pageattr.o tlb.o obj-$(CONFIG_DISCONTIGMEM) += discontig.o obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o diff -prauN pgcl-2.5.70-bk9-1/arch/i386/mm/init.c pgcl-2.5.70-bk9-2/arch/i386/mm/init.c --- pgcl-2.5.70-bk9-1/arch/i386/mm/init.c 2003-06-04 10:12:54.000000000 -0700 +++ pgcl-2.5.70-bk9-2/arch/i386/mm/init.c 2003-06-04 14:20:10.000000000 -0700 @@ -41,7 +41,6 @@ #include #include -DEFINE_PER_CPU(struct mmu_gather, mmu_gathers); unsigned long highstart_pfn, highend_pfn; struct page *zero_page; @@ -477,6 +476,7 @@ void __init mem_init(void) zero_page = alloc_page(GFP_ATOMIC|GFP_DMA); clear_page(page_address(zero_page)); SetPageReserved(zero_page); + tlb_init(); totalram_pages--; reservedpages = 0; diff -prauN pgcl-2.5.70-bk9-1/arch/i386/mm/pgtable.c pgcl-2.5.70-bk9-2/arch/i386/mm/pgtable.c --- pgcl-2.5.70-bk9-1/arch/i386/mm/pgtable.c 2003-06-04 10:12:54.000000000 -0700 +++ pgcl-2.5.70-bk9-2/arch/i386/mm/pgtable.c 2003-06-04 14:32:52.000000000 -0700 @@ -142,20 +142,6 @@ pte_t *pte_alloc_one_kernel(struct mm_st return pte; } -struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address) -{ - struct page *pte; - -#ifdef CONFIG_HIGHPTE - pte = alloc_pages(GFP_KERNEL|__GFP_HIGHMEM|__GFP_REPEAT, 0); -#else - pte = alloc_pages(GFP_KERNEL|__GFP_REPEAT, 0); -#endif - if (pte) - clear_highpage(pte); - return pte; -} - void pmd_ctor(void *pmd, kmem_cache_t *cache, unsigned long flags) { memset(pmd, 0, PTRS_PER_PMD*sizeof(pmd_t)); diff -prauN pgcl-2.5.70-bk9-1/arch/i386/mm/tlb.c pgcl-2.5.70-bk9-2/arch/i386/mm/tlb.c --- pgcl-2.5.70-bk9-1/arch/i386/mm/tlb.c 1969-12-31 16:00:00.000000000 -0800 +++ pgcl-2.5.70-bk9-2/arch/i386/mm/tlb.c 2003-06-04 21:28:28.000000000 -0700 @@ -0,0 +1,351 @@ +/* + * arch/i386/mm/tlb.c + * (C) June 2003 William Irwin, IBM + * Routines for pagetable cacheing and release. + */ +#include +#include +#include +#include +#include + +#define __GFP_PTE (GFP_KERNEL|__GFP_REPEAT) +#ifdef CONFIG_HIGHMEM +#define GFP_PTE (__GFP_PTE|__GFP_HIGHMEM) +#else +#define GFP_PTE __GFP_PTE +#endif + +#define PG_PTE PG_arch_1 +#define NR_PTE 128 +#define NR_NONPTE 512 +#define MAX_ZONE_ID (MAX_NUMNODES * MAX_NR_ZONES) + +#define PagePTE(page) test_bit(PG_PTE, &(page)->flags) +#define SetPagePTE(page) set_bit(PG_PTE, &(page)->flags) +#define ClearPagePTE(page) clear_bit(PG_PTE, &(page)->flags) +#define PageZoneID(page) ((page)->flags >> ZONE_SHIFT) + +struct mmu_gather { + struct mm_struct *mm; + + /* number of active ptes needing a TLB flush before reuse */ + int nr_pte_active; + + /* whether some ptes were unmapped */ + unsigned int need_flush; + + /* non-zero means full mm flush */ + unsigned int fullmm; + + /* number freed for RSS adjustment */ + unsigned long freed; + + /* number of ready ptes */ + int nr_pte_ready; + + struct list_head active_list[MAX_ZONE_ID], ready_list[MAX_ZONE_ID]; + int active_count[MAX_ZONE_ID], ready_count[MAX_ZONE_ID]; + + int nr_nonpte; + struct page *nonpte[NR_NONPTE]; +}; + +static DEFINE_PER_CPU(struct mmu_gather, mmu_gathers); + +void tlb_init(void) +{ + int cpu; + for (cpu = 0; cpu < NR_CPUS; ++cpu) { + int zone; + struct mmu_gather *tlb = &per_cpu(mmu_gathers, cpu); + + for (zone = 0; zone < MAX_ZONE_ID; ++zone) { + INIT_LIST_HEAD(&tlb->active_list[zone]); + INIT_LIST_HEAD(&tlb->ready_list[zone]); + } + } +} + +struct mm_struct *tlb_mm(struct mmu_gather *tlb) +{ + return tlb->mm; +} + +void tlb_inc_freed(struct mmu_gather *tlb) +{ + tlb->freed++; +} + +/* + * When an mmu_gather fills, we must flush the entire mm, in no + * small part because whole-mm flushes are the sole bulk TLB + * invalidation primitive on i386. + */ +void tlb_flush(struct mmu_gather *tlb) +{ + flush_tlb_mm(tlb->mm); +} + +struct mmu_gather *tlb_gather_mmu(struct mm_struct *mm, unsigned int flush) +{ + struct mmu_gather *tlb = &per_cpu(mmu_gathers, get_cpu()); + tlb->mm = mm; + tlb->fullmm = flush; + put_cpu(); + return tlb; +} + +void tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *pte, unsigned long addr) +{ + tlb->need_flush = 1; +} + +static void tlb_flush_ready(struct mmu_gather *tlb) +{ + int count, zone = 0; + while (tlb->nr_pte_ready >= NR_PTE) { + BUG_ON(zone >= MAX_ZONE_ID); + if (!list_empty(&tlb->ready_list[zone])) { + BUG_ON(!zone_table[zone]); + free_pages_bulk(zone_table[zone], + tlb->ready_count[zone], + &tlb->ready_list[zone], + 0); + tlb->nr_pte_ready -= tlb->ready_count[zone]; + tlb->ready_count[zone] = 0; + BUG_ON(tlb->nr_pte_ready < 0); + BUG_ON(!list_empty(&tlb->ready_list[zone])); + } + zone++; + } + for (count = 0; zone < MAX_ZONE_ID; ++zone) { + BUG_ON(tlb->ready_count[zone] < 0); + count += tlb->ready_count[zone]; + } + BUG_ON(count != tlb->nr_pte_ready); +} + +void tlb_flush_mmu(struct mmu_gather *tlb, unsigned long start, + unsigned long end) +{ + int zone; + + if (!tlb->need_flush && tlb->nr_nonpte < NR_NONPTE) { + BUG_ON(tlb->nr_nonpte < 0); + BUG_ON(tlb->nr_pte_active < 0); + BUG_ON(tlb->nr_pte_ready < 0); + return; + } + + tlb->need_flush = 0; + tlb_flush(tlb); + BUG_ON(tlb->nr_nonpte < 0); + if (tlb->nr_nonpte) { + free_pages_and_swap_cache(tlb->nonpte, tlb->nr_nonpte); + tlb->nr_nonpte = 0; + } + + for (zone = 0; zone < MAX_ZONE_ID; ++zone) { + if (list_empty(&tlb->active_list[zone])) { + BUG_ON(tlb->active_count[zone]); + continue; + } + + list_splice_init(&tlb->active_list[zone], + &tlb->ready_list[zone]); + BUG_ON(tlb->active_count[zone] < 0); + BUG_ON(tlb->ready_count[zone] < 0); + tlb->ready_count[zone] += tlb->active_count[zone]; + tlb->active_count[zone] = 0; + } + tlb->nr_pte_ready += tlb->nr_pte_active; + tlb->nr_pte_active = 0; + if (tlb->nr_pte_ready >= NR_PTE) + tlb_flush_ready(tlb); +} + +void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, + unsigned long end) +{ + if (tlb->mm->rss >= tlb->freed) + tlb->mm->rss -= tlb->freed; + else + tlb->mm->rss = 0; + tlb_flush_mmu(tlb, start, end); +} + +static void tlb_remove_nonpte_page(struct mmu_gather *tlb, struct page *page) +{ + BUG_ON(tlb->nr_nonpte >= NR_NONPTE); + tlb->nonpte[tlb->nr_nonpte] = page; + tlb->nr_nonpte++; + if (tlb->nr_nonpte == NR_NONPTE) + tlb_flush_mmu(tlb, 0, 0); +} + +static void tlb_remove_pte_page(struct mmu_gather *tlb, struct page *page) +{ + int zone; + + if (!atomic_dec_and_test(&page->count)) + return; + + zone = PageZoneID(page); + ClearPagePTE(page); + BUG_ON(tlb->nr_pte_active < 0); + BUG_ON(tlb->active_count[zone] < 0); + tlb->nr_pte_active++; + tlb->active_count[zone]++; + list_add(&page->list, &tlb->active_list[zone]); +} + +void tlb_remove_page(struct mmu_gather *tlb, struct page *page) +{ + tlb->need_flush = 1; + if (PagePTE(page)) + tlb_remove_pte_page(tlb, page); + else + tlb_remove_nonpte_page(tlb, page); +} + +static struct page *pte_alloc_fresh(void) +{ + struct page *page = alloc_page(GFP_PTE); + if (page) { + clear_highpage(page); + BUG_ON(PagePTE(page)); + SetPagePTE(page); + } + return page; +} + +/* + * This needs to be restructured to discourage fallback to lowmem when + * nodes > 0 have lowmem. + */ +static struct page *pte_alloc_ready(void) +{ + struct mmu_gather *tlb = &per_cpu(mmu_gathers, get_cpu()); + struct page *page; + + BUG_ON(tlb->nr_pte_ready < 0); + if (!tlb->nr_pte_ready) { + BUG_ON(tlb->nr_pte_active < 0); + BUG_ON(tlb->nr_nonpte < 0); + page = NULL; + } else { + int zone; + for (zone = MAX_ZONE_ID - 1; zone >= 0; --zone) { + if (!list_empty(&tlb->ready_list[zone])) + break; + } + + BUG_ON(zone < 0); + BUG_ON(list_empty(&tlb->ready_list[zone])); + + page = list_entry(tlb->ready_list[zone].next, struct page, list); + BUG_ON(PagePTE(page)); + SetPagePTE(page); + list_del(&page->list); + atomic_set(&page->count, 1); + tlb->ready_count[zone]--; + tlb->nr_pte_ready--; + BUG_ON(tlb->ready_count[zone] < 0); + BUG_ON(tlb->nr_pte_ready < 0); + } + put_cpu(); + return page; +} + +struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address) +{ + struct page *page = pte_alloc_ready(); + return page ? page : pte_alloc_fresh(); +} + +/* + * pmd freeing occurs as part of freeing the pgd on PAE, and is not + * meaningful for non-PAE. + */ +void pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd) +{ + tlb->need_flush = 1; +} + +/* + * oddly declared in pgalloc.h; in general these are TLB-related pmd + * and pte twiddlings. + */ +void pmd_populate(struct mm_struct *mm, pmd_t *pmd, struct page *page) +{ + unsigned long pfn, pmd_off = (unsigned long)pmd; + int k; + + pmd_off = (pmd_off/sizeof(pmd_t)) % PAGE_MMUCOUNT; + pfn = page_to_pfn(page); + pmd -= pmd_off; + + if (PAGE_MMUCOUNT > 1) { + struct page *old_page = NULL; + + if (atomic_read(&page->count) != 1) { + WARN_ON(1); + printk(KERN_DEBUG "bad pte refcount = %d\n", + atomic_read(&page->count)); + } + + for (k = 0; k < PAGE_MMUCOUNT; ++k) { + if (pmd_present(pmd[k]) || !pmd_none(pmd[k])) { + if (old_page) + WARN_ON(old_page != pmd_page(pmd[k])); + else + old_page = pmd_page(pmd[k]); + } + } + + if (!old_page || old_page == page) + atomic_set(&page->count, PAGE_MMUCOUNT); + else { + /* + * old_page->index can legitimately be 0 + * but something's corrupt if it's mapping's wrong + */ + BUG_ON((struct mm_struct *)old_page->mapping != mm); + + /* + * errant callers can potentially do things + * out-of-order + */ + WARN_ON((struct mm_struct *)page->mapping != mm); + /* if (old_page->mapping != mm) + pgtable_add_rmap(page, mm, page->index); */ + pgtable_remove_rmap(page); + put_page(page); + atomic_set(&old_page->count, PAGE_MMUCOUNT); + for (k = 0; k < PAGE_MMUCOUNT; ++k) { + unsigned long long pmdval; + pmdval = page_to_pfn(old_page) + k; + pmdval <<= MMUPAGE_SHIFT; + if (pmd_present(pmd[k]) || !pmd_none(pmd[k])) { + WARN_ON(old_page != pmd_page(pmd[k])); + continue; + } else + set_pmd(&pmd[k], __pmd(_PAGE_TABLE + pmdval)); + } + return; + } + } + + for (k = 0; k < PAGE_MMUCOUNT; ++k) { + unsigned long long pmdval; + pmdval = (unsigned long long)(pfn + k) << MMUPAGE_SHIFT; + if (likely(pmd_none(pmd[k]) || !pmd_present(pmd[k]))) + set_pmd(&pmd[k], __pmd(_PAGE_TABLE + pmdval)); + else { + WARN_ON(1); + printk(KERN_DEBUG "pmdval=%Lx\n", (u64)pmd_val(pmd[k])); + put_page(page); /* a reference will be omitted */ + } + } +} diff -prauN pgcl-2.5.70-bk9-1/include/asm-generic/tlb.h pgcl-2.5.70-bk9-2/include/asm-generic/tlb.h --- pgcl-2.5.70-bk9-1/include/asm-generic/tlb.h 2003-05-26 18:00:19.000000000 -0700 +++ pgcl-2.5.70-bk9-2/include/asm-generic/tlb.h 2003-06-04 19:38:52.000000000 -0700 @@ -46,6 +46,16 @@ struct mmu_gather { /* Users of the generic TLB shootdown code must declare this storage space. */ DECLARE_PER_CPU(struct mmu_gather, mmu_gathers); +static inline struct mm_struct *tlb_mm(struct mmu_gather *tlb) +{ + return tlb->mm; +} + +static inline void tlb_inc_freed(struct mmu_gather *tlb) +{ + tlb->freed++; +} + /* tlb_gather_mmu * Return a pointer to an initialized struct mmu_gather. */ diff -prauN pgcl-2.5.70-bk9-1/include/asm-i386/pgalloc.h pgcl-2.5.70-bk9-2/include/asm-i386/pgalloc.h --- pgcl-2.5.70-bk9-1/include/asm-i386/pgalloc.h 2003-06-04 10:12:55.000000000 -0700 +++ pgcl-2.5.70-bk9-2/include/asm-i386/pgalloc.h 2003-06-04 18:46:26.000000000 -0700 @@ -9,92 +9,32 @@ #include /* for struct page */ #include /* to make asm-generic/rmap.h happy */ #include /* for pgtable_remove_rmap() */ +/* + * allocating and freeing a pmd is trivial: the 1-entry pmd is + * inside the pgd, so has no extra memory associated with it. + * (In the PAE case we free the pmds as part of the pgd.) + */ + +#define pmd_alloc_one(mm, addr) ({ BUG(); ((pmd_t *)2); }) +#define pmd_free(x) do { } while (0) +#define __pmd_free_tlb(tlb,x) do { } while (0) +#define pgd_populate(mm, pmd, pte) BUG() + +#define check_pgt_cache() do { } while (0) #define pmd_populate_kernel(mm, pmd, pte) \ set_pmd(pmd, __pmd(_PAGE_TABLE + __pa(pte))) -static inline void pgtable_remove_rmap(struct page *); -static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, struct page *page) -{ - unsigned long pfn, pmd_off = (unsigned long)pmd; - int k; +struct mmu_gather; - pmd_off = (pmd_off/sizeof(pmd_t)) % PAGE_MMUCOUNT; - pfn = page_to_pfn(page); - pmd -= pmd_off; - - if (PAGE_MMUCOUNT > 1) { - struct page *old_page = NULL; - - if (atomic_read(&page->count) != 1) { - WARN_ON(1); - printk(KERN_DEBUG "bad pte refcount = %d\n", - atomic_read(&page->count)); - } - - for (k = 0; k < PAGE_MMUCOUNT; ++k) { - if (pmd_present(pmd[k]) || !pmd_none(pmd[k])) { - if (old_page) - WARN_ON(old_page != pmd_page(pmd[k])); - else - old_page = pmd_page(pmd[k]); - } - } - - if (!old_page || old_page == page) - atomic_set(&page->count, PAGE_MMUCOUNT); - else { - /* - * old_page->index can legitimately be 0 - * but something's corrupt if it's mapping's wrong - */ - BUG_ON((struct mm_struct *)old_page->mapping != mm); - - /* - * errant callers can potentially do things - * out-of-order - */ - WARN_ON((struct mm_struct *)page->mapping != mm); - /* if (old_page->mapping != mm) - pgtable_add_rmap(page, mm, page->index); */ - pgtable_remove_rmap(page); - put_page(page); - atomic_set(&old_page->count, PAGE_MMUCOUNT); - for (k = 0; k < PAGE_MMUCOUNT; ++k) { - unsigned long long pmdval; - pmdval = page_to_pfn(old_page) + k; - pmdval <<= MMUPAGE_SHIFT; - if (pmd_present(pmd[k]) || !pmd_none(pmd[k])) { - WARN_ON(old_page != pmd_page(pmd[k])); - continue; - } else - set_pmd(&pmd[k], __pmd(_PAGE_TABLE + pmdval)); - } - return; - } - } - - for (k = 0; k < PAGE_MMUCOUNT; ++k) { - unsigned long long pmdval; - pmdval = (unsigned long long)(pfn + k) << MMUPAGE_SHIFT; - if (likely(pmd_none(pmd[k]) || !pmd_present(pmd[k]))) - set_pmd(&pmd[k], __pmd(_PAGE_TABLE + pmdval)); - else { - WARN_ON(1); - printk(KERN_DEBUG "pmdval=%Lx\n", (u64)pmd_val(pmd[k])); - put_page(page); /* a reference will be omitted */ - } - } -} /* * Allocate and free page tables. */ - -extern pgd_t *pgd_alloc(struct mm_struct *); -extern void pgd_free(pgd_t *pgd); - -extern pte_t *pte_alloc_one_kernel(struct mm_struct *, unsigned long); -extern struct page *pte_alloc_one(struct mm_struct *, unsigned long); +void pmd_populate(struct mm_struct *mm, pmd_t *pmd, struct page *page); +pgd_t *pgd_alloc(struct mm_struct *); +void pgd_free(pgd_t *pgd); +pte_t *pte_alloc_one_kernel(struct mm_struct *, unsigned long); +struct page *pte_alloc_one(struct mm_struct *, unsigned long); static inline void pte_free_kernel(pte_t *pte) { @@ -103,40 +43,9 @@ static inline void pte_free_kernel(pte_t static inline void pte_free(struct page *page) { - if (PAGE_MMUCOUNT == 1) - __free_page(page); - else - put_page(page); + put_page(page); } - -struct mmu_gather; -static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page); -static inline void __pte_free_tlb(struct mmu_gather *tlb, struct page *page) -{ - /* restore the reference count so later frees don't BUG() */ - if (PAGE_MMUCOUNT > 1) { - if (atomic_dec_and_test(&page->count)) - atomic_set(&page->count, 1); - else - return; - } - tlb_remove_page(tlb, page); -} - -/* - * allocating and freeing a pmd is trivial: the 1-entry pmd is - * inside the pgd, so has no extra memory associated with it. - * (In the PAE case we free the pmds as part of the pgd.) - */ - -#define pmd_alloc_one(mm, addr) ({ BUG(); ((pmd_t *)2); }) -#define pmd_free(x) do { } while (0) -#define __pmd_free_tlb(tlb,x) do { } while (0) -#define pgd_populate(mm, pmd, pte) BUG() - -#define check_pgt_cache() do { } while (0) - #include #endif /* _I386_PGALLOC_H */ diff -prauN pgcl-2.5.70-bk9-1/include/asm-i386/tlb.h pgcl-2.5.70-bk9-2/include/asm-i386/tlb.h --- pgcl-2.5.70-bk9-1/include/asm-i386/tlb.h 2003-05-26 18:00:27.000000000 -0700 +++ pgcl-2.5.70-bk9-2/include/asm-i386/tlb.h 2003-06-04 18:48:24.000000000 -0700 @@ -1,20 +1,45 @@ #ifndef _I386_TLB_H #define _I386_TLB_H +#include +#include +#include +#include +#include +#include +#include + /* * x86 doesn't need any special per-pte or * per-vma handling.. */ -#define tlb_start_vma(tlb, vma) do { } while (0) -#define tlb_end_vma(tlb, vma) do { } while (0) -#define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0) +struct vm_area_struct; +struct mmu_gather; -/* - * .. because we flush the whole mm when it - * fills up. - */ -#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm) +void tlb_flush(struct mmu_gather *); +struct mm_struct *tlb_mm(struct mmu_gather *tlb); +void tlb_inc_freed(struct mmu_gather *tlb); +struct mmu_gather *tlb_gather_mmu(struct mm_struct *, unsigned int flush); +void tlb_flush_mmu(struct mmu_gather *, unsigned long, unsigned long); +void tlb_finish_mmu(struct mmu_gather *, unsigned long, unsigned long); +void tlb_remove_page(struct mmu_gather *, struct page *); +void tlb_remove_tlb_entry(struct mmu_gather *, pte_t *, unsigned long addr); +void pmd_free_tlb(struct mmu_gather *, pmd_t *); +void tlb_init(void); + +static inline void tlb_start_vma(struct mmu_gather *tlb, + struct vm_area_struct *vma) +{ +} + +static inline void tlb_end_vma(struct mmu_gather *tlb, + struct vm_area_struct *vma) +{ +} -#include +static inline void pte_free_tlb(struct mmu_gather *tlb, struct page *page) +{ + tlb_remove_page(tlb, page); +} -#endif +#endif /* _I386_TLB_H */ diff -prauN pgcl-2.5.70-bk9-1/include/asm-i386/tlbflush.h pgcl-2.5.70-bk9-2/include/asm-i386/tlbflush.h --- pgcl-2.5.70-bk9-1/include/asm-i386/tlbflush.h 2003-06-04 10:12:55.000000000 -0700 +++ pgcl-2.5.70-bk9-2/include/asm-i386/tlbflush.h 2003-06-04 13:23:28.000000000 -0700 @@ -119,10 +119,10 @@ static inline void flush_tlb_range(struc #define local_flush_tlb() \ __flush_tlb() -extern void flush_tlb_all(void); -extern void flush_tlb_current_task(void); -extern void flush_tlb_mm(struct mm_struct *); -extern void flush_tlb_page(struct vm_area_struct *, unsigned long); +void flush_tlb_all(void); +void flush_tlb_current_task(void); +void flush_tlb_mm(struct mm_struct *); +void flush_tlb_page(struct vm_area_struct *, unsigned long); #define flush_tlb() flush_tlb_current_task() diff -prauN pgcl-2.5.70-bk9-1/include/linux/gfp.h pgcl-2.5.70-bk9-2/include/linux/gfp.h --- pgcl-2.5.70-bk9-1/include/linux/gfp.h 2003-05-26 18:00:26.000000000 -0700 +++ pgcl-2.5.70-bk9-2/include/linux/gfp.h 2003-06-04 13:03:31.000000000 -0700 @@ -74,8 +74,9 @@ static inline struct page * alloc_pages_ #define alloc_page(gfp_mask) \ alloc_pages_node(numa_node_id(), gfp_mask, 0) -extern unsigned long FASTCALL(__get_free_pages(unsigned int gfp_mask, unsigned int order)); -extern unsigned long FASTCALL(get_zeroed_page(unsigned int gfp_mask)); +unsigned long FASTCALL(__get_free_pages(unsigned int gfp_mask, unsigned int order)); +unsigned long FASTCALL(get_zeroed_page(unsigned int gfp_mask)); +int free_pages_bulk(struct zone *zone, int count, struct list_head *list, unsigned int order); #define __get_free_page(gfp_mask) \ __get_free_pages((gfp_mask),0) diff -prauN pgcl-2.5.70-bk9-1/mm/memory.c pgcl-2.5.70-bk9-2/mm/memory.c --- pgcl-2.5.70-bk9-1/mm/memory.c 2003-06-04 10:12:55.000000000 -0700 +++ pgcl-2.5.70-bk9-2/mm/memory.c 2003-06-04 19:21:09.000000000 -0700 @@ -127,7 +127,7 @@ static inline void free_one_pgd(struct m */ void clear_page_tables(struct mmu_gather *tlb, unsigned long first, int nr) { - pgd_t * page_dir = tlb->mm->pgd; + pgd_t * page_dir = tlb_mm(tlb)->pgd; page_dir += first; do { @@ -432,21 +432,15 @@ zap_pte_range(struct mmu_gather *tlb, pm if (page->mapping && pte_young(pte) && !PageSwapCache(page)) mark_page_accessed(page); - tlb->freed++; + tlb_inc_freed(tlb); page_remove_rmap(page, ptep); tlb_remove_page(tlb, page); } } } else { - if (!pte_file(pte)) { - if (pte_to_swp_entry(pte).val == 0x8073756dUL) - printk(KERN_DEBUG "detected fsckup " - "early, leaking stuff to " - "work around it\n"); - else - free_swap_and_cache(pte_to_swp_entry(pte)); - } pte_clear(ptep); + if (!pte_file(pte)) + free_swap_and_cache(pte_to_swp_entry(pte)); } } pte_unmap(ptep-1); @@ -545,7 +539,7 @@ int unmap_vmas(struct mmu_gather **tlbp, unsigned long end_addr, unsigned long *nr_accounted) { unsigned long zap_bytes = ZAP_BLOCK_SIZE; - unsigned long tlb_start; /* For tlb_finish_mmu */ + unsigned long tlb_start = 0; /* For tlb_finish_mmu */ int tlb_start_valid = 0; int ret = 0; @@ -592,7 +586,9 @@ int unmap_vmas(struct mmu_gather **tlbp, if ((long)zap_bytes > 0) continue; if (need_resched()) { - tlb_finish_mmu(*tlbp, tlb_start, start); + tlb_finish_mmu(*tlbp, + tlb_start_valid ? tlb_start : 0, + start); cond_resched_lock(&mm->page_table_lock); *tlbp = tlb_gather_mmu(mm, 0); tlb_start_valid = 0; diff -prauN pgcl-2.5.70-bk9-1/mm/mmap.c pgcl-2.5.70-bk9-2/mm/mmap.c --- pgcl-2.5.70-bk9-1/mm/mmap.c 2003-06-04 10:12:55.000000000 -0700 +++ pgcl-2.5.70-bk9-2/mm/mmap.c 2003-06-04 14:28:24.000000000 -0700 @@ -1064,7 +1064,7 @@ static void free_pgtables(struct mmu_gat unsigned long first = start & PGDIR_MASK; unsigned long last = end + PGDIR_SIZE - 1; unsigned long start_index, end_index; - struct mm_struct *mm = tlb->mm; + struct mm_struct *mm = tlb_mm(tlb); if (!prev) { prev = mm->mmap; diff -prauN pgcl-2.5.70-bk9-1/mm/page_alloc.c pgcl-2.5.70-bk9-2/mm/page_alloc.c --- pgcl-2.5.70-bk9-1/mm/page_alloc.c 2003-06-04 10:12:55.000000000 -0700 +++ pgcl-2.5.70-bk9-2/mm/page_alloc.c 2003-06-04 13:00:57.000000000 -0700 @@ -233,7 +233,7 @@ static inline void free_pages_check(cons * And clear the zone's pages_scanned counter, to hold off the "all pages are * pinned" detection logic. */ -static int +int free_pages_bulk(struct zone *zone, int count, struct list_head *list, unsigned int order) {