diff -Naur -X dontdiff linux-2.4.20-ref/Documentation/Configure.help linux-2.4.20/Documentation/Configure.help --- linux-2.4.20-ref/Documentation/Configure.help Thu Nov 28 17:53:08 2002 +++ linux-2.4.20/Documentation/Configure.help Mon Jun 9 14:55:53 2003 @@ -389,6 +389,11 @@ Select this if you have a 32-bit processor and more than 4 gigabytes of physical RAM. +Hot-add memory support +CONFIG_X86_MEM_HOTADD + Say Y here if you have a Intel i386 based system with memory hotplug + capability. + HIGHMEM I/O support CONFIG_HIGHIO If you want to be able to do I/O to high memory pages, say Y. diff -Naur -X dontdiff linux-2.4.20-ref/arch/i386/boot/setup.S linux-2.4.20/arch/i386/boot/setup.S --- linux-2.4.20-ref/arch/i386/boot/setup.S Fri Aug 2 19:39:42 2002 +++ linux-2.4.20/arch/i386/boot/setup.S Wed Jun 11 10:30:41 2003 @@ -157,9 +157,13 @@ # can be located anywhere in # low memory 0x10000 or higher. +#ifndef CONFIG_X86_MEM_HOTADD ramdisk_max: .long __MAXMEM-1 # (Header version 0x0203 or later) # The highest safe address for # the contents of an initrd +#else +ramdisk_max: .long __MAX_RDMEM-1 +#endif trampoline: call start_of_setup .space 1024 diff -Naur -X dontdiff linux-2.4.20-ref/arch/i386/config.in linux-2.4.20/arch/i386/config.in --- linux-2.4.20-ref/arch/i386/config.in Thu Nov 28 17:53:09 2002 +++ linux-2.4.20/arch/i386/config.in Mon Jun 9 14:41:17 2003 @@ -200,6 +200,7 @@ fi if [ "$CONFIG_HIGHMEM" = "y" ]; then + bool 'Hot-add memory support' CONFIG_X86_MEM_HOTADD bool 'HIGHMEM I/O support' CONFIG_HIGHIO fi diff -Naur -X dontdiff linux-2.4.20-ref/arch/i386/kernel/setup.c linux-2.4.20/arch/i386/kernel/setup.c --- linux-2.4.20-ref/arch/i386/kernel/setup.c Thu Nov 28 17:53:09 2002 +++ linux-2.4.20/arch/i386/kernel/setup.c Wed Jun 11 10:49:10 2003 @@ -144,6 +144,12 @@ /* user-defined highmem size */ static unsigned int highmem_pages __initdata = -1; +#ifdef CONFIG_X86_MEM_HOTADD +/* Virtaul address space reserved for memory hot-add operation */ +unsigned long hotadd_reserve_size; +unsigned long hotadd_reserve_start; +#endif /* CONFIG_X86_MEM_HOTADD */ + /* * Setup options */ @@ -730,6 +736,7 @@ print_memory_map(who); } /* setup_memory_region */ +static int mem_hotadd_disabled = 0; static void __init parse_cmdline_early (char ** cmdline_p) { @@ -762,6 +769,11 @@ from += 8+4; e820.nr_map = 0; userdef = 1; +#ifdef CONFIG_X86_MEM_HOTADD + } else if (!memcmp(from+4, "nohotadd", 8)) { + from += 8+4; + mem_hotadd_disabled = 1; +#endif /* CONFIG_X86_MEM_HOTADD */ } else { /* If the user specifies memory size, we * limit the BIOS-provided memory map to @@ -792,6 +804,7 @@ else if (!memcmp(from, "acpismp=force", 13)) enable_acpi_smp_table = 1; +#ifndef CONFIG_X86_MEM_HOTADD /* * highmem=size forces highmem to be exactly 'size' bytes. * This works even on boxes that have no highmem otherwise. @@ -799,6 +812,7 @@ */ else if (!memcmp(from, "highmem=", 8)) highmem_pages = memparse(from+8, &from) >> PAGE_SHIFT; +#endif /* !CONFIG_X86_MEM_HOTADD */ nextchar: c = *(from++); if (!c) @@ -823,7 +837,10 @@ * Reserved space for vmalloc and iomap - defined in asm/page.h */ #define MAXMEM_PFN PFN_DOWN(MAXMEM) + +#ifndef CONFIG_X86_MEM_HOTADD #define MAX_NONPAE_PFN (1 << 20) +#endif /* * Find the highest page frame number we have available @@ -906,6 +923,53 @@ return max_low_pfn; } +#ifdef CONFIG_X86_MEM_HOTADD +/* + * Reserve enough virtual addresses to store page structures for + * hot-added memory. + * Needs work: Should the reserve_size depend on a configuration + * option? + */ +static void __init find_hotadd_reserve_size(unsigned long max_pfn) +{ + if (! mem_hotadd_disabled) { + +#ifndef CONFIG_X86_PAE + if (max_pfn > MAX_NONPAE_PFN) + hotadd_reserve_size = 0; + else + hotadd_reserve_size = (MAX_NONPAE_PFN - max_pfn) * + sizeof(struct page); +#else /* CONFIG_X86_PAE */ + if (max_pfn >= MAX_PAE_PFN) + hotadd_reserve_size = 0; + else + hotadd_reserve_size = (MAX_PAE_PFN - max_pfn) * + sizeof(struct page); + +#endif /* CONFIG_X86_PAE */ + + if (hotadd_reserve_size % PMD_SIZE) + hotadd_reserve_size += (PMD_SIZE - + (hotadd_reserve_size % PMD_SIZE)); + } +} + +static unsigned long +__init set_hotadd_reserve_start(unsigned long max_low_pfn) +{ + if (hotadd_reserve_size) { + hotadd_reserve_start = (unsigned long)__va(max_low_pfn * PAGE_SIZE); + return (max_low_pfn + (hotadd_reserve_size >> PAGE_SHIFT)); + } + else { + hotadd_reserve_start = 0; + return 0; + } +} +#endif /* CONFIG_X86_MEM_HOTADD */ + + /* * Register fully available low RAM pages with the bootmem allocator. */ @@ -946,10 +1010,20 @@ } } +#if defined (CONFIG_BLK_DEV_INITRD) && defined(CONFIG_X86_MEM_HOTADD) +static unsigned long __initdata initrd_high_start; +static unsigned long __initdata initrd_high_end; +static int initrd_high = 0; +#endif /* CONFIG_BLK_DEV_INITRD && CONFIG_X86_MEM_HOTADD */ + static unsigned long __init setup_memory(void) { unsigned long bootmap_size, start_pfn, max_low_pfn; +#ifdef CONFIG_X86_MEM_HOTADD + unsigned long saved_low_pfn = 0; +#endif /* CONFIG_X86_MEM_HOTADD */ + /* * partially used pages are not usable - thus * we are rounding upwards: @@ -958,6 +1032,10 @@ find_max_pfn(); +#ifdef CONFIG_X86_MEM_HOTADD + find_hotadd_reserve_size(max_pfn); +#endif + max_low_pfn = find_max_low_pfn(); #ifdef CONFIG_HIGHMEM @@ -986,6 +1064,10 @@ reserve_bootmem(HIGH_MEMORY, (PFN_PHYS(start_pfn) + bootmap_size + PAGE_SIZE-1) - (HIGH_MEMORY)); +#ifdef CONFIG_X86_MEM_HOTADD + saved_low_pfn = set_hotadd_reserve_start(max_low_pfn); +#endif + /* * reserve physical page 0 - it's a special BIOS page on many boxes, * enabling clean reboots, SMP operation, laptop functions. @@ -1015,6 +1097,35 @@ INITRD_START ? INITRD_START + PAGE_OFFSET : 0; initrd_end = initrd_start+INITRD_SIZE; } +#ifdef CONFIG_X86_MEM_HOTADD + else if (saved_low_pfn && (INITRD_START + INITRD_SIZE <= + (saved_low_pfn << PAGE_SHIFT))) { + unsigned long initrd_low_start; + unsigned long initrd_low_end; + unsigned long initrd_low_size; + + initrd_high = 1; + initrd_low_size = initrd_low_start = 0; + initrd_high_start = 0; + if ((max_low_pfn << PAGE_SHIFT) > INITRD_START) { + + initrd_low_start = INITRD_START + PAGE_OFFSET; + initrd_low_size = (max_low_pfn << PAGE_SHIFT) - + initrd_start; + initrd_low_end = initrd_low_start + + initrd_low_size; + reserve_bootmem(INITRD_START, initrd_low_size); + } + initrd_high_end = INITRD_START + INITRD_SIZE + + PAGE_OFFSET; + initrd_high_start = initrd_high_end - + (INITRD_SIZE - initrd_low_size); + initrd_start = (initrd_low_start) ? initrd_low_start : + initrd_high_start; + initrd_end = initrd_high_end; + + } +#endif else { printk(KERN_ERR "initrd extends beyond end of memory " "(0x%08lx > 0x%08lx)\ndisabling initrd\n", @@ -1147,6 +1258,11 @@ smp_alloc_memory(); /* AP processor realmode stacks in low memory*/ #endif paging_init(); + +#if defined (CONFIG_BLK_DEV_INITRD) && defined(CONFIG_X86_MEM_HOTADD) + if (initrd_high && initrd_high_start) + initrd_map_high (initrd_high_start, initrd_high_end); +#endif #ifdef CONFIG_X86_LOCAL_APIC /* * get boot-time SMP configuration: diff -Naur -X dontdiff linux-2.4.20-ref/arch/i386/mm/Makefile linux-2.4.20/arch/i386/mm/Makefile --- linux-2.4.20-ref/arch/i386/mm/Makefile Thu Nov 28 17:53:09 2002 +++ linux-2.4.20/arch/i386/mm/Makefile Mon Jun 9 14:39:33 2003 @@ -12,4 +12,9 @@ obj-y := init.o fault.o ioremap.o extable.o pageattr.o export-objs := pageattr.o +ifeq ($(CONFIG_X86_MEM_HOTADD),y) +obj-y += mem_hotadd.o +export-objs += mem_hotadd.o +endif + include $(TOPDIR)/Rules.make diff -Naur -X dontdiff linux-2.4.20-ref/arch/i386/mm/fault.c linux-2.4.20/arch/i386/mm/fault.c --- linux-2.4.20-ref/arch/i386/mm/fault.c Thu Nov 28 17:53:09 2002 +++ linux-2.4.20/arch/i386/mm/fault.c Mon Jun 9 14:44:49 2003 @@ -391,6 +391,9 @@ if (!pmd_present(*pmd_k)) goto no_context; set_pmd(pmd, *pmd_k); + /* If PMD points to a 4MB page, no PTEs to update */ + if (pmd_has_pse(*pmd_k)) + return; pte_k = pte_offset(pmd_k, address); if (!pte_present(*pte_k)) diff -Naur -X dontdiff linux-2.4.20-ref/arch/i386/mm/init.c linux-2.4.20/arch/i386/mm/init.c --- linux-2.4.20-ref/arch/i386/mm/init.c Thu Nov 28 17:53:09 2002 +++ linux-2.4.20/arch/i386/mm/init.c Wed Jun 11 10:14:11 2003 @@ -40,8 +40,14 @@ mmu_gather_t mmu_gathers[NR_CPUS]; unsigned long highstart_pfn, highend_pfn; + +#ifndef CONFIG_X86_MEM_HOTADD static unsigned long totalram_pages; static unsigned long totalhigh_pages; +#else +unsigned long totalram_pages; +unsigned long totalhigh_pages; +#endif int do_check_pgt_cache(int low, int high) { @@ -592,16 +598,95 @@ } #ifdef CONFIG_BLK_DEV_INITRD + +#ifdef CONFIG_X86_MEM_HOTADD +void initrd_map_high(unsigned long start, unsigned long end) +{ + unsigned long align_start = start & PMD_MASK; + unsigned long align_end = (end + PMD_SIZE - 1) & PMD_MASK; + unsigned long base; + pgd_t *pgd; + pmd_t *pmd; + pte_t *pte, *pte_base; + int count = (align_end - align_start) / PMD_SIZE; + int i; + + pgd = swapper_pg_dir + __pgd_offset(align_start); + pmd = pmd_offset(pgd, align_start); + base = align_start; + if (cpu_has_pse) { + while (count--) { + set_pmd(pmd,__pmd(_KERNPG_TABLE + _PAGE_PSE + __pa(base))); + base += PMD_SIZE; + pmd++; + } + } + else { + while (count--) { + pte_base = pte = alloc_bootmem_low_pages(PAGE_SIZE); + for (i=0; i < PTRS_PER_PTE; i++, pte++) { + set_pte(pte, mk_pte_phys(__pa(base), PAGE_KERNEL)); + base += PAGE_SIZE; + } + set_pmd(pmd, __pmd(_KERNPG_TABLE + __pa(pte_base))); + pmd++; + } + } +} + +static void initrd_unmap_high(unsigned long start, unsigned long end) +{ + unsigned long align_start = start & PMD_MASK; + unsigned long align_end = (end + PMD_SIZE - 1) & PMD_MASK; + unsigned long base; + pgd_t *pgd; + pmd_t *pmd; + pte_t *pte; + int count = (align_end - align_start) / PMD_SIZE; + + pgd = swapper_pg_dir + __pgd_offset(align_start); + pmd = pmd_offset(pgd, align_start); + base = align_start; + + while (count--) { + if (!pmd_has_pse(*pmd)) { + pte = (pmd_val(*pmd) & PAGE_MASK); + free_bootmem((unsigned long)pte, PAGE_SIZE); + } + pmd_clear(pmd); + base += PMD_SIZE; + pmd++; + } +} +#endif + void free_initrd_mem(unsigned long start, unsigned long end) { + unsigned long high_start = __va(max_low_pfn << PAGE_SHIFT); + if (start < end) printk (KERN_INFO "Freeing initrd memory: %ldk freed\n", (end - start) >> 10); + +#ifndef CONFIG_X86_MEM_HOTADD for (; start < end; start += PAGE_SIZE) { ClearPageReserved(virt_to_page(start)); set_page_count(virt_to_page(start), 1); free_page(start); totalram_pages++; } +#else + + for (; (start < end) && (start < high_start); start += PAGE_SIZE) { + ClearPageReserved(virt_to_page(start)); + set_page_count(virt_to_page(start), 1); + free_page(start); + totalram_pages++; + } + + if (start < end) + initrd_unmap_high(start, end); +#endif + } #endif diff -Naur -X dontdiff linux-2.4.20-ref/arch/i386/mm/mem_hotadd.c linux-2.4.20/arch/i386/mm/mem_hotadd.c --- linux-2.4.20-ref/arch/i386/mm/mem_hotadd.c Wed Dec 31 18:00:00 1969 +++ linux-2.4.20/arch/i386/mm/mem_hotadd.c Wed Jun 11 17:58:06 2003 @@ -0,0 +1,423 @@ +/* + * arch/i386/mem_hotadd.c + * (c) 2002 Hewlett-Packard Development Company, L.P. + */ + +#include +#include + +#ifdef CONFIG_MODULES +#include +#include +#endif + +/* + * The functions in this file integrate added memory into the system. + * Flow: + * 1. Calculate size of data structures needed + * 2. Make part of the new memory accesible by setting PGDs. Allocate + * data structures in this area + * 3. Initialize data structures - pg_data_t and mem map + * 4. Reserve memory used up by data structures + * 5. Add rest of the memory to free list + * 6. Update global variables + */ + +int mem_hotadd_count = 0; +static DECLARE_MUTEX(mem_hotadd_sem); + +extern int init_pgdat(int, pg_data_t *, unsigned long, unsigned long, + unsigned long *, unsigned long *, struct page *, int); + +extern unsigned long hotadd_reserve_size; +extern unsigned long hotadd_reserve_start; +extern unsigned long totalram_pages; +extern unsigned long totalhigh_pages; + +static unsigned long hotadd_vaddr_start = -1; +static unsigned long hotadd_vaddr_left = -1; + +static void hotadd_mem_cleanup(unsigned long, unsigned long); + + +/* + * hotadd_mem_bootstrap(): Map part of the newly added memory to the + * kernel address space, starting at vaddr_start. + * Return size of the mapped memory (in bytes). + */ + +static int +hotadd_mem_bootstrap (kaddr_t paddr_start, unsigned long totalpages) +{ + int required; + kaddr_t paddr; + unsigned long vaddr, vaddr_start, vaddr_end; + int i,j,k; + pgd_t *pgd; + pmd_t *pmd; + pte_t *pte, *pte_base; + + /* + * Calculate data structure sizes: pg_data_t and memory map + * Also get the number of PGDs required to access that amount + * of memory + */ +#if 0 + int map_size; + map_size = (sizeof(struct page) * (totalpages+1)); + required = map_size + sizeof(pg_data_t); +#endif + + vaddr_start = hotadd_vaddr_start; + required = (sizeof(struct page) * totalpages); + + if (hotadd_vaddr_left < required) + return -ENOSPC; + + + printk ("hotadd_mem_bootstrap: Initializing %lu memory pages for data structures\n", (required / PAGE_SIZE)); + /* + * Create PGD and PMD entries and set them in the global page directory + */ + vaddr_end = vaddr_start + required; + vaddr = vaddr_start; + paddr = paddr_start; + + while (vaddr < vaddr_end) { + + i = __pgd_offset(vaddr); + j = __pmd_offset(vaddr); + k = __pte_offset(vaddr); + pgd = (pgd_t *)(swapper_pg_dir + i); + pmd = pmd_offset(pgd, vaddr); + + if ( ((vaddr % PMD_SIZE) == 0) && + ((paddr % PMD_SIZE) == 0) && + (required > PMD_SIZE) && + (cpu_has_pse)) { + kaddr_t __pe; + + __pe = _KERNPG_TABLE + _PAGE_PSE + paddr; + + /* Make it "global" too if supported */ + if (cpu_has_pge) { + __pe += _PAGE_GLOBAL; + } + set_pmd(pmd, __pmd(__pe)); + paddr += PMD_SIZE; + vaddr += PMD_SIZE; + required -= PMD_SIZE; + continue; + } + else { + int k_max = (required + PAGE_SIZE - 1) / PAGE_SIZE; + if ((k + k_max) > PTRS_PER_PTE) + k_max = PTRS_PER_PTE; + else + k_max += k; + + if (pmd_val(*pmd)) + pte_base = (pte_t *) __va(pmd_val(*pmd) & PAGE_MASK); + else { + pte_base = (pte_t *) kmalloc(PAGE_SIZE,GFP_KERNEL); + if (pte_base == NULL) { + printk("hotadd_mem_bootstrap: " + "kmalloc failed. " + "Cleaning up..\n"); + hotadd_mem_cleanup(vaddr_start, vaddr); + return -ENOMEM; + } + memset(pte_base,0,PAGE_SIZE); + } + pte = pte_base + k; + for (; k < k_max; pte++, k++) { + set_pte(pte, mk_pte_phys(paddr, PAGE_KERNEL)); + paddr += PAGE_SIZE; + vaddr += PAGE_SIZE; + required -= PAGE_SIZE; + } + if (! pmd_val(*pmd)) + set_pmd(pmd, __pmd(_KERNPG_TABLE + + __pa(pte_base))); + } + } + + return (vaddr_end - vaddr_start); +} + +/* + * hotadd_init_pgdat(): Initialize the pg_data_t and page structures + * needed for newly added memory. Reserve used pages and add rest of + * the pages to free list. + */ +static int +hotadd_init_pgdat(unsigned long totalpages, kaddr_t paddr_start, int used) +{ + struct page *map; + pg_data_t *pgdat, *temp_pgdat; + unsigned long zone_sizes[MAX_NR_ZONES]; + unsigned long zhole_sizes[MAX_NR_ZONES]; + unsigned long vaddr_start; + int map_size; + int i, ret = 0; + + vaddr_start = hotadd_vaddr_start; + + for (i=0; i> PAGE_SHIFT), + totalpages, zone_sizes, zhole_sizes, map, 0); + + if (ret < 0) { + kfree (pgdat); + return ret; + } + + /* Mark as reserved all pages corresponding to memory used earlier */ + for (i=0; i <= ((used + PAGE_SIZE - 1) / PAGE_SIZE); i++) + SetPageReserved(map+i); + + printk ("hotadd_init_pgdat: Reserved %d pages\n", i); + + /* Add the new pg_data_t to the linked list */ + temp_pgdat = pgdat_list; + while (temp_pgdat->node_next) + temp_pgdat = temp_pgdat->node_next; + + temp_pgdat->node_next = pgdat; + mb(); + + mem_hotadd_count++; + mb(); + + /* Add all (un-reserved) pages to free list */ + for (i=0; iflags)); + if (! PageReserved(map+i)) { + set_page_count(map+i, 1); + __free_page(map+i); + } + } + return 0; +} + +static void +hotadd_init_done(unsigned long totalpages, int used) +{ + unsigned long next_start; + + printk("hotadd_init_done: Updating global variables\n"); + + next_start = (hotadd_vaddr_start + used + PAGE_SIZE - 1) & PAGE_MASK; + hotadd_vaddr_left -= (next_start - hotadd_vaddr_start); + hotadd_vaddr_start = next_start; + + /* + * Update globals. + */ + numnodes++; + num_physpages += totalpages; + totalram_pages += totalpages; + totalhigh_pages += totalpages; + max_mapnr += totalpages; + highend_pfn += totalpages; +} + +static void +hotadd_mem_cleanup(unsigned long vaddr_start, unsigned long vaddr_end) +{ + unsigned long vaddr = vaddr_start; + int i, j; + pgd_t *pgd; + pmd_t *pmd; + pte_t *pte_base, *pte; + + i = __pgd_offset(vaddr); + j = __pte_offset(vaddr); + pgd = (pgd_t *)(swapper_pg_dir + i); + pmd = pmd_offset(pgd, vaddr); + + if (! pmd_val(*pmd)) + BUG(); + + if (pmd_has_pse(*pmd)) { + pmd_clear(pmd); + vaddr += PMD_SIZE; + } + else { + pte_base = (pte_t *) __va(pmd_val(*pmd) & PAGE_MASK); + + pte = pte_base + j; + for (; j < PTRS_PER_PTE; pte++, j++) { + pte_clear(pte); + vaddr += PAGE_SIZE; + } + } + + while (vaddr < vaddr_end) { + + i = __pgd_offset(vaddr); + pgd = (pgd_t *)(swapper_pg_dir + i); + pmd = pmd_offset(pgd, vaddr); + + if (! pmd_val(*pmd)) + BUG(); + + if (! pmd_has_pse(*pmd)) { + pte_base = (pte_t *) __va(pmd_val(*pmd) & PAGE_MASK); + kfree(pte_base); + } + pmd_clear(pmd); + vaddr += PMD_SIZE; + } +} + + +/* + * hotadd_mem_init(): Integrate hot-added memory into the system. + */ +int hotadd_mem_init(unsigned long long start, unsigned long long size, + int flags) +{ +#ifndef CONFIG_X86_PAE + unsigned long paddr_start = (unsigned long)start; +#else + unsigned long long paddr_start = start; +#endif + pg_data_t *pgdat; + unsigned long pages = size / PAGE_SIZE; + unsigned long totalpages; + unsigned long start_pfn = (paddr_start >> PAGE_SHIFT); + unsigned long end_pfn = start_pfn + pages - 1; + unsigned long old_flags; + int used; + int ret = 0; + + printk ("Attempting to hot-add memory. Start address = 0x%Lx, Size = 0x%Lx (%lu pages)\n", start, size, pages); + + printk ("Free pages before hot-add: %u\n\n", nr_free_pages()); + + if (paddr_start & ~PAGE_MASK) + return -EINVAL; + + /* + * Adding to DMA or Normal zones is not allowed. + */ + if (paddr_start < __pa(high_memory)) + return -EINVAL; + +#ifndef CONFIG_X86_PAE + if (start_pfn >= MAX_NONPAE_PFN) + return -E2BIG; +#else + if (start_pfn >= MAX_PAE_PFN) + return -E2BIG; +#endif + if (numnodes == MAX_NR_NODES) + return -E2BIG; + +#ifndef CONFIG_X86_PAE + totalpages = (end_pfn < MAX_NONPAE_PFN) ? pages : + (MAX_NONPAE_PFN - start_pfn); + +#else + totalpages = (end_pfn < MAX_PAE_PFN) ? pages : + (MAX_PAE_PFN - start_pfn); +#endif + end_pfn = start_pfn + totalpages - 1; + + if (totalpages < pages) { + printk ("hotadd_mem_init: Can only add %lu pages out of %lu\n", totalpages, pages); + } + + down(&mem_hotadd_sem); + + /* + * Check for overlapping address ranges (should never happen..) + */ + pgdat = pgdat_list; + while (pgdat) { + kaddr_t pgdat_start = (pgdat->node_start_pfn); + kaddr_t pgdat_end = pgdat_start + pgdat->node_size - 1; + + if ((start_pfn >= pgdat_start) && (start_pfn <= pgdat_end)) { + printk ("hotadd_mem_init: Overlapping memory region\n"); + ret = -EEXIST; + goto err; + } + + if ((end_pfn >= pgdat_start) && (end_pfn <= pgdat_end)) { + printk ("hotadd_mem_init: Overlapping memory region\n"); + ret = -EEXIST; + goto err; + } + pgdat = pgdat->node_next; + } + + if (hotadd_vaddr_start == -1) + hotadd_vaddr_start = hotadd_reserve_start; + if (hotadd_vaddr_left == -1) + hotadd_vaddr_left = hotadd_reserve_size; + + /* Return if the reserved virtual address range is used up */ + if (hotadd_vaddr_left == 0) { + ret = -ENOMEM; + goto err; + } + + old_flags = current->flags; + current->flags |= PF_MEMALLOC; + + used = hotadd_mem_bootstrap(paddr_start, totalpages); + + if (used < 0) { + printk("hotadd_mem_init: Failed to initialize memory for data structures\n"); + ret = used; + goto done; + } + + ret = hotadd_init_pgdat(totalpages, paddr_start, used); + if (ret < 0) { + printk ("hotadd_init_pgdat failed. Cleaning up...\n"); + hotadd_mem_cleanup(hotadd_vaddr_start, + (hotadd_vaddr_start + used)); + goto done; + } + + hotadd_init_done(totalpages, used); + + printk ("Memory hot-add operation completed successfully\n"); + printk ("Free pages after hot-add: %u\n\n", nr_free_pages()); +done: + current->flags = old_flags; +err: + up(&mem_hotadd_sem); + return ret; +} + +EXPORT_SYMBOL(hotadd_mem_init); +EXPORT_SYMBOL(mem_hotadd_count); diff -Naur -X dontdiff linux-2.4.20-ref/include/asm-i386/io.h linux-2.4.20/include/asm-i386/io.h --- linux-2.4.20-ref/include/asm-i386/io.h Mon Jun 9 14:03:55 2003 +++ linux-2.4.20/include/asm-i386/io.h Thu Jun 12 10:49:18 2003 @@ -99,11 +99,13 @@ /* * Change "struct page" to physical address. */ +#ifndef CONFIG_X86_MEM_HOTADD #ifdef CONFIG_HIGHMEM64G #define page_to_phys(page) ((u64)(page - mem_map) << PAGE_SHIFT) #else #define page_to_phys(page) ((page - mem_map) << PAGE_SHIFT) #endif +#endif extern void * __ioremap(unsigned long offset, unsigned long size, unsigned long flags); diff -Naur -X dontdiff linux-2.4.20-ref/include/asm-i386/mem_hotadd.h linux-2.4.20/include/asm-i386/mem_hotadd.h --- linux-2.4.20-ref/include/asm-i386/mem_hotadd.h Wed Dec 31 18:00:00 1969 +++ linux-2.4.20/include/asm-i386/mem_hotadd.h Thu Jun 12 10:49:00 2003 @@ -0,0 +1,161 @@ +/* + * linux/include/asm-i386/mem_hotadd.h + * (c) 2002 Hewlett-Packard Development Company, L.P. + */ + +#ifndef __ASM_MEM_HOTADD_H +#define __ASM_MEM_HOTADD_H + +/* + * Re-defines macros defined elsewhere to handle the case of multiple + * pg_data_t structures and memory maps + */ + + +extern int mem_hotadd_count; + +#ifdef CONFIG_X86_PAE +#define kaddr_t unsigned long long +#else +#define kaddr_t unsigned long +#endif + +#if 0 +/* + * These definitions are moved here from arch/i386/kernel/setup.c + * because of some dependencies in the hot-add code + */ +#define PFN_UP(x) (((x) + PAGE_SIZE-1) >> PAGE_SHIFT) +#define PFN_DOWN(x) ((x) >> PAGE_SHIFT) +#define PFN_PHYS(x) ((x) << PAGE_SHIFT) + +#ifndef __ASSEMBLY__ +extern unsigned long hotadd_reserve_size; +#endif +/* + * 128MB for vmalloc and initrd + */ +#define VMALLOC_RESERVE (unsigned long)(128 << 20) +#define MAXMEM (unsigned long)(-PAGE_OFFSET-VMALLOC_RESERVE-hotadd_reserve_size) +#define MAXMEM_PFN PFN_DOWN(MAXMEM) +#endif + +#define MAX_NONPAE_PFN (1 << 20) +#define MAX_PAE_PFN (1 << 23) + +/* + * Return a pointer to the pg_data_t structure corresponding to + * a given physical address + */ +static inline pg_data_t * __pfn_to_pgdat(unsigned long pfn) +{ + pg_data_t *temp; + + if (mem_hotadd_count == 0) + return (pgdat_list); + + temp = pgdat_list; + while (temp) { + if ((pfn >= temp->node_start_pfn) && + (pfn < (temp->node_start_pfn + temp->node_size))) + return temp; + temp = temp->node_next; + } + + return 0; +} + +/* + * Return a poniter to the pg_data_t structure correspondig to + * a page structure + */ +static inline pg_data_t * __page_to_pgdat(struct page *page) +{ + pg_data_t *temp; + + if ((mem_hotadd_count == 0) && ((page - pgdat_list->node_mem_map) < pgdat_list->node_size)) + return (pgdat_list); + + temp = pgdat_list; + while (temp) { + if ((page - temp->node_mem_map) < temp->node_size) + return (temp); + temp=temp->node_next; + } + return (pg_data_t *)0; +} + +#if 0 + +#define __page_to_pgdat(p) (page_zone(p)->zone_pgdat) +#define __page_to_mem_map(p) \ + ({pg_data_t *temp = (__page_to_pgdat(p)); \ + (temp) ? (temp)->node_mem_map : 0}) +#endif + +/* + * Return a pointer to the mem map array corresponding to + * a page structure + */ +static inline struct page * __page_to_mem_map(struct page *page) +{ + pg_data_t *temp = __page_to_pgdat(page); + if (temp) + return temp->node_mem_map; + else + return (struct page *)0; +} + +/* + * Return the page frame number of a given page structure + */ +static inline unsigned long __page_to_mapnr(struct page *page) +{ + unsigned long mapnr; + pg_data_t *temp = (page_zone(page)->zone_pgdat); + + mapnr = (page - temp->node_mem_map); + mapnr += temp->node_start_mapnr; + + return mapnr; +} + +static inline struct page * addr_to_page(kaddr_t addr) +{ + unsigned long pfn = (addr >> PAGE_SHIFT); + pg_data_t *temp = __pfn_to_pgdat(pfn); + + return (temp->node_mem_map + (pfn - temp->node_start_pfn)); +} + +/* + * Convert a "struct page" to a physical address + */ +static inline kaddr_t page_to_phys(struct page *page) +{ + kaddr_t addr; + pg_data_t *temp = (page_zone(page)->zone_pgdat); + addr = (page - temp->node_mem_map); + addr = (kaddr_t)((addr + temp->node_start_pfn) << PAGE_SHIFT); + + return addr; +} + +/* + * Return the page structure corresponding to a PTE + */ +static inline struct page * pte_page(pte_t pte) +{ + unsigned long pfn = (((kaddr_t)pte_val(pte)) >> PAGE_SHIFT); + pg_data_t *temp = __pfn_to_pgdat(pfn); + + return (temp->node_mem_map + (pfn - temp->node_start_pfn)); + +} + +#define VALID_PAGE(page) ((__page_to_mem_map(page) != 0)) + +#define mk_pte(page, pgprot) __mk_pte((__page_to_mapnr(page)), (pgprot)) + +extern int hotadd_mem_init(unsigned long long, unsigned long long, int); +#endif /* ASM_MEM_HOTADD_H */ diff -Naur -X dontdiff linux-2.4.20-ref/include/asm-i386/page.h linux-2.4.20/include/asm-i386/page.h --- linux-2.4.20-ref/include/asm-i386/page.h Mon Jun 9 14:03:34 2003 +++ linux-2.4.20/include/asm-i386/page.h Thu Jun 12 10:49:00 2003 @@ -127,12 +127,30 @@ #define PAGE_OFFSET ((unsigned long)__PAGE_OFFSET) #define VMALLOC_RESERVE ((unsigned long)__VMALLOC_RESERVE) + +#ifdef CONFIG_X86_MEM_HOTADD +#ifndef __ASSEMBLY__ +extern unsigned long hotadd_reserve_size; +#endif /* __ASSEMBLY__ */ + +#define __MAX_RDMEM (-__PAGE_OFFSET-__VMALLOC_RESERVE) +#define __MAXMEM (-__PAGE_OFFSET-__VMALLOC_RESERVE-hotadd_reserve_size) +#define MAXMEM ((unsigned long)(-PAGE_OFFSET-VMALLOC_RESERVE-hotadd_reserve_size)) + +#else /* !CONFIG_X86_MEM_HOTADD */ + #define __MAXMEM (-__PAGE_OFFSET-__VMALLOC_RESERVE) #define MAXMEM ((unsigned long)(-PAGE_OFFSET-VMALLOC_RESERVE)) +#endif /* CONFIG_X86_MEM_HOTADD */ + #define __pa(x) ((unsigned long)(x)-PAGE_OFFSET) #define __va(x) ((void *)((unsigned long)(x)+PAGE_OFFSET)) #define virt_to_page(kaddr) (mem_map + (__pa(kaddr) >> PAGE_SHIFT)) + +/* Defined in hotadd.h to handle multiple mem maps */ +#ifndef CONFIG_X86_MEM_HOTADD #define VALID_PAGE(page) ((page - mem_map) < max_mapnr) +#endif /* !CONFIG_X86_MEM_HOTADD */ #define VM_DATA_DEFAULT_FLAGS (VM_READ | VM_WRITE | VM_EXEC | \ VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC) diff -Naur -X dontdiff linux-2.4.20-ref/include/asm-i386/pci.h linux-2.4.20/include/asm-i386/pci.h --- linux-2.4.20-ref/include/asm-i386/pci.h Mon Jun 9 14:04:00 2003 +++ linux-2.4.20/include/asm-i386/pci.h Thu Jun 12 10:49:38 2003 @@ -34,6 +34,10 @@ #include #include +#ifdef CONFIG_X86_MEM_HOTADD +#include +#endif + struct pci_dev; /* The PCI address space does equal the physical memory @@ -103,7 +107,11 @@ if (direction == PCI_DMA_NONE) out_of_line_bug(); +#ifndef CONFIG_X86_MEM_HOTADD return (dma_addr_t)(page - mem_map) * PAGE_SIZE + offset; +#else + return ((dma_addr_t)page_to_phys(page)) + (dma_addr_t)offset; +#endif /* CONFIG_X86_MEM_HOTADD */ } static inline void pci_unmap_page(struct pci_dev *hwdev, dma_addr_t dma_address, @@ -237,6 +245,7 @@ (dma64_addr_t) offset); } +#ifndef CONFIG_X86_MEM_HOTADD static __inline__ struct page * pci_dac_dma_to_page(struct pci_dev *pdev, dma64_addr_t dma_addr) { @@ -244,6 +253,13 @@ return mem_map + poff; } +#else +static __inline__ struct page * +pci_dac_dma_to_page(struct pci_dev *pdev, dma64_addr_t dma_addr) +{ + return (addr_to_page(dma_addr)); +} +#endif static __inline__ unsigned long pci_dac_dma_to_offset(struct pci_dev *pdev, dma64_addr_t dma_addr) diff -Naur -X dontdiff linux-2.4.20-ref/include/asm-i386/pgtable-2level.h linux-2.4.20/include/asm-i386/pgtable-2level.h --- linux-2.4.20-ref/include/asm-i386/pgtable-2level.h Thu Nov 28 17:53:15 2002 +++ linux-2.4.20/include/asm-i386/pgtable-2level.h Thu Jun 12 10:49:00 2003 @@ -58,7 +58,11 @@ } #define ptep_get_and_clear(xp) __pte(xchg(&(xp)->pte_low, 0)) #define pte_same(a, b) ((a).pte_low == (b).pte_low) + +#ifndef CONFIG_X86_MEM_HOTADD #define pte_page(x) (mem_map+((unsigned long)(((x).pte_low >> PAGE_SHIFT)))) +#endif /* CONFIG_X86_MEM_HOTADD */ + #define pte_none(x) (!(x).pte_low) #define __mk_pte(page_nr,pgprot) __pte(((page_nr) << PAGE_SHIFT) | pgprot_val(pgprot)) diff -Naur -X dontdiff linux-2.4.20-ref/include/asm-i386/pgtable-3level.h linux-2.4.20/include/asm-i386/pgtable-3level.h --- linux-2.4.20-ref/include/asm-i386/pgtable-3level.h Thu Nov 28 17:53:15 2002 +++ linux-2.4.20/include/asm-i386/pgtable-3level.h Thu Jun 12 10:49:00 2003 @@ -89,7 +89,10 @@ return a.pte_low == b.pte_low && a.pte_high == b.pte_high; } +#ifndef CONFIG_X86_MEM_HOTADD #define pte_page(x) (mem_map+(((x).pte_low >> PAGE_SHIFT) | ((x).pte_high << (32 - PAGE_SHIFT)))) +#endif + #define pte_none(x) (!(x).pte_low && !(x).pte_high) static inline pte_t __mk_pte(unsigned long page_nr, pgprot_t pgprot) diff -Naur -X dontdiff linux-2.4.20-ref/include/asm-i386/pgtable.h linux-2.4.20/include/asm-i386/pgtable.h --- linux-2.4.20-ref/include/asm-i386/pgtable.h Mon Jun 9 14:03:34 2003 +++ linux-2.4.20/include/asm-i386/pgtable.h Thu Jun 12 10:49:00 2003 @@ -158,9 +158,18 @@ * area for the same reason. ;) */ #define VMALLOC_OFFSET (8*1024*1024) + +#ifdef CONFIG_X86_MEM_HOTADD +extern unsigned long hotadd_reserve_size; +#define VMALLOC_START (((unsigned long) high_memory + hotadd_reserve_size + \ + 2*VMALLOC_OFFSET-1) & ~(VMALLOC_OFFSET-1)) +#else /* CONFIG_X86_MEM_HOTADD */ #define VMALLOC_START (((unsigned long) high_memory + 2*VMALLOC_OFFSET-1) & \ ~(VMALLOC_OFFSET-1)) +#endif /* CONFIG_X86_MEM_HOTADD */ + #define VMALLOC_VMADDR(x) ((unsigned long)(x)) + #if CONFIG_HIGHMEM # define VMALLOC_END (PKMAP_BASE-2*PAGE_SIZE) #else @@ -271,6 +280,7 @@ #define pmd_present(x) (pmd_val(x) & _PAGE_PRESENT) #define pmd_clear(xp) do { set_pmd(xp, __pmd(0)); } while (0) #define pmd_bad(x) ((pmd_val(x) & (~PAGE_MASK & ~_PAGE_USER)) != _KERNPG_TABLE) +#define pmd_has_pse(x) (pmd_val(x) & _PAGE_PSE) #define pages_to_mb(x) ((x) >> (20-PAGE_SHIFT)) @@ -306,7 +316,9 @@ * and a page entry and page directory to the page they refer to. */ +#ifndef CONFIG_X86_MEM_HOTADD #define mk_pte(page, pgprot) __mk_pte((page) - mem_map, (pgprot)) +#endif /* CONFIG_X86_MEM_HOTADD */ /* This takes a physical page address that is used by the remapping functions */ #define mk_pte_phys(physpage, pgprot) __mk_pte((physpage) >> PAGE_SHIFT, pgprot) diff -Naur -X dontdiff linux-2.4.20-ref/include/linux/blkdev.h linux-2.4.20/include/linux/blkdev.h --- linux-2.4.20-ref/include/linux/blkdev.h Mon Jun 9 14:04:01 2003 +++ linux-2.4.20/include/linux/blkdev.h Thu Jun 12 10:49:25 2003 @@ -154,10 +154,10 @@ { struct page *page = bh->b_page; -#ifndef CONFIG_DISCONTIGMEM +#if (! defined CONFIG_DISCONTIGMEM) && (! defined CONFIG_X86_MEM_HOTADD) if (page - mem_map <= q->bounce_pfn) #else - if ((page - page_zone(page)->zone_mem_map) + (page_zone(page)->zone_start_paddr >> PAGE_SHIFT) <= q->bounce_pfn) + if ((page - page_zone(page)->zone_mem_map) + (page_zone(page)->zone_start_pfn) <= q->bounce_pfn) #endif return bh; diff -Naur -X dontdiff linux-2.4.20-ref/include/linux/mm.h linux-2.4.20/include/linux/mm.h --- linux-2.4.20-ref/include/linux/mm.h Mon Jun 9 14:03:37 2003 +++ linux-2.4.20/include/linux/mm.h Thu Jun 12 10:49:02 2003 @@ -336,6 +336,10 @@ page->flags |= zone_num << ZONE_SHIFT; } +#ifdef CONFIG_X86_MEM_HOTADD +#include +#endif + /* * In order to avoid #ifdefs within C code itself, we define * set_page_address to a noop for non-highmem machines, where @@ -364,8 +368,8 @@ #else /* CONFIG_HIGHMEM || WANT_PAGE_VIRTUAL */ #define page_address(page) \ - __va( (((page) - page_zone(page)->zone_mem_map) << PAGE_SHIFT) \ - + page_zone(page)->zone_start_paddr) + __va( ((page) - page_zone(page)->zone_mem_map) \ + + page_zone(page)->zone_start_pfn << PAGE_SHIFT) #endif /* CONFIG_HIGHMEM || WANT_PAGE_VIRTUAL */ diff -Naur -X dontdiff linux-2.4.20-ref/include/linux/mmzone.h linux-2.4.20/include/linux/mmzone.h --- linux-2.4.20-ref/include/linux/mmzone.h Mon Jun 9 14:03:34 2003 +++ linux-2.4.20/include/linux/mmzone.h Thu Jun 12 10:49:00 2003 @@ -82,7 +82,7 @@ */ struct pglist_data *zone_pgdat; struct page *zone_mem_map; - unsigned long zone_start_paddr; + unsigned long long zone_start_pfn; unsigned long zone_start_mapnr; /* @@ -133,7 +133,7 @@ struct page *node_mem_map; unsigned long *valid_addr_bitmap; struct bootmem_data *bdata; - unsigned long node_start_paddr; + unsigned long node_start_pfn; unsigned long node_start_mapnr; unsigned long node_size; int node_id; @@ -212,11 +212,33 @@ for(zone = pgdat_list->node_zones; zone; zone = next_zone(zone)) +static inline zone_t * next_node_zone(zone_t *zone) +{ + pg_data_t *pgdat = zone->zone_pgdat; + int index = zone - (pgdat->node_zones); + + if (pgdat->node_next) { + pgdat = pgdat->node_next; + zone = pgdat->node_zones + index; + } + else + zone = NULL; + return zone; +} + +#define for_each_node_zone(zone) \ + for(; zone; zone = next_node_zone(zone)) + #ifndef CONFIG_DISCONTIGMEM #define NODE_DATA(nid) (&contig_page_data) #define NODE_MEM_MAP(nid) mem_map + +#ifndef CONFIG_X86_MEM_HOTADD #define MAX_NR_NODES 1 +#else +#define MAX_NR_NODES (255 / MAX_NR_ZONES) +#endif #else /* !CONFIG_DISCONTIGMEM */ diff -Naur -X dontdiff linux-2.4.20-ref/mm/numa.c linux-2.4.20/mm/numa.c --- linux-2.4.20-ref/mm/numa.c Mon Sep 17 18:15:02 2001 +++ linux-2.4.20/mm/numa.c Wed Jun 11 17:07:33 2003 @@ -19,14 +19,15 @@ /* * This is meant to be invoked by platforms whose physical memory starts * at a considerably higher value than 0. Examples are Super-H, ARM, m68k. - * Should be invoked with paramters (0, 0, unsigned long *[], start_paddr). + * Should be invoked with paramters (0, 0, unsigned long *[], start_pfn). */ void __init free_area_init_node(int nid, pg_data_t *pgdat, struct page *pmap, unsigned long *zones_size, unsigned long zone_start_paddr, unsigned long *zholes_size) { free_area_init_core(0, &contig_page_data, &mem_map, zones_size, - zone_start_paddr, zholes_size, pmap); + (zone_start_paddr >> PAGE_SHIFT), zholes_size, + pmap); } #endif /* !CONFIG_DISCONTIGMEM */ @@ -68,8 +69,8 @@ if (mem_map == (mem_map_t *)NULL) mem_map = (mem_map_t *)PAGE_OFFSET; - free_area_init_core(nid, pgdat, &discard, zones_size, zone_start_paddr, - zholes_size, pmap); + free_area_init_core(nid, pgdat, &discard, zones_size, + (zone_start_paddr >> PAGE_SHIFT), zholes_size, pmap); pgdat->node_id = nid; /* diff -Naur -X dontdiff linux-2.4.20-ref/mm/page_alloc.c linux-2.4.20/mm/page_alloc.c --- linux-2.4.20-ref/mm/page_alloc.c Thu Nov 28 17:53:15 2002 +++ linux-2.4.20/mm/page_alloc.c Wed Jun 11 18:01:30 2003 @@ -28,6 +28,7 @@ LIST_HEAD(inactive_list); LIST_HEAD(active_list); pg_data_t *pgdat_list; +EXPORT_SYMBOL(pgdat_list); /* * @@ -39,17 +40,23 @@ EXPORT_SYMBOL(zone_table); static char *zone_names[MAX_NR_ZONES] = { "DMA", "Normal", "HighMem" }; -static int zone_balance_ratio[MAX_NR_ZONES] __initdata = { 128, 128, 128, }; -static int zone_balance_min[MAX_NR_ZONES] __initdata = { 20 , 20, 20, }; -static int zone_balance_max[MAX_NR_ZONES] __initdata = { 255 , 255, 255, }; +static int zone_balance_ratio[MAX_NR_ZONES] = { 128, 128, 128, }; +static int zone_balance_min[MAX_NR_ZONES] = { 20 , 20, 20, }; +static int zone_balance_max[MAX_NR_ZONES] = { 255 , 255, 255, }; + +#ifdef CONFIG_X86_MEM_HOTADD +#define MEM_MAP(x) (page_zone(x)->zone_pgdat->node_mem_map) +#else +#define MEM_MAP(x) (mem_map) +#endif /* * Temporary debugging check. */ #define BAD_RANGE(zone, page) \ ( \ - (((page) - mem_map) >= ((zone)->zone_start_mapnr+(zone)->size)) \ - || (((page) - mem_map) < (zone)->zone_start_mapnr) \ + (((page) - MEM_MAP(page)) >= ((zone)->zone_start_mapnr+(zone)->size))\ + || (((page) - MEM_MAP(page)) < (zone)->zone_start_mapnr) \ || ((zone) != page_zone(page)) \ ) @@ -338,19 +345,31 @@ min = 1UL << order; for (;;) { zone_t *z = *(zone++); + int min_incr = 1; if (!z) break; - min += z->pages_low; - if (z->free_pages > min) { - page = rmqueue(z, order); - if (page) - return page; + for_each_node_zone(z) { + if (!z->size) + continue; + if (min_incr) { + min += z->pages_low; + min_incr = 0; + } + if (z->free_pages > min) { + page = rmqueue(z, order); + if (page) + return page; + } } } - classzone->need_balance = 1; - mb(); + for_each_node_zone(classzone) { + if (classzone->size) { + classzone->need_balance = 1; + mb(); + } + } if (waitqueue_active(&kswapd_wait)) wake_up_interruptible(&kswapd_wait); @@ -358,18 +377,26 @@ min = 1UL << order; for (;;) { unsigned long local_min; + int min_incr = 1; zone_t *z = *(zone++); if (!z) break; - local_min = z->pages_min; - if (!(gfp_mask & __GFP_WAIT)) - local_min >>= 2; - min += local_min; - if (z->free_pages > min) { - page = rmqueue(z, order); - if (page) - return page; + for_each_node_zone(z) { + if (!z->size) + continue; + if (min_incr) { + local_min = z->pages_min; + if (!(gfp_mask & __GFP_WAIT)) + local_min >>= 2; + min += local_min; + min_incr = 0; + } + if (z->free_pages > min) { + page = rmqueue(z, order); + if (page) + return page; + } } } @@ -383,9 +410,13 @@ if (!z) break; - page = rmqueue(z, order); - if (page) - return page; + for_each_node_zone(z) { + if (!z->size) + continue; + page = rmqueue(z, order); + if (page) + return page; + } } return NULL; } @@ -394,22 +425,35 @@ if (!(gfp_mask & __GFP_WAIT)) return NULL; - page = balance_classzone(classzone, gfp_mask, order, &freed); - if (page) - return page; + classzone = *(zonelist->zones); + for_each_node_zone(classzone) { + if (!classzone->size) + continue; + page = balance_classzone(classzone, gfp_mask, order, &freed); + if (page) + return page; + } zone = zonelist->zones; min = 1UL << order; for (;;) { zone_t *z = *(zone++); + int min_incr = 1; if (!z) break; - min += z->pages_min; - if (z->free_pages > min) { - page = rmqueue(z, order); - if (page) - return page; + for_each_node_zone(z) { + if (!z->size) + continue; + if (min_incr) { + min += z->pages_min; + min_incr = 0; + } + if (z->free_pages > min) { + page = rmqueue(z, order); + if (page) + return page; + } } } @@ -612,12 +656,12 @@ */ case ZONE_HIGHMEM: zone = pgdat->node_zones + ZONE_HIGHMEM; - if (zone->size) { #ifndef CONFIG_HIGHMEM + if (zone->size) { BUG(); -#endif - zonelist->zones[j++] = zone; } +#endif + zonelist->zones[j++] = zone; case ZONE_NORMAL: zone = pgdat->node_zones + ZONE_NORMAL; if (zone->size) @@ -675,56 +719,23 @@ #define LONG_ALIGN(x) (((x)+(sizeof(long))-1)&~((sizeof(long))-1)) -/* - * Set up the zone data structures: - * - mark all pages reserved - * - mark all memory queues empty - * - clear the memory bitmaps - */ -void __init free_area_init_core(int nid, pg_data_t *pgdat, struct page **gmap, - unsigned long *zones_size, unsigned long zone_start_paddr, - unsigned long *zholes_size, struct page *lmem_map) + +int init_pgdat(int nid, pg_data_t *pgdat, unsigned long zone_start_pfn, + unsigned long totalpages, unsigned long *zones_size, + unsigned long *zholes_size, struct page *lmem_map, + int boot_flag) { unsigned long i, j; - unsigned long map_size; - unsigned long totalpages, offset, realtotalpages; + unsigned long offset; const unsigned long zone_required_alignment = 1UL << (MAX_ORDER-1); - if (zone_start_paddr & ~PAGE_MASK) - BUG(); - - totalpages = 0; - for (i = 0; i < MAX_NR_ZONES; i++) { - unsigned long size = zones_size[i]; - totalpages += size; - } - realtotalpages = totalpages; - if (zholes_size) - for (i = 0; i < MAX_NR_ZONES; i++) - realtotalpages -= zholes_size[i]; - - printk("On node %d totalpages: %lu\n", nid, realtotalpages); - - /* - * Some architectures (with lots of mem and discontinous memory - * maps) have to search for a good mem_map area: - * For discontigmem, the conceptual mem map array starts from - * PAGE_OFFSET, we need to align the actual array onto a mem map - * boundary, so that MAP_NR works. - */ - map_size = (totalpages + 1)*sizeof(struct page); - if (lmem_map == (struct page *)0) { - lmem_map = (struct page *) alloc_bootmem_node(pgdat, map_size); - lmem_map = (struct page *)(PAGE_OFFSET + - MAP_ALIGN((unsigned long)lmem_map - PAGE_OFFSET)); - } - *gmap = pgdat->node_mem_map = lmem_map; + pgdat->node_mem_map = lmem_map; pgdat->node_size = totalpages; - pgdat->node_start_paddr = zone_start_paddr; - pgdat->node_start_mapnr = (lmem_map - mem_map); + pgdat->node_start_pfn = zone_start_pfn; + pgdat->node_start_mapnr = zone_start_pfn; pgdat->nr_zones = 0; - offset = lmem_map - mem_map; + offset = 0; for (j = 0; j < MAX_NR_ZONES; j++) { zone_t *zone = pgdat->node_zones + j; unsigned long mask; @@ -752,9 +763,21 @@ zone->wait_table_size = wait_table_size(size); zone->wait_table_shift = BITS_PER_LONG - wait_table_bits(zone->wait_table_size); - zone->wait_table = (wait_queue_head_t *) - alloc_bootmem_node(pgdat, zone->wait_table_size + if (boot_flag) { + zone->wait_table = (wait_queue_head_t *) + alloc_bootmem_node(pgdat, zone->wait_table_size * sizeof(wait_queue_head_t)); + } + else { + zone->wait_table = (wait_queue_head_t *) + kmalloc(zone->wait_table_size * + sizeof(wait_queue_head_t),GFP_KERNEL); + if (zone->wait_table == NULL) + goto err_cleanup; + + memset(zone->wait_table, 0, zone->wait_table_size * + sizeof(wait_queue_head_t)); + } for(i = 0; i < zone->wait_table_size; ++i) init_waitqueue_head(zone->wait_table + i); @@ -770,11 +793,11 @@ zone->pages_low = mask*2; zone->pages_high = mask*3; - zone->zone_mem_map = mem_map + offset; + zone->zone_mem_map = lmem_map + offset; zone->zone_start_mapnr = offset; - zone->zone_start_paddr = zone_start_paddr; + zone->zone_start_pfn = zone_start_pfn; - if ((zone_start_paddr >> PAGE_SHIFT) & (zone_required_alignment-1)) + if ((zone_start_pfn) & (zone_required_alignment-1)) printk("BUG: wrong zone alignment, it will crash\n"); /* @@ -783,14 +806,16 @@ * done. Non-atomic initialization, single-pass. */ for (i = 0; i < size; i++) { - struct page *page = mem_map + offset + i; + struct page *page = lmem_map + offset + i; set_page_zone(page, nid * MAX_NR_ZONES + j); - set_page_count(page, 0); - SetPageReserved(page); + if (boot_flag) { + set_page_count(page, 0); + SetPageReserved(page); + } INIT_LIST_HEAD(&page->list); if (j != ZONE_HIGHMEM) - set_page_address(page, __va(zone_start_paddr)); - zone_start_paddr += PAGE_SIZE; + set_page_address(page, __va(zone_start_pfn << PAGE_SHIFT)); + zone_start_pfn++; } offset += size; @@ -828,11 +853,86 @@ */ bitmap_size = (size-1) >> (i+4); bitmap_size = LONG_ALIGN(bitmap_size+1); - zone->free_area[i].map = - (unsigned long *) alloc_bootmem_node(pgdat, bitmap_size); + if (boot_flag) { + zone->free_area[i].map = + (unsigned long *) alloc_bootmem_node(pgdat, bitmap_size); + } + else { + zone->free_area[i].map = + (unsigned long *)kmalloc(bitmap_size,GFP_KERNEL); + if (zone->free_area[i].map == NULL) + goto err_cleanup; + + memset(zone->free_area[i].map,0,bitmap_size); + } } } build_zonelists(pgdat); + + return 0; + +err_cleanup: + if (boot_flag) + BUG(); + printk ("init_pgdat: kmalloc failed. Cleaning up...\n"); + for (i=0; inode_zones + i; + if (! zone->size) + continue; + + if (zone->wait_table) + kfree(zone->wait_table); + for (j=0; j < MAX_ORDER; j++) { + if (zone->free_area[i].map) + kfree(zone->free_area[i].map); + } + } + + return -ENOMEM; +} + +/* + * Set up the zone data structures: + * - mark all pages reserved + * - mark all memory queues empty + * - clear the memory bitmaps + */ +void __init free_area_init_core(int nid, pg_data_t *pgdat, struct page **gmap, + unsigned long *zones_size, unsigned long zone_start_pfn, + unsigned long *zholes_size, struct page *lmem_map) +{ + unsigned long map_size; + unsigned long totalpages, realtotalpages; + int i; + + totalpages = 0; + for (i = 0; i < MAX_NR_ZONES; i++) { + unsigned long size = zones_size[i]; + totalpages += size; + } + realtotalpages = totalpages; + if (zholes_size) + for (i = 0; i < MAX_NR_ZONES; i++) + realtotalpages -= zholes_size[i]; + + printk("On node %d totalpages: %lu\n", nid, realtotalpages); + + /* + * Some architectures (with lots of mem and discontinous memory + * maps) have to search for a good mem_map area: + * For discontigmem, the conceptual mem map array starts from + * PAGE_OFFSET, we need to align the actual array onto a mem map + * boundary, so that MAP_NR works. + */ + map_size = (totalpages + 1)*sizeof(struct page); + if (lmem_map == (struct page *)0) { + lmem_map = (struct page *) alloc_bootmem_node(pgdat, map_size); + lmem_map = (struct page *)(PAGE_OFFSET + + MAP_ALIGN((unsigned long)lmem_map - PAGE_OFFSET)); + } + *gmap = lmem_map; + init_pgdat(0, pgdat, zone_start_pfn, totalpages, + zones_size, zholes_size, lmem_map, 1); } void __init free_area_init(unsigned long *zones_size)