From: Jeremy Fitzhardinge - fix typo - add a self-pointer to the PDA, so that finding its linear address is easy - add type checking to PDA write operations - add byte-sized read/writes to the PDA Signed-off-by: Jeremy Fitzhardinge Cc: Andi Kleen Signed-off-by: Andrew Morton --- arch/i386/kernel/cpu/common.c | 109 ++++++++++++++++---------------- arch/i386/kernel/head.S | 31 +++++++-- arch/i386/kernel/smpboot.c | 37 ++++++---- include/asm-i386/current.h | 7 -- include/asm-i386/pda.h | 20 +++++ include/asm-i386/processor.h | 4 - include/asm-i386/smp.h | 4 - 7 files changed, 128 insertions(+), 84 deletions(-) diff -puN arch/i386/kernel/cpu/common.c~x86-remaining-pda-patches arch/i386/kernel/cpu/common.c --- a/arch/i386/kernel/cpu/common.c~x86-remaining-pda-patches +++ a/arch/i386/kernel/cpu/common.c @@ -13,7 +13,6 @@ #include #include #include -#include #ifdef CONFIG_X86_LOCAL_APIC #include #include @@ -596,7 +595,7 @@ struct pt_regs * __devinit idle_regs(str return regs; } -__cpuinit int alloc_gdt(int cpu) +static __cpuinit int alloc_gdt(int cpu) { struct Xgt_desc_struct *cpu_gdt_descr = &per_cpu(cpu_gdt_descr, cpu); struct desc_struct *gdt; @@ -642,17 +641,12 @@ __cpuinit int alloc_gdt(int cpu) return 1; } -static __cpuinit void pda_init(int cpu, struct task_struct *curr) -{ - struct i386_pda *pda = cpu_pda(cpu); - - memset(pda, 0, sizeof(*pda)); - - pda->cpu_number = cpu; - pda->pcurrent = curr; - - printk("cpu %d current %p\n", cpu, curr); -} +/* Initial PDA used by boot CPU */ +struct i386_pda boot_pda = { + ._pda = &boot_pda, + .cpu_number = 0, + .pcurrent = &init_task, +}; static inline void set_kernel_gs(void) { @@ -662,11 +656,10 @@ static inline void set_kernel_gs(void) asm volatile ("mov %0, %%gs" : : "r" (__KERNEL_PDA) : "memory"); } -/* Initialize the CPU's GDT and PDA */ -static __cpuinit void init_gdt(void) +/* Initialize the CPU's GDT and PDA. The boot CPU does this for + itself, but secondaries find this done for them. */ +__cpuinit int init_gdt(int cpu, struct task_struct *idle) { - int cpu = early_smp_processor_id(); - struct task_struct *curr = early_current(); struct Xgt_desc_struct *cpu_gdt_descr = &per_cpu(cpu_gdt_descr, cpu); __u32 stk16_off = (__u32)&per_cpu(cpu_16bit_stack, cpu); struct desc_struct *gdt; @@ -676,8 +669,7 @@ static __cpuinit void init_gdt(void) allocated. */ if (!alloc_gdt(cpu)) { printk(KERN_CRIT "CPU%d failed to allocate GDT or PDA\n", cpu); - for (;;) - local_irq_enable(); + return 0; } gdt = (struct desc_struct *)cpu_gdt_descr->address; @@ -703,53 +695,33 @@ static __cpuinit void init_gdt(void) (unsigned long)pda, sizeof(*pda) - 1, 0x80 | DESCTYPE_S | 0x2, 0); /* present read-write data segment */ - load_gdt(cpu_gdt_descr); - set_kernel_gs(); - - /* Do this once everything GDT-related has been set up. */ - pda_init(cpu, curr); -} - -/* Set up a very early PDA for the boot CPU so that smp_processor_id() - and current will work. */ -void __init smp_setup_processor_id(void) -{ - static __initdata struct i386_pda boot_pda; - pack_descriptor((u32 *)&cpu_gdt_table[GDT_ENTRY_PDA].a, - (u32 *)&cpu_gdt_table[GDT_ENTRY_PDA].b, - (unsigned long)&boot_pda, sizeof(struct i386_pda) - 1, - 0x80 | DESCTYPE_S | 0x2, 0); /* present read-write data segment */ - - boot_pda.pcurrent = early_current(); + memset(pda, 0, sizeof(*pda)); + pda->_pda = pda; + pda->cpu_number = cpu; + pda->pcurrent = idle; - /* Set %gs for this CPU's PDA */ - set_kernel_gs(); + return 1; } -/* - * cpu_init() initializes state that is per-CPU. Some data is already - * initialized (naturally) in the bootstrap process, such as the GDT - * and IDT. We reload them nevertheless, this function acts as a - * 'CPU state barrier', nothing should get across. - */ -void __cpuinit cpu_init(void) +/* Common CPU init for both boot and secondary CPUs */ +static void __cpuinit _cpu_init(int cpu, struct task_struct *curr) { - int cpu = early_smp_processor_id(); - struct task_struct *curr = early_current(); - struct tss_struct * t = &per_cpu(init_tss, cpu); struct thread_struct *thread = &curr->thread; + struct Xgt_desc_struct *cpu_gdt_descr = &per_cpu(cpu_gdt_descr, cpu); + + /* Reinit these anyway, even if they've already been done (on + the boot CPU, this will transition from the boot gdt+pda to + the real ones). */ + load_gdt(cpu_gdt_descr); + set_kernel_gs(); if (cpu_test_and_set(cpu, cpu_initialized)) { printk(KERN_WARNING "CPU#%d already initialized!\n", cpu); for (;;) local_irq_enable(); } - /* Init the GDT and PDA early, before calling printk(), - since it may end up using the PDA indirectly. */ - init_gdt(); - printk(KERN_INFO "Initializing CPU#%d\n", cpu); if (cpu_has_vme || cpu_has_tsc || cpu_has_de) @@ -801,6 +773,37 @@ void __cpuinit cpu_init(void) mxcsr_feature_mask_init(); } +/* Entrypoint to initialize secondary CPU */ +void __cpuinit secondary_cpu_init(void) +{ + int cpu = smp_processor_id(); + struct task_struct *curr = current; + + _cpu_init(cpu, curr); +} + +/* + * cpu_init() initializes state that is per-CPU. Some data is already + * initialized (naturally) in the bootstrap process, such as the GDT + * and IDT. We reload them nevertheless, this function acts as a + * 'CPU state barrier', nothing should get across. + */ +void __cpuinit cpu_init(void) +{ + int cpu = smp_processor_id(); + struct task_struct *curr = current; + + /* Set up the real GDT and PDA, so we can transition from the + boot versions. */ + if (!init_gdt(cpu, curr)) { + /* failed to allocate something; not much we can do... */ + for (;;) + local_irq_enable(); + } + + _cpu_init(cpu, curr); +} + #ifdef CONFIG_HOTPLUG_CPU void __cpuinit cpu_uninit(void) { diff -puN arch/i386/kernel/head.S~x86-remaining-pda-patches arch/i386/kernel/head.S --- a/arch/i386/kernel/head.S~x86-remaining-pda-patches +++ a/arch/i386/kernel/head.S @@ -302,6 +302,7 @@ is386: movl $2,%ecx # set MP movl %eax,%cr0 call check_x87 + call setup_pda lgdt cpu_gdt_descr lidt idt_descr ljmp $(__KERNEL_CS),$1f @@ -312,10 +313,13 @@ is386: movl $2,%ecx # set MP movl %eax,%ds movl %eax,%es - xorl %eax,%eax # Clear FS/GS and LDT + xorl %eax,%eax # Clear FS and LDT movl %eax,%fs - movl %eax,%gs lldt %ax + + movl $(__KERNEL_PDA),%eax + mov %eax,%gs + cld # gcc2 wants the direction flag cleared at all times pushl %eax # fake return address #ifdef CONFIG_SMP @@ -346,6 +350,23 @@ check_x87: ret /* + * Point the GDT at this CPU's PDA. On boot this will be + * cpu_gdt_table and boot_pda; for secondary CPUs, these will be + * that CPU's GDT and PDA. + */ +setup_pda: + /* get the PDA pointer */ + movl start_pda, %eax + + /* slot the PDA address into the GDT */ + mov cpu_gdt_descr+2, %ecx + mov %ax, (__KERNEL_PDA+0+2)(%ecx) /* base & 0x0000ffff */ + shr $16, %eax + mov %al, (__KERNEL_PDA+4+0)(%ecx) /* base & 0x00ff0000 */ + mov %ah, (__KERNEL_PDA+4+3)(%ecx) /* base & 0xff000000 */ + ret + +/* * setup_idt * * sets up a idt with 256 entries pointing to @@ -484,6 +505,8 @@ ENTRY(empty_zero_page) * This starts the data section. */ .data +ENTRY(start_pda) + .long boot_pda ENTRY(stack_start) .long init_thread_union+THREAD_SIZE @@ -525,7 +548,7 @@ idt_descr: # boot GDT descriptor (later on used by CPU#0): .word 0 # 32 bit align gdt_desc.address -cpu_gdt_descr: +ENTRY(cpu_gdt_descr) .word GDT_ENTRIES*8-1 .long cpu_gdt_table @@ -585,7 +608,7 @@ ENTRY(cpu_gdt_table) .quad 0x004092000000ffff /* 0xc8 APM DS data */ .quad 0x0000920000000000 /* 0xd0 - ESPFIX 16-bit SS */ - .quad 0x0000000000000000 /* 0xd8 - PDA */ + .quad 0x00cf92000000ffff /* 0xd8 - PDA */ .quad 0x0000000000000000 /* 0xe0 - unused */ .quad 0x0000000000000000 /* 0xe8 - unused */ .quad 0x0000000000000000 /* 0xf0 - unused */ diff -puN arch/i386/kernel/smpboot.c~x86-remaining-pda-patches arch/i386/kernel/smpboot.c --- a/arch/i386/kernel/smpboot.c~x86-remaining-pda-patches +++ a/arch/i386/kernel/smpboot.c @@ -534,11 +534,11 @@ set_cpu_sibling_map(int cpu) static void __devinit start_secondary(void *unused) { /* - * Don't put *anything* before cpu_init(), SMP + * Don't put *anything* before secondary_cpu_init(), SMP * booting is too fragile that we want to limit the * things done here to the most necessary things. */ - cpu_init(); + secondary_cpu_init(); preempt_disable(); smp_callin(); while (!cpu_isset(smp_processor_id(), smp_commenced_mask)) @@ -588,8 +588,6 @@ static void __devinit start_secondary(vo */ void __devinit initialize_secondary(void) { - struct task_struct *curr = early_current(); - /* * We don't actually need to load the full TSS, * basically just the stack pointer and the eip. @@ -599,13 +597,16 @@ void __devinit initialize_secondary(void "movl %0,%%esp\n\t" "jmp *%1" : - :"r" (curr->thread.esp),"r" (curr->thread.eip)); + :"m" (current->thread.esp),"m" (current->thread.eip)); } +/* Static state in head.S used to set up a CPU */ extern struct { void * esp; unsigned short ss; } stack_start; +extern struct i386_pda *start_pda; +extern struct Xgt_desc_struct cpu_gdt_descr; #ifdef CONFIG_NUMA @@ -931,17 +932,6 @@ static int __devinit do_boot_cpu(int api unsigned long start_eip; unsigned short nmi_high = 0, nmi_low = 0; - /* Pre-allocate the CPU's GDT and PDA so it doesn't have to do - any memory allocation during the delicate CPU-bringup - phase. */ - if (!alloc_gdt(cpu)) { - printk(KERN_INFO "Couldn't allocate GDT/PDA for CPU %d\n", cpu); - return -1; /* ? */ - } - - ++cpucount; - alternatives_smp_switch(1); - /* * We can't use kernel_thread since we must avoid to * reschedule the child. @@ -949,15 +939,30 @@ static int __devinit do_boot_cpu(int api idle = alloc_idle_task(cpu); if (IS_ERR(idle)) panic("failed fork for CPU %d", cpu); + + /* Pre-allocate and initialize the CPU's GDT and PDA so it + doesn't have to do any memory allocation during the + delicate CPU-bringup phase. */ + if (!init_gdt(cpu, idle)) { + printk(KERN_INFO "Couldn't allocate GDT/PDA for CPU %d\n", cpu); + return -1; /* ? */ + } + idle->thread.eip = (unsigned long) start_secondary; /* start_eip had better be page-aligned! */ start_eip = setup_trampoline(); + ++cpucount; + alternatives_smp_switch(1); + /* So we see what's up */ printk("Booting processor %d/%d eip %lx\n", cpu, apicid, start_eip); /* Stack for startup_32 can be just as for start_secondary onwards */ stack_start.esp = (void *) idle->thread.esp; + start_pda = cpu_pda(cpu); + cpu_gdt_descr = per_cpu(cpu_gdt_descr, cpu); + irq_ctx_init(cpu); /* diff -puN include/asm-i386/current.h~x86-remaining-pda-patches include/asm-i386/current.h --- a/include/asm-i386/current.h~x86-remaining-pda-patches +++ a/include/asm-i386/current.h @@ -1,16 +1,11 @@ #ifndef _I386_CURRENT_H #define _I386_CURRENT_H -#include #include +#include struct task_struct; -static __always_inline struct task_struct *early_current(void) -{ - return current_thread_info()->task; -} - static __always_inline struct task_struct *get_current(void) { return read_pda(pcurrent); diff -puN include/asm-i386/pda.h~x86-remaining-pda-patches include/asm-i386/pda.h --- a/include/asm-i386/pda.h~x86-remaining-pda-patches +++ a/include/asm-i386/pda.h @@ -1,8 +1,12 @@ #ifndef _I386_PDA_H #define _I386_PDA_H +#include + struct i386_pda { + struct i386_pda *_pda; /* pointer to self */ + struct task_struct *pcurrent; /* current process */ int cpu_number; }; @@ -22,6 +26,12 @@ extern struct i386_pda _proxy_pda; typedef typeof(_proxy_pda.field) T__; \ if (0) { T__ tmp__; tmp__ = (val); } \ switch (sizeof(_proxy_pda.field)) { \ + case 1: \ + asm(op "b %1,%%gs:%c2" \ + : "+m" (_proxy_pda.field) \ + :"ri" ((T__)val), \ + "i"(pda_offset(field))); \ + break; \ case 2: \ asm(op "w %1,%%gs:%c2" \ : "+m" (_proxy_pda.field) \ @@ -42,6 +52,12 @@ extern struct i386_pda _proxy_pda; ({ \ typeof(_proxy_pda.field) ret__; \ switch (sizeof(_proxy_pda.field)) { \ + case 1: \ + asm(op "b %%gs:%c1,%0" \ + : "=r" (ret__) \ + : "i" (pda_offset(field)), \ + "m" (_proxy_pda.field)); \ + break; \ case 2: \ asm(op "w %%gs:%c1,%0" \ : "=r" (ret__) \ @@ -58,6 +74,10 @@ extern struct i386_pda _proxy_pda; } \ ret__; }) +/* Return a pointer to a pda field */ +#define pda_addr(field) \ + ((typeof(_proxy_pda.field) *)((unsigned char *)read_pda(_pda) + \ + pda_offset(field))) #define read_pda(field) pda_from_op("mov",field) #define write_pda(field,val) pda_to_op("mov",field,val) diff -puN include/asm-i386/processor.h~x86-remaining-pda-patches include/asm-i386/processor.h --- a/include/asm-i386/processor.h~x86-remaining-pda-patches +++ a/include/asm-i386/processor.h @@ -734,6 +734,8 @@ extern void select_idle_routine(const st extern unsigned long boot_option_idle_override; extern void enable_sep_cpu(void); extern int sysenter_setup(void); -extern int alloc_gdt(int cpu); + +extern int init_gdt(int cpu, struct task_struct *idle); +extern void secondary_cpu_init(void); #endif /* __ASM_I386_PROCESSOR_H */ diff -puN include/asm-i386/smp.h~x86-remaining-pda-patches include/asm-i386/smp.h --- a/include/asm-i386/smp.h~x86-remaining-pda-patches +++ a/include/asm-i386/smp.h @@ -58,9 +58,6 @@ extern void cpu_uninit(void); * so this is correct in the x86 case. */ #define raw_smp_processor_id() (read_pda(cpu_number)) -/* This is valid from the very earliest point in boot that we care - about. */ -#define early_smp_processor_id() (current_thread_info()->cpu) extern cpumask_t cpu_callout_map; extern cpumask_t cpu_callin_map; @@ -95,7 +92,6 @@ extern unsigned int num_processors; #else /* CONFIG_SMP */ #define cpu_physical_id(cpu) boot_cpu_physical_apicid -#define early_smp_processor_id() 0 #define NO_PROC_ID 0xFF /* No processor magic marker */ _