Linux 多核启动过程
以这篇博文来纪念自己与“Linux kernel多核启动”相处的两个多月。
本文章以2.6.33.1的linux内核在x86_64平台上为例进行说明。
本文参考了http://tldp.org/HOWTO/Linux-i386-Boot-Code-HOWTO/smpboot.html
Linux kernel启动的过程概览
init/main.c:start_kernel()
|
\|/
init/main.c:rest_init
{
……
kernel_thread(kernel_init, NULL, CLONES_FS | CLONE_SIGHAND)
……
cpu_idle()
}
|
\|/
init/main.c:kernel_init//从上面代码可以看出，kernel_init是一个内核线程
|
\|/
init/main.c:init_post //会在最后调用启动脚本
{
……
823 /*
824 * We try each of these until one succeeds.
825 *
826 * The Bourne shell can be used instead of init if we are
827 * trying to recover a really broken machine.
828 */
829 if (execute_command) {
830 run_init_process(execute_command);
831 printk(KERN_WARNING "Failed to execute %s. Attempting "
832 "defaults...\n", execute_command);
833 }
834 run_init_process("/sbin/init");
835 run_init_process("/etc/init");
836 run_init_process("/bin/init");
837 run_init_process("/bin/sh");
838
839 panic("No init found. Try passing init= option to kernel.");
……
}
我们再来看看内核启动多核的详细过程。
init/main.c:start_kernel()
|
\|/
init/main.c:rest_init
{
……
kernel_thread(kernel_init, NULL, CLONES_FS | CLONE_SIGHAND)
……
}
|
\|/
kernel_init
|
\|/
/* called by boot processor to activate the rest */
init/main.c: smp_init()
{
……
for_each_present_cpu(cpu) {
if (num_onlien_cpus() >= setup_max_cpus)
break;
if ( !cpu_online(cpu))
cpu_up(cpu);
}
/* Any cleanup work */
printk(KERN_INFO "Brought up %ld CPUs\n", (long)num_online_cpus());
smp_cpu_done(setup_max_cpus);
……
}
--------------------------
cpu_up = native_cpu_up是一个回调函数。
注册地方是在：arch/x86/kernel/smp.c
struct smp_ops smp_ops = {
……
.cpu_up = native_cpu_up,
……
}
--------------------------
|
\|/
arch/x86/kernel/smpboot.c:native_cpu_up(unsigned int cpu)
|
\|/
arch/x86/kernel/smpboot.c: do_boot_cpu(int apicid, int cpu)
|
\|/
wakeup_secondary_cpu_via_init(apicid, start_ip)
在启动多核的过程中有两个bitmap很重要，一个是cpu_callin_mask，另一个是cpu_callout_mask。
cpu_callin_mask代表某个cpu是否已经启动，它的某个bit被与之对应的cpu在启动后置位，标记已经启动。
cpu_callout_mask在do_boot_cpu中被置位，并在检查到对应cpu已经启动后重新清零。
我们下面来详细看看do_boot_cpu(int apicid, int cpu)与wakeup_secondary_cpu_via_init(apicid, start_ip)
/* * Activate a secondary processor. */notrace static void __cpuinit start_secondary(void *unused){/* * Don't put *anything* before cpu_init(), SMP booting is too * fragile that we want to limit the things done here to the * most necessary things. */vmi_bringup();cpu_init();preempt_disable();smp_callin();/* otherwise gcc will move up smp_processor_id before the cpu_init */barrier();/* * Check TSC synchronization with the BP: */check_tsc_sync_target();if (nmi_watchdog == NMI_IO_APIC) {disable_8259A_irq(0);enable_NMI_through_LVT0();enable_8259A_irq(0);}#ifdef CONFIG_X86_32while (low_mappings)cpu_relax();__flush_tlb_all();#endif/* This must be done before setting cpu_online_mask */set_cpu_sibling_map(raw_smp_processor_id());wmb();/* * We need to hold call_lock, so there is no inconsistency * between the time smp_call_function() determines number of * IPI recipients, and the time when the determination is made * for which cpus receive the IPI. Holding this * lock helps us to not include this cpu in a currently in progress * smp_call_function(). * * We need to hold vector_lock so there the set of online cpus * does not change while we are assigning vectors to cpus. Holding * this lock ensures we don't half assign or remove an irq from a cpu. */ipi_call_lock();lock_vector_lock();__setup_vector_irq(smp_processor_id());set_cpu_online(smp_processor_id(), true);unlock_vector_lock();ipi_call_unlock();per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;/* enable local interrupts */local_irq_enable();x86_cpuinit.setup_percpu_clockev();wmb();cpu_idle();}
