From ded1f6ab43a03be74a53649cf388b5424d447584 Mon Sep 17 00:00:00 2001 From: Naga Chumbalkar Date: Fri, 8 Jul 2011 08:36:34 +0000 Subject: [PATCH 1/6] x86: print APIC data a little later during boot To view IOAPIC data you could boot with "apic=debug". When booting in such a way then the kernel will dump the IO-APIC's registers, for example: NR Dst Mask Trig IRR Pol Stat Dmod Deli Vect: 00 000 1 0 0 0 0 0 0 00 01 000 0 0 0 0 0 0 0 31 02 000 0 0 0 0 0 0 0 30 03 000 0 0 0 0 0 0 0 33 04 000 0 0 0 0 0 0 0 34 05 000 0 0 0 0 0 0 0 35 06 000 0 0 0 0 0 0 0 36 07 000 0 0 0 0 0 0 0 37 08 000 0 0 0 0 0 0 0 38 09 000 0 1 0 0 0 0 0 39 0a 000 0 0 0 0 0 0 0 3A 0b 000 0 0 0 0 0 0 0 3B 0c 000 0 0 0 0 0 0 0 3C 0d 000 0 0 0 0 0 0 0 3D 0e 000 0 0 0 0 0 0 0 3E 0f 000 0 0 0 0 0 0 0 3F 10 000 1 0 0 0 0 0 0 00 11 000 1 0 0 0 0 0 0 00 12 000 1 0 0 0 0 0 0 00 13 000 1 0 0 0 0 0 0 00 14 000 1 0 0 0 0 0 0 00 15 000 1 0 0 0 0 0 0 00 16 000 1 0 0 0 0 0 0 00 17 000 1 0 0 0 0 0 0 00 Delaying the call to print_ICs() gives better results: NR Dst Mask Trig IRR Pol Stat Dmod Deli Vect: 00 000 1 0 0 0 0 0 0 00 01 000 0 0 0 0 0 0 0 31 02 000 0 0 0 0 0 0 0 30 03 000 1 0 0 0 0 0 0 33 04 000 1 0 0 0 0 0 0 34 05 000 1 0 0 0 0 0 0 35 06 000 1 0 0 0 0 0 0 36 07 000 1 0 0 0 0 0 0 37 08 000 0 0 0 0 0 0 0 38 09 000 0 1 0 0 0 0 0 39 0a 000 1 0 0 0 0 0 0 3A 0b 000 1 0 0 0 0 0 0 3B 0c 000 0 0 0 0 0 0 0 3C 0d 000 1 0 0 0 0 0 0 3D 0e 000 1 0 0 0 0 0 0 3E 0f 000 1 0 0 0 0 0 0 3F 10 000 1 1 0 1 0 0 0 29 11 000 1 0 0 0 0 0 0 00 12 000 1 0 0 0 0 0 0 00 13 000 1 0 0 0 0 0 0 00 14 000 0 1 0 1 0 0 0 51 15 000 1 0 0 0 0 0 0 00 16 000 0 1 0 1 0 0 0 61 17 000 0 1 0 1 0 0 0 59 Notice that the entries beyond interrupt input signal 0x0f also get populated and arent just the hw-initialization default of all zeroes. Signed-off-by: Naga Chumbalkar Link: http://lkml.kernel.org/r/20110708083555.2598.42216.sendpatchset@nchumbalkar.americas.hpqcorp.net Signed-off-by: Ingo Molnar --- arch/x86/kernel/apic/io_apic.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c index e5293394b548..98c8d7e11c40 100644 --- a/arch/x86/kernel/apic/io_apic.c +++ b/arch/x86/kernel/apic/io_apic.c @@ -1792,7 +1792,7 @@ __apicdebuginit(int) print_ICs(void) return 0; } -fs_initcall(print_ICs); +late_initcall(print_ICs); /* Where if anywhere is the i8259 connect in external int mode */ From bd6a46e087571897f0b2736917500b97d18dac13 Mon Sep 17 00:00:00 2001 From: Naga Chumbalkar Date: Fri, 8 Jul 2011 18:46:36 +0000 Subject: [PATCH 2/6] x86, ioapic: Format clean up for IOAPIC output When IOAPIC data is displayed in "dmesg" with the help of the boot parameter "apic=debug" certain values are not formatted correctly wrt their size. In the "dmesg" snippet below, note that the output for "max redirection entries", and "IO APIC version" which are each defined to be just 8-bits long are displayed as 2 bytes in length. Similarly, "Dst" under the "IRQ redirection table" should only be 8-bits long. IO APIC #0...... ... ... .... register #01: 00170020 ....... : max redirection entries: 0017 ....... : PRQ implemented: 0 ....... : IO APIC version: 0020 ... ... .... IRQ redirection table: NR Dst Mask Trig IRR Pol Stat Dmod Deli Vect: 00 000 1 0 0 0 0 0 0 00 01 000 0 0 0 0 0 0 0 31 02 000 0 0 0 0 0 0 0 30 03 000 1 0 0 0 0 0 0 33 ... ... Do some formatting clean up, so you will see output like below: IO APIC #0...... ... ... .... register #01: 00170020 ....... : max redirection entries: 17 ....... : PRQ implemented: 0 ....... : IO APIC version: 20 ... ... .... IRQ redirection table: NR Dst Mask Trig IRR Pol Stat Dmod Deli Vect: 00 00 1 0 0 0 0 0 0 00 01 00 0 0 0 0 0 0 0 31 02 00 0 0 0 0 0 0 0 30 03 00 1 0 0 0 0 0 0 33 ... ... Signed-off-by: Naga Chumbalkar Link: http://lkml.kernel.org/r/20110708184557.2734.61830.sendpatchset@nchumbalkar.americas.cpqcorp.net Signed-off-by: Ingo Molnar --- arch/x86/kernel/apic/io_apic.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c index 98c8d7e11c40..ed4abaa301d0 100644 --- a/arch/x86/kernel/apic/io_apic.c +++ b/arch/x86/kernel/apic/io_apic.c @@ -1522,10 +1522,12 @@ __apicdebuginit(void) print_IO_APIC(void) printk(KERN_DEBUG "....... : LTS : %X\n", reg_00.bits.LTS); printk(KERN_DEBUG ".... register #01: %08X\n", *(int *)®_01); - printk(KERN_DEBUG "....... : max redirection entries: %04X\n", reg_01.bits.entries); + printk(KERN_DEBUG "....... : max redirection entries: %02X\n", + reg_01.bits.entries); printk(KERN_DEBUG "....... : PRQ implemented: %X\n", reg_01.bits.PRQ); - printk(KERN_DEBUG "....... : IO APIC version: %04X\n", reg_01.bits.version); + printk(KERN_DEBUG "....... : IO APIC version: %02X\n", + reg_01.bits.version); /* * Some Intel chipsets with IO APIC VERSION of 0x1? don't have reg_02, @@ -1558,7 +1560,7 @@ __apicdebuginit(void) print_IO_APIC(void) entry = ioapic_read_entry(apic, i); - printk(KERN_DEBUG " %02x %03X ", + printk(KERN_DEBUG " %02x %02X ", i, entry.dest ); From 7fece83235a59b15d75d6c8ef2225c24abd4505b Mon Sep 17 00:00:00 2001 From: Naga Chumbalkar Date: Fri, 8 Jul 2011 18:46:42 +0000 Subject: [PATCH 3/6] x86, ioapic: Also print Dest field The code in setup_ioapic_irq() determines the Destination Field, so why not also include it in the debug printk output that gets displayed when the boot parameter "apic=debug" is used. Before the change, "dmesg" will show: IOAPIC[0]: Set routing entry (8-1 -> 0x31 -> IRQ 1 Mode:0 Active:0) IOAPIC[0]: Set routing entry (8-2 -> 0x30 -> IRQ 0 Mode:0 Active:0) IOAPIC[0]: Set routing entry (8-3 -> 0x33 -> IRQ 3 Mode:0 Active:0) ... After the change, you will see: IOAPIC[0]: Set routing entry (8-1 -> 0x31 -> IRQ 1 Mode:0 Active:0 Dest:0) IOAPIC[0]: Set routing entry (8-2 -> 0x30 -> IRQ 0 Mode:0 Active:0 Dest:0) IOAPIC[0]: Set routing entry (8-3 -> 0x33 -> IRQ 3 Mode:0 Active:0 Dest:0) ... Signed-off-by: Naga Chumbalkar Link: http://lkml.kernel.org/r/20110708184603.2734.91071.sendpatchset@nchumbalkar.americas.cpqcorp.net Signed-off-by: Ingo Molnar --- arch/x86/kernel/apic/io_apic.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c index ed4abaa301d0..5cba200609c0 100644 --- a/arch/x86/kernel/apic/io_apic.c +++ b/arch/x86/kernel/apic/io_apic.c @@ -1337,9 +1337,9 @@ static void setup_ioapic_irq(int apic_id, int pin, unsigned int irq, apic_printk(APIC_VERBOSE,KERN_DEBUG "IOAPIC[%d]: Set routing entry (%d-%d -> 0x%x -> " - "IRQ %d Mode:%i Active:%i)\n", + "IRQ %d Mode:%i Active:%i Dest:%d)\n", apic_id, mpc_ioapic_id(apic_id), pin, cfg->vector, - irq, trigger, polarity); + irq, trigger, polarity, dest); if (setup_ioapic_entry(mpc_ioapic_id(apic_id), irq, &entry, From 25970852280c9d5fb2de899769880d3e97332baa Mon Sep 17 00:00:00 2001 From: Naga Chumbalkar Date: Tue, 12 Jul 2011 05:59:07 +0000 Subject: [PATCH 4/6] x86, x2apic: Preserve high 32-bits of IA32_APIC_BASE MSR If there's no special reason to zero-out the "high" 32-bits of the IA32_APIC_BASE MSR, let's preserve it. The x2APIC Specification doesn't explicitly state any such requirement. (Sec 2.2 in: http://www.intel.com/Assets/PDF/manual/318148.pdf). Signed-off-by: Naga Chumbalkar Link: http://lkml.kernel.org/r/20110712055831.2498.78521.sendpatchset@nchumbalkar.americas.cpqcorp.net Reviewed-by: Cyrill Gorcunov Reviewed-by: Suresh Siddha Signed-off-by: H. Peter Anvin --- arch/x86/kernel/apic/apic.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index b9338b8cf420..f7b0c7a1dfaa 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -1429,7 +1429,7 @@ void enable_x2apic(void) rdmsr(MSR_IA32_APICBASE, msr, msr2); if (!(msr & X2APIC_ENABLE)) { printk_once(KERN_INFO "Enabling x2apic\n"); - wrmsr(MSR_IA32_APICBASE, msr | X2APIC_ENABLE, 0); + wrmsr(MSR_IA32_APICBASE, msr | X2APIC_ENABLE, msr2); } } #endif /* CONFIG_X86_X2APIC */ From 3040db92ee1b6c5b6b6d73f8cdcad54c0da11563 Mon Sep 17 00:00:00 2001 From: Naga Chumbalkar Date: Tue, 12 Jul 2011 21:17:41 +0000 Subject: [PATCH 5/6] x86, ioapic: Print IRTE when IR is enabled When "apic=debug" is used as a boot parameter, Linux prints the IOAPIC routing entries in "dmesg". Below is output from IOAPIC whose apic_id is 8: # dmesg | grep "routing entry" IOAPIC[8]: Set routing entry (8-1 -> 0x31 -> IRQ 1 Mode:0 Active:0 Dest:0) IOAPIC[8]: Set routing entry (8-2 -> 0x30 -> IRQ 0 Mode:0 Active:0 Dest:0) IOAPIC[8]: Set routing entry (8-3 -> 0x33 -> IRQ 3 Mode:0 Active:0 Dest:0) ... Similarly, when IR (interrupt remapping) is enabled, and the IRTE (interrupt remapping table entry) is set up we should display it. After the fix: # dmesg | grep IRTE IOAPIC[8]: Set IRTE entry (P:1 FPD:0 Dst_Mode:0 Redir_hint:1 Trig_Mode:0 Dlvry_Mode:0 Avail:0 Vector:31 Dest:00000000 SID:00F1 SQ:0 SVT:1) IOAPIC[8]: Set IRTE entry (P:1 FPD:0 Dst_Mode:0 Redir_hint:1 Trig_Mode:0 Dlvry_Mode:0 Avail:0 Vector:30 Dest:00000000 SID:00F1 SQ:0 SVT:1) IOAPIC[8]: Set IRTE entry (P:1 FPD:0 Dst_Mode:0 Redir_hint:1 Trig_Mode:0 Dlvry_Mode:0 Avail:0 Vector:33 Dest:00000000 SID:00F1 SQ:0 SVT:1) ... The IRTE is defined in Sec 9.5 of the Intel VT-d Specification. Signed-off-by: Naga Chumbalkar Link: http://lkml.kernel.org/r/20110712211704.2939.71291.sendpatchset@nchumbalkar.americas.cpqcorp.net Signed-off-by: H. Peter Anvin --- arch/x86/kernel/apic/io_apic.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c index 5cba200609c0..1ed5cbb9a14a 100644 --- a/arch/x86/kernel/apic/io_apic.c +++ b/arch/x86/kernel/apic/io_apic.c @@ -1295,6 +1295,16 @@ static int setup_ioapic_entry(int apic_id, int irq, * irq handler will do the explicit EOI to the io-apic. */ ir_entry->vector = pin; + + apic_printk(APIC_VERBOSE, KERN_DEBUG "IOAPIC[%d]: " + "Set IRTE entry (P:%d FPD:%d Dst_Mode:%d " + "Redir_hint:%d Trig_Mode:%d Dlvry_Mode:%X " + "Avail:%X Vector:%02X Dest:%08X " + "SID:%04X SQ:%X SVT:%X)\n", + apic_id, irte.present, irte.fpd, irte.dst_mode, + irte.redir_hint, irte.trigger_mode, irte.dlvry_mode, + irte.avail, irte.vector, irte.dest_id, + irte.sid, irte.sq, irte.svt); } else { entry->delivery_mode = apic->irq_delivery_mode; entry->dest_mode = apic->irq_dest_mode; From 42f0efc5aae2bd7e3bc420c0902c7024ef77391f Mon Sep 17 00:00:00 2001 From: Naga Chumbalkar Date: Tue, 12 Jul 2011 21:17:35 +0000 Subject: [PATCH 6/6] x86, ioapic: Print IR_IO_APIC_route_entry when IR is enabled When IR (interrupt remapping) is enabled print_IO_APIC() displays output according to legacy RTE (redirection table entry) definitons: NR Dst Mask Trig IRR Pol Stat Dmod Deli Vect: 00 00 1 0 0 0 0 0 0 00 01 00 0 0 0 0 0 0 0 01 02 00 0 0 0 0 0 0 0 02 03 00 1 0 0 0 0 0 0 03 04 00 1 0 0 0 0 0 0 04 05 00 1 0 0 0 0 0 0 05 06 00 1 0 0 0 0 0 0 06 ... The above output is as per Sec 3.2.4 of the IOAPIC datasheet: 82093AA I/O Advanced Programmable Interrupt Controller (IOAPIC): http://download.intel.com/design/chipsets/datashts/29056601.pdf Instead the output should display the fields as discussed in Sec 5.5.1 of the VT-d specification: (Intel Virtualization Technology for Directed I/O: http://download.intel.com/technology/computing/vptech/Intel(r)_VT_for_Direct_IO.pdf) After the fix: NR Indx Fmt Mask Trig IRR Pol Stat Indx2 Zero Vect: 00 0000 0 1 0 0 0 0 0 0 00 01 000F 1 0 0 0 0 0 0 0 01 02 0001 1 0 0 0 0 0 0 0 02 03 0002 1 1 0 0 0 0 0 0 03 04 0011 1 1 0 0 0 0 0 0 04 05 0004 1 1 0 0 0 0 0 0 05 06 0005 1 1 0 0 0 0 0 0 06 ... Signed-off-by: Naga Chumbalkar Link: http://lkml.kernel.org/r/20110712211658.2939.93123.sendpatchset@nchumbalkar.americas.cpqcorp.net Signed-off-by: H. Peter Anvin --- arch/x86/kernel/apic/io_apic.c | 67 ++++++++++++++++++++++++---------- 1 file changed, 48 insertions(+), 19 deletions(-) diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c index 1ed5cbb9a14a..8eb863e27ea6 100644 --- a/arch/x86/kernel/apic/io_apic.c +++ b/arch/x86/kernel/apic/io_apic.c @@ -1562,31 +1562,60 @@ __apicdebuginit(void) print_IO_APIC(void) printk(KERN_DEBUG ".... IRQ redirection table:\n"); - printk(KERN_DEBUG " NR Dst Mask Trig IRR Pol" - " Stat Dmod Deli Vect:\n"); + if (intr_remapping_enabled) { + printk(KERN_DEBUG " NR Indx Fmt Mask Trig IRR" + " Pol Stat Indx2 Zero Vect:\n"); + } else { + printk(KERN_DEBUG " NR Dst Mask Trig IRR Pol" + " Stat Dmod Deli Vect:\n"); + } for (i = 0; i <= reg_01.bits.entries; i++) { - struct IO_APIC_route_entry entry; + if (intr_remapping_enabled) { + struct IO_APIC_route_entry entry; + struct IR_IO_APIC_route_entry *ir_entry; - entry = ioapic_read_entry(apic, i); + entry = ioapic_read_entry(apic, i); + ir_entry = (struct IR_IO_APIC_route_entry *) &entry; + printk(KERN_DEBUG " %02x %04X ", + i, + ir_entry->index + ); + printk("%1d %1d %1d %1d %1d " + "%1d %1d %X %02X\n", + ir_entry->format, + ir_entry->mask, + ir_entry->trigger, + ir_entry->irr, + ir_entry->polarity, + ir_entry->delivery_status, + ir_entry->index2, + ir_entry->zero, + ir_entry->vector + ); + } else { + struct IO_APIC_route_entry entry; - printk(KERN_DEBUG " %02x %02X ", - i, - entry.dest - ); - - printk("%1d %1d %1d %1d %1d %1d %1d %02X\n", - entry.mask, - entry.trigger, - entry.irr, - entry.polarity, - entry.delivery_status, - entry.dest_mode, - entry.delivery_mode, - entry.vector - ); + entry = ioapic_read_entry(apic, i); + printk(KERN_DEBUG " %02x %02X ", + i, + entry.dest + ); + printk("%1d %1d %1d %1d %1d " + "%1d %1d %02X\n", + entry.mask, + entry.trigger, + entry.irr, + entry.polarity, + entry.delivery_status, + entry.dest_mode, + entry.delivery_mode, + entry.vector + ); + } } } + printk(KERN_DEBUG "IRQ to pin mappings:\n"); for_each_active_irq(irq) { struct irq_pin_list *entry;