Jan Beulich
2013-Aug-22 11:42 UTC
[PATCH] x86: move struct bug_frame instances out of line
Just like Linux did many years ago, move them into a separate (data)
section, such that they no longer pollute instruction caches and TLBs.
Assertion frames, requiring two pointers to be stored, occupy two slots
in the array, with the second slot mimicking a frame the location
pointer of which doesn''t match any address within .text or .init.text
(it effectively points back to the slot itself, which - being in a data
section - can''t be reached by non-buggy execution).
Signed-off-by: Jan Beulich <jbeulich@suse.com>
--- a/xen/arch/x86/efi/stub.c
+++ b/xen/arch/x86/efi/stub.c
@@ -1,7 +1,7 @@
#include <xen/efi.h>
#include <xen/errno.h>
#include <xen/init.h>
-#include <asm/bug.h>
+#include <xen/lib.h>
#ifndef efi_enabled
const bool_t efi_enabled = 0;
--- a/xen/arch/x86/traps.c
+++ b/xen/arch/x86/traps.c
@@ -952,11 +952,18 @@ static int emulate_forced_invalid_op(str
void do_invalid_op(struct cpu_user_regs *regs)
{
- struct bug_frame bug;
- struct bug_frame_str bug_str;
- const char *p, *filename, *predicate, *eip = (char *)regs->eip;
+ const struct bug_frame *bug;
+ u8 bug_insn[2];
+ const char *filename, *predicate, *eip = (char *)regs->eip;
unsigned long fixup;
int id, lineno;
+ static const struct bug_frame *const stop_frames[] = {
+ __stop_bug_frames_0,
+ __stop_bug_frames_1,
+ __stop_bug_frames_2,
+ __stop_bug_frames_3,
+ NULL
+ };
DEBUGGER_trap_entry(TRAP_invalid_op, regs);
@@ -968,70 +975,65 @@ void do_invalid_op(struct cpu_user_regs
return;
}
- if ( !is_kernel(eip) ||
- __copy_from_user(&bug, eip, sizeof(bug)) ||
- memcmp(bug.ud2, "\xf\xb", sizeof(bug.ud2)) ||
- (bug.ret != 0xc2) )
+ if ( (!is_kernel_text(eip) &&
+ (system_state > SYS_STATE_boot || !is_kernel_inittext(eip))) ||
+ __copy_from_user(bug_insn, eip, sizeof(bug_insn)) ||
+ memcmp(bug_insn, "\xf\xb", sizeof(bug_insn)) )
goto die;
- eip += sizeof(bug);
- /* Decode first pointer argument. */
- if ( !is_kernel(eip) ||
- __copy_from_user(&bug_str, eip, sizeof(bug_str)) ||
- (bug_str.mov != 0xbc) )
- goto die;
- p = bug_str(bug_str, eip);
- if ( !is_kernel(p) )
+ for ( bug = __start_bug_frames, id = 0; stop_frames[id]; ++bug )
+ {
+ while ( unlikely(bug == stop_frames[id]) )
+ ++id;
+ if ( bug_loc(bug) == eip )
+ break;
+ }
+ if ( !stop_frames[id] )
goto die;
- eip += sizeof(bug_str);
-
- id = bug.id & 3;
+ eip += sizeof(bug_insn);
if ( id == BUGFRAME_run_fn )
{
- void (*fn)(struct cpu_user_regs *) = (void *)p;
- (*fn)(regs);
+ void (*fn)(struct cpu_user_regs *) = bug_ptr(bug);
+
+ fn(regs);
regs->eip = (unsigned long)eip;
return;
}
/* WARN, BUG or ASSERT: decode the filename pointer and line number. */
- filename = p;
- lineno = bug.id >> 2;
+ filename = bug_ptr(bug);
+ if ( !is_kernel(filename) )
+ goto die;
+ lineno = bug_line(bug);
- if ( id == BUGFRAME_warn )
+ switch ( id )
{
+ case BUGFRAME_warn:
printk("Xen WARN at %.50s:%d\n", filename, lineno);
show_execution_state(regs);
regs->eip = (unsigned long)eip;
return;
- }
- if ( id == BUGFRAME_bug )
- {
+ case BUGFRAME_bug:
printk("Xen BUG at %.50s:%d\n", filename, lineno);
DEBUGGER_trap_fatal(TRAP_invalid_op, regs);
show_execution_state(regs);
panic("Xen BUG at %.50s:%d\n", filename, lineno);
- }
- /* ASSERT: decode the predicate string pointer. */
- ASSERT(id == BUGFRAME_assert);
- if ( !is_kernel(eip) ||
- __copy_from_user(&bug_str, eip, sizeof(bug_str)) ||
- (bug_str.mov != 0xbc) )
- goto die;
- predicate = bug_str(bug_str, eip);
- eip += sizeof(bug_str);
+ case BUGFRAME_assert:
+ /* ASSERT: decode the predicate string pointer. */
+ predicate = bug_msg(bug);
+ if ( !is_kernel(predicate) )
+ predicate = "<unknown>";
- if ( !is_kernel(predicate) )
- predicate = "<unknown>";
- printk("Assertion ''%s'' failed at %.50s:%d\n",
- predicate, filename, lineno);
- DEBUGGER_trap_fatal(TRAP_invalid_op, regs);
- show_execution_state(regs);
- panic("Assertion ''%s'' failed at %.50s:%d\n",
- predicate, filename, lineno);
+ printk("Assertion ''%s'' failed at
%.50s:%d\n",
+ predicate, filename, lineno);
+ DEBUGGER_trap_fatal(TRAP_invalid_op, regs);
+ show_execution_state(regs);
+ panic("Assertion ''%s'' failed at
%.50s:%d\n",
+ predicate, filename, lineno);
+ }
die:
if ( (fixup = search_exception_table(regs->eip)) != 0 )
--- a/xen/include/asm-x86/bug.h
+++ b/xen/include/asm-x86/bug.h
@@ -1,42 +1,60 @@
#ifndef __X86_BUG_H__
#define __X86_BUG_H__
-#include <asm/x86_64/bug.h>
+#define BUG_DISP_WIDTH 24
+#define BUG_LINE_LO_WIDTH (31 - BUG_DISP_WIDTH)
+#define BUG_LINE_HI_WIDTH (31 - BUG_DISP_WIDTH)
struct bug_frame {
- unsigned char ud2[2];
- unsigned char ret;
- unsigned short id; /* BUGFRAME_??? */
-} __attribute__((packed));
+ signed int loc_disp:BUG_DISP_WIDTH;
+ unsigned int line_hi:BUG_LINE_HI_WIDTH;
+ signed int ptr_disp:BUG_DISP_WIDTH;
+ unsigned int line_lo:BUG_LINE_LO_WIDTH;
+ signed int msg_disp[];
+};
+
+#define bug_loc(b) ((const void *)(b) + (b)->loc_disp)
+#define bug_ptr(b) ((const void *)(b) + (b)->ptr_disp)
+#define bug_line(b) ((((b)->line_hi + ((b)->loc_disp < 0)) <<
\
+ BUG_LINE_LO_WIDTH) + \
+ (b)->line_lo + ((b)->ptr_disp < 0))
+#define bug_msg(b) ((const char *)(b) + (b)->msg_disp[1])
#define BUGFRAME_run_fn 0
#define BUGFRAME_warn 1
#define BUGFRAME_bug 2
#define BUGFRAME_assert 3
-#define run_in_exception_handler(fn) \
- asm volatile ( \
- "ud2 ; ret %0" BUG_STR(1) \
- : : "i" (BUGFRAME_run_fn), \
- "i" (&(fn)) )
-
-#define WARN() \
- asm volatile ( \
- "ud2 ; ret %0" BUG_STR(1) \
- : : "i" (BUGFRAME_warn | (__LINE__<<2)), \
- "i" (__FILE__) )
-
-#define BUG() \
- asm volatile ( \
- "ud2 ; ret %0" BUG_STR(1) \
- : : "i" (BUGFRAME_bug | (__LINE__<<2)), \
- "i" (__FILE__) )
-
-#define assert_failed(p) \
- asm volatile ( \
- "ud2 ; ret %0" BUG_STR(1) BUG_STR(2) \
- : : "i" (BUGFRAME_assert | (__LINE__<<2)), \
- "i" (__FILE__), "i" (p) )
-
+#define BUG_FRAME(type, line, ptr, msg) do { \
+ BUILD_BUG_ON((line) >> (BUG_LINE_LO_WIDTH + BUG_LINE_HI_WIDTH));
\
+ asm volatile ( ".Lbug%=: ud2\n"
\
+ ".pushsection .bug_frames.%c0, \"a\",
@progbits\n" \
+ ".p2align 2\n"
\
+ ".Lfrm%=:\n"
\
+ ".long (.Lbug%= - .Lfrm%=) + %c4\n"
\
+ ".long (%c1 - .Lfrm%=) + %c3\n"
\
+ ".ifnes \"" msg "\",
\"\"\n" \
+ ".long 0, %c2 - .Lfrm%=\n"
\
+ ".endif\n"
\
+ ".popsection"
\
+ : \
+ : "i" (type), "i" (ptr), "i"
(msg), \
+ "i" ((line & ((1 << BUG_LINE_LO_WIDTH)
- 1)) \
+ << BUG_DISP_WIDTH),
\
+ "i" (((line) >> BUG_LINE_LO_WIDTH)
<< BUG_DISP_WIDTH)); \
+} while (0)
+
+#define WARN() BUG_FRAME(BUGFRAME_warn, __LINE__, __FILE__, "")
+#define BUG() BUG_FRAME(BUGFRAME_bug, __LINE__, __FILE__, "")
+
+#define run_in_exception_handler(fn) BUG_FRAME(BUGFRAME_run_fn, 0, fn,
"")
+
+#define assert_failed(msg) BUG_FRAME(BUGFRAME_assert, __LINE__, __FILE__, msg)
+
+extern const struct bug_frame __start_bug_frames[],
+ __stop_bug_frames_0[],
+ __stop_bug_frames_1[],
+ __stop_bug_frames_2[],
+ __stop_bug_frames_3[];
#endif /* __X86_BUG_H__ */
--- a/xen/include/asm-x86/x86_64/bug.h
+++ /dev/null
@@ -1,11 +0,0 @@
-#ifndef __X86_64_BUG_H__
-#define __X86_64_BUG_H__
-
-struct bug_frame_str {
- unsigned char mov;
- signed int str_disp;
-} __attribute__((packed));
-#define bug_str(b, rip) ((const char *)(rip) + (b).str_disp)
-#define BUG_STR(n) "; movl %" #n " - ., %%esp"
-
-#endif /* __X86_64_BUG_H__ */
--- a/xen/arch/x86/xen.lds.S
+++ b/xen/arch/x86/xen.lds.S
@@ -51,6 +51,18 @@ SECTIONS
} :text = 0x9090
.rodata : {
+ /* Bug frames table */
+ . = ALIGN(4);
+ __start_bug_frames = .;
+ *(.bug_frames.0)
+ __stop_bug_frames_0 = .;
+ *(.bug_frames.1)
+ __stop_bug_frames_1 = .;
+ *(.bug_frames.2)
+ __stop_bug_frames_2 = .;
+ *(.bug_frames.3)
+ __stop_bug_frames_3 = .;
+
*(.rodata)
*(.rodata.*)
} :text
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel
Keir Fraser
2013-Aug-22 18:46 UTC
Re: [PATCH] x86: move struct bug_frame instances out of line
On 22/08/2013 12:42, "Jan Beulich" <JBeulich@suse.com> wrote:> Just like Linux did many years ago, move them into a separate (data) > section, such that they no longer pollute instruction caches and TLBs. > > Assertion frames, requiring two pointers to be stored, occupy two slots > in the array, with the second slot mimicking a frame the location > pointer of which doesn''t match any address within .text or .init.text > (it effectively points back to the slot itself, which - being in a data > section - can''t be reached by non-buggy execution). > > Signed-off-by: Jan Beulich <jbeulich@suse.com>Acked-by: Keir Fraser <keir@xen.org>