/******************************************************************************
 * arch/x86/traps.c
 * 
 * Modifications to Linux original are copyright (c) 2002-2004, K A Fraser
 * 
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 * 
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 * 
 * You should have received a copy of the GNU General Public License
 * along with this program; If not, see <http://www.gnu.org/licenses/>.
 */

/*
 *  Copyright (C) 1991, 1992  Linus Torvalds
 *
 *  Pentium III FXSR, SSE support
 * Gareth Hughes <gareth@valinux.com>, May 2000
 */

#include <xen/init.h>
#include <xen/sched.h>
#include <xen/lib.h>
#include <xen/err.h>
#include <xen/errno.h>
#include <xen/mm.h>
#include <xen/console.h>
#include <xen/shutdown.h>
#include <xen/guest_access.h>
#include <asm/regs.h>
#include <xen/delay.h>
#include <xen/event.h>
#include <xen/spinlock.h>
#include <xen/irq.h>
#include <xen/perfc.h>
#include <xen/softirq.h>
#include <xen/domain_page.h>
#include <xen/symbols.h>
#include <xen/iocap.h>
#include <xen/nmi.h>
#include <xen/version.h>
#include <xen/kexec.h>
#include <xen/trace.h>
#include <xen/paging.h>
#include <xen/virtual_region.h>
#include <xen/watchdog.h>
#include <xen/livepatch.h>
#include <asm/system.h>
#include <asm/io.h>
#include <asm/atomic.h>
#include <xen/bitops.h>
#include <asm/desc.h>
#include <asm/debugreg.h>
#include <asm/smp.h>
#include <asm/flushtlb.h>
#include <asm/uaccess.h>
#include <asm/i387.h>
#include <asm/xstate.h>
#include <asm/debugger.h>
#include <asm/msr.h>
#include <asm/shared.h>
#include <asm/x86_emulate.h>
#include <asm/traps.h>
#include <asm/hvm/vpt.h>
#include <asm/hypercall.h>
#include <asm/mce.h>
#include <asm/apic.h>
#include <asm/mc146818rtc.h>
#include <asm/hpet.h>
#include <asm/vpmu.h>
#include <public/arch-x86/cpuid.h>
#include <asm/cpuid.h>
#include <xsm/xsm.h>

/*
 * opt_nmi: one of 'ignore', 'dom0', or 'fatal'.
 *  fatal:  Xen prints diagnostic message and then hangs.
 *  dom0:   The NMI is virtualised to DOM0.
 *  ignore: The NMI error is cleared and ignored.
 */
#ifdef NDEBUG
static char __read_mostly opt_nmi[10] = "dom0";
#else
static char __read_mostly opt_nmi[10] = "fatal";
#endif
string_param("nmi", opt_nmi);

DEFINE_PER_CPU(u64, efer);
static DEFINE_PER_CPU(unsigned long, last_extable_addr);

DEFINE_PER_CPU_READ_MOSTLY(u32, ler_msr);

DEFINE_PER_CPU_READ_MOSTLY(struct desc_struct *, gdt_table);
DEFINE_PER_CPU_READ_MOSTLY(struct desc_struct *, compat_gdt_table);

/* Master table, used by CPU0. */
idt_entry_t __section(".bss.page_aligned") __aligned(PAGE_SIZE)
    idt_table[IDT_ENTRIES];

/* Pointer to the IDT of every CPU. */
idt_entry_t *idt_tables[NR_CPUS] __read_mostly;

void (*ioemul_handle_quirk)(
    u8 opcode, char *io_emul_stub, struct cpu_user_regs *regs);

static int debug_stack_lines = 20;
integer_param("debug_stack_lines", debug_stack_lines);

static bool_t opt_ler;
boolean_param("ler", opt_ler);

#define stack_words_per_line 4
#define ESP_BEFORE_EXCEPTION(regs) ((unsigned long *)regs->rsp)

static void show_code(const struct cpu_user_regs *regs)
{
    unsigned char insns_before[8] = {}, insns_after[16] = {};
    unsigned int i, tmp, missing_before, missing_after;

    if ( guest_mode(regs) )
        return;

    stac();

    /*
     * Copy forward from regs->rip.  In the case of a fault, %ecx contains the
     * number of bytes remaining to copy.
     */
    asm volatile ("1: rep movsb; 2:"
                  _ASM_EXTABLE(1b, 2b)
                  : "=&c" (missing_after),
                    "=&D" (tmp), "=&S" (tmp)
                  : "0" (ARRAY_SIZE(insns_after)),
                    "1" (insns_after),
                    "2" (regs->rip));

    /*
     * Copy backwards from regs->rip - 1.  In the case of a fault, %ecx
     * contains the number of bytes remaining to copy.
     */
    asm volatile ("std;"
                  "1: rep movsb;"
                  "2: cld;"
                  _ASM_EXTABLE(1b, 2b)
                  : "=&c" (missing_before),
                    "=&D" (tmp), "=&S" (tmp)
                  : "0" (ARRAY_SIZE(insns_before)),
                    "1" (insns_before + ARRAY_SIZE(insns_before) - 1),
                    "2" (regs->rip - 1));
    clac();

    printk("Xen code around <%p> (%ps)%s:\n",
           _p(regs->rip), _p(regs->rip),
           (missing_before || missing_after) ? " [fault on access]" : "");

    /* Print bytes from insns_before[]. */
    for ( i = 0; i < ARRAY_SIZE(insns_before); ++i )
    {
        if ( i < missing_before )
            printk(" --");
        else
            printk(" %02x", insns_before[i]);
    }

    /* Print the byte under %rip. */
    if ( missing_after != ARRAY_SIZE(insns_after) )
        printk(" <%02x>", insns_after[0]);
    else
        printk(" <-->");

    /* Print bytes from insns_after[]. */
    for ( i = 1; i < ARRAY_SIZE(insns_after); ++i )
    {
        if ( i < (ARRAY_SIZE(insns_after) - missing_after) )
            printk(" %02x", insns_after[i]);
        else
            printk(" --");
    }

    printk("\n");
}

static void show_guest_stack(struct vcpu *v, const struct cpu_user_regs *regs)
{
    int i;
    unsigned long *stack, addr;
    unsigned long mask = STACK_SIZE;

    /* Avoid HVM as we don't know what the stack looks like. */
    if ( is_hvm_vcpu(v) )
        return;

    if ( is_pv_32bit_vcpu(v) )
    {
        compat_show_guest_stack(v, regs, debug_stack_lines);
        return;
    }

    stack = (unsigned long *)regs->rsp;
    printk("Guest stack trace from "__OP"sp=%p:\n  ", stack);

    if ( !access_ok(stack, sizeof(*stack)) )
    {
        printk("Guest-inaccessible memory.\n");
        return;
    }

    if ( v != current )
    {
        struct vcpu *vcpu;

        ASSERT(guest_kernel_mode(v, regs));
        vcpu = maddr_get_owner(read_cr3()) == v->domain ? v : NULL;
        if ( !vcpu )
        {
            stack = do_page_walk(v, (unsigned long)stack);
            if ( (unsigned long)stack < PAGE_SIZE )
            {
                printk("Inaccessible guest memory.\n");
                return;
            }
            mask = PAGE_SIZE;
        }
    }

    for ( i = 0; i < (debug_stack_lines*stack_words_per_line); i++ )
    {
        if ( (((long)stack - 1) ^ ((long)(stack + 1) - 1)) & mask )
            break;
        if ( __get_user(addr, stack) )
        {
            if ( i != 0 )
                printk("\n    ");
            printk("Fault while accessing guest memory.");
            i = 1;
            break;
        }
        if ( (i != 0) && ((i % stack_words_per_line) == 0) )
            printk("\n  ");
        printk(" %p", _p(addr));
        stack++;
    }
    if ( mask == PAGE_SIZE )
    {
        BUILD_BUG_ON(PAGE_SIZE == STACK_SIZE);
        unmap_domain_page(stack);
    }
    if ( i == 0 )
        printk("Stack empty.");
    printk("\n");
}

/*
 * Notes for get_stack_trace_bottom() and get_stack_dump_bottom()
 *
 * Stack pages 0, 1 and 2:
 *   These are all 1-page IST stacks.  Each of these stacks have an exception
 *   frame and saved register state at the top.  The interesting bound for a
 *   trace is the word adjacent to this, while the bound for a dump is the
 *   very top, including the exception frame.
 *
 * Stack pages 3, 4 and 5:
 *   None of these are particularly interesting.  With MEMORY_GUARD, page 5 is
 *   explicitly not present, so attempting to dump or trace it is
 *   counterproductive.  Without MEMORY_GUARD, it is possible for a call chain
 *   to use the entire primary stack and wander into page 5.  In this case,
 *   consider these pages an extension of the primary stack to aid debugging
 *   hopefully rare situations where the primary stack has effective been
 *   overflown.
 *
 * Stack pages 6 and 7:
 *   These form the primary stack, and have a cpu_info at the top.  For a
 *   trace, the interesting bound is adjacent to the cpu_info, while for a
 *   dump, the entire cpu_info is interesting.
 *
 * For the cases where the stack should not be inspected, pretend that the
 * passed stack pointer is already out of reasonable bounds.
 */
unsigned long get_stack_trace_bottom(unsigned long sp)
{
    switch ( get_stack_page(sp) )
    {
    case 0 ... 2:
        return ROUNDUP(sp, PAGE_SIZE) -
            offsetof(struct cpu_user_regs, es) - sizeof(unsigned long);

#ifndef MEMORY_GUARD
    case 3 ... 5:
#endif
    case 6 ... 7:
        return ROUNDUP(sp, STACK_SIZE) -
            sizeof(struct cpu_info) - sizeof(unsigned long);

    default:
        return sp - sizeof(unsigned long);
    }
}

unsigned long get_stack_dump_bottom(unsigned long sp)
{
    switch ( get_stack_page(sp) )
    {
    case 0 ... 2:
        return ROUNDUP(sp, PAGE_SIZE) - sizeof(unsigned long);

#ifndef MEMORY_GUARD
    case 3 ... 5:
#endif
    case 6 ... 7:
        return ROUNDUP(sp, STACK_SIZE) - sizeof(unsigned long);

    default:
        return sp - sizeof(unsigned long);
    }
}

#if !defined(CONFIG_FRAME_POINTER)

/*
 * Stack trace from pointers found in stack, unaided by frame pointers.  For
 * caller convenience, this has the same prototype as its alternative, and
 * simply ignores the base pointer parameter.
 */
static void _show_trace(unsigned long sp, unsigned long __maybe_unused bp)
{
    unsigned long *stack = (unsigned long *)sp, addr;
    unsigned long *bottom = (unsigned long *)get_stack_trace_bottom(sp);

    while ( stack <= bottom )
    {
        addr = *stack++;
        if ( is_active_kernel_text(addr) )
            printk("   [<%p>] %pS\n", _p(addr), _p(addr));
    }
}

#else

/* Stack trace from frames in the stack, using frame pointers */
static void _show_trace(unsigned long sp, unsigned long bp)
{
    unsigned long *frame, next, addr;

    /* Bounds for range of valid frame pointer. */
    unsigned long low = sp, high = get_stack_trace_bottom(sp);

    /* The initial frame pointer. */
    next = bp;

    for ( ; ; )
    {
        /* Valid frame pointer? */
        if ( (next < low) || (next >= high) )
        {
            /*
             * Exception stack frames have a different layout, denoted by an
             * inverted frame pointer.
             */
            next = ~next;
            if ( (next < low) || (next >= high) )
                break;
            frame = (unsigned long *)next;
            next  = frame[0];
            addr  = frame[(offsetof(struct cpu_user_regs, rip) -
                           offsetof(struct cpu_user_regs, rbp))
                         / BYTES_PER_LONG];
        }
        else
        {
            /* Ordinary stack frame. */
            frame = (unsigned long *)next;
            next  = frame[0];
            addr  = frame[1];
        }

        printk("   [<%p>] %pS\n", _p(addr), _p(addr));

        low = (unsigned long)&frame[2];
    }
}

#endif

static void show_trace(const struct cpu_user_regs *regs)
{
    unsigned long *sp = ESP_BEFORE_EXCEPTION(regs);

    printk("Xen call trace:\n");

    /*
     * If RIP looks sensible, or the top of the stack doesn't, print RIP at
     * the top of the stack trace.
     */
    if ( is_active_kernel_text(regs->rip) ||
         !is_active_kernel_text(*sp) )
        printk("   [<%p>] %pS\n", _p(regs->rip), _p(regs->rip));
    /*
     * Else RIP looks bad but the top of the stack looks good.  Perhaps we
     * followed a wild function pointer? Lets assume the top of the stack is a
     * return address; print it and skip past so _show_trace() doesn't print
     * it again.
     */
    else
    {
        printk("   [<%p>] %pS\n", _p(*sp), _p(*sp));
        sp++;
    }

    _show_trace((unsigned long)sp, regs->rbp);

    printk("\n");
}

void show_stack(const struct cpu_user_regs *regs)
{
    unsigned long *stack = ESP_BEFORE_EXCEPTION(regs), *stack_bottom, addr;
    int i;

    if ( guest_mode(regs) )
        return show_guest_stack(current, regs);

    printk("Xen stack trace from "__OP"sp=%p:\n  ", stack);

    stack_bottom = _p(get_stack_dump_bottom(regs->rsp));

    for ( i = 0; i < (debug_stack_lines*stack_words_per_line) &&
              (stack <= stack_bottom); i++ )
    {
        if ( (i != 0) && ((i % stack_words_per_line) == 0) )
            printk("\n  ");
        addr = *stack++;
        printk(" %p", _p(addr));
    }
    if ( i == 0 )
        printk("Stack empty.");
    printk("\n");

    show_trace(regs);
}

void show_stack_overflow(unsigned int cpu, const struct cpu_user_regs *regs)
{
    unsigned long esp = regs->rsp;
    unsigned long curr_stack_base = esp & ~(STACK_SIZE - 1);
#ifdef MEMORY_GUARD
    unsigned long esp_top, esp_bottom;
#endif

    if ( _p(curr_stack_base) != stack_base[cpu] )
        printk("Current stack base %p differs from expected %p\n",
               _p(curr_stack_base), stack_base[cpu]);

#ifdef MEMORY_GUARD
    esp_bottom = (esp | (STACK_SIZE - 1)) + 1;
    esp_top    = esp_bottom - PRIMARY_STACK_SIZE;

    printk("Valid stack range: %p-%p, sp=%p, tss.rsp0=%p\n",
           (void *)esp_top, (void *)esp_bottom, (void *)esp,
           (void *)per_cpu(init_tss, cpu).rsp0);

    /*
     * Trigger overflow trace if %esp is anywhere within the guard page, or
     * with fewer than 512 bytes remaining on the primary stack.
     */
    if ( (esp > (esp_top + 512)) ||
         (esp < (esp_top - PAGE_SIZE)) )
    {
        printk("No stack overflow detected. Skipping stack trace.\n");
        return;
    }

    if ( esp < esp_top )
        esp = esp_top;

    printk("Xen stack overflow (dumping trace %p-%p):\n",
           (void *)esp, (void *)esp_bottom);

    _show_trace(esp, regs->rbp);

    printk("\n");
#endif
}

void show_execution_state(const struct cpu_user_regs *regs)
{
    /* Prevent interleaving of output. */
    unsigned long flags = console_lock_recursive_irqsave();

    show_registers(regs);
    show_code(regs);
    show_stack(regs);

    console_unlock_recursive_irqrestore(flags);
}

void vcpu_show_execution_state(struct vcpu *v)
{
    unsigned long flags;

    printk("*** Dumping Dom%d vcpu#%d state: ***\n",
           v->domain->domain_id, v->vcpu_id);

    if ( v == current )
    {
        show_execution_state(guest_cpu_user_regs());
        return;
    }

    vcpu_pause(v); /* acceptably dangerous */

    /* Prevent interleaving of output. */
    flags = console_lock_recursive_irqsave();

    vcpu_show_registers(v);
    if ( guest_kernel_mode(v, &v->arch.user_regs) )
        show_guest_stack(v, &v->arch.user_regs);

    console_unlock_recursive_irqrestore(flags);

    vcpu_unpause(v);
}

static cpumask_t show_state_mask;
static bool_t opt_show_all;
boolean_param("async-show-all", opt_show_all);

static int nmi_show_execution_state(const struct cpu_user_regs *regs, int cpu)
{
    if ( !cpumask_test_cpu(cpu, &show_state_mask) )
        return 0;

    if ( opt_show_all )
        show_execution_state(regs);
    else
        printk(XENLOG_ERR "CPU%d @ %04x:%08lx (%pS)\n", cpu, regs->cs, regs->rip,
               guest_mode(regs) ? NULL : _p(regs->rip));
    cpumask_clear_cpu(cpu, &show_state_mask);

    return 1;
}

static const char *trapstr(unsigned int trapnr)
{
    static const char * const strings[] = {
        "divide error", "debug", "nmi", "bkpt", "overflow", "bounds",
        "invalid opcode", "device not available", "double fault",
        "coprocessor segment", "invalid tss", "segment not found",
        "stack error", "general protection fault", "page fault",
        "spurious interrupt", "coprocessor error", "alignment check",
        "machine check", "simd error", "virtualisation exception"
    };

    return trapnr < ARRAY_SIZE(strings) ? strings[trapnr] : "???";
}

/*
 * This is called for faults at very unexpected times (e.g., when interrupts
 * are disabled). In such situations we can't do much that is safe. We try to
 * print out some tracing and then we just spin.
 */
void fatal_trap(const struct cpu_user_regs *regs, bool_t show_remote)
{
    static DEFINE_PER_CPU(char, depth);
    unsigned int trapnr = regs->entry_vector;

    /* Set AC to reduce chance of further SMAP faults */
    stac();

    /*
     * In some cases, we can end up in a vicious cycle of fatal_trap()s
     * within fatal_trap()s. We give the problem a couple of iterations to
     * bottom out, and then we just panic.
     */
    if ( ++this_cpu(depth) < 3 )
    {
        watchdog_disable();
        console_start_sync();

        show_execution_state(regs);

        if ( trapnr == TRAP_page_fault )
        {
            unsigned long cr2 = read_cr2();
            printk("Faulting linear address: %p\n", _p(cr2));
            show_page_walk(cr2);
        }

        if ( show_remote )
        {
            unsigned int msecs, pending;

            cpumask_andnot(&show_state_mask, &cpu_online_map,
                           cpumask_of(smp_processor_id()));
            set_nmi_callback(nmi_show_execution_state);
            /* Ensure new callback is set before sending out the NMI. */
            smp_wmb();
            smp_send_nmi_allbutself();

            /* Wait at most 10ms for some other CPU to respond. */
            msecs = 10;
            pending = cpumask_weight(&show_state_mask);
            while ( pending && msecs-- )
            {
                unsigned int left;

                mdelay(1);
                left = cpumask_weight(&show_state_mask);
                if ( left < pending )
                {
                    pending = left;
                    msecs = 10;
                }
            }
        }
    }

    panic("FATAL TRAP: vector = %d (%s)\n"
          "[error_code=%04x] %s",
          trapnr, trapstr(trapnr), regs->error_code,
          (regs->eflags & X86_EFLAGS_IF) ? "" : ", IN INTERRUPT CONTEXT");
}

void pv_inject_event(const struct x86_event *event)
{
    struct vcpu *v = current;
    struct cpu_user_regs *regs = guest_cpu_user_regs();
    struct trap_bounce *tb;
    const struct trap_info *ti;
    const uint8_t vector = event->vector;
    unsigned int error_code = event->error_code;
    bool use_error_code;

    ASSERT(vector == event->vector); /* Confirm no truncation. */
    if ( event->type == X86_EVENTTYPE_HW_EXCEPTION )
    {
        ASSERT(vector < 32);
        use_error_code = TRAP_HAVE_EC & (1u << vector);
    }
    else
    {
        ASSERT(event->type == X86_EVENTTYPE_SW_INTERRUPT);
        use_error_code = false;
    }
    if ( use_error_code )
        ASSERT(error_code != X86_EVENT_NO_EC);
    else
        ASSERT(error_code == X86_EVENT_NO_EC);

    tb = &v->arch.pv_vcpu.trap_bounce;
    ti = &v->arch.pv_vcpu.trap_ctxt[vector];

    tb->flags = TBF_EXCEPTION;
    tb->cs    = ti->cs;
    tb->eip   = ti->address;

    if ( event->type == X86_EVENTTYPE_HW_EXCEPTION &&
         vector == TRAP_page_fault )
    {
        v->arch.pv_vcpu.ctrlreg[2] = event->cr2;
        arch_set_cr2(v, event->cr2);

        /* Re-set error_code.user flag appropriately for the guest. */
        error_code &= ~PFEC_user_mode;
        if ( !guest_kernel_mode(v, regs) )
            error_code |= PFEC_user_mode;

        trace_pv_page_fault(event->cr2, error_code);
    }
    else
        trace_pv_trap(vector, regs->rip, use_error_code, error_code);

    if ( use_error_code )
    {
        tb->flags |= TBF_EXCEPTION_ERRCODE;
        tb->error_code = error_code;
    }

    if ( TI_GET_IF(ti) )
        tb->flags |= TBF_INTERRUPT;

    if ( unlikely(null_trap_bounce(v, tb)) )
    {
        gprintk(XENLOG_WARNING,
                "Unhandled %s fault/trap [#%d, ec=%04x]\n",
                trapstr(vector), vector, error_code);

        if ( vector == TRAP_page_fault )
            show_page_walk(event->cr2);
    }
}

static inline void do_guest_trap(unsigned int trapnr,
                                 const struct cpu_user_regs *regs)
{
    const struct x86_event event = {
        .vector = trapnr,
        .type = X86_EVENTTYPE_HW_EXCEPTION,
        .error_code = (((trapnr < 32) && (TRAP_HAVE_EC & (1u << trapnr)))
                       ? regs->error_code : X86_EVENT_NO_EC),
    };

    pv_inject_event(&event);
}

static void instruction_done(struct cpu_user_regs *regs, unsigned long rip)
{
    regs->rip = rip;
    regs->eflags &= ~X86_EFLAGS_RF;
    if ( regs->eflags & X86_EFLAGS_TF )
    {
        current->arch.debugreg[6] |= DR_STEP | DR_STATUS_RESERVED_ONE;
        do_guest_trap(TRAP_debug, regs);
    }
}

static unsigned int check_guest_io_breakpoint(struct vcpu *v,
    unsigned int port, unsigned int len)
{
    unsigned int width, i, match = 0;
    unsigned long start;

    if ( !(v->arch.debugreg[5]) ||
         !(v->arch.pv_vcpu.ctrlreg[4] & X86_CR4_DE) )
        return 0;

    for ( i = 0; i < 4; i++ )
    {
        if ( !(v->arch.debugreg[5] &
               (3 << (i * DR_ENABLE_SIZE))) )
            continue;

        start = v->arch.debugreg[i];
        width = 0;

        switch ( (v->arch.debugreg[7] >>
                  (DR_CONTROL_SHIFT + i * DR_CONTROL_SIZE)) & 0xc )
        {
        case DR_LEN_1: width = 1; break;
        case DR_LEN_2: width = 2; break;
        case DR_LEN_4: width = 4; break;
        case DR_LEN_8: width = 8; break;
        }

        if ( (start < (port + len)) && ((start + width) > port) )
            match |= 1 << i;
    }

    return match;
}

/*
 * Called from asm to set up the MCE trapbounce info.
 * Returns 0 if no callback is set up, else 1.
 */
int set_guest_machinecheck_trapbounce(void)
{
    struct vcpu *v = current;
    struct trap_bounce *tb = &v->arch.pv_vcpu.trap_bounce;
 
    do_guest_trap(TRAP_machine_check, guest_cpu_user_regs());
    tb->flags &= ~TBF_EXCEPTION; /* not needed for MCE delivery path */
    return !null_trap_bounce(v, tb);
}

/*
 * Called from asm to set up the NMI trapbounce info.
 * Returns 0 if no callback is set up, else 1.
 */
int set_guest_nmi_trapbounce(void)
{
    struct vcpu *v = current;
    struct trap_bounce *tb = &v->arch.pv_vcpu.trap_bounce;
    do_guest_trap(TRAP_nmi, guest_cpu_user_regs());
    tb->flags &= ~TBF_EXCEPTION; /* not needed for NMI delivery path */
    return !null_trap_bounce(v, tb);
}

void do_reserved_trap(struct cpu_user_regs *regs)
{
    unsigned int trapnr = regs->entry_vector;

    if ( debugger_trap_fatal(trapnr, regs) )
        return;

    show_execution_state(regs);
    panic("FATAL RESERVED TRAP %#x: %s", trapnr, trapstr(trapnr));
}

void do_trap(struct cpu_user_regs *regs)
{
    struct vcpu *curr = current;
    unsigned int trapnr = regs->entry_vector;
    unsigned long fixup;

    if ( regs->error_code & X86_XEC_EXT )
        goto hardware_trap;

    if ( debugger_trap_entry(trapnr, regs) )
        return;

    if ( guest_mode(regs) )
    {
        do_guest_trap(trapnr, regs);
        return;
    }

    if ( ((trapnr == TRAP_copro_error) || (trapnr == TRAP_simd_error)) &&
         system_state >= SYS_STATE_active && is_hvm_vcpu(curr) &&
         curr->arch.hvm_vcpu.fpu_exception_callback )
    {
        curr->arch.hvm_vcpu.fpu_exception_callback(
            curr->arch.hvm_vcpu.fpu_exception_callback_arg, regs);
        return;
    }

    if ( likely((fixup = search_exception_table(regs)) != 0) )
    {
        dprintk(XENLOG_ERR, "Trap %u: %p [%ps] -> %p\n",
                trapnr, _p(regs->rip), _p(regs->rip), _p(fixup));
        this_cpu(last_extable_addr) = regs->rip;
        regs->rip = fixup;
        return;
    }

 hardware_trap:
    if ( debugger_trap_fatal(trapnr, regs) )
        return;

    show_execution_state(regs);
    panic("FATAL TRAP: vector = %d (%s)\n"
          "[error_code=%04x]",
          trapnr, trapstr(trapnr), regs->error_code);
}

/* Returns 0 if not handled, and non-0 for success. */
int rdmsr_hypervisor_regs(uint32_t idx, uint64_t *val)
{
    struct domain *d = current->domain;
    /* Optionally shift out of the way of Viridian architectural MSRs. */
    uint32_t base = is_viridian_domain(d) ? 0x40000200 : 0x40000000;

    switch ( idx - base )
    {
    case 0: /* Write hypercall page MSR.  Read as zero. */
    {
        *val = 0;
        return 1;
    }
    }

    return 0;
}

/* Returns 1 if handled, 0 if not and -Exx for error. */
int wrmsr_hypervisor_regs(uint32_t idx, uint64_t val)
{
    struct domain *d = current->domain;
    /* Optionally shift out of the way of Viridian architectural MSRs. */
    uint32_t base = is_viridian_domain(d) ? 0x40000200 : 0x40000000;

    switch ( idx - base )
    {
    case 0: /* Write hypercall page */
    {
        void *hypercall_page;
        unsigned long gmfn = val >> PAGE_SHIFT;
        unsigned int page_index = val & (PAGE_SIZE - 1);
        struct page_info *page;
        p2m_type_t t;

        if ( page_index > 0 )
        {
            gdprintk(XENLOG_WARNING,
                     "wrmsr hypercall page index %#x unsupported\n",
                     page_index);
            return 0;
        }

        page = get_page_from_gfn(d, gmfn, &t, P2M_ALLOC);

        if ( !page || !get_page_type(page, PGT_writable_page) )
        {
            if ( page )
                put_page(page);

            if ( p2m_is_paging(t) )
            {
                p2m_mem_paging_populate(d, gmfn);
                return -ERESTART;
            }

            gdprintk(XENLOG_WARNING,
                     "Bad GMFN %lx (MFN %lx) to MSR %08x\n",
                     gmfn, page ? page_to_mfn(page) : -1UL, base);
            return 0;
        }

        hypercall_page = __map_domain_page(page);
        hypercall_page_initialise(d, hypercall_page);
        unmap_domain_page(hypercall_page);

        put_page_and_type(page);
        return 1;
    }
    }

    return 0;
}

void cpuid_hypervisor_leaves(const struct vcpu *v, uint32_t leaf,
                             uint32_t subleaf, struct cpuid_leaf *res)
{
    const struct domain *d = v->domain;
    const struct cpuid_policy *p = d->arch.cpuid;
    uint32_t base = is_viridian_domain(d) ? 0x40000100 : 0x40000000;
    uint32_t idx  = leaf - base;
    unsigned int limit = is_viridian_domain(d) ? p->hv2_limit : p->hv_limit;

    if ( limit == 0 )
        /* Default number of leaves */
        limit = XEN_CPUID_MAX_NUM_LEAVES;
    else
        /* Clamp toolstack value between 2 and MAX_NUM_LEAVES. */
        limit = min(max(limit, 2u), XEN_CPUID_MAX_NUM_LEAVES + 0u);

    if ( idx > limit )
        return;

    switch ( idx )
    {
    case 0:
        res->a = base + limit; /* Largest leaf */
        res->b = XEN_CPUID_SIGNATURE_EBX;
        res->c = XEN_CPUID_SIGNATURE_ECX;
        res->d = XEN_CPUID_SIGNATURE_EDX;
        break;

    case 1:
        res->a = (xen_major_version() << 16) | xen_minor_version();
        break;

    case 2:
        res->a = 1;            /* Number of hypercall-transfer pages */
                               /* MSR base address */
        res->b = is_viridian_domain(d) ? 0x40000200 : 0x40000000;
        if ( is_pv_domain(d) ) /* Features */
            res->c |= XEN_CPUID_FEAT1_MMU_PT_UPDATE_PRESERVE_AD;
        break;

    case 3: /* Time leaf. */
        switch ( subleaf )
        {
        case 0: /* features */
            res->a = ((d->arch.vtsc << 0) |
                      (!!host_tsc_is_safe() << 1) |
                      (!!boot_cpu_has(X86_FEATURE_RDTSCP) << 2));
            res->b = d->arch.tsc_mode;
            res->c = d->arch.tsc_khz;
            res->d = d->arch.incarnation;
            break;

        case 1: /* scale and offset */
        {
            uint64_t offset;

            if ( !d->arch.vtsc )
                offset = d->arch.vtsc_offset;
            else
                /* offset already applied to value returned by virtual rdtscp */
                offset = 0;
            res->a = offset;
            res->b = offset >> 32;
            res->c = d->arch.vtsc_to_ns.mul_frac;
            res->d = (s8)d->arch.vtsc_to_ns.shift;
            break;
        }

        case 2: /* physical cpu_khz */
            res->a = cpu_khz;
            break;
        }
        break;

    case 4: /* HVM hypervisor leaf. */
        if ( !is_hvm_domain(d) || subleaf != 0 )
            break;

        if ( cpu_has_vmx_apic_reg_virt )
            res->a |= XEN_HVM_CPUID_APIC_ACCESS_VIRT;

        /*
         * We want to claim that x2APIC is virtualized if APIC MSR accesses
         * are not intercepted. When all three of these are true both rdmsr
         * and wrmsr in the guest will run without VMEXITs (see
         * vmx_vlapic_msr_changed()).
         */
        if ( cpu_has_vmx_virtualize_x2apic_mode &&
             cpu_has_vmx_apic_reg_virt &&
             cpu_has_vmx_virtual_intr_delivery )
            res->a |= XEN_HVM_CPUID_X2APIC_VIRT;

        /*
         * Indicate that memory mapped from other domains (either grants or
         * foreign pages) has valid IOMMU entries.
         */
        res->a |= XEN_HVM_CPUID_IOMMU_MAPPINGS;

        /* Indicate presence of vcpu id and set it in ebx */
        res->a |= XEN_HVM_CPUID_VCPU_ID_PRESENT;
        res->b = v->vcpu_id;
        break;

    default:
        ASSERT_UNREACHABLE();
    }
}

static int emulate_invalid_rdtscp(struct cpu_user_regs *regs)
{
    char opcode[3];
    unsigned long eip, rc;
    struct vcpu *v = current;

    eip = regs->rip;
    if ( (rc = copy_from_user(opcode, (char *)eip, sizeof(opcode))) != 0 )
    {
        pv_inject_page_fault(0, eip + sizeof(opcode) - rc);
        return EXCRET_fault_fixed;
    }
    if ( memcmp(opcode, "\xf\x1\xf9", sizeof(opcode)) )
        return 0;
    eip += sizeof(opcode);
    pv_soft_rdtsc(v, regs, 1);
    instruction_done(regs, eip);
    return EXCRET_fault_fixed;
}

static int emulate_forced_invalid_op(struct cpu_user_regs *regs)
{
    char sig[5], instr[2];
    unsigned long eip, rc;
    struct cpuid_leaf res;

    eip = regs->rip;

    /* Check for forced emulation signature: ud2 ; .ascii "xen". */
    if ( (rc = copy_from_user(sig, (char *)eip, sizeof(sig))) != 0 )
    {
        pv_inject_page_fault(0, eip + sizeof(sig) - rc);
        return EXCRET_fault_fixed;
    }
    if ( memcmp(sig, "\xf\xbxen", sizeof(sig)) )
        return 0;
    eip += sizeof(sig);

    /* We only emulate CPUID. */
    if ( ( rc = copy_from_user(instr, (char *)eip, sizeof(instr))) != 0 )
    {
        pv_inject_page_fault(0, eip + sizeof(instr) - rc);
        return EXCRET_fault_fixed;
    }
    if ( memcmp(instr, "\xf\xa2", sizeof(instr)) )
        return 0;

    /* If cpuid faulting is enabled and CPL>0 inject a #GP in place of #UD. */
    if ( current->arch.cpuid_faulting && !guest_kernel_mode(current, regs) )
    {
        regs->rip = eip;
        do_guest_trap(TRAP_gp_fault, regs);
        return EXCRET_fault_fixed;
    }

    eip += sizeof(instr);

    guest_cpuid(current, regs->eax, regs->ecx, &res);

    regs->rax = res.a;
    regs->rbx = res.b;
    regs->rcx = res.c;
    regs->rdx = res.d;

    instruction_done(regs, eip);

    trace_trap_one_addr(TRC_PV_FORCED_INVALID_OP, regs->rip);

    return EXCRET_fault_fixed;
}

void do_invalid_op(struct cpu_user_regs *regs)
{
    const struct bug_frame *bug = NULL;
    u8 bug_insn[2];
    const char *prefix = "", *filename, *predicate, *eip = (char *)regs->rip;
    unsigned long fixup;
    int id = -1, lineno;
    const struct virtual_region *region;

    if ( debugger_trap_entry(TRAP_invalid_op, regs) )
        return;

    if ( likely(guest_mode(regs)) )
    {
        if ( !emulate_invalid_rdtscp(regs) &&
             !emulate_forced_invalid_op(regs) )
            do_guest_trap(TRAP_invalid_op, regs);
        return;
    }

    if ( !is_active_kernel_text(regs->rip) ||
         __copy_from_user(bug_insn, eip, sizeof(bug_insn)) ||
         memcmp(bug_insn, "\xf\xb", sizeof(bug_insn)) )
        goto die;

    region = find_text_region(regs->rip);
    if ( region )
    {
        for ( id = 0; id < BUGFRAME_NR; id++ )
        {
            const struct bug_frame *b;
            unsigned int i;

            for ( i = 0, b = region->frame[id].bugs;
                  i < region->frame[id].n_bugs; b++, i++ )
            {
                if ( bug_loc(b) == eip )
                {
                    bug = b;
                    goto found;
                }
            }
        }
    }

 found:
    if ( !bug )
        goto die;
    eip += sizeof(bug_insn);
    if ( id == BUGFRAME_run_fn )
    {
        void (*fn)(struct cpu_user_regs *) = bug_ptr(bug);

        fn(regs);
        regs->rip = (unsigned long)eip;
        return;
    }

    /* WARN, BUG or ASSERT: decode the filename pointer and line number. */
    filename = bug_ptr(bug);
    if ( !is_kernel(filename) && !is_patch(filename) )
        goto die;
    fixup = strlen(filename);
    if ( fixup > 50 )
    {
        filename += fixup - 47;
        prefix = "...";
    }
    lineno = bug_line(bug);

    switch ( id )
    {
    case BUGFRAME_warn:
        printk("Xen WARN at %s%s:%d\n", prefix, filename, lineno);
        show_execution_state(regs);
        regs->rip = (unsigned long)eip;
        return;

    case BUGFRAME_bug:
        printk("Xen BUG at %s%s:%d\n", prefix, filename, lineno);

        if ( debugger_trap_fatal(TRAP_invalid_op, regs) )
            return;

        show_execution_state(regs);
        panic("Xen BUG at %s%s:%d", prefix, filename, lineno);

    case BUGFRAME_assert:
        /* ASSERT: decode the predicate string pointer. */
        predicate = bug_msg(bug);
        if ( !is_kernel(predicate) && !is_patch(predicate) )
            predicate = "<unknown>";

        printk("Assertion '%s' failed at %s%s:%d\n",
               predicate, prefix, filename, lineno);

        if ( debugger_trap_fatal(TRAP_invalid_op, regs) )
            return;

        show_execution_state(regs);
        panic("Assertion '%s' failed at %s%s:%d",
              predicate, prefix, filename, lineno);
    }

 die:
    if ( (fixup = search_exception_table(regs)) != 0 )
    {
        this_cpu(last_extable_addr) = regs->rip;
        regs->rip = fixup;
        return;
    }

    if ( debugger_trap_fatal(TRAP_invalid_op, regs) )
        return;

    show_execution_state(regs);
    panic("FATAL TRAP: vector = %d (invalid opcode)", TRAP_invalid_op);
}

void do_int3(struct cpu_user_regs *regs)
{
    if ( debugger_trap_entry(TRAP_int3, regs) )
        return;

    if ( !guest_mode(regs) )
    {
        unsigned long fixup;

        if ( (fixup = search_exception_table(regs)) != 0 )
        {
            this_cpu(last_extable_addr) = regs->rip;
            dprintk(XENLOG_DEBUG, "Trap %u: %p [%ps] -> %p\n",
                    TRAP_int3, _p(regs->rip), _p(regs->rip), _p(fixup));
            regs->rip = fixup;
            return;
        }

        if ( !debugger_trap_fatal(TRAP_int3, regs) )
            printk(XENLOG_DEBUG "Hit embedded breakpoint at %p [%ps]\n",
                   _p(regs->rip), _p(regs->rip));

        return;
    }

    do_guest_trap(TRAP_int3, regs);
}

static void reserved_bit_page_fault(
    unsigned long addr, struct cpu_user_regs *regs)
{
    printk("%pv: reserved bit in page table (ec=%04X)\n",
           current, regs->error_code);
    show_page_walk(addr);
    show_execution_state(regs);
}

static int handle_gdt_ldt_mapping_fault(
    unsigned long offset, struct cpu_user_regs *regs)
{
    struct vcpu *curr = current;
    /* Which vcpu's area did we fault in, and is it in the ldt sub-area? */
    unsigned int is_ldt_area = (offset >> (GDT_LDT_VCPU_VA_SHIFT-1)) & 1;
    unsigned int vcpu_area   = (offset >> GDT_LDT_VCPU_VA_SHIFT);

    /*
     * If the fault is in another vcpu's area, it cannot be due to
     * a GDT/LDT descriptor load. Thus we can reasonably exit immediately, and
     * indeed we have to since map_ldt_shadow_page() works correctly only on
     * accesses to a vcpu's own area.
     */
    if ( vcpu_area != curr->vcpu_id )
        return 0;

    /* Byte offset within the gdt/ldt sub-area. */
    offset &= (1UL << (GDT_LDT_VCPU_VA_SHIFT-1)) - 1UL;

    if ( likely(is_ldt_area) )
    {
        /* LDT fault: Copy a mapping from the guest's LDT, if it is valid. */
        if ( likely(map_ldt_shadow_page(offset >> PAGE_SHIFT)) )
        {
            if ( guest_mode(regs) )
                trace_trap_two_addr(TRC_PV_GDT_LDT_MAPPING_FAULT,
                                    regs->rip, offset);
        }
        else
        {
            /* In hypervisor mode? Leave it to the #PF handler to fix up. */
            if ( !guest_mode(regs) )
                return 0;

            /* Access would have become non-canonical? Pass #GP[sel] back. */
            if ( unlikely(!is_canonical_address(
                              curr->arch.pv_vcpu.ldt_base + offset)) )
            {
                uint16_t ec = (offset & ~(X86_XEC_EXT | X86_XEC_IDT)) | X86_XEC_TI;

                pv_inject_hw_exception(TRAP_gp_fault, ec);
            }
            else
                /* else pass the #PF back, with adjusted %cr2. */
                pv_inject_page_fault(regs->error_code,
                                     curr->arch.pv_vcpu.ldt_base + offset);
        }
    }
    else
    {
        /* GDT fault: handle the fault as #GP(selector). */
        regs->error_code = offset & ~(X86_XEC_EXT | X86_XEC_IDT | X86_XEC_TI);
        (void)do_general_protection(regs);
    }

    return EXCRET_fault_fixed;
}

#define IN_HYPERVISOR_RANGE(va) \
    (((va) >= HYPERVISOR_VIRT_START) && ((va) < HYPERVISOR_VIRT_END))

enum pf_type {
    real_fault,
    smep_fault,
    smap_fault,
    spurious_fault
};

static enum pf_type __page_fault_type(
    unsigned long addr, const struct cpu_user_regs *regs)
{
    unsigned long mfn, cr3 = read_cr3();
    l4_pgentry_t l4e, *l4t;
    l3_pgentry_t l3e, *l3t;
    l2_pgentry_t l2e, *l2t;
    l1_pgentry_t l1e, *l1t;
    unsigned int required_flags, disallowed_flags, page_user;
    unsigned int error_code = regs->error_code;

    /*
     * We do not take spurious page faults in IRQ handlers as we do not
     * modify page tables in IRQ context. We therefore bail here because
     * map_domain_page() is not IRQ-safe.
     */
    if ( in_irq() )
        return real_fault;

    /* Reserved bit violations are never spurious faults. */
    if ( error_code & PFEC_reserved_bit )
        return real_fault;

    required_flags  = _PAGE_PRESENT;
    if ( error_code & PFEC_write_access )
        required_flags |= _PAGE_RW;
    if ( error_code & PFEC_user_mode )
        required_flags |= _PAGE_USER;

    disallowed_flags = 0;
    if ( error_code & PFEC_insn_fetch )
        disallowed_flags |= _PAGE_NX_BIT;

    page_user = _PAGE_USER;

    mfn = cr3 >> PAGE_SHIFT;

    l4t = map_domain_page(_mfn(mfn));
    l4e = l4e_read_atomic(&l4t[l4_table_offset(addr)]);
    mfn = l4e_get_pfn(l4e);
    unmap_domain_page(l4t);
    if ( ((l4e_get_flags(l4e) & required_flags) != required_flags) ||
         (l4e_get_flags(l4e) & disallowed_flags) )
        return real_fault;
    page_user &= l4e_get_flags(l4e);

    l3t  = map_domain_page(_mfn(mfn));
    l3e = l3e_read_atomic(&l3t[l3_table_offset(addr)]);
    mfn = l3e_get_pfn(l3e);
    unmap_domain_page(l3t);
    if ( ((l3e_get_flags(l3e) & required_flags) != required_flags) ||
         (l3e_get_flags(l3e) & disallowed_flags) )
        return real_fault;
    page_user &= l3e_get_flags(l3e);
    if ( l3e_get_flags(l3e) & _PAGE_PSE )
        goto leaf;

    l2t = map_domain_page(_mfn(mfn));
    l2e = l2e_read_atomic(&l2t[l2_table_offset(addr)]);
    mfn = l2e_get_pfn(l2e);
    unmap_domain_page(l2t);
    if ( ((l2e_get_flags(l2e) & required_flags) != required_flags) ||
         (l2e_get_flags(l2e) & disallowed_flags) )
        return real_fault;
    page_user &= l2e_get_flags(l2e);
    if ( l2e_get_flags(l2e) & _PAGE_PSE )
        goto leaf;

    l1t = map_domain_page(_mfn(mfn));
    l1e = l1e_read_atomic(&l1t[l1_table_offset(addr)]);
    mfn = l1e_get_pfn(l1e);
    unmap_domain_page(l1t);
    if ( ((l1e_get_flags(l1e) & required_flags) != required_flags) ||
         (l1e_get_flags(l1e) & disallowed_flags) )
        return real_fault;
    page_user &= l1e_get_flags(l1e);

leaf:
    if ( page_user )
    {
        unsigned long cr4 = read_cr4();
        /*
         * Supervisor Mode Execution Prevention (SMEP):
         * Disallow supervisor execution from user-accessible mappings
         */
        if ( (cr4 & X86_CR4_SMEP) &&
             ((error_code & (PFEC_insn_fetch|PFEC_user_mode)) == PFEC_insn_fetch) )
            return smep_fault;

        /*
         * Supervisor Mode Access Prevention (SMAP):
         * Disallow supervisor access user-accessible mappings
         * A fault is considered as an SMAP violation if the following
         * conditions are true:
         *   - X86_CR4_SMAP is set in CR4
         *   - A user page is being accessed
         *   - CPL=3 or X86_EFLAGS_AC is clear
         *   - Page fault in kernel mode
         */
        if ( (cr4 & X86_CR4_SMAP) && !(error_code & PFEC_user_mode) &&
             (((regs->cs & 3) == 3) || !(regs->eflags & X86_EFLAGS_AC)) )
            return smap_fault;
    }

    return spurious_fault;
}

static enum pf_type spurious_page_fault(
    unsigned long addr, const struct cpu_user_regs *regs)
{
    unsigned long flags;
    enum pf_type pf_type;

    /*
     * Disabling interrupts prevents TLB flushing, and hence prevents
     * page tables from becoming invalid under our feet during the walk.
     */
    local_irq_save(flags);
    pf_type = __page_fault_type(addr, regs);
    local_irq_restore(flags);

    return pf_type;
}

static int fixup_page_fault(unsigned long addr, struct cpu_user_regs *regs)
{
    struct vcpu   *v = current;
    struct domain *d = v->domain;

    /* No fixups in interrupt context or when interrupts are disabled. */
    if ( in_irq() || !(regs->eflags & X86_EFLAGS_IF) )
        return 0;

    if ( !(regs->error_code & PFEC_page_present) &&
          (pagefault_by_memadd(addr, regs)) )
        return handle_memadd_fault(addr, regs);

    if ( unlikely(IN_HYPERVISOR_RANGE(addr)) )
    {
        if ( !(regs->error_code & (PFEC_user_mode | PFEC_reserved_bit)) &&
             (addr >= GDT_LDT_VIRT_START) && (addr < GDT_LDT_VIRT_END) )
            return handle_gdt_ldt_mapping_fault(
                addr - GDT_LDT_VIRT_START, regs);
        return 0;
    }

    if ( guest_kernel_mode(v, regs) &&
         !(regs->error_code & (PFEC_reserved_bit | PFEC_insn_fetch)) &&
         (regs->error_code & PFEC_write_access) )
    {
        if ( VM_ASSIST(d, writable_pagetables) &&
             /* Do not check if access-protection fault since the page may
                legitimately be not present in shadow page tables */
             (paging_mode_enabled(d) ||
              (regs->error_code & PFEC_page_present)) &&
             ptwr_do_page_fault(v, addr, regs) )
            return EXCRET_fault_fixed;

        if ( is_hardware_domain(d) && (regs->error_code & PFEC_page_present) &&
             mmio_ro_do_page_fault(v, addr, regs) )
            return EXCRET_fault_fixed;
    }

    /*
     * For non-external shadowed guests, we fix up both their own pagefaults
     * and Xen's, since they share the pagetables.  This includes hypervisor
     * faults, e.g. from copy_to_user().
     */
    if ( paging_mode_enabled(d) && !paging_mode_external(d) )
    {
        int ret = paging_fault(addr, regs);

        if ( ret == EXCRET_fault_fixed )
            trace_trap_two_addr(TRC_PV_PAGING_FIXUP, regs->rip, addr);
        return ret;
    }

    return 0;
}

/*
 * #PF error code:
 *  Bit 0: Protection violation (=1) ; Page not present (=0)
 *  Bit 1: Write access
 *  Bit 2: User mode (=1) ; Supervisor mode (=0)
 *  Bit 3: Reserved bit violation
 *  Bit 4: Instruction fetch
 */
void do_page_fault(struct cpu_user_regs *regs)
{
    unsigned long addr, fixup;
    unsigned int error_code;
    enum pf_type pf_type;

    addr = read_cr2();

    /* fixup_page_fault() might change regs->error_code, so cache it here. */
    error_code = regs->error_code;

    if ( debugger_trap_entry(TRAP_page_fault, regs) )
        return;

    perfc_incr(page_faults);

    if ( unlikely(fixup_page_fault(addr, regs) != 0) )
        return;

    if ( unlikely(!guest_mode(regs)) )
    {
        pf_type = spurious_page_fault(addr, regs);
        if ( (pf_type == smep_fault) || (pf_type == smap_fault) )
        {
            console_start_sync();
            printk("Xen SM%cP violation\n", (pf_type == smep_fault) ? 'E' : 'A');
            fatal_trap(regs, 0);
        }

        if ( pf_type != real_fault )
            return;

        if ( likely((fixup = search_exception_table(regs)) != 0) )
        {
            perfc_incr(copy_user_faults);
            if ( unlikely(regs->error_code & PFEC_reserved_bit) )
                reserved_bit_page_fault(addr, regs);
            this_cpu(last_extable_addr) = regs->rip;
            regs->rip = fixup;
            return;
        }

        if ( debugger_trap_fatal(TRAP_page_fault, regs) )
            return;

        show_execution_state(regs);
        show_page_walk(addr);
        panic("FATAL PAGE FAULT\n"
              "[error_code=%04x]\n"
              "Faulting linear address: %p",
              error_code, _p(addr));
    }

    if ( unlikely(current->domain->arch.suppress_spurious_page_faults) )
    {
        pf_type = spurious_page_fault(addr, regs);
        if ( (pf_type == smep_fault) || (pf_type == smap_fault))
        {
            printk(XENLOG_G_ERR "%pv fatal SM%cP violation\n",
                   current, (pf_type == smep_fault) ? 'E' : 'A');

            domain_crash(current->domain);
        }
        if ( pf_type != real_fault )
            return;
    }

    if ( unlikely(regs->error_code & PFEC_reserved_bit) )
        reserved_bit_page_fault(addr, regs);

    pv_inject_page_fault(regs->error_code, addr);
}

/*
 * Early #PF handler to print CR2, error code, and stack.
 * 
 * We also deal with spurious faults here, even though they should never happen
 * during early boot (an issue was seen once, but was most likely a hardware 
 * problem).
 */
void __init do_early_page_fault(struct cpu_user_regs *regs)
{
    static unsigned int __initdata stuck;
    static unsigned long __initdata prev_eip, prev_cr2;
    unsigned long cr2 = read_cr2();

    BUG_ON(smp_processor_id() != 0);

    if ( (regs->rip != prev_eip) || (cr2 != prev_cr2) )
    {
        prev_eip = regs->rip;
        prev_cr2 = cr2;
        stuck    = 0;
        return;
    }

    if ( stuck++ == 1000 )
    {
        console_start_sync();
        printk("Early fatal page fault at %04x:%p (cr2=%p, ec=%04x)\n",
               regs->cs, _p(regs->rip), _p(cr2), regs->error_code);
        fatal_trap(regs, 0);
    }
}

long do_fpu_taskswitch(int set)
{
    struct vcpu *v = current;

    if ( set )
    {
        v->arch.pv_vcpu.ctrlreg[0] |= X86_CR0_TS;
        stts();
    }
    else
    {
        v->arch.pv_vcpu.ctrlreg[0] &= ~X86_CR0_TS;
        if ( v->fpu_dirtied )
            clts();
    }

    return 0;
}

static int read_descriptor(unsigned int sel,
                           const struct vcpu *v,
                           unsigned long *base,
                           unsigned long *limit,
                           unsigned int *ar,
                           bool_t insn_fetch)
{
    struct desc_struct desc;

    if ( sel < 4)
        desc.b = desc.a = 0;
    else if ( __get_user(desc,
                         (const struct desc_struct *)(!(sel & 4)
                                                      ? GDT_VIRT_START(v)
                                                      : LDT_VIRT_START(v))
                         + (sel >> 3)) )
        return 0;
    if ( !insn_fetch )
        desc.b &= ~_SEGMENT_L;

    *ar = desc.b & 0x00f0ff00;
    if ( !(desc.b & _SEGMENT_L) )
    {
        *base = ((desc.a >> 16) + ((desc.b & 0xff) << 16) +
                 (desc.b & 0xff000000));
        *limit = (desc.a & 0xffff) | (desc.b & 0x000f0000);
        if ( desc.b & _SEGMENT_G )
            *limit = ((*limit + 1) << 12) - 1;
#ifndef NDEBUG
        if ( sel > 3 )
        {
            unsigned int a, l;
            unsigned char valid;

            asm volatile (
                "larl %2,%0 ; setz %1"
                : "=r" (a), "=qm" (valid) : "rm" (sel));
            BUG_ON(valid && ((a & 0x00f0ff00) != *ar));
            asm volatile (
                "lsll %2,%0 ; setz %1"
                : "=r" (l), "=qm" (valid) : "rm" (sel));
            BUG_ON(valid && (l != *limit));
        }
#endif
    }
    else
    {
        *base = 0UL;
        *limit = ~0UL;
    }

    return 1;
}

static int read_gate_descriptor(unsigned int gate_sel,
                                const struct vcpu *v,
                                unsigned int *sel,
                                unsigned long *off,
                                unsigned int *ar)
{
    struct desc_struct desc;
    const struct desc_struct *pdesc;


    pdesc = (const struct desc_struct *)
        (!(gate_sel & 4) ? GDT_VIRT_START(v) : LDT_VIRT_START(v))
        + (gate_sel >> 3);
    if ( (gate_sel < 4) ||
         ((gate_sel >= FIRST_RESERVED_GDT_BYTE) && !(gate_sel & 4)) ||
         __get_user(desc, pdesc) )
        return 0;

    *sel = (desc.a >> 16) & 0x0000fffc;
    *off = (desc.a & 0x0000ffff) | (desc.b & 0xffff0000);
    *ar = desc.b & 0x0000ffff;

    /*
     * check_descriptor() clears the DPL field and stores the
     * guest requested DPL in the selector's RPL field.
     */
    if ( *ar & _SEGMENT_DPL )
        return 0;
    *ar |= (desc.a >> (16 - 13)) & _SEGMENT_DPL;

    if ( !is_pv_32bit_vcpu(v) )
    {
        if ( (*ar & 0x1f00) != 0x0c00 ||
             (gate_sel >= FIRST_RESERVED_GDT_BYTE - 8 && !(gate_sel & 4)) ||
             __get_user(desc, pdesc + 1) ||
             (desc.b & 0x1f00) )
            return 0;

        *off |= (unsigned long)desc.a << 32;
        return 1;
    }

    switch ( *ar & 0x1f00 )
    {
    case 0x0400:
        *off &= 0xffff;
        break;
    case 0x0c00:
        break;
    default:
        return 0;
    }

    return 1;
}

static int pv_emul_virt_to_linear(unsigned long base, unsigned long offset,
                                  unsigned int bytes, unsigned long limit,
                                  enum x86_segment seg,
                                  struct x86_emulate_ctxt *ctxt,
                                  unsigned long *addr)
{
    int rc = X86EMUL_OKAY;

    *addr = base + offset;

    if ( ctxt->addr_size < 64 )
    {
        if ( limit < bytes - 1 || offset > limit - bytes + 1 )
            rc = X86EMUL_EXCEPTION;
        *addr = (uint32_t)*addr;
    }
    else if ( !__addr_ok(*addr) )
        rc = X86EMUL_EXCEPTION;

    if ( unlikely(rc == X86EMUL_EXCEPTION) )
        x86_emul_hw_exception(seg != x86_seg_ss ? TRAP_gp_fault
                                                : TRAP_stack_error,
                              0, ctxt);

    return rc;
}

struct priv_op_ctxt {
    struct x86_emulate_ctxt ctxt;
    struct {
        unsigned long base, limit;
    } cs;
    char *io_emul_stub;
    unsigned int bpmatch;
    unsigned int tsc;
#define TSC_BASE 1
#define TSC_AUX 2
};

static int priv_op_insn_fetch(enum x86_segment seg,
                              unsigned long offset,
                              void *p_data,
                              unsigned int bytes,
                              struct x86_emulate_ctxt *ctxt)
{
    const struct priv_op_ctxt *poc =
        container_of(ctxt, struct priv_op_ctxt, ctxt);
    unsigned int rc;
    unsigned long addr = poc->cs.base + offset;

    ASSERT(seg == x86_seg_cs);

    /* We don't mean to emulate any branches. */
    if ( !bytes )
        return X86EMUL_UNHANDLEABLE;

    rc = pv_emul_virt_to_linear(poc->cs.base, offset, bytes, poc->cs.limit,
                                x86_seg_cs, ctxt, &addr);
    if ( rc != X86EMUL_OKAY )
        return rc;

    if ( (rc = __copy_from_user(p_data, (void *)addr, bytes)) != 0 )
    {
        /*
         * TODO: This should report PFEC_insn_fetch when goc->insn_fetch &&
         * cpu_has_nx, but we'd then need a "fetch" variant of
         * __copy_from_user() respecting NX, SMEP, and protection keys.
         */
        x86_emul_pagefault(0, addr + bytes - rc, ctxt);
        return X86EMUL_EXCEPTION;
    }

    return X86EMUL_OKAY;
}

static int priv_op_read_segment(enum x86_segment seg,
                                struct segment_register *reg,
                                struct x86_emulate_ctxt *ctxt)
{
    /* Check if this is an attempt to access the I/O bitmap. */
    if ( seg == x86_seg_tr )
    {
        switch ( ctxt->opcode )
        {
        case 0x6c ... 0x6f: /* ins / outs */
        case 0xe4 ... 0xe7: /* in / out (immediate port) */
        case 0xec ... 0xef: /* in / out (port in %dx) */
            /* Defer the check to priv_op_{read,write}_io(). */
            return X86EMUL_DONE;
        }
    }

    if ( ctxt->addr_size < 64 )
    {
        unsigned long limit;
        unsigned int sel, ar;

        switch ( seg )
        {
        case x86_seg_cs: sel = ctxt->regs->cs; break;
        case x86_seg_ds: sel = read_sreg(ds);  break;
        case x86_seg_es: sel = read_sreg(es);  break;
        case x86_seg_fs: sel = read_sreg(fs);  break;
        case x86_seg_gs: sel = read_sreg(gs);  break;
        case x86_seg_ss: sel = ctxt->regs->ss; break;
        default: return X86EMUL_UNHANDLEABLE;
        }

        if ( !read_descriptor(sel, current, &reg->base, &limit, &ar, 0) )
            return X86EMUL_UNHANDLEABLE;

        reg->limit = limit;
        reg->attr.bytes = ar >> 8;
    }
    else
    {
        switch ( seg )
        {
        default:
            if ( !is_x86_user_segment(seg) )
                return X86EMUL_UNHANDLEABLE;
            reg->base = 0;
            break;
        case x86_seg_fs:
            reg->base = rdfsbase();
            break;
        case x86_seg_gs:
            reg->base = rdgsbase();
            break;
        }

        reg->limit = ~0U;

        reg->attr.bytes = 0;
        reg->attr.fields.type = _SEGMENT_WR >> 8;
        if ( seg == x86_seg_cs )
        {
            reg->attr.fields.type |= _SEGMENT_CODE >> 8;
            reg->attr.fields.l = 1;
        }
        else
            reg->attr.fields.db = 1;
        reg->attr.fields.s   = 1;
        reg->attr.fields.dpl = 3;
        reg->attr.fields.p   = 1;
        reg->attr.fields.g   = 1;
    }

    /*
     * For x86_emulate.c's mode_ring0() to work, fake a DPL of zero.
     * Also do this for consistency for non-conforming code segments.
     */
    if ( (seg == x86_seg_ss ||
          (seg == x86_seg_cs &&
           !(reg->attr.fields.type & (_SEGMENT_EC >> 8)))) &&
         guest_kernel_mode(current, ctxt->regs) )
        reg->attr.fields.dpl = 0;

    return X86EMUL_OKAY;
}

/* Perform IOPL check between the vcpu's shadowed IOPL, and the assumed cpl. */
static bool_t iopl_ok(const struct vcpu *v, const struct cpu_user_regs *regs)
{
    unsigned int cpl = guest_kernel_mode(v, regs) ?
        (VM_ASSIST(v->domain, architectural_iopl) ? 0 : 1) : 3;

    ASSERT((v->arch.pv_vcpu.iopl & ~X86_EFLAGS_IOPL) == 0);

    return IOPL(cpl) <= v->arch.pv_vcpu.iopl;
}

/* Has the guest requested sufficient permission for this I/O access? */
static int guest_io_okay(
    unsigned int port, unsigned int bytes,
    struct vcpu *v, struct cpu_user_regs *regs)
{
    /* If in user mode, switch to kernel mode just to read I/O bitmap. */
    int user_mode = !(v->arch.flags & TF_kernel_mode);
#define TOGGLE_MODE() if ( user_mode ) toggle_guest_pt(v)

    if ( iopl_ok(v, regs) )
        return 1;

    if ( (port + bytes) <= v->arch.pv_vcpu.iobmp_limit )
    {
        union { uint8_t bytes[2]; uint16_t mask; } x;

        /*
         * Grab permission bytes from guest space. Inaccessible bytes are
         * read as 0xff (no access allowed).
         */
        TOGGLE_MODE();
        switch ( __copy_from_guest_offset(x.bytes, v->arch.pv_vcpu.iobmp,
                                          port>>3, 2) )
        {
        default: x.bytes[0] = ~0;
            /* fallthrough */
        case 1:  x.bytes[1] = ~0;
            /* fallthrough */
        case 0:  break;
        }
        TOGGLE_MODE();

        if ( (x.mask & (((1<<bytes)-1) << (port&7))) == 0 )
            return 1;
    }

    return 0;
}

/* Has the administrator granted sufficient permission for this I/O access? */
static bool_t admin_io_okay(unsigned int port, unsigned int bytes,
                            const struct domain *d)
{
    /*
     * Port 0xcf8 (CONFIG_ADDRESS) is only visible for DWORD accesses.
     * We never permit direct access to that register.
     */
    if ( (port == 0xcf8) && (bytes == 4) )
        return 0;

    /* We also never permit direct access to the RTC/CMOS registers. */
    if ( ((port & ~1) == RTC_PORT(0)) )
        return 0;

    return ioports_access_permitted(d, port, port + bytes - 1);
}

static bool_t pci_cfg_ok(struct domain *currd, unsigned int start,
                         unsigned int size, uint32_t *write)
{
    uint32_t machine_bdf;

    if ( !is_hardware_domain(currd) )
        return 0;

    if ( !CF8_ENABLED(currd->arch.pci_cf8) )
        return 1;

    machine_bdf = CF8_BDF(currd->arch.pci_cf8);
    if ( write )
    {
        const unsigned long *ro_map = pci_get_ro_map(0);

        if ( ro_map && test_bit(machine_bdf, ro_map) )
            return 0;
    }
    start |= CF8_ADDR_LO(currd->arch.pci_cf8);
    /* AMD extended configuration space access? */
    if ( CF8_ADDR_HI(currd->arch.pci_cf8) &&
         boot_cpu_data.x86_vendor == X86_VENDOR_AMD &&
         boot_cpu_data.x86 >= 0x10 && boot_cpu_data.x86 <= 0x17 )
    {
        uint64_t msr_val;

        if ( rdmsr_safe(MSR_AMD64_NB_CFG, msr_val) )
            return 0;
        if ( msr_val & (1ULL << AMD64_NB_CFG_CF8_EXT_ENABLE_BIT) )
            start |= CF8_ADDR_HI(currd->arch.pci_cf8);
    }

    return !write ?
           xsm_pci_config_permission(XSM_HOOK, currd, machine_bdf,
                                     start, start + size - 1, 0) == 0 :
           pci_conf_write_intercept(0, machine_bdf, start, size, write) >= 0;
}

uint32_t guest_io_read(unsigned int port, unsigned int bytes,
                       struct domain *currd)
{
    uint32_t data = 0;
    unsigned int shift = 0;

    if ( admin_io_okay(port, bytes, currd) )
    {
        switch ( bytes )
        {
        case 1: return inb(port);
        case 2: return inw(port);
        case 4: return inl(port);
        }
    }

    while ( bytes != 0 )
    {
        unsigned int size = 1;
        uint32_t sub_data = ~0;

        if ( (port == 0x42) || (port == 0x43) || (port == 0x61) )
        {
            sub_data = pv_pit_handler(port, 0, 0);
        }
        else if ( port == RTC_PORT(0) )
        {
            sub_data = currd->arch.cmos_idx;
        }
        else if ( (port == RTC_PORT(1)) &&
                  ioports_access_permitted(currd, RTC_PORT(0), RTC_PORT(1)) )
        {
            unsigned long flags;

            spin_lock_irqsave(&rtc_lock, flags);
            outb(currd->arch.cmos_idx & 0x7f, RTC_PORT(0));
            sub_data = inb(RTC_PORT(1));
            spin_unlock_irqrestore(&rtc_lock, flags);
        }
        else if ( (port == 0xcf8) && (bytes == 4) )
        {
            size = 4;
            sub_data = currd->arch.pci_cf8;
        }
        else if ( (port & 0xfffc) == 0xcfc )
        {
            size = min(bytes, 4 - (port & 3));
            if ( size == 3 )
                size = 2;
            if ( pci_cfg_ok(currd, port & 3, size, NULL) )
                sub_data = pci_conf_read(currd->arch.pci_cf8, port & 3, size);
        }

        if ( size == 4 )
            return sub_data;

        data |= (sub_data & ((1u << (size * 8)) - 1)) << shift;
        shift += size * 8;
        port += size;
        bytes -= size;
    }

    return data;
}

void guest_io_write(unsigned int port, unsigned int bytes, uint32_t data,
                    struct domain *currd)
{
    if ( admin_io_okay(port, bytes, currd) )
    {
        switch ( bytes ) {
        case 1:
            outb((uint8_t)data, port);
            if ( pv_post_outb_hook )
                pv_post_outb_hook(port, (uint8_t)data);
            break;
        case 2:
            outw((uint16_t)data, port);
            break;
        case 4:
            outl(data, port);
            break;
        }
        return;
    }

    while ( bytes != 0 )
    {
        unsigned int size = 1;

        if ( (port == 0x42) || (port == 0x43) || (port == 0x61) )
        {
            pv_pit_handler(port, (uint8_t)data, 1);
        }
        else if ( port == RTC_PORT(0) )
        {
            currd->arch.cmos_idx = data;
        }
        else if ( (port == RTC_PORT(1)) &&
                  ioports_access_permitted(currd, RTC_PORT(0), RTC_PORT(1)) )
        {
            unsigned long flags;

            if ( pv_rtc_handler )
                pv_rtc_handler(currd->arch.cmos_idx & 0x7f, data);
            spin_lock_irqsave(&rtc_lock, flags);
            outb(currd->arch.cmos_idx & 0x7f, RTC_PORT(0));
            outb(data, RTC_PORT(1));
            spin_unlock_irqrestore(&rtc_lock, flags);
        }
        else if ( (port == 0xcf8) && (bytes == 4) )
        {
            size = 4;
            currd->arch.pci_cf8 = data;
        }
        else if ( (port & 0xfffc) == 0xcfc )
        {
            size = min(bytes, 4 - (port & 3));
            if ( size == 3 )
                size = 2;
            if ( pci_cfg_ok(currd, port & 3, size, &data) )
                pci_conf_write(currd->arch.pci_cf8, port & 3, size, data);
        }

        if ( size == 4 )
            return;

        port += size;
        bytes -= size;
        data >>= size * 8;
    }
}

/* I/O emulation support. Helper routines for, and type of, the stack stub.*/
void host_to_guest_gpr_switch(struct cpu_user_regs *);
unsigned long guest_to_host_gpr_switch(unsigned long);

void (*pv_post_outb_hook)(unsigned int port, u8 value);

typedef void io_emul_stub_t(struct cpu_user_regs *);

void __x86_indirect_thunk_rcx(void);

static io_emul_stub_t *io_emul_stub_setup(struct priv_op_ctxt *ctxt, u8 opcode,
                                          unsigned int port, unsigned int bytes)
{
    struct stubs *this_stubs = &this_cpu(stubs);
    unsigned long stub_va = this_stubs->addr + STUB_BUF_SIZE / 2;

    if ( !ctxt->io_emul_stub )
        ctxt->io_emul_stub =
            map_domain_page(_mfn(this_stubs->mfn)) + (stub_va & ~PAGE_MASK);

    /* movq $host_to_guest_gpr_switch,%rcx */
    ctxt->io_emul_stub[0] = 0x48;
    ctxt->io_emul_stub[1] = 0xb9;
    *(void **)&ctxt->io_emul_stub[2] = (void *)host_to_guest_gpr_switch;

#ifdef CONFIG_INDIRECT_THUNK
    /* callq __x86_indirect_thunk_rcx */
    ctxt->io_emul_stub[10] = 0xe8;
    *(int32_t *)&ctxt->io_emul_stub[11] =
        (long)__x86_indirect_thunk_rcx - (stub_va + 11 + 4);
#else
    /* callq *%rcx */
    ctxt->io_emul_stub[10] = 0xff;
    ctxt->io_emul_stub[11] = 0xd1;
    /* TODO: untangle ideal_nops from init/livepatch Kconfig options. */
    memcpy(&ctxt->io_emul_stub[12], "\x0f\x1f\x00", 3); /* P6_NOP3 */
#endif

    /* data16 or nop */
    ctxt->io_emul_stub[15] = (bytes != 2) ? 0x90 : 0x66;
    /* <io-access opcode> */
    ctxt->io_emul_stub[16] = opcode;
    /* imm8 or nop */
    ctxt->io_emul_stub[17] = !(opcode & 8) ? port : 0x90;
    /* ret (jumps to guest_to_host_gpr_switch) */
    ctxt->io_emul_stub[18] = 0xc3;
    BUILD_BUG_ON(STUB_BUF_SIZE / 2 < 19);

    if ( ioemul_handle_quirk )
    {
        BUILD_BUG_ON(STUB_BUF_SIZE / 2 < 15 + 10);
        ioemul_handle_quirk(opcode, &ctxt->io_emul_stub[15], ctxt->ctxt.regs);
    }

    /* Handy function-typed pointer to the stub. */
    return (void *)stub_va;
}

static int priv_op_read_io(unsigned int port, unsigned int bytes,
                           unsigned long *val, struct x86_emulate_ctxt *ctxt)
{
    struct priv_op_ctxt *poc = container_of(ctxt, struct priv_op_ctxt, ctxt);
    struct vcpu *curr = current;
    struct domain *currd = current->domain;

    /* INS must not come here. */
    ASSERT((ctxt->opcode & ~9) == 0xe4);

    if ( !guest_io_okay(port, bytes, curr, ctxt->regs) )
        return X86EMUL_UNHANDLEABLE;

    poc->bpmatch = check_guest_io_breakpoint(curr, port, bytes);

    if ( admin_io_okay(port, bytes, currd) )
    {
        io_emul_stub_t *io_emul =
            io_emul_stub_setup(poc, ctxt->opcode, port, bytes);

        io_emul(ctxt->regs);
        return X86EMUL_DONE;
    }

    *val = guest_io_read(port, bytes, currd);

    return X86EMUL_OKAY;
}

static int priv_op_write_io(unsigned int port, unsigned int bytes,
                            unsigned long val, struct x86_emulate_ctxt *ctxt)
{
    struct priv_op_ctxt *poc = container_of(ctxt, struct priv_op_ctxt, ctxt);
    struct vcpu *curr = current;
    struct domain *currd = current->domain;

    /* OUTS must not come here. */
    ASSERT((ctxt->opcode & ~9) == 0xe6);

    if ( !guest_io_okay(port, bytes, curr, ctxt->regs) )
        return X86EMUL_UNHANDLEABLE;

    poc->bpmatch = check_guest_io_breakpoint(curr, port, bytes);

    if ( admin_io_okay(port, bytes, currd) )
    {
        io_emul_stub_t *io_emul =
            io_emul_stub_setup(poc, ctxt->opcode, port, bytes);

        io_emul(ctxt->regs);
        if ( (bytes == 1) && pv_post_outb_hook )
            pv_post_outb_hook(port, val);
        return X86EMUL_DONE;
    }

    guest_io_write(port, bytes, val, currd);

    return X86EMUL_OKAY;
}

static int priv_op_rep_ins(uint16_t port,
                           enum x86_segment seg, unsigned long offset,
                           unsigned int bytes_per_rep, unsigned long *reps,
                           struct x86_emulate_ctxt *ctxt)
{
    struct priv_op_ctxt *poc = container_of(ctxt, struct priv_op_ctxt, ctxt);
    struct vcpu *curr = current;
    struct domain *currd = current->domain;
    unsigned long goal = *reps;
    struct segment_register sreg;
    int rc;

    ASSERT(seg == x86_seg_es);

    *reps = 0;

    if ( !guest_io_okay(port, bytes_per_rep, curr, ctxt->regs) )
        return X86EMUL_UNHANDLEABLE;

    rc = priv_op_read_segment(x86_seg_es, &sreg, ctxt);
    if ( rc != X86EMUL_OKAY )
        return rc;

    if ( !sreg.attr.fields.p )
        return X86EMUL_UNHANDLEABLE;
    if ( !sreg.attr.fields.s ||
         (sreg.attr.fields.type & (_SEGMENT_CODE >> 8)) ||
         !(sreg.attr.fields.type & (_SEGMENT_WR >> 8)) )
    {
        x86_emul_hw_exception(TRAP_gp_fault, 0, ctxt);
        return X86EMUL_EXCEPTION;
    }

    poc->bpmatch = check_guest_io_breakpoint(curr, port, bytes_per_rep);

    while ( *reps < goal )
    {
        unsigned int data = guest_io_read(port, bytes_per_rep, currd);
        unsigned long addr;

        rc = pv_emul_virt_to_linear(sreg.base, offset, bytes_per_rep,
                                    sreg.limit, x86_seg_es, ctxt, &addr);
        if ( rc != X86EMUL_OKAY )
            return rc;

        if ( (rc = __copy_to_user((void *)addr, &data, bytes_per_rep)) != 0 )
        {
            x86_emul_pagefault(PFEC_write_access,
                               addr + bytes_per_rep - rc, ctxt);
            return X86EMUL_EXCEPTION;
        }

        ++*reps;

        if ( poc->bpmatch || hypercall_preempt_check() )
            break;

        /* x86_emulate() clips the repetition count to ensure we don't wrap. */
        if ( unlikely(ctxt->regs->eflags & X86_EFLAGS_DF) )
            offset -= bytes_per_rep;
        else
            offset += bytes_per_rep;
    }

    return X86EMUL_OKAY;
}

static int priv_op_rep_outs(enum x86_segment seg, unsigned long offset,
                            uint16_t port,
                            unsigned int bytes_per_rep, unsigned long *reps,
                            struct x86_emulate_ctxt *ctxt)
{
    struct priv_op_ctxt *poc = container_of(ctxt, struct priv_op_ctxt, ctxt);
    struct vcpu *curr = current;
    struct domain *currd = current->domain;
    unsigned long goal = *reps;
    struct segment_register sreg;
    int rc;

    *reps = 0;

    if ( !guest_io_okay(port, bytes_per_rep, curr, ctxt->regs) )
        return X86EMUL_UNHANDLEABLE;

    rc = priv_op_read_segment(seg, &sreg, ctxt);
    if ( rc != X86EMUL_OKAY )
        return rc;

    if ( !sreg.attr.fields.p )
        return X86EMUL_UNHANDLEABLE;
    if ( !sreg.attr.fields.s ||
         ((sreg.attr.fields.type & (_SEGMENT_CODE >> 8)) &&
          !(sreg.attr.fields.type & (_SEGMENT_WR >> 8))) )
    {
        x86_emul_hw_exception(seg != x86_seg_ss ? TRAP_gp_fault
                                                : TRAP_stack_error,
                              0, ctxt);
        return X86EMUL_EXCEPTION;
    }

    poc->bpmatch = check_guest_io_breakpoint(curr, port, bytes_per_rep);

    while ( *reps < goal )
    {
        unsigned int data = 0;
        unsigned long addr;

        rc = pv_emul_virt_to_linear(sreg.base, offset, bytes_per_rep,
                                    sreg.limit, seg, ctxt, &addr);
        if ( rc != X86EMUL_OKAY )
            return rc;

        if ( (rc = __copy_from_user(&data, (void *)addr, bytes_per_rep)) != 0 )
        {
            x86_emul_pagefault(0, addr + bytes_per_rep - rc, ctxt);
            return X86EMUL_EXCEPTION;
        }

        guest_io_write(port, bytes_per_rep, data, currd);

        ++*reps;

        if ( poc->bpmatch || hypercall_preempt_check() )
            break;

        /* x86_emulate() clips the repetition count to ensure we don't wrap. */
        if ( unlikely(ctxt->regs->eflags & X86_EFLAGS_DF) )
            offset -= bytes_per_rep;
        else
            offset += bytes_per_rep;
    }

    return X86EMUL_OKAY;
}

static int priv_op_read_cr(unsigned int reg, unsigned long *val,
                           struct x86_emulate_ctxt *ctxt)
{
    const struct vcpu *curr = current;

    switch ( reg )
    {
    case 0: /* Read CR0 */
        *val = (read_cr0() & ~X86_CR0_TS) | curr->arch.pv_vcpu.ctrlreg[0];
        return X86EMUL_OKAY;

    case 2: /* Read CR2 */
    case 4: /* Read CR4 */
        *val = curr->arch.pv_vcpu.ctrlreg[reg];
        return X86EMUL_OKAY;

    case 3: /* Read CR3 */
    {
        const struct domain *currd = curr->domain;
        unsigned long mfn;

        if ( !is_pv_32bit_domain(currd) )
        {
            mfn = pagetable_get_pfn(curr->arch.guest_table);
            *val = xen_pfn_to_cr3(mfn_to_gmfn(currd, mfn));
        }
        else
        {
            l4_pgentry_t *pl4e =
                map_domain_page(_mfn(pagetable_get_pfn(curr->arch.guest_table)));

            mfn = l4e_get_pfn(*pl4e);
            unmap_domain_page(pl4e);
            *val = compat_pfn_to_cr3(mfn_to_gmfn(currd, mfn));
        }
        /* PTs should not be shared */
        BUG_ON(page_get_owner(mfn_to_page(mfn)) == dom_cow);
        return X86EMUL_OKAY;
    }
    }

    return X86EMUL_UNHANDLEABLE;
}

static int priv_op_write_cr(unsigned int reg, unsigned long val,
                            struct x86_emulate_ctxt *ctxt)
{
    struct vcpu *curr = current;

    switch ( reg )
    {
    case 0: /* Write CR0 */
        if ( (val ^ read_cr0()) & ~X86_CR0_TS )
        {
            gdprintk(XENLOG_WARNING,
                    "Attempt to change unmodifiable CR0 flags\n");
            break;
        }
        do_fpu_taskswitch(!!(val & X86_CR0_TS));
        return X86EMUL_OKAY;

    case 2: /* Write CR2 */
        curr->arch.pv_vcpu.ctrlreg[2] = val;
        arch_set_cr2(curr, val);
        return X86EMUL_OKAY;

    case 3: /* Write CR3 */
    {
        struct domain *currd = curr->domain;
        unsigned long gfn;
        struct page_info *page;
        int rc;

        gfn = !is_pv_32bit_domain(currd)
              ? xen_cr3_to_pfn(val) : compat_cr3_to_pfn(val);
        page = get_page_from_gfn(currd, gfn, NULL, P2M_ALLOC);
        if ( !page )
            break;
        rc = new_guest_cr3(page_to_mfn(page));
        put_page(page);

        switch ( rc )
        {
        case 0:
            return X86EMUL_OKAY;
        case -ERESTART: /* retry after preemption */
            return X86EMUL_RETRY;
        }
        break;
    }

    case 4: /* Write CR4 */
        curr->arch.pv_vcpu.ctrlreg[4] = pv_guest_cr4_fixup(curr, val);
        write_cr4(pv_guest_cr4_to_real_cr4(curr));
        ctxt_switch_levelling(curr);
        return X86EMUL_OKAY;
    }

    return X86EMUL_UNHANDLEABLE;
}

static int priv_op_read_dr(unsigned int reg, unsigned long *val,
                           struct x86_emulate_ctxt *ctxt)
{
    unsigned long res = do_get_debugreg(reg);

    if ( IS_ERR_VALUE(res) )
        return X86EMUL_UNHANDLEABLE;

    *val = res;

    return X86EMUL_OKAY;
}

static int priv_op_write_dr(unsigned int reg, unsigned long val,
                            struct x86_emulate_ctxt *ctxt)
{
    return do_set_debugreg(reg, val) == 0
           ? X86EMUL_OKAY : X86EMUL_UNHANDLEABLE;
}

static inline uint64_t guest_misc_enable(uint64_t val)
{
    val &= ~(MSR_IA32_MISC_ENABLE_PERF_AVAIL |
             MSR_IA32_MISC_ENABLE_MONITOR_ENABLE);
    val |= MSR_IA32_MISC_ENABLE_BTS_UNAVAIL |
           MSR_IA32_MISC_ENABLE_PEBS_UNAVAIL |
           MSR_IA32_MISC_ENABLE_XTPR_DISABLE;
    return val;
}

static inline bool is_cpufreq_controller(const struct domain *d)
{
    return ((cpufreq_controller == FREQCTL_dom0_kernel) &&
            is_hardware_domain(d));
}

static int priv_op_read_msr(unsigned int reg, uint64_t *val,
                            struct x86_emulate_ctxt *ctxt)
{
    struct priv_op_ctxt *poc = container_of(ctxt, struct priv_op_ctxt, ctxt);
    const struct vcpu *curr = current;
    const struct domain *currd = curr->domain;
    bool vpmu_msr = false;

    switch ( reg )
    {
        int rc;

    case MSR_FS_BASE:
        if ( is_pv_32bit_domain(currd) )
            break;
        *val = cpu_has_fsgsbase ? __rdfsbase() : curr->arch.pv_vcpu.fs_base;
        return X86EMUL_OKAY;

    case MSR_GS_BASE:
        if ( is_pv_32bit_domain(currd) )
            break;
        *val = cpu_has_fsgsbase ? __rdgsbase()
                                : curr->arch.pv_vcpu.gs_base_kernel;
        return X86EMUL_OKAY;

    case MSR_SHADOW_GS_BASE:
        if ( is_pv_32bit_domain(currd) )
            break;
        *val = curr->arch.pv_vcpu.gs_base_user;
        return X86EMUL_OKAY;

    /*
     * In order to fully retain original behavior, defer calling
     * pv_soft_rdtsc() until after emulation. This may want/need to be
     * reconsidered.
     */
    case MSR_IA32_TSC:
        poc->tsc |= TSC_BASE;
        goto normal;

    case MSR_TSC_AUX:
        poc->tsc |= TSC_AUX;
        if ( cpu_has_rdtscp )
            goto normal;
        *val = 0;
        return X86EMUL_OKAY;

    case MSR_EFER:
        *val = read_efer();
        if ( is_pv_32bit_domain(currd) )
            *val &= ~(EFER_LME | EFER_LMA | EFER_LMSLE);
        return X86EMUL_OKAY;

    case MSR_K7_FID_VID_CTL:
    case MSR_K7_FID_VID_STATUS:
    case MSR_K8_PSTATE_LIMIT:
    case MSR_K8_PSTATE_CTRL:
    case MSR_K8_PSTATE_STATUS:
    case MSR_K8_PSTATE0:
    case MSR_K8_PSTATE1:
    case MSR_K8_PSTATE2:
    case MSR_K8_PSTATE3:
    case MSR_K8_PSTATE4:
    case MSR_K8_PSTATE5:
    case MSR_K8_PSTATE6:
    case MSR_K8_PSTATE7:
        if ( boot_cpu_data.x86_vendor != X86_VENDOR_AMD )
            break;
        if ( unlikely(is_cpufreq_controller(currd)) )
            goto normal;
        *val = 0;
        return X86EMUL_OKAY;

    case MSR_IA32_UCODE_REV:
        BUILD_BUG_ON(MSR_IA32_UCODE_REV != MSR_AMD_PATCHLEVEL);
        if ( boot_cpu_data.x86_vendor == X86_VENDOR_INTEL )
        {
            if ( wrmsr_safe(MSR_IA32_UCODE_REV, 0) )
                break;
            /* As documented in the SDM: Do a CPUID 1 here */
            cpuid_eax(1);
        }
        goto normal;

    case MSR_IA32_MISC_ENABLE:
        if ( rdmsr_safe(reg, *val) )
            break;
        *val = guest_misc_enable(*val);
        return X86EMUL_OKAY;

    case MSR_AMD64_DR0_ADDRESS_MASK:
        if ( !boot_cpu_has(X86_FEATURE_DBEXT) )
            break;
        *val = curr->arch.pv_vcpu.dr_mask[0];
        return X86EMUL_OKAY;

    case MSR_AMD64_DR1_ADDRESS_MASK ... MSR_AMD64_DR3_ADDRESS_MASK:
        if ( !boot_cpu_has(X86_FEATURE_DBEXT) )
            break;
        *val = curr->arch.pv_vcpu.dr_mask[reg - MSR_AMD64_DR1_ADDRESS_MASK + 1];
        return X86EMUL_OKAY;

    case MSR_IA32_PERF_CAPABILITIES:
        /* No extra capabilities are supported. */
        *val = 0;
        return X86EMUL_OKAY;

    case MSR_PRED_CMD:
        /* Write-only */
        break;

    case MSR_SPEC_CTRL:
        if ( !currd->arch.cpuid->feat.ibrsb )
            break;
        *val = curr->arch.spec_ctrl;
        return X86EMUL_OKAY;

    case MSR_INTEL_PLATFORM_INFO:
        if ( !boot_cpu_has(X86_FEATURE_MSR_PLATFORM_INFO) )
            break;
        *val = 0;
        if ( this_cpu(cpuid_faulting_enabled) )
            *val |= MSR_PLATFORM_INFO_CPUID_FAULTING;
        return X86EMUL_OKAY;

    case MSR_ARCH_CAPABILITIES:
        /* Not implemented yet. */
        break;

    case MSR_INTEL_MISC_FEATURES_ENABLES:
        if ( !boot_cpu_has(X86_FEATURE_MSR_MISC_FEATURES) )
            break;
        *val = 0;
        if ( curr->arch.cpuid_faulting )
            *val |= MSR_MISC_FEATURES_CPUID_FAULTING;
        return X86EMUL_OKAY;

    case MSR_P6_PERFCTR(0)...MSR_P6_PERFCTR(7):
    case MSR_P6_EVNTSEL(0)...MSR_P6_EVNTSEL(3):
    case MSR_CORE_PERF_FIXED_CTR0...MSR_CORE_PERF_FIXED_CTR2:
    case MSR_CORE_PERF_FIXED_CTR_CTRL...MSR_CORE_PERF_GLOBAL_OVF_CTRL:
        if ( boot_cpu_data.x86_vendor == X86_VENDOR_INTEL )
        {
            vpmu_msr = true;
            /* fall through */
    case MSR_AMD_FAM15H_EVNTSEL0...MSR_AMD_FAM15H_PERFCTR5:
    case MSR_K7_EVNTSEL0...MSR_K7_PERFCTR3:
            if ( vpmu_msr || (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) )
            {
                if ( vpmu_do_rdmsr(reg, val) )
                    break;
                return X86EMUL_OKAY;
            }
        }
        /* fall through */
    default:
        if ( rdmsr_hypervisor_regs(reg, val) )
            return X86EMUL_OKAY;

        rc = vmce_rdmsr(reg, val);
        if ( rc < 0 )
            break;
        if ( rc )
            return X86EMUL_OKAY;
        /* fall through */
    normal:
        /* Everyone can read the MSR space. */
        /* gdprintk(XENLOG_WARNING, "Domain attempted RDMSR %08x\n", reg); */
        if ( rdmsr_safe(reg, *val) )
            break;
        return X86EMUL_OKAY;
    }

    return X86EMUL_UNHANDLEABLE;
}

#include "x86_64/mmconfig.h"

static int priv_op_write_msr(unsigned int reg, uint64_t val,
                             struct x86_emulate_ctxt *ctxt)
{
    struct vcpu *curr = current;
    const struct domain *currd = curr->domain;
    bool vpmu_msr = false;

    switch ( reg )
    {
        uint64_t temp;
        int rc;

    case MSR_FS_BASE:
        if ( is_pv_32bit_domain(currd) || !is_canonical_address(val) )
            break;
        wrfsbase(val);
        curr->arch.pv_vcpu.fs_base = val;
        return X86EMUL_OKAY;

    case MSR_GS_BASE:
        if ( is_pv_32bit_domain(currd) || !is_canonical_address(val) )
            break;
        wrgsbase(val);
        curr->arch.pv_vcpu.gs_base_kernel = val;
        return X86EMUL_OKAY;

    case MSR_SHADOW_GS_BASE:
        if ( is_pv_32bit_domain(currd) || !is_canonical_address(val) )
            break;
        wrmsrl(MSR_SHADOW_GS_BASE, val);
        curr->arch.pv_vcpu.gs_base_user = val;
        return X86EMUL_OKAY;

    case MSR_K7_FID_VID_STATUS:
    case MSR_K7_FID_VID_CTL:
    case MSR_K8_PSTATE_LIMIT:
    case MSR_K8_PSTATE_CTRL:
    case MSR_K8_PSTATE_STATUS:
    case MSR_K8_PSTATE0:
    case MSR_K8_PSTATE1:
    case MSR_K8_PSTATE2:
    case MSR_K8_PSTATE3:
    case MSR_K8_PSTATE4:
    case MSR_K8_PSTATE5:
    case MSR_K8_PSTATE6:
    case MSR_K8_PSTATE7:
    case MSR_K8_HWCR:
        if ( boot_cpu_data.x86_vendor != X86_VENDOR_AMD )
            break;
        if ( likely(!is_cpufreq_controller(currd)) ||
             wrmsr_safe(reg, val) == 0 )
            return X86EMUL_OKAY;
        break;

    case MSR_AMD64_NB_CFG:
        if ( boot_cpu_data.x86_vendor != X86_VENDOR_AMD ||
             boot_cpu_data.x86 < 0x10 || boot_cpu_data.x86 > 0x17 )
            break;
        if ( !is_hardware_domain(currd) || !is_pinned_vcpu(curr) )
            return X86EMUL_OKAY;
        if ( (rdmsr_safe(MSR_AMD64_NB_CFG, temp) != 0) ||
             ((val ^ temp) & ~(1ULL << AMD64_NB_CFG_CF8_EXT_ENABLE_BIT)) )
            goto invalid;
        if ( wrmsr_safe(MSR_AMD64_NB_CFG, val) == 0 )
            return X86EMUL_OKAY;
        break;

    case MSR_FAM10H_MMIO_CONF_BASE:
        if ( boot_cpu_data.x86_vendor != X86_VENDOR_AMD ||
             boot_cpu_data.x86 < 0x10 || boot_cpu_data.x86 > 0x17 )
            break;
        if ( !is_hardware_domain(currd) || !is_pinned_vcpu(curr) )
            return X86EMUL_OKAY;
        if ( rdmsr_safe(MSR_FAM10H_MMIO_CONF_BASE, temp) != 0 )
            break;
        if ( (pci_probe & PCI_PROBE_MASK) == PCI_PROBE_MMCONF ?
             temp != val :
             ((temp ^ val) &
              ~(FAM10H_MMIO_CONF_ENABLE |
                (FAM10H_MMIO_CONF_BUSRANGE_MASK <<
                 FAM10H_MMIO_CONF_BUSRANGE_SHIFT) |
                ((u64)FAM10H_MMIO_CONF_BASE_MASK <<
                 FAM10H_MMIO_CONF_BASE_SHIFT))) )
            goto invalid;
        if ( wrmsr_safe(MSR_FAM10H_MMIO_CONF_BASE, val) == 0 )
            return X86EMUL_OKAY;
        break;

    case MSR_IA32_UCODE_REV:
        if ( boot_cpu_data.x86_vendor != X86_VENDOR_INTEL )
            break;
        if ( !is_hardware_domain(currd) || !is_pinned_vcpu(curr) )
            return X86EMUL_OKAY;
        if ( rdmsr_safe(reg, temp) )
            break;
        if ( val )
            goto invalid;
        return X86EMUL_OKAY;

    case MSR_IA32_MISC_ENABLE:
        if ( rdmsr_safe(reg, temp) )
            break;
        if ( val != guest_misc_enable(temp) )
            goto invalid;
        return X86EMUL_OKAY;

    case MSR_IA32_MPERF:
    case MSR_IA32_APERF:
        if ( (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) &&
             (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) )
            break;
        if ( likely(!is_cpufreq_controller(currd)) ||
             wrmsr_safe(reg, val) == 0 )
            return X86EMUL_OKAY;
        break;

    case MSR_IA32_PERF_CTL:
        if ( boot_cpu_data.x86_vendor != X86_VENDOR_INTEL )
            break;
        if ( likely(!is_cpufreq_controller(currd)) ||
             wrmsr_safe(reg, val) == 0 )
            return X86EMUL_OKAY;
        break;

    case MSR_IA32_THERM_CONTROL:
    case MSR_IA32_ENERGY_PERF_BIAS:
        if ( boot_cpu_data.x86_vendor != X86_VENDOR_INTEL )
            break;
        if ( !is_hardware_domain(currd) || !is_pinned_vcpu(curr) ||
             wrmsr_safe(reg, val) == 0 )
            return X86EMUL_OKAY;
        break;

    case MSR_AMD64_DR0_ADDRESS_MASK:
        if ( !boot_cpu_has(X86_FEATURE_DBEXT) || (val >> 32) )
            break;
        curr->arch.pv_vcpu.dr_mask[0] = val;
        if ( curr->arch.debugreg[7] & DR7_ACTIVE_MASK )
            wrmsrl(MSR_AMD64_DR0_ADDRESS_MASK, val);
        return X86EMUL_OKAY;

    case MSR_AMD64_DR1_ADDRESS_MASK ... MSR_AMD64_DR3_ADDRESS_MASK:
        if ( !boot_cpu_has(X86_FEATURE_DBEXT) || (val >> 32) )
            break;
        curr->arch.pv_vcpu.dr_mask[reg - MSR_AMD64_DR1_ADDRESS_MASK + 1] = val;
        if ( curr->arch.debugreg[7] & DR7_ACTIVE_MASK )
            wrmsrl(reg, val);
        return X86EMUL_OKAY;

    case MSR_INTEL_PLATFORM_INFO:
    case MSR_ARCH_CAPABILITIES:
        /* The MSR is read-only. */
        break;

    case MSR_SPEC_CTRL:
        if ( !currd->arch.cpuid->feat.ibrsb )
            break; /* MSR available? */

        /*
         * Note: SPEC_CTRL_STIBP is specified as safe to use (i.e. ignored)
         * when STIBP isn't enumerated in hardware.
         */

        if ( val & ~(SPEC_CTRL_IBRS | SPEC_CTRL_STIBP) )
            break; /* Rsvd bit set? */

        curr->arch.spec_ctrl = val;
        return X86EMUL_OKAY;

    case MSR_PRED_CMD:
        if ( !currd->arch.cpuid->feat.ibrsb && !currd->arch.cpuid->extd.ibpb )
            break; /* MSR available? */

        /*
         * The only defined behaviour is when writing PRED_CMD_IBPB.  In
         * practice, real hardware accepts any value without faulting.
         */
        if ( val & PRED_CMD_IBPB )
            wrmsrl(MSR_PRED_CMD, PRED_CMD_IBPB);
        return X86EMUL_OKAY;

    case MSR_INTEL_MISC_FEATURES_ENABLES:
        if ( !boot_cpu_has(X86_FEATURE_MSR_MISC_FEATURES) ||
             (val & ~MSR_MISC_FEATURES_CPUID_FAULTING) )
            break;
        if ( (val & MSR_MISC_FEATURES_CPUID_FAULTING) &&
             !this_cpu(cpuid_faulting_enabled) )
            break;
        curr->arch.cpuid_faulting = !!(val & MSR_MISC_FEATURES_CPUID_FAULTING);
        return X86EMUL_OKAY;

    case MSR_P6_PERFCTR(0)...MSR_P6_PERFCTR(7):
    case MSR_P6_EVNTSEL(0)...MSR_P6_EVNTSEL(3):
    case MSR_CORE_PERF_FIXED_CTR0...MSR_CORE_PERF_FIXED_CTR2:
    case MSR_CORE_PERF_FIXED_CTR_CTRL...MSR_CORE_PERF_GLOBAL_OVF_CTRL:
        if ( boot_cpu_data.x86_vendor == X86_VENDOR_INTEL )
        {
            vpmu_msr = true;
    case MSR_AMD_FAM15H_EVNTSEL0...MSR_AMD_FAM15H_PERFCTR5:
    case MSR_K7_EVNTSEL0...MSR_K7_PERFCTR3:
            if ( vpmu_msr || (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) )
            {
                if ( (vpmu_mode & XENPMU_MODE_ALL) &&
                     !is_hardware_domain(currd) )
                    return X86EMUL_OKAY;

                if ( vpmu_do_wrmsr(reg, val, 0) )
                    break;
                return X86EMUL_OKAY;
            }
        }
        /* fall through */
    default:
        if ( wrmsr_hypervisor_regs(reg, val) == 1 )
            return X86EMUL_OKAY;

        rc = vmce_wrmsr(reg, val);
        if ( rc < 0 )
            break;
        if ( rc )
            return X86EMUL_OKAY;

        if ( (rdmsr_safe(reg, temp) != 0) || (val != temp) )
    invalid:
            gdprintk(XENLOG_WARNING,
                     "Domain attempted WRMSR %08x from 0x%016"PRIx64" to 0x%016"PRIx64"\n",
                     reg, temp, val);
        return X86EMUL_OKAY;
    }

    return X86EMUL_UNHANDLEABLE;
}

static int priv_op_wbinvd(struct x86_emulate_ctxt *ctxt)
{
    /* Ignore the instruction if unprivileged. */
    if ( !cache_flush_permitted(current->domain) )
        /*
         * Non-physdev domain attempted WBINVD; ignore for now since
         * newer linux uses this in some start-of-day timing loops.
         */
        ;
    else
        wbinvd();

    return X86EMUL_OKAY;
}

int pv_emul_cpuid(uint32_t leaf, uint32_t subleaf,
                  struct cpuid_leaf *res, struct x86_emulate_ctxt *ctxt)
{
    guest_cpuid(current, leaf, subleaf, res);

    return X86EMUL_OKAY;
}

static int priv_op_validate(const struct x86_emulate_state *state,
                            struct x86_emulate_ctxt *ctxt)
{
    switch ( ctxt->opcode )
    {
    case 0x6c ... 0x6f: /* ins / outs */
    case 0xe4 ... 0xe7: /* in / out (immediate port) */
    case 0xec ... 0xef: /* in / out (port in %dx) */
    case X86EMUL_OPC(0x0f, 0x06): /* clts */
    case X86EMUL_OPC(0x0f, 0x09): /* wbinvd */
    case X86EMUL_OPC(0x0f, 0x20) ...
         X86EMUL_OPC(0x0f, 0x23): /* mov to/from cr/dr */
    case X86EMUL_OPC(0x0f, 0x30): /* wrmsr */
    case X86EMUL_OPC(0x0f, 0x31): /* rdtsc */
    case X86EMUL_OPC(0x0f, 0x32): /* rdmsr */
    case X86EMUL_OPC(0x0f, 0xa2): /* cpuid */
        return X86EMUL_OKAY;

    case 0xfa: case 0xfb: /* cli / sti */
        if ( !iopl_ok(current, ctxt->regs) )
            break;
        /*
         * This is just too dangerous to allow, in my opinion. Consider if the
         * caller then tries to reenable interrupts using POPF: we can't trap
         * that and we'll end up with hard-to-debug lockups. Fast & loose will
         * do for us. :-)
        vcpu_info(current, evtchn_upcall_mask) = (ctxt->opcode == 0xfa);
         */
        return X86EMUL_DONE;

    case X86EMUL_OPC(0x0f, 0x01):
    {
        unsigned int modrm_rm, modrm_reg;

        if ( x86_insn_modrm(state, &modrm_rm, &modrm_reg) != 3 ||
             (modrm_rm & 7) != 1 )
            break;
        switch ( modrm_reg & 7 )
        {
        case 2: /* xsetbv */
        case 7: /* rdtscp */
            return X86EMUL_OKAY;
        }
        break;
    }
    }

    return X86EMUL_UNHANDLEABLE;
}

static const struct x86_emulate_ops priv_op_ops = {
    .insn_fetch          = priv_op_insn_fetch,
    .read                = x86emul_unhandleable_rw,
    .validate            = priv_op_validate,
    .read_io             = priv_op_read_io,
    .write_io            = priv_op_write_io,
    .rep_ins             = priv_op_rep_ins,
    .rep_outs            = priv_op_rep_outs,
    .read_segment        = priv_op_read_segment,
    .read_cr             = priv_op_read_cr,
    .write_cr            = priv_op_write_cr,
    .read_dr             = priv_op_read_dr,
    .write_dr            = priv_op_write_dr,
    .read_msr            = priv_op_read_msr,
    .write_msr           = priv_op_write_msr,
    .cpuid               = pv_emul_cpuid,
    .wbinvd              = priv_op_wbinvd,
};

static int emulate_privileged_op(struct cpu_user_regs *regs)
{
    struct vcpu *curr = current;
    struct domain *currd = curr->domain;
    struct priv_op_ctxt ctxt = {
        .ctxt.regs = regs,
        .ctxt.vendor = currd->arch.cpuid->x86_vendor,
        .ctxt.lma = !is_pv_32bit_domain(currd),
    };
    int rc;
    unsigned int eflags, ar;

    if ( !read_descriptor(regs->cs, curr, &ctxt.cs.base, &ctxt.cs.limit,
                          &ar, 1) ||
         !(ar & _SEGMENT_S) ||
         !(ar & _SEGMENT_P) ||
         !(ar & _SEGMENT_CODE) )
        return 0;

    /* Mirror virtualized state into EFLAGS. */
    ASSERT(regs->eflags & X86_EFLAGS_IF);
    if ( vcpu_info(curr, evtchn_upcall_mask) )
        regs->eflags &= ~X86_EFLAGS_IF;
    else
        regs->eflags |= X86_EFLAGS_IF;
    ASSERT(!(regs->eflags & X86_EFLAGS_IOPL));
    regs->eflags |= curr->arch.pv_vcpu.iopl;
    eflags = regs->eflags;

    ctxt.ctxt.addr_size = ar & _SEGMENT_L ? 64 : ar & _SEGMENT_DB ? 32 : 16;
    /* Leave zero in ctxt.ctxt.sp_size, as it's not needed. */
    rc = x86_emulate(&ctxt.ctxt, &priv_op_ops);

    if ( ctxt.io_emul_stub )
        unmap_domain_page(ctxt.io_emul_stub);

    /*
     * Un-mirror virtualized state from EFLAGS.
     * Nothing we allow to be emulated can change anything other than the
     * arithmetic bits, and the resume flag.
     */
    ASSERT(!((regs->eflags ^ eflags) &
             ~(X86_EFLAGS_RF | X86_EFLAGS_ARITH_MASK)));
    regs->eflags |= X86_EFLAGS_IF;
    regs->eflags &= ~X86_EFLAGS_IOPL;

    switch ( rc )
    {
    case X86EMUL_OKAY:
        if ( ctxt.tsc & TSC_BASE )
        {
            if ( ctxt.tsc & TSC_AUX )
                pv_soft_rdtsc(curr, regs, 1);
            else if ( currd->arch.vtsc )
                pv_soft_rdtsc(curr, regs, 0);
            else
                msr_split(regs, rdtsc());
        }

        if ( ctxt.ctxt.retire.singlestep )
            ctxt.bpmatch |= DR_STEP;
        if ( ctxt.bpmatch )
        {
            curr->arch.debugreg[6] |= ctxt.bpmatch | DR_STATUS_RESERVED_ONE;
            if ( !(curr->arch.pv_vcpu.trap_bounce.flags & TBF_EXCEPTION) )
                do_guest_trap(TRAP_debug, regs);
        }
        /* fall through */
    case X86EMUL_RETRY:
        return EXCRET_fault_fixed;

    case X86EMUL_EXCEPTION:
        pv_inject_event(&ctxt.ctxt.event);
        return EXCRET_fault_fixed;
    }

    return 0;
}

static inline int check_stack_limit(unsigned int ar, unsigned int limit,
                                    unsigned int esp, unsigned int decr)
{
    return (((esp - decr) < (esp - 1)) &&
            (!(ar & _SEGMENT_EC) ? (esp - 1) <= limit : (esp - decr) > limit));
}

struct gate_op_ctxt {
    struct x86_emulate_ctxt ctxt;
    struct {
        unsigned long base, limit;
    } cs;
    bool insn_fetch;
};

static int gate_op_read(
    enum x86_segment seg,
    unsigned long offset,
    void *p_data,
    unsigned int bytes,
    struct x86_emulate_ctxt *ctxt)
{
    const struct gate_op_ctxt *goc =
        container_of(ctxt, struct gate_op_ctxt, ctxt);
    unsigned int rc = bytes, sel = 0;
    unsigned long addr = offset, limit = 0;

    switch ( seg )
    {
    case x86_seg_cs:
        addr += goc->cs.base;
        limit = goc->cs.limit;
        break;
    case x86_seg_ds:
        sel = read_sreg(ds);
        break;
    case x86_seg_es:
        sel = read_sreg(es);
        break;
    case x86_seg_fs:
        sel = read_sreg(fs);
        break;
    case x86_seg_gs:
        sel = read_sreg(gs);
        break;
    case x86_seg_ss:
        sel = ctxt->regs->ss;
        break;
    default:
        return X86EMUL_UNHANDLEABLE;
    }
    if ( sel )
    {
        unsigned int ar;

        ASSERT(!goc->insn_fetch);
        if ( !read_descriptor(sel, current, &addr, &limit, &ar, 0) ||
             !(ar & _SEGMENT_S) ||
             !(ar & _SEGMENT_P) ||
             ((ar & _SEGMENT_CODE) && !(ar & _SEGMENT_WR)) )
            return X86EMUL_UNHANDLEABLE;
        addr += offset;
    }
    else if ( seg != x86_seg_cs )
        return X86EMUL_UNHANDLEABLE;

    /* We don't mean to emulate any branches. */
    if ( limit < bytes - 1 || offset > limit - bytes + 1 )
        return X86EMUL_UNHANDLEABLE;

    addr = (uint32_t)addr;

    if ( (rc = __copy_from_user(p_data, (void *)addr, bytes)) )
    {
        /*
         * TODO: This should report PFEC_insn_fetch when goc->insn_fetch &&
         * cpu_has_nx, but we'd then need a "fetch" variant of
         * __copy_from_user() respecting NX, SMEP, and protection keys.
         */
        x86_emul_pagefault(0, addr + bytes - rc, ctxt);
        return X86EMUL_EXCEPTION;
    }

    return X86EMUL_OKAY;
}

static void emulate_gate_op(struct cpu_user_regs *regs)
{
    struct vcpu *v = current;
    unsigned int sel, ar, dpl, nparm, insn_len;
    struct gate_op_ctxt ctxt = { .ctxt.regs = regs, .insn_fetch = true };
    struct x86_emulate_state *state;
    unsigned long off, base, limit;
    uint16_t opnd_sel = 0;
    int jump = -1, rc = X86EMUL_OKAY;

    /* Check whether this fault is due to the use of a call gate. */
    if ( !read_gate_descriptor(regs->error_code, v, &sel, &off, &ar) ||
         (((ar >> 13) & 3) < (regs->cs & 3)) ||
         ((ar & _SEGMENT_TYPE) != 0xc00) )
    {
        do_guest_trap(TRAP_gp_fault, regs);
        return;
    }
    if ( !(ar & _SEGMENT_P) )
    {
        do_guest_trap(TRAP_no_segment, regs);
        return;
    }
    dpl = (ar >> 13) & 3;
    nparm = ar & 0x1f;

    /*
     * Decode instruction (and perhaps operand) to determine RPL,
     * whether this is a jump or a call, and the call return offset.
     */
    if ( !read_descriptor(regs->cs, v, &ctxt.cs.base, &ctxt.cs.limit,
                          &ar, 0) ||
         !(ar & _SEGMENT_S) ||
         !(ar & _SEGMENT_P) ||
         !(ar & _SEGMENT_CODE) )
    {
        do_guest_trap(TRAP_gp_fault, regs);
        return;
    }

    ctxt.ctxt.addr_size = ar & _SEGMENT_DB ? 32 : 16;
    /* Leave zero in ctxt.ctxt.sp_size, as it's not needed for decoding. */
    state = x86_decode_insn(&ctxt.ctxt, gate_op_read);
    ctxt.insn_fetch = false;
    if ( IS_ERR_OR_NULL(state) )
    {
        if ( PTR_ERR(state) == -X86EMUL_EXCEPTION )
            pv_inject_event(&ctxt.ctxt.event);
        else
            do_guest_trap(TRAP_gp_fault, regs);
        return;
    }

    switch ( ctxt.ctxt.opcode )
    {
        unsigned int modrm_345;

    case 0xea:
        ++jump;
        /* fall through */
    case 0x9a:
        ++jump;
        opnd_sel = x86_insn_immediate(state, 1);
        break;
    case 0xff:
        if ( x86_insn_modrm(state, NULL, &modrm_345) >= 3 )
            break;
        switch ( modrm_345 & 7 )
        {
            enum x86_segment seg;

        case 5:
            ++jump;
            /* fall through */
        case 3:
            ++jump;
            base = x86_insn_operand_ea(state, &seg);
            rc = gate_op_read(seg,
                              base + (x86_insn_opsize(state) >> 3),
                              &opnd_sel, sizeof(opnd_sel), &ctxt.ctxt);
            break;
        }
        break;
    }

    insn_len = x86_insn_length(state, &ctxt.ctxt);
    x86_emulate_free_state(state);

    if ( rc == X86EMUL_EXCEPTION )
    {
        pv_inject_event(&ctxt.ctxt.event);
        return;
    }

    if ( rc != X86EMUL_OKAY ||
         jump < 0 ||
         (opnd_sel & ~3) != regs->error_code ||
         dpl < (opnd_sel & 3) )
    {
        do_guest_trap(TRAP_gp_fault, regs);
        return;
    }

    if ( !read_descriptor(sel, v, &base, &limit, &ar, 0) ||
         !(ar & _SEGMENT_S) ||
         !(ar & _SEGMENT_CODE) ||
         (!jump || (ar & _SEGMENT_EC) ?
          ((ar >> 13) & 3) > (regs->cs & 3) :
          ((ar >> 13) & 3) != (regs->cs & 3)) )
    {
        pv_inject_hw_exception(TRAP_gp_fault, sel);
        return;
    }
    if ( !(ar & _SEGMENT_P) )
    {
        pv_inject_hw_exception(TRAP_no_segment, sel);
        return;
    }
    if ( off > limit )
    {
        pv_inject_hw_exception(TRAP_gp_fault, 0);
        return;
    }

    if ( !jump )
    {
        unsigned int ss, esp, *stkp;
        int rc;
#define push(item) do \
        { \
            --stkp; \
            esp -= 4; \
            rc = __put_user(item, stkp); \
            if ( rc ) \
            { \
                pv_inject_page_fault(PFEC_write_access, \
                                     (unsigned long)(stkp + 1) - rc); \
                return; \
            } \
        } while ( 0 )

        if ( ((ar >> 13) & 3) < (regs->cs & 3) )
        {
            sel |= (ar >> 13) & 3;
            /* Inner stack known only for kernel ring. */
            if ( (sel & 3) != GUEST_KERNEL_RPL(v->domain) )
            {
                do_guest_trap(TRAP_gp_fault, regs);
                return;
            }
            esp = v->arch.pv_vcpu.kernel_sp;
            ss = v->arch.pv_vcpu.kernel_ss;
            if ( (ss & 3) != (sel & 3) ||
                 !read_descriptor(ss, v, &base, &limit, &ar, 0) ||
                 ((ar >> 13) & 3) != (sel & 3) ||
                 !(ar & _SEGMENT_S) ||
                 (ar & _SEGMENT_CODE) ||
                 !(ar & _SEGMENT_WR) )
            {
                pv_inject_hw_exception(TRAP_invalid_tss, ss & ~3);
                return;
            }
            if ( !(ar & _SEGMENT_P) ||
                 !check_stack_limit(ar, limit, esp, (4 + nparm) * 4) )
            {
                pv_inject_hw_exception(TRAP_stack_error, ss & ~3);
                return;
            }
            stkp = (unsigned int *)(unsigned long)((unsigned int)base + esp);
            if ( !compat_access_ok(stkp - 4 - nparm, 16 + nparm * 4) )
            {
                do_guest_trap(TRAP_gp_fault, regs);
                return;
            }
            push(regs->ss);
            push(regs->rsp);
            if ( nparm )
            {
                const unsigned int *ustkp;

                if ( !read_descriptor(regs->ss, v, &base, &limit, &ar, 0) ||
                     ((ar >> 13) & 3) != (regs->cs & 3) ||
                     !(ar & _SEGMENT_S) ||
                     (ar & _SEGMENT_CODE) ||
                     !(ar & _SEGMENT_WR) ||
                     !check_stack_limit(ar, limit, esp + nparm * 4, nparm * 4) )
                    return do_guest_trap(TRAP_gp_fault, regs);
                ustkp = (unsigned int *)(unsigned long)
                        ((unsigned int)base + regs->esp + nparm * 4);
                if ( !compat_access_ok(ustkp - nparm, 0 + nparm * 4) )
                {
                    do_guest_trap(TRAP_gp_fault, regs);
                    return;
                }
                do
                {
                    unsigned int parm;

                    --ustkp;
                    rc = __get_user(parm, ustkp);
                    if ( rc )
                    {
                        pv_inject_page_fault(0, (unsigned long)(ustkp + 1) - rc);
                        return;
                    }
                    push(parm);
                } while ( --nparm );
            }
        }
        else
        {
            sel |= (regs->cs & 3);
            esp = regs->rsp;
            ss = regs->ss;
            if ( !read_descriptor(ss, v, &base, &limit, &ar, 0) ||
                 ((ar >> 13) & 3) != (sel & 3) )
            {
                do_guest_trap(TRAP_gp_fault, regs);
                return;
            }
            if ( !check_stack_limit(ar, limit, esp, 2 * 4) )
            {
                pv_inject_hw_exception(TRAP_stack_error, 0);
                return;
            }
            stkp = (unsigned int *)(unsigned long)((unsigned int)base + esp);
            if ( !compat_access_ok(stkp - 2, 2 * 4) )
            {
                do_guest_trap(TRAP_gp_fault, regs);
                return;
            }
        }
        push(regs->cs);
        push(regs->rip + insn_len);
#undef push
        regs->rsp = esp;
        regs->ss = ss;
    }
    else
        sel |= (regs->cs & 3);

    regs->cs = sel;
    instruction_done(regs, off);
}

void do_general_protection(struct cpu_user_regs *regs)
{
    struct vcpu *v = current;
    unsigned long fixup;

    if ( debugger_trap_entry(TRAP_gp_fault, regs) )
        return;

    if ( regs->error_code & X86_XEC_EXT )
        goto hardware_gp;

    if ( !guest_mode(regs) )
        goto gp_in_kernel;

    /*
     * Cunning trick to allow arbitrary "INT n" handling.
     * 
     * We set DPL == 0 on all vectors in the IDT. This prevents any INT <n>
     * instruction from trapping to the appropriate vector, when that might not
     * be expected by Xen or the guest OS. For example, that entry might be for
     * a fault handler (unlike traps, faults don't increment EIP), or might
     * expect an error code on the stack (which a software trap never
     * provides), or might be a hardware interrupt handler that doesn't like
     * being called spuriously.
     * 
     * Instead, a GPF occurs with the faulting IDT vector in the error code.
     * Bit 1 is set to indicate that an IDT entry caused the fault. Bit 0 is 
     * clear (which got already checked above) to indicate that it's a software
     * fault, not a hardware one.
     * 
     * NOTE: Vectors 3 and 4 are dealt with from their own handler. This is
     * okay because they can only be triggered by an explicit DPL-checked
     * instruction. The DPL specified by the guest OS for these vectors is NOT
     * CHECKED!!
     */
    if ( regs->error_code & X86_XEC_IDT )
    {
        /* This fault must be due to <INT n> instruction. */
        const struct trap_info *ti;
        unsigned char vector = regs->error_code >> 3;
        ti = &v->arch.pv_vcpu.trap_ctxt[vector];
        if ( permit_softint(TI_GET_DPL(ti), v, regs) )
        {
            regs->rip += 2;
            pv_inject_sw_interrupt(vector);
            return;
        }
    }
    else if ( is_pv_32bit_vcpu(v) && regs->error_code )
    {
        emulate_gate_op(regs);
        return;
    }

    /* Emulate some simple privileged and I/O instructions. */
    if ( (regs->error_code == 0) &&
         emulate_privileged_op(regs) )
    {
        trace_trap_one_addr(TRC_PV_EMULATE_PRIVOP, regs->rip);
        return;
    }

    /* Pass on GPF as is. */
    do_guest_trap(TRAP_gp_fault, regs);
    return;

 gp_in_kernel:

    if ( likely((fixup = search_exception_table(regs)) != 0) )
    {
        dprintk(XENLOG_INFO, "GPF (%04x): %p [%ps] -> %p\n",
                regs->error_code, _p(regs->rip), _p(regs->rip), _p(fixup));
        this_cpu(last_extable_addr) = regs->rip;
        regs->rip = fixup;
        return;
    }

 hardware_gp:
    if ( debugger_trap_fatal(TRAP_gp_fault, regs) )
        return;

    show_execution_state(regs);
    panic("GENERAL PROTECTION FAULT\n[error_code=%04x]", regs->error_code);
}

static DEFINE_PER_CPU(struct softirq_trap, softirq_trap);

static void nmi_mce_softirq(void)
{
    int cpu = smp_processor_id();
    struct softirq_trap *st = &per_cpu(softirq_trap, cpu);

    BUG_ON(st->vcpu == NULL);

    /* Set the tmp value unconditionally, so that
     * the check in the iret hypercall works. */
    cpumask_copy(st->vcpu->cpu_hard_affinity_tmp,
                 st->vcpu->cpu_hard_affinity);

    if ((cpu != st->processor)
       || (st->processor != st->vcpu->processor))
    {
        /* We are on a different physical cpu.
         * Make sure to wakeup the vcpu on the
         * specified processor.
         */
        vcpu_set_hard_affinity(st->vcpu, cpumask_of(st->processor));

        /* Affinity is restored in the iret hypercall. */
    }

    /* Only used to defer wakeup of domain/vcpu to
     * a safe (non-NMI/MCE) context.
     */
    vcpu_kick(st->vcpu);
    st->vcpu = NULL;
}

static void pci_serr_softirq(void)
{
    printk("\n\nNMI - PCI system error (SERR)\n");
    outb(inb(0x61) & 0x0b, 0x61); /* re-enable the PCI SERR error line. */
}

void async_exception_cleanup(struct vcpu *curr)
{
    int trap;

    if ( !curr->async_exception_mask )
        return;

    /* Restore affinity.  */
    if ( !cpumask_empty(curr->cpu_hard_affinity_tmp) &&
         !cpumask_equal(curr->cpu_hard_affinity_tmp, curr->cpu_hard_affinity) )
    {
        vcpu_set_hard_affinity(curr, curr->cpu_hard_affinity_tmp);
        cpumask_clear(curr->cpu_hard_affinity_tmp);
    }

    if ( !(curr->async_exception_mask & (curr->async_exception_mask - 1)) )
        trap = __scanbit(curr->async_exception_mask, VCPU_TRAP_NONE);
    else
        for ( trap = VCPU_TRAP_NONE + 1; trap <= VCPU_TRAP_LAST; ++trap )
            if ( (curr->async_exception_mask ^
                  curr->async_exception_state(trap).old_mask) == (1 << trap) )
                break;
    if ( unlikely(trap > VCPU_TRAP_LAST) )
    {
        ASSERT_UNREACHABLE();
        return;
    }

    /* Restore previous asynchronous exception mask. */
    curr->async_exception_mask = curr->async_exception_state(trap).old_mask;
}

static void nmi_hwdom_report(unsigned int reason_idx)
{
    struct domain *d = hardware_domain;

    if ( !d || !d->vcpu || !d->vcpu[0] || !is_pv_domain(d) /* PVH fixme */ )
        return;

    set_bit(reason_idx, nmi_reason(d));

    send_guest_trap(d, 0, TRAP_nmi);
}

static void pci_serr_error(const struct cpu_user_regs *regs)
{
    outb((inb(0x61) & 0x0f) | 0x04, 0x61); /* clear-and-disable the PCI SERR error line. */

    switch ( opt_nmi[0] )
    {
    case 'd': /* 'dom0' */
        nmi_hwdom_report(_XEN_NMIREASON_pci_serr);
        /* fallthrough */
    case 'i': /* 'ignore' */
        /* Would like to print a diagnostic here but can't call printk()
           from NMI context -- raise a softirq instead. */
        raise_softirq(PCI_SERR_SOFTIRQ);
        break;
    default:  /* 'fatal' */
        console_force_unlock();
        printk("\n\nNMI - PCI system error (SERR)\n");
        fatal_trap(regs, 0);
    }
}

static void io_check_error(const struct cpu_user_regs *regs)
{
    switch ( opt_nmi[0] )
    {
    case 'd': /* 'dom0' */
        nmi_hwdom_report(_XEN_NMIREASON_io_error);
    case 'i': /* 'ignore' */
        break;
    default:  /* 'fatal' */
        console_force_unlock();
        printk("\n\nNMI - I/O ERROR\n");
        fatal_trap(regs, 0);
    }

    outb((inb(0x61) & 0x0f) | 0x08, 0x61); /* clear-and-disable IOCK */
    mdelay(1);
    outb((inb(0x61) & 0x07) | 0x00, 0x61); /* enable IOCK */
}

static void unknown_nmi_error(const struct cpu_user_regs *regs, unsigned char reason)
{
    switch ( opt_nmi[0] )
    {
    case 'd': /* 'dom0' */
        nmi_hwdom_report(_XEN_NMIREASON_unknown);
    case 'i': /* 'ignore' */
        break;
    default:  /* 'fatal' */
        console_force_unlock();
        printk("Uhhuh. NMI received for unknown reason %02x.\n", reason);
        printk("Do you have a strange power saving mode enabled?\n");
        fatal_trap(regs, 0);
    }
}

static int dummy_nmi_callback(const struct cpu_user_regs *regs, int cpu)
{
    return 0;
}
 
static nmi_callback_t *nmi_callback = dummy_nmi_callback;

void do_nmi(const struct cpu_user_regs *regs)
{
    unsigned int cpu = smp_processor_id();
    unsigned char reason = 0;
    bool_t handle_unknown = 0;

    ++nmi_count(cpu);

    if ( nmi_callback(regs, cpu) )
        return;

    /*
     * Accessing port 0x61 may trap to SMM which has been actually
     * observed on some production SKX servers. This SMI sometimes
     * takes enough time for the next NMI tick to happen. By reading
     * this port before we re-arm the NMI watchdog, we reduce the chance
     * of having an NMI watchdog expire while in the SMI handler.
     */
    if ( cpu == 0 )
        reason = inb(0x61);

    if ( (nmi_watchdog == NMI_NONE) ||
         (!nmi_watchdog_tick(regs) && watchdog_force) )
        handle_unknown = 1;

    /* Only the BSP gets external NMIs from the system. */
    if ( cpu == 0 )
    {
        if ( reason & 0x80 )
            pci_serr_error(regs);
        if ( reason & 0x40 )
            io_check_error(regs);
        if ( !(reason & 0xc0) && handle_unknown )
            unknown_nmi_error(regs, reason);
    }
}

nmi_callback_t *set_nmi_callback(nmi_callback_t *callback)
{
    nmi_callback_t *old_nmi_callback = nmi_callback;

    nmi_callback = callback;

    return old_nmi_callback;
}

void unset_nmi_callback(void)
{
    nmi_callback = dummy_nmi_callback;
}

void do_device_not_available(struct cpu_user_regs *regs)
{
    struct vcpu *curr = current;

    BUG_ON(!guest_mode(regs));

    vcpu_restore_fpu_lazy(curr);

    if ( curr->arch.pv_vcpu.ctrlreg[0] & X86_CR0_TS )
    {
        do_guest_trap(TRAP_no_device, regs);
        curr->arch.pv_vcpu.ctrlreg[0] &= ~X86_CR0_TS;
    }
    else
        TRACE_0D(TRC_PV_MATH_STATE_RESTORE);

    return;
}

u64 read_efer(void)
{
    return this_cpu(efer);
}

void write_efer(u64 val)
{
    this_cpu(efer) = val;
    wrmsrl(MSR_EFER, val);
}

static void ler_enable(void)
{
    u64 debugctl;

    if ( !this_cpu(ler_msr) )
        return;

    rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
    wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl | IA32_DEBUGCTLMSR_LBR);
}

void do_debug(struct cpu_user_regs *regs)
{
    struct vcpu *v = current;

    if ( debugger_trap_entry(TRAP_debug, regs) )
        return;

    if ( !guest_mode(regs) )
    {
        if ( regs->eflags & X86_EFLAGS_TF )
        {
            /* In SYSENTER entry path we can't zap TF until EFLAGS is saved. */
            if ( (regs->rip >= (unsigned long)sysenter_entry) &&
                 (regs->rip <= (unsigned long)sysenter_eflags_saved) )
            {
                if ( regs->rip == (unsigned long)sysenter_eflags_saved )
                    regs->eflags &= ~X86_EFLAGS_TF;
                goto out;
            }
            if ( !debugger_trap_fatal(TRAP_debug, regs) )
            {
                WARN();
                regs->eflags &= ~X86_EFLAGS_TF;
            }
        }
        else
        {
            /*
             * We ignore watchpoints when they trigger within Xen. This may
             * happen when a buffer is passed to us which previously had a
             * watchpoint set on it. No need to bump EIP; the only faulting
             * trap is an instruction breakpoint, which can't happen to us.
             */
            WARN_ON(!search_exception_table(regs));
        }
        goto out;
    }

    /* Save debug status register where guest OS can peek at it */
    v->arch.debugreg[6] = read_debugreg(6);

    ler_enable();
    do_guest_trap(TRAP_debug, regs);
    return;

 out:
    ler_enable();
    return;
}

static void __init noinline __set_intr_gate(unsigned int n, uint32_t dpl, void *addr)
{
    _set_gate(&idt_table[n], SYS_DESC_irq_gate, dpl, addr);
}

static void __init set_swint_gate(unsigned int n, void *addr)
{
    __set_intr_gate(n, 3, addr);
}

static void __init set_intr_gate(unsigned int n, void *addr)
{
    __set_intr_gate(n, 0, addr);
}

void load_TR(void)
{
    struct tss_struct *tss = &this_cpu(init_tss);
    struct desc_ptr old_gdt, tss_gdt = {
        .base = (long)(this_cpu(gdt_table) - FIRST_RESERVED_GDT_ENTRY),
        .limit = LAST_RESERVED_GDT_BYTE
    };

    _set_tssldt_desc(
        this_cpu(gdt_table) + TSS_ENTRY - FIRST_RESERVED_GDT_ENTRY,
        (unsigned long)tss,
        offsetof(struct tss_struct, __cacheline_filler) - 1,
        SYS_DESC_tss_avail);
    _set_tssldt_desc(
        this_cpu(compat_gdt_table) + TSS_ENTRY - FIRST_RESERVED_GDT_ENTRY,
        (unsigned long)tss,
        offsetof(struct tss_struct, __cacheline_filler) - 1,
        SYS_DESC_tss_busy);

    /* Switch to non-compat GDT (which has B bit clear) to execute LTR. */
    asm volatile (
        "sgdt %0; lgdt %2; ltr %w1; lgdt %0"
        : "=m" (old_gdt) : "rm" (TSS_ENTRY << 3), "m" (tss_gdt) : "memory" );
}

void percpu_traps_init(void)
{
    subarch_percpu_traps_init();

    if ( !opt_ler )
        return;

    switch ( boot_cpu_data.x86_vendor )
    {
    case X86_VENDOR_INTEL:
        switch ( boot_cpu_data.x86 )
        {
        case 6:
            this_cpu(ler_msr) = MSR_IA32_LASTINTFROMIP;
            break;
        case 15:
            this_cpu(ler_msr) = MSR_P4_LER_FROM_LIP;
            break;
        }
        break;
    case X86_VENDOR_AMD:
        switch ( boot_cpu_data.x86 )
        {
        case 6:
        case 0xf ... 0x17:
            this_cpu(ler_msr) = MSR_IA32_LASTINTFROMIP;
            break;
        }
        break;
    }

    ler_enable();
}

void __init init_idt_traps(void)
{
    /*
     * Note that interrupt gates are always used, rather than trap gates. We
     * must have interrupts disabled until DS/ES/FS/GS are saved because the
     * first activation must have the "bad" value(s) for these registers and
     * we may lose them if another activation is installed before they are
     * saved. The page-fault handler also needs interrupts disabled until %cr2
     * has been read and saved on the stack.
     */
    set_intr_gate(TRAP_divide_error,&divide_error);
    set_intr_gate(TRAP_debug,&debug);
    set_intr_gate(TRAP_nmi,&nmi);
    set_swint_gate(TRAP_int3,&int3);         /* usable from all privileges */
    set_swint_gate(TRAP_overflow,&overflow); /* usable from all privileges */
    set_intr_gate(TRAP_bounds,&bounds);
    set_intr_gate(TRAP_invalid_op,&invalid_op);
    set_intr_gate(TRAP_no_device,&device_not_available);
    set_intr_gate(TRAP_double_fault,&double_fault);
    set_intr_gate(TRAP_invalid_tss,&invalid_TSS);
    set_intr_gate(TRAP_no_segment,&segment_not_present);
    set_intr_gate(TRAP_stack_error,&stack_segment);
    set_intr_gate(TRAP_gp_fault,&general_protection);
    set_intr_gate(TRAP_page_fault,&early_page_fault);
    set_intr_gate(TRAP_copro_error,&coprocessor_error);
    set_intr_gate(TRAP_alignment_check,&alignment_check);
    set_intr_gate(TRAP_machine_check,&machine_check);
    set_intr_gate(TRAP_simd_error,&simd_coprocessor_error);

    /* Specify dedicated interrupt stacks for NMI, #DF, and #MC. */
    set_ist(&idt_table[TRAP_double_fault],  IST_DF);
    set_ist(&idt_table[TRAP_nmi],           IST_NMI);
    set_ist(&idt_table[TRAP_machine_check], IST_MCE);

    /* CPU0 uses the master IDT. */
    idt_tables[0] = idt_table;

    this_cpu(gdt_table) = boot_cpu_gdt_table;
    this_cpu(compat_gdt_table) = boot_cpu_compat_gdt_table;
}

extern void (*const autogen_entrypoints[NR_VECTORS])(void);
void __init trap_init(void)
{
    unsigned int vector;

    /* Replace early pagefault with real pagefault handler. */
    set_intr_gate(TRAP_page_fault, &page_fault);

    /* The 32-on-64 hypercall vector is only accessible from ring 1. */
    _set_gate(idt_table + HYPERCALL_VECTOR,
              SYS_DESC_trap_gate, 1, entry_int82);

    /* Fast trap for int80 (faster than taking the #GP-fixup path). */
    _set_gate(idt_table + 0x80, SYS_DESC_trap_gate, 3, &int80_direct_trap);

    for ( vector = 0; vector < NR_VECTORS; ++vector )
    {
        if ( autogen_entrypoints[vector] )
        {
            /* Found autogen entry: check we won't clobber an existing trap. */
            ASSERT(idt_table[vector].b == 0);
            set_intr_gate(vector, autogen_entrypoints[vector]);
        }
        else
        {
            /* No entry point: confirm we have an existing trap in place. */
            ASSERT(idt_table[vector].b != 0);
        }
    }

    percpu_traps_init();

    cpu_init();

    open_softirq(NMI_MCE_SOFTIRQ, nmi_mce_softirq);
    open_softirq(PCI_SERR_SOFTIRQ, pci_serr_softirq);
}

long register_guest_nmi_callback(unsigned long address)
{
    struct vcpu *v = current;
    struct domain *d = v->domain;
    struct trap_info *t = &v->arch.pv_vcpu.trap_ctxt[TRAP_nmi];

    if ( !is_canonical_address(address) )
        return -EINVAL;

    t->vector  = TRAP_nmi;
    t->flags   = 0;
    t->cs      = (is_pv_32bit_domain(d) ?
                  FLAT_COMPAT_KERNEL_CS : FLAT_KERNEL_CS);
    t->address = address;
    TI_SET_IF(t, 1);

    /*
     * If no handler was registered we can 'lose the NMI edge'. Re-assert it
     * now.
     */
    if ( (v->vcpu_id == 0) && (arch_get_nmi_reason(d) != 0) )
        v->nmi_pending = 1;

    return 0;
}

long unregister_guest_nmi_callback(void)
{
    struct vcpu *v = current;
    struct trap_info *t = &v->arch.pv_vcpu.trap_ctxt[TRAP_nmi];

    memset(t, 0, sizeof(*t));

    return 0;
}

int guest_has_trap_callback(struct domain *d, uint16_t vcpuid, unsigned int trap_nr)
{
    struct vcpu *v;
    struct trap_info *t;

    BUG_ON(d == NULL);
    BUG_ON(vcpuid >= d->max_vcpus);

    /* Sanity check - XXX should be more fine grained. */
    BUG_ON(trap_nr >= NR_VECTORS);

    v = d->vcpu[vcpuid];
    t = &v->arch.pv_vcpu.trap_ctxt[trap_nr];

    return (t->address != 0);
}


int send_guest_trap(struct domain *d, uint16_t vcpuid, unsigned int trap_nr)
{
    struct vcpu *v;
    struct softirq_trap *st = &per_cpu(softirq_trap, smp_processor_id());

    BUG_ON(d == NULL);
    BUG_ON(vcpuid >= d->max_vcpus);
    v = d->vcpu[vcpuid];

    switch (trap_nr) {
    case TRAP_nmi:
        if ( cmpxchgptr(&st->vcpu, NULL, v) )
            return -EBUSY;
        if ( !test_and_set_bool(v->nmi_pending) ) {
               st->domain = d;
               st->processor = v->processor;

               /* not safe to wake up a vcpu here */
               raise_softirq(NMI_MCE_SOFTIRQ);
               return 0;
        }
        st->vcpu = NULL;
        break;

    case TRAP_machine_check:
        if ( cmpxchgptr(&st->vcpu, NULL, v) )
            return -EBUSY;

        /* We are called by the machine check (exception or polling) handlers
         * on the physical CPU that reported a machine check error. */

        if ( !test_and_set_bool(v->mce_pending) ) {
                st->domain = d;
                st->processor = v->processor;

                /* not safe to wake up a vcpu here */
                raise_softirq(NMI_MCE_SOFTIRQ);
                return 0;
        }
        st->vcpu = NULL;
        break;
    }

    /* delivery failed */
    return -EIO;
}


long do_set_trap_table(XEN_GUEST_HANDLE_PARAM(const_trap_info_t) traps)
{
    struct trap_info cur;
    struct vcpu *curr = current;
    struct trap_info *dst = curr->arch.pv_vcpu.trap_ctxt;
    long rc = 0;

    /* If no table is presented then clear the entire virtual IDT. */
    if ( guest_handle_is_null(traps) )
    {
        memset(dst, 0, NR_VECTORS * sizeof(*dst));
        init_int80_direct_trap(curr);
        return 0;
    }

    for ( ; ; )
    {
        if ( copy_from_guest(&cur, traps, 1) )
        {
            rc = -EFAULT;
            break;
        }

        if ( cur.address == 0 )
            break;

        if ( !is_canonical_address(cur.address) )
            return -EINVAL;

        fixup_guest_code_selector(curr->domain, cur.cs);

        memcpy(&dst[cur.vector], &cur, sizeof(cur));

        if ( cur.vector == 0x80 )
            init_int80_direct_trap(curr);

        guest_handle_add_offset(traps, 1);

        if ( hypercall_preempt_check() )
        {
            rc = hypercall_create_continuation(
                __HYPERVISOR_set_trap_table, "h", traps);
            break;
        }
    }

    return rc;
}

void activate_debugregs(const struct vcpu *curr)
{
    ASSERT(curr == current);

    write_debugreg(0, curr->arch.debugreg[0]);
    write_debugreg(1, curr->arch.debugreg[1]);
    write_debugreg(2, curr->arch.debugreg[2]);
    write_debugreg(3, curr->arch.debugreg[3]);
    write_debugreg(6, curr->arch.debugreg[6]);

    /*
     * Avoid writing the subsequently getting replaced value when getting
     * called from set_debugreg() below. Eventual future callers will need
     * to take this into account.
     */
    if ( curr->arch.debugreg[7] & DR7_ACTIVE_MASK )
        write_debugreg(7, curr->arch.debugreg[7]);

    if ( boot_cpu_has(X86_FEATURE_DBEXT) )
    {
        wrmsrl(MSR_AMD64_DR0_ADDRESS_MASK, curr->arch.pv_vcpu.dr_mask[0]);
        wrmsrl(MSR_AMD64_DR1_ADDRESS_MASK, curr->arch.pv_vcpu.dr_mask[1]);
        wrmsrl(MSR_AMD64_DR2_ADDRESS_MASK, curr->arch.pv_vcpu.dr_mask[2]);
        wrmsrl(MSR_AMD64_DR3_ADDRESS_MASK, curr->arch.pv_vcpu.dr_mask[3]);
    }
}

long set_debugreg(struct vcpu *v, unsigned int reg, unsigned long value)
{
    int i;
    struct vcpu *curr = current;

    switch ( reg )
    {
    case 0: 
        if ( !access_ok(value, sizeof(long)) )
            return -EPERM;
        if ( v == curr ) 
            write_debugreg(0, value);
        break;
    case 1: 
        if ( !access_ok(value, sizeof(long)) )
            return -EPERM;
        if ( v == curr ) 
            write_debugreg(1, value);
        break;
    case 2: 
        if ( !access_ok(value, sizeof(long)) )
            return -EPERM;
        if ( v == curr ) 
            write_debugreg(2, value);
        break;
    case 3:
        if ( !access_ok(value, sizeof(long)) )
            return -EPERM;
        if ( v == curr ) 
            write_debugreg(3, value);
        break;
    case 6:
        /*
         * DR6: Bits 4-11,16-31 reserved (set to 1).
         *      Bit 12 reserved (set to 0).
         */
        value &= ~DR_STATUS_RESERVED_ZERO; /* reserved bits => 0 */
        value |=  DR_STATUS_RESERVED_ONE;  /* reserved bits => 1 */
        if ( v == curr ) 
            write_debugreg(6, value);
        break;
    case 7:
        /*
         * DR7: Bit 10 reserved (set to 1).
         *      Bits 11-12,14-15 reserved (set to 0).
         */
        value &= ~DR_CONTROL_RESERVED_ZERO; /* reserved bits => 0 */
        value |=  DR_CONTROL_RESERVED_ONE;  /* reserved bits => 1 */
        /*
         * Privileged bits:
         *      GD (bit 13): must be 0.
         */
        if ( value & DR_GENERAL_DETECT )
            return -EPERM;
        /* DR7.{G,L}E = 0 => debugging disabled for this domain. */
        if ( value & DR7_ACTIVE_MASK )
        {
            unsigned int io_enable = 0;

            for ( i = DR_CONTROL_SHIFT; i < 32; i += DR_CONTROL_SIZE )
            {
                if ( ((value >> i) & 3) == DR_IO )
                {
                    if ( !(v->arch.pv_vcpu.ctrlreg[4] & X86_CR4_DE) )
                        return -EPERM;
                    io_enable |= value & (3 << ((i - 16) >> 1));
                }
            }

            /* Guest DR5 is a handy stash for I/O intercept information. */
            v->arch.debugreg[5] = io_enable;
            value &= ~io_enable;

            /*
             * If DR7 was previously clear then we need to load all other
             * debug registers at this point as they were not restored during
             * context switch.
             */
            if ( (v == curr) &&
                 !(v->arch.debugreg[7] & DR7_ACTIVE_MASK) )
            {
                activate_debugregs(v);
                break;
            }
        }
        if ( v == curr )
            write_debugreg(7, value);
        break;
    default:
        return -EINVAL;
    }

    v->arch.debugreg[reg] = value;
    return 0;
}

long do_set_debugreg(int reg, unsigned long value)
{
    return set_debugreg(current, reg, value);
}

unsigned long do_get_debugreg(int reg)
{
    struct vcpu *curr = current;

    switch ( reg )
    {
    case 0 ... 3:
    case 6:
        return curr->arch.debugreg[reg];
    case 7:
        return (curr->arch.debugreg[7] |
                curr->arch.debugreg[5]);
    case 4 ... 5:
        return ((curr->arch.pv_vcpu.ctrlreg[4] & X86_CR4_DE) ?
                curr->arch.debugreg[reg + 2] : 0);
    }

    return -EINVAL;
}

void asm_domain_crash_synchronous(unsigned long addr)
{
    /*
     * We need clear AC bit here because in entry.S AC is set
     * by ASM_STAC to temporarily allow accesses to user pages
     * which is prevented by SMAP by default.
     *
     * For some code paths, where this function is called, clac()
     * is not needed, but adding clac() here instead of each place
     * asm_domain_crash_synchronous() is called can reduce the code
     * redundancy, and it is harmless as well.
     */
    clac();

    if ( addr == 0 )
        addr = this_cpu(last_extable_addr);

    printk("domain_crash_sync called from entry.S: fault at %p %pS\n",
           _p(addr), _p(addr));

    __domain_crash_synchronous();
}

/*
 * Local variables:
 * mode: C
 * c-file-style: "BSD"
 * c-basic-offset: 4
 * tab-width: 4
 * indent-tabs-mode: nil
 * End:
 */
