With on_selected_cpus() rewritten to avoid taking a global lock, also
use it for TLB flushes rather than the existing hand-rolled
functionality.

This improves performance by allowing TLB flushes on behalf of different
guests to happen at the same time.

Signed-off-by: Ross Lagerwall <ross.lagerwall@citrix.com>
---
 tools/xentrace/xenalyze.c              |  2 --
 xen/arch/x86/include/asm/irq-vectors.h |  1 -
 xen/arch/x86/include/asm/irq.h         |  1 -
 xen/arch/x86/smp.c                     | 30 +++++++++++---------------
 xen/arch/x86/smpboot.c                 |  1 -
 5 files changed, 13 insertions(+), 22 deletions(-)

diff --git a/tools/xentrace/xenalyze.c b/tools/xentrace/xenalyze.c
index 876d59d42ca5..a32f5f378a84 100644
--- a/tools/xentrace/xenalyze.c
+++ b/tools/xentrace/xenalyze.c
@@ -866,7 +866,6 @@ const char * hvm_svm_exit_reason_name[HVM_SVM_EXIT_REASON_MAX] = {
 /* General hvm information */
 #define SPURIOUS_APIC_VECTOR  0xff
 #define ERROR_APIC_VECTOR     0xfe
-#define INVALIDATE_TLB_VECTOR 0xfd
 #define EVENT_CHECK_VECTOR    0xfc
 #define CALL_FUNCTION_VECTOR  0xfb
 #define THERMAL_APIC_VECTOR   0xfa
@@ -878,7 +877,6 @@ const char * hvm_svm_exit_reason_name[HVM_SVM_EXIT_REASON_MAX] = {
 const char * hvm_extint_vector_name[EXTERNAL_INTERRUPT_MAX] = {
     [SPURIOUS_APIC_VECTOR] = "SPURIOS_APIC",
     [ERROR_APIC_VECTOR] =    "ERROR_APIC",
-    [INVALIDATE_TLB_VECTOR]= "INVALIDATE_TLB",
     [EVENT_CHECK_VECTOR]=    "EVENT_CHECK",
     [CALL_FUNCTION_VECTOR]=  "CALL_FUNCTION",
     [THERMAL_APIC_VECTOR]=   "THERMAL_APIC",
diff --git a/xen/arch/x86/include/asm/irq-vectors.h b/xen/arch/x86/include/asm/irq-vectors.h
index d75d1c56716a..a1eb160d500b 100644
--- a/xen/arch/x86/include/asm/irq-vectors.h
+++ b/xen/arch/x86/include/asm/irq-vectors.h
@@ -4,7 +4,6 @@
 /* Processor-initiated interrupts are all high priority. */
 #define SPURIOUS_APIC_VECTOR	0xff
 #define ERROR_APIC_VECTOR	0xfe
-#define INVALIDATE_TLB_VECTOR	0xfd
 #define EVENT_CHECK_VECTOR	0xfc
 #define CALL_FUNCTION_VECTOR	0xfb
 #define LOCAL_TIMER_VECTOR	0xfa
diff --git a/xen/arch/x86/include/asm/irq.h b/xen/arch/x86/include/asm/irq.h
index 7315150b66b4..57cf2f23eb8d 100644
--- a/xen/arch/x86/include/asm/irq.h
+++ b/xen/arch/x86/include/asm/irq.h
@@ -111,7 +111,6 @@ extern int opt_irq_vector_map;
 #define platform_legacy_irq(irq)	((irq) < NR_ISA_IRQS)
 
 void cf_check event_check_interrupt(void);
-void cf_check invalidate_interrupt(void);
 void cf_check call_function_interrupt(void);
 void cf_check irq_move_cleanup_interrupt(void);
 
diff --git a/xen/arch/x86/smp.c b/xen/arch/x86/smp.c
index 7936294f5fcd..a2a9abbb2f4e 100644
--- a/xen/arch/x86/smp.c
+++ b/xen/arch/x86/smp.c
@@ -242,21 +242,20 @@ void cf_check send_IPI_mask_phys(const cpumask_t *mask, int vector)
     local_irq_restore(flags);
 }
 
-static DEFINE_SPINLOCK(flush_lock);
-static cpumask_t flush_cpumask;
-static const void *flush_va;
-static unsigned int flush_flags;
+struct flush_data {
+    const void *va;
+    unsigned int flags;
+};
 
-void cf_check invalidate_interrupt(void)
+static void cf_check invalidate_cb(void *info)
 {
-    unsigned int flags = flush_flags;
-    ack_APIC_irq();
-    perfc_incr(ipis);
+    struct flush_data *data = info;
+    unsigned int flags = data->flags;
+
     if ( (flags & FLUSH_VCPU_STATE) && __sync_local_execstate() )
         flags &= ~(FLUSH_TLB | FLUSH_TLB_GLOBAL | FLUSH_ROOT_PGTBL);
     if ( flags & ~(FLUSH_VCPU_STATE | FLUSH_ORDER_MASK) )
-        flush_area_local(flush_va, flags);
-    cpumask_clear_cpu(smp_processor_id(), &flush_cpumask);
+        flush_area_local(data->va, flags);
 }
 
 void flush_area_mask(const cpumask_t *mask, const void *va, unsigned int flags)
@@ -275,21 +274,18 @@ void flush_area_mask(const cpumask_t *mask, const void *va, unsigned int flags)
     if ( (flags & ~FLUSH_ORDER_MASK) &&
          !cpumask_subset(mask, cpumask_of(cpu)) )
     {
+        cpumask_t flush_cpumask;
+        struct flush_data data = { .va = va, .flags = flags };
+
         if ( cpu_has_hypervisor &&
              !(flags & ~(FLUSH_TLB | FLUSH_TLB_GLOBAL | FLUSH_VA_VALID |
                          FLUSH_ORDER_MASK)) &&
              !hypervisor_flush_tlb(mask, va, flags) )
             return;
 
-        spin_lock(&flush_lock);
         cpumask_and(&flush_cpumask, mask, &cpu_online_map);
         cpumask_clear_cpu(cpu, &flush_cpumask);
-        flush_va      = va;
-        flush_flags   = flags;
-        send_IPI_mask(&flush_cpumask, INVALIDATE_TLB_VECTOR);
-        while ( !cpumask_empty(&flush_cpumask) )
-            cpu_relax();
-        spin_unlock(&flush_lock);
+        on_selected_cpus(&flush_cpumask, invalidate_cb, &data, 1);
     }
 }
 
diff --git a/xen/arch/x86/smpboot.c b/xen/arch/x86/smpboot.c
index 491cbbba33ae..51e6982a1d25 100644
--- a/xen/arch/x86/smpboot.c
+++ b/xen/arch/x86/smpboot.c
@@ -1482,6 +1482,5 @@ void __init smp_intr_init(void)
     /* Direct IPI vectors. */
     set_direct_apic_vector(IRQ_MOVE_CLEANUP_VECTOR, irq_move_cleanup_interrupt);
     set_direct_apic_vector(EVENT_CHECK_VECTOR, event_check_interrupt);
-    set_direct_apic_vector(INVALIDATE_TLB_VECTOR, invalidate_interrupt);
     set_direct_apic_vector(CALL_FUNCTION_VECTOR, call_function_interrupt);
 }
-- 
2.53.0