Xen devel - Sep 2010 - [PATCH] sched_credit: Raise bar for inter-socket migrations on mostly idle systems

The credit scheduler ties to keep work balanced, even on a mostly
idle system.  Unfortunately, if you have one VM burning cpu and
another VM idle, the effect is that the busy VM will flip back
and forth between sockets.

This patch addresses this, by only migrating to a different socket
if the number of idle processors is twice that of the socket the vcpu is
currently on.

This will only affect mostly-idle systems; as the system becomes more busy,
other load-balancing code will come into effect.

Signed-off-by: George Dunlap <george.dunlap@eu.citrix.com>

diff -r 632c02167f97 -r 425cb3e36513 xen/common/sched_credit.c
--- a/xen/common/sched_credit.c	Fri Sep 17 16:59:21 2010 +0100
+++ b/xen/common/sched_credit.c	Fri Sep 17 17:51:29 2010 +0100
@@ -499,26 +499,36 @@
         cpumask_t cpu_idlers;
         cpumask_t nxt_idlers;
         int nxt, weight_cpu, weight_nxt;
+        int migrate_factor;
 
         nxt = cycle_cpu(cpu, cpus);
 
         if ( cpu_isset(cpu, per_cpu(cpu_core_map, nxt)) )
         {
+            /* We''re on the same socket, so check the busy-ness of
threads.
+             * Migrate if # of idlers is less at all */
             ASSERT( cpu_isset(nxt, per_cpu(cpu_core_map, cpu)) );
+            migrate_factor = 1;
             cpus_and(cpu_idlers, idlers, per_cpu(cpu_sibling_map, cpu));
             cpus_and(nxt_idlers, idlers, per_cpu(cpu_sibling_map, nxt));
         }
         else
         {
+            /* We''re on different sockets, so check the busy-ness of
cores.
+             * Migrate only if the other core is twice as idle */
             ASSERT( !cpu_isset(nxt, per_cpu(cpu_core_map, cpu)) );
+            migrate_factor = 2;
             cpus_and(cpu_idlers, idlers, per_cpu(cpu_core_map, cpu));
             cpus_and(nxt_idlers, idlers, per_cpu(cpu_core_map, nxt));
         }
 
         weight_cpu = cpus_weight(cpu_idlers);
         weight_nxt = cpus_weight(nxt_idlers);
-        if ( ( (weight_cpu < weight_nxt) ^ sched_smt_power_savings )
-                && (weight_cpu != weight_nxt) )
+        /* smt_power_savings: consolidate work rather than spreading it */
+        if ( ( sched_smt_power_savings
+               && (weight_cpu > weight_nxt) )
+             || ( !sched_smt_power_savings
+                  && (weight_cpu * migrate_factor < weight_nxt) ) )
         {
             cpu = cycle_cpu(CSCHED_PCPU(nxt)->idle_bias, nxt_idlers);
             if ( commit )

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel