Index: linux-2.6.10/include/asm-ia64/spinlock.h
===================================================================
--- linux-2.6.10.orig/include/asm-ia64/spinlock.h 2005-02-17 18:47:00.000000000 -0800
+++ linux-2.6.10/include/asm-ia64/spinlock.h 2005-02-22 10:31:22.000000000 -0800
@@ -11,7 +11,7 @@
#include <linux/compiler.h>
#include <linux/kernel.h>
-
+#include <linux/config.h>
#include <asm/atomic.h>
#include <asm/bitops.h>
#include <asm/intrinsics.h>
@@ -24,10 +24,13 @@ typedef struct {
#endif
} spinlock_t;
+/* These definitions do not mean much. Lots of code for IA64 depends on
+ * zeroed memory containing unlocked spinlocks
+ */
#define SPIN_LOCK_UNLOCKED (spinlock_t) { 0 }
#define spin_lock_init(x) ((x)->lock = 0)
-#ifdef ASM_SUPPORTED
+#if defined(ASM_SUPPORTED) & !defined(CONFIG_HBO_LOCKS)
/*
* Try to get the lock. If we fail to get the lock, make a non-standard call to
* ia64_spinlock_contention(). We do not use a normal call because that would force all
@@ -94,7 +97,31 @@ _raw_spin_lock_flags (spinlock_t *lock,
#endif
}
#define _raw_spin_lock(lock) _raw_spin_lock_flags(lock, 0)
+#error CRAP CRAP
#else /* !ASM_SUPPORTED */
+
+#ifdef CONFIG_HBO_LOCKS
+
+void hbo_spinlock_contention(__u32 *, unsigned long, unsigned long);
+void hbo_spinlock_wait_remote(__u32 *, unsigned long);
+
+#define _raw_spin_lock_flags(__x, __flags) \
+do { \
+ __u32 *ia64_spinlock_ptr = (__u32 *) (__x); \
+ __u64 ia64_spinlock_val; \
+ \
+ if (unlikely(system_state == SYSTEM_RUNNING && local_node_data->remote_lock_addr == ia64_spinlock_ptr)) \
+ hbo_spinlock_wait_remote(ia64_spinlock_ptr, __flags); \
+ \
+ ia64_spinlock_val = ia64_cmpxchg4_acq(ia64_spinlock_ptr, numa_node_id()+ 1, 0); \
+ \
+ if (unlikely(ia64_spinlock_val)) \
+ hbo_spinlock_contention(ia64_spinlock_ptr, ia64_spinlock_val, __flags); \
+} while (0)
+
+#define _raw_spin_lock(lock) _raw_spin_lock_flags(lock, 0)
+
+#else
#define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock)
# define _raw_spin_lock(x) \
do { \
@@ -109,11 +136,16 @@ do { \
} while (ia64_spinlock_val); \
} \
} while (0)
+#endif
#endif /* !ASM_SUPPORTED */
#define spin_is_locked(x) ((x)->lock != 0)
#define _raw_spin_unlock(x) do { barrier(); ((spinlock_t *) x)->lock = 0; } while (0)
+#ifdef CONFIG_HBO_LOCKS
+#define _raw_spin_trylock(x) (cmpxchg_acq(&(x)->lock, 0, numa_node_id() + 1) == 0)
+#else
#define _raw_spin_trylock(x) (cmpxchg_acq(&(x)->lock, 0, 1) == 0)
+#endif
#define spin_unlock_wait(x) do { barrier(); } while ((x)->lock)
typedef struct {
Index: linux-2.6.10/arch/ia64/Kconfig
===================================================================
--- linux-2.6.10.orig/arch/ia64/Kconfig 2005-02-17 18:46:27.000000000 -0800
+++ linux-2.6.10/arch/ia64/Kconfig 2005-02-18 18:38:59.000000000 -0800
@@ -272,6 +272,18 @@ config PREEMPT
Say Y here if you are building a kernel for a desktop, embedded
or real-time system. Say N if you are unsure.
+config HBO_LOCKS
+ bool "HBO Locks"
+ help
+ depends on (SMP && NUMA)
+ default y
+ help
+ HBO locks reduces contention for spinlocks by saving the node id when the
+ lock is taken. The cpu waiting on the spinlock can then select an appropriate
+ backoff algorithm depending on the distance to the node. This also insures that
+ it is highly likely that another cpu on the same node gets to a spinlock before
+ remote nodes to avoid too much cacheline bouncing.
+
config HAVE_DEC_LOCK
bool
depends on (SMP || PREEMPT)
Index: linux-2.6.10/arch/ia64/mm/numa.c
===================================================================
--- linux-2.6.10.orig/arch/ia64/mm/numa.c 2005-02-17 18:46:28.000000000 -0800
+++ linux-2.6.10/arch/ia64/mm/numa.c 2005-02-22 10:33:10.000000000 -0800
@@ -47,3 +47,81 @@ paddr_to_nid(unsigned long paddr)
return (i < num_node_memblks) ? node_memblk[i].nid : (num_node_memblks ? -1 : 0);
}
+
+#ifdef CONFIG_HBO_LOCKS
+
+static inline void maybe_enable_irq(unsigned long flags)
+{
+ if (flags & IA64_PSR_I_BIT)
+ local_irq_enable();
+}
+
+static inline void maybe_disable_irq(unsigned long flags)
+{
+ if (flags & IA64_PSR_I_BIT)
+ local_irq_disable();
+}
+
+void hbo_spinlock_contention(__u32 *lock, unsigned long lockval, unsigned long flags)
+{
+ int node = numa_node_id()+ 1;
+
+ do {
+ int backoff = 100;
+ int cap = 2500;
+ int remote = 0;
+
+ maybe_enable_irq(flags);
+
+ if (lockval != node) {
+ /* remote backoff */
+ backoff = 200;
+ cap = 20000;
+ /*
+ * Make sure that no other cpu of this node tries
+ * to acquire the same remote lock.
+ */
+ if (system_state == SYSTEM_RUNNING) {
+ local_node_data->remote_lock_addr = lock;
+ remote = 1;
+ }
+ }
+
+ do {
+ int i;
+
+ for(i = 0; i < backoff; i++)
+ cpu_relax();
+
+ /* Increase backoff for next cycle */
+ backoff = min(backoff << 1, cap);
+
+ ia64_barrier();
+
+ /* No cmpxchg so that we will not get an exclusive cache line */
+ lockval = *lock;
+
+ } while (lockval);
+
+ maybe_disable_irq(flags);
+
+ /* Lock was unlocked. Now use cmpxchg acquiring an exclusive cache line */
+ lockval = ia64_cmpxchg4_acq(lock, node, 0);
+
+ /* Release eventually spinning other cpus on this node since either the lock has been
+ * acquired by this cpu or the node holding the lock may have changed.
+ */
+ if (remote)
+ local_node_data->remote_lock_addr = NULL;
+
+ } while (lockval);
+}
+
+void hbo_spinlock_wait_remote(__u32 *lock, unsigned long flags)
+{
+ maybe_enable_irq(flags);
+ while (local_node_data->remote_lock_addr == lock)
+ cpu_relax();
+ maybe_disable_irq(flags);
+}
+#endif
Index: linux-2.6.10/include/asm-ia64/nodedata.h
===================================================================
--- linux-2.6.10.orig/include/asm-ia64/nodedata.h 2005-02-17 18:47:00.000000000 -0800
+++ linux-2.6.10/include/asm-ia64/nodedata.h 2005-02-18 18:49:48.000000000 -0800
@@ -27,6 +27,7 @@ struct pglist_data;
struct ia64_node_data {
short active_cpu_count;
short node;
+ __u32 *remote_lock_addr; /* active off node spinlock */
struct pglist_data *pg_data_ptrs[MAX_NUMNODES];
};
Index: linux-2.6.10/Makefile
===================================================================
--- linux-2.6.10.orig/Makefile 2005-02-17 18:47:21.000000000 -0800
+++ linux-2.6.10/Makefile 2005-02-22 11:24:51.000000000 -0800
@@ -1,7 +1,7 @@
VERSION = 2
PATCHLEVEL = 6
SUBLEVEL = 11
-EXTRAVERSION = -rc4-bk5
+EXTRAVERSION = -rc4-bk5-hbo
NAME=Woozy Numbat
# *DOCUMENTATION*