patch-2.4.21 linux-2.4.21/arch/sparc64/kernel/smp.c

• Lines: 395
• Date: 2003-06-13 07:51:32.000000000 -0700
• Orig file: linux-2.4.20/arch/sparc64/kernel/smp.c
• Orig date: 2002-11-28 15:53:12.000000000 -0800

diff -urN linux-2.4.20/arch/sparc64/kernel/smp.c linux-2.4.21/arch/sparc64/kernel/smp.c
@@ -18,6 +18,7 @@
 #include <linux/fs.h>
 #include <linux/seq_file.h>
 #include <linux/cache.h>
+#include <linux/timer.h>
 
 #include <asm/head.h>
 #include <asm/ptrace.h>
@@ -137,7 +138,6 @@
 void __init smp_callin(void)
 {
 	int cpuid = hard_smp_processor_id();
-	unsigned long pstate;
 	extern int bigkernel;
 	extern unsigned long kern_locked_tte_data;
 
@@ -155,50 +155,6 @@
 
 	cpu_probe();
 
-	/* Guarentee that the following sequences execute
-	 * uninterrupted.
-	 */
-	__asm__ __volatile__("rdpr	%%pstate, %0\n\t"
-			     "wrpr	%0, %1, %%pstate"
-			     : "=r" (pstate)
-			     : "i" (PSTATE_IE));
-
-	/* Set things up so user can access tick register for profiling
-	 * purposes.  Also workaround BB_ERRATA_1 by doing a dummy
-	 * read back of %tick after writing it.
-	 */
-	__asm__ __volatile__(
-		"sethi	%%hi(0x80000000), %%g1\n\t"
-		"ba,pt	%%xcc, 1f\n\t"
-		"sllx	%%g1, 32, %%g1\n\t"
-		".align	64\n"
-	"1:	rd	%%tick, %%g2\n\t"
-		"add	%%g2, 6, %%g2\n\t"
-		"andn	%%g2, %%g1, %%g2\n\t"
-		"wrpr	%%g2, 0, %%tick\n\t"
-		"rdpr	%%tick, %%g0"
-	: /* no outputs */
-	: /* no inputs */
-	: "g1", "g2");
-
-	if (SPARC64_USE_STICK) {
-		/* Let the user get at STICK too. */
-		__asm__ __volatile__(
-			"sethi	%%hi(0x80000000), %%g1\n\t"
-			"sllx	%%g1, 32, %%g1\n\t"
-			"rd	%%asr24, %%g2\n\t"
-			"andn	%%g2, %%g1, %%g2\n\t"
-			"wr	%%g2, 0, %%asr24"
-		: /* no outputs */
-		: /* no inputs */
-		: "g1", "g2");
-	}
-
-	/* Restore PSTATE_IE. */
-	__asm__ __volatile__("wrpr	%0, 0x0, %%pstate"
-			     : /* no outputs */
-			     : "r" (pstate));
-
 	smp_setup_percpu_timer();
 
 	__sti();
@@ -225,10 +181,6 @@
 extern int cpu_idle(void);
 extern void init_IRQ(void);
 
-void initialize_secondary(void)
-{
-}
-
 int start_secondary(void *unused)
 {
 	trap_init();
@@ -242,6 +194,158 @@
 	panic("SMP bolixed\n");
 }
 
+static unsigned long current_tick_offset;
+
+/* This tick register synchronization scheme is taken entirely from
+ * the ia64 port, see arch/ia64/kernel/smpboot.c for details and credit.
+ *
+ * The only change I've made is to rework it so that the master
+ * initiates the synchonization instead of the slave. -DaveM
+ */
+
+#define MASTER	0
+#define SLAVE	(SMP_CACHE_BYTES/sizeof(unsigned long))
+
+#define NUM_ROUNDS	64	/* magic value */
+#define NUM_ITERS	5	/* likewise */
+
+static spinlock_t itc_sync_lock = SPIN_LOCK_UNLOCKED;
+static unsigned long go[SLAVE + 1];
+
+#define DEBUG_TICK_SYNC	0
+
+static inline long get_delta (long *rt, long *master)
+{
+	unsigned long best_t0 = 0, best_t1 = ~0UL, best_tm = 0;
+	unsigned long tcenter, t0, t1, tm;
+	unsigned long i;
+
+	for (i = 0; i < NUM_ITERS; i++) {
+		t0 = tick_ops->get_tick();
+		go[MASTER] = 1;
+		membar("#StoreLoad");
+		while (!(tm = go[SLAVE]))
+			membar("#LoadLoad");
+		go[SLAVE] = 0;
+		membar("#StoreStore");
+		t1 = tick_ops->get_tick();
+
+		if (t1 - t0 < best_t1 - best_t0)
+			best_t0 = t0, best_t1 = t1, best_tm = tm;
+	}
+
+	*rt = best_t1 - best_t0;
+	*master = best_tm - best_t0;
+
+	/* average best_t0 and best_t1 without overflow: */
+	tcenter = (best_t0/2 + best_t1/2);
+	if (best_t0 % 2 + best_t1 % 2 == 2)
+		tcenter++;
+	return tcenter - best_tm;
+}
+
+void smp_synchronize_tick_client(void)
+{
+	long i, delta, adj, adjust_latency = 0, done = 0;
+	unsigned long flags, rt, master_time_stamp, bound;
+#if DEBUG_TICK_SYNC
+	struct {
+		long rt;	/* roundtrip time */
+		long master;	/* master's timestamp */
+		long diff;	/* difference between midpoint and master's timestamp */
+		long lat;	/* estimate of itc adjustment latency */
+	} t[NUM_ROUNDS];
+#endif
+
+	go[MASTER] = 1;
+
+	while (go[MASTER])
+		membar("#LoadLoad");
+
+	local_irq_save(flags);
+	{
+		for (i = 0; i < NUM_ROUNDS; i++) {
+			delta = get_delta(&rt, &master_time_stamp);
+			if (delta == 0) {
+				done = 1;	/* let's lock on to this... */
+				bound = rt;
+			}
+
+			if (!done) {
+				if (i > 0) {
+					adjust_latency += -delta;
+					adj = -delta + adjust_latency/4;
+				} else
+					adj = -delta;
+
+				tick_ops->add_tick(adj, current_tick_offset);
+			}
+#if DEBUG_TICK_SYNC
+			t[i].rt = rt;
+			t[i].master = master_time_stamp;
+			t[i].diff = delta;
+			t[i].lat = adjust_latency/4;
+#endif
+		}
+	}
+	local_irq_restore(flags);
+
+#if DEBUG_TICK_SYNC
+	for (i = 0; i < NUM_ROUNDS; i++)
+		printk("rt=%5ld master=%5ld diff=%5ld adjlat=%5ld\n",
+		       t[i].rt, t[i].master, t[i].diff, t[i].lat);
+#endif
+
+	printk(KERN_INFO "CPU %d: synchronized TICK with master CPU (last diff %ld cycles,"
+	       "maxerr %lu cycles)\n", smp_processor_id(), delta, rt);
+}
+
+static void smp_start_sync_tick_client(int cpu);
+
+static void smp_synchronize_one_tick(int cpu)
+{
+	unsigned long flags, i;
+
+	go[MASTER] = 0;
+
+	smp_start_sync_tick_client(cpu);
+
+	/* wait for client to be ready */
+	while (!go[MASTER])
+		membar("#LoadLoad");
+
+	/* now let the client proceed into his loop */
+	go[MASTER] = 0;
+	membar("#StoreLoad");
+
+	spin_lock_irqsave(&itc_sync_lock, flags);
+	{
+		for (i = 0; i < NUM_ROUNDS*NUM_ITERS; i++) {
+			while (!go[MASTER])
+				membar("#LoadLoad");
+			go[MASTER] = 0;
+			membar("#StoreStore");
+			go[SLAVE] = tick_ops->get_tick();
+			membar("#StoreLoad");
+		}
+	}
+	spin_unlock_irqrestore(&itc_sync_lock, flags);
+}
+
+static void smp_synchronize_tick(void)
+{
+	int cpu = smp_processor_id();
+	int i;
+
+	for (i = 0; i < NR_CPUS; i++) {
+		if (cpu_present_map & (1UL << i)) {
+			if (i == cpu)
+				continue;
+			smp_synchronize_one_tick(i);
+		}
+	}
+}
+
 extern struct prom_cpuinfo linux_cpus[64];
 
 extern unsigned long sparc64_cpu_startup;
@@ -340,6 +444,8 @@
 	}
 	smp_processors_ready = 1;
 	membar("#StoreStore | #StoreLoad");
+
+	smp_synchronize_tick();
 }
 
 static void spitfire_xcall_helper(u64 data0, u64 data1, u64 data2, u64 pstate, unsigned long cpu)
@@ -542,6 +648,15 @@
 	}
 }
 
+extern unsigned long xcall_sync_tick;
+
+static void smp_start_sync_tick_client(int cpu)
+{
+	smp_cross_call_masked(&xcall_sync_tick,
+			      0, 0, 0,
+			      (1UL << cpu));
+}
+
 /* Send cross call to all processors except self. */
 #define smp_cross_call(func, ctx, data1, data2) \
 	smp_cross_call_masked(func, ctx, data1, data2, cpu_present_map)
@@ -987,8 +1102,6 @@
 
 extern void sparc64_do_profile(unsigned long pc, unsigned long o7);
 
-static unsigned long current_tick_offset;
-
 #define prof_multiplier(__cpu)		cpu_data[(__cpu)].multiplier
 #define prof_counter(__cpu)		cpu_data[(__cpu)].counter
 
@@ -1002,12 +1115,7 @@
 	 * Check for level 14 softint.
 	 */
 	{
-		unsigned long tick_mask;
-
-		if (SPARC64_USE_STICK)
-			tick_mask = (1UL << 16);
-		else
-			tick_mask = (1UL << 0);
+		unsigned long tick_mask = tick_ops->softint_mask;
 
 		if (!(get_softint() & tick_mask)) {
 			extern void handler_irq(int, struct pt_regs *);
@@ -1022,17 +1130,17 @@
 		if (!user)
 			sparc64_do_profile(regs->tpc, regs->u_regs[UREG_RETPC]);
 		if (!--prof_counter(cpu)) {
-			if (cpu == boot_cpu_id) {
-				irq_enter(cpu, 0);
+			irq_enter(cpu, 0);
 
+			if (cpu == boot_cpu_id) {
 				kstat.irqs[cpu][0]++;
 				timer_tick_interrupt(regs);
-
-				irq_exit(cpu, 0);
 			}
 
 			update_process_times(user);
 
+			irq_exit(cpu, 0);
+
 			prof_counter(cpu) = prof_multiplier(cpu);
 		}
 
@@ -1044,47 +1152,14 @@
 				     : "=r" (pstate)
 				     : "i" (PSTATE_IE));
 
-		/* Workaround for Spitfire Errata (#54 I think??), I discovered
-		 * this via Sun BugID 4008234, mentioned in Solaris-2.5.1 patch
-		 * number 103640.
-		 *
-		 * On Blackbird writes to %tick_cmpr can fail, the
-		 * workaround seems to be to execute the wr instruction
-		 * at the start of an I-cache line, and perform a dummy
-		 * read back from %tick_cmpr right after writing to it. -DaveM
-		 *
-		 * Just to be anal we add a workaround for Spitfire
-		 * Errata 50 by preventing pipeline bypasses on the
-		 * final read of the %tick register into a compare
-		 * instruction.  The Errata 50 description states
-		 * that %tick is not prone to this bug, but I am not
-		 * taking any chances.
-		 */
-		if (!SPARC64_USE_STICK) {
-		__asm__ __volatile__("rd	%%tick_cmpr, %0\n\t"
-				     "ba,pt	%%xcc, 1f\n\t"
-				     " add	%0, %2, %0\n\t"
-				     ".align	64\n"
-				  "1: wr	%0, 0x0, %%tick_cmpr\n\t"
-				     "rd	%%tick_cmpr, %%g0\n\t"
-				     "rd	%%tick, %1\n\t"
-				     "mov	%1, %1"
-				     : "=&r" (compare), "=r" (tick)
-				     : "r" (current_tick_offset));
-		} else {
-		__asm__ __volatile__("rd	%%asr25, %0\n\t"
-				     "add	%0, %2, %0\n\t"
-				     "wr	%0, 0x0, %%asr25\n\t"
-				     "rd	%%asr24, %1\n\t"
-				     : "=&r" (compare), "=r" (tick)
-				     : "r" (current_tick_offset));
-		}
+		compare = tick_ops->add_compare(current_tick_offset);
+		tick = tick_ops->get_tick();
 
 		/* Restore PSTATE_IE. */
 		__asm__ __volatile__("wrpr	%0, 0x0, %%pstate"
 				     : /* no outputs */
 				     : "r" (pstate));
-	} while (tick >= compare);
+	} while (time_after_eq(tick, compare));
 }
 
 static void __init smp_setup_percpu_timer(void)
@@ -1102,35 +1177,7 @@
 			     : "=r" (pstate)
 			     : "i" (PSTATE_IE));
 
-	/* Workaround for Spitfire Errata (#54 I think??), I discovered
-	 * this via Sun BugID 4008234, mentioned in Solaris-2.5.1 patch
-	 * number 103640.
-	 *
-	 * On Blackbird writes to %tick_cmpr can fail, the
-	 * workaround seems to be to execute the wr instruction
-	 * at the start of an I-cache line, and perform a dummy
-	 * read back from %tick_cmpr right after writing to it. -DaveM
-	 */
-	if (!SPARC64_USE_STICK) {
-	__asm__ __volatile__(
-		"rd	%%tick, %%g1\n\t"
-		"ba,pt	%%xcc, 1f\n\t"
-		" add	%%g1, %0, %%g1\n\t"
-		".align	64\n"
-	"1:	wr	%%g1, 0x0, %%tick_cmpr\n\t"
-		"rd	%%tick_cmpr, %%g0"
-	: /* no outputs */
-	: "r" (current_tick_offset)
-	: "g1");
-	} else {
-	__asm__ __volatile__(
-		"rd	%%asr24, %%g1\n\t"
-		"add	%%g1, %0, %%g1\n\t"
-		"wr	%%g1, 0x0, %%asr25"
-	: /* no outputs */
-	: "r" (current_tick_offset)
-	: "g1");
-	}
+	tick_ops->init_tick(current_tick_offset);
 
 	/* Restore PSTATE_IE. */
 	__asm__ __volatile__("wrpr	%0, 0x0, %%pstate"