Name: Hold cpucontrol Lock Around Module Removal
Author: Vatsa (Srivatsa Vaddagiri)
Status: Booted on 2.6.2-rc2-bk2
Depends: 

D: We want to hold the cpucontrol lock around module removal since we
D: play with cpus_allowed.  This solution is not ideal, since we leave
D: the race with sys_sched_setaffinity(), but it prevents a new race.
D: 
D: I have another patch which actually uses a kthread for module removal
D: which means that we never play with a user task's affinity.

diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .10262-linux-2.6.2-rc1-bk1/kernel/module.c .10262-linux-2.6.2-rc1-bk1.updated/kernel/module.c
--- .10262-linux-2.6.2-rc1-bk1/kernel/module.c	2004-01-24 23:02:55.000000000 +1100
+++ .10262-linux-2.6.2-rc1-bk1.updated/kernel/module.c	2004-01-24 23:02:57.000000000 +1100
@@ -536,26 +536,24 @@ static void stopref_set_state(enum stopr
 }
 
 /* Stop the machine.  Disables irqs. */
-static int stop_refcounts(void)
+static int stop_refcounts(cpumask_t *old_allowed)
 {
 	unsigned int i, cpu;
-	cpumask_t old_allowed;
 	int ret = 0;
 
+	/* No CPUs can come up or down during this. */
+	lock_cpu_hotplug();
+
 	/* One thread per cpu.  We'll do our own. */
 	cpu = smp_processor_id();
 
-	/* FIXME: racy with set_cpus_allowed. */
-	old_allowed = current->cpus_allowed;
+	*old_allowed = current->cpus_allowed;
 	set_cpus_allowed(current, cpumask_of_cpu(cpu));
 
 	atomic_set(&stopref_thread_ack, 0);
 	stopref_num_threads = 0;
 	stopref_state = STOPREF_WAIT;
 
-	/* No CPUs can come up or down during this. */
-	lock_cpu_hotplug();
-
 	for (i = 0; i < NR_CPUS; i++) {
 		if (i == cpu || !cpu_online(i))
 			continue;
@@ -572,6 +570,7 @@ static int stop_refcounts(void)
 	/* If some failed, kill them all. */
 	if (ret < 0) {
 		stopref_set_state(STOPREF_EXIT, 1);
+		set_cpus_allowed(current, *old_allowed);
 		unlock_cpu_hotplug();
 		return ret;
 	}
@@ -590,20 +589,21 @@ static int stop_refcounts(void)
 }
 
 /* Restart the machine.  Re-enables irqs. */
-static void restart_refcounts(void)
+static void restart_refcounts(cpumask_t *old_allowed)
 {
 	stopref_set_state(STOPREF_EXIT, 0);
 	local_irq_enable();
 	preempt_enable();
+	set_cpus_allowed(current, *old_allowed);
 	unlock_cpu_hotplug();
 }
 #else /* ...!SMP */
-static inline int stop_refcounts(void)
+static inline int stop_refcounts(cpumask_t *old_allowed)
 {
 	local_irq_disable();
 	return 0;
 }
-static inline void restart_refcounts(void)
+static inline void restart_refcounts(cpumask_t *old_allowed)
 {
 	local_irq_enable();
 }
@@ -664,6 +664,7 @@ sys_delete_module(const char __user *nam
 	struct module *mod;
 	char name[MODULE_NAME_LEN];
 	int ret, forced = 0;
+	cpumask_t old_allowed;
 
 	if (!capable(CAP_SYS_MODULE))
 		return -EPERM;
@@ -708,7 +709,7 @@ sys_delete_module(const char __user *nam
 	}
 	/* Stop the machine so refcounts can't move: irqs disabled. */
 	DEBUGP("Stopping refcounts...\n");
-	ret = stop_refcounts();
+	ret = stop_refcounts(&old_allowed);
 	if (ret != 0)
 		goto out;
 
@@ -717,7 +718,7 @@ sys_delete_module(const char __user *nam
 		forced = try_force(flags);
 		if (!forced) {
 			ret = -EWOULDBLOCK;
-			restart_refcounts();
+			restart_refcounts(&old_allowed);
 			goto out;
 		}
 	}
@@ -725,7 +726,7 @@ sys_delete_module(const char __user *nam
 	/* Mark it as dying. */
 	mod->waiter = current;
 	mod->state = MODULE_STATE_GOING;
-	restart_refcounts();
+	restart_refcounts(&old_allowed);
 
 	/* Never wait if forced. */
 	if (!forced && module_refcount(mod) != 0)