[linux-2.6] / lib / percpu_counter.c

/*
 * Fast batching percpu counters.
 */

#include <linux/percpu_counter.h>
#include <linux/notifier.h>
#include <linux/mutex.h>
#include <linux/init.h>
#include <linux/cpu.h>
#include <linux/module.h>

static LIST_HEAD(percpu_counters);
static DEFINE_MUTEX(percpu_counters_lock);

void percpu_counter_set(struct percpu_counter *fbc, s64 amount)
{
	int cpu;

	spin_lock(&fbc->lock);
	for_each_possible_cpu(cpu) {
		s32 *pcount = per_cpu_ptr(fbc->counters, cpu);
		*pcount = 0;
	}
	fbc->count = amount;
	spin_unlock(&fbc->lock);
}
EXPORT_SYMBOL(percpu_counter_set);

void __percpu_counter_add(struct percpu_counter *fbc, s64 amount, s32 batch)
{
	s64 count;
	s32 *pcount;
	int cpu = get_cpu();

	pcount = per_cpu_ptr(fbc->counters, cpu);
	count = *pcount + amount;
	if (count >= batch || count <= -batch) {
		spin_lock(&fbc->lock);
		fbc->count += count;
		*pcount = 0;
		spin_unlock(&fbc->lock);
	} else {
		*pcount = count;
	}
	put_cpu();
}
EXPORT_SYMBOL(__percpu_counter_add);

/*
 * Add up all the per-cpu counts, return the result.  This is a more accurate
 * but much slower version of percpu_counter_read_positive()
 */
s64 __percpu_counter_sum(struct percpu_counter *fbc)
{
	s64 ret;
	int cpu;

	spin_lock(&fbc->lock);
	ret = fbc->count;
	for_each_online_cpu(cpu) {
		s32 *pcount = per_cpu_ptr(fbc->counters, cpu);
		ret += *pcount;
	}
	spin_unlock(&fbc->lock);
	return ret;
}
EXPORT_SYMBOL(__percpu_counter_sum);

int __percpu_counter_init(struct percpu_counter *fbc, s64 amount,
			  struct lock_class_key *key)
{
	spin_lock_init(&fbc->lock);
	lockdep_set_class(&fbc->lock, key);
	fbc->count = amount;
	fbc->counters = alloc_percpu(s32);
	if (!fbc->counters)
		return -ENOMEM;
#ifdef CONFIG_HOTPLUG_CPU
	mutex_lock(&percpu_counters_lock);
	list_add(&fbc->list, &percpu_counters);
	mutex_unlock(&percpu_counters_lock);
#endif
	return 0;
}
EXPORT_SYMBOL(__percpu_counter_init);

void percpu_counter_destroy(struct percpu_counter *fbc)
{
	if (!fbc->counters)
		return;

#ifdef CONFIG_HOTPLUG_CPU
	mutex_lock(&percpu_counters_lock);
	list_del(&fbc->list);
	mutex_unlock(&percpu_counters_lock);
#endif
	free_percpu(fbc->counters);
	fbc->counters = NULL;
}
EXPORT_SYMBOL(percpu_counter_destroy);

int percpu_counter_batch __read_mostly = 32;
EXPORT_SYMBOL(percpu_counter_batch);

static void compute_batch_value(void)
{
	int nr = num_online_cpus();

	percpu_counter_batch = max(32, nr*2);
}

static int __cpuinit percpu_counter_hotcpu_callback(struct notifier_block *nb,
					unsigned long action, void *hcpu)
{
#ifdef CONFIG_HOTPLUG_CPU
	unsigned int cpu;
	struct percpu_counter *fbc;

	compute_batch_value();
	if (action != CPU_DEAD)
		return NOTIFY_OK;

	cpu = (unsigned long)hcpu;
	mutex_lock(&percpu_counters_lock);
	list_for_each_entry(fbc, &percpu_counters, list) {
		s32 *pcount;
		unsigned long flags;

		spin_lock_irqsave(&fbc->lock, flags);
		pcount = per_cpu_ptr(fbc->counters, cpu);
		fbc->count += *pcount;
		*pcount = 0;
		spin_unlock_irqrestore(&fbc->lock, flags);
	}
	mutex_unlock(&percpu_counters_lock);
#endif
	return NOTIFY_OK;
}

static int __init percpu_counter_startup(void)
{
	compute_batch_value();
	hotcpu_notifier(percpu_counter_hotcpu_callback, 0);
	return 0;
}
module_init(percpu_counter_startup);
Commit	Line	Data
3cbc5640 RT	1	/*
	2	* Fast batching percpu counters.
	3	*/
	4
	5	#include <linux/percpu_counter.h>
c67ad917 AM	6	#include <linux/notifier.h>
	7	#include <linux/mutex.h>
	8	#include <linux/init.h>
	9	#include <linux/cpu.h>
3cbc5640 RT	10	#include <linux/module.h>
3cbc5640 RT	11
c67ad917 AM	12	static LIST_HEAD(percpu_counters);
c67ad917 AM	13	static DEFINE_MUTEX(percpu_counters_lock);
c67ad917	14
3a587f47 PZ	15	void percpu_counter_set(struct percpu_counter *fbc, s64 amount)
	16	{
	17	int cpu;
	18
	19	spin_lock(&fbc->lock);
	20	for_each_possible_cpu(cpu) {
	21	s32 *pcount = per_cpu_ptr(fbc->counters, cpu);
	22	*pcount = 0;
	23	}
	24	fbc->count = amount;
	25	spin_unlock(&fbc->lock);
	26	}
	27	EXPORT_SYMBOL(percpu_counter_set);
	28
20e89767	29	void __percpu_counter_add(struct percpu_counter *fbc, s64 amount, s32 batch)
3cbc5640	30	{
20e89767	31	s64 count;
0216bfcf	32	s32 *pcount;
3cbc5640 RT	33	int cpu = get_cpu();
	34
	35	pcount = per_cpu_ptr(fbc->counters, cpu);
	36	count = *pcount + amount;
252e0ba6	37	if (count >= batch \|\| count <= -batch) {
3cbc5640 RT	38	spin_lock(&fbc->lock);
	39	fbc->count += count;
	40	*pcount = 0;
	41	spin_unlock(&fbc->lock);
	42	} else {
	43	*pcount = count;
	44	}
	45	put_cpu();
	46	}
252e0ba6	47	EXPORT_SYMBOL(__percpu_counter_add);
3cbc5640 RT	48
	49	/*
	50	* Add up all the per-cpu counts, return the result. This is a more accurate
	51	* but much slower version of percpu_counter_read_positive()
	52	*/
02d21168	53	s64 __percpu_counter_sum(struct percpu_counter *fbc)
3cbc5640	54	{
0216bfcf	55	s64 ret;
3cbc5640 RT	56	int cpu;
	57
	58	spin_lock(&fbc->lock);
	59	ret = fbc->count;
b4ef0296	60	for_each_online_cpu(cpu) {
0216bfcf	61	s32 *pcount = per_cpu_ptr(fbc->counters, cpu);
3cbc5640 RT	62	ret += *pcount;
	63	}
	64	spin_unlock(&fbc->lock);
bf1d89c8	65	return ret;
3cbc5640	66	}
bf1d89c8	67	EXPORT_SYMBOL(__percpu_counter_sum);
c67ad917	68
ea319518 PZ	69	int __percpu_counter_init(struct percpu_counter *fbc, s64 amount,
ea319518 PZ	70	struct lock_class_key *key)
c67ad917 AM	71	{
c67ad917 AM	72	spin_lock_init(&fbc->lock);
ea319518	73	lockdep_set_class(&fbc->lock, key);
c67ad917 AM	74	fbc->count = amount;
c67ad917 AM	75	fbc->counters = alloc_percpu(s32);
833f4077 PZ	76	if (!fbc->counters)
833f4077 PZ	77	return -ENOMEM;
c67ad917 AM	78	#ifdef CONFIG_HOTPLUG_CPU
	79	mutex_lock(&percpu_counters_lock);
	80	list_add(&fbc->list, &percpu_counters);
	81	mutex_unlock(&percpu_counters_lock);
	82	#endif
833f4077	83	return 0;
c67ad917	84	}
ea319518	85	EXPORT_SYMBOL(__percpu_counter_init);
c67ad917 AM	86
	87	void percpu_counter_destroy(struct percpu_counter *fbc)
	88	{
833f4077 PZ	89	if (!fbc->counters)
	90	return;
	91
c67ad917 AM	92	#ifdef CONFIG_HOTPLUG_CPU
	93	mutex_lock(&percpu_counters_lock);
	94	list_del(&fbc->list);
	95	mutex_unlock(&percpu_counters_lock);
	96	#endif
fd3d664f ED	97	free_percpu(fbc->counters);
fd3d664f ED	98	fbc->counters = NULL;
c67ad917 AM	99	}
	100	EXPORT_SYMBOL(percpu_counter_destroy);
	101
179f7ebf ED	102	int percpu_counter_batch __read_mostly = 32;
	103	EXPORT_SYMBOL(percpu_counter_batch);
	104
	105	static void compute_batch_value(void)
	106	{
	107	int nr = num_online_cpus();
	108
	109	percpu_counter_batch = max(32, nr*2);
	110	}
	111
c67ad917 AM	112	static int __cpuinit percpu_counter_hotcpu_callback(struct notifier_block *nb,
	113	unsigned long action, void *hcpu)
	114	{
179f7ebf	115	#ifdef CONFIG_HOTPLUG_CPU
c67ad917 AM	116	unsigned int cpu;
	117	struct percpu_counter *fbc;
	118
179f7ebf	119	compute_batch_value();
c67ad917 AM	120	if (action != CPU_DEAD)
	121	return NOTIFY_OK;
	122
	123	cpu = (unsigned long)hcpu;
	124	mutex_lock(&percpu_counters_lock);
	125	list_for_each_entry(fbc, &percpu_counters, list) {
	126	s32 *pcount;
d2b20b11	127	unsigned long flags;
c67ad917	128
d2b20b11	129	spin_lock_irqsave(&fbc->lock, flags);
c67ad917 AM	130	pcount = per_cpu_ptr(fbc->counters, cpu);
	131	fbc->count += *pcount;
	132	*pcount = 0;
d2b20b11	133	spin_unlock_irqrestore(&fbc->lock, flags);
c67ad917 AM	134	}
c67ad917 AM	135	mutex_unlock(&percpu_counters_lock);
179f7ebf	136	#endif
c67ad917 AM	137	return NOTIFY_OK;
	138	}
	139
	140	static int __init percpu_counter_startup(void)
	141	{
179f7ebf	142	compute_batch_value();
c67ad917 AM	143	hotcpu_notifier(percpu_counter_hotcpu_callback, 0);
	144	return 0;
	145	}
	146	module_init(percpu_counter_startup);