53 	struct blk_stat_callback *cb;  in blk_stat_add()  local
62 	list_for_each_entry_rcu(cb, &q->stats->callbacks, list) {  in blk_stat_add()
63 		if (!blk_stat_is_active(cb))  in blk_stat_add()
66 		bucket = cb->bucket_fn(rq);  in blk_stat_add()
70 		stat = &per_cpu_ptr(cb->cpu_stat, cpu)[bucket];  in blk_stat_add()
79 	struct blk_stat_callback *cb = from_timer(cb, t, timer);  in blk_stat_timer_fn()  local
83 	for (bucket = 0; bucket < cb->buckets; bucket++)  in blk_stat_timer_fn()
84 		blk_rq_stat_init(&cb->stat[bucket]);  in blk_stat_timer_fn()
89 		cpu_stat = per_cpu_ptr(cb->cpu_stat, cpu);  in blk_stat_timer_fn()
90 		for (bucket = 0; bucket < cb->buckets; bucket++) {  in blk_stat_timer_fn()
91 			blk_rq_stat_sum(&cb->stat[bucket], &cpu_stat[bucket]);  in blk_stat_timer_fn()
96 	cb->timer_fn(cb);  in blk_stat_timer_fn()
104 	struct blk_stat_callback *cb;  in blk_stat_alloc_callback()  local
106 	cb = kmalloc(sizeof(*cb), GFP_KERNEL);  in blk_stat_alloc_callback()
107 	if (!cb)  in blk_stat_alloc_callback()
110 	cb->stat = kmalloc_array(buckets, sizeof(struct blk_rq_stat),  in blk_stat_alloc_callback()
112 	if (!cb->stat) {  in blk_stat_alloc_callback()
113 		kfree(cb);  in blk_stat_alloc_callback()
116 	cb->cpu_stat = __alloc_percpu(buckets * sizeof(struct blk_rq_stat),  in blk_stat_alloc_callback()
118 	if (!cb->cpu_stat) {  in blk_stat_alloc_callback()
119 		kfree(cb->stat);  in blk_stat_alloc_callback()
120 		kfree(cb);  in blk_stat_alloc_callback()
124 	cb->timer_fn = timer_fn;  in blk_stat_alloc_callback()
125 	cb->bucket_fn = bucket_fn;  in blk_stat_alloc_callback()
126 	cb->data = data;  in blk_stat_alloc_callback()
127 	cb->buckets = buckets;  in blk_stat_alloc_callback()
128 	timer_setup(&cb->timer, blk_stat_timer_fn, 0);  in blk_stat_alloc_callback()
130 	return cb;  in blk_stat_alloc_callback()
134 			   struct blk_stat_callback *cb)  in blk_stat_add_callback()  argument
143 		cpu_stat = per_cpu_ptr(cb->cpu_stat, cpu);  in blk_stat_add_callback()
144 		for (bucket = 0; bucket < cb->buckets; bucket++)  in blk_stat_add_callback()
149 	list_add_tail_rcu(&cb->list, &q->stats->callbacks);  in blk_stat_add_callback()
155 			      struct blk_stat_callback *cb)  in blk_stat_remove_callback()  argument
160 	list_del_rcu(&cb->list);  in blk_stat_remove_callback()
165 	del_timer_sync(&cb->timer);  in blk_stat_remove_callback()
170 	struct blk_stat_callback *cb;  in blk_stat_free_callback_rcu()  local
172 	cb = container_of(head, struct blk_stat_callback, rcu);  in blk_stat_free_callback_rcu()
173 	free_percpu(cb->cpu_stat);  in blk_stat_free_callback_rcu()
174 	kfree(cb->stat);  in blk_stat_free_callback_rcu()
175 	kfree(cb);  in blk_stat_free_callback_rcu()
178 void blk_stat_free_callback(struct blk_stat_callback *cb)  in blk_stat_free_callback()  argument
180 	if (cb)  in blk_stat_free_callback()
181 		call_rcu(&cb->rcu, blk_stat_free_callback_rcu);  in blk_stat_free_callback()