39 static void ioc_exit_icq(struct io_cq *icq)  in ioc_exit_icq()  argument
41 	struct elevator_type *et = icq->q->elevator->type;  in ioc_exit_icq()
43 	if (icq->flags & ICQ_EXITED)  in ioc_exit_icq()
47 		et->ops.exit_icq(icq);  in ioc_exit_icq()
49 	icq->flags |= ICQ_EXITED;  in ioc_exit_icq()
54 	struct io_cq *icq;  in ioc_exit_icqs()  local
57 	hlist_for_each_entry(icq, &ioc->icq_list, ioc_node)  in ioc_exit_icqs()
58 		ioc_exit_icq(icq);  in ioc_exit_icqs()
66 static void ioc_destroy_icq(struct io_cq *icq)  in ioc_destroy_icq()  argument
68 	struct io_context *ioc = icq->ioc;  in ioc_destroy_icq()
69 	struct request_queue *q = icq->q;  in ioc_destroy_icq()
75 	if (icq->flags & ICQ_DESTROYED)  in ioc_destroy_icq()
78 	radix_tree_delete(&ioc->icq_tree, icq->q->id);  in ioc_destroy_icq()
79 	hlist_del_init(&icq->ioc_node);  in ioc_destroy_icq()
80 	list_del_init(&icq->q_node);  in ioc_destroy_icq()
87 	if (rcu_access_pointer(ioc->icq_hint) == icq)  in ioc_destroy_icq()
90 	ioc_exit_icq(icq);  in ioc_destroy_icq()
96 	icq->__rcu_icq_cache = et->icq_cache;  in ioc_destroy_icq()
97 	icq->flags |= ICQ_DESTROYED;  in ioc_destroy_icq()
98 	kfree_rcu(icq, __rcu_head);  in ioc_destroy_icq()
112 		struct io_cq *icq = hlist_entry(ioc->icq_list.first,  in ioc_release_fn()  local
114 		struct request_queue *q = icq->q;  in ioc_release_fn()
117 			ioc_destroy_icq(icq);  in ioc_release_fn()
128 			ioc_destroy_icq(icq);  in ioc_release_fn()
168 		struct io_cq *icq =  in ioc_clear_queue()  local
175 		spin_lock(&icq->ioc->lock);  in ioc_clear_queue()
176 		ioc_destroy_icq(icq);  in ioc_clear_queue()
177 		spin_unlock(&icq->ioc->lock);  in ioc_clear_queue()
320 	struct io_cq *icq;  in ioc_lookup_icq()  local
331 	icq = rcu_dereference(ioc->icq_hint);  in ioc_lookup_icq()
332 	if (icq && icq->q == q)  in ioc_lookup_icq()
335 	icq = radix_tree_lookup(&ioc->icq_tree, q->id);  in ioc_lookup_icq()
336 	if (icq && icq->q == q)  in ioc_lookup_icq()
337 		rcu_assign_pointer(ioc->icq_hint, icq);	/* allowed to race */  in ioc_lookup_icq()
339 		icq = NULL;  in ioc_lookup_icq()
342 	return icq;  in ioc_lookup_icq()
360 	struct io_cq *icq;  in ioc_create_icq()  local
363 	icq = kmem_cache_alloc_node(et->icq_cache, GFP_ATOMIC | __GFP_ZERO,  in ioc_create_icq()
365 	if (!icq)  in ioc_create_icq()
369 		kmem_cache_free(et->icq_cache, icq);  in ioc_create_icq()
373 	icq->ioc = ioc;  in ioc_create_icq()
374 	icq->q = q;  in ioc_create_icq()
375 	INIT_LIST_HEAD(&icq->q_node);  in ioc_create_icq()
376 	INIT_HLIST_NODE(&icq->ioc_node);  in ioc_create_icq()
382 	if (likely(!radix_tree_insert(&ioc->icq_tree, q->id, icq))) {  in ioc_create_icq()
383 		hlist_add_head(&icq->ioc_node, &ioc->icq_list);  in ioc_create_icq()
384 		list_add(&icq->q_node, &q->icq_list);  in ioc_create_icq()
386 			et->ops.init_icq(icq);  in ioc_create_icq()
388 		kmem_cache_free(et->icq_cache, icq);  in ioc_create_icq()
389 		icq = ioc_lookup_icq(q);  in ioc_create_icq()
390 		if (!icq)  in ioc_create_icq()
397 	return icq;  in ioc_create_icq()
403 	struct io_cq *icq = NULL;  in ioc_find_get_icq()  local
424 		icq = ioc_lookup_icq(q);  in ioc_find_get_icq()
428 	if (!icq) {  in ioc_find_get_icq()
429 		icq = ioc_create_icq(q);  in ioc_find_get_icq()
430 		if (!icq) {  in ioc_find_get_icq()
435 	return icq;  in ioc_find_get_icq()