318 void mei_io_cb_free(struct mei_cl_cb *cb)  in mei_io_cb_free()  argument
320 	if (cb == NULL)  in mei_io_cb_free()
323 	list_del(&cb->list);  in mei_io_cb_free()
324 	kvfree(cb->buf.data);  in mei_io_cb_free()
325 	kfree(cb->ext_hdr);  in mei_io_cb_free()
326 	kfree(cb);  in mei_io_cb_free()
337 static inline void mei_tx_cb_enqueue(struct mei_cl_cb *cb,  in mei_tx_cb_enqueue()  argument
340 	list_add_tail(&cb->list, head);  in mei_tx_cb_enqueue()
341 	cb->cl->tx_cb_queued++;  in mei_tx_cb_enqueue()
351 static inline void mei_tx_cb_dequeue(struct mei_cl_cb *cb)  in mei_tx_cb_dequeue()  argument
353 	if (!WARN_ON(cb->cl->tx_cb_queued == 0))  in mei_tx_cb_dequeue()
354 		cb->cl->tx_cb_queued--;  in mei_tx_cb_dequeue()
356 	mei_io_cb_free(cb);  in mei_tx_cb_dequeue()
393 	struct mei_cl_cb *cb;  in mei_io_cb_init()  local
395 	cb = kzalloc(sizeof(*cb), GFP_KERNEL);  in mei_io_cb_init()
396 	if (!cb)  in mei_io_cb_init()
399 	INIT_LIST_HEAD(&cb->list);  in mei_io_cb_init()
400 	cb->fp = fp;  in mei_io_cb_init()
401 	cb->cl = cl;  in mei_io_cb_init()
402 	cb->buf_idx = 0;  in mei_io_cb_init()
403 	cb->fop_type = type;  in mei_io_cb_init()
404 	cb->vtag = 0;  in mei_io_cb_init()
405 	cb->ext_hdr = NULL;  in mei_io_cb_init()
407 	return cb;  in mei_io_cb_init()
419 	struct mei_cl_cb *cb, *next;  in mei_io_list_flush_cl()  local
421 	list_for_each_entry_safe(cb, next, head, list) {  in mei_io_list_flush_cl()
422 		if (cl == cb->cl) {  in mei_io_list_flush_cl()
423 			list_del_init(&cb->list);  in mei_io_list_flush_cl()
424 			if (cb->fop_type == MEI_FOP_READ)  in mei_io_list_flush_cl()
425 				mei_io_cb_free(cb);  in mei_io_list_flush_cl()
441 	struct mei_cl_cb *cb, *next;  in mei_io_tx_list_free_cl()  local
443 	list_for_each_entry_safe(cb, next, head, list) {  in mei_io_tx_list_free_cl()
444 		if (cl == cb->cl && (!fp || fp == cb->fp))  in mei_io_tx_list_free_cl()
445 			mei_tx_cb_dequeue(cb);  in mei_io_tx_list_free_cl()
457 	struct mei_cl_cb *cb, *next;  in mei_io_list_free_fp()  local
459 	list_for_each_entry_safe(cb, next, head, list)  in mei_io_list_free_fp()
460 		if (!fp || fp == cb->fp)  in mei_io_list_free_fp()
461 			mei_io_cb_free(cb);  in mei_io_list_free_fp()
471 	struct mei_cl_cb *cb;  in mei_cl_free_pending()  local
473 	cb = list_first_entry_or_null(&cl->rd_pending, struct mei_cl_cb, list);  in mei_cl_free_pending()
474 	mei_io_cb_free(cb);  in mei_cl_free_pending()
491 	struct mei_cl_cb *cb;  in mei_cl_alloc_cb()  local
493 	cb = mei_io_cb_init(cl, fop_type, fp);  in mei_cl_alloc_cb()
494 	if (!cb)  in mei_cl_alloc_cb()
498 		return cb;  in mei_cl_alloc_cb()
500 	cb->buf.data = kvmalloc(roundup(length, MEI_SLOT_SIZE), GFP_KERNEL);  in mei_cl_alloc_cb()
501 	if (!cb->buf.data) {  in mei_cl_alloc_cb()
502 		mei_io_cb_free(cb);  in mei_cl_alloc_cb()
505 	cb->buf.size = length;  in mei_cl_alloc_cb()
507 	return cb;  in mei_cl_alloc_cb()
526 	struct mei_cl_cb *cb;  in mei_cl_enqueue_ctrl_wr_cb()  local
532 	cb = mei_cl_alloc_cb(cl, length, fop_type, fp);  in mei_cl_enqueue_ctrl_wr_cb()
533 	if (!cb)  in mei_cl_enqueue_ctrl_wr_cb()
536 	list_add_tail(&cb->list, &cl->dev->ctrl_wr_list);  in mei_cl_enqueue_ctrl_wr_cb()
537 	return cb;  in mei_cl_enqueue_ctrl_wr_cb()
551 	struct mei_cl_cb *cb;  in mei_cl_read_cb()  local
555 	list_for_each_entry(cb, &cl->rd_completed, list)  in mei_cl_read_cb()
556 		if (!fp || fp == cb->fp) {  in mei_cl_read_cb()
557 			ret_cb = cb;  in mei_cl_read_cb()
860 static int mei_cl_send_disconnect(struct mei_cl *cl, struct mei_cl_cb *cb)  in mei_cl_send_disconnect()  argument
874 	list_move_tail(&cb->list, &dev->ctrl_rd_list);  in mei_cl_send_disconnect()
891 int mei_cl_irq_disconnect(struct mei_cl *cl, struct mei_cl_cb *cb,  in mei_cl_irq_disconnect()  argument
907 	ret = mei_cl_send_disconnect(cl, cb);  in mei_cl_irq_disconnect()
909 		list_move_tail(&cb->list, cmpl_list);  in mei_cl_irq_disconnect()
925 	struct mei_cl_cb *cb;  in __mei_cl_disconnect()  local
932 	cb = mei_cl_enqueue_ctrl_wr_cb(cl, 0, MEI_FOP_DISCONNECT, NULL);  in __mei_cl_disconnect()
933 	if (!cb) {  in __mei_cl_disconnect()
939 		rets = mei_cl_send_disconnect(cl, cb);  in __mei_cl_disconnect()
966 	mei_io_cb_free(cb);  in __mei_cl_disconnect()
1034 	struct mei_cl_cb *cb;  in mei_cl_is_other_connecting()  local
1038 	list_for_each_entry(cb, &dev->ctrl_rd_list, list) {  in mei_cl_is_other_connecting()
1039 		if (cb->fop_type == MEI_FOP_CONNECT &&  in mei_cl_is_other_connecting()
1040 		    mei_cl_me_id(cl) == mei_cl_me_id(cb->cl))  in mei_cl_is_other_connecting()
1055 static int mei_cl_send_connect(struct mei_cl *cl, struct mei_cl_cb *cb)  in mei_cl_send_connect()  argument
1069 	list_move_tail(&cb->list, &dev->ctrl_rd_list);  in mei_cl_send_connect()
1084 int mei_cl_irq_connect(struct mei_cl *cl, struct mei_cl_cb *cb,  in mei_cl_irq_connect()  argument
1103 	rets = mei_cl_send_connect(cl, cb);  in mei_cl_irq_connect()
1105 		list_move_tail(&cb->list, cmpl_list);  in mei_cl_irq_connect()
1125 	struct mei_cl_cb *cb;  in mei_cl_connect()  local
1150 	cb = mei_cl_enqueue_ctrl_wr_cb(cl, 0, MEI_FOP_CONNECT, fp);  in mei_cl_connect()
1151 	if (!cb) {  in mei_cl_connect()
1158 		rets = mei_cl_send_connect(cl, cb);  in mei_cl_connect()
1195 	mei_io_cb_free(cb);  in mei_cl_connect()
1407 void mei_cl_add_rd_completed(struct mei_cl *cl, struct mei_cl_cb *cb)  in mei_cl_add_rd_completed()  argument
1412 		fp = mei_cl_fp_by_vtag(cl, cb->vtag);  in mei_cl_add_rd_completed()
1415 			mei_io_cb_free(cb);  in mei_cl_add_rd_completed()
1418 		cb->fp = fp;  in mei_cl_add_rd_completed()
1419 		mei_cl_reset_read_by_vtag(cl, cb->vtag);  in mei_cl_add_rd_completed()
1424 	list_add_tail(&cb->list, &cl->rd_completed);  in mei_cl_add_rd_completed()
1435 void mei_cl_del_rd_completed(struct mei_cl *cl, struct mei_cl_cb *cb)  in mei_cl_del_rd_completed()  argument
1438 	mei_io_cb_free(cb);  in mei_cl_del_rd_completed()
1481 int mei_cl_irq_notify(struct mei_cl *cl, struct mei_cl_cb *cb,  in mei_cl_irq_notify()  argument
1498 	request = mei_cl_notify_fop2req(cb->fop_type);  in mei_cl_irq_notify()
1502 		list_move_tail(&cb->list, cmpl_list);  in mei_cl_irq_notify()
1506 	list_move_tail(&cb->list, &dev->ctrl_rd_list);  in mei_cl_irq_notify()
1525 	struct mei_cl_cb *cb;  in mei_cl_notify_request()  local
1550 	cb = mei_cl_enqueue_ctrl_wr_cb(cl, 0, fop_type, fp);  in mei_cl_notify_request()
1551 	if (!cb) {  in mei_cl_notify_request()
1561 		list_move_tail(&cb->list, &dev->ctrl_rd_list);  in mei_cl_notify_request()
1582 	mei_io_cb_free(cb);  in mei_cl_notify_request()
1677 	struct mei_cl_cb *cb;  in mei_cl_read_start()  local
1702 	cb = mei_cl_enqueue_ctrl_wr_cb(cl, length, MEI_FOP_READ, fp);  in mei_cl_read_start()
1703 	if (!cb)  in mei_cl_read_start()
1721 		list_move_tail(&cb->list, &cl->rd_pending);  in mei_cl_read_start()
1731 		mei_io_cb_free(cb);  in mei_cl_read_start()
1765 static struct mei_msg_hdr *mei_msg_hdr_init(const struct mei_cl_cb *cb)  in mei_msg_hdr_init()  argument
1773 	if (!cb)  in mei_msg_hdr_init()
1777 	is_vtag = (cb->vtag && cb->buf_idx == 0);  in mei_msg_hdr_init()
1778 	is_hbm = cb->cl->me_cl->client_id == 0;  in mei_msg_hdr_init()
1779 	is_gsc = ((!is_hbm) && cb->cl->dev->hbm_f_gsc_supported && mei_ext_hdr_is_gsc(cb->ext_hdr));  in mei_msg_hdr_init()
1793 		hdr_len += mei_ext_hdr_len(cb->ext_hdr);  in mei_msg_hdr_init()
1800 	mei_hdr->host_addr = mei_cl_host_addr(cb->cl);  in mei_msg_hdr_init()
1801 	mei_hdr->me_addr = mei_cl_me_id(cb->cl);  in mei_msg_hdr_init()
1802 	mei_hdr->internal = cb->internal;  in mei_msg_hdr_init()
1813 		meta->size += mei_ext_hdr_set_vtag(next_ext, cb->vtag);  in mei_msg_hdr_init()
1819 		meta->size += mei_ext_hdr_set_gsc(next_ext, cb->ext_hdr);  in mei_msg_hdr_init()
1838 int mei_cl_irq_write(struct mei_cl *cl, struct mei_cl_cb *cb,  in mei_cl_irq_write()  argument
1860 	buf = &cb->buf;  in mei_cl_irq_write()
1862 	first_chunk = cb->buf_idx == 0;  in mei_cl_irq_write()
1874 		buf_len = buf->size - cb->buf_idx;  in mei_cl_irq_write()
1875 		data = buf->data + cb->buf_idx;  in mei_cl_irq_write()
1887 	mei_hdr = mei_msg_hdr_init(cb);  in mei_cl_irq_write()
1923 		mei_dma_ring_write(dev, buf->data + cb->buf_idx, buf_len);  in mei_cl_irq_write()
1931 	cb->buf_idx += buf_len;  in mei_cl_irq_write()
1941 		list_move_tail(&cb->list, &dev->write_waiting_list);  in mei_cl_irq_write()
1949 	list_move_tail(&cb->list, cmpl_list);  in mei_cl_irq_write()
1965 ssize_t mei_cl_write(struct mei_cl *cl, struct mei_cl_cb *cb, unsigned long timeout)  in mei_cl_write()  argument
1984 	if (WARN_ON(!cb))  in mei_cl_write()
1989 	buf = &cb->buf;  in mei_cl_write()
1994 	blocking = cb->blocking;  in mei_cl_write()
2004 	cb->buf_idx = 0;  in mei_cl_write()
2012 	mei_hdr = mei_msg_hdr_init(cb);  in mei_cl_write()
2075 	cb->buf_idx = buf_len;  in mei_cl_write()
2081 		mei_tx_cb_enqueue(cb, &dev->write_waiting_list);  in mei_cl_write()
2083 		mei_tx_cb_enqueue(cb, &dev->write_list);  in mei_cl_write()
2085 	cb = NULL;  in mei_cl_write()
2120 	mei_io_cb_free(cb);  in mei_cl_write()
2133 void mei_cl_complete(struct mei_cl *cl, struct mei_cl_cb *cb)  in mei_cl_complete()  argument
2137 	switch (cb->fop_type) {  in mei_cl_complete()
2139 		mei_tx_cb_dequeue(cb);  in mei_cl_complete()
2150 		mei_cl_add_rd_completed(cl, cb);  in mei_cl_complete()
2169 		mei_io_cb_free(cb);  in mei_cl_complete()
2211 int mei_cl_irq_dma_map(struct mei_cl *cl, struct mei_cl_cb *cb,  in mei_cl_irq_dma_map()  argument
2230 		list_move_tail(&cb->list, cmpl_list);  in mei_cl_irq_dma_map()
2234 	list_move_tail(&cb->list, &dev->ctrl_rd_list);  in mei_cl_irq_dma_map()
2247 int mei_cl_irq_dma_unmap(struct mei_cl *cl, struct mei_cl_cb *cb,  in mei_cl_irq_dma_unmap()  argument
2266 		list_move_tail(&cb->list, cmpl_list);  in mei_cl_irq_dma_unmap()
2270 	list_move_tail(&cb->list, &dev->ctrl_rd_list);  in mei_cl_irq_dma_unmap()
2318 	struct mei_cl_cb *cb;  in mei_cl_dma_alloc_and_map()  local
2359 	cb = mei_cl_enqueue_ctrl_wr_cb(cl, 0, MEI_FOP_DMA_MAP, fp);  in mei_cl_dma_alloc_and_map()
2360 	if (!cb) {  in mei_cl_dma_alloc_and_map()
2370 		list_move_tail(&cb->list, &dev->ctrl_rd_list);  in mei_cl_dma_alloc_and_map()
2394 	mei_io_cb_free(cb);  in mei_cl_dma_alloc_and_map()
2411 	struct mei_cl_cb *cb;  in mei_cl_dma_unmap()  local
2438 	cb = mei_cl_enqueue_ctrl_wr_cb(cl, 0, MEI_FOP_DMA_UNMAP, fp);  in mei_cl_dma_unmap()
2439 	if (!cb) {  in mei_cl_dma_unmap()
2449 		list_move_tail(&cb->list, &dev->ctrl_rd_list);  in mei_cl_dma_unmap()
2472 	mei_io_cb_free(cb);  in mei_cl_dma_unmap()