35 static void populate_mtts(struct mlx5_vdpa_direct_mr *mr, __be64 *mtt)  in populate_mtts()  argument
38 	int nsg = mr->nsg;  in populate_mtts()
44 	for_each_sg(mr->sg_head.sgl, sg, mr->nent, i) {  in populate_mtts()
47 		     nsg--, dma_addr += BIT(mr->log_size), dma_len -= BIT(mr->log_size))  in populate_mtts()
64 				  struct mlx5_vdpa_direct_mr *mr,  in fill_create_direct_mr()  argument
72 	MLX5_SET(mkc, mkc, lw, !!(mr->perm & VHOST_MAP_WO));  in fill_create_direct_mr()
73 	MLX5_SET(mkc, mkc, lr, !!(mr->perm & VHOST_MAP_RO));  in fill_create_direct_mr()
77 	MLX5_SET64(mkc, mkc, start_addr, mr->offset);  in fill_create_direct_mr()
78 	MLX5_SET64(mkc, mkc, len, mr->end - mr->start);  in fill_create_direct_mr()
79 	MLX5_SET(mkc, mkc, log_page_size, mr->log_size);  in fill_create_direct_mr()
81 		 get_octo_len(mr->end - mr->start, mr->log_size));  in fill_create_direct_mr()
83 		 get_octo_len(mr->end - mr->start, mr->log_size));  in fill_create_direct_mr()
84 	populate_mtts(mr, MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt));  in fill_create_direct_mr()
91 				 struct mlx5_vdpa_direct_mr *mr,  in create_direct_mr_end()  argument
96 	mr->mr = mlx5_idx_to_mkey(mkey_index);  in create_direct_mr_end()
100 				   struct mlx5_vdpa_direct_mr *mr,  in fill_destroy_direct_mr()  argument
107 	MLX5_SET(destroy_mkey_in, in, mkey_index, mlx5_mkey_to_idx(mr->mr));  in fill_destroy_direct_mr()
110 static void destroy_direct_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_direct_mr *mr)  in destroy_direct_mr()  argument
112 	if (!mr->mr)  in destroy_direct_mr()
115 	mlx5_vdpa_destroy_mkey(mvdev, mr->mr);  in destroy_direct_mr()
118 static u64 map_start(struct vhost_iotlb_map *map, struct mlx5_vdpa_direct_mr *mr)  in map_start()  argument
120 	return max_t(u64, map->start, mr->start);  in map_start()
123 static u64 map_end(struct vhost_iotlb_map *map, struct mlx5_vdpa_direct_mr *mr)  in map_end()  argument
125 	return min_t(u64, map->last + 1, mr->end);  in map_end()
128 static u64 maplen(struct vhost_iotlb_map *map, struct mlx5_vdpa_direct_mr *mr)  in maplen()  argument
130 	return map_end(map, mr) - map_start(map, mr);  in maplen()
189 			klm->key = cpu_to_be32(dmr->mr);  in fill_indir()
211 static int create_direct_keys(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mr)  in create_direct_keys()  argument
218 	cmds = kvcalloc(mr->num_directs, sizeof(*cmds), GFP_KERNEL);  in create_direct_keys()
222 	list_for_each_entry(dmr, &mr->head, list) {  in create_direct_keys()
244 	err = mlx5_vdpa_exec_async_cmds(mvdev, cmds, mr->num_directs);  in create_direct_keys()
252 	list_for_each_entry(dmr, &mr->head, list) {  in create_direct_keys()
282 static int destroy_direct_keys(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mr)  in DEFINE_FREE()
290 	cmds = kvcalloc(mr->num_directs, sizeof(*cmds), GFP_KERNEL);  in DEFINE_FREE()
291 	cmd_mem = kvcalloc(mr->num_directs, sizeof(*cmd_mem), GFP_KERNEL);  in DEFINE_FREE()
295 	list_for_each_entry(dmr, &mr->head, list) {  in DEFINE_FREE()
304 	err = mlx5_vdpa_exec_async_cmds(mvdev, cmds, mr->num_directs);  in DEFINE_FREE()
312 	list_for_each_entry(dmr, &mr->head, list) {  in DEFINE_FREE()
315 		dmr->mr = 0;  in DEFINE_FREE()
326 static int create_indirect_key(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mr)  in create_indirect_key()  argument
335 	start = indir_start_addr(mr);  in create_indirect_key()
336 	len = indir_len(mr);  in create_indirect_key()
340 	inlen = MLX5_ST_SZ_BYTES(create_mkey_in) + klm_byte_size(mr->num_klms);  in create_indirect_key()
354 	MLX5_SET(mkc, mkc, translations_octword_size, klm_byte_size(mr->num_klms) / 16);  in create_indirect_key()
355 	MLX5_SET(create_mkey_in, in, translations_octword_actual_size, mr->num_klms);  in create_indirect_key()
356 	fill_indir(mvdev, mr, in);  in create_indirect_key()
357 	err = mlx5_vdpa_create_mkey(mvdev, &mr->mkey, in, inlen);  in create_indirect_key()
367 static int map_direct_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_direct_mr *mr,  in map_direct_mr()  argument
383 	for (map = vhost_iotlb_itree_first(iotlb, mr->start, mr->end - 1);  in map_direct_mr()
384 	     map; map = vhost_iotlb_itree_next(map, mr->start, mr->end - 1)) {  in map_direct_mr()
385 		size = maplen(map, mr);  in map_direct_mr()
391 	nsg = MLX5_DIV_ROUND_UP_POW2(mr->end - mr->start, log_entity_size);  in map_direct_mr()
393 	err = sg_alloc_table(&mr->sg_head, nsg, GFP_KERNEL);  in map_direct_mr()
397 	sg = mr->sg_head.sgl;  in map_direct_mr()
398 	for (map = vhost_iotlb_itree_first(iotlb, mr->start, mr->end - 1);  in map_direct_mr()
399 	     map; map = vhost_iotlb_itree_next(map, mr->start, mr->end - 1)) {  in map_direct_mr()
400 		offset = mr->start > map->start ? mr->start - map->start : 0;  in map_direct_mr()
402 		paend = map->addr + offset + maplen(map, mr);  in map_direct_mr()
418 	mr->log_size = log_entity_size;  in map_direct_mr()
419 	mr->nsg = nsg;  in map_direct_mr()
420 	mr->nent = dma_map_sg_attrs(dma, mr->sg_head.sgl, mr->nsg, DMA_BIDIRECTIONAL, 0);  in map_direct_mr()
421 	if (!mr->nent) {  in map_direct_mr()
429 	sg_free_table(&mr->sg_head);  in map_direct_mr()
433 static void unmap_direct_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_direct_mr *mr)  in unmap_direct_mr()  argument
437 	destroy_direct_mr(mvdev, mr);  in unmap_direct_mr()
438 	dma_unmap_sg_attrs(dma, mr->sg_head.sgl, mr->nsg, DMA_BIDIRECTIONAL, 0);  in unmap_direct_mr()
439 	sg_free_table(&mr->sg_head);  in unmap_direct_mr()
443 			    struct mlx5_vdpa_mr *mr,  in add_direct_chain()  argument
476 		mr->num_directs++;  in add_direct_chain()
477 		mr->num_klms++;  in add_direct_chain()
480 	list_splice_tail(&tmp, &mr->head);  in add_direct_chain()
484 	list_for_each_entry_safe(dmr, n, &mr->head, list) {  in add_direct_chain()
499 			  struct mlx5_vdpa_mr *mr,  in create_user_mr()  argument
513 	INIT_LIST_HEAD(&mr->head);  in create_user_mr()
527 					mr->num_klms += nnuls;  in create_user_mr()
529 				err = add_direct_chain(mvdev, mr, ps, pe - ps, pperm, iotlb);  in create_user_mr()
538 	err = add_direct_chain(mvdev, mr, ps, pe - ps, pperm, iotlb);  in create_user_mr()
542 	err = create_direct_keys(mvdev, mr);  in create_user_mr()
550 	err = create_indirect_key(mvdev, mr);  in create_user_mr()
554 	mr->user_mr = true;  in create_user_mr()
558 	list_for_each_entry_safe_reverse(dmr, n, &mr->head, list) {  in create_user_mr()
566 static int create_dma_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mr)  in create_dma_mr()  argument
586 	err = mlx5_vdpa_create_mkey(mvdev, &mr->mkey, in, inlen);  in create_dma_mr()
588 		mr->user_mr = false;  in create_dma_mr()
594 static void destroy_dma_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mr)  in destroy_dma_mr()  argument
596 	mlx5_vdpa_destroy_mkey(mvdev, mr->mkey);  in destroy_dma_mr()
628 static void destroy_user_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mr)  in destroy_user_mr()  argument
633 	destroy_indirect_key(mvdev, mr);  in destroy_user_mr()
634 	destroy_direct_keys(mvdev, mr);  in destroy_user_mr()
635 	list_for_each_entry_safe_reverse(dmr, n, &mr->head, list) {  in destroy_user_mr()
642 static void _mlx5_vdpa_destroy_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mr)  in _mlx5_vdpa_destroy_mr()  argument
644 	if (WARN_ON(!mr))  in _mlx5_vdpa_destroy_mr()
647 	if (mr->user_mr)  in _mlx5_vdpa_destroy_mr()
648 		destroy_user_mr(mvdev, mr);  in _mlx5_vdpa_destroy_mr()
650 		destroy_dma_mr(mvdev, mr);  in _mlx5_vdpa_destroy_mr()
652 	vhost_iotlb_free(mr->iotlb);  in _mlx5_vdpa_destroy_mr()
654 	list_del(&mr->mr_list);  in _mlx5_vdpa_destroy_mr()
656 	kfree(mr);  in _mlx5_vdpa_destroy_mr()
668 	struct mlx5_vdpa_mr *mr, *tmp;  in mlx5_vdpa_mr_gc_handler()  local
683 	list_for_each_entry_safe(mr, tmp, &mres->mr_gc_list_head, mr_list) {  in mlx5_vdpa_mr_gc_handler()
684 		_mlx5_vdpa_destroy_mr(mvdev, mr);  in mlx5_vdpa_mr_gc_handler()
691 			      struct mlx5_vdpa_mr *mr)  in _mlx5_vdpa_put_mr()  argument
695 	if (!mr)  in _mlx5_vdpa_put_mr()
698 	if (refcount_dec_and_test(&mr->refcount)) {  in _mlx5_vdpa_put_mr()
699 		list_move_tail(&mr->mr_list, &mres->mr_gc_list_head);  in _mlx5_vdpa_put_mr()
706 		      struct mlx5_vdpa_mr *mr)  in mlx5_vdpa_put_mr()  argument
709 	_mlx5_vdpa_put_mr(mvdev, mr);  in mlx5_vdpa_put_mr()
714 			      struct mlx5_vdpa_mr *mr)  in _mlx5_vdpa_get_mr()  argument
716 	if (!mr)  in _mlx5_vdpa_get_mr()
719 	refcount_inc(&mr->refcount);  in _mlx5_vdpa_get_mr()
723 		      struct mlx5_vdpa_mr *mr)  in mlx5_vdpa_get_mr()  argument
726 	_mlx5_vdpa_get_mr(mvdev, mr);  in mlx5_vdpa_get_mr()
734 	struct mlx5_vdpa_mr *old_mr = mvdev->mres.mr[asid];  in mlx5_vdpa_update_mr()
739 	mvdev->mres.mr[asid] = new_mr;  in mlx5_vdpa_update_mr()
746 	struct mlx5_vdpa_mr *mr;  in mlx5_vdpa_show_mr_leaks()  local
750 	list_for_each_entry(mr, &mvdev->mres.mr_list_head, mr_list) {  in mlx5_vdpa_show_mr_leaks()
754 				       mr, mr->mkey, refcount_read(&mr->refcount));  in mlx5_vdpa_show_mr_leaks()
775 				struct mlx5_vdpa_mr *mr,  in _mlx5_vdpa_create_mr()  argument
781 		err = create_user_mr(mvdev, mr, iotlb);  in _mlx5_vdpa_create_mr()
783 		err = create_dma_mr(mvdev, mr);  in _mlx5_vdpa_create_mr()
788 	mr->iotlb = vhost_iotlb_alloc(0, 0);  in _mlx5_vdpa_create_mr()
789 	if (!mr->iotlb) {  in _mlx5_vdpa_create_mr()
794 	err = dup_iotlb(mr->iotlb, iotlb);  in _mlx5_vdpa_create_mr()
798 	list_add_tail(&mr->mr_list, &mvdev->mres.mr_list_head);  in _mlx5_vdpa_create_mr()
803 	vhost_iotlb_free(mr->iotlb);  in _mlx5_vdpa_create_mr()
807 		destroy_user_mr(mvdev, mr);  in _mlx5_vdpa_create_mr()
809 		destroy_dma_mr(mvdev, mr);  in _mlx5_vdpa_create_mr()
817 	struct mlx5_vdpa_mr *mr;  in mlx5_vdpa_create_mr()  local
820 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);  in mlx5_vdpa_create_mr()
821 	if (!mr)  in mlx5_vdpa_create_mr()
825 	err = _mlx5_vdpa_create_mr(mvdev, mr, iotlb);  in mlx5_vdpa_create_mr()
831 	refcount_set(&mr->refcount, 1);  in mlx5_vdpa_create_mr()
833 	return mr;  in mlx5_vdpa_create_mr()
836 	kfree(mr);  in mlx5_vdpa_create_mr()
861 	struct mlx5_vdpa_mr *mr;  in mlx5_vdpa_create_dma_mr()  local
863 	mr = mlx5_vdpa_create_mr(mvdev, NULL);  in mlx5_vdpa_create_dma_mr()
864 	if (IS_ERR(mr))  in mlx5_vdpa_create_dma_mr()
865 		return PTR_ERR(mr);  in mlx5_vdpa_create_dma_mr()
867 	mlx5_vdpa_update_mr(mvdev, mr, 0);  in mlx5_vdpa_create_dma_mr()