3  * GICv3 ITS emulation
28 static int vgic_its_save_tables_v0(struct vgic_its *its);
29 static int vgic_its_restore_tables_v0(struct vgic_its *its);
30 static int vgic_its_commit_v0(struct vgic_its *its);
71  * If this LPI is already mapped on another ITS, we increase its refcount
188  * struct vgic_its_abi - ITS abi ops and settings
192  * @save_tables: save the ITS tables into guest RAM
193  * @restore_tables: restore the ITS internal structs from tables
202 	int (*save_tables)(struct vgic_its *its);
203 	int (*restore_tables)(struct vgic_its *its);
204 	int (*commit)(struct vgic_its *its);
223 inline const struct vgic_its_abi *vgic_its_get_abi(struct vgic_its *its)  in vgic_its_get_abi()  argument
225 	return &its_table_abi_versions[its->abi_rev];  in vgic_its_get_abi()
228 static int vgic_its_set_abi(struct vgic_its *its, u32 rev)  in vgic_its_set_abi()  argument
232 	its->abi_rev = rev;  in vgic_its_set_abi()
233 	abi = vgic_its_get_abi(its);  in vgic_its_set_abi()
234 	return abi->commit(its);  in vgic_its_set_abi()
238  * Find and returns a device in the device table for an ITS.
241 static struct its_device *find_its_device(struct vgic_its *its, u32 device_id)  in find_its_device()  argument
245 	list_for_each_entry(device, &its->device_list, dev_list)  in find_its_device()
254  * Device ID/Event ID pair on an ITS.
257 static struct its_ite *find_ite(struct vgic_its *its, u32 device_id,  in find_ite()  argument
263 	device = find_its_device(its, device_id);  in find_ite()
275 #define for_each_lpi_its(dev, ite, its) \  argument
276 	list_for_each_entry(dev, &(its)->device_list, dev_list) \
289  * Finds and returns a collection in the ITS collection table.
292 static struct its_collection *find_collection(struct vgic_its *its, int coll_id)  in find_collection()  argument
296 	list_for_each_entry(collection, &its->collection_list, coll_list) {  in find_collection()
381  * Promotes the ITS view of affinity of an ITTE (which redistributor this LPI
401 static void update_affinity_collection(struct kvm *kvm, struct vgic_its *its,  in update_affinity_collection()  argument
407 	for_each_lpi_its(device, ite, its) {  in update_affinity_collection()
472 					      struct vgic_its *its,  in vgic_mmio_read_its_typer()  argument
475 	const struct vgic_its_abi *abi = vgic_its_get_abi(its);  in vgic_mmio_read_its_typer()
494 					     struct vgic_its *its,  in vgic_mmio_read_its_iidr()  argument
499 	val = (its->abi_rev << GITS_IIDR_REV_SHIFT) & GITS_IIDR_REV_MASK;  in vgic_mmio_read_its_iidr()
505 					    struct vgic_its *its,  in vgic_mmio_uaccess_write_its_iidr()  argument
513 	return vgic_its_set_abi(its, rev);  in vgic_mmio_uaccess_write_its_iidr()
517 					       struct vgic_its *its,  in vgic_mmio_read_its_idregs()  argument
559 	return iodev->its;  in __vgic_doorbell_to_its()
572 	struct vgic_its *its;  in vgic_its_check_cache()  local
578 	its = __vgic_doorbell_to_its(kvm, db);  in vgic_its_check_cache()
579 	if (IS_ERR(its))  in vgic_its_check_cache()
584 	irq = xa_load(&its->translation_cache, cache_key);  in vgic_its_check_cache()
593 static void vgic_its_cache_translation(struct kvm *kvm, struct vgic_its *its,  in vgic_its_cache_translation()  argument
608 	lockdep_assert_held(&its->its_lock);  in vgic_its_cache_translation()
611 	old = xa_store(&its->translation_cache, cache_key, irq, GFP_KERNEL_ACCOUNT);  in vgic_its_cache_translation()
631 static void vgic_its_invalidate_cache(struct vgic_its *its)  in vgic_its_invalidate_cache()  argument
633 	struct kvm *kvm = its->dev->kvm;  in vgic_its_invalidate_cache()
637 	xa_for_each(&its->translation_cache, idx, irq) {  in vgic_its_invalidate_cache()
638 		xa_erase(&its->translation_cache, idx);  in vgic_its_invalidate_cache()
646 	struct vgic_its *its;  in vgic_its_invalidate_all_caches()  local
654 		its = dev->private;  in vgic_its_invalidate_all_caches()
655 		vgic_its_invalidate_cache(its);  in vgic_its_invalidate_all_caches()
661 int vgic_its_resolve_lpi(struct kvm *kvm, struct vgic_its *its,  in vgic_its_resolve_lpi()  argument
667 	if (!its->enabled)  in vgic_its_resolve_lpi()
670 	ite = find_ite(its, devid, eventid);  in vgic_its_resolve_lpi()
681 	vgic_its_cache_translation(kvm, its, devid, eventid, ite->irq);  in vgic_its_resolve_lpi()
706  * Returns 0 on success, a positive error value for any ITS mapping
709 static int vgic_its_trigger_msi(struct kvm *kvm, struct vgic_its *its,  in vgic_its_trigger_msi()  argument
716 	err = vgic_its_resolve_lpi(kvm, its, devid, eventid, &irq);  in vgic_its_trigger_msi()
751  * Queries the KVM IO bus framework to get the ITS pointer from the given
758 	struct vgic_its *its;  in vgic_its_inject_msi()  local
764 	its = vgic_msi_to_its(kvm, msi);  in vgic_its_inject_msi()
765 	if (IS_ERR(its))  in vgic_its_inject_msi()
766 		return PTR_ERR(its);  in vgic_its_inject_msi()
768 	mutex_lock(&its->its_lock);  in vgic_its_inject_msi()
769 	ret = vgic_its_trigger_msi(kvm, its, msi->devid, msi->data);  in vgic_its_inject_msi()
770 	mutex_unlock(&its->its_lock);  in vgic_its_inject_msi()
821 static int vgic_its_cmd_handle_discard(struct kvm *kvm, struct vgic_its *its,  in vgic_its_cmd_handle_discard()  argument
828 	ite = find_ite(its, device_id, event_id);  in vgic_its_cmd_handle_discard()
830 		struct its_device *device = find_its_device(its, device_id);  in vgic_its_cmd_handle_discard()
831 		int ite_esz = vgic_its_get_abi(its)->ite_esz;  in vgic_its_cmd_handle_discard()
838 		vgic_its_invalidate_cache(its);  in vgic_its_cmd_handle_discard()
842 		return vgic_its_write_entry_lock(its, gpa, 0ULL, ite);  in vgic_its_cmd_handle_discard()
852 static int vgic_its_cmd_handle_movi(struct kvm *kvm, struct vgic_its *its,  in vgic_its_cmd_handle_movi()  argument
862 	ite = find_ite(its, device_id, event_id);  in vgic_its_cmd_handle_movi()
869 	collection = find_collection(its, coll_id);  in vgic_its_cmd_handle_movi()
876 	vgic_its_invalidate_cache(its);  in vgic_its_cmd_handle_movi()
881 static bool __is_visible_gfn_locked(struct vgic_its *its, gpa_t gpa)  in __is_visible_gfn_locked()  argument
887 	idx = srcu_read_lock(&its->dev->kvm->srcu);  in __is_visible_gfn_locked()
888 	ret = kvm_is_visible_gfn(its->dev->kvm, gfn);  in __is_visible_gfn_locked()
889 	srcu_read_unlock(&its->dev->kvm->srcu, idx);  in __is_visible_gfn_locked()
900 static bool vgic_its_check_id(struct vgic_its *its, u64 baser, u32 id,  in vgic_its_check_id()  argument
915 		/* as GITS_TYPER.CIL == 0, ITS supports 16-bit collection ID */  in vgic_its_check_id()
934 		return __is_visible_gfn_locked(its, addr);  in vgic_its_check_id()
943 	if (kvm_read_guest_lock(its->dev->kvm,  in vgic_its_check_id()
964 	return __is_visible_gfn_locked(its, indirect_ptr);  in vgic_its_check_id()
971 static bool vgic_its_check_event_id(struct vgic_its *its, struct its_device *device,  in vgic_its_check_event_id()  argument
974 	const struct vgic_its_abi *abi = vgic_its_get_abi(its);  in vgic_its_check_event_id()
983 	return __is_visible_gfn_locked(its, gpa);  in vgic_its_check_event_id()
987  * Add a new collection into the ITS collection table.
990 static int vgic_its_alloc_collection(struct vgic_its *its,  in vgic_its_alloc_collection()  argument
1003 	list_add_tail(&collection->coll_list, &its->collection_list);  in vgic_its_alloc_collection()
1009 static void vgic_its_free_collection(struct vgic_its *its, u32 coll_id)  in vgic_its_free_collection()  argument
1020 	collection = find_collection(its, coll_id);  in vgic_its_free_collection()
1024 	for_each_lpi_its(device, ite, its)  in vgic_its_free_collection()
1055 static int vgic_its_cmd_handle_mapi(struct kvm *kvm, struct vgic_its *its,  in vgic_its_cmd_handle_mapi()  argument
1068 	device = find_its_device(its, device_id);  in vgic_its_cmd_handle_mapi()
1072 	if (!vgic_its_check_event_id(its, device, event_id))  in vgic_its_cmd_handle_mapi()
1084 	if (find_ite(its, device_id, event_id))  in vgic_its_cmd_handle_mapi()
1087 	collection = find_collection(its, coll_id);  in vgic_its_cmd_handle_mapi()
1091 		if (!vgic_its_check_id(its, its->baser_coll_table, coll_id, NULL))  in vgic_its_cmd_handle_mapi()
1094 		ret = vgic_its_alloc_collection(its, &collection, coll_id);  in vgic_its_cmd_handle_mapi()
1103 			vgic_its_free_collection(its, coll_id);  in vgic_its_cmd_handle_mapi()
1113 			vgic_its_free_collection(its, coll_id);  in vgic_its_cmd_handle_mapi()
1123 static void vgic_its_free_device(struct kvm *kvm, struct vgic_its *its,  in vgic_its_free_device()  argument
1136 	vgic_its_invalidate_cache(its);  in vgic_its_free_device()
1142 /* its lock must be held */
1143 static void vgic_its_free_device_list(struct kvm *kvm, struct vgic_its *its)  in vgic_its_free_device_list()  argument
1147 	list_for_each_entry_safe(cur, temp, &its->device_list, dev_list)  in vgic_its_free_device_list()
1148 		vgic_its_free_device(kvm, its, cur);  in vgic_its_free_device_list()
1151 /* its lock must be held */
1152 static void vgic_its_free_collection_list(struct kvm *kvm, struct vgic_its *its)  in vgic_its_free_collection_list()  argument
1156 	list_for_each_entry_safe(cur, temp, &its->collection_list, coll_list)  in vgic_its_free_collection_list()
1157 		vgic_its_free_collection(its, cur->collection_id);  in vgic_its_free_collection_list()
1161 static struct its_device *vgic_its_alloc_device(struct vgic_its *its,  in vgic_its_alloc_device()  argument
1176 	list_add_tail(&device->dev_list, &its->device_list);  in vgic_its_alloc_device()
1184 static int vgic_its_cmd_handle_mapd(struct kvm *kvm, struct vgic_its *its,  in vgic_its_cmd_handle_mapd()  argument
1194 	if (!vgic_its_check_id(its, its->baser_device_table, device_id, &gpa))  in vgic_its_cmd_handle_mapd()
1200 	device = find_its_device(its, device_id);  in vgic_its_cmd_handle_mapd()
1208 		vgic_its_free_device(kvm, its, device);  in vgic_its_cmd_handle_mapd()
1215 		return vgic_its_write_entry_lock(its, gpa, 0ULL, dte);  in vgic_its_cmd_handle_mapd()
1217 	device = vgic_its_alloc_device(its, device_id, itt_addr,  in vgic_its_cmd_handle_mapd()
1227 static int vgic_its_cmd_handle_mapc(struct kvm *kvm, struct vgic_its *its,  in vgic_its_cmd_handle_mapc()  argument
1238 		vgic_its_free_collection(its, coll_id);  in vgic_its_cmd_handle_mapc()
1239 		vgic_its_invalidate_cache(its);  in vgic_its_cmd_handle_mapc()
1247 		collection = find_collection(its, coll_id);  in vgic_its_cmd_handle_mapc()
1252 			if (!vgic_its_check_id(its, its->baser_coll_table,  in vgic_its_cmd_handle_mapc()
1256 			ret = vgic_its_alloc_collection(its, &collection,  in vgic_its_cmd_handle_mapc()
1263 			update_affinity_collection(kvm, its, collection);  in vgic_its_cmd_handle_mapc()
1274 static int vgic_its_cmd_handle_clear(struct kvm *kvm, struct vgic_its *its,  in vgic_its_cmd_handle_clear()  argument
1282 	ite = find_ite(its, device_id, event_id);  in vgic_its_cmd_handle_clear()
1304 static int vgic_its_cmd_handle_inv(struct kvm *kvm, struct vgic_its *its,  in vgic_its_cmd_handle_inv()  argument
1312 	ite = find_ite(its, device_id, event_id);  in vgic_its_cmd_handle_inv()
1324  * collection, and we don't need to hold the its_lock, since no ITS is
1357 static int vgic_its_cmd_handle_invall(struct kvm *kvm, struct vgic_its *its,  in vgic_its_cmd_handle_invall()  argument
1364 	collection = find_collection(its, coll_id);  in vgic_its_cmd_handle_invall()
1382 static int vgic_its_cmd_handle_movall(struct kvm *kvm, struct vgic_its *its,  in vgic_its_cmd_handle_movall()  argument
1410 	vgic_its_invalidate_cache(its);  in vgic_its_cmd_handle_movall()
1419 static int vgic_its_cmd_handle_int(struct kvm *kvm, struct vgic_its *its,  in vgic_its_cmd_handle_int()  argument
1425 	return vgic_its_trigger_msi(kvm, its, msi_devid, msi_data);  in vgic_its_cmd_handle_int()
1429  * This function is called with the its_cmd lock held, but the ITS data
1432 static int vgic_its_handle_command(struct kvm *kvm, struct vgic_its *its,  in vgic_its_handle_command()  argument
1437 	mutex_lock(&its->its_lock);  in vgic_its_handle_command()
1440 		ret = vgic_its_cmd_handle_mapd(kvm, its, its_cmd);  in vgic_its_handle_command()
1443 		ret = vgic_its_cmd_handle_mapc(kvm, its, its_cmd);  in vgic_its_handle_command()
1446 		ret = vgic_its_cmd_handle_mapi(kvm, its, its_cmd);  in vgic_its_handle_command()
1449 		ret = vgic_its_cmd_handle_mapi(kvm, its, its_cmd);  in vgic_its_handle_command()
1452 		ret = vgic_its_cmd_handle_movi(kvm, its, its_cmd);  in vgic_its_handle_command()
1455 		ret = vgic_its_cmd_handle_discard(kvm, its, its_cmd);  in vgic_its_handle_command()
1458 		ret = vgic_its_cmd_handle_clear(kvm, its, its_cmd);  in vgic_its_handle_command()
1461 		ret = vgic_its_cmd_handle_movall(kvm, its, its_cmd);  in vgic_its_handle_command()
1464 		ret = vgic_its_cmd_handle_int(kvm, its, its_cmd);  in vgic_its_handle_command()
1467 		ret = vgic_its_cmd_handle_inv(kvm, its, its_cmd);  in vgic_its_handle_command()
1470 		ret = vgic_its_cmd_handle_invall(kvm, its, its_cmd);  in vgic_its_handle_command()
1477 	mutex_unlock(&its->its_lock);  in vgic_its_handle_command()
1494 	/* We support only one (ITS) page size: 64K */  in vgic_sanitise_its_baser()
1519 					       struct vgic_its *its,  in vgic_mmio_read_its_cbaser()  argument
1522 	return extract_bytes(its->cbaser, addr & 7, len);  in vgic_mmio_read_its_cbaser()
1525 static void vgic_mmio_write_its_cbaser(struct kvm *kvm, struct vgic_its *its,  in vgic_mmio_write_its_cbaser()  argument
1530 	if (its->enabled)  in vgic_mmio_write_its_cbaser()
1533 	mutex_lock(&its->cmd_lock);  in vgic_mmio_write_its_cbaser()
1534 	its->cbaser = update_64bit_reg(its->cbaser, addr & 7, len, val);  in vgic_mmio_write_its_cbaser()
1535 	its->cbaser = vgic_sanitise_its_cbaser(its->cbaser);  in vgic_mmio_write_its_cbaser()
1536 	its->creadr = 0;  in vgic_mmio_write_its_cbaser()
1541 	its->cwriter = its->creadr;  in vgic_mmio_write_its_cbaser()
1542 	mutex_unlock(&its->cmd_lock);  in vgic_mmio_write_its_cbaser()
1550 static void vgic_its_process_commands(struct kvm *kvm, struct vgic_its *its)  in vgic_its_process_commands()  argument
1555 	/* Commands are only processed when the ITS is enabled. */  in vgic_its_process_commands()
1556 	if (!its->enabled)  in vgic_its_process_commands()
1559 	cbaser = GITS_CBASER_ADDRESS(its->cbaser);  in vgic_its_process_commands()
1561 	while (its->cwriter != its->creadr) {  in vgic_its_process_commands()
1562 		int ret = kvm_read_guest_lock(kvm, cbaser + its->creadr,  in vgic_its_process_commands()
1572 			vgic_its_handle_command(kvm, its, cmd_buf);  in vgic_its_process_commands()
1574 		its->creadr += ITS_CMD_SIZE;  in vgic_its_process_commands()
1575 		if (its->creadr == ITS_CMD_BUFFER_SIZE(its->cbaser))  in vgic_its_process_commands()
1576 			its->creadr = 0;  in vgic_its_process_commands()
1584  * per ITS handling commands at a given time.
1586 static void vgic_mmio_write_its_cwriter(struct kvm *kvm, struct vgic_its *its,  in vgic_mmio_write_its_cwriter()  argument
1592 	if (!its)  in vgic_mmio_write_its_cwriter()
1595 	mutex_lock(&its->cmd_lock);  in vgic_mmio_write_its_cwriter()
1597 	reg = update_64bit_reg(its->cwriter, addr & 7, len, val);  in vgic_mmio_write_its_cwriter()
1599 	if (reg >= ITS_CMD_BUFFER_SIZE(its->cbaser)) {  in vgic_mmio_write_its_cwriter()
1600 		mutex_unlock(&its->cmd_lock);  in vgic_mmio_write_its_cwriter()
1603 	its->cwriter = reg;  in vgic_mmio_write_its_cwriter()
1605 	vgic_its_process_commands(kvm, its);  in vgic_mmio_write_its_cwriter()
1607 	mutex_unlock(&its->cmd_lock);  in vgic_mmio_write_its_cwriter()
1611 						struct vgic_its *its,  in vgic_mmio_read_its_cwriter()  argument
1614 	return extract_bytes(its->cwriter, addr & 0x7, len);  in vgic_mmio_read_its_cwriter()
1618 					       struct vgic_its *its,  in vgic_mmio_read_its_creadr()  argument
1621 	return extract_bytes(its->creadr, addr & 0x7, len);  in vgic_mmio_read_its_creadr()
1625 					      struct vgic_its *its,  in vgic_mmio_uaccess_write_its_creadr()  argument
1632 	mutex_lock(&its->cmd_lock);  in vgic_mmio_uaccess_write_its_creadr()
1634 	if (its->enabled) {  in vgic_mmio_uaccess_write_its_creadr()
1640 	if (cmd_offset >= ITS_CMD_BUFFER_SIZE(its->cbaser)) {  in vgic_mmio_uaccess_write_its_creadr()
1645 	its->creadr = cmd_offset;  in vgic_mmio_uaccess_write_its_creadr()
1647 	mutex_unlock(&its->cmd_lock);  in vgic_mmio_uaccess_write_its_creadr()
1653 					      struct vgic_its *its,  in vgic_mmio_read_its_baser()  argument
1660 		reg = its->baser_device_table;  in vgic_mmio_read_its_baser()
1663 		reg = its->baser_coll_table;  in vgic_mmio_read_its_baser()
1675 				      struct vgic_its *its,  in vgic_mmio_write_its_baser()  argument
1679 	const struct vgic_its_abi *abi = vgic_its_get_abi(its);  in vgic_mmio_write_its_baser()
1684 	if (its->enabled)  in vgic_mmio_write_its_baser()
1689 		regptr = &its->baser_device_table;  in vgic_mmio_write_its_baser()
1694 		regptr = &its->baser_coll_table;  in vgic_mmio_write_its_baser()
1715 		mutex_lock(&its->its_lock);  in vgic_mmio_write_its_baser()
1718 			vgic_its_free_device_list(kvm, its);  in vgic_mmio_write_its_baser()
1721 			vgic_its_free_collection_list(kvm, its);  in vgic_mmio_write_its_baser()
1724 		mutex_unlock(&its->its_lock);  in vgic_mmio_write_its_baser()
1729 					     struct vgic_its *its,  in vgic_mmio_read_its_ctlr()  argument
1734 	mutex_lock(&its->cmd_lock);  in vgic_mmio_read_its_ctlr()
1735 	if (its->creadr == its->cwriter)  in vgic_mmio_read_its_ctlr()
1737 	if (its->enabled)  in vgic_mmio_read_its_ctlr()
1739 	mutex_unlock(&its->cmd_lock);  in vgic_mmio_read_its_ctlr()
1744 static void vgic_mmio_write_its_ctlr(struct kvm *kvm, struct vgic_its *its,  in vgic_mmio_write_its_ctlr()  argument
1748 	mutex_lock(&its->cmd_lock);  in vgic_mmio_write_its_ctlr()
1751 	 * It is UNPREDICTABLE to enable the ITS if any of the CBASER or  in vgic_mmio_write_its_ctlr()
1754 	if (!its->enabled && (val & GITS_CTLR_ENABLE) &&  in vgic_mmio_write_its_ctlr()
1755 		(!(its->baser_device_table & GITS_BASER_VALID) ||  in vgic_mmio_write_its_ctlr()
1756 		 !(its->baser_coll_table & GITS_BASER_VALID) ||  in vgic_mmio_write_its_ctlr()
1757 		 !(its->cbaser & GITS_CBASER_VALID)))  in vgic_mmio_write_its_ctlr()
1760 	its->enabled = !!(val & GITS_CTLR_ENABLE);  in vgic_mmio_write_its_ctlr()
1761 	if (!its->enabled)  in vgic_mmio_write_its_ctlr()
1762 		vgic_its_invalidate_cache(its);  in vgic_mmio_write_its_ctlr()
1766 	 * if the ITS is disabled or no commands have been queued.  in vgic_mmio_write_its_ctlr()
1768 	vgic_its_process_commands(kvm, its);  in vgic_mmio_write_its_ctlr()
1771 	mutex_unlock(&its->cmd_lock);  in vgic_mmio_write_its_ctlr()
1793 static void its_mmio_write_wi(struct kvm *kvm, struct vgic_its *its,  in its_mmio_write_wi()  argument
1835 static int vgic_register_its_iodev(struct kvm *kvm, struct vgic_its *its,  in vgic_register_its_iodev()  argument
1838 	struct vgic_io_device *iodev = &its->iodev;  in vgic_register_its_iodev()
1842 	if (!IS_VGIC_ADDR_UNDEF(its->vgic_its_base)) {  in vgic_register_its_iodev()
1847 	its->vgic_its_base = addr;  in vgic_register_its_iodev()
1852 	iodev->base_addr = its->vgic_its_base;  in vgic_register_its_iodev()
1854 	iodev->its = its;  in vgic_register_its_iodev()
1877 	struct vgic_its *its;  in vgic_its_create()  local
1882 	its = kzalloc(sizeof(struct vgic_its), GFP_KERNEL_ACCOUNT);  in vgic_its_create()
1883 	if (!its)  in vgic_its_create()
1892 			kfree(its);  in vgic_its_create()
1897 	mutex_init(&its->its_lock);  in vgic_its_create()
1898 	mutex_init(&its->cmd_lock);  in vgic_its_create()
1902 	mutex_lock(&its->cmd_lock);  in vgic_its_create()
1903 	mutex_lock(&its->its_lock);  in vgic_its_create()
1904 	mutex_unlock(&its->its_lock);  in vgic_its_create()
1905 	mutex_unlock(&its->cmd_lock);  in vgic_its_create()
1908 	its->vgic_its_base = VGIC_ADDR_UNDEF;  in vgic_its_create()
1910 	INIT_LIST_HEAD(&its->device_list);  in vgic_its_create()
1911 	INIT_LIST_HEAD(&its->collection_list);  in vgic_its_create()
1912 	xa_init(&its->translation_cache);  in vgic_its_create()
1916 	its->enabled = false;  in vgic_its_create()
1917 	its->dev = dev;  in vgic_its_create()
1919 	its->baser_device_table = INITIAL_BASER_VALUE			|  in vgic_its_create()
1921 	its->baser_coll_table = INITIAL_BASER_VALUE |  in vgic_its_create()
1925 	dev->private = its;  in vgic_its_create()
1927 	ret = vgic_its_set_abi(its, NR_ITS_ABIS - 1);  in vgic_its_create()
1937 	struct vgic_its *its = kvm_dev->private;  in vgic_its_destroy()  local
1939 	mutex_lock(&its->its_lock);  in vgic_its_destroy()
1941 	vgic_its_free_device_list(kvm, its);  in vgic_its_destroy()
1942 	vgic_its_free_collection_list(kvm, its);  in vgic_its_destroy()
1943 	vgic_its_invalidate_cache(its);  in vgic_its_destroy()
1944 	xa_destroy(&its->translation_cache);  in vgic_its_destroy()
1946 	mutex_unlock(&its->its_lock);  in vgic_its_destroy()
1947 	kfree(its);  in vgic_its_destroy()
1977 	struct vgic_its *its;  in vgic_its_attr_regs_access()  local
1982 	its = dev->private;  in vgic_its_attr_regs_access()
1987 	 * 64-bit ITS registers, the userspace ABI requires 64-bit  in vgic_its_attr_regs_access()
2009 	if (IS_VGIC_ADDR_UNDEF(its->vgic_its_base)) {  in vgic_its_attr_regs_access()
2022 	addr = its->vgic_its_base + offset;  in vgic_its_attr_regs_access()
2028 			ret = region->uaccess_its_write(dev->kvm, its, addr,  in vgic_its_attr_regs_access()
2031 			region->its_write(dev->kvm, its, addr, len, *reg);  in vgic_its_attr_regs_access()
2033 		*reg = region->its_read(dev->kvm, its, addr, len);  in vgic_its_attr_regs_access()
2071  * @its: its handle
2079 typedef int (*entry_fn_t)(struct vgic_its *its, u32 id, void *entry,
2086  * @its: its handle
2098 static int scan_its_table(struct vgic_its *its, gpa_t base, int size, u32 esz,  in scan_its_table()  argument
2101 	struct kvm *kvm = its->dev->kvm;  in scan_its_table()
2118 		next_offset = fn(its, id, entry, opaque);  in scan_its_table()
2136 static int vgic_its_save_ite(struct vgic_its *its, struct its_device *dev,  in vgic_its_save_ite()  argument
2148 	return vgic_its_write_entry_lock(its, gpa, val, ite);  in vgic_its_save_ite()
2154  * @its: its handle
2159 static int vgic_its_restore_ite(struct vgic_its *its, u32 event_id,  in vgic_its_restore_ite()  argument
2164 	struct kvm *kvm = its->dev->kvm;  in vgic_its_restore_ite()
2190 	collection = find_collection(its, coll_id);  in vgic_its_restore_ite()
2194 	if (!vgic_its_check_event_id(its, dev, event_id))  in vgic_its_restore_ite()
2226 static int vgic_its_save_itt(struct vgic_its *its, struct its_device *device)  in vgic_its_save_itt()  argument
2228 	const struct vgic_its_abi *abi = vgic_its_get_abi(its);  in vgic_its_save_itt()
2248 		ret = vgic_its_save_ite(its, device, ite, gpa);  in vgic_its_save_itt()
2258  * @its: its handle
2263 static int vgic_its_restore_itt(struct vgic_its *its, struct its_device *dev)  in vgic_its_restore_itt()  argument
2265 	const struct vgic_its_abi *abi = vgic_its_get_abi(its);  in vgic_its_restore_itt()
2271 	ret = scan_its_table(its, base, max_size, ite_esz, 0,  in vgic_its_restore_itt()
2284  * @its: ITS handle
2285  * @dev: ITS device
2288 static int vgic_its_save_dte(struct vgic_its *its, struct its_device *dev,  in vgic_its_save_dte()  argument
2295 	next_offset = compute_next_devid_offset(&its->device_list, dev);  in vgic_its_save_dte()
2302 	return vgic_its_write_entry_lock(its, ptr, val, dte);  in vgic_its_save_dte()
2308  * @its: its handle
2316 static int vgic_its_restore_dte(struct vgic_its *its, u32 id,  in vgic_its_restore_dte()  argument
2320 	u64 baser = its->baser_device_table;  in vgic_its_restore_dte()
2341 	if (!vgic_its_check_id(its, baser, id, NULL))  in vgic_its_restore_dte()
2344 	dev = vgic_its_alloc_device(its, id, itt_addr, num_eventid_bits);  in vgic_its_restore_dte()
2348 	ret = vgic_its_restore_itt(its, dev);  in vgic_its_restore_dte()
2350 		vgic_its_free_device(its->dev->kvm, its, dev);  in vgic_its_restore_dte()
2376 static int vgic_its_save_device_tables(struct vgic_its *its)  in vgic_its_save_device_tables()  argument
2378 	u64 baser = its->baser_device_table;  in vgic_its_save_device_tables()
2384 	list_sort(NULL, &its->device_list, vgic_its_device_cmp);  in vgic_its_save_device_tables()
2386 	list_for_each_entry(dev, &its->device_list, dev_list) {  in vgic_its_save_device_tables()
2390 		if (!vgic_its_check_id(its, baser,  in vgic_its_save_device_tables()
2394 		ret = vgic_its_save_itt(its, dev);  in vgic_its_save_device_tables()
2398 		ret = vgic_its_save_dte(its, dev, eaddr);  in vgic_its_save_device_tables()
2408  * @its: its handle
2417 static int handle_l1_dte(struct vgic_its *its, u32 id, void *addr,  in handle_l1_dte()  argument
2420 	const struct vgic_its_abi *abi = vgic_its_get_abi(its);  in handle_l1_dte()
2434 	ret = scan_its_table(its, gpa, SZ_64K, dte_esz,  in handle_l1_dte()
2444 static int vgic_its_restore_device_tables(struct vgic_its *its)  in vgic_its_restore_device_tables()  argument
2446 	const struct vgic_its_abi *abi = vgic_its_get_abi(its);  in vgic_its_restore_device_tables()
2447 	u64 baser = its->baser_device_table;  in vgic_its_restore_device_tables()
2459 		ret = scan_its_table(its, l1_gpa, l1_tbl_size, l1_esz, 0,  in vgic_its_restore_device_tables()
2463 		ret = scan_its_table(its, l1_gpa, l1_tbl_size, l1_esz, 0,  in vgic_its_restore_device_tables()
2472 		vgic_its_free_device_list(its->dev->kvm, its);  in vgic_its_restore_device_tables()
2477 static int vgic_its_save_cte(struct vgic_its *its,  in vgic_its_save_cte()  argument
2488 	return vgic_its_write_entry_lock(its, gpa, val, cte);  in vgic_its_save_cte()
2492  * Restore a collection entry into the ITS collection table.
2496 static int vgic_its_restore_cte(struct vgic_its *its, gpa_t gpa)  in vgic_its_restore_cte()  argument
2499 	struct kvm *kvm = its->dev->kvm;  in vgic_its_restore_cte()
2504 	ret = vgic_its_read_entry_lock(its, gpa, &val, cte);  in vgic_its_restore_cte()
2518 	collection = find_collection(its, coll_id);  in vgic_its_restore_cte()
2522 	if (!vgic_its_check_id(its, its->baser_coll_table, coll_id, NULL))  in vgic_its_restore_cte()
2525 	ret = vgic_its_alloc_collection(its, &collection, coll_id);  in vgic_its_restore_cte()
2536 static int vgic_its_save_collection_table(struct vgic_its *its)  in vgic_its_save_collection_table()  argument
2538 	const struct vgic_its_abi *abi = vgic_its_get_abi(its);  in vgic_its_save_collection_table()
2539 	u64 baser = its->baser_coll_table;  in vgic_its_save_collection_table()
2550 	list_for_each_entry(collection, &its->collection_list, coll_list) {  in vgic_its_save_collection_table()
2551 		ret = vgic_its_save_cte(its, collection, gpa);  in vgic_its_save_collection_table()
2565 	return vgic_its_write_entry_lock(its, gpa, 0ULL, cte);  in vgic_its_save_collection_table()
2570  * in guest memory and restores the ITS internal state. Requires the
2573 static int vgic_its_restore_collection_table(struct vgic_its *its)  in vgic_its_restore_collection_table()  argument
2575 	const struct vgic_its_abi *abi = vgic_its_get_abi(its);  in vgic_its_restore_collection_table()
2576 	u64 baser = its->baser_coll_table;  in vgic_its_restore_collection_table()
2590 		ret = vgic_its_restore_cte(its, gpa);  in vgic_its_restore_collection_table()
2601 		vgic_its_free_collection_list(its->dev->kvm, its);  in vgic_its_restore_collection_table()
2607  * vgic_its_save_tables_v0 - Save the ITS tables into guest ARM
2610 static int vgic_its_save_tables_v0(struct vgic_its *its)  in vgic_its_save_tables_v0()  argument
2614 	ret = vgic_its_save_device_tables(its);  in vgic_its_save_tables_v0()
2618 	return vgic_its_save_collection_table(its);  in vgic_its_save_tables_v0()
2622  * vgic_its_restore_tables_v0 - Restore the ITS tables from guest RAM
2626 static int vgic_its_restore_tables_v0(struct vgic_its *its)  in vgic_its_restore_tables_v0()  argument
2630 	ret = vgic_its_restore_collection_table(its);  in vgic_its_restore_tables_v0()
2634 	ret = vgic_its_restore_device_tables(its);  in vgic_its_restore_tables_v0()
2636 		vgic_its_free_collection_list(its->dev->kvm, its);  in vgic_its_restore_tables_v0()
2640 static int vgic_its_commit_v0(struct vgic_its *its)  in vgic_its_commit_v0()  argument
2644 	abi = vgic_its_get_abi(its);  in vgic_its_commit_v0()
2645 	its->baser_coll_table &= ~GITS_BASER_ENTRY_SIZE_MASK;  in vgic_its_commit_v0()
2646 	its->baser_device_table &= ~GITS_BASER_ENTRY_SIZE_MASK;  in vgic_its_commit_v0()
2648 	its->baser_coll_table |= (GIC_ENCODE_SZ(abi->cte_esz, 5)  in vgic_its_commit_v0()
2651 	its->baser_device_table |= (GIC_ENCODE_SZ(abi->dte_esz, 5)  in vgic_its_commit_v0()
2656 static void vgic_its_reset(struct kvm *kvm, struct vgic_its *its)  in vgic_its_reset()  argument
2659 	its->baser_coll_table &= ~GITS_BASER_VALID;  in vgic_its_reset()
2660 	its->baser_device_table &= ~GITS_BASER_VALID;  in vgic_its_reset()
2661 	its->cbaser = 0;  in vgic_its_reset()
2662 	its->creadr = 0;  in vgic_its_reset()
2663 	its->cwriter = 0;  in vgic_its_reset()
2664 	its->enabled = 0;  in vgic_its_reset()
2665 	vgic_its_free_device_list(kvm, its);  in vgic_its_reset()
2666 	vgic_its_free_collection_list(kvm, its);  in vgic_its_reset()
2697 static int vgic_its_ctrl(struct kvm *kvm, struct vgic_its *its, u64 attr)  in vgic_its_ctrl()  argument
2699 	const struct vgic_its_abi *abi = vgic_its_get_abi(its);  in vgic_its_ctrl()
2713 	mutex_lock(&its->its_lock);  in vgic_its_ctrl()
2717 		vgic_its_reset(kvm, its);  in vgic_its_ctrl()
2720 		ret = abi->save_tables(its);  in vgic_its_ctrl()
2723 		ret = abi->restore_tables(its);  in vgic_its_ctrl()
2727 	mutex_unlock(&its->its_lock);  in vgic_its_ctrl()
2740  * dirty guest pages. When vgic/its tables are being saved, the backup
2754 	struct vgic_its *its = dev->private;  in vgic_its_set_attr()  local
2769 		ret = vgic_check_iorange(dev->kvm, its->vgic_its_base,  in vgic_its_set_attr()
2774 		return vgic_register_its_iodev(dev->kvm, its, addr);  in vgic_its_set_attr()
2777 		return vgic_its_ctrl(dev->kvm, its, attr->attr);  in vgic_its_set_attr()
2796 		struct vgic_its *its = dev->private;  in vgic_its_get_attr()  local
2797 		u64 addr = its->vgic_its_base;  in vgic_its_get_attr()
2826 	.name = "kvm-arm-vgic-its",