570 	u64 *ddtp = NULL;  in riscv_iommu_get_dc()  local
604 	for (ddtp = iommu->ddt_root; depth-- > 0;) {  in riscv_iommu_get_dc()
610 		ddtp += (devid >> split) & 0x1FF;  in riscv_iommu_get_dc()
617 			ddt = READ_ONCE(*(unsigned long *)ddtp);  in riscv_iommu_get_dc()
619 				ddtp = __va(ppn_to_phys(ddt));  in riscv_iommu_get_dc()
628 			old = cmpxchg_relaxed((unsigned long *)ddtp, ddt, new);  in riscv_iommu_get_dc()
631 				ddtp = (u64 *)ptr;  in riscv_iommu_get_dc()
645 	ddtp += (devid & ((64 << base_format) - 1)) << (3 - base_format);  in riscv_iommu_get_dc()
647 	return (struct riscv_iommu_dc *)ddtp;  in riscv_iommu_get_dc()
665 	u64 ddtp; \
666 	riscv_iommu_readq_timeout((iommu), RISCV_IOMMU_REG_DDTP, ddtp, \
667 				  !(ddtp & RISCV_IOMMU_DDTP_BUSY), 10, \
669 	ddtp; })
673 	u64 ddtp;  in riscv_iommu_iodir_alloc()  local
676 	ddtp = riscv_iommu_read_ddtp(iommu);  in riscv_iommu_iodir_alloc()
677 	if (ddtp & RISCV_IOMMU_DDTP_BUSY)  in riscv_iommu_iodir_alloc()
684 	mode = FIELD_GET(RISCV_IOMMU_DDTP_IOMMU_MODE, ddtp);  in riscv_iommu_iodir_alloc()
690 		ddtp = riscv_iommu_read_ddtp(iommu);  in riscv_iommu_iodir_alloc()
691 		if (ddtp & RISCV_IOMMU_DDTP_BUSY)  in riscv_iommu_iodir_alloc()
694 		iommu->ddt_phys = ppn_to_phys(ddtp);  in riscv_iommu_iodir_alloc()
722 	u64 ddtp, rq_ddtp;  in riscv_iommu_iodir_set_mode()  local
726 	ddtp = riscv_iommu_read_ddtp(iommu);  in riscv_iommu_iodir_set_mode()
727 	if (ddtp & RISCV_IOMMU_DDTP_BUSY)  in riscv_iommu_iodir_set_mode()
731 	mode = FIELD_GET(RISCV_IOMMU_DDTP_IOMMU_MODE, ddtp);  in riscv_iommu_iodir_set_mode()
744 		ddtp = riscv_iommu_read_ddtp(iommu);  in riscv_iommu_iodir_set_mode()
745 		if (ddtp & RISCV_IOMMU_DDTP_BUSY) {  in riscv_iommu_iodir_set_mode()
747 				rq_mode, ddtp);  in riscv_iommu_iodir_set_mode()
752 		mode = FIELD_GET(RISCV_IOMMU_DDTP_IOMMU_MODE, ddtp);  in riscv_iommu_iodir_set_mode()
758 		if (rq_mode < RISCV_IOMMU_DDTP_IOMMU_MODE_1LVL && rq_ddtp != ddtp) {  in riscv_iommu_iodir_set_mode()
760 				ddtp, rq_ddtp);  in riscv_iommu_iodir_set_mode()
1548 	u64 ddtp;  in riscv_iommu_init_check()  local
1555 	ddtp = riscv_iommu_readq(iommu, RISCV_IOMMU_REG_DDTP);  in riscv_iommu_init_check()
1556 	if (ddtp & RISCV_IOMMU_DDTP_BUSY)  in riscv_iommu_init_check()
1559 	if (FIELD_GET(RISCV_IOMMU_DDTP_IOMMU_MODE, ddtp) >  in riscv_iommu_init_check()