7 #include <linux/irq.h>
62  * Index the VM's xarray of mapped LPIs and return a reference to the IRQ
64  * finished with the IRQ.
69 	struct vgic_irq *irq = NULL;  in vgic_get_lpi()  local
73 	irq = xa_load(&dist->lpi_xa, intid);  in vgic_get_lpi()
74 	if (!vgic_try_get_irq_kref(irq))  in vgic_get_lpi()
75 		irq = NULL;  in vgic_get_lpi()
79 	return irq;  in vgic_get_lpi()
85  * to call vgic_put_irq() once it's finished with this IRQ.
126 void vgic_put_irq(struct kvm *kvm, struct vgic_irq *irq)  in vgic_put_irq()  argument
131 	if (irq->intid < VGIC_MIN_LPI)  in vgic_put_irq()
134 	if (!kref_put(&irq->refcount, vgic_irq_release))  in vgic_put_irq()
138 	__xa_erase(&dist->lpi_xa, irq->intid);  in vgic_put_irq()
141 	kfree_rcu(irq, rcu);  in vgic_put_irq()
147 	struct vgic_irq *irq, *tmp;  in vgic_flush_pending_lpis()  local
152 	list_for_each_entry_safe(irq, tmp, &vgic_cpu->ap_list_head, ap_list) {  in vgic_flush_pending_lpis()
153 		if (irq->intid >= VGIC_MIN_LPI) {  in vgic_flush_pending_lpis()
154 			raw_spin_lock(&irq->irq_lock);  in vgic_flush_pending_lpis()
155 			list_del(&irq->ap_list);  in vgic_flush_pending_lpis()
156 			irq->vcpu = NULL;  in vgic_flush_pending_lpis()
157 			raw_spin_unlock(&irq->irq_lock);  in vgic_flush_pending_lpis()
158 			vgic_put_irq(vcpu->kvm, irq);  in vgic_flush_pending_lpis()
165 void vgic_irq_set_phys_pending(struct vgic_irq *irq, bool pending)  in vgic_irq_set_phys_pending()  argument
167 	WARN_ON(irq_set_irqchip_state(irq->host_irq,  in vgic_irq_set_phys_pending()
172 bool vgic_get_phys_line_level(struct vgic_irq *irq)  in vgic_get_phys_line_level()  argument
176 	BUG_ON(!irq->hw);  in vgic_get_phys_line_level()
178 	if (irq->ops && irq->ops->get_input_level)  in vgic_get_phys_line_level()
179 		return irq->ops->get_input_level(irq->intid);  in vgic_get_phys_line_level()
181 	WARN_ON(irq_get_irqchip_state(irq->host_irq,  in vgic_get_phys_line_level()
188 void vgic_irq_set_phys_active(struct vgic_irq *irq, bool active)  in vgic_irq_set_phys_active()  argument
191 	BUG_ON(!irq->hw);  in vgic_irq_set_phys_active()
192 	WARN_ON(irq_set_irqchip_state(irq->host_irq,  in vgic_irq_set_phys_active()
198  * vgic_target_oracle - compute the target vcpu for an irq
200  * @irq:	The irq to route. Must be already locked.
206  * Requires the IRQ lock to be held.
208 static struct kvm_vcpu *vgic_target_oracle(struct vgic_irq *irq)  in vgic_target_oracle()  argument
210 	lockdep_assert_held(&irq->irq_lock);  in vgic_target_oracle()
213 	if (irq->active)  in vgic_target_oracle()
214 		return irq->vcpu ? : irq->target_vcpu;  in vgic_target_oracle()
217 	 * If the IRQ is not active but enabled and pending, we should direct  in vgic_target_oracle()
222 	if (irq->enabled && irq_is_pending(irq)) {  in vgic_target_oracle()
223 		if (unlikely(irq->target_vcpu &&  in vgic_target_oracle()
224 			     !irq->target_vcpu->kvm->arch.vgic.enabled))  in vgic_target_oracle()
227 		return irq->target_vcpu;  in vgic_target_oracle()
230 	/* If neither active nor pending and enabled, then this IRQ should not  in vgic_target_oracle()
302  * rising edge, and in-kernel connected IRQ lines can only be controlled by
305 static bool vgic_validate_injection(struct vgic_irq *irq, bool level, void *owner)  in vgic_validate_injection()  argument
307 	if (irq->owner != owner)  in vgic_validate_injection()
310 	switch (irq->config) {  in vgic_validate_injection()
312 		return irq->line_level != level;  in vgic_validate_injection()
321  * Check whether an IRQ needs to (and can) be queued to a VCPU's ap list.
323  * Returns true when the IRQ was queued, false otherwise.
325  * Needs to be entered with the IRQ lock already held, but will return
328 bool vgic_queue_irq_unlock(struct kvm *kvm, struct vgic_irq *irq,  in vgic_queue_irq_unlock()  argument
329 			   unsigned long flags) __releases(&irq->irq_lock)  in vgic_queue_irq_unlock()
333 	lockdep_assert_held(&irq->irq_lock);  in vgic_queue_irq_unlock()
336 	vcpu = vgic_target_oracle(irq);  in vgic_queue_irq_unlock()
337 	if (irq->vcpu || !vcpu) {  in vgic_queue_irq_unlock()
339 		 * If this IRQ is already on a VCPU's ap_list, then it  in vgic_queue_irq_unlock()
343 		 * Otherwise, if the irq is not pending and enabled, it does  in vgic_queue_irq_unlock()
347 		raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in vgic_queue_irq_unlock()
353 		 * while the IRQ is already on the VCPU's AP list, the  in vgic_queue_irq_unlock()
366 	 * We must unlock the irq lock to take the ap_list_lock where  in vgic_queue_irq_unlock()
369 	raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in vgic_queue_irq_unlock()
374 	raw_spin_lock(&irq->irq_lock);  in vgic_queue_irq_unlock()
380 	 * 1) The irq lost its pending state or was disabled behind our  in vgic_queue_irq_unlock()
382 	 * 2) Someone changed the affinity on this irq behind our  in vgic_queue_irq_unlock()
388 	if (unlikely(irq->vcpu || vcpu != vgic_target_oracle(irq))) {  in vgic_queue_irq_unlock()
389 		raw_spin_unlock(&irq->irq_lock);  in vgic_queue_irq_unlock()
393 		raw_spin_lock_irqsave(&irq->irq_lock, flags);  in vgic_queue_irq_unlock()
398 	 * Grab a reference to the irq to reflect the fact that it is  in vgic_queue_irq_unlock()
400 	 * reference on the irq.  in vgic_queue_irq_unlock()
402 	vgic_get_irq_kref(irq);  in vgic_queue_irq_unlock()
403 	list_add_tail(&irq->ap_list, &vcpu->arch.vgic_cpu.ap_list_head);  in vgic_queue_irq_unlock()
404 	irq->vcpu = vcpu;  in vgic_queue_irq_unlock()
406 	raw_spin_unlock(&irq->irq_lock);  in vgic_queue_irq_unlock()
416  * kvm_vgic_inject_irq - Inject an IRQ from a device to the vgic
424  * @owner:   The opaque pointer to the owner of the IRQ being raised to verify
425  *           that the caller is allowed to inject this IRQ.  Userspace
435 	struct vgic_irq *irq;  in kvm_vgic_inject_irq()  local
449 		irq = vgic_get_vcpu_irq(vcpu, intid);  in kvm_vgic_inject_irq()
451 		irq = vgic_get_irq(kvm, intid);  in kvm_vgic_inject_irq()
452 	if (!irq)  in kvm_vgic_inject_irq()
455 	raw_spin_lock_irqsave(&irq->irq_lock, flags);  in kvm_vgic_inject_irq()
457 	if (!vgic_validate_injection(irq, level, owner)) {  in kvm_vgic_inject_irq()
459 		raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in kvm_vgic_inject_irq()
460 		vgic_put_irq(kvm, irq);  in kvm_vgic_inject_irq()
464 	if (irq->config == VGIC_CONFIG_LEVEL)  in kvm_vgic_inject_irq()
465 		irq->line_level = level;  in kvm_vgic_inject_irq()
467 		irq->pending_latch = true;  in kvm_vgic_inject_irq()
469 	vgic_queue_irq_unlock(kvm, irq, flags);  in kvm_vgic_inject_irq()
470 	vgic_put_irq(kvm, irq);  in kvm_vgic_inject_irq()
475 /* @irq->irq_lock must be held */
476 static int kvm_vgic_map_irq(struct kvm_vcpu *vcpu, struct vgic_irq *irq,  in kvm_vgic_map_irq()  argument
484 	 * Find the physical IRQ number corresponding to @host_irq  in kvm_vgic_map_irq()
495 	irq->hw = true;  in kvm_vgic_map_irq()
496 	irq->host_irq = host_irq;  in kvm_vgic_map_irq()
497 	irq->hwintid = data->hwirq;  in kvm_vgic_map_irq()
498 	irq->ops = ops;  in kvm_vgic_map_irq()
502 /* @irq->irq_lock must be held */
503 static inline void kvm_vgic_unmap_irq(struct vgic_irq *irq)  in kvm_vgic_unmap_irq()  argument
505 	irq->hw = false;  in kvm_vgic_unmap_irq()
506 	irq->hwintid = 0;  in kvm_vgic_unmap_irq()
507 	irq->ops = NULL;  in kvm_vgic_unmap_irq()
513 	struct vgic_irq *irq = vgic_get_vcpu_irq(vcpu, vintid);  in kvm_vgic_map_phys_irq()  local
517 	BUG_ON(!irq);  in kvm_vgic_map_phys_irq()
519 	raw_spin_lock_irqsave(&irq->irq_lock, flags);  in kvm_vgic_map_phys_irq()
520 	ret = kvm_vgic_map_irq(vcpu, irq, host_irq, ops);  in kvm_vgic_map_phys_irq()
521 	raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in kvm_vgic_map_phys_irq()
522 	vgic_put_irq(vcpu->kvm, irq);  in kvm_vgic_map_phys_irq()
528  * kvm_vgic_reset_mapped_irq - Reset a mapped IRQ
538 	struct vgic_irq *irq = vgic_get_vcpu_irq(vcpu, vintid);  in kvm_vgic_reset_mapped_irq()  local
541 	if (!irq->hw)  in kvm_vgic_reset_mapped_irq()
544 	raw_spin_lock_irqsave(&irq->irq_lock, flags);  in kvm_vgic_reset_mapped_irq()
545 	irq->active = false;  in kvm_vgic_reset_mapped_irq()
546 	irq->pending_latch = false;  in kvm_vgic_reset_mapped_irq()
547 	irq->line_level = false;  in kvm_vgic_reset_mapped_irq()
548 	raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in kvm_vgic_reset_mapped_irq()
550 	vgic_put_irq(vcpu->kvm, irq);  in kvm_vgic_reset_mapped_irq()
555 	struct vgic_irq *irq;  in kvm_vgic_unmap_phys_irq()  local
561 	irq = vgic_get_vcpu_irq(vcpu, vintid);  in kvm_vgic_unmap_phys_irq()
562 	BUG_ON(!irq);  in kvm_vgic_unmap_phys_irq()
564 	raw_spin_lock_irqsave(&irq->irq_lock, flags);  in kvm_vgic_unmap_phys_irq()
565 	kvm_vgic_unmap_irq(irq);  in kvm_vgic_unmap_phys_irq()
566 	raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in kvm_vgic_unmap_phys_irq()
567 	vgic_put_irq(vcpu->kvm, irq);  in kvm_vgic_unmap_phys_irq()
574 	struct vgic_irq *irq = vgic_get_vcpu_irq(vcpu, vintid);  in kvm_vgic_get_map()  local
578 	raw_spin_lock_irqsave(&irq->irq_lock, flags);  in kvm_vgic_get_map()
579 	if (irq->hw)  in kvm_vgic_get_map()
580 		ret = irq->hwintid;  in kvm_vgic_get_map()
581 	raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in kvm_vgic_get_map()
583 	vgic_put_irq(vcpu->kvm, irq);  in kvm_vgic_get_map()
599 	struct vgic_irq *irq;  in kvm_vgic_set_owner()  local
610 	irq = vgic_get_vcpu_irq(vcpu, intid);  in kvm_vgic_set_owner()
611 	raw_spin_lock_irqsave(&irq->irq_lock, flags);  in kvm_vgic_set_owner()
612 	if (irq->owner && irq->owner != owner)  in kvm_vgic_set_owner()
615 		irq->owner = owner;  in kvm_vgic_set_owner()
616 	raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in kvm_vgic_set_owner()
632 	struct vgic_irq *irq, *tmp;  in vgic_prune_ap_list()  local
639 	list_for_each_entry_safe(irq, tmp, &vgic_cpu->ap_list_head, ap_list) {  in vgic_prune_ap_list()
643 		raw_spin_lock(&irq->irq_lock);  in vgic_prune_ap_list()
645 		BUG_ON(vcpu != irq->vcpu);  in vgic_prune_ap_list()
647 		target_vcpu = vgic_target_oracle(irq);  in vgic_prune_ap_list()
654 			list_del(&irq->ap_list);  in vgic_prune_ap_list()
655 			irq->vcpu = NULL;  in vgic_prune_ap_list()
656 			raw_spin_unlock(&irq->irq_lock);  in vgic_prune_ap_list()
662 			 * we remove the irq from the list, we drop  in vgic_prune_ap_list()
665 			vgic_put_irq(vcpu->kvm, irq);  in vgic_prune_ap_list()
671 			raw_spin_unlock(&irq->irq_lock);  in vgic_prune_ap_list()
677 		raw_spin_unlock(&irq->irq_lock);  in vgic_prune_ap_list()
695 		raw_spin_lock(&irq->irq_lock);  in vgic_prune_ap_list()
706 		if (target_vcpu == vgic_target_oracle(irq)) {  in vgic_prune_ap_list()
709 			list_del(&irq->ap_list);  in vgic_prune_ap_list()
710 			irq->vcpu = target_vcpu;  in vgic_prune_ap_list()
711 			list_add_tail(&irq->ap_list, &new_cpu->ap_list_head);  in vgic_prune_ap_list()
715 		raw_spin_unlock(&irq->irq_lock);  in vgic_prune_ap_list()
740 				    struct vgic_irq *irq, int lr)  in vgic_populate_lr()  argument
742 	lockdep_assert_held(&irq->irq_lock);  in vgic_populate_lr()
745 		vgic_v2_populate_lr(vcpu, irq, lr);  in vgic_populate_lr()
747 		vgic_v3_populate_lr(vcpu, irq, lr);  in vgic_populate_lr()
771 	struct vgic_irq *irq;  in compute_ap_list_depth()  local
778 	list_for_each_entry(irq, &vgic_cpu->ap_list_head, ap_list) {  in compute_ap_list_depth()
781 		raw_spin_lock(&irq->irq_lock);  in compute_ap_list_depth()
783 		w = vgic_irq_get_lr_count(irq);  in compute_ap_list_depth()
784 		raw_spin_unlock(&irq->irq_lock);  in compute_ap_list_depth()
796 	struct vgic_irq *irq;  in vgic_flush_lr_state()  local
810 	list_for_each_entry(irq, &vgic_cpu->ap_list_head, ap_list) {  in vgic_flush_lr_state()
811 		raw_spin_lock(&irq->irq_lock);  in vgic_flush_lr_state()
815 		 * guarantee that they are all seen before any IRQ of  in vgic_flush_lr_state()
820 		if (multi_sgi && irq->priority > prio) {  in vgic_flush_lr_state()
821 			_raw_spin_unlock(&irq->irq_lock);  in vgic_flush_lr_state()
825 		if (likely(vgic_target_oracle(irq) == vcpu)) {  in vgic_flush_lr_state()
826 			vgic_populate_lr(vcpu, irq, count++);  in vgic_flush_lr_state()
828 			if (irq->source)  in vgic_flush_lr_state()
829 				prio = irq->priority;  in vgic_flush_lr_state()
832 		raw_spin_unlock(&irq->irq_lock);  in vgic_flush_lr_state()
835 			if (!list_is_last(&irq->ap_list,  in vgic_flush_lr_state()
965 	struct vgic_irq *irq;  in kvm_vgic_vcpu_pending_irq()  local
980 	list_for_each_entry(irq, &vgic_cpu->ap_list_head, ap_list) {  in kvm_vgic_vcpu_pending_irq()
981 		raw_spin_lock(&irq->irq_lock);  in kvm_vgic_vcpu_pending_irq()
982 		pending = irq_is_pending(irq) && irq->enabled &&  in kvm_vgic_vcpu_pending_irq()
983 			  !irq->active &&  in kvm_vgic_vcpu_pending_irq()
984 			  irq->priority < vmcr.pmr;  in kvm_vgic_vcpu_pending_irq()
985 		raw_spin_unlock(&irq->irq_lock);  in kvm_vgic_vcpu_pending_irq()
1015 	struct vgic_irq *irq;  in kvm_vgic_map_is_active()  local
1022 	irq = vgic_get_vcpu_irq(vcpu, vintid);  in kvm_vgic_map_is_active()
1023 	raw_spin_lock_irqsave(&irq->irq_lock, flags);  in kvm_vgic_map_is_active()
1024 	map_is_active = irq->hw && irq->active;  in kvm_vgic_map_is_active()
1025 	raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in kvm_vgic_map_is_active()
1026 	vgic_put_irq(vcpu->kvm, irq);  in kvm_vgic_map_is_active()
1051 void vgic_irq_handle_resampling(struct vgic_irq *irq,  in vgic_irq_handle_resampling()  argument
1054 	if (vgic_irq_is_mapped_level(irq)) {  in vgic_irq_handle_resampling()
1057 		if (unlikely(vgic_irq_needs_resampling(irq))) {  in vgic_irq_handle_resampling()
1058 			resample = !(irq->active || irq->pending_latch);  in vgic_irq_handle_resampling()
1059 		} else if (lr_pending || (lr_deactivated && irq->line_level)) {  in vgic_irq_handle_resampling()
1060 			irq->line_level = vgic_get_phys_line_level(irq);  in vgic_irq_handle_resampling()
1061 			resample = !irq->line_level;  in vgic_irq_handle_resampling()
1065 			vgic_irq_set_phys_active(irq, false);  in vgic_irq_handle_resampling()