88 	struct nvme_ns *ns = req->q->queuedata;  in nvme_failover_req()  local
93 	nvme_mpath_clear_current_path(ns);  in nvme_failover_req()
100 	if (nvme_is_ana_error(status) && ns->ctrl->ana_log_buf) {  in nvme_failover_req()
101 		set_bit(NVME_NS_ANA_PENDING, &ns->flags);  in nvme_failover_req()
102 		queue_work(nvme_wq, &ns->ctrl->ana_work);  in nvme_failover_req()
105 	spin_lock_irqsave(&ns->head->requeue_lock, flags);  in nvme_failover_req()
107 		bio_set_dev(bio, ns->head->disk->part0);  in nvme_failover_req()
121 	blk_steal_bios(&ns->head->requeue_list, req);  in nvme_failover_req()
122 	spin_unlock_irqrestore(&ns->head->requeue_lock, flags);  in nvme_failover_req()
126 	kblockd_schedule_work(&ns->head->requeue_work);  in nvme_failover_req()
131 	struct nvme_ns *ns = rq->q->queuedata;  in nvme_mpath_start_request()  local
132 	struct gendisk *disk = ns->head->disk;  in nvme_mpath_start_request()
134 	if (READ_ONCE(ns->head->subsys->iopolicy) == NVME_IOPOLICY_QD) {  in nvme_mpath_start_request()
135 		atomic_inc(&ns->ctrl->nr_active);  in nvme_mpath_start_request()
150 	struct nvme_ns *ns = rq->q->queuedata;  in nvme_mpath_end_request()  local
153 		atomic_dec_if_positive(&ns->ctrl->nr_active);  in nvme_mpath_end_request()
157 	bdev_end_io_acct(ns->head->disk->part0, req_op(rq),  in nvme_mpath_end_request()
164 	struct nvme_ns *ns;  in nvme_kick_requeue_lists()  local
168 	list_for_each_entry_srcu(ns, &ctrl->namespaces, list,  in nvme_kick_requeue_lists()
170 		if (!ns->head->disk)  in nvme_kick_requeue_lists()
172 		kblockd_schedule_work(&ns->head->requeue_work);  in nvme_kick_requeue_lists()
173 		if (nvme_ctrl_state(ns->ctrl) == NVME_CTRL_LIVE)  in nvme_kick_requeue_lists()
174 			disk_uevent(ns->head->disk, KOBJ_CHANGE);  in nvme_kick_requeue_lists()
188 bool nvme_mpath_clear_current_path(struct nvme_ns *ns)  in nvme_mpath_clear_current_path()  argument
190 	struct nvme_ns_head *head = ns->head;  in nvme_mpath_clear_current_path()
198 		if (ns == rcu_access_pointer(head->current_path[node])) {  in nvme_mpath_clear_current_path()
209 	struct nvme_ns *ns;  in nvme_mpath_clear_ctrl_paths()  local
213 	list_for_each_entry_srcu(ns, &ctrl->namespaces, list,  in nvme_mpath_clear_ctrl_paths()
215 		nvme_mpath_clear_current_path(ns);  in nvme_mpath_clear_ctrl_paths()
216 		kblockd_schedule_work(&ns->head->requeue_work);  in nvme_mpath_clear_ctrl_paths()
221 void nvme_mpath_revalidate_paths(struct nvme_ns *ns)  in nvme_mpath_revalidate_paths()  argument
223 	struct nvme_ns_head *head = ns->head;  in nvme_mpath_revalidate_paths()
229 	list_for_each_entry_srcu(ns, &head->list, siblings,  in nvme_mpath_revalidate_paths()
231 		if (capacity != get_capacity(ns->disk))  in nvme_mpath_revalidate_paths()
232 			clear_bit(NVME_NS_READY, &ns->flags);  in nvme_mpath_revalidate_paths()
241 static bool nvme_path_is_disabled(struct nvme_ns *ns)  in nvme_path_is_disabled()  argument
243 	enum nvme_ctrl_state state = nvme_ctrl_state(ns->ctrl);  in nvme_path_is_disabled()
252 	if (test_bit(NVME_NS_ANA_PENDING, &ns->flags) ||  in nvme_path_is_disabled()
253 	    !test_bit(NVME_NS_READY, &ns->flags))  in nvme_path_is_disabled()
261 	struct nvme_ns *found = NULL, *fallback = NULL, *ns;  in __nvme_find_path()  local
263 	list_for_each_entry_srcu(ns, &head->list, siblings,  in __nvme_find_path()
265 		if (nvme_path_is_disabled(ns))  in __nvme_find_path()
268 		if (ns->ctrl->numa_node != NUMA_NO_NODE &&  in __nvme_find_path()
270 			distance = node_distance(node, ns->ctrl->numa_node);  in __nvme_find_path()
274 		switch (ns->ana_state) {  in __nvme_find_path()
278 				found = ns;  in __nvme_find_path()
284 				fallback = ns;  in __nvme_find_path()
300 		struct nvme_ns *ns)  in nvme_next_ns()  argument
302 	ns = list_next_or_null_rcu(&head->list, &ns->siblings, struct nvme_ns,  in nvme_next_ns()
304 	if (ns)  in nvme_next_ns()
305 		return ns;  in nvme_next_ns()
311 	struct nvme_ns *ns, *found = NULL;  in nvme_round_robin_path()  local
325 	for (ns = nvme_next_ns(head, old);  in nvme_round_robin_path()
326 	     ns && ns != old;  in nvme_round_robin_path()
327 	     ns = nvme_next_ns(head, ns)) {  in nvme_round_robin_path()
328 		if (nvme_path_is_disabled(ns))  in nvme_round_robin_path()
331 		if (ns->ana_state == NVME_ANA_OPTIMIZED) {  in nvme_round_robin_path()
332 			found = ns;  in nvme_round_robin_path()
335 		if (ns->ana_state == NVME_ANA_NONOPTIMIZED)  in nvme_round_robin_path()
336 			found = ns;  in nvme_round_robin_path()
359 	struct nvme_ns *best_opt = NULL, *best_nonopt = NULL, *ns;  in nvme_queue_depth_path()  local
363 	list_for_each_entry_srcu(ns, &head->list, siblings,  in nvme_queue_depth_path()
365 		if (nvme_path_is_disabled(ns))  in nvme_queue_depth_path()
368 		depth = atomic_read(&ns->ctrl->nr_active);  in nvme_queue_depth_path()
370 		switch (ns->ana_state) {  in nvme_queue_depth_path()
374 				best_opt = ns;  in nvme_queue_depth_path()
380 				best_nonopt = ns;  in nvme_queue_depth_path()
394 static inline bool nvme_path_is_optimized(struct nvme_ns *ns)  in nvme_path_is_optimized()  argument
396 	return nvme_ctrl_state(ns->ctrl) == NVME_CTRL_LIVE &&  in nvme_path_is_optimized()
397 		ns->ana_state == NVME_ANA_OPTIMIZED;  in nvme_path_is_optimized()
403 	struct nvme_ns *ns;  in nvme_numa_path()  local
405 	ns = srcu_dereference(head->current_path[node], &head->srcu);  in nvme_numa_path()
406 	if (unlikely(!ns))  in nvme_numa_path()
408 	if (unlikely(!nvme_path_is_optimized(ns)))  in nvme_numa_path()
410 	return ns;  in nvme_numa_path()
427 	struct nvme_ns *ns;  in nvme_available_path()  local
432 	list_for_each_entry_srcu(ns, &head->list, siblings,  in nvme_available_path()
434 		if (test_bit(NVME_CTRL_FAILFAST_EXPIRED, &ns->ctrl->flags))  in nvme_available_path()
436 		switch (nvme_ctrl_state(ns->ctrl)) {  in nvme_available_path()
452 	struct nvme_ns *ns;  in nvme_ns_head_submit_bio()  local
465 	ns = nvme_find_path(head);  in nvme_ns_head_submit_bio()
466 	if (likely(ns)) {  in nvme_ns_head_submit_bio()
467 		bio_set_dev(bio, ns->disk->part0);  in nvme_ns_head_submit_bio()
469 		trace_block_bio_remap(bio, disk_devt(ns->head->disk),  in nvme_ns_head_submit_bio()
503 	struct nvme_ns *ns;  in nvme_ns_head_get_unique_id()  local
507 	ns = nvme_find_path(head);  in nvme_ns_head_get_unique_id()
508 	if (ns)  in nvme_ns_head_get_unique_id()
509 		ret = nvme_ns_get_unique_id(ns, id, type);  in nvme_ns_head_get_unique_id()
519 	struct nvme_ns *ns;  in nvme_ns_head_report_zones()  local
523 	ns = nvme_find_path(head);  in nvme_ns_head_report_zones()
524 	if (ns)  in nvme_ns_head_report_zones()
525 		ret = nvme_ns_report_zones(ns, sector, nr_zones, cb, data);  in nvme_ns_head_report_zones()
665 static void nvme_mpath_set_live(struct nvme_ns *ns)  in nvme_mpath_set_live()  argument
667 	struct nvme_ns_head *head = ns->head;  in nvme_mpath_set_live()
690 	if (nvme_path_is_optimized(ns)) {  in nvme_mpath_set_live()
754 		struct nvme_ns *ns)  in nvme_update_ns_ana_state()  argument
756 	ns->ana_grpid = le32_to_cpu(desc->grpid);  in nvme_update_ns_ana_state()
757 	ns->ana_state = desc->state;  in nvme_update_ns_ana_state()
758 	clear_bit(NVME_NS_ANA_PENDING, &ns->flags);  in nvme_update_ns_ana_state()
768 	if (nvme_state_is_live(ns->ana_state) &&  in nvme_update_ns_ana_state()
769 	    nvme_ctrl_state(ns->ctrl) == NVME_CTRL_LIVE)  in nvme_update_ns_ana_state()
770 		nvme_mpath_set_live(ns);  in nvme_update_ns_ana_state()
778 	struct nvme_ns *ns;  in nvme_update_ana_state()  local
792 	list_for_each_entry_srcu(ns, &ctrl->namespaces, list,  in nvme_update_ana_state()
797 		if (ns->head->ns_id < nsid)  in nvme_update_ana_state()
799 		if (ns->head->ns_id == nsid)  in nvme_update_ana_state()
800 			nvme_update_ns_ana_state(desc, ns);  in nvme_update_ana_state()
803 		if (ns->head->ns_id > nsid)  in nvme_update_ana_state()
952 	struct nvme_ns *ns = nvme_get_ns_from_dev(dev);  in ana_state_show()  local
954 	return sysfs_emit(buf, "%s\n", nvme_ana_state_names[ns->ana_state]);  in ana_state_show()
970 void nvme_mpath_add_disk(struct nvme_ns *ns, __le32 anagrpid)  in nvme_mpath_add_disk()  argument
972 	if (nvme_ctrl_use_ana(ns->ctrl)) {  in nvme_mpath_add_disk()
978 		mutex_lock(&ns->ctrl->ana_lock);  in nvme_mpath_add_disk()
979 		ns->ana_grpid = le32_to_cpu(anagrpid);  in nvme_mpath_add_disk()
980 		nvme_parse_ana_log(ns->ctrl, &desc, nvme_lookup_ana_group_desc);  in nvme_mpath_add_disk()
981 		mutex_unlock(&ns->ctrl->ana_lock);  in nvme_mpath_add_disk()
984 			nvme_update_ns_ana_state(&desc, ns);  in nvme_mpath_add_disk()
987 			set_bit(NVME_NS_ANA_PENDING, &ns->flags);  in nvme_mpath_add_disk()
988 			queue_work(nvme_wq, &ns->ctrl->ana_work);  in nvme_mpath_add_disk()
991 		ns->ana_state = NVME_ANA_OPTIMIZED;  in nvme_mpath_add_disk()
992 		nvme_mpath_set_live(ns);  in nvme_mpath_add_disk()
996 	if (blk_queue_is_zoned(ns->queue) && ns->head->disk)  in nvme_mpath_add_disk()
997 		ns->head->disk->nr_zones = ns->disk->nr_zones;  in nvme_mpath_add_disk()