Lines Matching refs:dd
108 static int mtip_block_initialize(struct driver_data *dd);
133 static bool mtip_check_surprise_removal(struct driver_data *dd) in mtip_check_surprise_removal() argument
137 if (dd->sr) in mtip_check_surprise_removal()
141 pci_read_config_word(dd->pdev, 0x00, &vendor_id); in mtip_check_surprise_removal()
143 dd->sr = true; in mtip_check_surprise_removal()
144 if (dd->disk) in mtip_check_surprise_removal()
145 blk_mark_disk_dead(dd->disk); in mtip_check_surprise_removal()
152 static struct mtip_cmd *mtip_cmd_from_tag(struct driver_data *dd, in mtip_cmd_from_tag() argument
155 return blk_mq_rq_to_pdu(blk_mq_tag_to_rq(dd->tags.tags[0], tag)); in mtip_cmd_from_tag()
167 static int mtip_hba_reset(struct driver_data *dd) in mtip_hba_reset() argument
172 writel(HOST_RESET, dd->mmio + HOST_CTL); in mtip_hba_reset()
175 readl(dd->mmio + HOST_CTL); in mtip_hba_reset()
184 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)) in mtip_hba_reset()
187 } while ((readl(dd->mmio + HOST_CTL) & HOST_RESET) in mtip_hba_reset()
190 if (readl(dd->mmio + HOST_CTL) & HOST_RESET) in mtip_hba_reset()
326 if (readl(port->dd->mmio + HOST_CAP) & HOST_CAP_64) { in mtip_init_port()
342 for (i = 0; i < port->dd->slot_groups; i++) in mtip_init_port()
349 writel(readl(port->dd->mmio + HOST_IRQ_STAT), in mtip_init_port()
350 port->dd->mmio + HOST_IRQ_STAT); in mtip_init_port()
377 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) in mtip_restart_port()
385 dev_warn(&port->dd->pdev->dev, in mtip_restart_port()
388 if (mtip_hba_reset(port->dd)) in mtip_restart_port()
389 dev_err(&port->dd->pdev->dev, in mtip_restart_port()
396 dev_warn(&port->dd->pdev->dev, "Issuing COM reset\n"); in mtip_restart_port()
408 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) in mtip_restart_port()
422 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) in mtip_restart_port()
426 dev_warn(&port->dd->pdev->dev, in mtip_restart_port()
434 static int mtip_device_reset(struct driver_data *dd) in mtip_device_reset() argument
438 if (mtip_check_surprise_removal(dd)) in mtip_device_reset()
441 if (mtip_hba_reset(dd) < 0) in mtip_device_reset()
445 mtip_init_port(dd->port); in mtip_device_reset()
446 mtip_start_port(dd->port); in mtip_device_reset()
449 writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN, in mtip_device_reset()
450 dd->mmio + HOST_CTL); in mtip_device_reset()
457 static void print_tags(struct driver_data *dd, in print_tags() argument
469 dev_warn(&dd->pdev->dev, in print_tags()
495 static void mtip_handle_tfe(struct driver_data *dd) in mtip_handle_tfe() argument
508 dev_warn(&dd->pdev->dev, "Taskfile error\n"); in mtip_handle_tfe()
510 port = dd->port; in mtip_handle_tfe()
513 cmd = mtip_cmd_from_tag(dd, MTIP_TAG_INTERNAL); in mtip_handle_tfe()
523 for (group = 0; group < dd->slot_groups; group++) { in mtip_handle_tfe()
526 dev_warn(&dd->pdev->dev, "g=%u, comp=%x\n", group, completed); in mtip_handle_tfe()
541 cmd = mtip_cmd_from_tag(dd, tag); in mtip_handle_tfe()
548 print_tags(dd, "completed (TFE)", tagaccum, cmd_cnt); in mtip_handle_tfe()
555 rv = mtip_read_log_page(dd->port, ATA_LOG_SATA_NCQ, in mtip_handle_tfe()
556 dd->port->log_buf, in mtip_handle_tfe()
557 dd->port->log_buf_dma, 1); in mtip_handle_tfe()
559 dev_warn(&dd->pdev->dev, in mtip_handle_tfe()
563 buf = (unsigned char *)dd->port->log_buf; in mtip_handle_tfe()
565 dev_info(&dd->pdev->dev, in mtip_handle_tfe()
567 set_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag); in mtip_handle_tfe()
572 dev_info(&dd->pdev->dev, in mtip_handle_tfe()
574 set_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag); in mtip_handle_tfe()
579 set_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag); in mtip_handle_tfe()
580 dev_info(&dd->pdev->dev, in mtip_handle_tfe()
591 for (group = 0; group < dd->slot_groups; group++) { in mtip_handle_tfe()
595 cmd = mtip_cmd_from_tag(dd, tag); in mtip_handle_tfe()
607 dev_warn(&dd->pdev->dev, in mtip_handle_tfe()
634 dev_warn(&port->dd->pdev->dev, in mtip_handle_tfe()
640 print_tags(dd, "reissued (TFE)", tagaccum, cmd_cnt); in mtip_handle_tfe()
649 struct driver_data *dd = port->dd; in mtip_workq_sdbfx() local
669 command = mtip_cmd_from_tag(dd, tag); in mtip_workq_sdbfx()
676 if (atomic_dec_return(&dd->irq_workers_active) == 0) in mtip_workq_sdbfx()
677 writel(0xffffffff, dd->mmio + HOST_IRQ_STAT); in mtip_workq_sdbfx()
683 static inline void mtip_process_legacy(struct driver_data *dd, u32 port_stat) in mtip_process_legacy() argument
685 struct mtip_port *port = dd->port; in mtip_process_legacy()
686 struct mtip_cmd *cmd = mtip_cmd_from_tag(dd, MTIP_TAG_INTERNAL); in mtip_process_legacy()
700 static inline void mtip_process_errors(struct driver_data *dd, u32 port_stat) in mtip_process_errors() argument
703 dev_warn(&dd->pdev->dev, in mtip_process_errors()
705 writel((1 << 26), dd->port->mmio + PORT_SCR_ERR); in mtip_process_errors()
709 dev_warn(&dd->pdev->dev, in mtip_process_errors()
711 writel((1 << 16), dd->port->mmio + PORT_SCR_ERR); in mtip_process_errors()
715 dev_warn(&dd->pdev->dev, in mtip_process_errors()
718 if (mtip_check_surprise_removal(dd)) in mtip_process_errors()
722 set_bit(MTIP_PF_EH_ACTIVE_BIT, &dd->port->flags); in mtip_process_errors()
723 wake_up_interruptible(&dd->port->svc_wait); in mtip_process_errors()
729 struct driver_data *dd = (struct driver_data *) data; in mtip_handle_irq() local
730 struct mtip_port *port = dd->port; in mtip_handle_irq()
736 hba_stat = readl(dd->mmio + HOST_IRQ_STAT); in mtip_handle_irq()
743 mtip_check_surprise_removal(dd); in mtip_handle_irq()
751 WARN_ON_ONCE(atomic_read(&dd->irq_workers_active) != 0); in mtip_handle_irq()
756 twork = &dd->work[i]; in mtip_handle_irq()
762 atomic_set(&dd->irq_workers_active, workers); in mtip_handle_irq()
765 twork = &dd->work[i]; in mtip_handle_irq()
769 dd->isr_workq, in mtip_handle_irq()
773 if (likely(dd->work[0].completed)) in mtip_handle_irq()
775 dd->work[0].completed); in mtip_handle_irq()
787 if (unlikely(mtip_check_surprise_removal(dd))) { in mtip_handle_irq()
792 &dd->dd_flag)) in mtip_handle_irq()
795 mtip_process_errors(dd, port_stat & PORT_IRQ_ERR); in mtip_handle_irq()
799 mtip_process_legacy(dd, port_stat & PORT_IRQ_LEGACY); in mtip_handle_irq()
804 writel(hba_stat, dd->mmio + HOST_IRQ_STAT); in mtip_handle_irq()
821 struct driver_data *dd = instance; in mtip_irq_handler() local
823 return mtip_handle_irq(dd); in mtip_irq_handler()
852 clear_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag); in mtip_pause_ncq()
853 clear_bit(MTIP_DDF_REBUILD_FAILED_BIT, &port->dd->dd_flag); in mtip_pause_ncq()
873 for (n = 1; n < port->dd->slot_groups; n++) in mtip_commands_active()
894 blk_mq_quiesce_queue(port->dd->queue); in mtip_quiesce_io()
906 if (mtip_check_surprise_removal(port->dd)) in mtip_quiesce_io()
914 blk_mq_unquiesce_queue(port->dd->queue); in mtip_quiesce_io()
917 blk_mq_unquiesce_queue(port->dd->queue); in mtip_quiesce_io()
955 struct driver_data *dd = port->dd; in mtip_exec_internal_command() local
967 dev_err(&dd->pdev->dev, "SG buffer is not 8 byte aligned\n"); in mtip_exec_internal_command()
971 if (mtip_check_surprise_removal(dd)) in mtip_exec_internal_command()
974 rq = blk_mq_alloc_request(dd->queue, REQ_OP_DRV_IN, BLK_MQ_REQ_RESERVED); in mtip_exec_internal_command()
990 dev_warn(&dd->pdev->dev, "Failed to quiesce IO\n"); in mtip_exec_internal_command()
1009 dev_err(&dd->pdev->dev, "Internal command [%02X] failed %d\n", in mtip_exec_internal_command()
1013 if (mtip_check_surprise_removal(dd) || in mtip_exec_internal_command()
1015 &dd->dd_flag)) { in mtip_exec_internal_command()
1016 dev_err(&dd->pdev->dev, in mtip_exec_internal_command()
1022 mtip_device_reset(dd); /* recover from timeout issue */ in mtip_exec_internal_command()
1030 if (!test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)) { in mtip_exec_internal_command()
1031 mtip_device_reset(dd); in mtip_exec_internal_command()
1068 static void mtip_set_timeout(struct driver_data *dd, in mtip_set_timeout() argument
1079 *timeout = ((*(dd->port->identify + 90) * 2) * 60000); in mtip_set_timeout()
1081 *timeout = ((*(dd->port->identify + 89) * 2) * 60000); in mtip_set_timeout()
1123 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) in mtip_get_identify()
1170 set_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag); in mtip_get_identify()
1172 clear_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag); in mtip_get_identify()
1213 mtip_set_timeout(port->dd, &fis, &timeout, 0); in mtip_standby_immediate()
1226 dev_warn(&port->dd->pdev->dev, in mtip_standby_immediate()
1325 dev_warn(&port->dd->pdev->dev, "IDENTIFY DATA not valid\n"); in mtip_get_smart_attr()
1329 dev_warn(&port->dd->pdev->dev, "SMART not supported\n"); in mtip_get_smart_attr()
1333 dev_warn(&port->dd->pdev->dev, "SMART not enabled\n"); in mtip_get_smart_attr()
1340 dev_warn(&port->dd->pdev->dev, "Failed to ge SMART data\n"); in mtip_get_smart_attr()
1352 dev_warn(&port->dd->pdev->dev, in mtip_get_smart_attr()
1370 static bool mtip_hw_get_capacity(struct driver_data *dd, sector_t *sectors) in mtip_hw_get_capacity() argument
1372 struct mtip_port *port = dd->port; in mtip_hw_get_capacity()
1401 dev_info(&port->dd->pdev->dev, in mtip_dump_identify()
1405 dev_info(&port->dd->pdev->dev, in mtip_dump_identify()
1409 dev_info(&port->dd->pdev->dev, "Model: %s\n", cbuf); in mtip_dump_identify()
1411 dev_info(&port->dd->pdev->dev, "Security: %04x %s\n", in mtip_dump_identify()
1415 if (mtip_hw_get_capacity(port->dd, §ors)) in mtip_dump_identify()
1416 dev_info(&port->dd->pdev->dev, in mtip_dump_identify()
1421 pci_read_config_word(port->dd->pdev, PCI_REVISION_ID, &revid); in mtip_dump_identify()
1433 dev_info(&port->dd->pdev->dev, in mtip_dump_identify()
1446 static inline void fill_command_sg(struct driver_data *dd, in fill_command_sg() argument
1460 dev_err(&dd->pdev->dev, in fill_command_sg()
1494 mtip_set_timeout(port->dd, &fis, &to, 0); in exec_drive_task()
1559 buf = dma_alloc_coherent(&port->dd->pdev->dev, in exec_drive_command()
1564 dev_err(&port->dd->pdev->dev, in exec_drive_command()
1584 mtip_set_timeout(port->dd, &fis, &to, 0); in exec_drive_command()
1636 dma_free_coherent(&port->dd->pdev->dev, in exec_drive_command()
1692 static int exec_drive_taskfile(struct driver_data *dd, in exec_drive_taskfile() argument
1726 outbuf_dma = dma_map_single(&dd->pdev->dev, outbuf, in exec_drive_taskfile()
1728 if (dma_mapping_error(&dd->pdev->dev, outbuf_dma)) { in exec_drive_taskfile()
1742 inbuf_dma = dma_map_single(&dd->pdev->dev, inbuf, in exec_drive_taskfile()
1744 if (dma_mapping_error(&dd->pdev->dev, inbuf_dma)) { in exec_drive_taskfile()
1755 reply = (dd->port->rxfis + RX_FIS_PIO_SETUP); in exec_drive_taskfile()
1758 reply = (dd->port->rxfis + RX_FIS_PIO_SETUP); in exec_drive_taskfile()
1761 reply = (dd->port->rxfis + RX_FIS_D2H_REG); in exec_drive_taskfile()
1803 dev_warn(&dd->pdev->dev, in exec_drive_taskfile()
1831 mtip_set_timeout(dd, &fis, &timeout, erasemode); in exec_drive_taskfile()
1840 if (mtip_exec_internal_command(dd->port, in exec_drive_taskfile()
1851 task_file_data = readl(dd->port->mmio+PORT_TFDATA); in exec_drive_taskfile()
1854 reply = dd->port->rxfis + RX_FIS_PIO_SETUP; in exec_drive_taskfile()
1857 reply = dd->port->rxfis + RX_FIS_D2H_REG; in exec_drive_taskfile()
1863 dma_unmap_single(&dd->pdev->dev, inbuf_dma, taskin, in exec_drive_taskfile()
1866 dma_unmap_single(&dd->pdev->dev, outbuf_dma, taskout, in exec_drive_taskfile()
1914 dma_unmap_single(&dd->pdev->dev, inbuf_dma, taskin, in exec_drive_taskfile()
1917 dma_unmap_single(&dd->pdev->dev, outbuf_dma, taskout, in exec_drive_taskfile()
1942 static int mtip_hw_ioctl(struct driver_data *dd, unsigned int cmd, in mtip_hw_ioctl() argument
1948 if (copy_to_user((void __user *)arg, dd->port->identify, in mtip_hw_ioctl()
1964 if (exec_drive_command(dd->port, in mtip_hw_ioctl()
1988 if (exec_drive_task(dd->port, drive_command)) in mtip_hw_ioctl()
2009 ret = exec_drive_taskfile(dd, (void __user *) arg, in mtip_hw_ioctl()
2045 static void mtip_hw_submit_io(struct driver_data *dd, struct request *rq, in mtip_hw_submit_io() argument
2050 dd->port->command_list + sizeof(struct mtip_cmd_hdr) * rq->tag; in mtip_hw_submit_io()
2052 struct mtip_port *port = dd->port; in mtip_hw_submit_io()
2060 nents = dma_map_sg(&dd->pdev->dev, command->sg, nents, dma_dir); in mtip_hw_submit_io()
2094 fill_command_sg(dd, command, nents); in mtip_hw_submit_io()
2101 if (test_bit(MTIP_PF_HOST_CAP_64, &dd->port->flags)) in mtip_hw_submit_io()
2136 struct driver_data *dd = dev_to_disk(dev)->private_data; in mtip_hw_show_status() local
2139 if (test_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag)) in mtip_hw_show_status()
2141 else if (test_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag)) in mtip_hw_show_status()
2168 struct driver_data *dd = (struct driver_data *)f->private_data; in mtip_hw_read_registers() local
2183 for (n = dd->slot_groups-1; n >= 0; n--) in mtip_hw_read_registers()
2185 readl(dd->port->s_active[n])); in mtip_hw_read_registers()
2190 for (n = dd->slot_groups-1; n >= 0; n--) in mtip_hw_read_registers()
2192 readl(dd->port->cmd_issue[n])); in mtip_hw_read_registers()
2197 for (n = dd->slot_groups-1; n >= 0; n--) in mtip_hw_read_registers()
2199 readl(dd->port->completed[n])); in mtip_hw_read_registers()
2203 readl(dd->port->mmio + PORT_IRQ_STAT)); in mtip_hw_read_registers()
2205 readl(dd->mmio + HOST_IRQ_STAT)); in mtip_hw_read_registers()
2210 for (n = dd->slot_groups-1; n >= 0; n--) { in mtip_hw_read_registers()
2213 dd->port->cmds_to_issue[n/2] >> (32*(n&1)); in mtip_hw_read_registers()
2215 group_allocated = dd->port->cmds_to_issue[n]; in mtip_hw_read_registers()
2232 struct driver_data *dd = (struct driver_data *)f->private_data; in mtip_hw_read_flags() local
2245 dd->port->flags); in mtip_hw_read_flags()
2247 dd->dd_flag); in mtip_hw_read_flags()
2270 static void mtip_hw_debugfs_init(struct driver_data *dd) in mtip_hw_debugfs_init() argument
2272 dd->dfs_node = debugfs_create_dir(dd->disk->disk_name, dfs_parent); in mtip_hw_debugfs_init()
2273 debugfs_create_file("flags", 0444, dd->dfs_node, dd, &mtip_flags_fops); in mtip_hw_debugfs_init()
2274 debugfs_create_file("registers", 0444, dd->dfs_node, dd, in mtip_hw_debugfs_init()
2278 static void mtip_hw_debugfs_exit(struct driver_data *dd) in mtip_hw_debugfs_exit() argument
2280 debugfs_remove_recursive(dd->dfs_node); in mtip_hw_debugfs_exit()
2291 static inline void hba_setup(struct driver_data *dd) in hba_setup() argument
2294 hwdata = readl(dd->mmio + HOST_HSORG); in hba_setup()
2300 dd->mmio + HOST_HSORG); in hba_setup()
2303 static int mtip_device_unaligned_constrained(struct driver_data *dd) in mtip_device_unaligned_constrained() argument
2305 return (dd->pdev->device == P420M_DEVICE_ID ? 1 : 0); in mtip_device_unaligned_constrained()
2318 static void mtip_detect_product(struct driver_data *dd) in mtip_detect_product() argument
2330 hwdata = readl(dd->mmio + HOST_HSORG); in mtip_detect_product()
2332 dd->product_type = MTIP_PRODUCT_UNKNOWN; in mtip_detect_product()
2333 dd->slot_groups = 1; in mtip_detect_product()
2336 dd->product_type = MTIP_PRODUCT_ASICFPGA; in mtip_detect_product()
2339 dev_info(&dd->pdev->dev, in mtip_detect_product()
2347 dev_warn(&dd->pdev->dev, in mtip_detect_product()
2352 dd->slot_groups = slotgroups; in mtip_detect_product()
2356 dev_warn(&dd->pdev->dev, "Unrecognized product id\n"); in mtip_detect_product()
2368 static int mtip_ftl_rebuild_poll(struct driver_data *dd) in mtip_ftl_rebuild_poll() argument
2372 dev_warn(&dd->pdev->dev, in mtip_ftl_rebuild_poll()
2380 &dd->dd_flag))) in mtip_ftl_rebuild_poll()
2382 if (mtip_check_surprise_removal(dd)) in mtip_ftl_rebuild_poll()
2385 if (mtip_get_identify(dd->port, NULL) < 0) in mtip_ftl_rebuild_poll()
2388 if (*(dd->port->identify + MTIP_FTL_REBUILD_OFFSET) == in mtip_ftl_rebuild_poll()
2393 dev_warn(&dd->pdev->dev, in mtip_ftl_rebuild_poll()
2399 dev_warn(&dd->pdev->dev, in mtip_ftl_rebuild_poll()
2402 mtip_block_initialize(dd); in mtip_ftl_rebuild_poll()
2408 dev_err(&dd->pdev->dev, in mtip_ftl_rebuild_poll()
2417 struct driver_data *dd = rq->q->queuedata; in mtip_softirq_done_fn() local
2420 dma_unmap_sg(&dd->pdev->dev, cmd->sg, cmd->scatter_ents, in mtip_softirq_done_fn()
2424 atomic_inc(&dd->port->cmd_slot_unal); in mtip_softirq_done_fn()
2432 struct driver_data *dd = data; in mtip_abort_cmd() local
2436 clear_bit(req->tag, dd->port->cmds_to_issue); in mtip_abort_cmd()
2444 struct driver_data *dd = data; in mtip_queue_cmd() local
2446 set_bit(req->tag, dd->port->cmds_to_issue); in mtip_queue_cmd()
2462 struct driver_data *dd = (struct driver_data *)data; in mtip_service_thread() local
2464 unsigned int num_cmd_slots = dd->slot_groups * 32; in mtip_service_thread()
2465 struct mtip_port *port = dd->port; in mtip_service_thread()
2485 &dd->dd_flag))) in mtip_service_thread()
2493 mtip_handle_tfe(dd); in mtip_service_thread()
2505 } while (atomic_read(&dd->irq_workers_active) != 0 && in mtip_service_thread()
2508 if (atomic_read(&dd->irq_workers_active) != 0) in mtip_service_thread()
2509 dev_warn(&dd->pdev->dev, in mtip_service_thread()
2512 blk_mq_quiesce_queue(dd->queue); in mtip_service_thread()
2514 blk_mq_tagset_busy_iter(&dd->tags, mtip_queue_cmd, dd); in mtip_service_thread()
2516 set_bit(MTIP_PF_ISSUE_CMDS_BIT, &dd->port->flags); in mtip_service_thread()
2518 if (mtip_device_reset(dd)) in mtip_service_thread()
2519 blk_mq_tagset_busy_iter(&dd->tags, in mtip_service_thread()
2520 mtip_abort_cmd, dd); in mtip_service_thread()
2522 clear_bit(MTIP_PF_TO_ACTIVE_BIT, &dd->port->flags); in mtip_service_thread()
2524 blk_mq_unquiesce_queue(dd->queue); in mtip_service_thread()
2559 if (mtip_ftl_rebuild_poll(dd) == 0) in mtip_service_thread()
2576 static void mtip_dma_free(struct driver_data *dd) in mtip_dma_free() argument
2578 struct mtip_port *port = dd->port; in mtip_dma_free()
2581 dma_free_coherent(&dd->pdev->dev, BLOCK_DMA_ALLOC_SZ, in mtip_dma_free()
2585 dma_free_coherent(&dd->pdev->dev, AHCI_CMD_TBL_SZ, in mtip_dma_free()
2598 static int mtip_dma_alloc(struct driver_data *dd) in mtip_dma_alloc() argument
2600 struct mtip_port *port = dd->port; in mtip_dma_alloc()
2604 dma_alloc_coherent(&dd->pdev->dev, BLOCK_DMA_ALLOC_SZ, in mtip_dma_alloc()
2611 dma_alloc_coherent(&dd->pdev->dev, AHCI_CMD_TBL_SZ, in mtip_dma_alloc()
2614 dma_free_coherent(&dd->pdev->dev, BLOCK_DMA_ALLOC_SZ, in mtip_dma_alloc()
2634 static int mtip_hw_get_identify(struct driver_data *dd) in mtip_hw_get_identify() argument
2640 if (mtip_get_identify(dd->port, NULL) < 0) in mtip_hw_get_identify()
2643 if (*(dd->port->identify + MTIP_FTL_REBUILD_OFFSET) == in mtip_hw_get_identify()
2645 set_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags); in mtip_hw_get_identify()
2648 mtip_dump_identify(dd->port); in mtip_hw_get_identify()
2651 rv = mtip_read_log_page(dd->port, ATA_LOG_SATA_NCQ, in mtip_hw_get_identify()
2652 dd->port->log_buf, in mtip_hw_get_identify()
2653 dd->port->log_buf_dma, 1); in mtip_hw_get_identify()
2655 dev_warn(&dd->pdev->dev, in mtip_hw_get_identify()
2659 buf = (unsigned char *)dd->port->log_buf; in mtip_hw_get_identify()
2661 dev_info(&dd->pdev->dev, in mtip_hw_get_identify()
2663 set_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag); in mtip_hw_get_identify()
2666 dev_info(&dd->pdev->dev, in mtip_hw_get_identify()
2668 set_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag); in mtip_hw_get_identify()
2671 dev_info(&dd->pdev->dev, in mtip_hw_get_identify()
2673 set_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag); in mtip_hw_get_identify()
2679 if (mtip_get_smart_attr(dd->port, 242, &attr242)) in mtip_hw_get_identify()
2680 dev_warn(&dd->pdev->dev, in mtip_hw_get_identify()
2683 dev_info(&dd->pdev->dev, in mtip_hw_get_identify()
2698 static int mtip_hw_init(struct driver_data *dd) in mtip_hw_init() argument
2704 dd->mmio = pcim_iomap_region(dd->pdev, MTIP_ABAR, MTIP_DRV_NAME); in mtip_hw_init()
2705 if (IS_ERR(dd->mmio)) { in mtip_hw_init()
2706 dev_err(&dd->pdev->dev, "Unable to request / ioremap PCI region\n"); in mtip_hw_init()
2707 return PTR_ERR(dd->mmio); in mtip_hw_init()
2711 mtip_detect_product(dd); in mtip_hw_init()
2712 if (dd->product_type == MTIP_PRODUCT_UNKNOWN) { in mtip_hw_init()
2717 hba_setup(dd); in mtip_hw_init()
2719 dd->port = kzalloc_node(sizeof(struct mtip_port), GFP_KERNEL, in mtip_hw_init()
2720 dd->numa_node); in mtip_hw_init()
2721 if (!dd->port) in mtip_hw_init()
2726 dd->work[i].port = dd->port; in mtip_hw_init()
2729 if (mtip_device_unaligned_constrained(dd)) in mtip_hw_init()
2730 dd->unal_qdepth = MTIP_MAX_UNALIGNED_SLOTS; in mtip_hw_init()
2732 dd->unal_qdepth = 0; in mtip_hw_init()
2734 atomic_set(&dd->port->cmd_slot_unal, dd->unal_qdepth); in mtip_hw_init()
2738 spin_lock_init(&dd->port->cmd_issue_lock[i]); in mtip_hw_init()
2741 dd->port->mmio = dd->mmio + PORT_OFFSET; in mtip_hw_init()
2742 dd->port->dd = dd; in mtip_hw_init()
2745 rv = mtip_dma_alloc(dd); in mtip_hw_init()
2750 for (i = 0; i < dd->slot_groups; i++) { in mtip_hw_init()
2751 dd->port->s_active[i] = in mtip_hw_init()
2752 dd->port->mmio + i*0x80 + PORT_SCR_ACT; in mtip_hw_init()
2753 dd->port->cmd_issue[i] = in mtip_hw_init()
2754 dd->port->mmio + i*0x80 + PORT_COMMAND_ISSUE; in mtip_hw_init()
2755 dd->port->completed[i] = in mtip_hw_init()
2756 dd->port->mmio + i*0x80 + PORT_SDBV; in mtip_hw_init()
2761 while (((readl(dd->port->mmio + PORT_SCR_STAT) & 0x0F) != 0x03) && in mtip_hw_init()
2765 if (unlikely(mtip_check_surprise_removal(dd))) { in mtip_hw_init()
2767 dev_warn(&dd->pdev->dev, in mtip_hw_init()
2773 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) { in mtip_hw_init()
2775 dev_warn(&dd->pdev->dev, in mtip_hw_init()
2783 if (!(readl(dd->mmio + HOST_CAP) & HOST_CAP_NZDMA)) { in mtip_hw_init()
2784 if (mtip_hba_reset(dd) < 0) { in mtip_hw_init()
2785 dev_err(&dd->pdev->dev, in mtip_hw_init()
2792 writel(readl(dd->mmio + HOST_IRQ_STAT), in mtip_hw_init()
2793 dd->mmio + HOST_IRQ_STAT); in mtip_hw_init()
2796 mtip_init_port(dd->port); in mtip_hw_init()
2797 mtip_start_port(dd->port); in mtip_hw_init()
2800 rv = request_irq(dd->pdev->irq, mtip_irq_handler, IRQF_SHARED, in mtip_hw_init()
2801 dev_driver_string(&dd->pdev->dev), dd); in mtip_hw_init()
2803 dev_err(&dd->pdev->dev, in mtip_hw_init()
2804 "Unable to allocate IRQ %d\n", dd->pdev->irq); in mtip_hw_init()
2807 irq_set_affinity_hint(dd->pdev->irq, get_cpu_mask(dd->isr_binding)); in mtip_hw_init()
2810 writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN, in mtip_hw_init()
2811 dd->mmio + HOST_CTL); in mtip_hw_init()
2813 init_waitqueue_head(&dd->port->svc_wait); in mtip_hw_init()
2815 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)) { in mtip_hw_init()
2824 writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN, in mtip_hw_init()
2825 dd->mmio + HOST_CTL); in mtip_hw_init()
2828 irq_set_affinity_hint(dd->pdev->irq, NULL); in mtip_hw_init()
2829 free_irq(dd->pdev->irq, dd); in mtip_hw_init()
2832 mtip_deinit_port(dd->port); in mtip_hw_init()
2833 mtip_dma_free(dd); in mtip_hw_init()
2837 kfree(dd->port); in mtip_hw_init()
2842 static int mtip_standby_drive(struct driver_data *dd) in mtip_standby_drive() argument
2846 if (dd->sr || !dd->port) in mtip_standby_drive()
2852 if (!test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags) && in mtip_standby_drive()
2853 !test_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag) && in mtip_standby_drive()
2854 !test_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag)) { in mtip_standby_drive()
2855 rv = mtip_standby_immediate(dd->port); in mtip_standby_drive()
2857 dev_warn(&dd->pdev->dev, in mtip_standby_drive()
2871 static int mtip_hw_exit(struct driver_data *dd) in mtip_hw_exit() argument
2873 if (!dd->sr) { in mtip_hw_exit()
2875 mtip_deinit_port(dd->port); in mtip_hw_exit()
2878 writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN, in mtip_hw_exit()
2879 dd->mmio + HOST_CTL); in mtip_hw_exit()
2883 irq_set_affinity_hint(dd->pdev->irq, NULL); in mtip_hw_exit()
2884 free_irq(dd->pdev->irq, dd); in mtip_hw_exit()
2888 mtip_dma_free(dd); in mtip_hw_exit()
2891 kfree(dd->port); in mtip_hw_exit()
2892 dd->port = NULL; in mtip_hw_exit()
2908 static int mtip_hw_shutdown(struct driver_data *dd) in mtip_hw_shutdown() argument
2914 mtip_standby_drive(dd); in mtip_hw_shutdown()
2931 static int mtip_hw_suspend(struct driver_data *dd) in mtip_hw_suspend() argument
2937 if (mtip_standby_drive(dd) != 0) { in mtip_hw_suspend()
2938 dev_err(&dd->pdev->dev, in mtip_hw_suspend()
2944 writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN, in mtip_hw_suspend()
2945 dd->mmio + HOST_CTL); in mtip_hw_suspend()
2946 mtip_deinit_port(dd->port); in mtip_hw_suspend()
2963 static int mtip_hw_resume(struct driver_data *dd) in mtip_hw_resume() argument
2966 hba_setup(dd); in mtip_hw_resume()
2969 if (mtip_hba_reset(dd) != 0) { in mtip_hw_resume()
2970 dev_err(&dd->pdev->dev, in mtip_hw_resume()
2979 mtip_init_port(dd->port); in mtip_hw_resume()
2980 mtip_start_port(dd->port); in mtip_hw_resume()
2983 writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN, in mtip_hw_resume()
2984 dd->mmio + HOST_CTL); in mtip_hw_resume()
3038 struct driver_data *dd = dev->bd_disk->private_data; in mtip_block_ioctl() local
3043 if (!dd) in mtip_block_ioctl()
3046 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) in mtip_block_ioctl()
3053 return mtip_hw_ioctl(dd, cmd, arg); in mtip_block_ioctl()
3076 struct driver_data *dd = dev->bd_disk->private_data; in mtip_block_compat_ioctl() local
3081 if (!dd) in mtip_block_compat_ioctl()
3084 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) in mtip_block_compat_ioctl()
3113 ret = exec_drive_taskfile(dd, (void __user *) arg, in mtip_block_compat_ioctl()
3130 return mtip_hw_ioctl(dd, cmd, arg); in mtip_block_compat_ioctl()
3156 struct driver_data *dd = dev->bd_disk->private_data; in mtip_block_getgeo() local
3159 if (!dd) in mtip_block_getgeo()
3162 if (!(mtip_hw_get_capacity(dd, &capacity))) { in mtip_block_getgeo()
3163 dev_warn(&dd->pdev->dev, in mtip_block_getgeo()
3177 struct driver_data *dd = disk->private_data; in mtip_block_free_disk() local
3179 ida_free(&rssd_index_ida, dd->index); in mtip_block_free_disk()
3180 kfree(dd); in mtip_block_free_disk()
3199 static inline bool is_se_active(struct driver_data *dd) in is_se_active() argument
3201 if (unlikely(test_bit(MTIP_PF_SE_ACTIVE_BIT, &dd->port->flags))) { in is_se_active()
3202 if (dd->port->ic_pause_timer) { in is_se_active()
3203 unsigned long to = dd->port->ic_pause_timer + in is_se_active()
3207 &dd->port->flags); in is_se_active()
3208 clear_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag); in is_se_active()
3209 dd->port->ic_pause_timer = 0; in is_se_active()
3210 wake_up_interruptible(&dd->port->svc_wait); in is_se_active()
3219 static inline bool is_stopped(struct driver_data *dd, struct request *rq) in is_stopped() argument
3221 if (likely(!(dd->dd_flag & MTIP_DDF_STOP_IO))) in is_stopped()
3224 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)) in is_stopped()
3226 if (test_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag)) in is_stopped()
3228 if (test_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag) && in is_stopped()
3231 if (test_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag)) in is_stopped()
3233 if (test_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag)) in is_stopped()
3242 struct driver_data *dd = hctx->queue->queuedata; in mtip_check_unal_depth() local
3245 if (rq_data_dir(rq) == READ || !dd->unal_qdepth) in mtip_check_unal_depth()
3257 if (cmd->unaligned && atomic_dec_if_positive(&dd->port->cmd_slot_unal) >= 0) in mtip_check_unal_depth()
3266 struct driver_data *dd = hctx->queue->queuedata; in mtip_issue_reserved_cmd() local
3270 dd->port->command_list + sizeof(struct mtip_cmd_hdr) * rq->tag; in mtip_issue_reserved_cmd()
3273 if (mtip_commands_active(dd->port)) in mtip_issue_reserved_cmd()
3277 if (test_bit(MTIP_PF_HOST_CAP_64, &dd->port->flags)) in mtip_issue_reserved_cmd()
3296 mtip_issue_non_ncq_command(dd->port, rq->tag); in mtip_issue_reserved_cmd()
3303 struct driver_data *dd = hctx->queue->queuedata; in mtip_queue_rq() local
3313 if (is_se_active(dd) || is_stopped(dd, rq)) in mtip_queue_rq()
3318 mtip_hw_submit_io(dd, rq, cmd, hctx); in mtip_queue_rq()
3325 struct driver_data *dd = set->driver_data; in mtip_free_cmd() local
3331 dma_free_coherent(&dd->pdev->dev, CMD_DMA_ALLOC_SZ, cmd->command, in mtip_free_cmd()
3338 struct driver_data *dd = set->driver_data; in mtip_init_cmd() local
3341 cmd->command = dma_alloc_coherent(&dd->pdev->dev, CMD_DMA_ALLOC_SZ, in mtip_init_cmd()
3352 struct driver_data *dd = req->q->queuedata; in mtip_cmd_timeout() local
3362 if (test_bit(req->tag, dd->port->cmds_to_issue)) in mtip_cmd_timeout()
3365 if (test_and_set_bit(MTIP_PF_TO_ACTIVE_BIT, &dd->port->flags)) in mtip_cmd_timeout()
3368 wake_up_interruptible(&dd->port->svc_wait); in mtip_cmd_timeout()
3392 static int mtip_block_initialize(struct driver_data *dd) in mtip_block_initialize() argument
3404 if (dd->disk) in mtip_block_initialize()
3407 if (mtip_hw_init(dd)) { in mtip_block_initialize()
3412 memset(&dd->tags, 0, sizeof(dd->tags)); in mtip_block_initialize()
3413 dd->tags.ops = &mtip_mq_ops; in mtip_block_initialize()
3414 dd->tags.nr_hw_queues = 1; in mtip_block_initialize()
3415 dd->tags.queue_depth = MTIP_MAX_COMMAND_SLOTS; in mtip_block_initialize()
3416 dd->tags.reserved_tags = 1; in mtip_block_initialize()
3417 dd->tags.cmd_size = sizeof(struct mtip_cmd); in mtip_block_initialize()
3418 dd->tags.numa_node = dd->numa_node; in mtip_block_initialize()
3419 dd->tags.driver_data = dd; in mtip_block_initialize()
3420 dd->tags.timeout = MTIP_NCQ_CMD_TIMEOUT_MS; in mtip_block_initialize()
3422 rv = blk_mq_alloc_tag_set(&dd->tags); in mtip_block_initialize()
3424 dev_err(&dd->pdev->dev, in mtip_block_initialize()
3429 dd->disk = blk_mq_alloc_disk(&dd->tags, &lim, dd); in mtip_block_initialize()
3430 if (IS_ERR(dd->disk)) { in mtip_block_initialize()
3431 dev_err(&dd->pdev->dev, in mtip_block_initialize()
3436 dd->queue = dd->disk->queue; in mtip_block_initialize()
3445 dd->disk->disk_name, in mtip_block_initialize()
3450 dd->disk->major = dd->major; in mtip_block_initialize()
3451 dd->disk->first_minor = index * MTIP_MAX_MINORS; in mtip_block_initialize()
3452 dd->disk->minors = MTIP_MAX_MINORS; in mtip_block_initialize()
3453 dd->disk->fops = &mtip_block_ops; in mtip_block_initialize()
3454 dd->disk->private_data = dd; in mtip_block_initialize()
3455 dd->index = index; in mtip_block_initialize()
3457 mtip_hw_debugfs_init(dd); in mtip_block_initialize()
3461 wait_for_rebuild = mtip_hw_get_identify(dd); in mtip_block_initialize()
3463 dev_err(&dd->pdev->dev, in mtip_block_initialize()
3477 dma_set_max_seg_size(&dd->pdev->dev, 0x400000); in mtip_block_initialize()
3480 if (!(mtip_hw_get_capacity(dd, &capacity))) { in mtip_block_initialize()
3481 dev_warn(&dd->pdev->dev, in mtip_block_initialize()
3486 set_capacity(dd->disk, capacity); in mtip_block_initialize()
3489 rv = device_add_disk(&dd->pdev->dev, dd->disk, mtip_disk_attr_groups); in mtip_block_initialize()
3493 if (dd->mtip_svc_handler) { in mtip_block_initialize()
3494 set_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag); in mtip_block_initialize()
3499 dd->mtip_svc_handler = kthread_create_on_node(mtip_service_thread, in mtip_block_initialize()
3500 dd, dd->numa_node, in mtip_block_initialize()
3503 if (IS_ERR(dd->mtip_svc_handler)) { in mtip_block_initialize()
3504 dev_err(&dd->pdev->dev, "service thread failed to start\n"); in mtip_block_initialize()
3505 dd->mtip_svc_handler = NULL; in mtip_block_initialize()
3509 wake_up_process(dd->mtip_svc_handler); in mtip_block_initialize()
3517 del_gendisk(dd->disk); in mtip_block_initialize()
3520 mtip_hw_debugfs_exit(dd); in mtip_block_initialize()
3524 put_disk(dd->disk); in mtip_block_initialize()
3526 blk_mq_free_tag_set(&dd->tags); in mtip_block_initialize()
3528 mtip_hw_exit(dd); /* De-initialize the protocol layer. */ in mtip_block_initialize()
3545 static int mtip_block_shutdown(struct driver_data *dd) in mtip_block_shutdown() argument
3547 mtip_hw_shutdown(dd); in mtip_block_shutdown()
3549 dev_info(&dd->pdev->dev, in mtip_block_shutdown()
3550 "Shutting down %s ...\n", dd->disk->disk_name); in mtip_block_shutdown()
3552 if (test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag)) in mtip_block_shutdown()
3553 del_gendisk(dd->disk); in mtip_block_shutdown()
3555 blk_mq_free_tag_set(&dd->tags); in mtip_block_shutdown()
3556 put_disk(dd->disk); in mtip_block_shutdown()
3560 static int mtip_block_suspend(struct driver_data *dd) in mtip_block_suspend() argument
3562 dev_info(&dd->pdev->dev, in mtip_block_suspend()
3563 "Suspending %s ...\n", dd->disk->disk_name); in mtip_block_suspend()
3564 mtip_hw_suspend(dd); in mtip_block_suspend()
3568 static int mtip_block_resume(struct driver_data *dd) in mtip_block_resume() argument
3570 dev_info(&dd->pdev->dev, "Resuming %s ...\n", in mtip_block_resume()
3571 dd->disk->disk_name); in mtip_block_resume()
3572 mtip_hw_resume(dd); in mtip_block_resume()
3626 static void mtip_disable_link_opts(struct driver_data *dd, struct pci_dev *pdev) in mtip_disable_link_opts() argument
3634 dev_info(&dd->pdev->dev, in mtip_disable_link_opts()
3645 static void mtip_fix_ero_nosnoop(struct driver_data *dd, struct pci_dev *pdev) in mtip_fix_ero_nosnoop() argument
3654 mtip_disable_link_opts(dd, pdev->bus->self); in mtip_fix_ero_nosnoop()
3665 mtip_disable_link_opts(dd, in mtip_fix_ero_nosnoop()
3685 struct driver_data *dd = NULL; in mtip_pci_probe() local
3704 dd = kzalloc_node(sizeof(struct driver_data), GFP_KERNEL, my_node); in mtip_pci_probe()
3705 if (!dd) in mtip_pci_probe()
3709 pci_set_drvdata(pdev, dd); in mtip_pci_probe()
3724 dd->major = mtip_major; in mtip_pci_probe()
3725 dd->instance = instance; in mtip_pci_probe()
3726 dd->pdev = pdev; in mtip_pci_probe()
3727 dd->numa_node = my_node; in mtip_pci_probe()
3729 memset(dd->workq_name, 0, 32); in mtip_pci_probe()
3730 snprintf(dd->workq_name, 31, "mtipq%d", dd->instance); in mtip_pci_probe()
3732 dd->isr_workq = create_workqueue(dd->workq_name); in mtip_pci_probe()
3733 if (!dd->isr_workq) { in mtip_pci_probe()
3734 dev_warn(&pdev->dev, "Can't create wq %d\n", dd->instance); in mtip_pci_probe()
3741 node_mask = cpumask_of_node(dd->numa_node); in mtip_pci_probe()
3750 dd->numa_node, in mtip_pci_probe()
3752 nr_cpus_node(dd->numa_node), in mtip_pci_probe()
3757 dd->isr_binding = get_least_used_cpu_on_node(dd->numa_node); in mtip_pci_probe()
3759 cpu_to_node(dd->isr_binding), dd->isr_binding); in mtip_pci_probe()
3762 dd->work[0].cpu_binding = dd->isr_binding; in mtip_pci_probe()
3763 dd->work[1].cpu_binding = get_least_used_cpu_on_node(dd->numa_node); in mtip_pci_probe()
3764 dd->work[2].cpu_binding = get_least_used_cpu_on_node(dd->numa_node); in mtip_pci_probe()
3765 dd->work[3].cpu_binding = dd->work[0].cpu_binding; in mtip_pci_probe()
3766 dd->work[4].cpu_binding = dd->work[1].cpu_binding; in mtip_pci_probe()
3767 dd->work[5].cpu_binding = dd->work[2].cpu_binding; in mtip_pci_probe()
3768 dd->work[6].cpu_binding = dd->work[2].cpu_binding; in mtip_pci_probe()
3769 dd->work[7].cpu_binding = dd->work[1].cpu_binding; in mtip_pci_probe()
3775 if (dd->work[i].cpu_binding == cpu) { in mtip_pci_probe()
3784 INIT_WORK(&dd->work[0].work, mtip_workq_sdbf0); in mtip_pci_probe()
3785 INIT_WORK(&dd->work[1].work, mtip_workq_sdbf1); in mtip_pci_probe()
3786 INIT_WORK(&dd->work[2].work, mtip_workq_sdbf2); in mtip_pci_probe()
3787 INIT_WORK(&dd->work[3].work, mtip_workq_sdbf3); in mtip_pci_probe()
3788 INIT_WORK(&dd->work[4].work, mtip_workq_sdbf4); in mtip_pci_probe()
3789 INIT_WORK(&dd->work[5].work, mtip_workq_sdbf5); in mtip_pci_probe()
3790 INIT_WORK(&dd->work[6].work, mtip_workq_sdbf6); in mtip_pci_probe()
3791 INIT_WORK(&dd->work[7].work, mtip_workq_sdbf7); in mtip_pci_probe()
3801 mtip_fix_ero_nosnoop(dd, pdev); in mtip_pci_probe()
3804 rv = mtip_block_initialize(dd); in mtip_pci_probe()
3817 set_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag); in mtip_pci_probe()
3827 if (dd->isr_workq) { in mtip_pci_probe()
3828 destroy_workqueue(dd->isr_workq); in mtip_pci_probe()
3829 drop_cpu(dd->work[0].cpu_binding); in mtip_pci_probe()
3830 drop_cpu(dd->work[1].cpu_binding); in mtip_pci_probe()
3831 drop_cpu(dd->work[2].cpu_binding); in mtip_pci_probe()
3837 kfree(dd); in mtip_pci_probe()
3853 struct driver_data *dd = pci_get_drvdata(pdev); in mtip_pci_remove() local
3856 mtip_check_surprise_removal(dd); in mtip_pci_remove()
3857 synchronize_irq(dd->pdev->irq); in mtip_pci_remove()
3863 } while (atomic_read(&dd->irq_workers_active) != 0 && in mtip_pci_remove()
3866 if (atomic_read(&dd->irq_workers_active) != 0) { in mtip_pci_remove()
3867 dev_warn(&dd->pdev->dev, in mtip_pci_remove()
3871 set_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag); in mtip_pci_remove()
3873 if (test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag)) in mtip_pci_remove()
3874 del_gendisk(dd->disk); in mtip_pci_remove()
3876 mtip_hw_debugfs_exit(dd); in mtip_pci_remove()
3878 if (dd->mtip_svc_handler) { in mtip_pci_remove()
3879 set_bit(MTIP_PF_SVC_THD_STOP_BIT, &dd->port->flags); in mtip_pci_remove()
3880 wake_up_interruptible(&dd->port->svc_wait); in mtip_pci_remove()
3881 kthread_stop(dd->mtip_svc_handler); in mtip_pci_remove()
3884 if (!dd->sr) { in mtip_pci_remove()
3889 if (!mtip_quiesce_io(dd->port, MTIP_QUIESCE_IO_TIMEOUT_MS)) in mtip_pci_remove()
3890 mtip_standby_drive(dd); in mtip_pci_remove()
3893 dev_info(&dd->pdev->dev, "device %s surprise removal\n", in mtip_pci_remove()
3894 dd->disk->disk_name); in mtip_pci_remove()
3896 blk_mq_free_tag_set(&dd->tags); in mtip_pci_remove()
3899 mtip_hw_exit(dd); in mtip_pci_remove()
3901 if (dd->isr_workq) { in mtip_pci_remove()
3902 destroy_workqueue(dd->isr_workq); in mtip_pci_remove()
3903 drop_cpu(dd->work[0].cpu_binding); in mtip_pci_remove()
3904 drop_cpu(dd->work[1].cpu_binding); in mtip_pci_remove()
3905 drop_cpu(dd->work[2].cpu_binding); in mtip_pci_remove()
3913 put_disk(dd->disk); in mtip_pci_remove()
3926 struct driver_data *dd = dev_get_drvdata(dev); in mtip_pci_suspend() local
3928 set_bit(MTIP_DDF_RESUME_BIT, &dd->dd_flag); in mtip_pci_suspend()
3931 rv = mtip_block_suspend(dd); in mtip_pci_suspend()
3948 struct driver_data *dd = dev_get_drvdata(dev); in mtip_pci_resume() local
3954 rv = mtip_block_resume(dd); in mtip_pci_resume()
3958 clear_bit(MTIP_DDF_RESUME_BIT, &dd->dd_flag); in mtip_pci_resume()
3971 struct driver_data *dd = pci_get_drvdata(pdev); in mtip_pci_shutdown() local
3972 if (dd) in mtip_pci_shutdown()
3973 mtip_block_shutdown(dd); in mtip_pci_shutdown()