1b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 2b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * NVM Express device driver 36eb0d698efa9c2a35ec3ca958699717c603f85eeMatthew Wilcox * Copyright (c) 2011-2014, Intel Corporation. 4b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 5b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * This program is free software; you can redistribute it and/or modify it 6b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * under the terms and conditions of the GNU General Public License, 7b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * version 2, as published by the Free Software Foundation. 8b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 9b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * This program is distributed in the hope it will be useful, but WITHOUT 10b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * more details. 13b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 14b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 15b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/nvme.h> 16b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/bio.h> 178de055350fbaa96b6563892c195a60be583faa9cMatthew Wilcox#include <linux/bitops.h> 18b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/blkdev.h> 1942f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch#include <linux/cpu.h> 20fd63e9ceeeae58cfe877c2d49d41c1bf7532303cMatthew Wilcox#include <linux/delay.h> 21b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/errno.h> 22b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/fs.h> 23b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/genhd.h> 244cc09e2dc4cbe6009c935b6f12a8376f09124bc5Keith Busch#include <linux/hdreg.h> 255aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox#include <linux/idr.h> 26b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/init.h> 27b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/interrupt.h> 28b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/io.h> 29b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/kdev_t.h> 301fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox#include <linux/kthread.h> 31b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/kernel.h> 32b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/mm.h> 33b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/module.h> 34b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/moduleparam.h> 35b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/pci.h> 3642f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch#include <linux/percpu.h> 37be7b62754e097adc0cb16c25c9ee86ee20de62fbMatthew Wilcox#include <linux/poison.h> 38c3bfe7176c035a0a2c70bc79180fb13a6c57142aMatthew Wilcox#include <linux/ptrace.h> 39b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/sched.h> 40b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/slab.h> 41b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/types.h> 425d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma#include <scsi/sg.h> 43797a796a13df6b84a4791e57306737059b5b2384Hitoshi Mitake#include <asm-generic/io-64-nonatomic-lo-hi.h> 44797a796a13df6b84a4791e57306737059b5b2384Hitoshi Mitake 453291fa57cb1b004c1a4823beb28b5cc72555f1a5Keith Busch#include <trace/events/block.h> 463291fa57cb1b004c1a4823beb28b5cc72555f1a5Keith Busch 479d43cf646eab948db81913379dacb1dcecacb1ebKeith Busch#define NVME_Q_DEPTH 1024 48b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define SQ_SIZE(depth) (depth * sizeof(struct nvme_command)) 49b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define CQ_SIZE(depth) (depth * sizeof(struct nvme_completion)) 509d43cf646eab948db81913379dacb1dcecacb1ebKeith Busch#define ADMIN_TIMEOUT (admin_timeout * HZ) 519d43cf646eab948db81913379dacb1dcecacb1ebKeith Busch#define IOD_TIMEOUT (retry_time * HZ) 529d43cf646eab948db81913379dacb1dcecacb1ebKeith Busch 539d43cf646eab948db81913379dacb1dcecacb1ebKeith Buschstatic unsigned char admin_timeout = 60; 549d43cf646eab948db81913379dacb1dcecacb1ebKeith Buschmodule_param(admin_timeout, byte, 0644); 559d43cf646eab948db81913379dacb1dcecacb1ebKeith BuschMODULE_PARM_DESC(admin_timeout, "timeout in seconds for admin commands"); 56b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 57bd67608a6127c994e897c49cc4f72d9095925301Matthew Wilcoxunsigned char nvme_io_timeout = 30; 58bd67608a6127c994e897c49cc4f72d9095925301Matthew Wilcoxmodule_param_named(io_timeout, nvme_io_timeout, byte, 0644); 59b355084a891985d4cd0ca23b1a83366af2c4232dKeith BuschMODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O"); 60b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 6161e4ce086df0a64a555880089e3b782517c828c0Keith Buschstatic unsigned char retry_time = 30; 6261e4ce086df0a64a555880089e3b782517c828c0Keith Buschmodule_param(retry_time, byte, 0644); 6361e4ce086df0a64a555880089e3b782517c828c0Keith BuschMODULE_PARM_DESC(retry_time, "time in seconds to retry failed I/O"); 6461e4ce086df0a64a555880089e3b782517c828c0Keith Busch 65b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_major; 66b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxmodule_param(nvme_major, int, 0); 67b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 6858ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcoxstatic int use_threaded_interrupts; 6958ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcoxmodule_param(use_threaded_interrupts, int, 0); 7058ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox 711fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcoxstatic DEFINE_SPINLOCK(dev_list_lock); 721fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcoxstatic LIST_HEAD(dev_list); 731fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcoxstatic struct task_struct *nvme_thread; 749a6b94584de1a0467d85b435df9c744c5c45a270Keith Buschstatic struct workqueue_struct *nvme_workq; 75b9afca3efb18a9b8392cb544a3e29e8b1168400cDan McLeranstatic wait_queue_head_t nvme_kthread_wait; 76f3db22feb5de6b98b7bae924c2d4b6c8d65bedaeKeith Buschstatic struct notifier_block nvme_nb; 771fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 78d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Buschstatic void nvme_reset_failed_dev(struct work_struct *ws); 79d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch 804d115420707afcabe77d2535e092356df6664b70Keith Buschstruct async_cmd_info { 814d115420707afcabe77d2535e092356df6664b70Keith Busch struct kthread_work work; 824d115420707afcabe77d2535e092356df6664b70Keith Busch struct kthread_worker *worker; 834d115420707afcabe77d2535e092356df6664b70Keith Busch u32 result; 844d115420707afcabe77d2535e092356df6664b70Keith Busch int status; 854d115420707afcabe77d2535e092356df6664b70Keith Busch void *ctx; 864d115420707afcabe77d2535e092356df6664b70Keith Busch}; 871fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 88b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 89b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * An NVM Express queue. Each device has at least two (one for admin 90b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * commands and one for I/O commands). 91b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 92b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstruct nvme_queue { 935a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch struct rcu_head r_head; 94b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct device *q_dmadev; 95091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox struct nvme_dev *dev; 963193f07bb7ae723f33ac8e8b9db317a4f68d7d18Matthew Wilcox char irqname[24]; /* nvme4294967295-65535\0 */ 97b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox spinlock_t q_lock; 98b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_command *sq_cmds; 99b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox volatile struct nvme_completion *cqes; 100b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_addr_t sq_dma_addr; 101b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_addr_t cq_dma_addr; 102b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox wait_queue_head_t sq_full; 1031fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox wait_queue_t sq_cong_wait; 104b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct bio_list sq_cong; 105edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch struct list_head iod_bio; 106b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 __iomem *q_db; 107b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 q_depth; 108b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 cq_vector; 109b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 sq_head; 110b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 sq_tail; 111b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 cq_head; 112c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch u16 qid; 113e9539f47525ecee05c9f22c3565885f3e9492c52Matthew Wilcox u8 cq_phase; 114e9539f47525ecee05c9f22c3565885f3e9492c52Matthew Wilcox u8 cqe_seen; 115224042742582c9938788b81165180c876e997a07Keith Busch u8 q_suspended; 11642f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch cpumask_var_t cpu_mask; 1174d115420707afcabe77d2535e092356df6664b70Keith Busch struct async_cmd_info cmdinfo; 118b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unsigned long cmdid_data[]; 119b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 120b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 121b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 122b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Check we didin't inadvertently grow the command struct 123b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 124b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic inline void _nvme_check_size(void) 125b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 126b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64); 127b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_create_cq) != 64); 128b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_create_sq) != 64); 129b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64); 130b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_features) != 64); 131f8ebf8409abfdaeeb8c847381629a2a8b8e3d816Vishal Verma BUILD_BUG_ON(sizeof(struct nvme_format_cmd) != 64); 132c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch BUILD_BUG_ON(sizeof(struct nvme_abort_cmd) != 64); 133b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_command) != 64); 134b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != 4096); 135b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_id_ns) != 4096); 136b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64); 1376ecec74520d8a357726e6c12f99080dbe7b347ddKeith Busch BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512); 138b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 139b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 140edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Buschtypedef void (*nvme_completion_fn)(struct nvme_queue *, void *, 141c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox struct nvme_completion *); 142c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox 143e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcoxstruct nvme_cmd_info { 144c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox nvme_completion_fn fn; 145c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox void *ctx; 146e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox unsigned long timeout; 147c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch int aborted; 148e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox}; 149e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox 150e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcoxstatic struct nvme_cmd_info *nvme_cmd_info(struct nvme_queue *nvmeq) 151e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox{ 152e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox return (void *)&nvmeq->cmdid_data[BITS_TO_LONGS(nvmeq->q_depth)]; 153e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox} 154e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox 155224042742582c9938788b81165180c876e997a07Keith Buschstatic unsigned nvme_queue_extra(int depth) 156224042742582c9938788b81165180c876e997a07Keith Busch{ 157224042742582c9938788b81165180c876e997a07Keith Busch return DIV_ROUND_UP(depth, 8) + (depth * sizeof(struct nvme_cmd_info)); 158224042742582c9938788b81165180c876e997a07Keith Busch} 159224042742582c9938788b81165180c876e997a07Keith Busch 160b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/** 161714a7a22884b74862540bc84955274d86b2f6040Matthew Wilcox * alloc_cmdid() - Allocate a Command ID 162714a7a22884b74862540bc84955274d86b2f6040Matthew Wilcox * @nvmeq: The queue that will be used for this command 163714a7a22884b74862540bc84955274d86b2f6040Matthew Wilcox * @ctx: A pointer that will be passed to the handler 164c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox * @handler: The function to call on completion 165b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 166b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Allocate a Command ID for a queue. The data passed in will 167b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * be passed to the completion handler. This is implemented by using 168b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * the bottom two bits of the ctx pointer to store the handler ID. 169b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Passing in a pointer that's not 4-byte aligned will cause a BUG. 170b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * We can change this if it becomes a problem. 171184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox * 172184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox * May be called with local interrupts disabled and the q_lock held, 173184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox * or with interrupts enabled and no locks held. 174b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 175c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcoxstatic int alloc_cmdid(struct nvme_queue *nvmeq, void *ctx, 176c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox nvme_completion_fn handler, unsigned timeout) 177b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 178e6d15f79f997a98b3a69abbc462fc9041cc1a7b4Matthew Wilcox int depth = nvmeq->q_depth - 1; 179e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox struct nvme_cmd_info *info = nvme_cmd_info(nvmeq); 180b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int cmdid; 181b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 182b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox do { 183b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cmdid = find_first_zero_bit(nvmeq->cmdid_data, depth); 184b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (cmdid >= depth) 185b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -EBUSY; 186b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } while (test_and_set_bit(cmdid, nvmeq->cmdid_data)); 187b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 188c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox info[cmdid].fn = handler; 189c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox info[cmdid].ctx = ctx; 190e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox info[cmdid].timeout = jiffies + timeout; 191c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch info[cmdid].aborted = 0; 192b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return cmdid; 193b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 194b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 195b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int alloc_cmdid_killable(struct nvme_queue *nvmeq, void *ctx, 196c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox nvme_completion_fn handler, unsigned timeout) 197b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 198b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int cmdid; 199b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox wait_event_killable(nvmeq->sq_full, 200e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox (cmdid = alloc_cmdid(nvmeq, ctx, handler, timeout)) >= 0); 201b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return (cmdid < 0) ? -EINTR : cmdid; 202b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 203b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 204c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox/* Special values must be less than 0x1000 */ 205c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox#define CMD_CTX_BASE ((void *)POISON_POINTER_DELTA) 206d2d8703481f60d67f49e3177196cbe474b11377cMatthew Wilcox#define CMD_CTX_CANCELLED (0x30C + CMD_CTX_BASE) 207d2d8703481f60d67f49e3177196cbe474b11377cMatthew Wilcox#define CMD_CTX_COMPLETED (0x310 + CMD_CTX_BASE) 208d2d8703481f60d67f49e3177196cbe474b11377cMatthew Wilcox#define CMD_CTX_INVALID (0x314 + CMD_CTX_BASE) 20953562be74bd06bbe74d2acf3caca5398f8eeb160Keith Busch#define CMD_CTX_ABORT (0x318 + CMD_CTX_BASE) 210be7b62754e097adc0cb16c25c9ee86ee20de62fbMatthew Wilcox 211edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Buschstatic void special_completion(struct nvme_queue *nvmeq, void *ctx, 212c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox struct nvme_completion *cqe) 213c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox{ 214c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox if (ctx == CMD_CTX_CANCELLED) 215c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox return; 216c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch if (ctx == CMD_CTX_ABORT) { 217edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch ++nvmeq->dev->abort_limit; 218c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch return; 219c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch } 220c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox if (ctx == CMD_CTX_COMPLETED) { 221edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch dev_warn(nvmeq->q_dmadev, 222c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox "completed id %d twice on queue %d\n", 223c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox cqe->command_id, le16_to_cpup(&cqe->sq_id)); 224c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox return; 225c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox } 226c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox if (ctx == CMD_CTX_INVALID) { 227edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch dev_warn(nvmeq->q_dmadev, 228c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox "invalid id %d completed on queue %d\n", 229c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox cqe->command_id, le16_to_cpup(&cqe->sq_id)); 230c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox return; 231c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox } 232c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox 233edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch dev_warn(nvmeq->q_dmadev, "Unknown special completion %p\n", ctx); 234c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox} 235c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox 236edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Buschstatic void async_completion(struct nvme_queue *nvmeq, void *ctx, 2374d115420707afcabe77d2535e092356df6664b70Keith Busch struct nvme_completion *cqe) 2384d115420707afcabe77d2535e092356df6664b70Keith Busch{ 2394d115420707afcabe77d2535e092356df6664b70Keith Busch struct async_cmd_info *cmdinfo = ctx; 2404d115420707afcabe77d2535e092356df6664b70Keith Busch cmdinfo->result = le32_to_cpup(&cqe->result); 2414d115420707afcabe77d2535e092356df6664b70Keith Busch cmdinfo->status = le16_to_cpup(&cqe->status) >> 1; 2424d115420707afcabe77d2535e092356df6664b70Keith Busch queue_kthread_work(cmdinfo->worker, &cmdinfo->work); 2434d115420707afcabe77d2535e092356df6664b70Keith Busch} 2444d115420707afcabe77d2535e092356df6664b70Keith Busch 245184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox/* 246184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox * Called with local interrupts disabled and the q_lock held. May not sleep. 247184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox */ 248c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcoxstatic void *free_cmdid(struct nvme_queue *nvmeq, int cmdid, 249c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox nvme_completion_fn *fn) 250b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 251c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox void *ctx; 252e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox struct nvme_cmd_info *info = nvme_cmd_info(nvmeq); 253b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 25494bbac4052eb93219ca0aa370ca741486b25fb98Keith Busch if (cmdid >= nvmeq->q_depth || !info[cmdid].fn) { 25594bbac4052eb93219ca0aa370ca741486b25fb98Keith Busch if (fn) 25694bbac4052eb93219ca0aa370ca741486b25fb98Keith Busch *fn = special_completion; 25748e3d39816416b3bf03dee3a796c0c04427c1a31Matthew Wilcox return CMD_CTX_INVALID; 258c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox } 259859361a228258edf4821d9f5635825033eca78e8Keith Busch if (fn) 260859361a228258edf4821d9f5635825033eca78e8Keith Busch *fn = info[cmdid].fn; 261c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox ctx = info[cmdid].ctx; 262c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox info[cmdid].fn = special_completion; 263e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox info[cmdid].ctx = CMD_CTX_COMPLETED; 264b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox clear_bit(cmdid, nvmeq->cmdid_data); 265b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox wake_up(&nvmeq->sq_full); 266c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox return ctx; 267b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 268b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 269c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcoxstatic void *cancel_cmdid(struct nvme_queue *nvmeq, int cmdid, 270c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox nvme_completion_fn *fn) 2713c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox{ 272c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox void *ctx; 273e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox struct nvme_cmd_info *info = nvme_cmd_info(nvmeq); 274c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox if (fn) 275c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox *fn = info[cmdid].fn; 276c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox ctx = info[cmdid].ctx; 277c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox info[cmdid].fn = special_completion; 278e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox info[cmdid].ctx = CMD_CTX_CANCELLED; 279c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox return ctx; 2803c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox} 2813c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox 2825a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Buschstatic struct nvme_queue *raw_nvmeq(struct nvme_dev *dev, int qid) 283b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 2845a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch return rcu_dereference_raw(dev->queues[qid]); 285b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 286b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2874f5099af4f3d5f999d8ab7784472d93e810e3912Keith Buschstatic struct nvme_queue *get_nvmeq(struct nvme_dev *dev) __acquires(RCU) 2885a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch{ 289a51afb54339c5e9ee72df66ae0f2ac5aacfed365Keith Busch struct nvme_queue *nvmeq; 29042f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch unsigned queue_id = get_cpu_var(*dev->io_queue); 291a51afb54339c5e9ee72df66ae0f2ac5aacfed365Keith Busch 2925a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch rcu_read_lock(); 293a51afb54339c5e9ee72df66ae0f2ac5aacfed365Keith Busch nvmeq = rcu_dereference(dev->queues[queue_id]); 294a51afb54339c5e9ee72df66ae0f2ac5aacfed365Keith Busch if (nvmeq) 295a51afb54339c5e9ee72df66ae0f2ac5aacfed365Keith Busch return nvmeq; 296a51afb54339c5e9ee72df66ae0f2ac5aacfed365Keith Busch 297a51afb54339c5e9ee72df66ae0f2ac5aacfed365Keith Busch rcu_read_unlock(); 298a51afb54339c5e9ee72df66ae0f2ac5aacfed365Keith Busch put_cpu_var(*dev->io_queue); 299a51afb54339c5e9ee72df66ae0f2ac5aacfed365Keith Busch return NULL; 3005a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch} 3015a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch 3024f5099af4f3d5f999d8ab7784472d93e810e3912Keith Buschstatic void put_nvmeq(struct nvme_queue *nvmeq) __releases(RCU) 303b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 3045a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch rcu_read_unlock(); 30542f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch put_cpu_var(nvmeq->dev->io_queue); 306b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 307b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 3084f5099af4f3d5f999d8ab7784472d93e810e3912Keith Buschstatic struct nvme_queue *lock_nvmeq(struct nvme_dev *dev, int q_idx) 3094f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch __acquires(RCU) 310b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 311a51afb54339c5e9ee72df66ae0f2ac5aacfed365Keith Busch struct nvme_queue *nvmeq; 312a51afb54339c5e9ee72df66ae0f2ac5aacfed365Keith Busch 3134f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch rcu_read_lock(); 314a51afb54339c5e9ee72df66ae0f2ac5aacfed365Keith Busch nvmeq = rcu_dereference(dev->queues[q_idx]); 315a51afb54339c5e9ee72df66ae0f2ac5aacfed365Keith Busch if (nvmeq) 316a51afb54339c5e9ee72df66ae0f2ac5aacfed365Keith Busch return nvmeq; 317a51afb54339c5e9ee72df66ae0f2ac5aacfed365Keith Busch 318a51afb54339c5e9ee72df66ae0f2ac5aacfed365Keith Busch rcu_read_unlock(); 319a51afb54339c5e9ee72df66ae0f2ac5aacfed365Keith Busch return NULL; 3204f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch} 3214f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch 3224f5099af4f3d5f999d8ab7784472d93e810e3912Keith Buschstatic void unlock_nvmeq(struct nvme_queue *nvmeq) __releases(RCU) 3234f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch{ 3244f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch rcu_read_unlock(); 325b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 326b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 327b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/** 328714a7a22884b74862540bc84955274d86b2f6040Matthew Wilcox * nvme_submit_cmd() - Copy a command into a queue and ring the doorbell 329b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * @nvmeq: The queue to use 330b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * @cmd: The command to send 331b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 332b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Safe to use from interrupt context 333b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 334b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_submit_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd) 335b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 336b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unsigned long flags; 337b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 tail; 338b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox spin_lock_irqsave(&nvmeq->q_lock, flags); 3394f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch if (nvmeq->q_suspended) { 3404f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch spin_unlock_irqrestore(&nvmeq->q_lock, flags); 3414f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch return -EBUSY; 3424f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch } 343b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox tail = nvmeq->sq_tail; 344b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memcpy(&nvmeq->sq_cmds[tail], cmd, sizeof(*cmd)); 345b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (++tail == nvmeq->q_depth) 346b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox tail = 0; 3477547881d0951384f9833ec3a80fac8f3f16f3b98Matthew Wilcox writel(tail, nvmeq->q_db); 348b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->sq_tail = tail; 349b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox spin_unlock_irqrestore(&nvmeq->q_lock, flags); 350b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 351b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 352b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 353b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 354eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcoxstatic __le64 **iod_list(struct nvme_iod *iod) 355e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews{ 356eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return ((void *)iod) + iod->offset; 357e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews} 358e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews 359eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox/* 360eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox * Will slightly overestimate the number of pages needed. This is OK 361eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox * as it only leads to a small amount of wasted memory for the lifetime of 362eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox * the I/O. 363eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox */ 364eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcoxstatic int nvme_npages(unsigned size) 365eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox{ 366eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox unsigned nprps = DIV_ROUND_UP(size + PAGE_SIZE, PAGE_SIZE); 367eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return DIV_ROUND_UP(8 * nprps, PAGE_SIZE - 8); 368eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox} 369b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 370eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcoxstatic struct nvme_iod * 371eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcoxnvme_alloc_iod(unsigned nseg, unsigned nbytes, gfp_t gfp) 372b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 373eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox struct nvme_iod *iod = kmalloc(sizeof(struct nvme_iod) + 374eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox sizeof(__le64 *) * nvme_npages(nbytes) + 375eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox sizeof(struct scatterlist) * nseg, gfp); 376eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox 377eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox if (iod) { 378eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->offset = offsetof(struct nvme_iod, sg[nseg]); 379eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->npages = -1; 380eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->length = nbytes; 3812b1960341576bf51c01b12fefeb1cc53820923e7Keith Busch iod->nents = 0; 382edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch iod->first_dma = 0ULL; 3836198221fa0df0298513b35796f63f242ea97134eKeith Busch iod->start_time = jiffies; 384eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox } 385eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox 386eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return iod; 387b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 388b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 3895d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermavoid nvme_free_iod(struct nvme_dev *dev, struct nvme_iod *iod) 390b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 391eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox const int last_prp = PAGE_SIZE / 8 - 1; 392eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox int i; 393eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox __le64 **list = iod_list(iod); 394eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox dma_addr_t prp_dma = iod->first_dma; 395eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox 396eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox if (iod->npages == 0) 397eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox dma_pool_free(dev->prp_small_pool, list[0], prp_dma); 398eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox for (i = 0; i < iod->npages; i++) { 399eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox __le64 *prp_list = list[i]; 400eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox dma_addr_t next_prp_dma = le64_to_cpu(prp_list[last_prp]); 401eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox dma_pool_free(dev->prp_page_pool, prp_list, prp_dma); 402eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox prp_dma = next_prp_dma; 403eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox } 404eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox kfree(iod); 405b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 406b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 4076198221fa0df0298513b35796f63f242ea97134eKeith Buschstatic void nvme_start_io_acct(struct bio *bio) 4086198221fa0df0298513b35796f63f242ea97134eKeith Busch{ 4096198221fa0df0298513b35796f63f242ea97134eKeith Busch struct gendisk *disk = bio->bi_bdev->bd_disk; 410b4e75cbf1364c4bbce3599c3279892a55b6ede07Sam Bradshaw if (blk_queue_io_stat(disk->queue)) { 411b4e75cbf1364c4bbce3599c3279892a55b6ede07Sam Bradshaw const int rw = bio_data_dir(bio); 412b4e75cbf1364c4bbce3599c3279892a55b6ede07Sam Bradshaw int cpu = part_stat_lock(); 413b4e75cbf1364c4bbce3599c3279892a55b6ede07Sam Bradshaw part_round_stats(cpu, &disk->part0); 414b4e75cbf1364c4bbce3599c3279892a55b6ede07Sam Bradshaw part_stat_inc(cpu, &disk->part0, ios[rw]); 415b4e75cbf1364c4bbce3599c3279892a55b6ede07Sam Bradshaw part_stat_add(cpu, &disk->part0, sectors[rw], 416b4e75cbf1364c4bbce3599c3279892a55b6ede07Sam Bradshaw bio_sectors(bio)); 417b4e75cbf1364c4bbce3599c3279892a55b6ede07Sam Bradshaw part_inc_in_flight(&disk->part0, rw); 418b4e75cbf1364c4bbce3599c3279892a55b6ede07Sam Bradshaw part_stat_unlock(); 419b4e75cbf1364c4bbce3599c3279892a55b6ede07Sam Bradshaw } 4206198221fa0df0298513b35796f63f242ea97134eKeith Busch} 4216198221fa0df0298513b35796f63f242ea97134eKeith Busch 4226198221fa0df0298513b35796f63f242ea97134eKeith Buschstatic void nvme_end_io_acct(struct bio *bio, unsigned long start_time) 4236198221fa0df0298513b35796f63f242ea97134eKeith Busch{ 4246198221fa0df0298513b35796f63f242ea97134eKeith Busch struct gendisk *disk = bio->bi_bdev->bd_disk; 425b4e75cbf1364c4bbce3599c3279892a55b6ede07Sam Bradshaw if (blk_queue_io_stat(disk->queue)) { 426b4e75cbf1364c4bbce3599c3279892a55b6ede07Sam Bradshaw const int rw = bio_data_dir(bio); 427b4e75cbf1364c4bbce3599c3279892a55b6ede07Sam Bradshaw unsigned long duration = jiffies - start_time; 428b4e75cbf1364c4bbce3599c3279892a55b6ede07Sam Bradshaw int cpu = part_stat_lock(); 429b4e75cbf1364c4bbce3599c3279892a55b6ede07Sam Bradshaw part_stat_add(cpu, &disk->part0, ticks[rw], duration); 430b4e75cbf1364c4bbce3599c3279892a55b6ede07Sam Bradshaw part_round_stats(cpu, &disk->part0); 431b4e75cbf1364c4bbce3599c3279892a55b6ede07Sam Bradshaw part_dec_in_flight(&disk->part0, rw); 432b4e75cbf1364c4bbce3599c3279892a55b6ede07Sam Bradshaw part_stat_unlock(); 433b4e75cbf1364c4bbce3599c3279892a55b6ede07Sam Bradshaw } 4346198221fa0df0298513b35796f63f242ea97134eKeith Busch} 4356198221fa0df0298513b35796f63f242ea97134eKeith Busch 436edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Buschstatic void bio_completion(struct nvme_queue *nvmeq, void *ctx, 437b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_completion *cqe) 438b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 439eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox struct nvme_iod *iod = ctx; 440eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox struct bio *bio = iod->private; 441b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 status = le16_to_cpup(&cqe->status) >> 1; 4423291fa57cb1b004c1a4823beb28b5cc72555f1a5Keith Busch int error = 0; 443b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 444edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch if (unlikely(status)) { 445edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch if (!(status & NVME_SC_DNR || 446edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch bio->bi_rw & REQ_FAILFAST_MASK) && 447edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch (jiffies - iod->start_time) < IOD_TIMEOUT) { 448edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch if (!waitqueue_active(&nvmeq->sq_full)) 449edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch add_wait_queue(&nvmeq->sq_full, 450edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch &nvmeq->sq_cong_wait); 451edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch list_add_tail(&iod->node, &nvmeq->iod_bio); 452edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch wake_up(&nvmeq->sq_full); 453edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch return; 454edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch } 4553291fa57cb1b004c1a4823beb28b5cc72555f1a5Keith Busch error = -EIO; 456edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch } 4579e59d091b0eb04f223ed037348e3d9e36f30e72bKeith Busch if (iod->nents) { 458edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch dma_unmap_sg(nvmeq->q_dmadev, iod->sg, iod->nents, 459b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox bio_data_dir(bio) ? DMA_TO_DEVICE : DMA_FROM_DEVICE); 4609e59d091b0eb04f223ed037348e3d9e36f30e72bKeith Busch nvme_end_io_acct(bio, iod->start_time); 4619e59d091b0eb04f223ed037348e3d9e36f30e72bKeith Busch } 462edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch nvme_free_iod(nvmeq->dev, iod); 4633291fa57cb1b004c1a4823beb28b5cc72555f1a5Keith Busch 4643291fa57cb1b004c1a4823beb28b5cc72555f1a5Keith Busch trace_block_bio_complete(bdev_get_queue(bio->bi_bdev), bio, error); 4653291fa57cb1b004c1a4823beb28b5cc72555f1a5Keith Busch bio_endio(bio, error); 466b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 467b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 468184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox/* length is in bytes. gfp flags indicates whether we may sleep. */ 469edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Buschint nvme_setup_prps(struct nvme_dev *dev, struct nvme_iod *iod, int total_len, 470edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch gfp_t gfp) 471ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox{ 47299802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox struct dma_pool *pool; 473eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox int length = total_len; 474eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox struct scatterlist *sg = iod->sg; 475ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox int dma_len = sg_dma_len(sg); 476ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox u64 dma_addr = sg_dma_address(sg); 477ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox int offset = offset_in_page(dma_addr); 478e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews __le64 *prp_list; 479eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox __le64 **list = iod_list(iod); 480e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_addr_t prp_dma; 481eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox int nprps, i; 482ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox 483ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox length -= (PAGE_SIZE - offset); 484ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox if (length <= 0) 485eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return total_len; 486ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox 487ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox dma_len -= (PAGE_SIZE - offset); 488ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox if (dma_len) { 489ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox dma_addr += (PAGE_SIZE - offset); 490ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox } else { 491ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox sg = sg_next(sg); 492ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox dma_addr = sg_dma_address(sg); 493ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox dma_len = sg_dma_len(sg); 494ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox } 495ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox 496ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox if (length <= PAGE_SIZE) { 497edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch iod->first_dma = dma_addr; 498eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return total_len; 499e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews } 500e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews 501e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews nprps = DIV_ROUND_UP(length, PAGE_SIZE); 50299802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox if (nprps <= (256 / 8)) { 50399802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox pool = dev->prp_small_pool; 504eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->npages = 0; 50599802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox } else { 50699802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox pool = dev->prp_page_pool; 507eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->npages = 1; 50899802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox } 50999802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox 510b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox prp_list = dma_pool_alloc(pool, gfp, &prp_dma); 511b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox if (!prp_list) { 512edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch iod->first_dma = dma_addr; 513eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->npages = -1; 514eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return (total_len - length) + PAGE_SIZE; 515b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox } 516eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox list[0] = prp_list; 517eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->first_dma = prp_dma; 518e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews i = 0; 519e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews for (;;) { 5207523d834dd1573610078eb1ac0933f6490232f90Matthew Wilcox if (i == PAGE_SIZE / 8) { 521e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews __le64 *old_prp_list = prp_list; 522b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox prp_list = dma_pool_alloc(pool, gfp, &prp_dma); 523eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox if (!prp_list) 524eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return total_len - length; 525eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox list[iod->npages++] = prp_list; 5267523d834dd1573610078eb1ac0933f6490232f90Matthew Wilcox prp_list[0] = old_prp_list[i - 1]; 5277523d834dd1573610078eb1ac0933f6490232f90Matthew Wilcox old_prp_list[i - 1] = cpu_to_le64(prp_dma); 5287523d834dd1573610078eb1ac0933f6490232f90Matthew Wilcox i = 1; 529e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews } 530e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews prp_list[i++] = cpu_to_le64(dma_addr); 531e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_len -= PAGE_SIZE; 532e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_addr += PAGE_SIZE; 533e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews length -= PAGE_SIZE; 534e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews if (length <= 0) 535e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews break; 536e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews if (dma_len > 0) 537e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews continue; 538e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews BUG_ON(dma_len < 0); 539e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews sg = sg_next(sg); 540e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_addr = sg_dma_address(sg); 541e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_len = sg_dma_len(sg); 542ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox } 543ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox 544eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return total_len; 545ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox} 546ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox 547427e97080196548557b288517537ab7eb48c309fKeith Buschstatic int nvme_split_and_submit(struct bio *bio, struct nvme_queue *nvmeq, 54820d0189b1012a37d2533a87fb451f7852f2418d1Kent Overstreet int len) 549427e97080196548557b288517537ab7eb48c309fKeith Busch{ 55020d0189b1012a37d2533a87fb451f7852f2418d1Kent Overstreet struct bio *split = bio_split(bio, len >> 9, GFP_ATOMIC, NULL); 55120d0189b1012a37d2533a87fb451f7852f2418d1Kent Overstreet if (!split) 552427e97080196548557b288517537ab7eb48c309fKeith Busch return -ENOMEM; 553427e97080196548557b288517537ab7eb48c309fKeith Busch 5543291fa57cb1b004c1a4823beb28b5cc72555f1a5Keith Busch trace_block_split(bdev_get_queue(bio->bi_bdev), bio, 5553291fa57cb1b004c1a4823beb28b5cc72555f1a5Keith Busch split->bi_iter.bi_sector); 55620d0189b1012a37d2533a87fb451f7852f2418d1Kent Overstreet bio_chain(split, bio); 55720d0189b1012a37d2533a87fb451f7852f2418d1Kent Overstreet 558edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch if (!waitqueue_active(&nvmeq->sq_full)) 559427e97080196548557b288517537ab7eb48c309fKeith Busch add_wait_queue(&nvmeq->sq_full, &nvmeq->sq_cong_wait); 56020d0189b1012a37d2533a87fb451f7852f2418d1Kent Overstreet bio_list_add(&nvmeq->sq_cong, split); 56120d0189b1012a37d2533a87fb451f7852f2418d1Kent Overstreet bio_list_add(&nvmeq->sq_cong, bio); 562edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch wake_up(&nvmeq->sq_full); 563427e97080196548557b288517537ab7eb48c309fKeith Busch 564427e97080196548557b288517537ab7eb48c309fKeith Busch return 0; 565427e97080196548557b288517537ab7eb48c309fKeith Busch} 566427e97080196548557b288517537ab7eb48c309fKeith Busch 5671ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox/* NVMe scatterlists require no holes in the virtual address */ 5681ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox#define BIOVEC_NOT_VIRT_MERGEABLE(vec1, vec2) ((vec2)->bv_offset || \ 5691ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox (((vec1)->bv_offset + (vec1)->bv_len) % PAGE_SIZE)) 5701ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox 571427e97080196548557b288517537ab7eb48c309fKeith Buschstatic int nvme_map_bio(struct nvme_queue *nvmeq, struct nvme_iod *iod, 572b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct bio *bio, enum dma_data_direction dma_dir, int psegs) 573b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 5747988613b0e5b2638caf6cd493cc78e9595eba19cKent Overstreet struct bio_vec bvec, bvprv; 5757988613b0e5b2638caf6cd493cc78e9595eba19cKent Overstreet struct bvec_iter iter; 576768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox struct scatterlist *sg = NULL; 5777988613b0e5b2638caf6cd493cc78e9595eba19cKent Overstreet int length = 0, nsegs = 0, split_len = bio->bi_iter.bi_size; 5787988613b0e5b2638caf6cd493cc78e9595eba19cKent Overstreet int first = 1; 579159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch 580159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch if (nvmeq->dev->stripe_size) 581159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch split_len = nvmeq->dev->stripe_size - 5824f024f3797c43cb4b73cd2c50cec728842d0e49eKent Overstreet ((bio->bi_iter.bi_sector << 9) & 5834f024f3797c43cb4b73cd2c50cec728842d0e49eKent Overstreet (nvmeq->dev->stripe_size - 1)); 584b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 585eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox sg_init_table(iod->sg, psegs); 5867988613b0e5b2638caf6cd493cc78e9595eba19cKent Overstreet bio_for_each_segment(bvec, bio, iter) { 5877988613b0e5b2638caf6cd493cc78e9595eba19cKent Overstreet if (!first && BIOVEC_PHYS_MERGEABLE(&bvprv, &bvec)) { 5887988613b0e5b2638caf6cd493cc78e9595eba19cKent Overstreet sg->length += bvec.bv_len; 589768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox } else { 5907988613b0e5b2638caf6cd493cc78e9595eba19cKent Overstreet if (!first && BIOVEC_NOT_VIRT_MERGEABLE(&bvprv, &bvec)) 5917988613b0e5b2638caf6cd493cc78e9595eba19cKent Overstreet return nvme_split_and_submit(bio, nvmeq, 59220d0189b1012a37d2533a87fb451f7852f2418d1Kent Overstreet length); 593427e97080196548557b288517537ab7eb48c309fKeith Busch 594eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox sg = sg ? sg + 1 : iod->sg; 5957988613b0e5b2638caf6cd493cc78e9595eba19cKent Overstreet sg_set_page(sg, bvec.bv_page, 5967988613b0e5b2638caf6cd493cc78e9595eba19cKent Overstreet bvec.bv_len, bvec.bv_offset); 597768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox nsegs++; 598768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox } 599159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch 6007988613b0e5b2638caf6cd493cc78e9595eba19cKent Overstreet if (split_len - length < bvec.bv_len) 60120d0189b1012a37d2533a87fb451f7852f2418d1Kent Overstreet return nvme_split_and_submit(bio, nvmeq, split_len); 6027988613b0e5b2638caf6cd493cc78e9595eba19cKent Overstreet length += bvec.bv_len; 603768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox bvprv = bvec; 6047988613b0e5b2638caf6cd493cc78e9595eba19cKent Overstreet first = 0; 605b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 606eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->nents = nsegs; 607768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox sg_mark_end(sg); 608427e97080196548557b288517537ab7eb48c309fKeith Busch if (dma_map_sg(nvmeq->q_dmadev, iod->sg, iod->nents, dma_dir) == 0) 6091ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox return -ENOMEM; 610427e97080196548557b288517537ab7eb48c309fKeith Busch 6114f024f3797c43cb4b73cd2c50cec728842d0e49eKent Overstreet BUG_ON(length != bio->bi_iter.bi_size); 6121ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox return length; 613b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 614b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 6150e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Buschstatic int nvme_submit_discard(struct nvme_queue *nvmeq, struct nvme_ns *ns, 6160e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch struct bio *bio, struct nvme_iod *iod, int cmdid) 6170e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch{ 618edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch struct nvme_dsm_range *range = 619edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch (struct nvme_dsm_range *)iod_list(iod)[0]; 6200e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch struct nvme_command *cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail]; 6210e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch 6220e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch range->cattr = cpu_to_le32(0); 6234f024f3797c43cb4b73cd2c50cec728842d0e49eKent Overstreet range->nlb = cpu_to_le32(bio->bi_iter.bi_size >> ns->lba_shift); 6244f024f3797c43cb4b73cd2c50cec728842d0e49eKent Overstreet range->slba = cpu_to_le64(nvme_block_nr(ns, bio->bi_iter.bi_sector)); 6250e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch 6260e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch memset(cmnd, 0, sizeof(*cmnd)); 6270e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch cmnd->dsm.opcode = nvme_cmd_dsm; 6280e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch cmnd->dsm.command_id = cmdid; 6290e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch cmnd->dsm.nsid = cpu_to_le32(ns->ns_id); 6300e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch cmnd->dsm.prp1 = cpu_to_le64(iod->first_dma); 6310e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch cmnd->dsm.nr = 0; 6320e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD); 6330e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch 6340e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch if (++nvmeq->sq_tail == nvmeq->q_depth) 6350e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch nvmeq->sq_tail = 0; 6360e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch writel(nvmeq->sq_tail, nvmeq->q_db); 6370e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch 6380e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch return 0; 6390e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch} 6400e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch 64100df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcoxstatic int nvme_submit_flush(struct nvme_queue *nvmeq, struct nvme_ns *ns, 64200df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox int cmdid) 64300df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox{ 64400df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox struct nvme_command *cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail]; 64500df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 64600df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox memset(cmnd, 0, sizeof(*cmnd)); 64700df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox cmnd->common.opcode = nvme_cmd_flush; 64800df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox cmnd->common.command_id = cmdid; 64900df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox cmnd->common.nsid = cpu_to_le32(ns->ns_id); 65000df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 65100df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox if (++nvmeq->sq_tail == nvmeq->q_depth) 65200df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox nvmeq->sq_tail = 0; 65300df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox writel(nvmeq->sq_tail, nvmeq->q_db); 65400df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 65500df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox return 0; 65600df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox} 65700df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 658edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Buschstatic int nvme_submit_iod(struct nvme_queue *nvmeq, struct nvme_iod *iod) 659b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 660edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch struct bio *bio = iod->private; 661edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch struct nvme_ns *ns = bio->bi_bdev->bd_disk->private_data; 662ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox struct nvme_command *cmnd; 663edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch int cmdid; 664b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 control; 665b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 dsmgmt; 66600df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 667ff976d724a74e4522e9ca2de1fb37ac4520f454fMatthew Wilcox cmdid = alloc_cmdid(nvmeq, iod, bio_completion, NVME_IO_TIMEOUT); 668b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (unlikely(cmdid < 0)) 669edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch return cmdid; 670b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 671edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch if (bio->bi_rw & REQ_DISCARD) 672edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch return nvme_submit_discard(nvmeq, ns, bio, iod, cmdid); 67353562be74bd06bbe74d2acf3caca5398f8eeb160Keith Busch if (bio->bi_rw & REQ_FLUSH) 67400df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox return nvme_submit_flush(nvmeq, ns, cmdid); 67500df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 676b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox control = 0; 677b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (bio->bi_rw & REQ_FUA) 678b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox control |= NVME_RW_FUA; 679b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (bio->bi_rw & (REQ_FAILFAST_DEV | REQ_RAHEAD)) 680b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox control |= NVME_RW_LR; 681b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 682b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dsmgmt = 0; 683b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (bio->bi_rw & REQ_RAHEAD) 684b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH; 685b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 686ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail]; 687b8deb62cf271fa9381edc8cf52bcae2f0225c55aMatthew Wilcox memset(cmnd, 0, sizeof(*cmnd)); 688b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 689edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch cmnd->rw.opcode = bio_data_dir(bio) ? nvme_cmd_write : nvme_cmd_read; 690ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.command_id = cmdid; 691ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.nsid = cpu_to_le32(ns->ns_id); 692edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch cmnd->rw.prp1 = cpu_to_le64(sg_dma_address(iod->sg)); 693edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch cmnd->rw.prp2 = cpu_to_le64(iod->first_dma); 6944f024f3797c43cb4b73cd2c50cec728842d0e49eKent Overstreet cmnd->rw.slba = cpu_to_le64(nvme_block_nr(ns, bio->bi_iter.bi_sector)); 695edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch cmnd->rw.length = 696edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch cpu_to_le16((bio->bi_iter.bi_size >> ns->lba_shift) - 1); 697ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.control = cpu_to_le16(control); 698ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt); 699b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 700b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (++nvmeq->sq_tail == nvmeq->q_depth) 701b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->sq_tail = 0; 7027547881d0951384f9833ec3a80fac8f3f16f3b98Matthew Wilcox writel(nvmeq->sq_tail, nvmeq->q_db); 703b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 7041974b1ae8852324a75fb8cfecbc7b758fd5a2c3cMatthew Wilcox return 0; 705edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch} 706edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch 70753562be74bd06bbe74d2acf3caca5398f8eeb160Keith Buschstatic int nvme_split_flush_data(struct nvme_queue *nvmeq, struct bio *bio) 70853562be74bd06bbe74d2acf3caca5398f8eeb160Keith Busch{ 70953562be74bd06bbe74d2acf3caca5398f8eeb160Keith Busch struct bio *split = bio_clone(bio, GFP_ATOMIC); 71053562be74bd06bbe74d2acf3caca5398f8eeb160Keith Busch if (!split) 71153562be74bd06bbe74d2acf3caca5398f8eeb160Keith Busch return -ENOMEM; 71253562be74bd06bbe74d2acf3caca5398f8eeb160Keith Busch 71353562be74bd06bbe74d2acf3caca5398f8eeb160Keith Busch split->bi_iter.bi_size = 0; 71453562be74bd06bbe74d2acf3caca5398f8eeb160Keith Busch split->bi_phys_segments = 0; 71553562be74bd06bbe74d2acf3caca5398f8eeb160Keith Busch bio->bi_rw &= ~REQ_FLUSH; 71653562be74bd06bbe74d2acf3caca5398f8eeb160Keith Busch bio_chain(split, bio); 71753562be74bd06bbe74d2acf3caca5398f8eeb160Keith Busch 71853562be74bd06bbe74d2acf3caca5398f8eeb160Keith Busch if (!waitqueue_active(&nvmeq->sq_full)) 71953562be74bd06bbe74d2acf3caca5398f8eeb160Keith Busch add_wait_queue(&nvmeq->sq_full, &nvmeq->sq_cong_wait); 72053562be74bd06bbe74d2acf3caca5398f8eeb160Keith Busch bio_list_add(&nvmeq->sq_cong, split); 72153562be74bd06bbe74d2acf3caca5398f8eeb160Keith Busch bio_list_add(&nvmeq->sq_cong, bio); 72253562be74bd06bbe74d2acf3caca5398f8eeb160Keith Busch wake_up_process(nvme_thread); 72353562be74bd06bbe74d2acf3caca5398f8eeb160Keith Busch 72453562be74bd06bbe74d2acf3caca5398f8eeb160Keith Busch return 0; 72553562be74bd06bbe74d2acf3caca5398f8eeb160Keith Busch} 72653562be74bd06bbe74d2acf3caca5398f8eeb160Keith Busch 727edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch/* 728edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch * Called with local interrupts disabled and the q_lock held. May not sleep. 729edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch */ 730edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Buschstatic int nvme_submit_bio_queue(struct nvme_queue *nvmeq, struct nvme_ns *ns, 731edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch struct bio *bio) 732edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch{ 733edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch struct nvme_iod *iod; 734edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch int psegs = bio_phys_segments(ns->queue, bio); 735edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch int result; 736edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch 73753562be74bd06bbe74d2acf3caca5398f8eeb160Keith Busch if ((bio->bi_rw & REQ_FLUSH) && psegs) 73853562be74bd06bbe74d2acf3caca5398f8eeb160Keith Busch return nvme_split_flush_data(nvmeq, bio); 739edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch 740edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch iod = nvme_alloc_iod(psegs, bio->bi_iter.bi_size, GFP_ATOMIC); 741edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch if (!iod) 742edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch return -ENOMEM; 743edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch 744edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch iod->private = bio; 745edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch if (bio->bi_rw & REQ_DISCARD) { 746edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch void *range; 747edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch /* 748edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch * We reuse the small pool to allocate the 16-byte range here 749edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch * as it is not worth having a special pool for these or 750edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch * additional cases to handle freeing the iod. 751edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch */ 752edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch range = dma_pool_alloc(nvmeq->dev->prp_small_pool, 753edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch GFP_ATOMIC, 754edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch &iod->first_dma); 755edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch if (!range) { 756edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch result = -ENOMEM; 757edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch goto free_iod; 758edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch } 759edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch iod_list(iod)[0] = (__le64 *)range; 760edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch iod->npages = 0; 761edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch } else if (psegs) { 762edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch result = nvme_map_bio(nvmeq, iod, bio, 763edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch bio_data_dir(bio) ? DMA_TO_DEVICE : DMA_FROM_DEVICE, 764edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch psegs); 765edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch if (result <= 0) 766edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch goto free_iod; 767edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch if (nvme_setup_prps(nvmeq->dev, iod, result, GFP_ATOMIC) != 768edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch result) { 769edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch result = -ENOMEM; 770edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch goto free_iod; 771edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch } 772edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch nvme_start_io_acct(bio); 773edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch } 774edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch if (unlikely(nvme_submit_iod(nvmeq, iod))) { 775edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch if (!waitqueue_active(&nvmeq->sq_full)) 776edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch add_wait_queue(&nvmeq->sq_full, &nvmeq->sq_cong_wait); 777edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch list_add_tail(&iod->node, &nvmeq->iod_bio); 778edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch } 779edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch return 0; 7801974b1ae8852324a75fb8cfecbc7b758fd5a2c3cMatthew Wilcox 781eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox free_iod: 782eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox nvme_free_iod(nvmeq->dev, iod); 783eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox return result; 784b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 785b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 786e9539f47525ecee05c9f22c3565885f3e9492c52Matthew Wilcoxstatic int nvme_process_cq(struct nvme_queue *nvmeq) 787b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 788821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox u16 head, phase; 789b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 790b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox head = nvmeq->cq_head; 791821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox phase = nvmeq->cq_phase; 792b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 793b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox for (;;) { 794c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox void *ctx; 795c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox nvme_completion_fn fn; 796b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_completion cqe = nvmeq->cqes[head]; 797821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox if ((le16_to_cpu(cqe.status) & 1) != phase) 798b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox break; 799b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->sq_head = le16_to_cpu(cqe.sq_head); 800b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (++head == nvmeq->q_depth) { 801b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox head = 0; 802821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox phase = !phase; 803b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 804b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 805c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox ctx = free_cmdid(nvmeq, cqe.command_id, &fn); 806edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch fn(nvmeq, ctx, &cqe); 807b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 808b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 809b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox /* If the controller ignores the cq head doorbell and continuously 810b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * writes to the queue, it is theoretically possible to wrap around 811b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * the queue twice and mistakenly return IRQ_NONE. Linux only 812b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * requires that 0.1% of your interrupts are handled, so this isn't 813b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * a big problem. 814b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 815821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox if (head == nvmeq->cq_head && phase == nvmeq->cq_phase) 816e9539f47525ecee05c9f22c3565885f3e9492c52Matthew Wilcox return 0; 817b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 818b80d5ccca3a012e91ca64a2a0b13049163a6a698Haiyan Hu writel(head, nvmeq->q_db + nvmeq->dev->db_stride); 819b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->cq_head = head; 820821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox nvmeq->cq_phase = phase; 821b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 822e9539f47525ecee05c9f22c3565885f3e9492c52Matthew Wilcox nvmeq->cqe_seen = 1; 823e9539f47525ecee05c9f22c3565885f3e9492c52Matthew Wilcox return 1; 824b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 825b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 8267d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcoxstatic void nvme_make_request(struct request_queue *q, struct bio *bio) 8277d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox{ 8287d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox struct nvme_ns *ns = q->queuedata; 8297d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox struct nvme_queue *nvmeq = get_nvmeq(ns->dev); 8307d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox int result = -EBUSY; 8317d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox 832cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch if (!nvmeq) { 833cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch bio_endio(bio, -EIO); 834cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch return; 835cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch } 836cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch 8377d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox spin_lock_irq(&nvmeq->q_lock); 838224042742582c9938788b81165180c876e997a07Keith Busch if (!nvmeq->q_suspended && bio_list_empty(&nvmeq->sq_cong)) 8397d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox result = nvme_submit_bio_queue(nvmeq, ns, bio); 8407d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox if (unlikely(result)) { 841edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch if (!waitqueue_active(&nvmeq->sq_full)) 8427d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox add_wait_queue(&nvmeq->sq_full, &nvmeq->sq_cong_wait); 8437d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox bio_list_add(&nvmeq->sq_cong, bio); 8447d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox } 8457d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox 8467d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox nvme_process_cq(nvmeq); 8477d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox spin_unlock_irq(&nvmeq->q_lock); 8487d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox put_nvmeq(nvmeq); 8497d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox} 8507d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox 851b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic irqreturn_t nvme_irq(int irq, void *data) 852b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 85358ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox irqreturn_t result; 85458ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox struct nvme_queue *nvmeq = data; 85558ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox spin_lock(&nvmeq->q_lock); 856e9539f47525ecee05c9f22c3565885f3e9492c52Matthew Wilcox nvme_process_cq(nvmeq); 857e9539f47525ecee05c9f22c3565885f3e9492c52Matthew Wilcox result = nvmeq->cqe_seen ? IRQ_HANDLED : IRQ_NONE; 858e9539f47525ecee05c9f22c3565885f3e9492c52Matthew Wilcox nvmeq->cqe_seen = 0; 85958ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox spin_unlock(&nvmeq->q_lock); 86058ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox return result; 86158ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox} 86258ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox 86358ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcoxstatic irqreturn_t nvme_irq_check(int irq, void *data) 86458ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox{ 86558ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox struct nvme_queue *nvmeq = data; 86658ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox struct nvme_completion cqe = nvmeq->cqes[nvmeq->cq_head]; 86758ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox if ((le16_to_cpu(cqe.status) & 1) != nvmeq->cq_phase) 86858ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox return IRQ_NONE; 86958ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox return IRQ_WAKE_THREAD; 87058ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox} 87158ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox 8723c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcoxstatic void nvme_abort_command(struct nvme_queue *nvmeq, int cmdid) 8733c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox{ 8743c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox spin_lock_irq(&nvmeq->q_lock); 875c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox cancel_cmdid(nvmeq, cmdid, NULL); 8763c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox spin_unlock_irq(&nvmeq->q_lock); 8773c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox} 8783c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox 879c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcoxstruct sync_cmd_info { 880c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox struct task_struct *task; 881c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox u32 result; 882c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox int status; 883c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox}; 884c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox 885edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Buschstatic void sync_completion(struct nvme_queue *nvmeq, void *ctx, 886c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox struct nvme_completion *cqe) 887c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox{ 888c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox struct sync_cmd_info *cmdinfo = ctx; 889c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox cmdinfo->result = le32_to_cpup(&cqe->result); 890c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox cmdinfo->status = le16_to_cpup(&cqe->status) >> 1; 891c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox wake_up_process(cmdinfo->task); 892c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox} 893c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox 894b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 895b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Returns 0 on success. If the result is negative, it's a Linux error code; 896b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * if the result is positive, it's an NVM Express status code 897b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 8984f5099af4f3d5f999d8ab7784472d93e810e3912Keith Buschstatic int nvme_submit_sync_cmd(struct nvme_dev *dev, int q_idx, 8994f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch struct nvme_command *cmd, 9005d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma u32 *result, unsigned timeout) 901b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 9024f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch int cmdid, ret; 903b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct sync_cmd_info cmdinfo; 9044f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch struct nvme_queue *nvmeq; 9054f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch 9064f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch nvmeq = lock_nvmeq(dev, q_idx); 907a51afb54339c5e9ee72df66ae0f2ac5aacfed365Keith Busch if (!nvmeq) 9084f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch return -ENODEV; 909b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 910b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cmdinfo.task = current; 911b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cmdinfo.status = -EINTR; 912b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 9134f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch cmdid = alloc_cmdid(nvmeq, &cmdinfo, sync_completion, timeout); 9144f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch if (cmdid < 0) { 9154f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch unlock_nvmeq(nvmeq); 916b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return cmdid; 9174f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch } 918b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cmd->common.command_id = cmdid; 919b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 9203c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox set_current_state(TASK_KILLABLE); 9214f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch ret = nvme_submit_cmd(nvmeq, cmd); 9224f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch if (ret) { 9234f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch free_cmdid(nvmeq, cmdid, NULL); 9244f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch unlock_nvmeq(nvmeq); 9254f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch set_current_state(TASK_RUNNING); 9264f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch return ret; 9274f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch } 9284f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch unlock_nvmeq(nvmeq); 92978f8d2577bd79ce9d62f4d9e6d3b895bd1dd1d1dKeith Busch schedule_timeout(timeout); 930b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 9313c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox if (cmdinfo.status == -EINTR) { 9324f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch nvmeq = lock_nvmeq(dev, q_idx); 933a51afb54339c5e9ee72df66ae0f2ac5aacfed365Keith Busch if (nvmeq) { 9344f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch nvme_abort_command(nvmeq, cmdid); 935a51afb54339c5e9ee72df66ae0f2ac5aacfed365Keith Busch unlock_nvmeq(nvmeq); 936a51afb54339c5e9ee72df66ae0f2ac5aacfed365Keith Busch } 9373c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox return -EINTR; 9383c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox } 9393c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox 940b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (result) 941b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox *result = cmdinfo.result; 942b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 943b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return cmdinfo.status; 944b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 945b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 9464d115420707afcabe77d2535e092356df6664b70Keith Buschstatic int nvme_submit_async_cmd(struct nvme_queue *nvmeq, 9474d115420707afcabe77d2535e092356df6664b70Keith Busch struct nvme_command *cmd, 9484d115420707afcabe77d2535e092356df6664b70Keith Busch struct async_cmd_info *cmdinfo, unsigned timeout) 9494d115420707afcabe77d2535e092356df6664b70Keith Busch{ 9504d115420707afcabe77d2535e092356df6664b70Keith Busch int cmdid; 9514d115420707afcabe77d2535e092356df6664b70Keith Busch 9524d115420707afcabe77d2535e092356df6664b70Keith Busch cmdid = alloc_cmdid_killable(nvmeq, cmdinfo, async_completion, timeout); 9534d115420707afcabe77d2535e092356df6664b70Keith Busch if (cmdid < 0) 9544d115420707afcabe77d2535e092356df6664b70Keith Busch return cmdid; 9554d115420707afcabe77d2535e092356df6664b70Keith Busch cmdinfo->status = -EINTR; 9564d115420707afcabe77d2535e092356df6664b70Keith Busch cmd->common.command_id = cmdid; 9574f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch return nvme_submit_cmd(nvmeq, cmd); 9584d115420707afcabe77d2535e092356df6664b70Keith Busch} 9594d115420707afcabe77d2535e092356df6664b70Keith Busch 9605d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermaint nvme_submit_admin_cmd(struct nvme_dev *dev, struct nvme_command *cmd, 961b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 *result) 962b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 9634f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch return nvme_submit_sync_cmd(dev, 0, cmd, result, ADMIN_TIMEOUT); 9644f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch} 9654f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch 9664f5099af4f3d5f999d8ab7784472d93e810e3912Keith Buschint nvme_submit_io_cmd(struct nvme_dev *dev, struct nvme_command *cmd, 9674f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch u32 *result) 9684f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch{ 9694f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch return nvme_submit_sync_cmd(dev, smp_processor_id() + 1, cmd, result, 9704f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch NVME_IO_TIMEOUT); 971b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 972b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 9734d115420707afcabe77d2535e092356df6664b70Keith Buschstatic int nvme_submit_admin_cmd_async(struct nvme_dev *dev, 9744d115420707afcabe77d2535e092356df6664b70Keith Busch struct nvme_command *cmd, struct async_cmd_info *cmdinfo) 9754d115420707afcabe77d2535e092356df6664b70Keith Busch{ 9765a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch return nvme_submit_async_cmd(raw_nvmeq(dev, 0), cmd, cmdinfo, 9774d115420707afcabe77d2535e092356df6664b70Keith Busch ADMIN_TIMEOUT); 9784d115420707afcabe77d2535e092356df6664b70Keith Busch} 9794d115420707afcabe77d2535e092356df6664b70Keith Busch 980b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id) 981b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 982b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int status; 983b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_command c; 984b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 985b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset(&c, 0, sizeof(c)); 986b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.delete_queue.opcode = opcode; 987b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.delete_queue.qid = cpu_to_le16(id); 988b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 989b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox status = nvme_submit_admin_cmd(dev, &c, NULL); 990b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (status) 991b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -EIO; 992b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 993b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 994b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 995b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_alloc_cq(struct nvme_dev *dev, u16 qid, 996b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq) 997b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 998b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int status; 999b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_command c; 1000b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int flags = NVME_QUEUE_PHYS_CONTIG | NVME_CQ_IRQ_ENABLED; 1001b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1002b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset(&c, 0, sizeof(c)); 1003b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.opcode = nvme_admin_create_cq; 1004b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.prp1 = cpu_to_le64(nvmeq->cq_dma_addr); 1005b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.cqid = cpu_to_le16(qid); 1006b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1); 1007b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.cq_flags = cpu_to_le16(flags); 1008b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.irq_vector = cpu_to_le16(nvmeq->cq_vector); 1009b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1010b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox status = nvme_submit_admin_cmd(dev, &c, NULL); 1011b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (status) 1012b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -EIO; 1013b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 1014b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1015b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1016b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_alloc_sq(struct nvme_dev *dev, u16 qid, 1017b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq) 1018b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1019b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int status; 1020b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_command c; 1021b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int flags = NVME_QUEUE_PHYS_CONTIG | NVME_SQ_PRIO_MEDIUM; 1022b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1023b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset(&c, 0, sizeof(c)); 1024b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.opcode = nvme_admin_create_sq; 1025b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.prp1 = cpu_to_le64(nvmeq->sq_dma_addr); 1026b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.sqid = cpu_to_le16(qid); 1027b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1); 1028b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.sq_flags = cpu_to_le16(flags); 1029b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.cqid = cpu_to_le16(qid); 1030b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1031b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox status = nvme_submit_admin_cmd(dev, &c, NULL); 1032b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (status) 1033b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -EIO; 1034b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 1035b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1036b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1037b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_delete_cq(struct nvme_dev *dev, u16 cqid) 1038b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1039b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return adapter_delete_queue(dev, nvme_admin_delete_cq, cqid); 1040b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1041b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1042b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_delete_sq(struct nvme_dev *dev, u16 sqid) 1043b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1044b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return adapter_delete_queue(dev, nvme_admin_delete_sq, sqid); 1045b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1046b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 10475d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermaint nvme_identify(struct nvme_dev *dev, unsigned nsid, unsigned cns, 1048bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox dma_addr_t dma_addr) 1049bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox{ 1050bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox struct nvme_command c; 1051bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox 1052bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox memset(&c, 0, sizeof(c)); 1053bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox c.identify.opcode = nvme_admin_identify; 1054bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox c.identify.nsid = cpu_to_le32(nsid); 1055bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox c.identify.prp1 = cpu_to_le64(dma_addr); 1056bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox c.identify.cns = cpu_to_le32(cns); 1057bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox 1058bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox return nvme_submit_admin_cmd(dev, &c, NULL); 1059bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox} 1060bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox 10615d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermaint nvme_get_features(struct nvme_dev *dev, unsigned fid, unsigned nsid, 106208df1e05657fc6712e520e7c09cc6c86160ceb35Keith Busch dma_addr_t dma_addr, u32 *result) 1063bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox{ 1064bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox struct nvme_command c; 1065bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox 1066bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox memset(&c, 0, sizeof(c)); 1067bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox c.features.opcode = nvme_admin_get_features; 1068a42ceccef0c43b46ff6bc1b12a7c1076ef243df1Keith Busch c.features.nsid = cpu_to_le32(nsid); 1069bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox c.features.prp1 = cpu_to_le64(dma_addr); 1070bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox c.features.fid = cpu_to_le32(fid); 1071bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox 107208df1e05657fc6712e520e7c09cc6c86160ceb35Keith Busch return nvme_submit_admin_cmd(dev, &c, result); 1073df3481399042200792822b6243e36a95a557b57eMatthew Wilcox} 1074df3481399042200792822b6243e36a95a557b57eMatthew Wilcox 10755d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermaint nvme_set_features(struct nvme_dev *dev, unsigned fid, unsigned dword11, 10765d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma dma_addr_t dma_addr, u32 *result) 1077df3481399042200792822b6243e36a95a557b57eMatthew Wilcox{ 1078df3481399042200792822b6243e36a95a557b57eMatthew Wilcox struct nvme_command c; 1079df3481399042200792822b6243e36a95a557b57eMatthew Wilcox 1080df3481399042200792822b6243e36a95a557b57eMatthew Wilcox memset(&c, 0, sizeof(c)); 1081df3481399042200792822b6243e36a95a557b57eMatthew Wilcox c.features.opcode = nvme_admin_set_features; 1082df3481399042200792822b6243e36a95a557b57eMatthew Wilcox c.features.prp1 = cpu_to_le64(dma_addr); 1083df3481399042200792822b6243e36a95a557b57eMatthew Wilcox c.features.fid = cpu_to_le32(fid); 1084df3481399042200792822b6243e36a95a557b57eMatthew Wilcox c.features.dword11 = cpu_to_le32(dword11); 1085df3481399042200792822b6243e36a95a557b57eMatthew Wilcox 1086bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox return nvme_submit_admin_cmd(dev, &c, result); 1087bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox} 1088bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox 1089a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox/** 1090c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch * nvme_abort_cmd - Attempt aborting a command 1091c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch * @cmdid: Command id of a timed out IO 1092c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch * @queue: The queue with timed out IO 1093c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch * 1094c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch * Schedule controller reset if the command was already aborted once before and 1095c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch * still hasn't been returned to the driver, or if this is the admin queue. 1096c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch */ 1097c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Buschstatic void nvme_abort_cmd(int cmdid, struct nvme_queue *nvmeq) 1098c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch{ 1099c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch int a_cmdid; 1100c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch struct nvme_command cmd; 1101c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch struct nvme_dev *dev = nvmeq->dev; 1102c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch struct nvme_cmd_info *info = nvme_cmd_info(nvmeq); 11035a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch struct nvme_queue *adminq; 1104c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch 1105c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch if (!nvmeq->qid || info[cmdid].aborted) { 1106c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch if (work_busy(&dev->reset_work)) 1107c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch return; 1108c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch list_del_init(&dev->node); 1109c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch dev_warn(&dev->pci_dev->dev, 1110c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch "I/O %d QID %d timeout, reset controller\n", cmdid, 1111c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch nvmeq->qid); 11129ca9737444f1a8602f74b85018d881e7e54b5bd1Tejun Heo dev->reset_workfn = nvme_reset_failed_dev; 1113c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch queue_work(nvme_workq, &dev->reset_work); 1114c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch return; 1115c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch } 1116c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch 1117c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch if (!dev->abort_limit) 1118c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch return; 1119c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch 11205a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch adminq = rcu_dereference(dev->queues[0]); 11215a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch a_cmdid = alloc_cmdid(adminq, CMD_CTX_ABORT, special_completion, 1122c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch ADMIN_TIMEOUT); 1123c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch if (a_cmdid < 0) 1124c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch return; 1125c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch 1126c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch memset(&cmd, 0, sizeof(cmd)); 1127c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch cmd.abort.opcode = nvme_admin_abort_cmd; 1128c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch cmd.abort.cid = cmdid; 1129c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch cmd.abort.sqid = cpu_to_le16(nvmeq->qid); 1130c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch cmd.abort.command_id = a_cmdid; 1131c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch 1132c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch --dev->abort_limit; 1133c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch info[cmdid].aborted = 1; 1134c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch info[cmdid].timeout = jiffies + ADMIN_TIMEOUT; 1135c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch 1136c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch dev_warn(nvmeq->q_dmadev, "Aborting I/O %d QID %d\n", cmdid, 1137c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch nvmeq->qid); 11385a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch nvme_submit_cmd(adminq, &cmd); 1139c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch} 1140c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch 1141c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch/** 1142a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox * nvme_cancel_ios - Cancel outstanding I/Os 1143a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox * @queue: The queue to cancel I/Os on 1144a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox * @timeout: True to only cancel I/Os which have timed out 1145a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox */ 1146a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcoxstatic void nvme_cancel_ios(struct nvme_queue *nvmeq, bool timeout) 1147a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox{ 1148a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox int depth = nvmeq->q_depth - 1; 1149a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox struct nvme_cmd_info *info = nvme_cmd_info(nvmeq); 1150a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox unsigned long now = jiffies; 1151a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox int cmdid; 1152a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox 1153a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox for_each_set_bit(cmdid, nvmeq->cmdid_data, depth) { 1154a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox void *ctx; 1155a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox nvme_completion_fn fn; 1156a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox static struct nvme_completion cqe = { 1157af2d9ca744af5e03390eeb3864d08ce75c860899Matthew Wilcox .status = cpu_to_le16(NVME_SC_ABORT_REQ << 1), 1158a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox }; 1159a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox 1160a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox if (timeout && !time_after(now, info[cmdid].timeout)) 1161a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox continue; 1162053ab702cc2702f25a97ead087ed344b864785b7Keith Busch if (info[cmdid].ctx == CMD_CTX_CANCELLED) 1163053ab702cc2702f25a97ead087ed344b864785b7Keith Busch continue; 1164c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch if (timeout && nvmeq->dev->initialized) { 1165c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch nvme_abort_cmd(cmdid, nvmeq); 1166c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch continue; 1167c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch } 1168c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch dev_warn(nvmeq->q_dmadev, "Cancelling I/O %d QID %d\n", cmdid, 1169c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch nvmeq->qid); 1170a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox ctx = cancel_cmdid(nvmeq, cmdid, &fn); 1171edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch fn(nvmeq, ctx, &cqe); 1172a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox } 1173a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox} 1174a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox 11755a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Buschstatic void nvme_free_queue(struct rcu_head *r) 11769e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox{ 11775a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch struct nvme_queue *nvmeq = container_of(r, struct nvme_queue, r_head); 11785a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch 1179224042742582c9938788b81165180c876e997a07Keith Busch spin_lock_irq(&nvmeq->q_lock); 1180224042742582c9938788b81165180c876e997a07Keith Busch while (bio_list_peek(&nvmeq->sq_cong)) { 1181224042742582c9938788b81165180c876e997a07Keith Busch struct bio *bio = bio_list_pop(&nvmeq->sq_cong); 1182224042742582c9938788b81165180c876e997a07Keith Busch bio_endio(bio, -EIO); 1183224042742582c9938788b81165180c876e997a07Keith Busch } 1184edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch while (!list_empty(&nvmeq->iod_bio)) { 1185edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch static struct nvme_completion cqe = { 1186edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch .status = cpu_to_le16( 1187edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch (NVME_SC_ABORT_REQ | NVME_SC_DNR) << 1), 1188edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch }; 1189edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch struct nvme_iod *iod = list_first_entry(&nvmeq->iod_bio, 1190edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch struct nvme_iod, 1191edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch node); 1192edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch list_del(&iod->node); 1193edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch bio_completion(nvmeq, iod, &cqe); 1194edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch } 1195224042742582c9938788b81165180c876e997a07Keith Busch spin_unlock_irq(&nvmeq->q_lock); 1196224042742582c9938788b81165180c876e997a07Keith Busch 11979e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth), 11989e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox (void *)nvmeq->cqes, nvmeq->cq_dma_addr); 11999e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth), 12009e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox nvmeq->sq_cmds, nvmeq->sq_dma_addr); 120142f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch if (nvmeq->qid) 120242f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch free_cpumask_var(nvmeq->cpu_mask); 12039e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox kfree(nvmeq); 12049e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox} 12059e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox 1206a1a5ef9998b1379780790b878457b0e0f48b25dbKeith Buschstatic void nvme_free_queues(struct nvme_dev *dev, int lowest) 1207224042742582c9938788b81165180c876e997a07Keith Busch{ 1208224042742582c9938788b81165180c876e997a07Keith Busch int i; 1209224042742582c9938788b81165180c876e997a07Keith Busch 1210a1a5ef9998b1379780790b878457b0e0f48b25dbKeith Busch for (i = dev->queue_count - 1; i >= lowest; i--) { 12115a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch struct nvme_queue *nvmeq = raw_nvmeq(dev, i); 12125a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch rcu_assign_pointer(dev->queues[i], NULL); 12135a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch call_rcu(&nvmeq->r_head, nvme_free_queue); 1214224042742582c9938788b81165180c876e997a07Keith Busch dev->queue_count--; 1215224042742582c9938788b81165180c876e997a07Keith Busch } 1216224042742582c9938788b81165180c876e997a07Keith Busch} 1217224042742582c9938788b81165180c876e997a07Keith Busch 12184d115420707afcabe77d2535e092356df6664b70Keith Busch/** 12194d115420707afcabe77d2535e092356df6664b70Keith Busch * nvme_suspend_queue - put queue into suspended state 12204d115420707afcabe77d2535e092356df6664b70Keith Busch * @nvmeq - queue to suspend 12214d115420707afcabe77d2535e092356df6664b70Keith Busch * 12224d115420707afcabe77d2535e092356df6664b70Keith Busch * Returns 1 if already suspended, 0 otherwise. 12234d115420707afcabe77d2535e092356df6664b70Keith Busch */ 12244d115420707afcabe77d2535e092356df6664b70Keith Buschstatic int nvme_suspend_queue(struct nvme_queue *nvmeq) 1225b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 12264d115420707afcabe77d2535e092356df6664b70Keith Busch int vector = nvmeq->dev->entry[nvmeq->cq_vector].vector; 1227b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1228a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox spin_lock_irq(&nvmeq->q_lock); 1229224042742582c9938788b81165180c876e997a07Keith Busch if (nvmeq->q_suspended) { 1230224042742582c9938788b81165180c876e997a07Keith Busch spin_unlock_irq(&nvmeq->q_lock); 12314d115420707afcabe77d2535e092356df6664b70Keith Busch return 1; 12323295874b6074d749516d6decd43afad7bf6e38ffKeith Busch } 1233224042742582c9938788b81165180c876e997a07Keith Busch nvmeq->q_suspended = 1; 123442f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch nvmeq->dev->online_queues--; 1235a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox spin_unlock_irq(&nvmeq->q_lock); 1236a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox 1237aba2080f3f1639f9202f1a52993669844abcfb80Matthew Wilcox irq_set_affinity_hint(vector, NULL); 1238aba2080f3f1639f9202f1a52993669844abcfb80Matthew Wilcox free_irq(vector, nvmeq); 1239b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 12404d115420707afcabe77d2535e092356df6664b70Keith Busch return 0; 12414d115420707afcabe77d2535e092356df6664b70Keith Busch} 1242b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 12434d115420707afcabe77d2535e092356df6664b70Keith Buschstatic void nvme_clear_queue(struct nvme_queue *nvmeq) 12444d115420707afcabe77d2535e092356df6664b70Keith Busch{ 1245224042742582c9938788b81165180c876e997a07Keith Busch spin_lock_irq(&nvmeq->q_lock); 1246224042742582c9938788b81165180c876e997a07Keith Busch nvme_process_cq(nvmeq); 1247224042742582c9938788b81165180c876e997a07Keith Busch nvme_cancel_ios(nvmeq, false); 1248224042742582c9938788b81165180c876e997a07Keith Busch spin_unlock_irq(&nvmeq->q_lock); 1249b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1250b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 12514d115420707afcabe77d2535e092356df6664b70Keith Buschstatic void nvme_disable_queue(struct nvme_dev *dev, int qid) 12524d115420707afcabe77d2535e092356df6664b70Keith Busch{ 12535a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch struct nvme_queue *nvmeq = raw_nvmeq(dev, qid); 12544d115420707afcabe77d2535e092356df6664b70Keith Busch 12554d115420707afcabe77d2535e092356df6664b70Keith Busch if (!nvmeq) 12564d115420707afcabe77d2535e092356df6664b70Keith Busch return; 12574d115420707afcabe77d2535e092356df6664b70Keith Busch if (nvme_suspend_queue(nvmeq)) 12584d115420707afcabe77d2535e092356df6664b70Keith Busch return; 12594d115420707afcabe77d2535e092356df6664b70Keith Busch 12600e53d18051725da46cbccfb7874a6422d4d4f274Keith Busch /* Don't tell the adapter to delete the admin queue. 12610e53d18051725da46cbccfb7874a6422d4d4f274Keith Busch * Don't tell a removed adapter to delete IO queues. */ 12620e53d18051725da46cbccfb7874a6422d4d4f274Keith Busch if (qid && readl(&dev->bar->csts) != -1) { 1263b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox adapter_delete_sq(dev, qid); 1264b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox adapter_delete_cq(dev, qid); 1265b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 12664d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_clear_queue(nvmeq); 1267b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1268b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1269b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev, int qid, 1270b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int depth, int vector) 1271b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1272b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct device *dmadev = &dev->pci_dev->dev; 1273224042742582c9938788b81165180c876e997a07Keith Busch unsigned extra = nvme_queue_extra(depth); 1274b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq = kzalloc(sizeof(*nvmeq) + extra, GFP_KERNEL); 1275b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!nvmeq) 1276b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 1277b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1278b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->cqes = dma_alloc_coherent(dmadev, CQ_SIZE(depth), 1279b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox &nvmeq->cq_dma_addr, GFP_KERNEL); 1280b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!nvmeq->cqes) 1281b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto free_nvmeq; 1282b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset((void *)nvmeq->cqes, 0, CQ_SIZE(depth)); 1283b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1284b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->sq_cmds = dma_alloc_coherent(dmadev, SQ_SIZE(depth), 1285b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox &nvmeq->sq_dma_addr, GFP_KERNEL); 1286b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!nvmeq->sq_cmds) 1287b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto free_cqdma; 1288b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 128942f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch if (qid && !zalloc_cpumask_var(&nvmeq->cpu_mask, GFP_KERNEL)) 129042f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch goto free_sqdma; 129142f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch 1292b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->q_dmadev = dmadev; 1293091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox nvmeq->dev = dev; 12943193f07bb7ae723f33ac8e8b9db317a4f68d7d18Matthew Wilcox snprintf(nvmeq->irqname, sizeof(nvmeq->irqname), "nvme%dq%d", 12953193f07bb7ae723f33ac8e8b9db317a4f68d7d18Matthew Wilcox dev->instance, qid); 1296b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox spin_lock_init(&nvmeq->q_lock); 1297b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->cq_head = 0; 1298821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox nvmeq->cq_phase = 1; 1299b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox init_waitqueue_head(&nvmeq->sq_full); 13001fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox init_waitqueue_entry(&nvmeq->sq_cong_wait, nvme_thread); 1301b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox bio_list_init(&nvmeq->sq_cong); 1302edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch INIT_LIST_HEAD(&nvmeq->iod_bio); 1303b80d5ccca3a012e91ca64a2a0b13049163a6a698Haiyan Hu nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride]; 1304b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->q_depth = depth; 1305b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->cq_vector = vector; 1306c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch nvmeq->qid = qid; 1307224042742582c9938788b81165180c876e997a07Keith Busch nvmeq->q_suspended = 1; 1308224042742582c9938788b81165180c876e997a07Keith Busch dev->queue_count++; 13095a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch rcu_assign_pointer(dev->queues[qid], nvmeq); 1310b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1311b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return nvmeq; 1312b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 131342f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch free_sqdma: 131442f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch dma_free_coherent(dmadev, SQ_SIZE(depth), (void *)nvmeq->sq_cmds, 131542f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch nvmeq->sq_dma_addr); 1316b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox free_cqdma: 131768b8eca5f882230e4991df0db0a3094f49420a90Keith Busch dma_free_coherent(dmadev, CQ_SIZE(depth), (void *)nvmeq->cqes, 1318b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->cq_dma_addr); 1319b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox free_nvmeq: 1320b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(nvmeq); 1321b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 1322b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1323b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 13243001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcoxstatic int queue_request_irq(struct nvme_dev *dev, struct nvme_queue *nvmeq, 13253001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox const char *name) 13263001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox{ 132758ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox if (use_threaded_interrupts) 132858ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox return request_threaded_irq(dev->entry[nvmeq->cq_vector].vector, 1329481e5bad84239b01415b888df2040c1860ae3cfdMichael Opdenacker nvme_irq_check, nvme_irq, IRQF_SHARED, 133058ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox name, nvmeq); 13313001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox return request_irq(dev->entry[nvmeq->cq_vector].vector, nvme_irq, 1332481e5bad84239b01415b888df2040c1860ae3cfdMichael Opdenacker IRQF_SHARED, name, nvmeq); 13333001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox} 13343001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox 1335224042742582c9938788b81165180c876e997a07Keith Buschstatic void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid) 1336b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1337224042742582c9938788b81165180c876e997a07Keith Busch struct nvme_dev *dev = nvmeq->dev; 1338224042742582c9938788b81165180c876e997a07Keith Busch unsigned extra = nvme_queue_extra(nvmeq->q_depth); 1339b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1340224042742582c9938788b81165180c876e997a07Keith Busch nvmeq->sq_tail = 0; 1341224042742582c9938788b81165180c876e997a07Keith Busch nvmeq->cq_head = 0; 1342224042742582c9938788b81165180c876e997a07Keith Busch nvmeq->cq_phase = 1; 1343b80d5ccca3a012e91ca64a2a0b13049163a6a698Haiyan Hu nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride]; 1344224042742582c9938788b81165180c876e997a07Keith Busch memset(nvmeq->cmdid_data, 0, extra); 1345224042742582c9938788b81165180c876e997a07Keith Busch memset((void *)nvmeq->cqes, 0, CQ_SIZE(nvmeq->q_depth)); 1346224042742582c9938788b81165180c876e997a07Keith Busch nvme_cancel_ios(nvmeq, false); 1347224042742582c9938788b81165180c876e997a07Keith Busch nvmeq->q_suspended = 0; 134842f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch dev->online_queues++; 1349224042742582c9938788b81165180c876e997a07Keith Busch} 1350224042742582c9938788b81165180c876e997a07Keith Busch 1351224042742582c9938788b81165180c876e997a07Keith Buschstatic int nvme_create_queue(struct nvme_queue *nvmeq, int qid) 1352224042742582c9938788b81165180c876e997a07Keith Busch{ 1353224042742582c9938788b81165180c876e997a07Keith Busch struct nvme_dev *dev = nvmeq->dev; 1354224042742582c9938788b81165180c876e997a07Keith Busch int result; 13553f85d50b609e8a5ef151656210203a6e94c19538Matthew Wilcox 1356b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox result = adapter_alloc_cq(dev, qid, nvmeq); 1357b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (result < 0) 1358224042742582c9938788b81165180c876e997a07Keith Busch return result; 1359b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1360b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox result = adapter_alloc_sq(dev, qid, nvmeq); 1361b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (result < 0) 1362b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto release_cq; 1363b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 13643193f07bb7ae723f33ac8e8b9db317a4f68d7d18Matthew Wilcox result = queue_request_irq(dev, nvmeq, nvmeq->irqname); 1365b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (result < 0) 1366b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto release_sq; 1367b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 13680a8d44cb33377969337fa6c1961b631605d5f453Matthew Wilcox spin_lock_irq(&nvmeq->q_lock); 1369224042742582c9938788b81165180c876e997a07Keith Busch nvme_init_queue(nvmeq, qid); 13700a8d44cb33377969337fa6c1961b631605d5f453Matthew Wilcox spin_unlock_irq(&nvmeq->q_lock); 1371224042742582c9938788b81165180c876e997a07Keith Busch 1372224042742582c9938788b81165180c876e997a07Keith Busch return result; 1373b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1374b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox release_sq: 1375b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox adapter_delete_sq(dev, qid); 1376b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox release_cq: 1377b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox adapter_delete_cq(dev, qid); 1378224042742582c9938788b81165180c876e997a07Keith Busch return result; 1379b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1380b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1381ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcoxstatic int nvme_wait_ready(struct nvme_dev *dev, u64 cap, bool enabled) 1382ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox{ 1383ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox unsigned long timeout; 1384ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox u32 bit = enabled ? NVME_CSTS_RDY : 0; 1385ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox 1386ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox timeout = ((NVME_CAP_TIMEOUT(cap) + 1) * HZ / 2) + jiffies; 1387ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox 1388ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox while ((readl(&dev->bar->csts) & NVME_CSTS_RDY) != bit) { 1389ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox msleep(100); 1390ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox if (fatal_signal_pending(current)) 1391ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox return -EINTR; 1392ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox if (time_after(jiffies, timeout)) { 1393ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox dev_err(&dev->pci_dev->dev, 139427e8166c31656f7780e682eaf6bc9c3b8dd03253Matthew Wilcox "Device not ready; aborting %s\n", enabled ? 139527e8166c31656f7780e682eaf6bc9c3b8dd03253Matthew Wilcox "initialisation" : "reset"); 1396ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox return -ENODEV; 1397ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox } 1398ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox } 1399ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox 1400ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox return 0; 1401ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox} 1402ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox 1403ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox/* 1404ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox * If the device has been passed off to us in an enabled state, just clear 1405ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox * the enabled bit. The spec says we should set the 'shutdown notification 1406ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox * bits', but doing so may cause the device to complete commands to the 1407ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox * admin queue ... and we don't know what memory that might be pointing at! 1408ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox */ 1409ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcoxstatic int nvme_disable_ctrl(struct nvme_dev *dev, u64 cap) 1410ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox{ 141144af146a84fa4a8e136d824207dcd356958a112bMatthew Wilcox u32 cc = readl(&dev->bar->cc); 141244af146a84fa4a8e136d824207dcd356958a112bMatthew Wilcox 141344af146a84fa4a8e136d824207dcd356958a112bMatthew Wilcox if (cc & NVME_CC_ENABLE) 141444af146a84fa4a8e136d824207dcd356958a112bMatthew Wilcox writel(cc & ~NVME_CC_ENABLE, &dev->bar->cc); 1415ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox return nvme_wait_ready(dev, cap, false); 1416ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox} 1417ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox 1418ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcoxstatic int nvme_enable_ctrl(struct nvme_dev *dev, u64 cap) 1419ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox{ 1420ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox return nvme_wait_ready(dev, cap, true); 1421ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox} 1422ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox 14231894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Buschstatic int nvme_shutdown_ctrl(struct nvme_dev *dev) 14241894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch{ 14251894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch unsigned long timeout; 14261894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch u32 cc; 14271894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch 14281894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch cc = (readl(&dev->bar->cc) & ~NVME_CC_SHN_MASK) | NVME_CC_SHN_NORMAL; 14291894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch writel(cc, &dev->bar->cc); 14301894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch 14311894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch timeout = 2 * HZ + jiffies; 14321894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch while ((readl(&dev->bar->csts) & NVME_CSTS_SHST_MASK) != 14331894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch NVME_CSTS_SHST_CMPLT) { 14341894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch msleep(100); 14351894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch if (fatal_signal_pending(current)) 14361894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch return -EINTR; 14371894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch if (time_after(jiffies, timeout)) { 14381894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch dev_err(&dev->pci_dev->dev, 14391894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch "Device shutdown incomplete; abort shutdown\n"); 14401894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch return -ENODEV; 14411894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch } 14421894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch } 14431894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch 14441894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch return 0; 14451894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch} 14461894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch 14478d85fce77edfc22f1d6dbf78e3af723b4b556f3dGreg Kroah-Hartmanstatic int nvme_configure_admin_queue(struct nvme_dev *dev) 1448b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1449ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox int result; 1450b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 aqa; 1451ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox u64 cap = readq(&dev->bar->cap); 1452b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq; 1453b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1454ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox result = nvme_disable_ctrl(dev, cap); 1455ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox if (result < 0) 1456ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox return result; 1457b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 14585a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch nvmeq = raw_nvmeq(dev, 0); 1459cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch if (!nvmeq) { 1460cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch nvmeq = nvme_alloc_queue(dev, 0, 64, 0); 1461cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch if (!nvmeq) 1462cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch return -ENOMEM; 1463cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch } 1464b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1465b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox aqa = nvmeq->q_depth - 1; 1466b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox aqa |= aqa << 16; 1467b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1468b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->ctrl_config = NVME_CC_ENABLE | NVME_CC_CSS_NVM; 1469b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->ctrl_config |= (PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT; 1470b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->ctrl_config |= NVME_CC_ARB_RR | NVME_CC_SHN_NONE; 14717f53f9d2424533256ae86f7df5661a17de743de8Matthew Wilcox dev->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES; 1472b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1473b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox writel(aqa, &dev->bar->aqa); 1474b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox writeq(nvmeq->sq_dma_addr, &dev->bar->asq); 1475b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox writeq(nvmeq->cq_dma_addr, &dev->bar->acq); 1476b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox writel(dev->ctrl_config, &dev->bar->cc); 1477b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1478ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox result = nvme_enable_ctrl(dev, cap); 1479025c557a71bd06a9f6d32259f00e2218b15bf0a4Keith Busch if (result) 1480cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch return result; 14819e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox 14823193f07bb7ae723f33ac8e8b9db317a4f68d7d18Matthew Wilcox result = queue_request_irq(dev, nvmeq, nvmeq->irqname); 1483025c557a71bd06a9f6d32259f00e2218b15bf0a4Keith Busch if (result) 1484cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch return result; 1485025c557a71bd06a9f6d32259f00e2218b15bf0a4Keith Busch 14860a8d44cb33377969337fa6c1961b631605d5f453Matthew Wilcox spin_lock_irq(&nvmeq->q_lock); 1487224042742582c9938788b81165180c876e997a07Keith Busch nvme_init_queue(nvmeq, 0); 14880a8d44cb33377969337fa6c1961b631605d5f453Matthew Wilcox spin_unlock_irq(&nvmeq->q_lock); 1489b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return result; 1490b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1491b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 14925d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermastruct nvme_iod *nvme_map_user_pages(struct nvme_dev *dev, int write, 1493eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox unsigned long addr, unsigned length) 1494b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 149536c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox int i, err, count, nents, offset; 14967fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox struct scatterlist *sg; 14977fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox struct page **pages; 1498eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox struct nvme_iod *iod; 149936c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox 150036c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox if (addr & 3) 1501eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return ERR_PTR(-EINVAL); 15025460fc03105fbed01fe27aa572d9f65bb410a61dDan Carpenter if (!length || length > INT_MAX - PAGE_SIZE) 1503eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return ERR_PTR(-EINVAL); 15047fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox 150536c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox offset = offset_in_page(addr); 15067fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox count = DIV_ROUND_UP(offset + length, PAGE_SIZE); 15077fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox pages = kcalloc(count, sizeof(*pages), GFP_KERNEL); 150822fff826e715e9727d3c7a69f15e602a9801b673Dan Carpenter if (!pages) 150922fff826e715e9727d3c7a69f15e602a9801b673Dan Carpenter return ERR_PTR(-ENOMEM); 151036c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox 151136c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox err = get_user_pages_fast(addr, count, 1, pages); 151236c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox if (err < count) { 151336c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox count = err; 151436c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox err = -EFAULT; 151536c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox goto put_pages; 151636c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox } 15177fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox 15186808c5fb7fc1574c7608a38c9819f1639d89c3d0Santosh Y err = -ENOMEM; 1519eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod = nvme_alloc_iod(count, length, GFP_KERNEL); 15206808c5fb7fc1574c7608a38c9819f1639d89c3d0Santosh Y if (!iod) 15216808c5fb7fc1574c7608a38c9819f1639d89c3d0Santosh Y goto put_pages; 15226808c5fb7fc1574c7608a38c9819f1639d89c3d0Santosh Y 1523eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox sg = iod->sg; 152436c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox sg_init_table(sg, count); 1525d0ba1e497bca83a3d353eb47c9658afc54d83228Matthew Wilcox for (i = 0; i < count; i++) { 1526d0ba1e497bca83a3d353eb47c9658afc54d83228Matthew Wilcox sg_set_page(&sg[i], pages[i], 15275460fc03105fbed01fe27aa572d9f65bb410a61dDan Carpenter min_t(unsigned, length, PAGE_SIZE - offset), 15285460fc03105fbed01fe27aa572d9f65bb410a61dDan Carpenter offset); 1529d0ba1e497bca83a3d353eb47c9658afc54d83228Matthew Wilcox length -= (PAGE_SIZE - offset); 1530d0ba1e497bca83a3d353eb47c9658afc54d83228Matthew Wilcox offset = 0; 15317fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox } 1532fe304c43c6d63e29ed4fc46a874d7a74313788c5Matthew Wilcox sg_mark_end(&sg[i - 1]); 15331c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox iod->nents = count; 15347fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox 15357fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox nents = dma_map_sg(&dev->pci_dev->dev, sg, count, 15367fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox write ? DMA_TO_DEVICE : DMA_FROM_DEVICE); 153736c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox if (!nents) 1538eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox goto free_iod; 1539b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 15407fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox kfree(pages); 1541eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return iod; 1542b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1543eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox free_iod: 1544eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox kfree(iod); 15457fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox put_pages: 15467fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox for (i = 0; i < count; i++) 15477fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox put_page(pages[i]); 15487fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox kfree(pages); 1549eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return ERR_PTR(err); 15507fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox} 1551b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 15525d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermavoid nvme_unmap_user_pages(struct nvme_dev *dev, int write, 15531c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox struct nvme_iod *iod) 15547fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox{ 15551c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox int i; 1556b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 15571c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox dma_unmap_sg(&dev->pci_dev->dev, iod->sg, iod->nents, 15581c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox write ? DMA_TO_DEVICE : DMA_FROM_DEVICE); 15597fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox 15601c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox for (i = 0; i < iod->nents; i++) 15611c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox put_page(sg_page(&iod->sg[i])); 15627fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox} 1563b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1564a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcoxstatic int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio) 1565a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox{ 1566a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox struct nvme_dev *dev = ns->dev; 1567a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox struct nvme_user_io io; 1568a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox struct nvme_command c; 1569f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch unsigned length, meta_len; 1570f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch int status, i; 1571f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch struct nvme_iod *iod, *meta_iod = NULL; 1572f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch dma_addr_t meta_dma_addr; 1573f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch void *meta, *uninitialized_var(meta_mem); 1574a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox 1575a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox if (copy_from_user(&io, uio, sizeof(io))) 1576a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox return -EFAULT; 15776c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox length = (io.nblocks + 1) << ns->lba_shift; 1578f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_len = (io.nblocks + 1) * ns->ms; 1579f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1580f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch if (meta_len && ((io.metadata & 3) || !io.metadata)) 1581f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch return -EINVAL; 15826c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox 15836c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox switch (io.opcode) { 15846c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox case nvme_cmd_write: 15856c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox case nvme_cmd_read: 15866bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox case nvme_cmd_compare: 1587eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod = nvme_map_user_pages(dev, io.opcode & 1, io.addr, length); 15886413214c5d424fd5aae6567848340f962ad2ce0fMatthew Wilcox break; 15896c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox default: 15906bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox return -EINVAL; 15916c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox } 15926c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox 1593eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox if (IS_ERR(iod)) 1594eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return PTR_ERR(iod); 1595a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox 1596a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox memset(&c, 0, sizeof(c)); 1597a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.opcode = io.opcode; 1598a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.flags = io.flags; 15996c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox c.rw.nsid = cpu_to_le32(ns->ns_id); 1600a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.slba = cpu_to_le64(io.slba); 16016c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox c.rw.length = cpu_to_le16(io.nblocks); 1602a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.control = cpu_to_le16(io.control); 16031c9b52651dad0ff1fa71fc6205c86d972f25bcc0Matthew Wilcox c.rw.dsmgmt = cpu_to_le32(io.dsmgmt); 16041c9b52651dad0ff1fa71fc6205c86d972f25bcc0Matthew Wilcox c.rw.reftag = cpu_to_le32(io.reftag); 16051c9b52651dad0ff1fa71fc6205c86d972f25bcc0Matthew Wilcox c.rw.apptag = cpu_to_le16(io.apptag); 16061c9b52651dad0ff1fa71fc6205c86d972f25bcc0Matthew Wilcox c.rw.appmask = cpu_to_le16(io.appmask); 1607f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1608f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch if (meta_len) { 16091b56749e541ad59068582f2a28297843e243b856Keith Busch meta_iod = nvme_map_user_pages(dev, io.opcode & 1, io.metadata, 16101b56749e541ad59068582f2a28297843e243b856Keith Busch meta_len); 1611f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch if (IS_ERR(meta_iod)) { 1612f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch status = PTR_ERR(meta_iod); 1613f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_iod = NULL; 1614f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch goto unmap; 1615f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch } 1616f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1617f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_mem = dma_alloc_coherent(&dev->pci_dev->dev, meta_len, 1618f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch &meta_dma_addr, GFP_KERNEL); 1619f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch if (!meta_mem) { 1620f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch status = -ENOMEM; 1621f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch goto unmap; 1622f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch } 1623f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1624f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch if (io.opcode & 1) { 1625f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch int meta_offset = 0; 1626f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1627f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch for (i = 0; i < meta_iod->nents; i++) { 1628f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta = kmap_atomic(sg_page(&meta_iod->sg[i])) + 1629f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_iod->sg[i].offset; 1630f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch memcpy(meta_mem + meta_offset, meta, 1631f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_iod->sg[i].length); 1632f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch kunmap_atomic(meta); 1633f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_offset += meta_iod->sg[i].length; 1634f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch } 1635f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch } 1636f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1637f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch c.rw.metadata = cpu_to_le64(meta_dma_addr); 1638f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch } 1639f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1640edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch length = nvme_setup_prps(dev, iod, length, GFP_KERNEL); 1641edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch c.rw.prp1 = cpu_to_le64(sg_dma_address(iod->sg)); 1642edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch c.rw.prp2 = cpu_to_le64(iod->first_dma); 1643a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox 1644b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox if (length != (io.nblocks + 1) << ns->lba_shift) 1645b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox status = -ENOMEM; 1646b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox else 16474f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch status = nvme_submit_io_cmd(dev, &c, NULL); 1648a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox 1649f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch if (meta_len) { 1650f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch if (status == NVME_SC_SUCCESS && !(io.opcode & 1)) { 1651f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch int meta_offset = 0; 1652f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1653f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch for (i = 0; i < meta_iod->nents; i++) { 1654f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta = kmap_atomic(sg_page(&meta_iod->sg[i])) + 1655f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_iod->sg[i].offset; 1656f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch memcpy(meta, meta_mem + meta_offset, 1657f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_iod->sg[i].length); 1658f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch kunmap_atomic(meta); 1659f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_offset += meta_iod->sg[i].length; 1660f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch } 1661f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch } 1662f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1663f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch dma_free_coherent(&dev->pci_dev->dev, meta_len, meta_mem, 1664f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_dma_addr); 1665f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch } 1666f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1667f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch unmap: 16681c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox nvme_unmap_user_pages(dev, io.opcode & 1, iod); 1669eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox nvme_free_iod(dev, iod); 1670f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1671f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch if (meta_iod) { 1672f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch nvme_unmap_user_pages(dev, io.opcode & 1, meta_iod); 1673f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch nvme_free_iod(dev, meta_iod); 1674f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch } 1675f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1676a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox return status; 1677a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox} 1678a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox 167950af8baec46a99a9b81a4600c0374f83a5a590a9Keith Buschstatic int nvme_user_admin_cmd(struct nvme_dev *dev, 16806bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox struct nvme_admin_cmd __user *ucmd) 16816ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox{ 16826bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox struct nvme_admin_cmd cmd; 16836ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox struct nvme_command c; 1684eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox int status, length; 1685c7d36ab8fa04c213328119a9c0d66985fe204ee5Keith Busch struct nvme_iod *uninitialized_var(iod); 168694f370cab6e5ac514b658c6b2b3aa308cefc5c7aKeith Busch unsigned timeout; 16876ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox 16886bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox if (!capable(CAP_SYS_ADMIN)) 16896bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox return -EACCES; 16906bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox if (copy_from_user(&cmd, ucmd, sizeof(cmd))) 16916ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox return -EFAULT; 16926ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox 16936ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox memset(&c, 0, sizeof(c)); 16946bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.opcode = cmd.opcode; 16956bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.flags = cmd.flags; 16966bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.nsid = cpu_to_le32(cmd.nsid); 16976bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw2[0] = cpu_to_le32(cmd.cdw2); 16986bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw2[1] = cpu_to_le32(cmd.cdw3); 16996bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw10[0] = cpu_to_le32(cmd.cdw10); 17006bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw10[1] = cpu_to_le32(cmd.cdw11); 17016bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw10[2] = cpu_to_le32(cmd.cdw12); 17026bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw10[3] = cpu_to_le32(cmd.cdw13); 17036bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw10[4] = cpu_to_le32(cmd.cdw14); 17046bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw10[5] = cpu_to_le32(cmd.cdw15); 17056bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox 17066bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox length = cmd.data_len; 17076bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox if (cmd.data_len) { 1708497421880acecd0281d3182d534f3d28c927caecMatthew Wilcox iod = nvme_map_user_pages(dev, cmd.opcode & 1, cmd.addr, 1709497421880acecd0281d3182d534f3d28c927caecMatthew Wilcox length); 1710eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox if (IS_ERR(iod)) 1711eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return PTR_ERR(iod); 1712edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch length = nvme_setup_prps(dev, iod, length, GFP_KERNEL); 1713edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch c.common.prp1 = cpu_to_le64(sg_dma_address(iod->sg)); 1714edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch c.common.prp2 = cpu_to_le64(iod->first_dma); 17156bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox } 17166bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox 171794f370cab6e5ac514b658c6b2b3aa308cefc5c7aKeith Busch timeout = cmd.timeout_ms ? msecs_to_jiffies(cmd.timeout_ms) : 171894f370cab6e5ac514b658c6b2b3aa308cefc5c7aKeith Busch ADMIN_TIMEOUT; 17196bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox if (length != cmd.data_len) 1720b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox status = -ENOMEM; 1721b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox else 17224f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch status = nvme_submit_sync_cmd(dev, 0, &c, &cmd.result, timeout); 1723eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox 17246bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox if (cmd.data_len) { 17251c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox nvme_unmap_user_pages(dev, cmd.opcode & 1, iod); 1726eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox nvme_free_iod(dev, iod); 17276bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox } 1728f4f117f64baf8840d22266d518227b2a186d294bKeith Busch 1729cf90bc4830b858487fe4b9b9ecd0031e23ca3e83Chayan Biswas if ((status >= 0) && copy_to_user(&ucmd->result, &cmd.result, 1730f4f117f64baf8840d22266d518227b2a186d294bKeith Busch sizeof(cmd.result))) 1731f4f117f64baf8840d22266d518227b2a186d294bKeith Busch status = -EFAULT; 1732f4f117f64baf8840d22266d518227b2a186d294bKeith Busch 17336ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox return status; 17346ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox} 17356ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox 1736b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, 1737b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unsigned long arg) 1738b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1739b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_ns *ns = bdev->bd_disk->private_data; 1740b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1741b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox switch (cmd) { 17426bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox case NVME_IOCTL_ID: 1743c3bfe7176c035a0a2c70bc79180fb13a6c57142aMatthew Wilcox force_successful_syscall_return(); 17446bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox return ns->ns_id; 17456bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox case NVME_IOCTL_ADMIN_CMD: 174650af8baec46a99a9b81a4600c0374f83a5a590a9Keith Busch return nvme_user_admin_cmd(ns->dev, (void __user *)arg); 1747a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox case NVME_IOCTL_SUBMIT_IO: 1748a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox return nvme_submit_io(ns, (void __user *)arg); 17495d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma case SG_GET_VERSION_NUM: 17505d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma return nvme_sg_get_version_num((void __user *)arg); 17515d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma case SG_IO: 17525d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma return nvme_sg_io(ns, (void __user *)arg); 1753b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox default: 1754b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -ENOTTY; 1755b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 1756b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1757b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1758320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch#ifdef CONFIG_COMPAT 1759320a382746e0ab1304476ea7e986a8d416ab99dbKeith Buschstatic int nvme_compat_ioctl(struct block_device *bdev, fmode_t mode, 1760320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch unsigned int cmd, unsigned long arg) 1761320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch{ 1762320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch struct nvme_ns *ns = bdev->bd_disk->private_data; 1763320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch 1764320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch switch (cmd) { 1765320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch case SG_IO: 1766320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch return nvme_sg_io32(ns, arg); 1767320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch } 1768320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch return nvme_ioctl(bdev, mode, cmd, arg); 1769320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch} 1770320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch#else 1771320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch#define nvme_compat_ioctl NULL 1772320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch#endif 1773320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch 17749ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Buschstatic int nvme_open(struct block_device *bdev, fmode_t mode) 17759ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch{ 17769ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch struct nvme_ns *ns = bdev->bd_disk->private_data; 17779ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch struct nvme_dev *dev = ns->dev; 17789ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch 17799ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch kref_get(&dev->kref); 17809ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch return 0; 17819ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch} 17829ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch 17839ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Buschstatic void nvme_free_dev(struct kref *kref); 17849ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch 17859ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Buschstatic void nvme_release(struct gendisk *disk, fmode_t mode) 17869ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch{ 17879ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch struct nvme_ns *ns = disk->private_data; 17889ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch struct nvme_dev *dev = ns->dev; 17899ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch 17909ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch kref_put(&dev->kref, nvme_free_dev); 17919ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch} 17929ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch 17934cc09e2dc4cbe6009c935b6f12a8376f09124bc5Keith Buschstatic int nvme_getgeo(struct block_device *bd, struct hd_geometry *geo) 17944cc09e2dc4cbe6009c935b6f12a8376f09124bc5Keith Busch{ 17954cc09e2dc4cbe6009c935b6f12a8376f09124bc5Keith Busch /* some standard values */ 17964cc09e2dc4cbe6009c935b6f12a8376f09124bc5Keith Busch geo->heads = 1 << 6; 17974cc09e2dc4cbe6009c935b6f12a8376f09124bc5Keith Busch geo->sectors = 1 << 5; 17984cc09e2dc4cbe6009c935b6f12a8376f09124bc5Keith Busch geo->cylinders = get_capacity(bd->bd_disk) >> 11; 17994cc09e2dc4cbe6009c935b6f12a8376f09124bc5Keith Busch return 0; 18004cc09e2dc4cbe6009c935b6f12a8376f09124bc5Keith Busch} 18014cc09e2dc4cbe6009c935b6f12a8376f09124bc5Keith Busch 1802b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic const struct block_device_operations nvme_fops = { 1803b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .owner = THIS_MODULE, 1804b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .ioctl = nvme_ioctl, 1805320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch .compat_ioctl = nvme_compat_ioctl, 18069ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch .open = nvme_open, 18079ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch .release = nvme_release, 18084cc09e2dc4cbe6009c935b6f12a8376f09124bc5Keith Busch .getgeo = nvme_getgeo, 1809b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 1810b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1811edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Buschstatic void nvme_resubmit_iods(struct nvme_queue *nvmeq) 1812edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch{ 1813edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch struct nvme_iod *iod, *next; 1814edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch 1815edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch list_for_each_entry_safe(iod, next, &nvmeq->iod_bio, node) { 1816edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch if (unlikely(nvme_submit_iod(nvmeq, iod))) 1817edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch break; 1818edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch list_del(&iod->node); 1819edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch if (bio_list_empty(&nvmeq->sq_cong) && 1820edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch list_empty(&nvmeq->iod_bio)) 1821edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch remove_wait_queue(&nvmeq->sq_full, 1822edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch &nvmeq->sq_cong_wait); 1823edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch } 1824edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch} 1825edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch 18261fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcoxstatic void nvme_resubmit_bios(struct nvme_queue *nvmeq) 18271fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox{ 18281fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox while (bio_list_peek(&nvmeq->sq_cong)) { 18291fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox struct bio *bio = bio_list_pop(&nvmeq->sq_cong); 18301fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox struct nvme_ns *ns = bio->bi_bdev->bd_disk->private_data; 1831427e97080196548557b288517537ab7eb48c309fKeith Busch 1832edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch if (bio_list_empty(&nvmeq->sq_cong) && 1833edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch list_empty(&nvmeq->iod_bio)) 1834427e97080196548557b288517537ab7eb48c309fKeith Busch remove_wait_queue(&nvmeq->sq_full, 1835427e97080196548557b288517537ab7eb48c309fKeith Busch &nvmeq->sq_cong_wait); 18361fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox if (nvme_submit_bio_queue(nvmeq, ns, bio)) { 1837edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch if (!waitqueue_active(&nvmeq->sq_full)) 1838427e97080196548557b288517537ab7eb48c309fKeith Busch add_wait_queue(&nvmeq->sq_full, 1839427e97080196548557b288517537ab7eb48c309fKeith Busch &nvmeq->sq_cong_wait); 18401fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox bio_list_add_head(&nvmeq->sq_cong, bio); 18411fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox break; 18421fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox } 18431fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox } 18441fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox} 18451fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 18461fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcoxstatic int nvme_kthread(void *data) 18471fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox{ 1848d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch struct nvme_dev *dev, *next; 18491fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 18501fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox while (!kthread_should_stop()) { 1851564a232c059913d91b491e04c2b2d670b8f94615Arjan van de Ven set_current_state(TASK_INTERRUPTIBLE); 18521fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_lock(&dev_list_lock); 1853d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch list_for_each_entry_safe(dev, next, &dev_list, node) { 18541fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox int i; 1855d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch if (readl(&dev->bar->csts) & NVME_CSTS_CFS && 1856d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch dev->initialized) { 1857d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch if (work_busy(&dev->reset_work)) 1858d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch continue; 1859d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch list_del_init(&dev->node); 1860d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch dev_warn(&dev->pci_dev->dev, 1861d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch "Failed status, reset controller\n"); 18629ca9737444f1a8602f74b85018d881e7e54b5bd1Tejun Heo dev->reset_workfn = nvme_reset_failed_dev; 1863d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch queue_work(nvme_workq, &dev->reset_work); 1864d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch continue; 1865d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch } 18665a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch rcu_read_lock(); 18671fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox for (i = 0; i < dev->queue_count; i++) { 18685a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch struct nvme_queue *nvmeq = 18695a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch rcu_dereference(dev->queues[i]); 1870740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox if (!nvmeq) 1871740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox continue; 18721fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_lock_irq(&nvmeq->q_lock); 1873224042742582c9938788b81165180c876e997a07Keith Busch if (nvmeq->q_suspended) 1874224042742582c9938788b81165180c876e997a07Keith Busch goto unlock; 1875bc57a0f7a44cfcf3e9873f6c6b8dcecdca486b1fMatthew Wilcox nvme_process_cq(nvmeq); 1876a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox nvme_cancel_ios(nvmeq, true); 18771fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox nvme_resubmit_bios(nvmeq); 1878edd10d33283899fb15d99a290dcc9ceb3604ca78Keith Busch nvme_resubmit_iods(nvmeq); 1879224042742582c9938788b81165180c876e997a07Keith Busch unlock: 18801fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_unlock_irq(&nvmeq->q_lock); 18811fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox } 18825a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch rcu_read_unlock(); 18831fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox } 18841fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_unlock(&dev_list_lock); 1885acb7aa0db09b8abd38abeb84334a8a27a52fbb1bArjan van de Ven schedule_timeout(round_jiffies_relative(HZ)); 18861fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox } 18871fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox return 0; 18881fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox} 18891fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 18900e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Buschstatic void nvme_config_discard(struct nvme_ns *ns) 18910e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch{ 18920e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch u32 logical_block_size = queue_logical_block_size(ns->queue); 18930e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch ns->queue->limits.discard_zeroes_data = 0; 18940e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch ns->queue->limits.discard_alignment = logical_block_size; 18950e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch ns->queue->limits.discard_granularity = logical_block_size; 18960e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch ns->queue->limits.max_discard_sectors = 0xffffffff; 18970e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, ns->queue); 18980e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch} 18990e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch 1900c3bfe7176c035a0a2c70bc79180fb13a6c57142aMatthew Wilcoxstatic struct nvme_ns *nvme_alloc_ns(struct nvme_dev *dev, unsigned nsid, 1901b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_id_ns *id, struct nvme_lba_range_type *rt) 1902b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1903b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_ns *ns; 1904b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct gendisk *disk; 1905b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int lbaf; 1906b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1907b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (rt->attributes & NVME_LBART_ATTRIB_HIDE) 1908b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 1909b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1910b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns = kzalloc(sizeof(*ns), GFP_KERNEL); 1911b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!ns) 1912b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 1913b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->queue = blk_alloc_queue(GFP_KERNEL); 1914b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!ns->queue) 1915b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto out_free_ns; 19164eeb9215a0d5c9494ca8b20158cc8ee82618840cMatthew Wilcox ns->queue->queue_flags = QUEUE_FLAG_DEFAULT; 19174eeb9215a0d5c9494ca8b20158cc8ee82618840cMatthew Wilcox queue_flag_set_unlocked(QUEUE_FLAG_NOMERGES, ns->queue); 19184eeb9215a0d5c9494ca8b20158cc8ee82618840cMatthew Wilcox queue_flag_set_unlocked(QUEUE_FLAG_NONROT, ns->queue); 1919b277da0a8a594308e17881f4926879bd5fca2a2dMike Snitzer queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, ns->queue); 1920b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox blk_queue_make_request(ns->queue, nvme_make_request); 1921b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->dev = dev; 1922b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->queue->queuedata = ns; 1923b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1924469071a37afc8a627b6b2ddf29db0a097d864845Matthew Wilcox disk = alloc_disk(0); 1925b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!disk) 1926b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto out_free_queue; 19275aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox ns->ns_id = nsid; 1928b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->disk = disk; 1929b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox lbaf = id->flbas & 0xf; 1930b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->lba_shift = id->lbaf[lbaf].ds; 1931f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch ns->ms = le16_to_cpu(id->lbaf[lbaf].ms); 1932e9ef46369f5107e634a93b7fc4e62a1f53343197Keith Busch blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift); 19338fc23e032debd682f5ba9fc524a5846c10d2c522Keith Busch if (dev->max_hw_sectors) 19348fc23e032debd682f5ba9fc524a5846c10d2c522Keith Busch blk_queue_max_hw_sectors(ns->queue, dev->max_hw_sectors); 1935a7d2ce2832d84e0182585f63bf96ca7323b3aee7Keith Busch if (dev->vwc & NVME_CTRL_VWC_PRESENT) 1936a7d2ce2832d84e0182585f63bf96ca7323b3aee7Keith Busch blk_queue_flush(ns->queue, REQ_FLUSH | REQ_FUA); 1937b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1938b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk->major = nvme_major; 1939469071a37afc8a627b6b2ddf29db0a097d864845Matthew Wilcox disk->first_minor = 0; 1940b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk->fops = &nvme_fops; 1941b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk->private_data = ns; 1942b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk->queue = ns->queue; 1943388f037f4e7f0a24bac6b1a24f144f5d939f58cfMatthew Wilcox disk->driverfs_dev = &dev->pci_dev->dev; 1944469071a37afc8a627b6b2ddf29db0a097d864845Matthew Wilcox disk->flags = GENHD_FL_EXT_DEVT; 19455aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox sprintf(disk->disk_name, "nvme%dn%d", dev->instance, nsid); 1946b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox set_capacity(disk, le64_to_cpup(&id->nsze) << (ns->lba_shift - 9)); 1947b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 19480e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch if (dev->oncs & NVME_CTRL_ONCS_DSM) 19490e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch nvme_config_discard(ns); 19500e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch 1951b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return ns; 1952b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1953b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox out_free_queue: 1954b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox blk_cleanup_queue(ns->queue); 1955b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox out_free_ns: 1956b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(ns); 1957b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 1958b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1959b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 196042f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Buschstatic int nvme_find_closest_node(int node) 196142f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch{ 196242f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch int n, val, min_val = INT_MAX, best_node = node; 196342f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch 196442f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch for_each_online_node(n) { 196542f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch if (n == node) 196642f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch continue; 196742f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch val = node_distance(node, n); 196842f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch if (val < min_val) { 196942f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch min_val = val; 197042f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch best_node = n; 197142f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch } 197242f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch } 197342f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch return best_node; 197442f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch} 197542f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch 197642f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Buschstatic void nvme_set_queue_cpus(cpumask_t *qmask, struct nvme_queue *nvmeq, 197742f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch int count) 197842f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch{ 197942f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch int cpu; 198042f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch for_each_cpu(cpu, qmask) { 198142f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch if (cpumask_weight(nvmeq->cpu_mask) >= count) 198242f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch break; 198342f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch if (!cpumask_test_and_set_cpu(cpu, nvmeq->cpu_mask)) 198442f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch *per_cpu_ptr(nvmeq->dev->io_queue, cpu) = nvmeq->qid; 198542f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch } 198642f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch} 198742f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch 198842f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Buschstatic void nvme_add_cpus(cpumask_t *mask, const cpumask_t *unassigned_cpus, 198942f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch const cpumask_t *new_mask, struct nvme_queue *nvmeq, int cpus_per_queue) 199042f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch{ 199142f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch int next_cpu; 199242f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch for_each_cpu(next_cpu, new_mask) { 199342f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch cpumask_or(mask, mask, get_cpu_mask(next_cpu)); 199442f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch cpumask_or(mask, mask, topology_thread_cpumask(next_cpu)); 199542f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch cpumask_and(mask, mask, unassigned_cpus); 199642f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch nvme_set_queue_cpus(mask, nvmeq, cpus_per_queue); 199742f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch } 199842f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch} 199942f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch 200042f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Buschstatic void nvme_create_io_queues(struct nvme_dev *dev) 200142f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch{ 200242f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch unsigned i, max; 200342f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch 200442f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch max = min(dev->max_qid, num_online_cpus()); 200542f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch for (i = dev->queue_count; i <= max; i++) 200642f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch if (!nvme_alloc_queue(dev, i, dev->q_depth, i - 1)) 200742f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch break; 200842f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch 200942f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch max = min(dev->queue_count - 1, num_online_cpus()); 201042f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch for (i = dev->online_queues; i <= max; i++) 201142f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch if (nvme_create_queue(raw_nvmeq(dev, i), i)) 201242f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch break; 201342f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch} 201442f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch 201542f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch/* 201642f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch * If there are fewer queues than online cpus, this will try to optimally 201742f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch * assign a queue to multiple cpus by grouping cpus that are "close" together: 201842f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch * thread siblings, core, socket, closest node, then whatever else is 201942f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch * available. 202042f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch */ 202142f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Buschstatic void nvme_assign_io_queues(struct nvme_dev *dev) 202242f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch{ 202342f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch unsigned cpu, cpus_per_queue, queues, remainder, i; 202442f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch cpumask_var_t unassigned_cpus; 202542f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch 202642f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch nvme_create_io_queues(dev); 202742f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch 202842f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch queues = min(dev->online_queues - 1, num_online_cpus()); 202942f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch if (!queues) 203042f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch return; 203142f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch 203242f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch cpus_per_queue = num_online_cpus() / queues; 203342f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch remainder = queues - (num_online_cpus() - queues * cpus_per_queue); 203442f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch 203542f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch if (!alloc_cpumask_var(&unassigned_cpus, GFP_KERNEL)) 203642f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch return; 203742f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch 203842f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch cpumask_copy(unassigned_cpus, cpu_online_mask); 203942f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch cpu = cpumask_first(unassigned_cpus); 204042f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch for (i = 1; i <= queues; i++) { 204142f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch struct nvme_queue *nvmeq = lock_nvmeq(dev, i); 204242f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch cpumask_t mask; 204342f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch 204442f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch cpumask_clear(nvmeq->cpu_mask); 204542f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch if (!cpumask_weight(unassigned_cpus)) { 204642f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch unlock_nvmeq(nvmeq); 204742f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch break; 204842f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch } 204942f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch 205042f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch mask = *get_cpu_mask(cpu); 205142f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch nvme_set_queue_cpus(&mask, nvmeq, cpus_per_queue); 205242f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch if (cpus_weight(mask) < cpus_per_queue) 205342f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch nvme_add_cpus(&mask, unassigned_cpus, 205442f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch topology_thread_cpumask(cpu), 205542f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch nvmeq, cpus_per_queue); 205642f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch if (cpus_weight(mask) < cpus_per_queue) 205742f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch nvme_add_cpus(&mask, unassigned_cpus, 205842f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch topology_core_cpumask(cpu), 205942f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch nvmeq, cpus_per_queue); 206042f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch if (cpus_weight(mask) < cpus_per_queue) 206142f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch nvme_add_cpus(&mask, unassigned_cpus, 206242f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch cpumask_of_node(cpu_to_node(cpu)), 206342f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch nvmeq, cpus_per_queue); 206442f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch if (cpus_weight(mask) < cpus_per_queue) 206542f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch nvme_add_cpus(&mask, unassigned_cpus, 206642f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch cpumask_of_node( 206742f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch nvme_find_closest_node( 206842f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch cpu_to_node(cpu))), 206942f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch nvmeq, cpus_per_queue); 207042f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch if (cpus_weight(mask) < cpus_per_queue) 207142f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch nvme_add_cpus(&mask, unassigned_cpus, 207242f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch unassigned_cpus, 207342f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch nvmeq, cpus_per_queue); 207442f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch 207542f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch WARN(cpumask_weight(nvmeq->cpu_mask) != cpus_per_queue, 207642f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch "nvme%d qid:%d mis-matched queue-to-cpu assignment\n", 207742f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch dev->instance, i); 207842f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch 207942f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch irq_set_affinity_hint(dev->entry[nvmeq->cq_vector].vector, 208042f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch nvmeq->cpu_mask); 208142f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch cpumask_andnot(unassigned_cpus, unassigned_cpus, 208242f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch nvmeq->cpu_mask); 208342f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch cpu = cpumask_next(cpu, unassigned_cpus); 208442f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch if (remainder && !--remainder) 208542f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch cpus_per_queue++; 208642f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch unlock_nvmeq(nvmeq); 208742f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch } 208842f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch WARN(cpumask_weight(unassigned_cpus), "nvme%d unassigned online cpus\n", 208942f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch dev->instance); 209042f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch i = 0; 209142f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch cpumask_andnot(unassigned_cpus, cpu_possible_mask, cpu_online_mask); 209242f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch for_each_cpu(cpu, unassigned_cpus) 209342f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch *per_cpu_ptr(dev->io_queue, cpu) = (i++ % queues) + 1; 209442f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch free_cpumask_var(unassigned_cpus); 209542f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch} 209642f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch 2097b3b06812e199f248561ce7824a4a8a9cd573c05aMatthew Wilcoxstatic int set_queue_count(struct nvme_dev *dev, int count) 2098b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 2099b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int status; 2100b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 result; 2101b3b06812e199f248561ce7824a4a8a9cd573c05aMatthew Wilcox u32 q_count = (count - 1) | ((count - 1) << 16); 2102b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2103df3481399042200792822b6243e36a95a557b57eMatthew Wilcox status = nvme_set_features(dev, NVME_FEAT_NUM_QUEUES, q_count, 0, 2104bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox &result); 210527e8166c31656f7780e682eaf6bc9c3b8dd03253Matthew Wilcox if (status < 0) 210627e8166c31656f7780e682eaf6bc9c3b8dd03253Matthew Wilcox return status; 210727e8166c31656f7780e682eaf6bc9c3b8dd03253Matthew Wilcox if (status > 0) { 210827e8166c31656f7780e682eaf6bc9c3b8dd03253Matthew Wilcox dev_err(&dev->pci_dev->dev, "Could not set queue count (%d)\n", 210927e8166c31656f7780e682eaf6bc9c3b8dd03253Matthew Wilcox status); 211027e8166c31656f7780e682eaf6bc9c3b8dd03253Matthew Wilcox return -EBUSY; 211127e8166c31656f7780e682eaf6bc9c3b8dd03253Matthew Wilcox } 2112b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return min(result & 0xffff, result >> 16) + 1; 2113b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 2114b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 21159d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Buschstatic size_t db_bar_size(struct nvme_dev *dev, unsigned nr_io_queues) 21169d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch{ 2117b80d5ccca3a012e91ca64a2a0b13049163a6a698Haiyan Hu return 4096 + ((nr_io_queues + 1) * 8 * dev->db_stride); 21189d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch} 21199d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch 2120f3db22feb5de6b98b7bae924c2d4b6c8d65bedaeKeith Buschstatic void nvme_cpu_workfn(struct work_struct *work) 2121f3db22feb5de6b98b7bae924c2d4b6c8d65bedaeKeith Busch{ 2122f3db22feb5de6b98b7bae924c2d4b6c8d65bedaeKeith Busch struct nvme_dev *dev = container_of(work, struct nvme_dev, cpu_work); 2123f3db22feb5de6b98b7bae924c2d4b6c8d65bedaeKeith Busch if (dev->initialized) 2124f3db22feb5de6b98b7bae924c2d4b6c8d65bedaeKeith Busch nvme_assign_io_queues(dev); 2125f3db22feb5de6b98b7bae924c2d4b6c8d65bedaeKeith Busch} 2126f3db22feb5de6b98b7bae924c2d4b6c8d65bedaeKeith Busch 212733b1e95c90447ea73e37e837ea0268a894919f19Keith Buschstatic int nvme_cpu_notify(struct notifier_block *self, 212833b1e95c90447ea73e37e837ea0268a894919f19Keith Busch unsigned long action, void *hcpu) 212933b1e95c90447ea73e37e837ea0268a894919f19Keith Busch{ 2130f3db22feb5de6b98b7bae924c2d4b6c8d65bedaeKeith Busch struct nvme_dev *dev; 2131f3db22feb5de6b98b7bae924c2d4b6c8d65bedaeKeith Busch 213233b1e95c90447ea73e37e837ea0268a894919f19Keith Busch switch (action) { 213333b1e95c90447ea73e37e837ea0268a894919f19Keith Busch case CPU_ONLINE: 213433b1e95c90447ea73e37e837ea0268a894919f19Keith Busch case CPU_DEAD: 2135f3db22feb5de6b98b7bae924c2d4b6c8d65bedaeKeith Busch spin_lock(&dev_list_lock); 2136f3db22feb5de6b98b7bae924c2d4b6c8d65bedaeKeith Busch list_for_each_entry(dev, &dev_list, node) 2137f3db22feb5de6b98b7bae924c2d4b6c8d65bedaeKeith Busch schedule_work(&dev->cpu_work); 2138f3db22feb5de6b98b7bae924c2d4b6c8d65bedaeKeith Busch spin_unlock(&dev_list_lock); 213933b1e95c90447ea73e37e837ea0268a894919f19Keith Busch break; 214033b1e95c90447ea73e37e837ea0268a894919f19Keith Busch } 214133b1e95c90447ea73e37e837ea0268a894919f19Keith Busch return NOTIFY_OK; 214233b1e95c90447ea73e37e837ea0268a894919f19Keith Busch} 214333b1e95c90447ea73e37e837ea0268a894919f19Keith Busch 21448d85fce77edfc22f1d6dbf78e3af723b4b556f3dGreg Kroah-Hartmanstatic int nvme_setup_io_queues(struct nvme_dev *dev) 2145b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 21465a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch struct nvme_queue *adminq = raw_nvmeq(dev, 0); 2147fa08a396647767abd24a9e7015cb177121d0cf15Ramachandra Rao Gajula struct pci_dev *pdev = dev->pci_dev; 214842f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch int result, i, vecs, nr_io_queues, size; 2149b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 215042f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch nr_io_queues = num_possible_cpus(); 2151b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox result = set_queue_count(dev, nr_io_queues); 21521b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox if (result < 0) 21531b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox return result; 2154b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox if (result < nr_io_queues) 2155b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox nr_io_queues = result; 2156b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 21579d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch size = db_bar_size(dev, nr_io_queues); 21589d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch if (size > 8192) { 2159f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox iounmap(dev->bar); 21609d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch do { 21619d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch dev->bar = ioremap(pci_resource_start(pdev, 0), size); 21629d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch if (dev->bar) 21639d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch break; 21649d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch if (!--nr_io_queues) 21659d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch return -ENOMEM; 21669d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch size = db_bar_size(dev, nr_io_queues); 21679d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch } while (1); 2168f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox dev->dbs = ((void __iomem *)dev->bar) + 4096; 21695a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch adminq->q_db = dev->dbs; 2170f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox } 2171f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox 21729d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch /* Deregister the admin queue's interrupt */ 21733193f07bb7ae723f33ac8e8b9db317a4f68d7d18Matthew Wilcox free_irq(dev->entry[0].vector, adminq); 21749d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch 2175be577fabf3ec2c9593f4d3c044283be783b88c3bAlexander Gordeev for (i = 0; i < nr_io_queues; i++) 21761b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox dev->entry[i].entry = i; 2177be577fabf3ec2c9593f4d3c044283be783b88c3bAlexander Gordeev vecs = pci_enable_msix_range(pdev, dev->entry, 1, nr_io_queues); 2178be577fabf3ec2c9593f4d3c044283be783b88c3bAlexander Gordeev if (vecs < 0) { 2179be577fabf3ec2c9593f4d3c044283be783b88c3bAlexander Gordeev vecs = pci_enable_msi_range(pdev, 1, min(nr_io_queues, 32)); 2180be577fabf3ec2c9593f4d3c044283be783b88c3bAlexander Gordeev if (vecs < 0) { 2181be577fabf3ec2c9593f4d3c044283be783b88c3bAlexander Gordeev vecs = 1; 2182be577fabf3ec2c9593f4d3c044283be783b88c3bAlexander Gordeev } else { 2183be577fabf3ec2c9593f4d3c044283be783b88c3bAlexander Gordeev for (i = 0; i < vecs; i++) 2184be577fabf3ec2c9593f4d3c044283be783b88c3bAlexander Gordeev dev->entry[i].vector = i + pdev->irq; 2185fa08a396647767abd24a9e7015cb177121d0cf15Ramachandra Rao Gajula } 2186fa08a396647767abd24a9e7015cb177121d0cf15Ramachandra Rao Gajula } 2187fa08a396647767abd24a9e7015cb177121d0cf15Ramachandra Rao Gajula 2188063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox /* 2189063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox * Should investigate if there's a performance win from allocating 2190063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox * more queues than interrupt vectors; it might allow the submission 2191063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox * path to scale better, even if the receive path is limited by the 2192063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox * number of interrupts. 2193063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox */ 2194063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox nr_io_queues = vecs; 219542f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch dev->max_qid = nr_io_queues; 2196063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox 21973193f07bb7ae723f33ac8e8b9db317a4f68d7d18Matthew Wilcox result = queue_request_irq(dev, adminq, adminq->irqname); 21989d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch if (result) { 21993193f07bb7ae723f33ac8e8b9db317a4f68d7d18Matthew Wilcox adminq->q_suspended = 1; 2200224042742582c9938788b81165180c876e997a07Keith Busch goto free_queues; 22019d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch } 22021b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox 2203cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch /* Free previously allocated queues that are no longer usable */ 220442f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch nvme_free_queues(dev, nr_io_queues + 1); 220542f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch nvme_assign_io_queues(dev); 22069ecdc946212f7cd592986b2c519b470404caa6b8Matthew Wilcox 2207224042742582c9938788b81165180c876e997a07Keith Busch return 0; 2208b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2209224042742582c9938788b81165180c876e997a07Keith Busch free_queues: 2210a1a5ef9998b1379780790b878457b0e0f48b25dbKeith Busch nvme_free_queues(dev, 1); 2211224042742582c9938788b81165180c876e997a07Keith Busch return result; 2212b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 2213b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2214422ef0c7c81934c3344e1623d2dcfda5fa2fab8dMatthew Wilcox/* 2215422ef0c7c81934c3344e1623d2dcfda5fa2fab8dMatthew Wilcox * Return: error value if an error occurred setting up the queues or calling 2216422ef0c7c81934c3344e1623d2dcfda5fa2fab8dMatthew Wilcox * Identify Device. 0 if these succeeded, even if adding some of the 2217422ef0c7c81934c3344e1623d2dcfda5fa2fab8dMatthew Wilcox * namespaces failed. At the moment, these failures are silent. TBD which 2218422ef0c7c81934c3344e1623d2dcfda5fa2fab8dMatthew Wilcox * failures should be reported. 2219422ef0c7c81934c3344e1623d2dcfda5fa2fab8dMatthew Wilcox */ 22208d85fce77edfc22f1d6dbf78e3af723b4b556f3dGreg Kroah-Hartmanstatic int nvme_dev_add(struct nvme_dev *dev) 2221b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 222268608c268bf265b039daeaafee6c902dfef32024Matthew Wilcox struct pci_dev *pdev = dev->pci_dev; 2223c3bfe7176c035a0a2c70bc79180fb13a6c57142aMatthew Wilcox int res; 2224c3bfe7176c035a0a2c70bc79180fb13a6c57142aMatthew Wilcox unsigned nn, i; 2225cbb6218fd4ae8f98ddf0d66f0826a7a3a9c88298Keith Busch struct nvme_ns *ns; 222651814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox struct nvme_id_ctrl *ctrl; 2227bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox struct nvme_id_ns *id_ns; 2228bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox void *mem; 2229b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_addr_t dma_addr; 2230159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch int shift = NVME_CAP_MPSMIN(readq(&dev->bar->cap)) + 12; 2231b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 223268608c268bf265b039daeaafee6c902dfef32024Matthew Wilcox mem = dma_alloc_coherent(&pdev->dev, 8192, &dma_addr, GFP_KERNEL); 2233a9ef4343afbe67a6abf83c0f0294e80db48e513aKeith Busch if (!mem) 2234a9ef4343afbe67a6abf83c0f0294e80db48e513aKeith Busch return -ENOMEM; 2235b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2236bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox res = nvme_identify(dev, 0, 1, dma_addr); 2237b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (res) { 223827e8166c31656f7780e682eaf6bc9c3b8dd03253Matthew Wilcox dev_err(&pdev->dev, "Identify Controller failed (%d)\n", res); 2239b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox res = -EIO; 2240cbb6218fd4ae8f98ddf0d66f0826a7a3a9c88298Keith Busch goto out; 2241b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 2242b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2243bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox ctrl = mem; 224451814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox nn = le32_to_cpup(&ctrl->nn); 22450e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch dev->oncs = le16_to_cpup(&ctrl->oncs); 2246c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch dev->abort_limit = ctrl->acl + 1; 2247a7d2ce2832d84e0182585f63bf96ca7323b3aee7Keith Busch dev->vwc = ctrl->vwc; 224851814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox memcpy(dev->serial, ctrl->sn, sizeof(ctrl->sn)); 224951814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox memcpy(dev->model, ctrl->mn, sizeof(ctrl->mn)); 225051814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox memcpy(dev->firmware_rev, ctrl->fr, sizeof(ctrl->fr)); 2251159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch if (ctrl->mdts) 22528fc23e032debd682f5ba9fc524a5846c10d2c522Keith Busch dev->max_hw_sectors = 1 << (ctrl->mdts + shift - 9); 225368608c268bf265b039daeaafee6c902dfef32024Matthew Wilcox if ((pdev->vendor == PCI_VENDOR_ID_INTEL) && 225468608c268bf265b039daeaafee6c902dfef32024Matthew Wilcox (pdev->device == 0x0953) && ctrl->vs[3]) 2255159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch dev->stripe_size = 1 << (ctrl->vs[3] + shift); 2256b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2257bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox id_ns = mem; 22582b2c1896871838cdf549442e8ad0264be5fa74e3Matthew Wilcox for (i = 1; i <= nn; i++) { 2259bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox res = nvme_identify(dev, i, 0, dma_addr); 2260b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (res) 2261b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox continue; 2262b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2263bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox if (id_ns->ncap == 0) 2264b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox continue; 2265b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2266bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox res = nvme_get_features(dev, NVME_FEAT_LBA_RANGE, i, 226708df1e05657fc6712e520e7c09cc6c86160ceb35Keith Busch dma_addr + 4096, NULL); 2268b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (res) 2269122090366d1d5c6ec1bfb6dfdb3a6d121ff074aaKeith Busch memset(mem + 4096, 0, 4096); 2270b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2271bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox ns = nvme_alloc_ns(dev, i, mem, mem + 4096); 2272b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (ns) 2273b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox list_add_tail(&ns->list, &dev->namespaces); 2274b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 2275b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox list_for_each_entry(ns, &dev->namespaces, list) 2276b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox add_disk(ns->disk); 2277422ef0c7c81934c3344e1623d2dcfda5fa2fab8dMatthew Wilcox res = 0; 2278b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2279bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox out: 2280684f5c2025b067a23722e620d0b3b858d8dc5d01Matthew Wilcox dma_free_coherent(&dev->pci_dev->dev, 8192, mem, dma_addr); 2281b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return res; 2282b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 2283b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 22840877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Buschstatic int nvme_dev_map(struct nvme_dev *dev) 22850877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch{ 228642f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch u64 cap; 22870877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch int bars, result = -ENOMEM; 22880877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch struct pci_dev *pdev = dev->pci_dev; 22890877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch 22900877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch if (pci_enable_device_mem(pdev)) 22910877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch return result; 22920877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch 22930877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch dev->entry[0].vector = pdev->irq; 22940877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch pci_set_master(pdev); 22950877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch bars = pci_select_bars(pdev, IORESOURCE_MEM); 22960877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch if (pci_request_selected_regions(pdev, bars, "nvme")) 22970877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch goto disable_pci; 22980877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch 2299052d0efada8ccfccd924059b75f5fb119cd10f9aRussell King if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) && 2300052d0efada8ccfccd924059b75f5fb119cd10f9aRussell King dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32))) 2301052d0efada8ccfccd924059b75f5fb119cd10f9aRussell King goto disable; 23020877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch 23030877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch dev->bar = ioremap(pci_resource_start(pdev, 0), 8192); 23040877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch if (!dev->bar) 23050877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch goto disable; 23060e53d18051725da46cbccfb7874a6422d4d4f274Keith Busch if (readl(&dev->bar->csts) == -1) { 23070e53d18051725da46cbccfb7874a6422d4d4f274Keith Busch result = -ENODEV; 23080e53d18051725da46cbccfb7874a6422d4d4f274Keith Busch goto unmap; 23090e53d18051725da46cbccfb7874a6422d4d4f274Keith Busch } 231042f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch cap = readq(&dev->bar->cap); 231142f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch dev->q_depth = min_t(int, NVME_CAP_MQES(cap) + 1, NVME_Q_DEPTH); 231242f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch dev->db_stride = 1 << NVME_CAP_STRIDE(cap); 23130877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch dev->dbs = ((void __iomem *)dev->bar) + 4096; 23140877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch 23150877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch return 0; 23160877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch 23170e53d18051725da46cbccfb7874a6422d4d4f274Keith Busch unmap: 23180e53d18051725da46cbccfb7874a6422d4d4f274Keith Busch iounmap(dev->bar); 23190e53d18051725da46cbccfb7874a6422d4d4f274Keith Busch dev->bar = NULL; 23200877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch disable: 23210877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch pci_release_regions(pdev); 23220877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch disable_pci: 23230877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch pci_disable_device(pdev); 23240877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch return result; 23250877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch} 23260877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch 23270877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Buschstatic void nvme_dev_unmap(struct nvme_dev *dev) 23280877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch{ 23290877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch if (dev->pci_dev->msi_enabled) 23300877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch pci_disable_msi(dev->pci_dev); 23310877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch else if (dev->pci_dev->msix_enabled) 23320877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch pci_disable_msix(dev->pci_dev); 23330877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch 23340877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch if (dev->bar) { 23350877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch iounmap(dev->bar); 23360877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch dev->bar = NULL; 23379a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch pci_release_regions(dev->pci_dev); 23380877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch } 23390877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch 23400877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch if (pci_is_enabled(dev->pci_dev)) 23410877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch pci_disable_device(dev->pci_dev); 23420877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch} 23430877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch 23444d115420707afcabe77d2535e092356df6664b70Keith Buschstruct nvme_delq_ctx { 23454d115420707afcabe77d2535e092356df6664b70Keith Busch struct task_struct *waiter; 23464d115420707afcabe77d2535e092356df6664b70Keith Busch struct kthread_worker *worker; 23474d115420707afcabe77d2535e092356df6664b70Keith Busch atomic_t refcount; 23484d115420707afcabe77d2535e092356df6664b70Keith Busch}; 23494d115420707afcabe77d2535e092356df6664b70Keith Busch 23504d115420707afcabe77d2535e092356df6664b70Keith Buschstatic void nvme_wait_dq(struct nvme_delq_ctx *dq, struct nvme_dev *dev) 23514d115420707afcabe77d2535e092356df6664b70Keith Busch{ 23524d115420707afcabe77d2535e092356df6664b70Keith Busch dq->waiter = current; 23534d115420707afcabe77d2535e092356df6664b70Keith Busch mb(); 23544d115420707afcabe77d2535e092356df6664b70Keith Busch 23554d115420707afcabe77d2535e092356df6664b70Keith Busch for (;;) { 23564d115420707afcabe77d2535e092356df6664b70Keith Busch set_current_state(TASK_KILLABLE); 23574d115420707afcabe77d2535e092356df6664b70Keith Busch if (!atomic_read(&dq->refcount)) 23584d115420707afcabe77d2535e092356df6664b70Keith Busch break; 23594d115420707afcabe77d2535e092356df6664b70Keith Busch if (!schedule_timeout(ADMIN_TIMEOUT) || 23604d115420707afcabe77d2535e092356df6664b70Keith Busch fatal_signal_pending(current)) { 23614d115420707afcabe77d2535e092356df6664b70Keith Busch set_current_state(TASK_RUNNING); 23624d115420707afcabe77d2535e092356df6664b70Keith Busch 23634d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_disable_ctrl(dev, readq(&dev->bar->cap)); 23644d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_disable_queue(dev, 0); 23654d115420707afcabe77d2535e092356df6664b70Keith Busch 23664d115420707afcabe77d2535e092356df6664b70Keith Busch send_sig(SIGKILL, dq->worker->task, 1); 23674d115420707afcabe77d2535e092356df6664b70Keith Busch flush_kthread_worker(dq->worker); 23684d115420707afcabe77d2535e092356df6664b70Keith Busch return; 23694d115420707afcabe77d2535e092356df6664b70Keith Busch } 23704d115420707afcabe77d2535e092356df6664b70Keith Busch } 23714d115420707afcabe77d2535e092356df6664b70Keith Busch set_current_state(TASK_RUNNING); 23724d115420707afcabe77d2535e092356df6664b70Keith Busch} 23734d115420707afcabe77d2535e092356df6664b70Keith Busch 23744d115420707afcabe77d2535e092356df6664b70Keith Buschstatic void nvme_put_dq(struct nvme_delq_ctx *dq) 23754d115420707afcabe77d2535e092356df6664b70Keith Busch{ 23764d115420707afcabe77d2535e092356df6664b70Keith Busch atomic_dec(&dq->refcount); 23774d115420707afcabe77d2535e092356df6664b70Keith Busch if (dq->waiter) 23784d115420707afcabe77d2535e092356df6664b70Keith Busch wake_up_process(dq->waiter); 23794d115420707afcabe77d2535e092356df6664b70Keith Busch} 23804d115420707afcabe77d2535e092356df6664b70Keith Busch 23814d115420707afcabe77d2535e092356df6664b70Keith Buschstatic struct nvme_delq_ctx *nvme_get_dq(struct nvme_delq_ctx *dq) 23824d115420707afcabe77d2535e092356df6664b70Keith Busch{ 23834d115420707afcabe77d2535e092356df6664b70Keith Busch atomic_inc(&dq->refcount); 23844d115420707afcabe77d2535e092356df6664b70Keith Busch return dq; 23854d115420707afcabe77d2535e092356df6664b70Keith Busch} 23864d115420707afcabe77d2535e092356df6664b70Keith Busch 23874d115420707afcabe77d2535e092356df6664b70Keith Buschstatic void nvme_del_queue_end(struct nvme_queue *nvmeq) 23884d115420707afcabe77d2535e092356df6664b70Keith Busch{ 23894d115420707afcabe77d2535e092356df6664b70Keith Busch struct nvme_delq_ctx *dq = nvmeq->cmdinfo.ctx; 23904d115420707afcabe77d2535e092356df6664b70Keith Busch 23914d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_clear_queue(nvmeq); 23924d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_put_dq(dq); 23934d115420707afcabe77d2535e092356df6664b70Keith Busch} 23944d115420707afcabe77d2535e092356df6664b70Keith Busch 23954d115420707afcabe77d2535e092356df6664b70Keith Buschstatic int adapter_async_del_queue(struct nvme_queue *nvmeq, u8 opcode, 23964d115420707afcabe77d2535e092356df6664b70Keith Busch kthread_work_func_t fn) 23974d115420707afcabe77d2535e092356df6664b70Keith Busch{ 23984d115420707afcabe77d2535e092356df6664b70Keith Busch struct nvme_command c; 23994d115420707afcabe77d2535e092356df6664b70Keith Busch 24004d115420707afcabe77d2535e092356df6664b70Keith Busch memset(&c, 0, sizeof(c)); 24014d115420707afcabe77d2535e092356df6664b70Keith Busch c.delete_queue.opcode = opcode; 24024d115420707afcabe77d2535e092356df6664b70Keith Busch c.delete_queue.qid = cpu_to_le16(nvmeq->qid); 24034d115420707afcabe77d2535e092356df6664b70Keith Busch 24044d115420707afcabe77d2535e092356df6664b70Keith Busch init_kthread_work(&nvmeq->cmdinfo.work, fn); 24054d115420707afcabe77d2535e092356df6664b70Keith Busch return nvme_submit_admin_cmd_async(nvmeq->dev, &c, &nvmeq->cmdinfo); 24064d115420707afcabe77d2535e092356df6664b70Keith Busch} 24074d115420707afcabe77d2535e092356df6664b70Keith Busch 24084d115420707afcabe77d2535e092356df6664b70Keith Buschstatic void nvme_del_cq_work_handler(struct kthread_work *work) 24094d115420707afcabe77d2535e092356df6664b70Keith Busch{ 24104d115420707afcabe77d2535e092356df6664b70Keith Busch struct nvme_queue *nvmeq = container_of(work, struct nvme_queue, 24114d115420707afcabe77d2535e092356df6664b70Keith Busch cmdinfo.work); 24124d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_del_queue_end(nvmeq); 24134d115420707afcabe77d2535e092356df6664b70Keith Busch} 24144d115420707afcabe77d2535e092356df6664b70Keith Busch 24154d115420707afcabe77d2535e092356df6664b70Keith Buschstatic int nvme_delete_cq(struct nvme_queue *nvmeq) 24164d115420707afcabe77d2535e092356df6664b70Keith Busch{ 24174d115420707afcabe77d2535e092356df6664b70Keith Busch return adapter_async_del_queue(nvmeq, nvme_admin_delete_cq, 24184d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_del_cq_work_handler); 24194d115420707afcabe77d2535e092356df6664b70Keith Busch} 24204d115420707afcabe77d2535e092356df6664b70Keith Busch 24214d115420707afcabe77d2535e092356df6664b70Keith Buschstatic void nvme_del_sq_work_handler(struct kthread_work *work) 24224d115420707afcabe77d2535e092356df6664b70Keith Busch{ 24234d115420707afcabe77d2535e092356df6664b70Keith Busch struct nvme_queue *nvmeq = container_of(work, struct nvme_queue, 24244d115420707afcabe77d2535e092356df6664b70Keith Busch cmdinfo.work); 24254d115420707afcabe77d2535e092356df6664b70Keith Busch int status = nvmeq->cmdinfo.status; 24264d115420707afcabe77d2535e092356df6664b70Keith Busch 24274d115420707afcabe77d2535e092356df6664b70Keith Busch if (!status) 24284d115420707afcabe77d2535e092356df6664b70Keith Busch status = nvme_delete_cq(nvmeq); 24294d115420707afcabe77d2535e092356df6664b70Keith Busch if (status) 24304d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_del_queue_end(nvmeq); 24314d115420707afcabe77d2535e092356df6664b70Keith Busch} 24324d115420707afcabe77d2535e092356df6664b70Keith Busch 24334d115420707afcabe77d2535e092356df6664b70Keith Buschstatic int nvme_delete_sq(struct nvme_queue *nvmeq) 24344d115420707afcabe77d2535e092356df6664b70Keith Busch{ 24354d115420707afcabe77d2535e092356df6664b70Keith Busch return adapter_async_del_queue(nvmeq, nvme_admin_delete_sq, 24364d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_del_sq_work_handler); 24374d115420707afcabe77d2535e092356df6664b70Keith Busch} 24384d115420707afcabe77d2535e092356df6664b70Keith Busch 24394d115420707afcabe77d2535e092356df6664b70Keith Buschstatic void nvme_del_queue_start(struct kthread_work *work) 24404d115420707afcabe77d2535e092356df6664b70Keith Busch{ 24414d115420707afcabe77d2535e092356df6664b70Keith Busch struct nvme_queue *nvmeq = container_of(work, struct nvme_queue, 24424d115420707afcabe77d2535e092356df6664b70Keith Busch cmdinfo.work); 24434d115420707afcabe77d2535e092356df6664b70Keith Busch allow_signal(SIGKILL); 24444d115420707afcabe77d2535e092356df6664b70Keith Busch if (nvme_delete_sq(nvmeq)) 24454d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_del_queue_end(nvmeq); 24464d115420707afcabe77d2535e092356df6664b70Keith Busch} 24474d115420707afcabe77d2535e092356df6664b70Keith Busch 24484d115420707afcabe77d2535e092356df6664b70Keith Buschstatic void nvme_disable_io_queues(struct nvme_dev *dev) 24494d115420707afcabe77d2535e092356df6664b70Keith Busch{ 24504d115420707afcabe77d2535e092356df6664b70Keith Busch int i; 24514d115420707afcabe77d2535e092356df6664b70Keith Busch DEFINE_KTHREAD_WORKER_ONSTACK(worker); 24524d115420707afcabe77d2535e092356df6664b70Keith Busch struct nvme_delq_ctx dq; 24534d115420707afcabe77d2535e092356df6664b70Keith Busch struct task_struct *kworker_task = kthread_run(kthread_worker_fn, 24544d115420707afcabe77d2535e092356df6664b70Keith Busch &worker, "nvme%d", dev->instance); 24554d115420707afcabe77d2535e092356df6664b70Keith Busch 24564d115420707afcabe77d2535e092356df6664b70Keith Busch if (IS_ERR(kworker_task)) { 24574d115420707afcabe77d2535e092356df6664b70Keith Busch dev_err(&dev->pci_dev->dev, 24584d115420707afcabe77d2535e092356df6664b70Keith Busch "Failed to create queue del task\n"); 24594d115420707afcabe77d2535e092356df6664b70Keith Busch for (i = dev->queue_count - 1; i > 0; i--) 24604d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_disable_queue(dev, i); 24614d115420707afcabe77d2535e092356df6664b70Keith Busch return; 24624d115420707afcabe77d2535e092356df6664b70Keith Busch } 24634d115420707afcabe77d2535e092356df6664b70Keith Busch 24644d115420707afcabe77d2535e092356df6664b70Keith Busch dq.waiter = NULL; 24654d115420707afcabe77d2535e092356df6664b70Keith Busch atomic_set(&dq.refcount, 0); 24664d115420707afcabe77d2535e092356df6664b70Keith Busch dq.worker = &worker; 24674d115420707afcabe77d2535e092356df6664b70Keith Busch for (i = dev->queue_count - 1; i > 0; i--) { 24685a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch struct nvme_queue *nvmeq = raw_nvmeq(dev, i); 24694d115420707afcabe77d2535e092356df6664b70Keith Busch 24704d115420707afcabe77d2535e092356df6664b70Keith Busch if (nvme_suspend_queue(nvmeq)) 24714d115420707afcabe77d2535e092356df6664b70Keith Busch continue; 24724d115420707afcabe77d2535e092356df6664b70Keith Busch nvmeq->cmdinfo.ctx = nvme_get_dq(&dq); 24734d115420707afcabe77d2535e092356df6664b70Keith Busch nvmeq->cmdinfo.worker = dq.worker; 24744d115420707afcabe77d2535e092356df6664b70Keith Busch init_kthread_work(&nvmeq->cmdinfo.work, nvme_del_queue_start); 24754d115420707afcabe77d2535e092356df6664b70Keith Busch queue_kthread_work(dq.worker, &nvmeq->cmdinfo.work); 24764d115420707afcabe77d2535e092356df6664b70Keith Busch } 24774d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_wait_dq(&dq, dev); 24784d115420707afcabe77d2535e092356df6664b70Keith Busch kthread_stop(kworker_task); 24794d115420707afcabe77d2535e092356df6664b70Keith Busch} 24804d115420707afcabe77d2535e092356df6664b70Keith Busch 2481b9afca3efb18a9b8392cb544a3e29e8b1168400cDan McLeran/* 2482b9afca3efb18a9b8392cb544a3e29e8b1168400cDan McLeran* Remove the node from the device list and check 2483b9afca3efb18a9b8392cb544a3e29e8b1168400cDan McLeran* for whether or not we need to stop the nvme_thread. 2484b9afca3efb18a9b8392cb544a3e29e8b1168400cDan McLeran*/ 2485b9afca3efb18a9b8392cb544a3e29e8b1168400cDan McLeranstatic void nvme_dev_list_remove(struct nvme_dev *dev) 2486b9afca3efb18a9b8392cb544a3e29e8b1168400cDan McLeran{ 2487b9afca3efb18a9b8392cb544a3e29e8b1168400cDan McLeran struct task_struct *tmp = NULL; 2488b9afca3efb18a9b8392cb544a3e29e8b1168400cDan McLeran 2489b9afca3efb18a9b8392cb544a3e29e8b1168400cDan McLeran spin_lock(&dev_list_lock); 2490b9afca3efb18a9b8392cb544a3e29e8b1168400cDan McLeran list_del_init(&dev->node); 2491b9afca3efb18a9b8392cb544a3e29e8b1168400cDan McLeran if (list_empty(&dev_list) && !IS_ERR_OR_NULL(nvme_thread)) { 2492b9afca3efb18a9b8392cb544a3e29e8b1168400cDan McLeran tmp = nvme_thread; 2493b9afca3efb18a9b8392cb544a3e29e8b1168400cDan McLeran nvme_thread = NULL; 2494b9afca3efb18a9b8392cb544a3e29e8b1168400cDan McLeran } 2495b9afca3efb18a9b8392cb544a3e29e8b1168400cDan McLeran spin_unlock(&dev_list_lock); 2496b9afca3efb18a9b8392cb544a3e29e8b1168400cDan McLeran 2497b9afca3efb18a9b8392cb544a3e29e8b1168400cDan McLeran if (tmp) 2498b9afca3efb18a9b8392cb544a3e29e8b1168400cDan McLeran kthread_stop(tmp); 2499b9afca3efb18a9b8392cb544a3e29e8b1168400cDan McLeran} 2500b9afca3efb18a9b8392cb544a3e29e8b1168400cDan McLeran 2501f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Buschstatic void nvme_dev_shutdown(struct nvme_dev *dev) 2502b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 2503224042742582c9938788b81165180c876e997a07Keith Busch int i; 2504224042742582c9938788b81165180c876e997a07Keith Busch 2505d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch dev->initialized = 0; 2506b9afca3efb18a9b8392cb544a3e29e8b1168400cDan McLeran nvme_dev_list_remove(dev); 25071fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 25084d115420707afcabe77d2535e092356df6664b70Keith Busch if (!dev->bar || (dev->bar && readl(&dev->bar->csts) == -1)) { 25094d115420707afcabe77d2535e092356df6664b70Keith Busch for (i = dev->queue_count - 1; i >= 0; i--) { 25105a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch struct nvme_queue *nvmeq = raw_nvmeq(dev, i); 25114d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_suspend_queue(nvmeq); 25124d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_clear_queue(nvmeq); 25134d115420707afcabe77d2535e092356df6664b70Keith Busch } 25144d115420707afcabe77d2535e092356df6664b70Keith Busch } else { 25154d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_disable_io_queues(dev); 25161894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch nvme_shutdown_ctrl(dev); 25174d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_disable_queue(dev, 0); 25184d115420707afcabe77d2535e092356df6664b70Keith Busch } 2519f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch nvme_dev_unmap(dev); 2520f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch} 2521f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch 2522f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Buschstatic void nvme_dev_remove(struct nvme_dev *dev) 2523f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch{ 25249ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch struct nvme_ns *ns; 2525f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch 25269ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch list_for_each_entry(ns, &dev->namespaces, list) { 25279ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch if (ns->disk->flags & GENHD_FL_UP) 25289ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch del_gendisk(ns->disk); 25299ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch if (!blk_queue_dying(ns->queue)) 25309ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch blk_cleanup_queue(ns->queue); 2531b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 2532b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 2533b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2534091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcoxstatic int nvme_setup_prp_pools(struct nvme_dev *dev) 2535091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox{ 2536091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox struct device *dmadev = &dev->pci_dev->dev; 2537091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox dev->prp_page_pool = dma_pool_create("prp list page", dmadev, 2538091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox PAGE_SIZE, PAGE_SIZE, 0); 2539091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox if (!dev->prp_page_pool) 2540091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox return -ENOMEM; 2541091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox 254299802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox /* Optimisation for I/Os between 4k and 128k */ 254399802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox dev->prp_small_pool = dma_pool_create("prp list 256", dmadev, 254499802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox 256, 256, 0); 254599802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox if (!dev->prp_small_pool) { 254699802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox dma_pool_destroy(dev->prp_page_pool); 254799802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox return -ENOMEM; 254899802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox } 2549091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox return 0; 2550091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox} 2551091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox 2552091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcoxstatic void nvme_release_prp_pools(struct nvme_dev *dev) 2553091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox{ 2554091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox dma_pool_destroy(dev->prp_page_pool); 255599802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox dma_pool_destroy(dev->prp_small_pool); 2556091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox} 2557091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox 2558cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anhstatic DEFINE_IDA(nvme_instance_ida); 2559cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh 2560cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anhstatic int nvme_set_instance(struct nvme_dev *dev) 2561b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 2562cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh int instance, error; 2563cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh 2564cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh do { 2565cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh if (!ida_pre_get(&nvme_instance_ida, GFP_KERNEL)) 2566cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh return -ENODEV; 2567cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh 2568cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh spin_lock(&dev_list_lock); 2569cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh error = ida_get_new(&nvme_instance_ida, &instance); 2570cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh spin_unlock(&dev_list_lock); 2571cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh } while (error == -EAGAIN); 2572cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh 2573cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh if (error) 2574cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh return -ENODEV; 2575cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh 2576cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh dev->instance = instance; 2577cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh return 0; 2578b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 2579b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2580b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void nvme_release_instance(struct nvme_dev *dev) 2581b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 2582cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh spin_lock(&dev_list_lock); 2583cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh ida_remove(&nvme_instance_ida, dev->instance); 2584cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh spin_unlock(&dev_list_lock); 2585b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 2586b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 25879ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Buschstatic void nvme_free_namespaces(struct nvme_dev *dev) 25889ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch{ 25899ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch struct nvme_ns *ns, *next; 25909ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch 25919ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch list_for_each_entry_safe(ns, next, &dev->namespaces, list) { 25929ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch list_del(&ns->list); 25939ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch put_disk(ns->disk); 25949ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch kfree(ns); 25959ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch } 25969ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch} 25979ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch 25985e82e952f04681c10f35e02ee0a4a43ec027137aKeith Buschstatic void nvme_free_dev(struct kref *kref) 25995e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch{ 26005e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch struct nvme_dev *dev = container_of(kref, struct nvme_dev, kref); 26019ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch 26029ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch nvme_free_namespaces(dev); 260342f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch free_percpu(dev->io_queue); 26045e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch kfree(dev->queues); 26055e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch kfree(dev->entry); 26065e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch kfree(dev); 26075e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch} 26085e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch 26095e82e952f04681c10f35e02ee0a4a43ec027137aKeith Buschstatic int nvme_dev_open(struct inode *inode, struct file *f) 26105e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch{ 26115e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch struct nvme_dev *dev = container_of(f->private_data, struct nvme_dev, 26125e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch miscdev); 26135e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch kref_get(&dev->kref); 26145e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch f->private_data = dev; 26155e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch return 0; 26165e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch} 26175e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch 26185e82e952f04681c10f35e02ee0a4a43ec027137aKeith Buschstatic int nvme_dev_release(struct inode *inode, struct file *f) 26195e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch{ 26205e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch struct nvme_dev *dev = f->private_data; 26215e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch kref_put(&dev->kref, nvme_free_dev); 26225e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch return 0; 26235e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch} 26245e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch 26255e82e952f04681c10f35e02ee0a4a43ec027137aKeith Buschstatic long nvme_dev_ioctl(struct file *f, unsigned int cmd, unsigned long arg) 26265e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch{ 26275e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch struct nvme_dev *dev = f->private_data; 26285e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch switch (cmd) { 26295e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch case NVME_IOCTL_ADMIN_CMD: 26305e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch return nvme_user_admin_cmd(dev, (void __user *)arg); 26315e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch default: 26325e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch return -ENOTTY; 26335e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch } 26345e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch} 26355e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch 26365e82e952f04681c10f35e02ee0a4a43ec027137aKeith Buschstatic const struct file_operations nvme_dev_fops = { 26375e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch .owner = THIS_MODULE, 26385e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch .open = nvme_dev_open, 26395e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch .release = nvme_dev_release, 26405e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch .unlocked_ioctl = nvme_dev_ioctl, 26415e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch .compat_ioctl = nvme_dev_ioctl, 26425e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch}; 26435e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch 2644f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Buschstatic int nvme_dev_start(struct nvme_dev *dev) 2645f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch{ 2646f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch int result; 2647b9afca3efb18a9b8392cb544a3e29e8b1168400cDan McLeran bool start_thread = false; 2648f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch 2649f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch result = nvme_dev_map(dev); 2650f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch if (result) 2651f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch return result; 2652f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch 2653f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch result = nvme_configure_admin_queue(dev); 2654f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch if (result) 2655f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch goto unmap; 2656f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch 2657f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch spin_lock(&dev_list_lock); 2658b9afca3efb18a9b8392cb544a3e29e8b1168400cDan McLeran if (list_empty(&dev_list) && IS_ERR_OR_NULL(nvme_thread)) { 2659b9afca3efb18a9b8392cb544a3e29e8b1168400cDan McLeran start_thread = true; 2660b9afca3efb18a9b8392cb544a3e29e8b1168400cDan McLeran nvme_thread = NULL; 2661b9afca3efb18a9b8392cb544a3e29e8b1168400cDan McLeran } 2662f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch list_add(&dev->node, &dev_list); 2663f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch spin_unlock(&dev_list_lock); 2664f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch 2665b9afca3efb18a9b8392cb544a3e29e8b1168400cDan McLeran if (start_thread) { 2666b9afca3efb18a9b8392cb544a3e29e8b1168400cDan McLeran nvme_thread = kthread_run(nvme_kthread, NULL, "nvme"); 2667b9afca3efb18a9b8392cb544a3e29e8b1168400cDan McLeran wake_up(&nvme_kthread_wait); 2668b9afca3efb18a9b8392cb544a3e29e8b1168400cDan McLeran } else 2669b9afca3efb18a9b8392cb544a3e29e8b1168400cDan McLeran wait_event_killable(nvme_kthread_wait, nvme_thread); 2670b9afca3efb18a9b8392cb544a3e29e8b1168400cDan McLeran 2671b9afca3efb18a9b8392cb544a3e29e8b1168400cDan McLeran if (IS_ERR_OR_NULL(nvme_thread)) { 2672b9afca3efb18a9b8392cb544a3e29e8b1168400cDan McLeran result = nvme_thread ? PTR_ERR(nvme_thread) : -EINTR; 2673b9afca3efb18a9b8392cb544a3e29e8b1168400cDan McLeran goto disable; 2674b9afca3efb18a9b8392cb544a3e29e8b1168400cDan McLeran } 2675b9afca3efb18a9b8392cb544a3e29e8b1168400cDan McLeran 2676f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch result = nvme_setup_io_queues(dev); 2677d82e8bfdef9afae83b894be49af4644d9ac3c359Keith Busch if (result && result != -EBUSY) 2678f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch goto disable; 2679f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch 2680d82e8bfdef9afae83b894be49af4644d9ac3c359Keith Busch return result; 2681f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch 2682f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch disable: 2683a1a5ef9998b1379780790b878457b0e0f48b25dbKeith Busch nvme_disable_queue(dev, 0); 2684b9afca3efb18a9b8392cb544a3e29e8b1168400cDan McLeran nvme_dev_list_remove(dev); 2685f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch unmap: 2686f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch nvme_dev_unmap(dev); 2687f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch return result; 2688f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch} 2689f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch 26909a6b94584de1a0467d85b435df9c744c5c45a270Keith Buschstatic int nvme_remove_dead_ctrl(void *arg) 26919a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch{ 26929a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch struct nvme_dev *dev = (struct nvme_dev *)arg; 26939a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch struct pci_dev *pdev = dev->pci_dev; 26949a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch 26959a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch if (pci_get_drvdata(pdev)) 26969a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch pci_stop_and_remove_bus_device(pdev); 26979a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch kref_put(&dev->kref, nvme_free_dev); 26989a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch return 0; 26999a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch} 27009a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch 27019a6b94584de1a0467d85b435df9c744c5c45a270Keith Buschstatic void nvme_remove_disks(struct work_struct *ws) 27029a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch{ 27039a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch struct nvme_dev *dev = container_of(ws, struct nvme_dev, reset_work); 27049a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch 27059a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch nvme_dev_remove(dev); 27065a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch nvme_free_queues(dev, 1); 27079a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch} 27089a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch 27099a6b94584de1a0467d85b435df9c744c5c45a270Keith Buschstatic int nvme_dev_resume(struct nvme_dev *dev) 27109a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch{ 27119a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch int ret; 27129a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch 27139a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch ret = nvme_dev_start(dev); 27149a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch if (ret && ret != -EBUSY) 27159a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch return ret; 27169a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch if (ret == -EBUSY) { 27179a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch spin_lock(&dev_list_lock); 27189ca9737444f1a8602f74b85018d881e7e54b5bd1Tejun Heo dev->reset_workfn = nvme_remove_disks; 27199a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch queue_work(nvme_workq, &dev->reset_work); 27209a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch spin_unlock(&dev_list_lock); 27219a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch } 2722d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch dev->initialized = 1; 27239a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch return 0; 27249a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch} 27259a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch 27269a6b94584de1a0467d85b435df9c744c5c45a270Keith Buschstatic void nvme_dev_reset(struct nvme_dev *dev) 27279a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch{ 27289a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch nvme_dev_shutdown(dev); 27299a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch if (nvme_dev_resume(dev)) { 27309a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch dev_err(&dev->pci_dev->dev, "Device failed to resume\n"); 27319a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch kref_get(&dev->kref); 27329a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch if (IS_ERR(kthread_run(nvme_remove_dead_ctrl, dev, "nvme%d", 27339a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch dev->instance))) { 27349a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch dev_err(&dev->pci_dev->dev, 27359a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch "Failed to start controller remove task\n"); 27369a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch kref_put(&dev->kref, nvme_free_dev); 27379a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch } 27389a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch } 27399a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch} 27409a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch 27419a6b94584de1a0467d85b435df9c744c5c45a270Keith Buschstatic void nvme_reset_failed_dev(struct work_struct *ws) 27429a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch{ 27439a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch struct nvme_dev *dev = container_of(ws, struct nvme_dev, reset_work); 27449a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch nvme_dev_reset(dev); 27459a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch} 27469a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch 27479ca9737444f1a8602f74b85018d881e7e54b5bd1Tejun Heostatic void nvme_reset_workfn(struct work_struct *work) 27489ca9737444f1a8602f74b85018d881e7e54b5bd1Tejun Heo{ 27499ca9737444f1a8602f74b85018d881e7e54b5bd1Tejun Heo struct nvme_dev *dev = container_of(work, struct nvme_dev, reset_work); 27509ca9737444f1a8602f74b85018d881e7e54b5bd1Tejun Heo dev->reset_workfn(work); 27519ca9737444f1a8602f74b85018d881e7e54b5bd1Tejun Heo} 27529ca9737444f1a8602f74b85018d881e7e54b5bd1Tejun Heo 27538d85fce77edfc22f1d6dbf78e3af723b4b556f3dGreg Kroah-Hartmanstatic int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id) 2754b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 27550877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch int result = -ENOMEM; 2756b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_dev *dev; 2757b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2758b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev = kzalloc(sizeof(*dev), GFP_KERNEL); 2759b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!dev) 2760b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -ENOMEM; 2761b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->entry = kcalloc(num_possible_cpus(), sizeof(*dev->entry), 2762b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox GFP_KERNEL); 2763b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!dev->entry) 2764b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto free; 27651b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox dev->queues = kcalloc(num_possible_cpus() + 1, sizeof(void *), 27661b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox GFP_KERNEL); 2767b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!dev->queues) 2768b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto free; 276942f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch dev->io_queue = alloc_percpu(unsigned short); 277042f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch if (!dev->io_queue) 277142f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch goto free; 2772b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2773b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox INIT_LIST_HEAD(&dev->namespaces); 27749ca9737444f1a8602f74b85018d881e7e54b5bd1Tejun Heo dev->reset_workfn = nvme_reset_failed_dev; 27759ca9737444f1a8602f74b85018d881e7e54b5bd1Tejun Heo INIT_WORK(&dev->reset_work, nvme_reset_workfn); 2776f3db22feb5de6b98b7bae924c2d4b6c8d65bedaeKeith Busch INIT_WORK(&dev->cpu_work, nvme_cpu_workfn); 2777b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->pci_dev = pdev; 27789a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch pci_set_drvdata(pdev, dev); 2779cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh result = nvme_set_instance(dev); 2780cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh if (result) 27810877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch goto free; 2782b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2783091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox result = nvme_setup_prp_pools(dev); 2784091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox if (result) 27850877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch goto release; 2786091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox 2787fb35e914b3f88cda9ee6f9d776910c35269c4ecfKeith Busch kref_init(&dev->kref); 2788f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch result = nvme_dev_start(dev); 2789d82e8bfdef9afae83b894be49af4644d9ac3c359Keith Busch if (result) { 2790d82e8bfdef9afae83b894be49af4644d9ac3c359Keith Busch if (result == -EBUSY) 2791d82e8bfdef9afae83b894be49af4644d9ac3c359Keith Busch goto create_cdev; 27920877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch goto release_pools; 2793d82e8bfdef9afae83b894be49af4644d9ac3c359Keith Busch } 2794b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2795740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox result = nvme_dev_add(dev); 2796d82e8bfdef9afae83b894be49af4644d9ac3c359Keith Busch if (result) 2797f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch goto shutdown; 2798740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox 2799d82e8bfdef9afae83b894be49af4644d9ac3c359Keith Busch create_cdev: 28005e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch scnprintf(dev->name, sizeof(dev->name), "nvme%d", dev->instance); 28015e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch dev->miscdev.minor = MISC_DYNAMIC_MINOR; 28025e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch dev->miscdev.parent = &pdev->dev; 28035e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch dev->miscdev.name = dev->name; 28045e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch dev->miscdev.fops = &nvme_dev_fops; 28055e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch result = misc_register(&dev->miscdev); 28065e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch if (result) 28075e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch goto remove; 28085e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch 2809d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch dev->initialized = 1; 2810b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 2811b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 28125e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch remove: 28135e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch nvme_dev_remove(dev); 28149ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch nvme_free_namespaces(dev); 2815f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch shutdown: 2816f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch nvme_dev_shutdown(dev); 28170877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch release_pools: 2818a1a5ef9998b1379780790b878457b0e0f48b25dbKeith Busch nvme_free_queues(dev, 0); 2819091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox nvme_release_prp_pools(dev); 28200877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch release: 28210877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch nvme_release_instance(dev); 2822b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox free: 282342f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch free_percpu(dev->io_queue); 2824b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(dev->queues); 2825b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(dev->entry); 2826b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(dev); 2827b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return result; 2828b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 2829b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2830f0d54a541751af2c6a16d0c081788cc2aef922a5Keith Buschstatic void nvme_reset_notify(struct pci_dev *pdev, bool prepare) 2831f0d54a541751af2c6a16d0c081788cc2aef922a5Keith Busch{ 2832f0d54a541751af2c6a16d0c081788cc2aef922a5Keith Busch struct nvme_dev *dev = pci_get_drvdata(pdev); 2833f0d54a541751af2c6a16d0c081788cc2aef922a5Keith Busch 2834f0d54a541751af2c6a16d0c081788cc2aef922a5Keith Busch if (prepare) 2835f0d54a541751af2c6a16d0c081788cc2aef922a5Keith Busch nvme_dev_shutdown(dev); 2836f0d54a541751af2c6a16d0c081788cc2aef922a5Keith Busch else 2837f0d54a541751af2c6a16d0c081788cc2aef922a5Keith Busch nvme_dev_resume(dev); 2838f0d54a541751af2c6a16d0c081788cc2aef922a5Keith Busch} 2839f0d54a541751af2c6a16d0c081788cc2aef922a5Keith Busch 284009ece1424f52a2d3ad0ab715e96b0fb342af4f03Keith Buschstatic void nvme_shutdown(struct pci_dev *pdev) 284109ece1424f52a2d3ad0ab715e96b0fb342af4f03Keith Busch{ 284209ece1424f52a2d3ad0ab715e96b0fb342af4f03Keith Busch struct nvme_dev *dev = pci_get_drvdata(pdev); 284309ece1424f52a2d3ad0ab715e96b0fb342af4f03Keith Busch nvme_dev_shutdown(dev); 284409ece1424f52a2d3ad0ab715e96b0fb342af4f03Keith Busch} 284509ece1424f52a2d3ad0ab715e96b0fb342af4f03Keith Busch 28468d85fce77edfc22f1d6dbf78e3af723b4b556f3dGreg Kroah-Hartmanstatic void nvme_remove(struct pci_dev *pdev) 2847b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 2848b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_dev *dev = pci_get_drvdata(pdev); 28499a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch 28509a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch spin_lock(&dev_list_lock); 28519a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch list_del_init(&dev->node); 28529a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch spin_unlock(&dev_list_lock); 28539a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch 28549a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch pci_set_drvdata(pdev, NULL); 28559a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch flush_work(&dev->reset_work); 2856f3db22feb5de6b98b7bae924c2d4b6c8d65bedaeKeith Busch flush_work(&dev->cpu_work); 28575e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch misc_deregister(&dev->miscdev); 28589a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch nvme_dev_remove(dev); 28599a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch nvme_dev_shutdown(dev); 2860a1a5ef9998b1379780790b878457b0e0f48b25dbKeith Busch nvme_free_queues(dev, 0); 28615a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch rcu_barrier(); 28629a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch nvme_release_instance(dev); 28639a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch nvme_release_prp_pools(dev); 28645e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch kref_put(&dev->kref, nvme_free_dev); 2865b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 2866b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2867b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* These functions are yet to be implemented */ 2868b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_error_detected NULL 2869b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_dump_registers NULL 2870b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_link_reset NULL 2871b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_slot_reset NULL 2872b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_error_resume NULL 2873cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch 2874671a6018db1d359254c8201704cf967efb42fc25Jingoo Han#ifdef CONFIG_PM_SLEEP 2875cd63894630ab17a192bf97427d16dbec10710a6aKeith Buschstatic int nvme_suspend(struct device *dev) 2876cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch{ 2877cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch struct pci_dev *pdev = to_pci_dev(dev); 2878cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch struct nvme_dev *ndev = pci_get_drvdata(pdev); 2879cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch 2880cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch nvme_dev_shutdown(ndev); 2881cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch return 0; 2882cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch} 2883cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch 2884cd63894630ab17a192bf97427d16dbec10710a6aKeith Buschstatic int nvme_resume(struct device *dev) 2885cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch{ 2886cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch struct pci_dev *pdev = to_pci_dev(dev); 2887cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch struct nvme_dev *ndev = pci_get_drvdata(pdev); 2888cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch 28899a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch if (nvme_dev_resume(ndev) && !work_busy(&ndev->reset_work)) { 28909ca9737444f1a8602f74b85018d881e7e54b5bd1Tejun Heo ndev->reset_workfn = nvme_reset_failed_dev; 28919a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch queue_work(nvme_workq, &ndev->reset_work); 28929a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch } 28939a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch return 0; 2894cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch} 2895671a6018db1d359254c8201704cf967efb42fc25Jingoo Han#endif 2896cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch 2897cd63894630ab17a192bf97427d16dbec10710a6aKeith Buschstatic SIMPLE_DEV_PM_OPS(nvme_dev_pm_ops, nvme_suspend, nvme_resume); 2898b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 28991d3520357df99baf4ad89f86268ac96cd38092d9Stephen Hemmingerstatic const struct pci_error_handlers nvme_err_handler = { 2900b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .error_detected = nvme_error_detected, 2901b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .mmio_enabled = nvme_dump_registers, 2902b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .link_reset = nvme_link_reset, 2903b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .slot_reset = nvme_slot_reset, 2904b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .resume = nvme_error_resume, 2905f0d54a541751af2c6a16d0c081788cc2aef922a5Keith Busch .reset_notify = nvme_reset_notify, 2906b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 2907b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2908b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* Move to pci_ids.h later */ 2909b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define PCI_CLASS_STORAGE_EXPRESS 0x010802 2910b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 29116eb0d698efa9c2a35ec3ca958699717c603f85eeMatthew Wilcoxstatic const struct pci_device_id nvme_id_table[] = { 2912b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox { PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) }, 2913b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox { 0, } 2914b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 2915b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew WilcoxMODULE_DEVICE_TABLE(pci, nvme_id_table); 2916b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2917b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic struct pci_driver nvme_driver = { 2918b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .name = "nvme", 2919b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .id_table = nvme_id_table, 2920b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .probe = nvme_probe, 29218d85fce77edfc22f1d6dbf78e3af723b4b556f3dGreg Kroah-Hartman .remove = nvme_remove, 292209ece1424f52a2d3ad0ab715e96b0fb342af4f03Keith Busch .shutdown = nvme_shutdown, 2923cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch .driver = { 2924cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch .pm = &nvme_dev_pm_ops, 2925cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch }, 2926b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .err_handler = &nvme_err_handler, 2927b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 2928b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2929b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int __init nvme_init(void) 2930b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 29310ac13140d796eb1e2f8956aea97a6e5e4ebcf981Matthew Wilcox int result; 29321fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 2933b9afca3efb18a9b8392cb544a3e29e8b1168400cDan McLeran init_waitqueue_head(&nvme_kthread_wait); 2934b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 29359a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch nvme_workq = create_singlethread_workqueue("nvme"); 29369a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch if (!nvme_workq) 2937b9afca3efb18a9b8392cb544a3e29e8b1168400cDan McLeran return -ENOMEM; 29389a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch 29395c42ea1643a630060f9e71e06d3933d244970967Keith Busch result = register_blkdev(nvme_major, "nvme"); 29405c42ea1643a630060f9e71e06d3933d244970967Keith Busch if (result < 0) 29419a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch goto kill_workq; 29425c42ea1643a630060f9e71e06d3933d244970967Keith Busch else if (result > 0) 29430ac13140d796eb1e2f8956aea97a6e5e4ebcf981Matthew Wilcox nvme_major = result; 2944b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2945f3db22feb5de6b98b7bae924c2d4b6c8d65bedaeKeith Busch nvme_nb.notifier_call = &nvme_cpu_notify; 2946f3db22feb5de6b98b7bae924c2d4b6c8d65bedaeKeith Busch result = register_hotcpu_notifier(&nvme_nb); 29471fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox if (result) 29481fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox goto unregister_blkdev; 2949f3db22feb5de6b98b7bae924c2d4b6c8d65bedaeKeith Busch 2950f3db22feb5de6b98b7bae924c2d4b6c8d65bedaeKeith Busch result = pci_register_driver(&nvme_driver); 2951f3db22feb5de6b98b7bae924c2d4b6c8d65bedaeKeith Busch if (result) 2952f3db22feb5de6b98b7bae924c2d4b6c8d65bedaeKeith Busch goto unregister_hotcpu; 29531fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox return 0; 2954b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2955f3db22feb5de6b98b7bae924c2d4b6c8d65bedaeKeith Busch unregister_hotcpu: 2956f3db22feb5de6b98b7bae924c2d4b6c8d65bedaeKeith Busch unregister_hotcpu_notifier(&nvme_nb); 29571fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox unregister_blkdev: 2958b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unregister_blkdev(nvme_major, "nvme"); 29599a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch kill_workq: 29609a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch destroy_workqueue(nvme_workq); 2961b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return result; 2962b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 2963b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2964b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void __exit nvme_exit(void) 2965b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 2966b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox pci_unregister_driver(&nvme_driver); 2967f3db22feb5de6b98b7bae924c2d4b6c8d65bedaeKeith Busch unregister_hotcpu_notifier(&nvme_nb); 2968b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unregister_blkdev(nvme_major, "nvme"); 29699a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch destroy_workqueue(nvme_workq); 2970b9afca3efb18a9b8392cb544a3e29e8b1168400cDan McLeran BUG_ON(nvme_thread && !IS_ERR(nvme_thread)); 297121bd78bcf4208e84deab0d34f9d4e034d0580d0cMatthew Wilcox _nvme_check_size(); 2972b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 2973b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2974b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew WilcoxMODULE_AUTHOR("Matthew Wilcox <willy@linux.intel.com>"); 2975b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew WilcoxMODULE_LICENSE("GPL"); 29766eb0d698efa9c2a35ec3ca958699717c603f85eeMatthew WilcoxMODULE_VERSION("0.9"); 2977b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxmodule_init(nvme_init); 2978b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxmodule_exit(nvme_exit); 2979