nvme-core.c revision b355084a891985d4cd0ca23b1a83366af2c4232d
1b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 2b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * NVM Express device driver 36eb0d698efa9c2a35ec3ca958699717c603f85eeMatthew Wilcox * Copyright (c) 2011-2014, Intel Corporation. 4b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 5b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * This program is free software; you can redistribute it and/or modify it 6b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * under the terms and conditions of the GNU General Public License, 7b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * version 2, as published by the Free Software Foundation. 8b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 9b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * This program is distributed in the hope it will be useful, but WITHOUT 10b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * more details. 13b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 14b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * You should have received a copy of the GNU General Public License along with 15b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * this program; if not, write to the Free Software Foundation, Inc., 16b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. 17b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 18b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 19b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/nvme.h> 20b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/bio.h> 218de055350fbaa96b6563892c195a60be583faa9cMatthew Wilcox#include <linux/bitops.h> 22b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/blkdev.h> 2342f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch#include <linux/cpu.h> 24fd63e9ceeeae58cfe877c2d49d41c1bf7532303cMatthew Wilcox#include <linux/delay.h> 25b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/errno.h> 26b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/fs.h> 27b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/genhd.h> 285aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox#include <linux/idr.h> 29b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/init.h> 30b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/interrupt.h> 31b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/io.h> 32b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/kdev_t.h> 331fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox#include <linux/kthread.h> 34b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/kernel.h> 35b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/mm.h> 36b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/module.h> 37b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/moduleparam.h> 38b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/pci.h> 3942f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch#include <linux/percpu.h> 40be7b62754e097adc0cb16c25c9ee86ee20de62fbMatthew Wilcox#include <linux/poison.h> 41c3bfe7176c035a0a2c70bc79180fb13a6c57142aMatthew Wilcox#include <linux/ptrace.h> 42b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/sched.h> 43b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/slab.h> 44b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/types.h> 455d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma#include <scsi/sg.h> 46797a796a13df6b84a4791e57306737059b5b2384Hitoshi Mitake#include <asm-generic/io-64-nonatomic-lo-hi.h> 47797a796a13df6b84a4791e57306737059b5b2384Hitoshi Mitake 48b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define NVME_Q_DEPTH 1024 49b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define SQ_SIZE(depth) (depth * sizeof(struct nvme_command)) 50b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define CQ_SIZE(depth) (depth * sizeof(struct nvme_completion)) 51e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox#define ADMIN_TIMEOUT (60 * HZ) 52b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 53b355084a891985d4cd0ca23b1a83366af2c4232dKeith Buschunsigned char io_timeout = 30; 54b355084a891985d4cd0ca23b1a83366af2c4232dKeith Buschmodule_param(io_timeout, byte, 0644); 55b355084a891985d4cd0ca23b1a83366af2c4232dKeith BuschMODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O"); 56b355084a891985d4cd0ca23b1a83366af2c4232dKeith Busch 57b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_major; 58b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxmodule_param(nvme_major, int, 0); 59b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 6058ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcoxstatic int use_threaded_interrupts; 6158ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcoxmodule_param(use_threaded_interrupts, int, 0); 6258ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox 631fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcoxstatic DEFINE_SPINLOCK(dev_list_lock); 641fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcoxstatic LIST_HEAD(dev_list); 651fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcoxstatic struct task_struct *nvme_thread; 669a6b94584de1a0467d85b435df9c744c5c45a270Keith Buschstatic struct workqueue_struct *nvme_workq; 671fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 68d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Buschstatic void nvme_reset_failed_dev(struct work_struct *ws); 69d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch 704d115420707afcabe77d2535e092356df6664b70Keith Buschstruct async_cmd_info { 714d115420707afcabe77d2535e092356df6664b70Keith Busch struct kthread_work work; 724d115420707afcabe77d2535e092356df6664b70Keith Busch struct kthread_worker *worker; 734d115420707afcabe77d2535e092356df6664b70Keith Busch u32 result; 744d115420707afcabe77d2535e092356df6664b70Keith Busch int status; 754d115420707afcabe77d2535e092356df6664b70Keith Busch void *ctx; 764d115420707afcabe77d2535e092356df6664b70Keith Busch}; 771fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 78b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 79b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * An NVM Express queue. Each device has at least two (one for admin 80b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * commands and one for I/O commands). 81b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 82b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstruct nvme_queue { 835a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch struct rcu_head r_head; 84b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct device *q_dmadev; 85091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox struct nvme_dev *dev; 863193f07bb7ae723f33ac8e8b9db317a4f68d7d18Matthew Wilcox char irqname[24]; /* nvme4294967295-65535\0 */ 87b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox spinlock_t q_lock; 88b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_command *sq_cmds; 89b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox volatile struct nvme_completion *cqes; 90b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_addr_t sq_dma_addr; 91b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_addr_t cq_dma_addr; 92b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox wait_queue_head_t sq_full; 931fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox wait_queue_t sq_cong_wait; 94b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct bio_list sq_cong; 95b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 __iomem *q_db; 96b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 q_depth; 97b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 cq_vector; 98b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 sq_head; 99b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 sq_tail; 100b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 cq_head; 101c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch u16 qid; 102e9539f47525ecee05c9f22c3565885f3e9492c52Matthew Wilcox u8 cq_phase; 103e9539f47525ecee05c9f22c3565885f3e9492c52Matthew Wilcox u8 cqe_seen; 104224042742582c9938788b81165180c876e997a07Keith Busch u8 q_suspended; 10542f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch cpumask_var_t cpu_mask; 1064d115420707afcabe77d2535e092356df6664b70Keith Busch struct async_cmd_info cmdinfo; 107b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unsigned long cmdid_data[]; 108b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 109b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 110b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 111b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Check we didin't inadvertently grow the command struct 112b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 113b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic inline void _nvme_check_size(void) 114b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 115b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64); 116b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_create_cq) != 64); 117b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_create_sq) != 64); 118b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64); 119b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_features) != 64); 120f8ebf8409abfdaeeb8c847381629a2a8b8e3d816Vishal Verma BUILD_BUG_ON(sizeof(struct nvme_format_cmd) != 64); 121c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch BUILD_BUG_ON(sizeof(struct nvme_abort_cmd) != 64); 122b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_command) != 64); 123b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != 4096); 124b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_id_ns) != 4096); 125b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64); 1266ecec74520d8a357726e6c12f99080dbe7b347ddKeith Busch BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512); 127b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 128b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1295c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcoxtypedef void (*nvme_completion_fn)(struct nvme_dev *, void *, 130c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox struct nvme_completion *); 131c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox 132e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcoxstruct nvme_cmd_info { 133c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox nvme_completion_fn fn; 134c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox void *ctx; 135e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox unsigned long timeout; 136c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch int aborted; 137e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox}; 138e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox 139e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcoxstatic struct nvme_cmd_info *nvme_cmd_info(struct nvme_queue *nvmeq) 140e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox{ 141e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox return (void *)&nvmeq->cmdid_data[BITS_TO_LONGS(nvmeq->q_depth)]; 142e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox} 143e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox 144224042742582c9938788b81165180c876e997a07Keith Buschstatic unsigned nvme_queue_extra(int depth) 145224042742582c9938788b81165180c876e997a07Keith Busch{ 146224042742582c9938788b81165180c876e997a07Keith Busch return DIV_ROUND_UP(depth, 8) + (depth * sizeof(struct nvme_cmd_info)); 147224042742582c9938788b81165180c876e997a07Keith Busch} 148224042742582c9938788b81165180c876e997a07Keith Busch 149b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/** 150714a7a22884b74862540bc84955274d86b2f6040Matthew Wilcox * alloc_cmdid() - Allocate a Command ID 151714a7a22884b74862540bc84955274d86b2f6040Matthew Wilcox * @nvmeq: The queue that will be used for this command 152714a7a22884b74862540bc84955274d86b2f6040Matthew Wilcox * @ctx: A pointer that will be passed to the handler 153c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox * @handler: The function to call on completion 154b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 155b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Allocate a Command ID for a queue. The data passed in will 156b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * be passed to the completion handler. This is implemented by using 157b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * the bottom two bits of the ctx pointer to store the handler ID. 158b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Passing in a pointer that's not 4-byte aligned will cause a BUG. 159b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * We can change this if it becomes a problem. 160184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox * 161184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox * May be called with local interrupts disabled and the q_lock held, 162184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox * or with interrupts enabled and no locks held. 163b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 164c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcoxstatic int alloc_cmdid(struct nvme_queue *nvmeq, void *ctx, 165c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox nvme_completion_fn handler, unsigned timeout) 166b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 167e6d15f79f997a98b3a69abbc462fc9041cc1a7b4Matthew Wilcox int depth = nvmeq->q_depth - 1; 168e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox struct nvme_cmd_info *info = nvme_cmd_info(nvmeq); 169b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int cmdid; 170b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 171b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox do { 172b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cmdid = find_first_zero_bit(nvmeq->cmdid_data, depth); 173b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (cmdid >= depth) 174b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -EBUSY; 175b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } while (test_and_set_bit(cmdid, nvmeq->cmdid_data)); 176b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 177c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox info[cmdid].fn = handler; 178c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox info[cmdid].ctx = ctx; 179e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox info[cmdid].timeout = jiffies + timeout; 180c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch info[cmdid].aborted = 0; 181b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return cmdid; 182b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 183b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 184b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int alloc_cmdid_killable(struct nvme_queue *nvmeq, void *ctx, 185c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox nvme_completion_fn handler, unsigned timeout) 186b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 187b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int cmdid; 188b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox wait_event_killable(nvmeq->sq_full, 189e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox (cmdid = alloc_cmdid(nvmeq, ctx, handler, timeout)) >= 0); 190b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return (cmdid < 0) ? -EINTR : cmdid; 191b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 192b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 193c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox/* Special values must be less than 0x1000 */ 194c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox#define CMD_CTX_BASE ((void *)POISON_POINTER_DELTA) 195d2d8703481f60d67f49e3177196cbe474b11377cMatthew Wilcox#define CMD_CTX_CANCELLED (0x30C + CMD_CTX_BASE) 196d2d8703481f60d67f49e3177196cbe474b11377cMatthew Wilcox#define CMD_CTX_COMPLETED (0x310 + CMD_CTX_BASE) 197d2d8703481f60d67f49e3177196cbe474b11377cMatthew Wilcox#define CMD_CTX_INVALID (0x314 + CMD_CTX_BASE) 19800df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox#define CMD_CTX_FLUSH (0x318 + CMD_CTX_BASE) 199c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch#define CMD_CTX_ABORT (0x31C + CMD_CTX_BASE) 200be7b62754e097adc0cb16c25c9ee86ee20de62fbMatthew Wilcox 2015c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcoxstatic void special_completion(struct nvme_dev *dev, void *ctx, 202c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox struct nvme_completion *cqe) 203c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox{ 204c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox if (ctx == CMD_CTX_CANCELLED) 205c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox return; 206c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox if (ctx == CMD_CTX_FLUSH) 207c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox return; 208c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch if (ctx == CMD_CTX_ABORT) { 209c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch ++dev->abort_limit; 210c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch return; 211c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch } 212c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox if (ctx == CMD_CTX_COMPLETED) { 2135c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcox dev_warn(&dev->pci_dev->dev, 214c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox "completed id %d twice on queue %d\n", 215c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox cqe->command_id, le16_to_cpup(&cqe->sq_id)); 216c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox return; 217c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox } 218c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox if (ctx == CMD_CTX_INVALID) { 2195c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcox dev_warn(&dev->pci_dev->dev, 220c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox "invalid id %d completed on queue %d\n", 221c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox cqe->command_id, le16_to_cpup(&cqe->sq_id)); 222c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox return; 223c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox } 224c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox 2255c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcox dev_warn(&dev->pci_dev->dev, "Unknown special completion %p\n", ctx); 226c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox} 227c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox 2284d115420707afcabe77d2535e092356df6664b70Keith Buschstatic void async_completion(struct nvme_dev *dev, void *ctx, 2294d115420707afcabe77d2535e092356df6664b70Keith Busch struct nvme_completion *cqe) 2304d115420707afcabe77d2535e092356df6664b70Keith Busch{ 2314d115420707afcabe77d2535e092356df6664b70Keith Busch struct async_cmd_info *cmdinfo = ctx; 2324d115420707afcabe77d2535e092356df6664b70Keith Busch cmdinfo->result = le32_to_cpup(&cqe->result); 2334d115420707afcabe77d2535e092356df6664b70Keith Busch cmdinfo->status = le16_to_cpup(&cqe->status) >> 1; 2344d115420707afcabe77d2535e092356df6664b70Keith Busch queue_kthread_work(cmdinfo->worker, &cmdinfo->work); 2354d115420707afcabe77d2535e092356df6664b70Keith Busch} 2364d115420707afcabe77d2535e092356df6664b70Keith Busch 237184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox/* 238184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox * Called with local interrupts disabled and the q_lock held. May not sleep. 239184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox */ 240c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcoxstatic void *free_cmdid(struct nvme_queue *nvmeq, int cmdid, 241c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox nvme_completion_fn *fn) 242b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 243c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox void *ctx; 244e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox struct nvme_cmd_info *info = nvme_cmd_info(nvmeq); 245b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 246c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox if (cmdid >= nvmeq->q_depth) { 247c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox *fn = special_completion; 24848e3d39816416b3bf03dee3a796c0c04427c1a31Matthew Wilcox return CMD_CTX_INVALID; 249c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox } 250859361a228258edf4821d9f5635825033eca78e8Keith Busch if (fn) 251859361a228258edf4821d9f5635825033eca78e8Keith Busch *fn = info[cmdid].fn; 252c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox ctx = info[cmdid].ctx; 253c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox info[cmdid].fn = special_completion; 254e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox info[cmdid].ctx = CMD_CTX_COMPLETED; 255b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox clear_bit(cmdid, nvmeq->cmdid_data); 256b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox wake_up(&nvmeq->sq_full); 257c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox return ctx; 258b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 259b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 260c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcoxstatic void *cancel_cmdid(struct nvme_queue *nvmeq, int cmdid, 261c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox nvme_completion_fn *fn) 2623c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox{ 263c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox void *ctx; 264e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox struct nvme_cmd_info *info = nvme_cmd_info(nvmeq); 265c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox if (fn) 266c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox *fn = info[cmdid].fn; 267c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox ctx = info[cmdid].ctx; 268c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox info[cmdid].fn = special_completion; 269e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox info[cmdid].ctx = CMD_CTX_CANCELLED; 270c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox return ctx; 2713c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox} 2723c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox 2735a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Buschstatic struct nvme_queue *raw_nvmeq(struct nvme_dev *dev, int qid) 274b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 2755a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch return rcu_dereference_raw(dev->queues[qid]); 276b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 277b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2784f5099af4f3d5f999d8ab7784472d93e810e3912Keith Buschstatic struct nvme_queue *get_nvmeq(struct nvme_dev *dev) __acquires(RCU) 2795a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch{ 28042f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch unsigned queue_id = get_cpu_var(*dev->io_queue); 2815a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch rcu_read_lock(); 28242f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch return rcu_dereference(dev->queues[queue_id]); 2835a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch} 2845a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch 2854f5099af4f3d5f999d8ab7784472d93e810e3912Keith Buschstatic void put_nvmeq(struct nvme_queue *nvmeq) __releases(RCU) 286b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 2875a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch rcu_read_unlock(); 28842f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch put_cpu_var(nvmeq->dev->io_queue); 289b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 290b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2914f5099af4f3d5f999d8ab7784472d93e810e3912Keith Buschstatic struct nvme_queue *lock_nvmeq(struct nvme_dev *dev, int q_idx) 2924f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch __acquires(RCU) 2934f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch{ 2944f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch rcu_read_lock(); 2954f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch return rcu_dereference(dev->queues[q_idx]); 2964f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch} 2974f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch 2984f5099af4f3d5f999d8ab7784472d93e810e3912Keith Buschstatic void unlock_nvmeq(struct nvme_queue *nvmeq) __releases(RCU) 2994f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch{ 3004f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch rcu_read_unlock(); 3014f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch} 3024f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch 303b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/** 304714a7a22884b74862540bc84955274d86b2f6040Matthew Wilcox * nvme_submit_cmd() - Copy a command into a queue and ring the doorbell 305b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * @nvmeq: The queue to use 306b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * @cmd: The command to send 307b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 308b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Safe to use from interrupt context 309b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 310b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_submit_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd) 311b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 312b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unsigned long flags; 313b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 tail; 314b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox spin_lock_irqsave(&nvmeq->q_lock, flags); 3154f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch if (nvmeq->q_suspended) { 3164f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch spin_unlock_irqrestore(&nvmeq->q_lock, flags); 3174f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch return -EBUSY; 3184f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch } 319b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox tail = nvmeq->sq_tail; 320b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memcpy(&nvmeq->sq_cmds[tail], cmd, sizeof(*cmd)); 321b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (++tail == nvmeq->q_depth) 322b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox tail = 0; 3237547881d0951384f9833ec3a80fac8f3f16f3b98Matthew Wilcox writel(tail, nvmeq->q_db); 324b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->sq_tail = tail; 325b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox spin_unlock_irqrestore(&nvmeq->q_lock, flags); 326b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 327b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 328b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 329b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 330eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcoxstatic __le64 **iod_list(struct nvme_iod *iod) 331e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews{ 332eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return ((void *)iod) + iod->offset; 333e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews} 334e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews 335eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox/* 336eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox * Will slightly overestimate the number of pages needed. This is OK 337eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox * as it only leads to a small amount of wasted memory for the lifetime of 338eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox * the I/O. 339eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox */ 340eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcoxstatic int nvme_npages(unsigned size) 341eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox{ 342eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox unsigned nprps = DIV_ROUND_UP(size + PAGE_SIZE, PAGE_SIZE); 343eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return DIV_ROUND_UP(8 * nprps, PAGE_SIZE - 8); 344eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox} 345b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 346eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcoxstatic struct nvme_iod * 347eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcoxnvme_alloc_iod(unsigned nseg, unsigned nbytes, gfp_t gfp) 348b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 349eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox struct nvme_iod *iod = kmalloc(sizeof(struct nvme_iod) + 350eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox sizeof(__le64 *) * nvme_npages(nbytes) + 351eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox sizeof(struct scatterlist) * nseg, gfp); 352eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox 353eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox if (iod) { 354eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->offset = offsetof(struct nvme_iod, sg[nseg]); 355eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->npages = -1; 356eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->length = nbytes; 3572b1960341576bf51c01b12fefeb1cc53820923e7Keith Busch iod->nents = 0; 3586198221fa0df0298513b35796f63f242ea97134eKeith Busch iod->start_time = jiffies; 359eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox } 360eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox 361eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return iod; 362b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 363b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 3645d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermavoid nvme_free_iod(struct nvme_dev *dev, struct nvme_iod *iod) 365b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 366eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox const int last_prp = PAGE_SIZE / 8 - 1; 367eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox int i; 368eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox __le64 **list = iod_list(iod); 369eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox dma_addr_t prp_dma = iod->first_dma; 370eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox 371eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox if (iod->npages == 0) 372eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox dma_pool_free(dev->prp_small_pool, list[0], prp_dma); 373eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox for (i = 0; i < iod->npages; i++) { 374eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox __le64 *prp_list = list[i]; 375eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox dma_addr_t next_prp_dma = le64_to_cpu(prp_list[last_prp]); 376eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox dma_pool_free(dev->prp_page_pool, prp_list, prp_dma); 377eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox prp_dma = next_prp_dma; 378eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox } 379eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox kfree(iod); 380b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 381b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 3826198221fa0df0298513b35796f63f242ea97134eKeith Buschstatic void nvme_start_io_acct(struct bio *bio) 3836198221fa0df0298513b35796f63f242ea97134eKeith Busch{ 3846198221fa0df0298513b35796f63f242ea97134eKeith Busch struct gendisk *disk = bio->bi_bdev->bd_disk; 3856198221fa0df0298513b35796f63f242ea97134eKeith Busch const int rw = bio_data_dir(bio); 3866198221fa0df0298513b35796f63f242ea97134eKeith Busch int cpu = part_stat_lock(); 3876198221fa0df0298513b35796f63f242ea97134eKeith Busch part_round_stats(cpu, &disk->part0); 3886198221fa0df0298513b35796f63f242ea97134eKeith Busch part_stat_inc(cpu, &disk->part0, ios[rw]); 3896198221fa0df0298513b35796f63f242ea97134eKeith Busch part_stat_add(cpu, &disk->part0, sectors[rw], bio_sectors(bio)); 3906198221fa0df0298513b35796f63f242ea97134eKeith Busch part_inc_in_flight(&disk->part0, rw); 3916198221fa0df0298513b35796f63f242ea97134eKeith Busch part_stat_unlock(); 3926198221fa0df0298513b35796f63f242ea97134eKeith Busch} 3936198221fa0df0298513b35796f63f242ea97134eKeith Busch 3946198221fa0df0298513b35796f63f242ea97134eKeith Buschstatic void nvme_end_io_acct(struct bio *bio, unsigned long start_time) 3956198221fa0df0298513b35796f63f242ea97134eKeith Busch{ 3966198221fa0df0298513b35796f63f242ea97134eKeith Busch struct gendisk *disk = bio->bi_bdev->bd_disk; 3976198221fa0df0298513b35796f63f242ea97134eKeith Busch const int rw = bio_data_dir(bio); 3986198221fa0df0298513b35796f63f242ea97134eKeith Busch unsigned long duration = jiffies - start_time; 3996198221fa0df0298513b35796f63f242ea97134eKeith Busch int cpu = part_stat_lock(); 4006198221fa0df0298513b35796f63f242ea97134eKeith Busch part_stat_add(cpu, &disk->part0, ticks[rw], duration); 4016198221fa0df0298513b35796f63f242ea97134eKeith Busch part_round_stats(cpu, &disk->part0); 4026198221fa0df0298513b35796f63f242ea97134eKeith Busch part_dec_in_flight(&disk->part0, rw); 4036198221fa0df0298513b35796f63f242ea97134eKeith Busch part_stat_unlock(); 4046198221fa0df0298513b35796f63f242ea97134eKeith Busch} 4056198221fa0df0298513b35796f63f242ea97134eKeith Busch 4065c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcoxstatic void bio_completion(struct nvme_dev *dev, void *ctx, 407b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_completion *cqe) 408b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 409eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox struct nvme_iod *iod = ctx; 410eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox struct bio *bio = iod->private; 411b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 status = le16_to_cpup(&cqe->status) >> 1; 412b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 4139e59d091b0eb04f223ed037348e3d9e36f30e72bKeith Busch if (iod->nents) { 4142b1960341576bf51c01b12fefeb1cc53820923e7Keith Busch dma_unmap_sg(&dev->pci_dev->dev, iod->sg, iod->nents, 415b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox bio_data_dir(bio) ? DMA_TO_DEVICE : DMA_FROM_DEVICE); 4169e59d091b0eb04f223ed037348e3d9e36f30e72bKeith Busch nvme_end_io_acct(bio, iod->start_time); 4179e59d091b0eb04f223ed037348e3d9e36f30e72bKeith Busch } 418eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox nvme_free_iod(dev, iod); 419427e97080196548557b288517537ab7eb48c309fKeith Busch if (status) 4201ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox bio_endio(bio, -EIO); 421427e97080196548557b288517537ab7eb48c309fKeith Busch else 4221ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox bio_endio(bio, 0); 423b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 424b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 425184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox/* length is in bytes. gfp flags indicates whether we may sleep. */ 4265d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermaint nvme_setup_prps(struct nvme_dev *dev, struct nvme_common_command *cmd, 4275d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma struct nvme_iod *iod, int total_len, gfp_t gfp) 428ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox{ 42999802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox struct dma_pool *pool; 430eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox int length = total_len; 431eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox struct scatterlist *sg = iod->sg; 432ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox int dma_len = sg_dma_len(sg); 433ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox u64 dma_addr = sg_dma_address(sg); 434ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox int offset = offset_in_page(dma_addr); 435e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews __le64 *prp_list; 436eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox __le64 **list = iod_list(iod); 437e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_addr_t prp_dma; 438eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox int nprps, i; 439ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox 440ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmd->prp1 = cpu_to_le64(dma_addr); 441ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox length -= (PAGE_SIZE - offset); 442ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox if (length <= 0) 443eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return total_len; 444ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox 445ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox dma_len -= (PAGE_SIZE - offset); 446ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox if (dma_len) { 447ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox dma_addr += (PAGE_SIZE - offset); 448ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox } else { 449ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox sg = sg_next(sg); 450ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox dma_addr = sg_dma_address(sg); 451ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox dma_len = sg_dma_len(sg); 452ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox } 453ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox 454ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox if (length <= PAGE_SIZE) { 455ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmd->prp2 = cpu_to_le64(dma_addr); 456eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return total_len; 457e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews } 458e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews 459e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews nprps = DIV_ROUND_UP(length, PAGE_SIZE); 46099802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox if (nprps <= (256 / 8)) { 46199802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox pool = dev->prp_small_pool; 462eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->npages = 0; 46399802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox } else { 46499802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox pool = dev->prp_page_pool; 465eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->npages = 1; 46699802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox } 46799802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox 468b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox prp_list = dma_pool_alloc(pool, gfp, &prp_dma); 469b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox if (!prp_list) { 470b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox cmd->prp2 = cpu_to_le64(dma_addr); 471eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->npages = -1; 472eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return (total_len - length) + PAGE_SIZE; 473b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox } 474eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox list[0] = prp_list; 475eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->first_dma = prp_dma; 476e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews cmd->prp2 = cpu_to_le64(prp_dma); 477e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews i = 0; 478e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews for (;;) { 4797523d834dd1573610078eb1ac0933f6490232f90Matthew Wilcox if (i == PAGE_SIZE / 8) { 480e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews __le64 *old_prp_list = prp_list; 481b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox prp_list = dma_pool_alloc(pool, gfp, &prp_dma); 482eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox if (!prp_list) 483eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return total_len - length; 484eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox list[iod->npages++] = prp_list; 4857523d834dd1573610078eb1ac0933f6490232f90Matthew Wilcox prp_list[0] = old_prp_list[i - 1]; 4867523d834dd1573610078eb1ac0933f6490232f90Matthew Wilcox old_prp_list[i - 1] = cpu_to_le64(prp_dma); 4877523d834dd1573610078eb1ac0933f6490232f90Matthew Wilcox i = 1; 488e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews } 489e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews prp_list[i++] = cpu_to_le64(dma_addr); 490e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_len -= PAGE_SIZE; 491e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_addr += PAGE_SIZE; 492e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews length -= PAGE_SIZE; 493e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews if (length <= 0) 494e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews break; 495e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews if (dma_len > 0) 496e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews continue; 497e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews BUG_ON(dma_len < 0); 498e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews sg = sg_next(sg); 499e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_addr = sg_dma_address(sg); 500e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_len = sg_dma_len(sg); 501ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox } 502ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox 503eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return total_len; 504ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox} 505ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox 506427e97080196548557b288517537ab7eb48c309fKeith Buschstatic int nvme_split_and_submit(struct bio *bio, struct nvme_queue *nvmeq, 50720d0189b1012a37d2533a87fb451f7852f2418d1Kent Overstreet int len) 508427e97080196548557b288517537ab7eb48c309fKeith Busch{ 50920d0189b1012a37d2533a87fb451f7852f2418d1Kent Overstreet struct bio *split = bio_split(bio, len >> 9, GFP_ATOMIC, NULL); 51020d0189b1012a37d2533a87fb451f7852f2418d1Kent Overstreet if (!split) 511427e97080196548557b288517537ab7eb48c309fKeith Busch return -ENOMEM; 512427e97080196548557b288517537ab7eb48c309fKeith Busch 51320d0189b1012a37d2533a87fb451f7852f2418d1Kent Overstreet bio_chain(split, bio); 51420d0189b1012a37d2533a87fb451f7852f2418d1Kent Overstreet 515427e97080196548557b288517537ab7eb48c309fKeith Busch if (bio_list_empty(&nvmeq->sq_cong)) 516427e97080196548557b288517537ab7eb48c309fKeith Busch add_wait_queue(&nvmeq->sq_full, &nvmeq->sq_cong_wait); 51720d0189b1012a37d2533a87fb451f7852f2418d1Kent Overstreet bio_list_add(&nvmeq->sq_cong, split); 51820d0189b1012a37d2533a87fb451f7852f2418d1Kent Overstreet bio_list_add(&nvmeq->sq_cong, bio); 519427e97080196548557b288517537ab7eb48c309fKeith Busch 520427e97080196548557b288517537ab7eb48c309fKeith Busch return 0; 521427e97080196548557b288517537ab7eb48c309fKeith Busch} 522427e97080196548557b288517537ab7eb48c309fKeith Busch 5231ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox/* NVMe scatterlists require no holes in the virtual address */ 5241ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox#define BIOVEC_NOT_VIRT_MERGEABLE(vec1, vec2) ((vec2)->bv_offset || \ 5251ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox (((vec1)->bv_offset + (vec1)->bv_len) % PAGE_SIZE)) 5261ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox 527427e97080196548557b288517537ab7eb48c309fKeith Buschstatic int nvme_map_bio(struct nvme_queue *nvmeq, struct nvme_iod *iod, 528b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct bio *bio, enum dma_data_direction dma_dir, int psegs) 529b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 5307988613b0e5b2638caf6cd493cc78e9595eba19cKent Overstreet struct bio_vec bvec, bvprv; 5317988613b0e5b2638caf6cd493cc78e9595eba19cKent Overstreet struct bvec_iter iter; 532768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox struct scatterlist *sg = NULL; 5337988613b0e5b2638caf6cd493cc78e9595eba19cKent Overstreet int length = 0, nsegs = 0, split_len = bio->bi_iter.bi_size; 5347988613b0e5b2638caf6cd493cc78e9595eba19cKent Overstreet int first = 1; 535159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch 536159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch if (nvmeq->dev->stripe_size) 537159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch split_len = nvmeq->dev->stripe_size - 5384f024f3797c43cb4b73cd2c50cec728842d0e49eKent Overstreet ((bio->bi_iter.bi_sector << 9) & 5394f024f3797c43cb4b73cd2c50cec728842d0e49eKent Overstreet (nvmeq->dev->stripe_size - 1)); 540b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 541eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox sg_init_table(iod->sg, psegs); 5427988613b0e5b2638caf6cd493cc78e9595eba19cKent Overstreet bio_for_each_segment(bvec, bio, iter) { 5437988613b0e5b2638caf6cd493cc78e9595eba19cKent Overstreet if (!first && BIOVEC_PHYS_MERGEABLE(&bvprv, &bvec)) { 5447988613b0e5b2638caf6cd493cc78e9595eba19cKent Overstreet sg->length += bvec.bv_len; 545768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox } else { 5467988613b0e5b2638caf6cd493cc78e9595eba19cKent Overstreet if (!first && BIOVEC_NOT_VIRT_MERGEABLE(&bvprv, &bvec)) 5477988613b0e5b2638caf6cd493cc78e9595eba19cKent Overstreet return nvme_split_and_submit(bio, nvmeq, 54820d0189b1012a37d2533a87fb451f7852f2418d1Kent Overstreet length); 549427e97080196548557b288517537ab7eb48c309fKeith Busch 550eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox sg = sg ? sg + 1 : iod->sg; 5517988613b0e5b2638caf6cd493cc78e9595eba19cKent Overstreet sg_set_page(sg, bvec.bv_page, 5527988613b0e5b2638caf6cd493cc78e9595eba19cKent Overstreet bvec.bv_len, bvec.bv_offset); 553768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox nsegs++; 554768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox } 555159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch 5567988613b0e5b2638caf6cd493cc78e9595eba19cKent Overstreet if (split_len - length < bvec.bv_len) 55720d0189b1012a37d2533a87fb451f7852f2418d1Kent Overstreet return nvme_split_and_submit(bio, nvmeq, split_len); 5587988613b0e5b2638caf6cd493cc78e9595eba19cKent Overstreet length += bvec.bv_len; 559768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox bvprv = bvec; 5607988613b0e5b2638caf6cd493cc78e9595eba19cKent Overstreet first = 0; 561b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 562eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->nents = nsegs; 563768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox sg_mark_end(sg); 564427e97080196548557b288517537ab7eb48c309fKeith Busch if (dma_map_sg(nvmeq->q_dmadev, iod->sg, iod->nents, dma_dir) == 0) 5651ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox return -ENOMEM; 566427e97080196548557b288517537ab7eb48c309fKeith Busch 5674f024f3797c43cb4b73cd2c50cec728842d0e49eKent Overstreet BUG_ON(length != bio->bi_iter.bi_size); 5681ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox return length; 569b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 570b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 5710e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch/* 5720e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch * We reuse the small pool to allocate the 16-byte range here as it is not 5730e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch * worth having a special pool for these or additional cases to handle freeing 5740e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch * the iod. 5750e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch */ 5760e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Buschstatic int nvme_submit_discard(struct nvme_queue *nvmeq, struct nvme_ns *ns, 5770e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch struct bio *bio, struct nvme_iod *iod, int cmdid) 5780e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch{ 5790e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch struct nvme_dsm_range *range; 5800e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch struct nvme_command *cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail]; 5810e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch 5820e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch range = dma_pool_alloc(nvmeq->dev->prp_small_pool, GFP_ATOMIC, 5830e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch &iod->first_dma); 5840e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch if (!range) 5850e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch return -ENOMEM; 5860e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch 5870e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch iod_list(iod)[0] = (__le64 *)range; 5880e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch iod->npages = 0; 5890e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch 5900e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch range->cattr = cpu_to_le32(0); 5914f024f3797c43cb4b73cd2c50cec728842d0e49eKent Overstreet range->nlb = cpu_to_le32(bio->bi_iter.bi_size >> ns->lba_shift); 5924f024f3797c43cb4b73cd2c50cec728842d0e49eKent Overstreet range->slba = cpu_to_le64(nvme_block_nr(ns, bio->bi_iter.bi_sector)); 5930e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch 5940e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch memset(cmnd, 0, sizeof(*cmnd)); 5950e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch cmnd->dsm.opcode = nvme_cmd_dsm; 5960e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch cmnd->dsm.command_id = cmdid; 5970e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch cmnd->dsm.nsid = cpu_to_le32(ns->ns_id); 5980e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch cmnd->dsm.prp1 = cpu_to_le64(iod->first_dma); 5990e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch cmnd->dsm.nr = 0; 6000e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD); 6010e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch 6020e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch if (++nvmeq->sq_tail == nvmeq->q_depth) 6030e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch nvmeq->sq_tail = 0; 6040e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch writel(nvmeq->sq_tail, nvmeq->q_db); 6050e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch 6060e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch return 0; 6070e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch} 6080e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch 60900df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcoxstatic int nvme_submit_flush(struct nvme_queue *nvmeq, struct nvme_ns *ns, 61000df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox int cmdid) 61100df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox{ 61200df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox struct nvme_command *cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail]; 61300df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 61400df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox memset(cmnd, 0, sizeof(*cmnd)); 61500df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox cmnd->common.opcode = nvme_cmd_flush; 61600df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox cmnd->common.command_id = cmdid; 61700df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox cmnd->common.nsid = cpu_to_le32(ns->ns_id); 61800df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 61900df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox if (++nvmeq->sq_tail == nvmeq->q_depth) 62000df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox nvmeq->sq_tail = 0; 62100df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox writel(nvmeq->sq_tail, nvmeq->q_db); 62200df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 62300df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox return 0; 62400df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox} 62500df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 6265d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermaint nvme_submit_flush_data(struct nvme_queue *nvmeq, struct nvme_ns *ns) 62700df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox{ 62800df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox int cmdid = alloc_cmdid(nvmeq, (void *)CMD_CTX_FLUSH, 629ff976d724a74e4522e9ca2de1fb37ac4520f454fMatthew Wilcox special_completion, NVME_IO_TIMEOUT); 63000df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox if (unlikely(cmdid < 0)) 63100df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox return cmdid; 63200df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 63300df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox return nvme_submit_flush(nvmeq, ns, cmdid); 63400df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox} 63500df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 636184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox/* 637184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox * Called with local interrupts disabled and the q_lock held. May not sleep. 638184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox */ 639b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_submit_bio_queue(struct nvme_queue *nvmeq, struct nvme_ns *ns, 640b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct bio *bio) 641b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 642ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox struct nvme_command *cmnd; 643eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox struct nvme_iod *iod; 644b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox enum dma_data_direction dma_dir; 6451287dabd345f447bbe0f7a99fc95ab89bcfc0f5dWei Yongjun int cmdid, length, result; 646b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 control; 647b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 dsmgmt; 648b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int psegs = bio_phys_segments(ns->queue, bio); 649b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 65000df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox if ((bio->bi_rw & REQ_FLUSH) && psegs) { 65100df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox result = nvme_submit_flush_data(nvmeq, ns); 65200df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox if (result) 65300df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox return result; 65400df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox } 65500df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 6561287dabd345f447bbe0f7a99fc95ab89bcfc0f5dWei Yongjun result = -ENOMEM; 6574f024f3797c43cb4b73cd2c50cec728842d0e49eKent Overstreet iod = nvme_alloc_iod(psegs, bio->bi_iter.bi_size, GFP_ATOMIC); 658eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox if (!iod) 659eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox goto nomem; 660eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->private = bio; 661b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 662eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox result = -EBUSY; 663ff976d724a74e4522e9ca2de1fb37ac4520f454fMatthew Wilcox cmdid = alloc_cmdid(nvmeq, iod, bio_completion, NVME_IO_TIMEOUT); 664b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (unlikely(cmdid < 0)) 665eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox goto free_iod; 666b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 6670e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch if (bio->bi_rw & REQ_DISCARD) { 6680e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch result = nvme_submit_discard(nvmeq, ns, bio, iod, cmdid); 6690e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch if (result) 6700e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch goto free_cmdid; 6710e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch return result; 6720e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch } 67300df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox if ((bio->bi_rw & REQ_FLUSH) && !psegs) 67400df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox return nvme_submit_flush(nvmeq, ns, cmdid); 67500df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 676b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox control = 0; 677b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (bio->bi_rw & REQ_FUA) 678b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox control |= NVME_RW_FUA; 679b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (bio->bi_rw & (REQ_FAILFAST_DEV | REQ_RAHEAD)) 680b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox control |= NVME_RW_LR; 681b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 682b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dsmgmt = 0; 683b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (bio->bi_rw & REQ_RAHEAD) 684b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH; 685b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 686ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail]; 687b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 688b8deb62cf271fa9381edc8cf52bcae2f0225c55aMatthew Wilcox memset(cmnd, 0, sizeof(*cmnd)); 689b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (bio_data_dir(bio)) { 690ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.opcode = nvme_cmd_write; 691b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_dir = DMA_TO_DEVICE; 692b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } else { 693ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.opcode = nvme_cmd_read; 694b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_dir = DMA_FROM_DEVICE; 695b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 696b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 697427e97080196548557b288517537ab7eb48c309fKeith Busch result = nvme_map_bio(nvmeq, iod, bio, dma_dir, psegs); 698427e97080196548557b288517537ab7eb48c309fKeith Busch if (result <= 0) 699859361a228258edf4821d9f5635825033eca78e8Keith Busch goto free_cmdid; 7001ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox length = result; 701b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 702ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.command_id = cmdid; 703ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.nsid = cpu_to_le32(ns->ns_id); 704eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox length = nvme_setup_prps(nvmeq->dev, &cmnd->common, iod, length, 705eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox GFP_ATOMIC); 7064f024f3797c43cb4b73cd2c50cec728842d0e49eKent Overstreet cmnd->rw.slba = cpu_to_le64(nvme_block_nr(ns, bio->bi_iter.bi_sector)); 7071ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox cmnd->rw.length = cpu_to_le16((length >> ns->lba_shift) - 1); 708ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.control = cpu_to_le16(control); 709ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt); 710b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 7116198221fa0df0298513b35796f63f242ea97134eKeith Busch nvme_start_io_acct(bio); 712b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (++nvmeq->sq_tail == nvmeq->q_depth) 713b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->sq_tail = 0; 7147547881d0951384f9833ec3a80fac8f3f16f3b98Matthew Wilcox writel(nvmeq->sq_tail, nvmeq->q_db); 715b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 7161974b1ae8852324a75fb8cfecbc7b758fd5a2c3cMatthew Wilcox return 0; 7171974b1ae8852324a75fb8cfecbc7b758fd5a2c3cMatthew Wilcox 718859361a228258edf4821d9f5635825033eca78e8Keith Busch free_cmdid: 719859361a228258edf4821d9f5635825033eca78e8Keith Busch free_cmdid(nvmeq, cmdid, NULL); 720eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox free_iod: 721eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox nvme_free_iod(nvmeq->dev, iod); 722eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox nomem: 723eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox return result; 724b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 725b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 726e9539f47525ecee05c9f22c3565885f3e9492c52Matthew Wilcoxstatic int nvme_process_cq(struct nvme_queue *nvmeq) 727b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 728821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox u16 head, phase; 729b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 730b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox head = nvmeq->cq_head; 731821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox phase = nvmeq->cq_phase; 732b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 733b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox for (;;) { 734c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox void *ctx; 735c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox nvme_completion_fn fn; 736b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_completion cqe = nvmeq->cqes[head]; 737821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox if ((le16_to_cpu(cqe.status) & 1) != phase) 738b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox break; 739b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->sq_head = le16_to_cpu(cqe.sq_head); 740b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (++head == nvmeq->q_depth) { 741b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox head = 0; 742821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox phase = !phase; 743b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 744b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 745c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox ctx = free_cmdid(nvmeq, cqe.command_id, &fn); 7465c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcox fn(nvmeq->dev, ctx, &cqe); 747b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 748b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 749b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox /* If the controller ignores the cq head doorbell and continuously 750b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * writes to the queue, it is theoretically possible to wrap around 751b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * the queue twice and mistakenly return IRQ_NONE. Linux only 752b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * requires that 0.1% of your interrupts are handled, so this isn't 753b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * a big problem. 754b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 755821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox if (head == nvmeq->cq_head && phase == nvmeq->cq_phase) 756e9539f47525ecee05c9f22c3565885f3e9492c52Matthew Wilcox return 0; 757b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 758b80d5ccca3a012e91ca64a2a0b13049163a6a698Haiyan Hu writel(head, nvmeq->q_db + nvmeq->dev->db_stride); 759b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->cq_head = head; 760821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox nvmeq->cq_phase = phase; 761b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 762e9539f47525ecee05c9f22c3565885f3e9492c52Matthew Wilcox nvmeq->cqe_seen = 1; 763e9539f47525ecee05c9f22c3565885f3e9492c52Matthew Wilcox return 1; 764b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 765b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 7667d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcoxstatic void nvme_make_request(struct request_queue *q, struct bio *bio) 7677d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox{ 7687d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox struct nvme_ns *ns = q->queuedata; 7697d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox struct nvme_queue *nvmeq = get_nvmeq(ns->dev); 7707d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox int result = -EBUSY; 7717d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox 772cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch if (!nvmeq) { 773cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch put_nvmeq(NULL); 774cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch bio_endio(bio, -EIO); 775cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch return; 776cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch } 777cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch 7787d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox spin_lock_irq(&nvmeq->q_lock); 779224042742582c9938788b81165180c876e997a07Keith Busch if (!nvmeq->q_suspended && bio_list_empty(&nvmeq->sq_cong)) 7807d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox result = nvme_submit_bio_queue(nvmeq, ns, bio); 7817d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox if (unlikely(result)) { 7827d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox if (bio_list_empty(&nvmeq->sq_cong)) 7837d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox add_wait_queue(&nvmeq->sq_full, &nvmeq->sq_cong_wait); 7847d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox bio_list_add(&nvmeq->sq_cong, bio); 7857d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox } 7867d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox 7877d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox nvme_process_cq(nvmeq); 7887d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox spin_unlock_irq(&nvmeq->q_lock); 7897d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox put_nvmeq(nvmeq); 7907d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox} 7917d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox 792b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic irqreturn_t nvme_irq(int irq, void *data) 793b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 79458ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox irqreturn_t result; 79558ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox struct nvme_queue *nvmeq = data; 79658ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox spin_lock(&nvmeq->q_lock); 797e9539f47525ecee05c9f22c3565885f3e9492c52Matthew Wilcox nvme_process_cq(nvmeq); 798e9539f47525ecee05c9f22c3565885f3e9492c52Matthew Wilcox result = nvmeq->cqe_seen ? IRQ_HANDLED : IRQ_NONE; 799e9539f47525ecee05c9f22c3565885f3e9492c52Matthew Wilcox nvmeq->cqe_seen = 0; 80058ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox spin_unlock(&nvmeq->q_lock); 80158ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox return result; 80258ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox} 80358ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox 80458ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcoxstatic irqreturn_t nvme_irq_check(int irq, void *data) 80558ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox{ 80658ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox struct nvme_queue *nvmeq = data; 80758ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox struct nvme_completion cqe = nvmeq->cqes[nvmeq->cq_head]; 80858ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox if ((le16_to_cpu(cqe.status) & 1) != nvmeq->cq_phase) 80958ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox return IRQ_NONE; 81058ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox return IRQ_WAKE_THREAD; 81158ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox} 81258ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox 8133c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcoxstatic void nvme_abort_command(struct nvme_queue *nvmeq, int cmdid) 8143c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox{ 8153c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox spin_lock_irq(&nvmeq->q_lock); 816c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox cancel_cmdid(nvmeq, cmdid, NULL); 8173c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox spin_unlock_irq(&nvmeq->q_lock); 8183c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox} 8193c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox 820c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcoxstruct sync_cmd_info { 821c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox struct task_struct *task; 822c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox u32 result; 823c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox int status; 824c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox}; 825c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox 8265c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcoxstatic void sync_completion(struct nvme_dev *dev, void *ctx, 827c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox struct nvme_completion *cqe) 828c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox{ 829c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox struct sync_cmd_info *cmdinfo = ctx; 830c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox cmdinfo->result = le32_to_cpup(&cqe->result); 831c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox cmdinfo->status = le16_to_cpup(&cqe->status) >> 1; 832c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox wake_up_process(cmdinfo->task); 833c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox} 834c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox 835b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 836b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Returns 0 on success. If the result is negative, it's a Linux error code; 837b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * if the result is positive, it's an NVM Express status code 838b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 8394f5099af4f3d5f999d8ab7784472d93e810e3912Keith Buschstatic int nvme_submit_sync_cmd(struct nvme_dev *dev, int q_idx, 8404f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch struct nvme_command *cmd, 8415d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma u32 *result, unsigned timeout) 842b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 8434f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch int cmdid, ret; 844b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct sync_cmd_info cmdinfo; 8454f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch struct nvme_queue *nvmeq; 8464f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch 8474f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch nvmeq = lock_nvmeq(dev, q_idx); 8484f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch if (!nvmeq) { 8494f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch unlock_nvmeq(nvmeq); 8504f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch return -ENODEV; 8514f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch } 852b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 853b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cmdinfo.task = current; 854b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cmdinfo.status = -EINTR; 855b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 8564f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch cmdid = alloc_cmdid(nvmeq, &cmdinfo, sync_completion, timeout); 8574f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch if (cmdid < 0) { 8584f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch unlock_nvmeq(nvmeq); 859b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return cmdid; 8604f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch } 861b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cmd->common.command_id = cmdid; 862b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 8633c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox set_current_state(TASK_KILLABLE); 8644f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch ret = nvme_submit_cmd(nvmeq, cmd); 8654f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch if (ret) { 8664f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch free_cmdid(nvmeq, cmdid, NULL); 8674f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch unlock_nvmeq(nvmeq); 8684f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch set_current_state(TASK_RUNNING); 8694f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch return ret; 8704f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch } 8714f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch unlock_nvmeq(nvmeq); 87278f8d2577bd79ce9d62f4d9e6d3b895bd1dd1d1dKeith Busch schedule_timeout(timeout); 873b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 8743c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox if (cmdinfo.status == -EINTR) { 8754f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch nvmeq = lock_nvmeq(dev, q_idx); 8764f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch if (nvmeq) 8774f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch nvme_abort_command(nvmeq, cmdid); 8784f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch unlock_nvmeq(nvmeq); 8793c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox return -EINTR; 8803c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox } 8813c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox 882b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (result) 883b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox *result = cmdinfo.result; 884b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 885b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return cmdinfo.status; 886b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 887b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 8884d115420707afcabe77d2535e092356df6664b70Keith Buschstatic int nvme_submit_async_cmd(struct nvme_queue *nvmeq, 8894d115420707afcabe77d2535e092356df6664b70Keith Busch struct nvme_command *cmd, 8904d115420707afcabe77d2535e092356df6664b70Keith Busch struct async_cmd_info *cmdinfo, unsigned timeout) 8914d115420707afcabe77d2535e092356df6664b70Keith Busch{ 8924d115420707afcabe77d2535e092356df6664b70Keith Busch int cmdid; 8934d115420707afcabe77d2535e092356df6664b70Keith Busch 8944d115420707afcabe77d2535e092356df6664b70Keith Busch cmdid = alloc_cmdid_killable(nvmeq, cmdinfo, async_completion, timeout); 8954d115420707afcabe77d2535e092356df6664b70Keith Busch if (cmdid < 0) 8964d115420707afcabe77d2535e092356df6664b70Keith Busch return cmdid; 8974d115420707afcabe77d2535e092356df6664b70Keith Busch cmdinfo->status = -EINTR; 8984d115420707afcabe77d2535e092356df6664b70Keith Busch cmd->common.command_id = cmdid; 8994f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch return nvme_submit_cmd(nvmeq, cmd); 9004d115420707afcabe77d2535e092356df6664b70Keith Busch} 9014d115420707afcabe77d2535e092356df6664b70Keith Busch 9025d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermaint nvme_submit_admin_cmd(struct nvme_dev *dev, struct nvme_command *cmd, 903b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 *result) 904b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 9054f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch return nvme_submit_sync_cmd(dev, 0, cmd, result, ADMIN_TIMEOUT); 9064f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch} 9074f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch 9084f5099af4f3d5f999d8ab7784472d93e810e3912Keith Buschint nvme_submit_io_cmd(struct nvme_dev *dev, struct nvme_command *cmd, 9094f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch u32 *result) 9104f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch{ 9114f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch return nvme_submit_sync_cmd(dev, smp_processor_id() + 1, cmd, result, 9124f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch NVME_IO_TIMEOUT); 913b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 914b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 9154d115420707afcabe77d2535e092356df6664b70Keith Buschstatic int nvme_submit_admin_cmd_async(struct nvme_dev *dev, 9164d115420707afcabe77d2535e092356df6664b70Keith Busch struct nvme_command *cmd, struct async_cmd_info *cmdinfo) 9174d115420707afcabe77d2535e092356df6664b70Keith Busch{ 9185a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch return nvme_submit_async_cmd(raw_nvmeq(dev, 0), cmd, cmdinfo, 9194d115420707afcabe77d2535e092356df6664b70Keith Busch ADMIN_TIMEOUT); 9204d115420707afcabe77d2535e092356df6664b70Keith Busch} 9214d115420707afcabe77d2535e092356df6664b70Keith Busch 922b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id) 923b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 924b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int status; 925b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_command c; 926b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 927b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset(&c, 0, sizeof(c)); 928b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.delete_queue.opcode = opcode; 929b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.delete_queue.qid = cpu_to_le16(id); 930b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 931b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox status = nvme_submit_admin_cmd(dev, &c, NULL); 932b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (status) 933b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -EIO; 934b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 935b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 936b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 937b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_alloc_cq(struct nvme_dev *dev, u16 qid, 938b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq) 939b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 940b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int status; 941b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_command c; 942b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int flags = NVME_QUEUE_PHYS_CONTIG | NVME_CQ_IRQ_ENABLED; 943b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 944b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset(&c, 0, sizeof(c)); 945b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.opcode = nvme_admin_create_cq; 946b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.prp1 = cpu_to_le64(nvmeq->cq_dma_addr); 947b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.cqid = cpu_to_le16(qid); 948b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1); 949b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.cq_flags = cpu_to_le16(flags); 950b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.irq_vector = cpu_to_le16(nvmeq->cq_vector); 951b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 952b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox status = nvme_submit_admin_cmd(dev, &c, NULL); 953b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (status) 954b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -EIO; 955b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 956b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 957b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 958b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_alloc_sq(struct nvme_dev *dev, u16 qid, 959b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq) 960b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 961b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int status; 962b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_command c; 963b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int flags = NVME_QUEUE_PHYS_CONTIG | NVME_SQ_PRIO_MEDIUM; 964b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 965b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset(&c, 0, sizeof(c)); 966b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.opcode = nvme_admin_create_sq; 967b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.prp1 = cpu_to_le64(nvmeq->sq_dma_addr); 968b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.sqid = cpu_to_le16(qid); 969b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1); 970b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.sq_flags = cpu_to_le16(flags); 971b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.cqid = cpu_to_le16(qid); 972b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 973b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox status = nvme_submit_admin_cmd(dev, &c, NULL); 974b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (status) 975b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -EIO; 976b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 977b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 978b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 979b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_delete_cq(struct nvme_dev *dev, u16 cqid) 980b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 981b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return adapter_delete_queue(dev, nvme_admin_delete_cq, cqid); 982b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 983b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 984b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_delete_sq(struct nvme_dev *dev, u16 sqid) 985b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 986b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return adapter_delete_queue(dev, nvme_admin_delete_sq, sqid); 987b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 988b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 9895d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermaint nvme_identify(struct nvme_dev *dev, unsigned nsid, unsigned cns, 990bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox dma_addr_t dma_addr) 991bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox{ 992bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox struct nvme_command c; 993bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox 994bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox memset(&c, 0, sizeof(c)); 995bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox c.identify.opcode = nvme_admin_identify; 996bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox c.identify.nsid = cpu_to_le32(nsid); 997bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox c.identify.prp1 = cpu_to_le64(dma_addr); 998bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox c.identify.cns = cpu_to_le32(cns); 999bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox 1000bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox return nvme_submit_admin_cmd(dev, &c, NULL); 1001bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox} 1002bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox 10035d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermaint nvme_get_features(struct nvme_dev *dev, unsigned fid, unsigned nsid, 100408df1e05657fc6712e520e7c09cc6c86160ceb35Keith Busch dma_addr_t dma_addr, u32 *result) 1005bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox{ 1006bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox struct nvme_command c; 1007bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox 1008bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox memset(&c, 0, sizeof(c)); 1009bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox c.features.opcode = nvme_admin_get_features; 1010a42ceccef0c43b46ff6bc1b12a7c1076ef243df1Keith Busch c.features.nsid = cpu_to_le32(nsid); 1011bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox c.features.prp1 = cpu_to_le64(dma_addr); 1012bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox c.features.fid = cpu_to_le32(fid); 1013bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox 101408df1e05657fc6712e520e7c09cc6c86160ceb35Keith Busch return nvme_submit_admin_cmd(dev, &c, result); 1015df3481399042200792822b6243e36a95a557b57eMatthew Wilcox} 1016df3481399042200792822b6243e36a95a557b57eMatthew Wilcox 10175d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermaint nvme_set_features(struct nvme_dev *dev, unsigned fid, unsigned dword11, 10185d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma dma_addr_t dma_addr, u32 *result) 1019df3481399042200792822b6243e36a95a557b57eMatthew Wilcox{ 1020df3481399042200792822b6243e36a95a557b57eMatthew Wilcox struct nvme_command c; 1021df3481399042200792822b6243e36a95a557b57eMatthew Wilcox 1022df3481399042200792822b6243e36a95a557b57eMatthew Wilcox memset(&c, 0, sizeof(c)); 1023df3481399042200792822b6243e36a95a557b57eMatthew Wilcox c.features.opcode = nvme_admin_set_features; 1024df3481399042200792822b6243e36a95a557b57eMatthew Wilcox c.features.prp1 = cpu_to_le64(dma_addr); 1025df3481399042200792822b6243e36a95a557b57eMatthew Wilcox c.features.fid = cpu_to_le32(fid); 1026df3481399042200792822b6243e36a95a557b57eMatthew Wilcox c.features.dword11 = cpu_to_le32(dword11); 1027df3481399042200792822b6243e36a95a557b57eMatthew Wilcox 1028bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox return nvme_submit_admin_cmd(dev, &c, result); 1029bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox} 1030bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox 1031a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox/** 1032c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch * nvme_abort_cmd - Attempt aborting a command 1033c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch * @cmdid: Command id of a timed out IO 1034c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch * @queue: The queue with timed out IO 1035c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch * 1036c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch * Schedule controller reset if the command was already aborted once before and 1037c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch * still hasn't been returned to the driver, or if this is the admin queue. 1038c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch */ 1039c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Buschstatic void nvme_abort_cmd(int cmdid, struct nvme_queue *nvmeq) 1040c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch{ 1041c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch int a_cmdid; 1042c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch struct nvme_command cmd; 1043c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch struct nvme_dev *dev = nvmeq->dev; 1044c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch struct nvme_cmd_info *info = nvme_cmd_info(nvmeq); 10455a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch struct nvme_queue *adminq; 1046c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch 1047c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch if (!nvmeq->qid || info[cmdid].aborted) { 1048c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch if (work_busy(&dev->reset_work)) 1049c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch return; 1050c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch list_del_init(&dev->node); 1051c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch dev_warn(&dev->pci_dev->dev, 1052c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch "I/O %d QID %d timeout, reset controller\n", cmdid, 1053c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch nvmeq->qid); 1054bdfd70fde389412be60f7e8aaed5732dc26fc8acMatthew Wilcox PREPARE_WORK(&dev->reset_work, nvme_reset_failed_dev); 1055c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch queue_work(nvme_workq, &dev->reset_work); 1056c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch return; 1057c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch } 1058c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch 1059c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch if (!dev->abort_limit) 1060c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch return; 1061c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch 10625a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch adminq = rcu_dereference(dev->queues[0]); 10635a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch a_cmdid = alloc_cmdid(adminq, CMD_CTX_ABORT, special_completion, 1064c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch ADMIN_TIMEOUT); 1065c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch if (a_cmdid < 0) 1066c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch return; 1067c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch 1068c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch memset(&cmd, 0, sizeof(cmd)); 1069c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch cmd.abort.opcode = nvme_admin_abort_cmd; 1070c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch cmd.abort.cid = cmdid; 1071c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch cmd.abort.sqid = cpu_to_le16(nvmeq->qid); 1072c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch cmd.abort.command_id = a_cmdid; 1073c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch 1074c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch --dev->abort_limit; 1075c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch info[cmdid].aborted = 1; 1076c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch info[cmdid].timeout = jiffies + ADMIN_TIMEOUT; 1077c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch 1078c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch dev_warn(nvmeq->q_dmadev, "Aborting I/O %d QID %d\n", cmdid, 1079c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch nvmeq->qid); 10805a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch nvme_submit_cmd(adminq, &cmd); 1081c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch} 1082c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch 1083c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch/** 1084a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox * nvme_cancel_ios - Cancel outstanding I/Os 1085a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox * @queue: The queue to cancel I/Os on 1086a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox * @timeout: True to only cancel I/Os which have timed out 1087a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox */ 1088a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcoxstatic void nvme_cancel_ios(struct nvme_queue *nvmeq, bool timeout) 1089a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox{ 1090a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox int depth = nvmeq->q_depth - 1; 1091a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox struct nvme_cmd_info *info = nvme_cmd_info(nvmeq); 1092a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox unsigned long now = jiffies; 1093a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox int cmdid; 1094a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox 1095a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox for_each_set_bit(cmdid, nvmeq->cmdid_data, depth) { 1096a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox void *ctx; 1097a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox nvme_completion_fn fn; 1098a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox static struct nvme_completion cqe = { 1099af2d9ca744af5e03390eeb3864d08ce75c860899Matthew Wilcox .status = cpu_to_le16(NVME_SC_ABORT_REQ << 1), 1100a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox }; 1101a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox 1102a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox if (timeout && !time_after(now, info[cmdid].timeout)) 1103a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox continue; 1104053ab702cc2702f25a97ead087ed344b864785b7Keith Busch if (info[cmdid].ctx == CMD_CTX_CANCELLED) 1105053ab702cc2702f25a97ead087ed344b864785b7Keith Busch continue; 1106c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch if (timeout && nvmeq->dev->initialized) { 1107c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch nvme_abort_cmd(cmdid, nvmeq); 1108c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch continue; 1109c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch } 1110c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch dev_warn(nvmeq->q_dmadev, "Cancelling I/O %d QID %d\n", cmdid, 1111c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch nvmeq->qid); 1112a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox ctx = cancel_cmdid(nvmeq, cmdid, &fn); 1113a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox fn(nvmeq->dev, ctx, &cqe); 1114a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox } 1115a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox} 1116a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox 11175a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Buschstatic void nvme_free_queue(struct rcu_head *r) 11189e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox{ 11195a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch struct nvme_queue *nvmeq = container_of(r, struct nvme_queue, r_head); 11205a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch 1121224042742582c9938788b81165180c876e997a07Keith Busch spin_lock_irq(&nvmeq->q_lock); 1122224042742582c9938788b81165180c876e997a07Keith Busch while (bio_list_peek(&nvmeq->sq_cong)) { 1123224042742582c9938788b81165180c876e997a07Keith Busch struct bio *bio = bio_list_pop(&nvmeq->sq_cong); 1124224042742582c9938788b81165180c876e997a07Keith Busch bio_endio(bio, -EIO); 1125224042742582c9938788b81165180c876e997a07Keith Busch } 1126224042742582c9938788b81165180c876e997a07Keith Busch spin_unlock_irq(&nvmeq->q_lock); 1127224042742582c9938788b81165180c876e997a07Keith Busch 11289e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth), 11299e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox (void *)nvmeq->cqes, nvmeq->cq_dma_addr); 11309e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth), 11319e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox nvmeq->sq_cmds, nvmeq->sq_dma_addr); 113242f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch if (nvmeq->qid) 113342f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch free_cpumask_var(nvmeq->cpu_mask); 11349e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox kfree(nvmeq); 11359e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox} 11369e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox 1137a1a5ef9998b1379780790b878457b0e0f48b25dbKeith Buschstatic void nvme_free_queues(struct nvme_dev *dev, int lowest) 1138224042742582c9938788b81165180c876e997a07Keith Busch{ 1139224042742582c9938788b81165180c876e997a07Keith Busch int i; 1140224042742582c9938788b81165180c876e997a07Keith Busch 1141a1a5ef9998b1379780790b878457b0e0f48b25dbKeith Busch for (i = dev->queue_count - 1; i >= lowest; i--) { 11425a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch struct nvme_queue *nvmeq = raw_nvmeq(dev, i); 11435a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch rcu_assign_pointer(dev->queues[i], NULL); 11445a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch call_rcu(&nvmeq->r_head, nvme_free_queue); 1145224042742582c9938788b81165180c876e997a07Keith Busch dev->queue_count--; 1146224042742582c9938788b81165180c876e997a07Keith Busch } 1147224042742582c9938788b81165180c876e997a07Keith Busch} 1148224042742582c9938788b81165180c876e997a07Keith Busch 11494d115420707afcabe77d2535e092356df6664b70Keith Busch/** 11504d115420707afcabe77d2535e092356df6664b70Keith Busch * nvme_suspend_queue - put queue into suspended state 11514d115420707afcabe77d2535e092356df6664b70Keith Busch * @nvmeq - queue to suspend 11524d115420707afcabe77d2535e092356df6664b70Keith Busch * 11534d115420707afcabe77d2535e092356df6664b70Keith Busch * Returns 1 if already suspended, 0 otherwise. 11544d115420707afcabe77d2535e092356df6664b70Keith Busch */ 11554d115420707afcabe77d2535e092356df6664b70Keith Buschstatic int nvme_suspend_queue(struct nvme_queue *nvmeq) 1156b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 11574d115420707afcabe77d2535e092356df6664b70Keith Busch int vector = nvmeq->dev->entry[nvmeq->cq_vector].vector; 1158b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1159a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox spin_lock_irq(&nvmeq->q_lock); 1160224042742582c9938788b81165180c876e997a07Keith Busch if (nvmeq->q_suspended) { 1161224042742582c9938788b81165180c876e997a07Keith Busch spin_unlock_irq(&nvmeq->q_lock); 11624d115420707afcabe77d2535e092356df6664b70Keith Busch return 1; 11633295874b6074d749516d6decd43afad7bf6e38ffKeith Busch } 1164224042742582c9938788b81165180c876e997a07Keith Busch nvmeq->q_suspended = 1; 116542f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch nvmeq->dev->online_queues--; 1166a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox spin_unlock_irq(&nvmeq->q_lock); 1167a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox 1168aba2080f3f1639f9202f1a52993669844abcfb80Matthew Wilcox irq_set_affinity_hint(vector, NULL); 1169aba2080f3f1639f9202f1a52993669844abcfb80Matthew Wilcox free_irq(vector, nvmeq); 1170b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 11714d115420707afcabe77d2535e092356df6664b70Keith Busch return 0; 11724d115420707afcabe77d2535e092356df6664b70Keith Busch} 1173b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 11744d115420707afcabe77d2535e092356df6664b70Keith Buschstatic void nvme_clear_queue(struct nvme_queue *nvmeq) 11754d115420707afcabe77d2535e092356df6664b70Keith Busch{ 1176224042742582c9938788b81165180c876e997a07Keith Busch spin_lock_irq(&nvmeq->q_lock); 1177224042742582c9938788b81165180c876e997a07Keith Busch nvme_process_cq(nvmeq); 1178224042742582c9938788b81165180c876e997a07Keith Busch nvme_cancel_ios(nvmeq, false); 1179224042742582c9938788b81165180c876e997a07Keith Busch spin_unlock_irq(&nvmeq->q_lock); 1180b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1181b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 11824d115420707afcabe77d2535e092356df6664b70Keith Buschstatic void nvme_disable_queue(struct nvme_dev *dev, int qid) 11834d115420707afcabe77d2535e092356df6664b70Keith Busch{ 11845a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch struct nvme_queue *nvmeq = raw_nvmeq(dev, qid); 11854d115420707afcabe77d2535e092356df6664b70Keith Busch 11864d115420707afcabe77d2535e092356df6664b70Keith Busch if (!nvmeq) 11874d115420707afcabe77d2535e092356df6664b70Keith Busch return; 11884d115420707afcabe77d2535e092356df6664b70Keith Busch if (nvme_suspend_queue(nvmeq)) 11894d115420707afcabe77d2535e092356df6664b70Keith Busch return; 11904d115420707afcabe77d2535e092356df6664b70Keith Busch 11910e53d18051725da46cbccfb7874a6422d4d4f274Keith Busch /* Don't tell the adapter to delete the admin queue. 11920e53d18051725da46cbccfb7874a6422d4d4f274Keith Busch * Don't tell a removed adapter to delete IO queues. */ 11930e53d18051725da46cbccfb7874a6422d4d4f274Keith Busch if (qid && readl(&dev->bar->csts) != -1) { 1194b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox adapter_delete_sq(dev, qid); 1195b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox adapter_delete_cq(dev, qid); 1196b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 11974d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_clear_queue(nvmeq); 1198b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1199b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1200b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev, int qid, 1201b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int depth, int vector) 1202b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1203b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct device *dmadev = &dev->pci_dev->dev; 1204224042742582c9938788b81165180c876e997a07Keith Busch unsigned extra = nvme_queue_extra(depth); 1205b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq = kzalloc(sizeof(*nvmeq) + extra, GFP_KERNEL); 1206b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!nvmeq) 1207b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 1208b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1209b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->cqes = dma_alloc_coherent(dmadev, CQ_SIZE(depth), 1210b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox &nvmeq->cq_dma_addr, GFP_KERNEL); 1211b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!nvmeq->cqes) 1212b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto free_nvmeq; 1213b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset((void *)nvmeq->cqes, 0, CQ_SIZE(depth)); 1214b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1215b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->sq_cmds = dma_alloc_coherent(dmadev, SQ_SIZE(depth), 1216b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox &nvmeq->sq_dma_addr, GFP_KERNEL); 1217b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!nvmeq->sq_cmds) 1218b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto free_cqdma; 1219b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 122042f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch if (qid && !zalloc_cpumask_var(&nvmeq->cpu_mask, GFP_KERNEL)) 122142f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch goto free_sqdma; 122242f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch 1223b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->q_dmadev = dmadev; 1224091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox nvmeq->dev = dev; 12253193f07bb7ae723f33ac8e8b9db317a4f68d7d18Matthew Wilcox snprintf(nvmeq->irqname, sizeof(nvmeq->irqname), "nvme%dq%d", 12263193f07bb7ae723f33ac8e8b9db317a4f68d7d18Matthew Wilcox dev->instance, qid); 1227b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox spin_lock_init(&nvmeq->q_lock); 1228b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->cq_head = 0; 1229821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox nvmeq->cq_phase = 1; 1230b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox init_waitqueue_head(&nvmeq->sq_full); 12311fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox init_waitqueue_entry(&nvmeq->sq_cong_wait, nvme_thread); 1232b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox bio_list_init(&nvmeq->sq_cong); 1233b80d5ccca3a012e91ca64a2a0b13049163a6a698Haiyan Hu nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride]; 1234b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->q_depth = depth; 1235b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->cq_vector = vector; 1236c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch nvmeq->qid = qid; 1237224042742582c9938788b81165180c876e997a07Keith Busch nvmeq->q_suspended = 1; 1238224042742582c9938788b81165180c876e997a07Keith Busch dev->queue_count++; 12395a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch rcu_assign_pointer(dev->queues[qid], nvmeq); 1240b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1241b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return nvmeq; 1242b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 124342f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch free_sqdma: 124442f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch dma_free_coherent(dmadev, SQ_SIZE(depth), (void *)nvmeq->sq_cmds, 124542f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch nvmeq->sq_dma_addr); 1246b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox free_cqdma: 124768b8eca5f882230e4991df0db0a3094f49420a90Keith Busch dma_free_coherent(dmadev, CQ_SIZE(depth), (void *)nvmeq->cqes, 1248b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->cq_dma_addr); 1249b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox free_nvmeq: 1250b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(nvmeq); 1251b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 1252b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1253b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 12543001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcoxstatic int queue_request_irq(struct nvme_dev *dev, struct nvme_queue *nvmeq, 12553001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox const char *name) 12563001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox{ 125758ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox if (use_threaded_interrupts) 125858ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox return request_threaded_irq(dev->entry[nvmeq->cq_vector].vector, 1259481e5bad84239b01415b888df2040c1860ae3cfdMichael Opdenacker nvme_irq_check, nvme_irq, IRQF_SHARED, 126058ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox name, nvmeq); 12613001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox return request_irq(dev->entry[nvmeq->cq_vector].vector, nvme_irq, 1262481e5bad84239b01415b888df2040c1860ae3cfdMichael Opdenacker IRQF_SHARED, name, nvmeq); 12633001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox} 12643001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox 1265224042742582c9938788b81165180c876e997a07Keith Buschstatic void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid) 1266b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1267224042742582c9938788b81165180c876e997a07Keith Busch struct nvme_dev *dev = nvmeq->dev; 1268224042742582c9938788b81165180c876e997a07Keith Busch unsigned extra = nvme_queue_extra(nvmeq->q_depth); 1269b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1270224042742582c9938788b81165180c876e997a07Keith Busch nvmeq->sq_tail = 0; 1271224042742582c9938788b81165180c876e997a07Keith Busch nvmeq->cq_head = 0; 1272224042742582c9938788b81165180c876e997a07Keith Busch nvmeq->cq_phase = 1; 1273b80d5ccca3a012e91ca64a2a0b13049163a6a698Haiyan Hu nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride]; 1274224042742582c9938788b81165180c876e997a07Keith Busch memset(nvmeq->cmdid_data, 0, extra); 1275224042742582c9938788b81165180c876e997a07Keith Busch memset((void *)nvmeq->cqes, 0, CQ_SIZE(nvmeq->q_depth)); 1276224042742582c9938788b81165180c876e997a07Keith Busch nvme_cancel_ios(nvmeq, false); 1277224042742582c9938788b81165180c876e997a07Keith Busch nvmeq->q_suspended = 0; 127842f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch dev->online_queues++; 1279224042742582c9938788b81165180c876e997a07Keith Busch} 1280224042742582c9938788b81165180c876e997a07Keith Busch 1281224042742582c9938788b81165180c876e997a07Keith Buschstatic int nvme_create_queue(struct nvme_queue *nvmeq, int qid) 1282224042742582c9938788b81165180c876e997a07Keith Busch{ 1283224042742582c9938788b81165180c876e997a07Keith Busch struct nvme_dev *dev = nvmeq->dev; 1284224042742582c9938788b81165180c876e997a07Keith Busch int result; 12853f85d50b609e8a5ef151656210203a6e94c19538Matthew Wilcox 1286b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox result = adapter_alloc_cq(dev, qid, nvmeq); 1287b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (result < 0) 1288224042742582c9938788b81165180c876e997a07Keith Busch return result; 1289b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1290b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox result = adapter_alloc_sq(dev, qid, nvmeq); 1291b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (result < 0) 1292b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto release_cq; 1293b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 12943193f07bb7ae723f33ac8e8b9db317a4f68d7d18Matthew Wilcox result = queue_request_irq(dev, nvmeq, nvmeq->irqname); 1295b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (result < 0) 1296b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto release_sq; 1297b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 12980a8d44cb33377969337fa6c1961b631605d5f453Matthew Wilcox spin_lock_irq(&nvmeq->q_lock); 1299224042742582c9938788b81165180c876e997a07Keith Busch nvme_init_queue(nvmeq, qid); 13000a8d44cb33377969337fa6c1961b631605d5f453Matthew Wilcox spin_unlock_irq(&nvmeq->q_lock); 1301224042742582c9938788b81165180c876e997a07Keith Busch 1302224042742582c9938788b81165180c876e997a07Keith Busch return result; 1303b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1304b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox release_sq: 1305b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox adapter_delete_sq(dev, qid); 1306b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox release_cq: 1307b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox adapter_delete_cq(dev, qid); 1308224042742582c9938788b81165180c876e997a07Keith Busch return result; 1309b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1310b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1311ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcoxstatic int nvme_wait_ready(struct nvme_dev *dev, u64 cap, bool enabled) 1312ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox{ 1313ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox unsigned long timeout; 1314ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox u32 bit = enabled ? NVME_CSTS_RDY : 0; 1315ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox 1316ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox timeout = ((NVME_CAP_TIMEOUT(cap) + 1) * HZ / 2) + jiffies; 1317ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox 1318ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox while ((readl(&dev->bar->csts) & NVME_CSTS_RDY) != bit) { 1319ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox msleep(100); 1320ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox if (fatal_signal_pending(current)) 1321ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox return -EINTR; 1322ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox if (time_after(jiffies, timeout)) { 1323ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox dev_err(&dev->pci_dev->dev, 1324ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox "Device not ready; aborting initialisation\n"); 1325ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox return -ENODEV; 1326ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox } 1327ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox } 1328ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox 1329ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox return 0; 1330ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox} 1331ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox 1332ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox/* 1333ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox * If the device has been passed off to us in an enabled state, just clear 1334ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox * the enabled bit. The spec says we should set the 'shutdown notification 1335ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox * bits', but doing so may cause the device to complete commands to the 1336ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox * admin queue ... and we don't know what memory that might be pointing at! 1337ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox */ 1338ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcoxstatic int nvme_disable_ctrl(struct nvme_dev *dev, u64 cap) 1339ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox{ 134044af146a84fa4a8e136d824207dcd356958a112bMatthew Wilcox u32 cc = readl(&dev->bar->cc); 134144af146a84fa4a8e136d824207dcd356958a112bMatthew Wilcox 134244af146a84fa4a8e136d824207dcd356958a112bMatthew Wilcox if (cc & NVME_CC_ENABLE) 134344af146a84fa4a8e136d824207dcd356958a112bMatthew Wilcox writel(cc & ~NVME_CC_ENABLE, &dev->bar->cc); 1344ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox return nvme_wait_ready(dev, cap, false); 1345ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox} 1346ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox 1347ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcoxstatic int nvme_enable_ctrl(struct nvme_dev *dev, u64 cap) 1348ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox{ 1349ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox return nvme_wait_ready(dev, cap, true); 1350ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox} 1351ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox 13521894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Buschstatic int nvme_shutdown_ctrl(struct nvme_dev *dev) 13531894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch{ 13541894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch unsigned long timeout; 13551894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch u32 cc; 13561894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch 13571894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch cc = (readl(&dev->bar->cc) & ~NVME_CC_SHN_MASK) | NVME_CC_SHN_NORMAL; 13581894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch writel(cc, &dev->bar->cc); 13591894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch 13601894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch timeout = 2 * HZ + jiffies; 13611894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch while ((readl(&dev->bar->csts) & NVME_CSTS_SHST_MASK) != 13621894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch NVME_CSTS_SHST_CMPLT) { 13631894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch msleep(100); 13641894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch if (fatal_signal_pending(current)) 13651894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch return -EINTR; 13661894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch if (time_after(jiffies, timeout)) { 13671894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch dev_err(&dev->pci_dev->dev, 13681894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch "Device shutdown incomplete; abort shutdown\n"); 13691894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch return -ENODEV; 13701894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch } 13711894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch } 13721894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch 13731894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch return 0; 13741894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch} 13751894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch 13768d85fce77edfc22f1d6dbf78e3af723b4b556f3dGreg Kroah-Hartmanstatic int nvme_configure_admin_queue(struct nvme_dev *dev) 1377b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1378ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox int result; 1379b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 aqa; 1380ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox u64 cap = readq(&dev->bar->cap); 1381b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq; 1382b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1383ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox result = nvme_disable_ctrl(dev, cap); 1384ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox if (result < 0) 1385ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox return result; 1386b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 13875a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch nvmeq = raw_nvmeq(dev, 0); 1388cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch if (!nvmeq) { 1389cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch nvmeq = nvme_alloc_queue(dev, 0, 64, 0); 1390cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch if (!nvmeq) 1391cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch return -ENOMEM; 1392cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch } 1393b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1394b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox aqa = nvmeq->q_depth - 1; 1395b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox aqa |= aqa << 16; 1396b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1397b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->ctrl_config = NVME_CC_ENABLE | NVME_CC_CSS_NVM; 1398b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->ctrl_config |= (PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT; 1399b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->ctrl_config |= NVME_CC_ARB_RR | NVME_CC_SHN_NONE; 14007f53f9d2424533256ae86f7df5661a17de743de8Matthew Wilcox dev->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES; 1401b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1402b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox writel(aqa, &dev->bar->aqa); 1403b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox writeq(nvmeq->sq_dma_addr, &dev->bar->asq); 1404b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox writeq(nvmeq->cq_dma_addr, &dev->bar->acq); 1405b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox writel(dev->ctrl_config, &dev->bar->cc); 1406b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1407ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox result = nvme_enable_ctrl(dev, cap); 1408025c557a71bd06a9f6d32259f00e2218b15bf0a4Keith Busch if (result) 1409cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch return result; 14109e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox 14113193f07bb7ae723f33ac8e8b9db317a4f68d7d18Matthew Wilcox result = queue_request_irq(dev, nvmeq, nvmeq->irqname); 1412025c557a71bd06a9f6d32259f00e2218b15bf0a4Keith Busch if (result) 1413cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch return result; 1414025c557a71bd06a9f6d32259f00e2218b15bf0a4Keith Busch 14150a8d44cb33377969337fa6c1961b631605d5f453Matthew Wilcox spin_lock_irq(&nvmeq->q_lock); 1416224042742582c9938788b81165180c876e997a07Keith Busch nvme_init_queue(nvmeq, 0); 14170a8d44cb33377969337fa6c1961b631605d5f453Matthew Wilcox spin_unlock_irq(&nvmeq->q_lock); 1418b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return result; 1419b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1420b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 14215d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermastruct nvme_iod *nvme_map_user_pages(struct nvme_dev *dev, int write, 1422eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox unsigned long addr, unsigned length) 1423b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 142436c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox int i, err, count, nents, offset; 14257fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox struct scatterlist *sg; 14267fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox struct page **pages; 1427eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox struct nvme_iod *iod; 142836c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox 142936c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox if (addr & 3) 1430eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return ERR_PTR(-EINVAL); 14315460fc03105fbed01fe27aa572d9f65bb410a61dDan Carpenter if (!length || length > INT_MAX - PAGE_SIZE) 1432eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return ERR_PTR(-EINVAL); 14337fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox 143436c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox offset = offset_in_page(addr); 14357fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox count = DIV_ROUND_UP(offset + length, PAGE_SIZE); 14367fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox pages = kcalloc(count, sizeof(*pages), GFP_KERNEL); 143722fff826e715e9727d3c7a69f15e602a9801b673Dan Carpenter if (!pages) 143822fff826e715e9727d3c7a69f15e602a9801b673Dan Carpenter return ERR_PTR(-ENOMEM); 143936c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox 144036c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox err = get_user_pages_fast(addr, count, 1, pages); 144136c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox if (err < count) { 144236c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox count = err; 144336c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox err = -EFAULT; 144436c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox goto put_pages; 144536c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox } 14467fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox 1447eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod = nvme_alloc_iod(count, length, GFP_KERNEL); 1448eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox sg = iod->sg; 144936c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox sg_init_table(sg, count); 1450d0ba1e497bca83a3d353eb47c9658afc54d83228Matthew Wilcox for (i = 0; i < count; i++) { 1451d0ba1e497bca83a3d353eb47c9658afc54d83228Matthew Wilcox sg_set_page(&sg[i], pages[i], 14525460fc03105fbed01fe27aa572d9f65bb410a61dDan Carpenter min_t(unsigned, length, PAGE_SIZE - offset), 14535460fc03105fbed01fe27aa572d9f65bb410a61dDan Carpenter offset); 1454d0ba1e497bca83a3d353eb47c9658afc54d83228Matthew Wilcox length -= (PAGE_SIZE - offset); 1455d0ba1e497bca83a3d353eb47c9658afc54d83228Matthew Wilcox offset = 0; 14567fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox } 1457fe304c43c6d63e29ed4fc46a874d7a74313788c5Matthew Wilcox sg_mark_end(&sg[i - 1]); 14581c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox iod->nents = count; 14597fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox 14607fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox err = -ENOMEM; 14617fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox nents = dma_map_sg(&dev->pci_dev->dev, sg, count, 14627fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox write ? DMA_TO_DEVICE : DMA_FROM_DEVICE); 146336c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox if (!nents) 1464eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox goto free_iod; 1465b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 14667fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox kfree(pages); 1467eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return iod; 1468b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1469eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox free_iod: 1470eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox kfree(iod); 14717fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox put_pages: 14727fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox for (i = 0; i < count; i++) 14737fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox put_page(pages[i]); 14747fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox kfree(pages); 1475eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return ERR_PTR(err); 14767fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox} 1477b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 14785d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermavoid nvme_unmap_user_pages(struct nvme_dev *dev, int write, 14791c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox struct nvme_iod *iod) 14807fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox{ 14811c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox int i; 1482b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 14831c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox dma_unmap_sg(&dev->pci_dev->dev, iod->sg, iod->nents, 14841c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox write ? DMA_TO_DEVICE : DMA_FROM_DEVICE); 14857fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox 14861c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox for (i = 0; i < iod->nents; i++) 14871c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox put_page(sg_page(&iod->sg[i])); 14887fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox} 1489b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1490a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcoxstatic int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio) 1491a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox{ 1492a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox struct nvme_dev *dev = ns->dev; 1493a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox struct nvme_user_io io; 1494a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox struct nvme_command c; 1495f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch unsigned length, meta_len; 1496f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch int status, i; 1497f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch struct nvme_iod *iod, *meta_iod = NULL; 1498f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch dma_addr_t meta_dma_addr; 1499f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch void *meta, *uninitialized_var(meta_mem); 1500a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox 1501a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox if (copy_from_user(&io, uio, sizeof(io))) 1502a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox return -EFAULT; 15036c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox length = (io.nblocks + 1) << ns->lba_shift; 1504f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_len = (io.nblocks + 1) * ns->ms; 1505f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1506f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch if (meta_len && ((io.metadata & 3) || !io.metadata)) 1507f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch return -EINVAL; 15086c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox 15096c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox switch (io.opcode) { 15106c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox case nvme_cmd_write: 15116c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox case nvme_cmd_read: 15126bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox case nvme_cmd_compare: 1513eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod = nvme_map_user_pages(dev, io.opcode & 1, io.addr, length); 15146413214c5d424fd5aae6567848340f962ad2ce0fMatthew Wilcox break; 15156c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox default: 15166bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox return -EINVAL; 15176c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox } 15186c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox 1519eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox if (IS_ERR(iod)) 1520eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return PTR_ERR(iod); 1521a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox 1522a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox memset(&c, 0, sizeof(c)); 1523a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.opcode = io.opcode; 1524a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.flags = io.flags; 15256c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox c.rw.nsid = cpu_to_le32(ns->ns_id); 1526a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.slba = cpu_to_le64(io.slba); 15276c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox c.rw.length = cpu_to_le16(io.nblocks); 1528a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.control = cpu_to_le16(io.control); 15291c9b52651dad0ff1fa71fc6205c86d972f25bcc0Matthew Wilcox c.rw.dsmgmt = cpu_to_le32(io.dsmgmt); 15301c9b52651dad0ff1fa71fc6205c86d972f25bcc0Matthew Wilcox c.rw.reftag = cpu_to_le32(io.reftag); 15311c9b52651dad0ff1fa71fc6205c86d972f25bcc0Matthew Wilcox c.rw.apptag = cpu_to_le16(io.apptag); 15321c9b52651dad0ff1fa71fc6205c86d972f25bcc0Matthew Wilcox c.rw.appmask = cpu_to_le16(io.appmask); 1533f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1534f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch if (meta_len) { 15351b56749e541ad59068582f2a28297843e243b856Keith Busch meta_iod = nvme_map_user_pages(dev, io.opcode & 1, io.metadata, 15361b56749e541ad59068582f2a28297843e243b856Keith Busch meta_len); 1537f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch if (IS_ERR(meta_iod)) { 1538f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch status = PTR_ERR(meta_iod); 1539f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_iod = NULL; 1540f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch goto unmap; 1541f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch } 1542f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1543f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_mem = dma_alloc_coherent(&dev->pci_dev->dev, meta_len, 1544f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch &meta_dma_addr, GFP_KERNEL); 1545f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch if (!meta_mem) { 1546f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch status = -ENOMEM; 1547f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch goto unmap; 1548f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch } 1549f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1550f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch if (io.opcode & 1) { 1551f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch int meta_offset = 0; 1552f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1553f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch for (i = 0; i < meta_iod->nents; i++) { 1554f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta = kmap_atomic(sg_page(&meta_iod->sg[i])) + 1555f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_iod->sg[i].offset; 1556f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch memcpy(meta_mem + meta_offset, meta, 1557f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_iod->sg[i].length); 1558f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch kunmap_atomic(meta); 1559f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_offset += meta_iod->sg[i].length; 1560f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch } 1561f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch } 1562f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1563f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch c.rw.metadata = cpu_to_le64(meta_dma_addr); 1564f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch } 1565f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1566eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox length = nvme_setup_prps(dev, &c.common, iod, length, GFP_KERNEL); 1567a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox 1568b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox if (length != (io.nblocks + 1) << ns->lba_shift) 1569b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox status = -ENOMEM; 1570b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox else 15714f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch status = nvme_submit_io_cmd(dev, &c, NULL); 1572a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox 1573f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch if (meta_len) { 1574f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch if (status == NVME_SC_SUCCESS && !(io.opcode & 1)) { 1575f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch int meta_offset = 0; 1576f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1577f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch for (i = 0; i < meta_iod->nents; i++) { 1578f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta = kmap_atomic(sg_page(&meta_iod->sg[i])) + 1579f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_iod->sg[i].offset; 1580f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch memcpy(meta, meta_mem + meta_offset, 1581f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_iod->sg[i].length); 1582f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch kunmap_atomic(meta); 1583f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_offset += meta_iod->sg[i].length; 1584f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch } 1585f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch } 1586f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1587f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch dma_free_coherent(&dev->pci_dev->dev, meta_len, meta_mem, 1588f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_dma_addr); 1589f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch } 1590f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1591f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch unmap: 15921c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox nvme_unmap_user_pages(dev, io.opcode & 1, iod); 1593eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox nvme_free_iod(dev, iod); 1594f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1595f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch if (meta_iod) { 1596f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch nvme_unmap_user_pages(dev, io.opcode & 1, meta_iod); 1597f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch nvme_free_iod(dev, meta_iod); 1598f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch } 1599f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1600a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox return status; 1601a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox} 1602a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox 160350af8baec46a99a9b81a4600c0374f83a5a590a9Keith Buschstatic int nvme_user_admin_cmd(struct nvme_dev *dev, 16046bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox struct nvme_admin_cmd __user *ucmd) 16056ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox{ 16066bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox struct nvme_admin_cmd cmd; 16076ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox struct nvme_command c; 1608eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox int status, length; 1609c7d36ab8fa04c213328119a9c0d66985fe204ee5Keith Busch struct nvme_iod *uninitialized_var(iod); 161094f370cab6e5ac514b658c6b2b3aa308cefc5c7aKeith Busch unsigned timeout; 16116ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox 16126bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox if (!capable(CAP_SYS_ADMIN)) 16136bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox return -EACCES; 16146bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox if (copy_from_user(&cmd, ucmd, sizeof(cmd))) 16156ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox return -EFAULT; 16166ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox 16176ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox memset(&c, 0, sizeof(c)); 16186bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.opcode = cmd.opcode; 16196bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.flags = cmd.flags; 16206bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.nsid = cpu_to_le32(cmd.nsid); 16216bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw2[0] = cpu_to_le32(cmd.cdw2); 16226bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw2[1] = cpu_to_le32(cmd.cdw3); 16236bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw10[0] = cpu_to_le32(cmd.cdw10); 16246bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw10[1] = cpu_to_le32(cmd.cdw11); 16256bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw10[2] = cpu_to_le32(cmd.cdw12); 16266bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw10[3] = cpu_to_le32(cmd.cdw13); 16276bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw10[4] = cpu_to_le32(cmd.cdw14); 16286bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw10[5] = cpu_to_le32(cmd.cdw15); 16296bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox 16306bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox length = cmd.data_len; 16316bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox if (cmd.data_len) { 1632497421880acecd0281d3182d534f3d28c927caecMatthew Wilcox iod = nvme_map_user_pages(dev, cmd.opcode & 1, cmd.addr, 1633497421880acecd0281d3182d534f3d28c927caecMatthew Wilcox length); 1634eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox if (IS_ERR(iod)) 1635eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return PTR_ERR(iod); 1636eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox length = nvme_setup_prps(dev, &c.common, iod, length, 1637eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox GFP_KERNEL); 16386bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox } 16396bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox 164094f370cab6e5ac514b658c6b2b3aa308cefc5c7aKeith Busch timeout = cmd.timeout_ms ? msecs_to_jiffies(cmd.timeout_ms) : 164194f370cab6e5ac514b658c6b2b3aa308cefc5c7aKeith Busch ADMIN_TIMEOUT; 16426bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox if (length != cmd.data_len) 1643b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox status = -ENOMEM; 1644b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox else 16454f5099af4f3d5f999d8ab7784472d93e810e3912Keith Busch status = nvme_submit_sync_cmd(dev, 0, &c, &cmd.result, timeout); 1646eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox 16476bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox if (cmd.data_len) { 16481c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox nvme_unmap_user_pages(dev, cmd.opcode & 1, iod); 1649eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox nvme_free_iod(dev, iod); 16506bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox } 1651f4f117f64baf8840d22266d518227b2a186d294bKeith Busch 1652cf90bc4830b858487fe4b9b9ecd0031e23ca3e83Chayan Biswas if ((status >= 0) && copy_to_user(&ucmd->result, &cmd.result, 1653f4f117f64baf8840d22266d518227b2a186d294bKeith Busch sizeof(cmd.result))) 1654f4f117f64baf8840d22266d518227b2a186d294bKeith Busch status = -EFAULT; 1655f4f117f64baf8840d22266d518227b2a186d294bKeith Busch 16566ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox return status; 16576ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox} 16586ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox 1659b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, 1660b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unsigned long arg) 1661b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1662b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_ns *ns = bdev->bd_disk->private_data; 1663b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1664b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox switch (cmd) { 16656bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox case NVME_IOCTL_ID: 1666c3bfe7176c035a0a2c70bc79180fb13a6c57142aMatthew Wilcox force_successful_syscall_return(); 16676bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox return ns->ns_id; 16686bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox case NVME_IOCTL_ADMIN_CMD: 166950af8baec46a99a9b81a4600c0374f83a5a590a9Keith Busch return nvme_user_admin_cmd(ns->dev, (void __user *)arg); 1670a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox case NVME_IOCTL_SUBMIT_IO: 1671a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox return nvme_submit_io(ns, (void __user *)arg); 16725d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma case SG_GET_VERSION_NUM: 16735d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma return nvme_sg_get_version_num((void __user *)arg); 16745d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma case SG_IO: 16755d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma return nvme_sg_io(ns, (void __user *)arg); 1676b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox default: 1677b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -ENOTTY; 1678b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 1679b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1680b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1681320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch#ifdef CONFIG_COMPAT 1682320a382746e0ab1304476ea7e986a8d416ab99dbKeith Buschstatic int nvme_compat_ioctl(struct block_device *bdev, fmode_t mode, 1683320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch unsigned int cmd, unsigned long arg) 1684320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch{ 1685320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch struct nvme_ns *ns = bdev->bd_disk->private_data; 1686320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch 1687320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch switch (cmd) { 1688320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch case SG_IO: 1689320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch return nvme_sg_io32(ns, arg); 1690320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch } 1691320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch return nvme_ioctl(bdev, mode, cmd, arg); 1692320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch} 1693320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch#else 1694320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch#define nvme_compat_ioctl NULL 1695320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch#endif 1696320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch 16979ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Buschstatic int nvme_open(struct block_device *bdev, fmode_t mode) 16989ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch{ 16999ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch struct nvme_ns *ns = bdev->bd_disk->private_data; 17009ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch struct nvme_dev *dev = ns->dev; 17019ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch 17029ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch kref_get(&dev->kref); 17039ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch return 0; 17049ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch} 17059ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch 17069ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Buschstatic void nvme_free_dev(struct kref *kref); 17079ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch 17089ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Buschstatic void nvme_release(struct gendisk *disk, fmode_t mode) 17099ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch{ 17109ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch struct nvme_ns *ns = disk->private_data; 17119ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch struct nvme_dev *dev = ns->dev; 17129ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch 17139ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch kref_put(&dev->kref, nvme_free_dev); 17149ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch} 17159ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch 1716b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic const struct block_device_operations nvme_fops = { 1717b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .owner = THIS_MODULE, 1718b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .ioctl = nvme_ioctl, 1719320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch .compat_ioctl = nvme_compat_ioctl, 17209ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch .open = nvme_open, 17219ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch .release = nvme_release, 1722b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 1723b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 17241fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcoxstatic void nvme_resubmit_bios(struct nvme_queue *nvmeq) 17251fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox{ 17261fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox while (bio_list_peek(&nvmeq->sq_cong)) { 17271fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox struct bio *bio = bio_list_pop(&nvmeq->sq_cong); 17281fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox struct nvme_ns *ns = bio->bi_bdev->bd_disk->private_data; 1729427e97080196548557b288517537ab7eb48c309fKeith Busch 1730427e97080196548557b288517537ab7eb48c309fKeith Busch if (bio_list_empty(&nvmeq->sq_cong)) 1731427e97080196548557b288517537ab7eb48c309fKeith Busch remove_wait_queue(&nvmeq->sq_full, 1732427e97080196548557b288517537ab7eb48c309fKeith Busch &nvmeq->sq_cong_wait); 17331fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox if (nvme_submit_bio_queue(nvmeq, ns, bio)) { 1734427e97080196548557b288517537ab7eb48c309fKeith Busch if (bio_list_empty(&nvmeq->sq_cong)) 1735427e97080196548557b288517537ab7eb48c309fKeith Busch add_wait_queue(&nvmeq->sq_full, 1736427e97080196548557b288517537ab7eb48c309fKeith Busch &nvmeq->sq_cong_wait); 17371fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox bio_list_add_head(&nvmeq->sq_cong, bio); 17381fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox break; 17391fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox } 17401fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox } 17411fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox} 17421fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 17431fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcoxstatic int nvme_kthread(void *data) 17441fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox{ 1745d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch struct nvme_dev *dev, *next; 17461fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 17471fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox while (!kthread_should_stop()) { 1748564a232c059913d91b491e04c2b2d670b8f94615Arjan van de Ven set_current_state(TASK_INTERRUPTIBLE); 17491fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_lock(&dev_list_lock); 1750d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch list_for_each_entry_safe(dev, next, &dev_list, node) { 17511fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox int i; 1752d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch if (readl(&dev->bar->csts) & NVME_CSTS_CFS && 1753d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch dev->initialized) { 1754d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch if (work_busy(&dev->reset_work)) 1755d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch continue; 1756d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch list_del_init(&dev->node); 1757d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch dev_warn(&dev->pci_dev->dev, 1758d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch "Failed status, reset controller\n"); 1759bdfd70fde389412be60f7e8aaed5732dc26fc8acMatthew Wilcox PREPARE_WORK(&dev->reset_work, 1760d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch nvme_reset_failed_dev); 1761d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch queue_work(nvme_workq, &dev->reset_work); 1762d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch continue; 1763d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch } 17645a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch rcu_read_lock(); 17651fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox for (i = 0; i < dev->queue_count; i++) { 17665a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch struct nvme_queue *nvmeq = 17675a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch rcu_dereference(dev->queues[i]); 1768740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox if (!nvmeq) 1769740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox continue; 17701fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_lock_irq(&nvmeq->q_lock); 1771224042742582c9938788b81165180c876e997a07Keith Busch if (nvmeq->q_suspended) 1772224042742582c9938788b81165180c876e997a07Keith Busch goto unlock; 1773bc57a0f7a44cfcf3e9873f6c6b8dcecdca486b1fMatthew Wilcox nvme_process_cq(nvmeq); 1774a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox nvme_cancel_ios(nvmeq, true); 17751fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox nvme_resubmit_bios(nvmeq); 1776224042742582c9938788b81165180c876e997a07Keith Busch unlock: 17771fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_unlock_irq(&nvmeq->q_lock); 17781fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox } 17795a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch rcu_read_unlock(); 17801fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox } 17811fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_unlock(&dev_list_lock); 1782acb7aa0db09b8abd38abeb84334a8a27a52fbb1bArjan van de Ven schedule_timeout(round_jiffies_relative(HZ)); 17831fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox } 17841fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox return 0; 17851fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox} 17861fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 17870e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Buschstatic void nvme_config_discard(struct nvme_ns *ns) 17880e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch{ 17890e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch u32 logical_block_size = queue_logical_block_size(ns->queue); 17900e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch ns->queue->limits.discard_zeroes_data = 0; 17910e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch ns->queue->limits.discard_alignment = logical_block_size; 17920e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch ns->queue->limits.discard_granularity = logical_block_size; 17930e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch ns->queue->limits.max_discard_sectors = 0xffffffff; 17940e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, ns->queue); 17950e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch} 17960e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch 1797c3bfe7176c035a0a2c70bc79180fb13a6c57142aMatthew Wilcoxstatic struct nvme_ns *nvme_alloc_ns(struct nvme_dev *dev, unsigned nsid, 1798b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_id_ns *id, struct nvme_lba_range_type *rt) 1799b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1800b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_ns *ns; 1801b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct gendisk *disk; 1802b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int lbaf; 1803b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1804b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (rt->attributes & NVME_LBART_ATTRIB_HIDE) 1805b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 1806b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1807b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns = kzalloc(sizeof(*ns), GFP_KERNEL); 1808b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!ns) 1809b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 1810b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->queue = blk_alloc_queue(GFP_KERNEL); 1811b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!ns->queue) 1812b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto out_free_ns; 18134eeb9215a0d5c9494ca8b20158cc8ee82618840cMatthew Wilcox ns->queue->queue_flags = QUEUE_FLAG_DEFAULT; 18144eeb9215a0d5c9494ca8b20158cc8ee82618840cMatthew Wilcox queue_flag_set_unlocked(QUEUE_FLAG_NOMERGES, ns->queue); 18154eeb9215a0d5c9494ca8b20158cc8ee82618840cMatthew Wilcox queue_flag_set_unlocked(QUEUE_FLAG_NONROT, ns->queue); 1816b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox blk_queue_make_request(ns->queue, nvme_make_request); 1817b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->dev = dev; 1818b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->queue->queuedata = ns; 1819b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1820469071a37afc8a627b6b2ddf29db0a097d864845Matthew Wilcox disk = alloc_disk(0); 1821b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!disk) 1822b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto out_free_queue; 18235aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox ns->ns_id = nsid; 1824b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->disk = disk; 1825b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox lbaf = id->flbas & 0xf; 1826b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->lba_shift = id->lbaf[lbaf].ds; 1827f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch ns->ms = le16_to_cpu(id->lbaf[lbaf].ms); 1828e9ef46369f5107e634a93b7fc4e62a1f53343197Keith Busch blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift); 18298fc23e032debd682f5ba9fc524a5846c10d2c522Keith Busch if (dev->max_hw_sectors) 18308fc23e032debd682f5ba9fc524a5846c10d2c522Keith Busch blk_queue_max_hw_sectors(ns->queue, dev->max_hw_sectors); 1831b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1832b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk->major = nvme_major; 1833469071a37afc8a627b6b2ddf29db0a097d864845Matthew Wilcox disk->first_minor = 0; 1834b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk->fops = &nvme_fops; 1835b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk->private_data = ns; 1836b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk->queue = ns->queue; 1837388f037f4e7f0a24bac6b1a24f144f5d939f58cfMatthew Wilcox disk->driverfs_dev = &dev->pci_dev->dev; 1838469071a37afc8a627b6b2ddf29db0a097d864845Matthew Wilcox disk->flags = GENHD_FL_EXT_DEVT; 18395aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox sprintf(disk->disk_name, "nvme%dn%d", dev->instance, nsid); 1840b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox set_capacity(disk, le64_to_cpup(&id->nsze) << (ns->lba_shift - 9)); 1841b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 18420e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch if (dev->oncs & NVME_CTRL_ONCS_DSM) 18430e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch nvme_config_discard(ns); 18440e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch 1845b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return ns; 1846b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1847b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox out_free_queue: 1848b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox blk_cleanup_queue(ns->queue); 1849b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox out_free_ns: 1850b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(ns); 1851b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 1852b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1853b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 185442f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Buschstatic int nvme_find_closest_node(int node) 185542f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch{ 185642f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch int n, val, min_val = INT_MAX, best_node = node; 185742f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch 185842f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch for_each_online_node(n) { 185942f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch if (n == node) 186042f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch continue; 186142f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch val = node_distance(node, n); 186242f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch if (val < min_val) { 186342f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch min_val = val; 186442f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch best_node = n; 186542f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch } 186642f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch } 186742f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch return best_node; 186842f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch} 186942f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch 187042f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Buschstatic void nvme_set_queue_cpus(cpumask_t *qmask, struct nvme_queue *nvmeq, 187142f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch int count) 187242f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch{ 187342f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch int cpu; 187442f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch for_each_cpu(cpu, qmask) { 187542f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch if (cpumask_weight(nvmeq->cpu_mask) >= count) 187642f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch break; 187742f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch if (!cpumask_test_and_set_cpu(cpu, nvmeq->cpu_mask)) 187842f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch *per_cpu_ptr(nvmeq->dev->io_queue, cpu) = nvmeq->qid; 187942f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch } 188042f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch} 188142f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch 188242f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Buschstatic void nvme_add_cpus(cpumask_t *mask, const cpumask_t *unassigned_cpus, 188342f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch const cpumask_t *new_mask, struct nvme_queue *nvmeq, int cpus_per_queue) 188442f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch{ 188542f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch int next_cpu; 188642f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch for_each_cpu(next_cpu, new_mask) { 188742f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch cpumask_or(mask, mask, get_cpu_mask(next_cpu)); 188842f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch cpumask_or(mask, mask, topology_thread_cpumask(next_cpu)); 188942f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch cpumask_and(mask, mask, unassigned_cpus); 189042f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch nvme_set_queue_cpus(mask, nvmeq, cpus_per_queue); 189142f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch } 189242f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch} 189342f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch 189442f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Buschstatic void nvme_create_io_queues(struct nvme_dev *dev) 189542f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch{ 189642f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch unsigned i, max; 189742f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch 189842f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch max = min(dev->max_qid, num_online_cpus()); 189942f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch for (i = dev->queue_count; i <= max; i++) 190042f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch if (!nvme_alloc_queue(dev, i, dev->q_depth, i - 1)) 190142f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch break; 190242f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch 190342f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch max = min(dev->queue_count - 1, num_online_cpus()); 190442f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch for (i = dev->online_queues; i <= max; i++) 190542f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch if (nvme_create_queue(raw_nvmeq(dev, i), i)) 190642f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch break; 190742f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch} 190842f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch 190942f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch/* 191042f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch * If there are fewer queues than online cpus, this will try to optimally 191142f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch * assign a queue to multiple cpus by grouping cpus that are "close" together: 191242f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch * thread siblings, core, socket, closest node, then whatever else is 191342f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch * available. 191442f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch */ 191542f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Buschstatic void nvme_assign_io_queues(struct nvme_dev *dev) 191642f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch{ 191742f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch unsigned cpu, cpus_per_queue, queues, remainder, i; 191842f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch cpumask_var_t unassigned_cpus; 191942f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch 192042f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch nvme_create_io_queues(dev); 192142f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch 192242f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch queues = min(dev->online_queues - 1, num_online_cpus()); 192342f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch if (!queues) 192442f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch return; 192542f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch 192642f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch cpus_per_queue = num_online_cpus() / queues; 192742f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch remainder = queues - (num_online_cpus() - queues * cpus_per_queue); 192842f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch 192942f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch if (!alloc_cpumask_var(&unassigned_cpus, GFP_KERNEL)) 193042f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch return; 193142f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch 193242f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch cpumask_copy(unassigned_cpus, cpu_online_mask); 193342f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch cpu = cpumask_first(unassigned_cpus); 193442f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch for (i = 1; i <= queues; i++) { 193542f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch struct nvme_queue *nvmeq = lock_nvmeq(dev, i); 193642f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch cpumask_t mask; 193742f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch 193842f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch cpumask_clear(nvmeq->cpu_mask); 193942f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch if (!cpumask_weight(unassigned_cpus)) { 194042f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch unlock_nvmeq(nvmeq); 194142f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch break; 194242f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch } 194342f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch 194442f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch mask = *get_cpu_mask(cpu); 194542f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch nvme_set_queue_cpus(&mask, nvmeq, cpus_per_queue); 194642f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch if (cpus_weight(mask) < cpus_per_queue) 194742f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch nvme_add_cpus(&mask, unassigned_cpus, 194842f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch topology_thread_cpumask(cpu), 194942f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch nvmeq, cpus_per_queue); 195042f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch if (cpus_weight(mask) < cpus_per_queue) 195142f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch nvme_add_cpus(&mask, unassigned_cpus, 195242f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch topology_core_cpumask(cpu), 195342f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch nvmeq, cpus_per_queue); 195442f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch if (cpus_weight(mask) < cpus_per_queue) 195542f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch nvme_add_cpus(&mask, unassigned_cpus, 195642f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch cpumask_of_node(cpu_to_node(cpu)), 195742f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch nvmeq, cpus_per_queue); 195842f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch if (cpus_weight(mask) < cpus_per_queue) 195942f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch nvme_add_cpus(&mask, unassigned_cpus, 196042f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch cpumask_of_node( 196142f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch nvme_find_closest_node( 196242f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch cpu_to_node(cpu))), 196342f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch nvmeq, cpus_per_queue); 196442f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch if (cpus_weight(mask) < cpus_per_queue) 196542f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch nvme_add_cpus(&mask, unassigned_cpus, 196642f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch unassigned_cpus, 196742f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch nvmeq, cpus_per_queue); 196842f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch 196942f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch WARN(cpumask_weight(nvmeq->cpu_mask) != cpus_per_queue, 197042f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch "nvme%d qid:%d mis-matched queue-to-cpu assignment\n", 197142f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch dev->instance, i); 197242f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch 197342f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch irq_set_affinity_hint(dev->entry[nvmeq->cq_vector].vector, 197442f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch nvmeq->cpu_mask); 197542f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch cpumask_andnot(unassigned_cpus, unassigned_cpus, 197642f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch nvmeq->cpu_mask); 197742f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch cpu = cpumask_next(cpu, unassigned_cpus); 197842f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch if (remainder && !--remainder) 197942f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch cpus_per_queue++; 198042f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch unlock_nvmeq(nvmeq); 198142f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch } 198242f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch WARN(cpumask_weight(unassigned_cpus), "nvme%d unassigned online cpus\n", 198342f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch dev->instance); 198442f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch i = 0; 198542f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch cpumask_andnot(unassigned_cpus, cpu_possible_mask, cpu_online_mask); 198642f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch for_each_cpu(cpu, unassigned_cpus) 198742f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch *per_cpu_ptr(dev->io_queue, cpu) = (i++ % queues) + 1; 198842f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch free_cpumask_var(unassigned_cpus); 198942f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch} 199042f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch 1991b3b06812e199f248561ce7824a4a8a9cd573c05aMatthew Wilcoxstatic int set_queue_count(struct nvme_dev *dev, int count) 1992b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1993b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int status; 1994b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 result; 1995b3b06812e199f248561ce7824a4a8a9cd573c05aMatthew Wilcox u32 q_count = (count - 1) | ((count - 1) << 16); 1996b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1997df3481399042200792822b6243e36a95a557b57eMatthew Wilcox status = nvme_set_features(dev, NVME_FEAT_NUM_QUEUES, q_count, 0, 1998bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox &result); 1999b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (status) 20007e03b124065507e72008ef294c30001eca74a031Keith Busch return status < 0 ? -EIO : -EBUSY; 2001b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return min(result & 0xffff, result >> 16) + 1; 2002b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 2003b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 20049d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Buschstatic size_t db_bar_size(struct nvme_dev *dev, unsigned nr_io_queues) 20059d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch{ 2006b80d5ccca3a012e91ca64a2a0b13049163a6a698Haiyan Hu return 4096 + ((nr_io_queues + 1) * 8 * dev->db_stride); 20079d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch} 20089d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch 200933b1e95c90447ea73e37e837ea0268a894919f19Keith Buschstatic int nvme_cpu_notify(struct notifier_block *self, 201033b1e95c90447ea73e37e837ea0268a894919f19Keith Busch unsigned long action, void *hcpu) 201133b1e95c90447ea73e37e837ea0268a894919f19Keith Busch{ 201233b1e95c90447ea73e37e837ea0268a894919f19Keith Busch struct nvme_dev *dev = container_of(self, struct nvme_dev, nb); 201333b1e95c90447ea73e37e837ea0268a894919f19Keith Busch switch (action) { 201433b1e95c90447ea73e37e837ea0268a894919f19Keith Busch case CPU_ONLINE: 201533b1e95c90447ea73e37e837ea0268a894919f19Keith Busch case CPU_DEAD: 201633b1e95c90447ea73e37e837ea0268a894919f19Keith Busch nvme_assign_io_queues(dev); 201733b1e95c90447ea73e37e837ea0268a894919f19Keith Busch break; 201833b1e95c90447ea73e37e837ea0268a894919f19Keith Busch } 201933b1e95c90447ea73e37e837ea0268a894919f19Keith Busch return NOTIFY_OK; 202033b1e95c90447ea73e37e837ea0268a894919f19Keith Busch} 202133b1e95c90447ea73e37e837ea0268a894919f19Keith Busch 20228d85fce77edfc22f1d6dbf78e3af723b4b556f3dGreg Kroah-Hartmanstatic int nvme_setup_io_queues(struct nvme_dev *dev) 2023b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 20245a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch struct nvme_queue *adminq = raw_nvmeq(dev, 0); 2025fa08a396647767abd24a9e7015cb177121d0cf15Ramachandra Rao Gajula struct pci_dev *pdev = dev->pci_dev; 202642f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch int result, i, vecs, nr_io_queues, size; 2027b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 202842f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch nr_io_queues = num_possible_cpus(); 2029b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox result = set_queue_count(dev, nr_io_queues); 20301b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox if (result < 0) 20311b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox return result; 2032b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox if (result < nr_io_queues) 2033b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox nr_io_queues = result; 2034b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 20359d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch size = db_bar_size(dev, nr_io_queues); 20369d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch if (size > 8192) { 2037f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox iounmap(dev->bar); 20389d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch do { 20399d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch dev->bar = ioremap(pci_resource_start(pdev, 0), size); 20409d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch if (dev->bar) 20419d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch break; 20429d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch if (!--nr_io_queues) 20439d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch return -ENOMEM; 20449d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch size = db_bar_size(dev, nr_io_queues); 20459d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch } while (1); 2046f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox dev->dbs = ((void __iomem *)dev->bar) + 4096; 20475a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch adminq->q_db = dev->dbs; 2048f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox } 2049f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox 20509d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch /* Deregister the admin queue's interrupt */ 20513193f07bb7ae723f33ac8e8b9db317a4f68d7d18Matthew Wilcox free_irq(dev->entry[0].vector, adminq); 20529d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch 2053063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox vecs = nr_io_queues; 2054063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox for (i = 0; i < vecs; i++) 20551b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox dev->entry[i].entry = i; 20561b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox for (;;) { 2057063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox result = pci_enable_msix(pdev, dev->entry, vecs); 2058063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox if (result <= 0) 20591b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox break; 2060063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox vecs = result; 20611b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox } 20621b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox 2063063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox if (result < 0) { 2064063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox vecs = nr_io_queues; 2065063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox if (vecs > 32) 2066063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox vecs = 32; 2067fa08a396647767abd24a9e7015cb177121d0cf15Ramachandra Rao Gajula for (;;) { 2068063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox result = pci_enable_msi_block(pdev, vecs); 2069fa08a396647767abd24a9e7015cb177121d0cf15Ramachandra Rao Gajula if (result == 0) { 2070063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox for (i = 0; i < vecs; i++) 2071fa08a396647767abd24a9e7015cb177121d0cf15Ramachandra Rao Gajula dev->entry[i].vector = i + pdev->irq; 2072fa08a396647767abd24a9e7015cb177121d0cf15Ramachandra Rao Gajula break; 2073063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox } else if (result < 0) { 2074063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox vecs = 1; 2075fa08a396647767abd24a9e7015cb177121d0cf15Ramachandra Rao Gajula break; 2076fa08a396647767abd24a9e7015cb177121d0cf15Ramachandra Rao Gajula } 2077063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox vecs = result; 2078fa08a396647767abd24a9e7015cb177121d0cf15Ramachandra Rao Gajula } 2079fa08a396647767abd24a9e7015cb177121d0cf15Ramachandra Rao Gajula } 2080fa08a396647767abd24a9e7015cb177121d0cf15Ramachandra Rao Gajula 2081063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox /* 2082063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox * Should investigate if there's a performance win from allocating 2083063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox * more queues than interrupt vectors; it might allow the submission 2084063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox * path to scale better, even if the receive path is limited by the 2085063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox * number of interrupts. 2086063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox */ 2087063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox nr_io_queues = vecs; 208842f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch dev->max_qid = nr_io_queues; 2089063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox 20903193f07bb7ae723f33ac8e8b9db317a4f68d7d18Matthew Wilcox result = queue_request_irq(dev, adminq, adminq->irqname); 20919d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch if (result) { 20923193f07bb7ae723f33ac8e8b9db317a4f68d7d18Matthew Wilcox adminq->q_suspended = 1; 2093224042742582c9938788b81165180c876e997a07Keith Busch goto free_queues; 20949d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch } 20951b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox 2096cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch /* Free previously allocated queues that are no longer usable */ 209742f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch nvme_free_queues(dev, nr_io_queues + 1); 209842f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch nvme_assign_io_queues(dev); 2099b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 210033b1e95c90447ea73e37e837ea0268a894919f19Keith Busch dev->nb.notifier_call = &nvme_cpu_notify; 210133b1e95c90447ea73e37e837ea0268a894919f19Keith Busch result = register_hotcpu_notifier(&dev->nb); 210233b1e95c90447ea73e37e837ea0268a894919f19Keith Busch if (result) 210333b1e95c90447ea73e37e837ea0268a894919f19Keith Busch goto free_queues; 210433b1e95c90447ea73e37e837ea0268a894919f19Keith Busch 2105224042742582c9938788b81165180c876e997a07Keith Busch return 0; 2106b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2107224042742582c9938788b81165180c876e997a07Keith Busch free_queues: 2108a1a5ef9998b1379780790b878457b0e0f48b25dbKeith Busch nvme_free_queues(dev, 1); 2109224042742582c9938788b81165180c876e997a07Keith Busch return result; 2110b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 2111b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2112422ef0c7c81934c3344e1623d2dcfda5fa2fab8dMatthew Wilcox/* 2113422ef0c7c81934c3344e1623d2dcfda5fa2fab8dMatthew Wilcox * Return: error value if an error occurred setting up the queues or calling 2114422ef0c7c81934c3344e1623d2dcfda5fa2fab8dMatthew Wilcox * Identify Device. 0 if these succeeded, even if adding some of the 2115422ef0c7c81934c3344e1623d2dcfda5fa2fab8dMatthew Wilcox * namespaces failed. At the moment, these failures are silent. TBD which 2116422ef0c7c81934c3344e1623d2dcfda5fa2fab8dMatthew Wilcox * failures should be reported. 2117422ef0c7c81934c3344e1623d2dcfda5fa2fab8dMatthew Wilcox */ 21188d85fce77edfc22f1d6dbf78e3af723b4b556f3dGreg Kroah-Hartmanstatic int nvme_dev_add(struct nvme_dev *dev) 2119b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 212068608c268bf265b039daeaafee6c902dfef32024Matthew Wilcox struct pci_dev *pdev = dev->pci_dev; 2121c3bfe7176c035a0a2c70bc79180fb13a6c57142aMatthew Wilcox int res; 2122c3bfe7176c035a0a2c70bc79180fb13a6c57142aMatthew Wilcox unsigned nn, i; 2123cbb6218fd4ae8f98ddf0d66f0826a7a3a9c88298Keith Busch struct nvme_ns *ns; 212451814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox struct nvme_id_ctrl *ctrl; 2125bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox struct nvme_id_ns *id_ns; 2126bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox void *mem; 2127b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_addr_t dma_addr; 2128159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch int shift = NVME_CAP_MPSMIN(readq(&dev->bar->cap)) + 12; 2129b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 213068608c268bf265b039daeaafee6c902dfef32024Matthew Wilcox mem = dma_alloc_coherent(&pdev->dev, 8192, &dma_addr, GFP_KERNEL); 2131a9ef4343afbe67a6abf83c0f0294e80db48e513aKeith Busch if (!mem) 2132a9ef4343afbe67a6abf83c0f0294e80db48e513aKeith Busch return -ENOMEM; 2133b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2134bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox res = nvme_identify(dev, 0, 1, dma_addr); 2135b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (res) { 2136b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox res = -EIO; 2137cbb6218fd4ae8f98ddf0d66f0826a7a3a9c88298Keith Busch goto out; 2138b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 2139b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2140bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox ctrl = mem; 214151814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox nn = le32_to_cpup(&ctrl->nn); 21420e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch dev->oncs = le16_to_cpup(&ctrl->oncs); 2143c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch dev->abort_limit = ctrl->acl + 1; 214451814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox memcpy(dev->serial, ctrl->sn, sizeof(ctrl->sn)); 214551814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox memcpy(dev->model, ctrl->mn, sizeof(ctrl->mn)); 214651814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox memcpy(dev->firmware_rev, ctrl->fr, sizeof(ctrl->fr)); 2147159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch if (ctrl->mdts) 21488fc23e032debd682f5ba9fc524a5846c10d2c522Keith Busch dev->max_hw_sectors = 1 << (ctrl->mdts + shift - 9); 214968608c268bf265b039daeaafee6c902dfef32024Matthew Wilcox if ((pdev->vendor == PCI_VENDOR_ID_INTEL) && 215068608c268bf265b039daeaafee6c902dfef32024Matthew Wilcox (pdev->device == 0x0953) && ctrl->vs[3]) 2151159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch dev->stripe_size = 1 << (ctrl->vs[3] + shift); 2152b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2153bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox id_ns = mem; 21542b2c1896871838cdf549442e8ad0264be5fa74e3Matthew Wilcox for (i = 1; i <= nn; i++) { 2155bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox res = nvme_identify(dev, i, 0, dma_addr); 2156b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (res) 2157b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox continue; 2158b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2159bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox if (id_ns->ncap == 0) 2160b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox continue; 2161b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2162bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox res = nvme_get_features(dev, NVME_FEAT_LBA_RANGE, i, 216308df1e05657fc6712e520e7c09cc6c86160ceb35Keith Busch dma_addr + 4096, NULL); 2164b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (res) 2165122090366d1d5c6ec1bfb6dfdb3a6d121ff074aaKeith Busch memset(mem + 4096, 0, 4096); 2166b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2167bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox ns = nvme_alloc_ns(dev, i, mem, mem + 4096); 2168b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (ns) 2169b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox list_add_tail(&ns->list, &dev->namespaces); 2170b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 2171b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox list_for_each_entry(ns, &dev->namespaces, list) 2172b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox add_disk(ns->disk); 2173422ef0c7c81934c3344e1623d2dcfda5fa2fab8dMatthew Wilcox res = 0; 2174b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2175bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox out: 2176684f5c2025b067a23722e620d0b3b858d8dc5d01Matthew Wilcox dma_free_coherent(&dev->pci_dev->dev, 8192, mem, dma_addr); 2177b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return res; 2178b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 2179b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 21800877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Buschstatic int nvme_dev_map(struct nvme_dev *dev) 21810877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch{ 218242f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch u64 cap; 21830877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch int bars, result = -ENOMEM; 21840877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch struct pci_dev *pdev = dev->pci_dev; 21850877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch 21860877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch if (pci_enable_device_mem(pdev)) 21870877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch return result; 21880877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch 21890877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch dev->entry[0].vector = pdev->irq; 21900877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch pci_set_master(pdev); 21910877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch bars = pci_select_bars(pdev, IORESOURCE_MEM); 21920877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch if (pci_request_selected_regions(pdev, bars, "nvme")) 21930877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch goto disable_pci; 21940877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch 2195052d0efada8ccfccd924059b75f5fb119cd10f9aRussell King if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) && 2196052d0efada8ccfccd924059b75f5fb119cd10f9aRussell King dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32))) 2197052d0efada8ccfccd924059b75f5fb119cd10f9aRussell King goto disable; 21980877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch 21990877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch dev->bar = ioremap(pci_resource_start(pdev, 0), 8192); 22000877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch if (!dev->bar) 22010877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch goto disable; 22020e53d18051725da46cbccfb7874a6422d4d4f274Keith Busch if (readl(&dev->bar->csts) == -1) { 22030e53d18051725da46cbccfb7874a6422d4d4f274Keith Busch result = -ENODEV; 22040e53d18051725da46cbccfb7874a6422d4d4f274Keith Busch goto unmap; 22050e53d18051725da46cbccfb7874a6422d4d4f274Keith Busch } 220642f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch cap = readq(&dev->bar->cap); 220742f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch dev->q_depth = min_t(int, NVME_CAP_MQES(cap) + 1, NVME_Q_DEPTH); 220842f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch dev->db_stride = 1 << NVME_CAP_STRIDE(cap); 22090877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch dev->dbs = ((void __iomem *)dev->bar) + 4096; 22100877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch 22110877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch return 0; 22120877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch 22130e53d18051725da46cbccfb7874a6422d4d4f274Keith Busch unmap: 22140e53d18051725da46cbccfb7874a6422d4d4f274Keith Busch iounmap(dev->bar); 22150e53d18051725da46cbccfb7874a6422d4d4f274Keith Busch dev->bar = NULL; 22160877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch disable: 22170877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch pci_release_regions(pdev); 22180877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch disable_pci: 22190877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch pci_disable_device(pdev); 22200877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch return result; 22210877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch} 22220877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch 22230877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Buschstatic void nvme_dev_unmap(struct nvme_dev *dev) 22240877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch{ 22250877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch if (dev->pci_dev->msi_enabled) 22260877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch pci_disable_msi(dev->pci_dev); 22270877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch else if (dev->pci_dev->msix_enabled) 22280877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch pci_disable_msix(dev->pci_dev); 22290877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch 22300877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch if (dev->bar) { 22310877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch iounmap(dev->bar); 22320877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch dev->bar = NULL; 22339a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch pci_release_regions(dev->pci_dev); 22340877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch } 22350877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch 22360877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch if (pci_is_enabled(dev->pci_dev)) 22370877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch pci_disable_device(dev->pci_dev); 22380877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch} 22390877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch 22404d115420707afcabe77d2535e092356df6664b70Keith Buschstruct nvme_delq_ctx { 22414d115420707afcabe77d2535e092356df6664b70Keith Busch struct task_struct *waiter; 22424d115420707afcabe77d2535e092356df6664b70Keith Busch struct kthread_worker *worker; 22434d115420707afcabe77d2535e092356df6664b70Keith Busch atomic_t refcount; 22444d115420707afcabe77d2535e092356df6664b70Keith Busch}; 22454d115420707afcabe77d2535e092356df6664b70Keith Busch 22464d115420707afcabe77d2535e092356df6664b70Keith Buschstatic void nvme_wait_dq(struct nvme_delq_ctx *dq, struct nvme_dev *dev) 22474d115420707afcabe77d2535e092356df6664b70Keith Busch{ 22484d115420707afcabe77d2535e092356df6664b70Keith Busch dq->waiter = current; 22494d115420707afcabe77d2535e092356df6664b70Keith Busch mb(); 22504d115420707afcabe77d2535e092356df6664b70Keith Busch 22514d115420707afcabe77d2535e092356df6664b70Keith Busch for (;;) { 22524d115420707afcabe77d2535e092356df6664b70Keith Busch set_current_state(TASK_KILLABLE); 22534d115420707afcabe77d2535e092356df6664b70Keith Busch if (!atomic_read(&dq->refcount)) 22544d115420707afcabe77d2535e092356df6664b70Keith Busch break; 22554d115420707afcabe77d2535e092356df6664b70Keith Busch if (!schedule_timeout(ADMIN_TIMEOUT) || 22564d115420707afcabe77d2535e092356df6664b70Keith Busch fatal_signal_pending(current)) { 22574d115420707afcabe77d2535e092356df6664b70Keith Busch set_current_state(TASK_RUNNING); 22584d115420707afcabe77d2535e092356df6664b70Keith Busch 22594d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_disable_ctrl(dev, readq(&dev->bar->cap)); 22604d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_disable_queue(dev, 0); 22614d115420707afcabe77d2535e092356df6664b70Keith Busch 22624d115420707afcabe77d2535e092356df6664b70Keith Busch send_sig(SIGKILL, dq->worker->task, 1); 22634d115420707afcabe77d2535e092356df6664b70Keith Busch flush_kthread_worker(dq->worker); 22644d115420707afcabe77d2535e092356df6664b70Keith Busch return; 22654d115420707afcabe77d2535e092356df6664b70Keith Busch } 22664d115420707afcabe77d2535e092356df6664b70Keith Busch } 22674d115420707afcabe77d2535e092356df6664b70Keith Busch set_current_state(TASK_RUNNING); 22684d115420707afcabe77d2535e092356df6664b70Keith Busch} 22694d115420707afcabe77d2535e092356df6664b70Keith Busch 22704d115420707afcabe77d2535e092356df6664b70Keith Buschstatic void nvme_put_dq(struct nvme_delq_ctx *dq) 22714d115420707afcabe77d2535e092356df6664b70Keith Busch{ 22724d115420707afcabe77d2535e092356df6664b70Keith Busch atomic_dec(&dq->refcount); 22734d115420707afcabe77d2535e092356df6664b70Keith Busch if (dq->waiter) 22744d115420707afcabe77d2535e092356df6664b70Keith Busch wake_up_process(dq->waiter); 22754d115420707afcabe77d2535e092356df6664b70Keith Busch} 22764d115420707afcabe77d2535e092356df6664b70Keith Busch 22774d115420707afcabe77d2535e092356df6664b70Keith Buschstatic struct nvme_delq_ctx *nvme_get_dq(struct nvme_delq_ctx *dq) 22784d115420707afcabe77d2535e092356df6664b70Keith Busch{ 22794d115420707afcabe77d2535e092356df6664b70Keith Busch atomic_inc(&dq->refcount); 22804d115420707afcabe77d2535e092356df6664b70Keith Busch return dq; 22814d115420707afcabe77d2535e092356df6664b70Keith Busch} 22824d115420707afcabe77d2535e092356df6664b70Keith Busch 22834d115420707afcabe77d2535e092356df6664b70Keith Buschstatic void nvme_del_queue_end(struct nvme_queue *nvmeq) 22844d115420707afcabe77d2535e092356df6664b70Keith Busch{ 22854d115420707afcabe77d2535e092356df6664b70Keith Busch struct nvme_delq_ctx *dq = nvmeq->cmdinfo.ctx; 22864d115420707afcabe77d2535e092356df6664b70Keith Busch 22874d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_clear_queue(nvmeq); 22884d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_put_dq(dq); 22894d115420707afcabe77d2535e092356df6664b70Keith Busch} 22904d115420707afcabe77d2535e092356df6664b70Keith Busch 22914d115420707afcabe77d2535e092356df6664b70Keith Buschstatic int adapter_async_del_queue(struct nvme_queue *nvmeq, u8 opcode, 22924d115420707afcabe77d2535e092356df6664b70Keith Busch kthread_work_func_t fn) 22934d115420707afcabe77d2535e092356df6664b70Keith Busch{ 22944d115420707afcabe77d2535e092356df6664b70Keith Busch struct nvme_command c; 22954d115420707afcabe77d2535e092356df6664b70Keith Busch 22964d115420707afcabe77d2535e092356df6664b70Keith Busch memset(&c, 0, sizeof(c)); 22974d115420707afcabe77d2535e092356df6664b70Keith Busch c.delete_queue.opcode = opcode; 22984d115420707afcabe77d2535e092356df6664b70Keith Busch c.delete_queue.qid = cpu_to_le16(nvmeq->qid); 22994d115420707afcabe77d2535e092356df6664b70Keith Busch 23004d115420707afcabe77d2535e092356df6664b70Keith Busch init_kthread_work(&nvmeq->cmdinfo.work, fn); 23014d115420707afcabe77d2535e092356df6664b70Keith Busch return nvme_submit_admin_cmd_async(nvmeq->dev, &c, &nvmeq->cmdinfo); 23024d115420707afcabe77d2535e092356df6664b70Keith Busch} 23034d115420707afcabe77d2535e092356df6664b70Keith Busch 23044d115420707afcabe77d2535e092356df6664b70Keith Buschstatic void nvme_del_cq_work_handler(struct kthread_work *work) 23054d115420707afcabe77d2535e092356df6664b70Keith Busch{ 23064d115420707afcabe77d2535e092356df6664b70Keith Busch struct nvme_queue *nvmeq = container_of(work, struct nvme_queue, 23074d115420707afcabe77d2535e092356df6664b70Keith Busch cmdinfo.work); 23084d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_del_queue_end(nvmeq); 23094d115420707afcabe77d2535e092356df6664b70Keith Busch} 23104d115420707afcabe77d2535e092356df6664b70Keith Busch 23114d115420707afcabe77d2535e092356df6664b70Keith Buschstatic int nvme_delete_cq(struct nvme_queue *nvmeq) 23124d115420707afcabe77d2535e092356df6664b70Keith Busch{ 23134d115420707afcabe77d2535e092356df6664b70Keith Busch return adapter_async_del_queue(nvmeq, nvme_admin_delete_cq, 23144d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_del_cq_work_handler); 23154d115420707afcabe77d2535e092356df6664b70Keith Busch} 23164d115420707afcabe77d2535e092356df6664b70Keith Busch 23174d115420707afcabe77d2535e092356df6664b70Keith Buschstatic void nvme_del_sq_work_handler(struct kthread_work *work) 23184d115420707afcabe77d2535e092356df6664b70Keith Busch{ 23194d115420707afcabe77d2535e092356df6664b70Keith Busch struct nvme_queue *nvmeq = container_of(work, struct nvme_queue, 23204d115420707afcabe77d2535e092356df6664b70Keith Busch cmdinfo.work); 23214d115420707afcabe77d2535e092356df6664b70Keith Busch int status = nvmeq->cmdinfo.status; 23224d115420707afcabe77d2535e092356df6664b70Keith Busch 23234d115420707afcabe77d2535e092356df6664b70Keith Busch if (!status) 23244d115420707afcabe77d2535e092356df6664b70Keith Busch status = nvme_delete_cq(nvmeq); 23254d115420707afcabe77d2535e092356df6664b70Keith Busch if (status) 23264d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_del_queue_end(nvmeq); 23274d115420707afcabe77d2535e092356df6664b70Keith Busch} 23284d115420707afcabe77d2535e092356df6664b70Keith Busch 23294d115420707afcabe77d2535e092356df6664b70Keith Buschstatic int nvme_delete_sq(struct nvme_queue *nvmeq) 23304d115420707afcabe77d2535e092356df6664b70Keith Busch{ 23314d115420707afcabe77d2535e092356df6664b70Keith Busch return adapter_async_del_queue(nvmeq, nvme_admin_delete_sq, 23324d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_del_sq_work_handler); 23334d115420707afcabe77d2535e092356df6664b70Keith Busch} 23344d115420707afcabe77d2535e092356df6664b70Keith Busch 23354d115420707afcabe77d2535e092356df6664b70Keith Buschstatic void nvme_del_queue_start(struct kthread_work *work) 23364d115420707afcabe77d2535e092356df6664b70Keith Busch{ 23374d115420707afcabe77d2535e092356df6664b70Keith Busch struct nvme_queue *nvmeq = container_of(work, struct nvme_queue, 23384d115420707afcabe77d2535e092356df6664b70Keith Busch cmdinfo.work); 23394d115420707afcabe77d2535e092356df6664b70Keith Busch allow_signal(SIGKILL); 23404d115420707afcabe77d2535e092356df6664b70Keith Busch if (nvme_delete_sq(nvmeq)) 23414d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_del_queue_end(nvmeq); 23424d115420707afcabe77d2535e092356df6664b70Keith Busch} 23434d115420707afcabe77d2535e092356df6664b70Keith Busch 23444d115420707afcabe77d2535e092356df6664b70Keith Buschstatic void nvme_disable_io_queues(struct nvme_dev *dev) 23454d115420707afcabe77d2535e092356df6664b70Keith Busch{ 23464d115420707afcabe77d2535e092356df6664b70Keith Busch int i; 23474d115420707afcabe77d2535e092356df6664b70Keith Busch DEFINE_KTHREAD_WORKER_ONSTACK(worker); 23484d115420707afcabe77d2535e092356df6664b70Keith Busch struct nvme_delq_ctx dq; 23494d115420707afcabe77d2535e092356df6664b70Keith Busch struct task_struct *kworker_task = kthread_run(kthread_worker_fn, 23504d115420707afcabe77d2535e092356df6664b70Keith Busch &worker, "nvme%d", dev->instance); 23514d115420707afcabe77d2535e092356df6664b70Keith Busch 23524d115420707afcabe77d2535e092356df6664b70Keith Busch if (IS_ERR(kworker_task)) { 23534d115420707afcabe77d2535e092356df6664b70Keith Busch dev_err(&dev->pci_dev->dev, 23544d115420707afcabe77d2535e092356df6664b70Keith Busch "Failed to create queue del task\n"); 23554d115420707afcabe77d2535e092356df6664b70Keith Busch for (i = dev->queue_count - 1; i > 0; i--) 23564d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_disable_queue(dev, i); 23574d115420707afcabe77d2535e092356df6664b70Keith Busch return; 23584d115420707afcabe77d2535e092356df6664b70Keith Busch } 23594d115420707afcabe77d2535e092356df6664b70Keith Busch 23604d115420707afcabe77d2535e092356df6664b70Keith Busch dq.waiter = NULL; 23614d115420707afcabe77d2535e092356df6664b70Keith Busch atomic_set(&dq.refcount, 0); 23624d115420707afcabe77d2535e092356df6664b70Keith Busch dq.worker = &worker; 23634d115420707afcabe77d2535e092356df6664b70Keith Busch for (i = dev->queue_count - 1; i > 0; i--) { 23645a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch struct nvme_queue *nvmeq = raw_nvmeq(dev, i); 23654d115420707afcabe77d2535e092356df6664b70Keith Busch 23664d115420707afcabe77d2535e092356df6664b70Keith Busch if (nvme_suspend_queue(nvmeq)) 23674d115420707afcabe77d2535e092356df6664b70Keith Busch continue; 23684d115420707afcabe77d2535e092356df6664b70Keith Busch nvmeq->cmdinfo.ctx = nvme_get_dq(&dq); 23694d115420707afcabe77d2535e092356df6664b70Keith Busch nvmeq->cmdinfo.worker = dq.worker; 23704d115420707afcabe77d2535e092356df6664b70Keith Busch init_kthread_work(&nvmeq->cmdinfo.work, nvme_del_queue_start); 23714d115420707afcabe77d2535e092356df6664b70Keith Busch queue_kthread_work(dq.worker, &nvmeq->cmdinfo.work); 23724d115420707afcabe77d2535e092356df6664b70Keith Busch } 23734d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_wait_dq(&dq, dev); 23744d115420707afcabe77d2535e092356df6664b70Keith Busch kthread_stop(kworker_task); 23754d115420707afcabe77d2535e092356df6664b70Keith Busch} 23764d115420707afcabe77d2535e092356df6664b70Keith Busch 2377f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Buschstatic void nvme_dev_shutdown(struct nvme_dev *dev) 2378b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 2379224042742582c9938788b81165180c876e997a07Keith Busch int i; 2380224042742582c9938788b81165180c876e997a07Keith Busch 2381d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch dev->initialized = 0; 238233b1e95c90447ea73e37e837ea0268a894919f19Keith Busch unregister_hotcpu_notifier(&dev->nb); 2383b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 23841fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_lock(&dev_list_lock); 2385f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch list_del_init(&dev->node); 23861fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_unlock(&dev_list_lock); 23871fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 23884d115420707afcabe77d2535e092356df6664b70Keith Busch if (!dev->bar || (dev->bar && readl(&dev->bar->csts) == -1)) { 23894d115420707afcabe77d2535e092356df6664b70Keith Busch for (i = dev->queue_count - 1; i >= 0; i--) { 23905a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch struct nvme_queue *nvmeq = raw_nvmeq(dev, i); 23914d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_suspend_queue(nvmeq); 23924d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_clear_queue(nvmeq); 23934d115420707afcabe77d2535e092356df6664b70Keith Busch } 23944d115420707afcabe77d2535e092356df6664b70Keith Busch } else { 23954d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_disable_io_queues(dev); 23961894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch nvme_shutdown_ctrl(dev); 23974d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_disable_queue(dev, 0); 23984d115420707afcabe77d2535e092356df6664b70Keith Busch } 2399f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch nvme_dev_unmap(dev); 2400f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch} 2401f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch 2402f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Buschstatic void nvme_dev_remove(struct nvme_dev *dev) 2403f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch{ 24049ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch struct nvme_ns *ns; 2405f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch 24069ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch list_for_each_entry(ns, &dev->namespaces, list) { 24079ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch if (ns->disk->flags & GENHD_FL_UP) 24089ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch del_gendisk(ns->disk); 24099ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch if (!blk_queue_dying(ns->queue)) 24109ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch blk_cleanup_queue(ns->queue); 2411b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 2412b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 2413b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2414091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcoxstatic int nvme_setup_prp_pools(struct nvme_dev *dev) 2415091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox{ 2416091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox struct device *dmadev = &dev->pci_dev->dev; 2417091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox dev->prp_page_pool = dma_pool_create("prp list page", dmadev, 2418091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox PAGE_SIZE, PAGE_SIZE, 0); 2419091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox if (!dev->prp_page_pool) 2420091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox return -ENOMEM; 2421091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox 242299802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox /* Optimisation for I/Os between 4k and 128k */ 242399802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox dev->prp_small_pool = dma_pool_create("prp list 256", dmadev, 242499802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox 256, 256, 0); 242599802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox if (!dev->prp_small_pool) { 242699802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox dma_pool_destroy(dev->prp_page_pool); 242799802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox return -ENOMEM; 242899802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox } 2429091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox return 0; 2430091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox} 2431091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox 2432091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcoxstatic void nvme_release_prp_pools(struct nvme_dev *dev) 2433091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox{ 2434091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox dma_pool_destroy(dev->prp_page_pool); 243599802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox dma_pool_destroy(dev->prp_small_pool); 2436091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox} 2437091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox 2438cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anhstatic DEFINE_IDA(nvme_instance_ida); 2439cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh 2440cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anhstatic int nvme_set_instance(struct nvme_dev *dev) 2441b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 2442cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh int instance, error; 2443cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh 2444cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh do { 2445cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh if (!ida_pre_get(&nvme_instance_ida, GFP_KERNEL)) 2446cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh return -ENODEV; 2447cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh 2448cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh spin_lock(&dev_list_lock); 2449cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh error = ida_get_new(&nvme_instance_ida, &instance); 2450cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh spin_unlock(&dev_list_lock); 2451cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh } while (error == -EAGAIN); 2452cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh 2453cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh if (error) 2454cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh return -ENODEV; 2455cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh 2456cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh dev->instance = instance; 2457cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh return 0; 2458b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 2459b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2460b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void nvme_release_instance(struct nvme_dev *dev) 2461b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 2462cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh spin_lock(&dev_list_lock); 2463cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh ida_remove(&nvme_instance_ida, dev->instance); 2464cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh spin_unlock(&dev_list_lock); 2465b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 2466b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 24679ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Buschstatic void nvme_free_namespaces(struct nvme_dev *dev) 24689ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch{ 24699ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch struct nvme_ns *ns, *next; 24709ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch 24719ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch list_for_each_entry_safe(ns, next, &dev->namespaces, list) { 24729ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch list_del(&ns->list); 24739ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch put_disk(ns->disk); 24749ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch kfree(ns); 24759ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch } 24769ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch} 24779ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch 24785e82e952f04681c10f35e02ee0a4a43ec027137aKeith Buschstatic void nvme_free_dev(struct kref *kref) 24795e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch{ 24805e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch struct nvme_dev *dev = container_of(kref, struct nvme_dev, kref); 24819ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch 24829ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch nvme_free_namespaces(dev); 248342f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch free_percpu(dev->io_queue); 24845e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch kfree(dev->queues); 24855e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch kfree(dev->entry); 24865e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch kfree(dev); 24875e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch} 24885e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch 24895e82e952f04681c10f35e02ee0a4a43ec027137aKeith Buschstatic int nvme_dev_open(struct inode *inode, struct file *f) 24905e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch{ 24915e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch struct nvme_dev *dev = container_of(f->private_data, struct nvme_dev, 24925e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch miscdev); 24935e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch kref_get(&dev->kref); 24945e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch f->private_data = dev; 24955e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch return 0; 24965e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch} 24975e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch 24985e82e952f04681c10f35e02ee0a4a43ec027137aKeith Buschstatic int nvme_dev_release(struct inode *inode, struct file *f) 24995e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch{ 25005e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch struct nvme_dev *dev = f->private_data; 25015e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch kref_put(&dev->kref, nvme_free_dev); 25025e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch return 0; 25035e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch} 25045e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch 25055e82e952f04681c10f35e02ee0a4a43ec027137aKeith Buschstatic long nvme_dev_ioctl(struct file *f, unsigned int cmd, unsigned long arg) 25065e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch{ 25075e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch struct nvme_dev *dev = f->private_data; 25085e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch switch (cmd) { 25095e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch case NVME_IOCTL_ADMIN_CMD: 25105e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch return nvme_user_admin_cmd(dev, (void __user *)arg); 25115e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch default: 25125e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch return -ENOTTY; 25135e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch } 25145e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch} 25155e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch 25165e82e952f04681c10f35e02ee0a4a43ec027137aKeith Buschstatic const struct file_operations nvme_dev_fops = { 25175e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch .owner = THIS_MODULE, 25185e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch .open = nvme_dev_open, 25195e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch .release = nvme_dev_release, 25205e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch .unlocked_ioctl = nvme_dev_ioctl, 25215e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch .compat_ioctl = nvme_dev_ioctl, 25225e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch}; 25235e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch 2524f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Buschstatic int nvme_dev_start(struct nvme_dev *dev) 2525f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch{ 2526f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch int result; 2527f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch 2528f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch result = nvme_dev_map(dev); 2529f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch if (result) 2530f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch return result; 2531f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch 2532f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch result = nvme_configure_admin_queue(dev); 2533f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch if (result) 2534f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch goto unmap; 2535f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch 2536f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch spin_lock(&dev_list_lock); 2537f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch list_add(&dev->node, &dev_list); 2538f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch spin_unlock(&dev_list_lock); 2539f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch 2540f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch result = nvme_setup_io_queues(dev); 2541d82e8bfdef9afae83b894be49af4644d9ac3c359Keith Busch if (result && result != -EBUSY) 2542f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch goto disable; 2543f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch 2544d82e8bfdef9afae83b894be49af4644d9ac3c359Keith Busch return result; 2545f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch 2546f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch disable: 2547a1a5ef9998b1379780790b878457b0e0f48b25dbKeith Busch nvme_disable_queue(dev, 0); 2548f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch spin_lock(&dev_list_lock); 2549f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch list_del_init(&dev->node); 2550f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch spin_unlock(&dev_list_lock); 2551f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch unmap: 2552f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch nvme_dev_unmap(dev); 2553f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch return result; 2554f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch} 2555f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch 25569a6b94584de1a0467d85b435df9c744c5c45a270Keith Buschstatic int nvme_remove_dead_ctrl(void *arg) 25579a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch{ 25589a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch struct nvme_dev *dev = (struct nvme_dev *)arg; 25599a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch struct pci_dev *pdev = dev->pci_dev; 25609a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch 25619a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch if (pci_get_drvdata(pdev)) 25629a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch pci_stop_and_remove_bus_device(pdev); 25639a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch kref_put(&dev->kref, nvme_free_dev); 25649a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch return 0; 25659a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch} 25669a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch 25679a6b94584de1a0467d85b435df9c744c5c45a270Keith Buschstatic void nvme_remove_disks(struct work_struct *ws) 25689a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch{ 25699a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch struct nvme_dev *dev = container_of(ws, struct nvme_dev, reset_work); 25709a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch 25719a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch nvme_dev_remove(dev); 25725a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch nvme_free_queues(dev, 1); 25739a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch} 25749a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch 25759a6b94584de1a0467d85b435df9c744c5c45a270Keith Buschstatic int nvme_dev_resume(struct nvme_dev *dev) 25769a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch{ 25779a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch int ret; 25789a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch 25799a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch ret = nvme_dev_start(dev); 25809a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch if (ret && ret != -EBUSY) 25819a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch return ret; 25829a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch if (ret == -EBUSY) { 25839a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch spin_lock(&dev_list_lock); 2584bdfd70fde389412be60f7e8aaed5732dc26fc8acMatthew Wilcox PREPARE_WORK(&dev->reset_work, nvme_remove_disks); 25859a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch queue_work(nvme_workq, &dev->reset_work); 25869a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch spin_unlock(&dev_list_lock); 25879a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch } 2588d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch dev->initialized = 1; 25899a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch return 0; 25909a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch} 25919a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch 25929a6b94584de1a0467d85b435df9c744c5c45a270Keith Buschstatic void nvme_dev_reset(struct nvme_dev *dev) 25939a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch{ 25949a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch nvme_dev_shutdown(dev); 25959a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch if (nvme_dev_resume(dev)) { 25969a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch dev_err(&dev->pci_dev->dev, "Device failed to resume\n"); 25979a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch kref_get(&dev->kref); 25989a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch if (IS_ERR(kthread_run(nvme_remove_dead_ctrl, dev, "nvme%d", 25999a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch dev->instance))) { 26009a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch dev_err(&dev->pci_dev->dev, 26019a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch "Failed to start controller remove task\n"); 26029a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch kref_put(&dev->kref, nvme_free_dev); 26039a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch } 26049a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch } 26059a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch} 26069a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch 26079a6b94584de1a0467d85b435df9c744c5c45a270Keith Buschstatic void nvme_reset_failed_dev(struct work_struct *ws) 26089a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch{ 26099a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch struct nvme_dev *dev = container_of(ws, struct nvme_dev, reset_work); 26109a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch nvme_dev_reset(dev); 26119a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch} 26129a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch 26138d85fce77edfc22f1d6dbf78e3af723b4b556f3dGreg Kroah-Hartmanstatic int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id) 2614b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 26150877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch int result = -ENOMEM; 2616b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_dev *dev; 2617b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2618b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev = kzalloc(sizeof(*dev), GFP_KERNEL); 2619b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!dev) 2620b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -ENOMEM; 2621b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->entry = kcalloc(num_possible_cpus(), sizeof(*dev->entry), 2622b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox GFP_KERNEL); 2623b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!dev->entry) 2624b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto free; 26251b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox dev->queues = kcalloc(num_possible_cpus() + 1, sizeof(void *), 26261b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox GFP_KERNEL); 2627b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!dev->queues) 2628b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto free; 262942f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch dev->io_queue = alloc_percpu(unsigned short); 263042f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch if (!dev->io_queue) 263142f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch goto free; 2632b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2633b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox INIT_LIST_HEAD(&dev->namespaces); 2634bdfd70fde389412be60f7e8aaed5732dc26fc8acMatthew Wilcox INIT_WORK(&dev->reset_work, nvme_reset_failed_dev); 2635b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->pci_dev = pdev; 26369a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch pci_set_drvdata(pdev, dev); 2637cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh result = nvme_set_instance(dev); 2638cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh if (result) 26390877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch goto free; 2640b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2641091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox result = nvme_setup_prp_pools(dev); 2642091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox if (result) 26430877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch goto release; 2644091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox 2645fb35e914b3f88cda9ee6f9d776910c35269c4ecfKeith Busch kref_init(&dev->kref); 2646f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch result = nvme_dev_start(dev); 2647d82e8bfdef9afae83b894be49af4644d9ac3c359Keith Busch if (result) { 2648d82e8bfdef9afae83b894be49af4644d9ac3c359Keith Busch if (result == -EBUSY) 2649d82e8bfdef9afae83b894be49af4644d9ac3c359Keith Busch goto create_cdev; 26500877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch goto release_pools; 2651d82e8bfdef9afae83b894be49af4644d9ac3c359Keith Busch } 2652b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2653740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox result = nvme_dev_add(dev); 2654d82e8bfdef9afae83b894be49af4644d9ac3c359Keith Busch if (result) 2655f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch goto shutdown; 2656740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox 2657d82e8bfdef9afae83b894be49af4644d9ac3c359Keith Busch create_cdev: 26585e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch scnprintf(dev->name, sizeof(dev->name), "nvme%d", dev->instance); 26595e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch dev->miscdev.minor = MISC_DYNAMIC_MINOR; 26605e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch dev->miscdev.parent = &pdev->dev; 26615e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch dev->miscdev.name = dev->name; 26625e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch dev->miscdev.fops = &nvme_dev_fops; 26635e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch result = misc_register(&dev->miscdev); 26645e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch if (result) 26655e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch goto remove; 26665e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch 2667d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch dev->initialized = 1; 2668b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 2669b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 26705e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch remove: 26715e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch nvme_dev_remove(dev); 26729ac27090f61ea6735a62b0a98c7669c833bcdc09Keith Busch nvme_free_namespaces(dev); 2673f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch shutdown: 2674f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch nvme_dev_shutdown(dev); 26750877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch release_pools: 2676a1a5ef9998b1379780790b878457b0e0f48b25dbKeith Busch nvme_free_queues(dev, 0); 2677091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox nvme_release_prp_pools(dev); 26780877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch release: 26790877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch nvme_release_instance(dev); 2680b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox free: 268142f614201e80ff4cfb8b285d7190149a8e1e6cecKeith Busch free_percpu(dev->io_queue); 2682b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(dev->queues); 2683b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(dev->entry); 2684b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(dev); 2685b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return result; 2686b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 2687b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 268809ece1424f52a2d3ad0ab715e96b0fb342af4f03Keith Buschstatic void nvme_shutdown(struct pci_dev *pdev) 268909ece1424f52a2d3ad0ab715e96b0fb342af4f03Keith Busch{ 269009ece1424f52a2d3ad0ab715e96b0fb342af4f03Keith Busch struct nvme_dev *dev = pci_get_drvdata(pdev); 269109ece1424f52a2d3ad0ab715e96b0fb342af4f03Keith Busch nvme_dev_shutdown(dev); 269209ece1424f52a2d3ad0ab715e96b0fb342af4f03Keith Busch} 269309ece1424f52a2d3ad0ab715e96b0fb342af4f03Keith Busch 26948d85fce77edfc22f1d6dbf78e3af723b4b556f3dGreg Kroah-Hartmanstatic void nvme_remove(struct pci_dev *pdev) 2695b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 2696b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_dev *dev = pci_get_drvdata(pdev); 26979a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch 26989a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch spin_lock(&dev_list_lock); 26999a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch list_del_init(&dev->node); 27009a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch spin_unlock(&dev_list_lock); 27019a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch 27029a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch pci_set_drvdata(pdev, NULL); 27039a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch flush_work(&dev->reset_work); 27045e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch misc_deregister(&dev->miscdev); 27059a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch nvme_dev_remove(dev); 27069a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch nvme_dev_shutdown(dev); 2707a1a5ef9998b1379780790b878457b0e0f48b25dbKeith Busch nvme_free_queues(dev, 0); 27085a92e700af2e5e0e6404988d6a7f2ed3dad3f46fKeith Busch rcu_barrier(); 27099a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch nvme_release_instance(dev); 27109a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch nvme_release_prp_pools(dev); 27115e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch kref_put(&dev->kref, nvme_free_dev); 2712b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 2713b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2714b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* These functions are yet to be implemented */ 2715b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_error_detected NULL 2716b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_dump_registers NULL 2717b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_link_reset NULL 2718b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_slot_reset NULL 2719b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_error_resume NULL 2720cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch 2721671a6018db1d359254c8201704cf967efb42fc25Jingoo Han#ifdef CONFIG_PM_SLEEP 2722cd63894630ab17a192bf97427d16dbec10710a6aKeith Buschstatic int nvme_suspend(struct device *dev) 2723cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch{ 2724cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch struct pci_dev *pdev = to_pci_dev(dev); 2725cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch struct nvme_dev *ndev = pci_get_drvdata(pdev); 2726cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch 2727cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch nvme_dev_shutdown(ndev); 2728cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch return 0; 2729cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch} 2730cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch 2731cd63894630ab17a192bf97427d16dbec10710a6aKeith Buschstatic int nvme_resume(struct device *dev) 2732cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch{ 2733cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch struct pci_dev *pdev = to_pci_dev(dev); 2734cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch struct nvme_dev *ndev = pci_get_drvdata(pdev); 2735cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch 27369a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch if (nvme_dev_resume(ndev) && !work_busy(&ndev->reset_work)) { 2737bdfd70fde389412be60f7e8aaed5732dc26fc8acMatthew Wilcox PREPARE_WORK(&ndev->reset_work, nvme_reset_failed_dev); 27389a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch queue_work(nvme_workq, &ndev->reset_work); 27399a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch } 27409a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch return 0; 2741cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch} 2742671a6018db1d359254c8201704cf967efb42fc25Jingoo Han#endif 2743cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch 2744cd63894630ab17a192bf97427d16dbec10710a6aKeith Buschstatic SIMPLE_DEV_PM_OPS(nvme_dev_pm_ops, nvme_suspend, nvme_resume); 2745b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 27461d3520357df99baf4ad89f86268ac96cd38092d9Stephen Hemmingerstatic const struct pci_error_handlers nvme_err_handler = { 2747b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .error_detected = nvme_error_detected, 2748b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .mmio_enabled = nvme_dump_registers, 2749b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .link_reset = nvme_link_reset, 2750b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .slot_reset = nvme_slot_reset, 2751b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .resume = nvme_error_resume, 2752b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 2753b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2754b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* Move to pci_ids.h later */ 2755b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define PCI_CLASS_STORAGE_EXPRESS 0x010802 2756b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 27576eb0d698efa9c2a35ec3ca958699717c603f85eeMatthew Wilcoxstatic const struct pci_device_id nvme_id_table[] = { 2758b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox { PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) }, 2759b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox { 0, } 2760b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 2761b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew WilcoxMODULE_DEVICE_TABLE(pci, nvme_id_table); 2762b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2763b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic struct pci_driver nvme_driver = { 2764b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .name = "nvme", 2765b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .id_table = nvme_id_table, 2766b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .probe = nvme_probe, 27678d85fce77edfc22f1d6dbf78e3af723b4b556f3dGreg Kroah-Hartman .remove = nvme_remove, 276809ece1424f52a2d3ad0ab715e96b0fb342af4f03Keith Busch .shutdown = nvme_shutdown, 2769cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch .driver = { 2770cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch .pm = &nvme_dev_pm_ops, 2771cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch }, 2772b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .err_handler = &nvme_err_handler, 2773b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 2774b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2775b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int __init nvme_init(void) 2776b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 27770ac13140d796eb1e2f8956aea97a6e5e4ebcf981Matthew Wilcox int result; 27781fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 27791fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox nvme_thread = kthread_run(nvme_kthread, NULL, "nvme"); 27801fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox if (IS_ERR(nvme_thread)) 27811fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox return PTR_ERR(nvme_thread); 2782b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 27839a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch result = -ENOMEM; 27849a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch nvme_workq = create_singlethread_workqueue("nvme"); 27859a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch if (!nvme_workq) 27869a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch goto kill_kthread; 27879a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch 27885c42ea1643a630060f9e71e06d3933d244970967Keith Busch result = register_blkdev(nvme_major, "nvme"); 27895c42ea1643a630060f9e71e06d3933d244970967Keith Busch if (result < 0) 27909a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch goto kill_workq; 27915c42ea1643a630060f9e71e06d3933d244970967Keith Busch else if (result > 0) 27920ac13140d796eb1e2f8956aea97a6e5e4ebcf981Matthew Wilcox nvme_major = result; 2793b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2794b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox result = pci_register_driver(&nvme_driver); 27951fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox if (result) 27961fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox goto unregister_blkdev; 27971fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox return 0; 2798b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 27991fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox unregister_blkdev: 2800b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unregister_blkdev(nvme_major, "nvme"); 28019a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch kill_workq: 28029a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch destroy_workqueue(nvme_workq); 28031fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox kill_kthread: 28041fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox kthread_stop(nvme_thread); 2805b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return result; 2806b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 2807b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2808b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void __exit nvme_exit(void) 2809b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 2810b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox pci_unregister_driver(&nvme_driver); 2811b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unregister_blkdev(nvme_major, "nvme"); 28129a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch destroy_workqueue(nvme_workq); 28131fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox kthread_stop(nvme_thread); 2814b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 2815b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2816b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew WilcoxMODULE_AUTHOR("Matthew Wilcox <willy@linux.intel.com>"); 2817b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew WilcoxMODULE_LICENSE("GPL"); 28186eb0d698efa9c2a35ec3ca958699717c603f85eeMatthew WilcoxMODULE_VERSION("0.9"); 2819b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxmodule_init(nvme_init); 2820b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxmodule_exit(nvme_exit); 2821