nvme-core.c revision 3193f07bb7ae723f33ac8e8b9db317a4f68d7d18
1b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 2b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * NVM Express device driver 3b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Copyright (c) 2011, Intel Corporation. 4b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 5b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * This program is free software; you can redistribute it and/or modify it 6b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * under the terms and conditions of the GNU General Public License, 7b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * version 2, as published by the Free Software Foundation. 8b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 9b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * This program is distributed in the hope it will be useful, but WITHOUT 10b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * more details. 13b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 14b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * You should have received a copy of the GNU General Public License along with 15b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * this program; if not, write to the Free Software Foundation, Inc., 16b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. 17b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 18b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 19b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/nvme.h> 20b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/bio.h> 218de055350fbaa96b6563892c195a60be583faa9cMatthew Wilcox#include <linux/bitops.h> 22b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/blkdev.h> 23fd63e9ceeeae58cfe877c2d49d41c1bf7532303cMatthew Wilcox#include <linux/delay.h> 24b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/errno.h> 25b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/fs.h> 26b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/genhd.h> 275aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox#include <linux/idr.h> 28b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/init.h> 29b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/interrupt.h> 30b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/io.h> 31b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/kdev_t.h> 321fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox#include <linux/kthread.h> 33b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/kernel.h> 34b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/mm.h> 35b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/module.h> 36b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/moduleparam.h> 37b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/pci.h> 38be7b62754e097adc0cb16c25c9ee86ee20de62fbMatthew Wilcox#include <linux/poison.h> 39c3bfe7176c035a0a2c70bc79180fb13a6c57142aMatthew Wilcox#include <linux/ptrace.h> 40b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/sched.h> 41b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/slab.h> 42b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/types.h> 435d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma#include <scsi/sg.h> 44797a796a13df6b84a4791e57306737059b5b2384Hitoshi Mitake#include <asm-generic/io-64-nonatomic-lo-hi.h> 45797a796a13df6b84a4791e57306737059b5b2384Hitoshi Mitake 46b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define NVME_Q_DEPTH 1024 47b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define SQ_SIZE(depth) (depth * sizeof(struct nvme_command)) 48b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define CQ_SIZE(depth) (depth * sizeof(struct nvme_completion)) 49e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox#define ADMIN_TIMEOUT (60 * HZ) 50b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 51b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_major; 52b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxmodule_param(nvme_major, int, 0); 53b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 5458ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcoxstatic int use_threaded_interrupts; 5558ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcoxmodule_param(use_threaded_interrupts, int, 0); 5658ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox 571fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcoxstatic DEFINE_SPINLOCK(dev_list_lock); 581fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcoxstatic LIST_HEAD(dev_list); 591fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcoxstatic struct task_struct *nvme_thread; 609a6b94584de1a0467d85b435df9c744c5c45a270Keith Buschstatic struct workqueue_struct *nvme_workq; 611fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 62d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Buschstatic void nvme_reset_failed_dev(struct work_struct *ws); 63d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch 644d115420707afcabe77d2535e092356df6664b70Keith Buschstruct async_cmd_info { 654d115420707afcabe77d2535e092356df6664b70Keith Busch struct kthread_work work; 664d115420707afcabe77d2535e092356df6664b70Keith Busch struct kthread_worker *worker; 674d115420707afcabe77d2535e092356df6664b70Keith Busch u32 result; 684d115420707afcabe77d2535e092356df6664b70Keith Busch int status; 694d115420707afcabe77d2535e092356df6664b70Keith Busch void *ctx; 704d115420707afcabe77d2535e092356df6664b70Keith Busch}; 714d115420707afcabe77d2535e092356df6664b70Keith Busch 72b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 73b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * An NVM Express queue. Each device has at least two (one for admin 74b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * commands and one for I/O commands). 75b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 76b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstruct nvme_queue { 77b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct device *q_dmadev; 78091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox struct nvme_dev *dev; 793193f07bb7ae723f33ac8e8b9db317a4f68d7d18Matthew Wilcox char irqname[24]; /* nvme4294967295-65535\0 */ 80b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox spinlock_t q_lock; 81b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_command *sq_cmds; 82b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox volatile struct nvme_completion *cqes; 83b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_addr_t sq_dma_addr; 84b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_addr_t cq_dma_addr; 85b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox wait_queue_head_t sq_full; 861fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox wait_queue_t sq_cong_wait; 87b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct bio_list sq_cong; 88b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 __iomem *q_db; 89b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 q_depth; 90b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 cq_vector; 91b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 sq_head; 92b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 sq_tail; 93b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 cq_head; 94c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch u16 qid; 95e9539f47525ecee05c9f22c3565885f3e9492c52Matthew Wilcox u8 cq_phase; 96e9539f47525ecee05c9f22c3565885f3e9492c52Matthew Wilcox u8 cqe_seen; 97224042742582c9938788b81165180c876e997a07Keith Busch u8 q_suspended; 984d115420707afcabe77d2535e092356df6664b70Keith Busch struct async_cmd_info cmdinfo; 99b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unsigned long cmdid_data[]; 100b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 101b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 102b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 103b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Check we didin't inadvertently grow the command struct 104b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 105b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic inline void _nvme_check_size(void) 106b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 107b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64); 108b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_create_cq) != 64); 109b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_create_sq) != 64); 110b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64); 111b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_features) != 64); 112f8ebf8409abfdaeeb8c847381629a2a8b8e3d816Vishal Verma BUILD_BUG_ON(sizeof(struct nvme_format_cmd) != 64); 113c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch BUILD_BUG_ON(sizeof(struct nvme_abort_cmd) != 64); 114b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_command) != 64); 115b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != 4096); 116b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_id_ns) != 4096); 117b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64); 1186ecec74520d8a357726e6c12f99080dbe7b347ddKeith Busch BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512); 119b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 120b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1215c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcoxtypedef void (*nvme_completion_fn)(struct nvme_dev *, void *, 122c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox struct nvme_completion *); 123c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox 124e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcoxstruct nvme_cmd_info { 125c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox nvme_completion_fn fn; 126c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox void *ctx; 127e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox unsigned long timeout; 128c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch int aborted; 129e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox}; 130e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox 131e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcoxstatic struct nvme_cmd_info *nvme_cmd_info(struct nvme_queue *nvmeq) 132e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox{ 133e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox return (void *)&nvmeq->cmdid_data[BITS_TO_LONGS(nvmeq->q_depth)]; 134e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox} 135e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox 136224042742582c9938788b81165180c876e997a07Keith Buschstatic unsigned nvme_queue_extra(int depth) 137224042742582c9938788b81165180c876e997a07Keith Busch{ 138224042742582c9938788b81165180c876e997a07Keith Busch return DIV_ROUND_UP(depth, 8) + (depth * sizeof(struct nvme_cmd_info)); 139224042742582c9938788b81165180c876e997a07Keith Busch} 140224042742582c9938788b81165180c876e997a07Keith Busch 141b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/** 142714a7a22884b74862540bc84955274d86b2f6040Matthew Wilcox * alloc_cmdid() - Allocate a Command ID 143714a7a22884b74862540bc84955274d86b2f6040Matthew Wilcox * @nvmeq: The queue that will be used for this command 144714a7a22884b74862540bc84955274d86b2f6040Matthew Wilcox * @ctx: A pointer that will be passed to the handler 145c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox * @handler: The function to call on completion 146b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 147b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Allocate a Command ID for a queue. The data passed in will 148b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * be passed to the completion handler. This is implemented by using 149b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * the bottom two bits of the ctx pointer to store the handler ID. 150b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Passing in a pointer that's not 4-byte aligned will cause a BUG. 151b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * We can change this if it becomes a problem. 152184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox * 153184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox * May be called with local interrupts disabled and the q_lock held, 154184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox * or with interrupts enabled and no locks held. 155b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 156c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcoxstatic int alloc_cmdid(struct nvme_queue *nvmeq, void *ctx, 157c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox nvme_completion_fn handler, unsigned timeout) 158b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 159e6d15f79f997a98b3a69abbc462fc9041cc1a7b4Matthew Wilcox int depth = nvmeq->q_depth - 1; 160e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox struct nvme_cmd_info *info = nvme_cmd_info(nvmeq); 161b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int cmdid; 162b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 163b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox do { 164b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cmdid = find_first_zero_bit(nvmeq->cmdid_data, depth); 165b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (cmdid >= depth) 166b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -EBUSY; 167b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } while (test_and_set_bit(cmdid, nvmeq->cmdid_data)); 168b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 169c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox info[cmdid].fn = handler; 170c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox info[cmdid].ctx = ctx; 171e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox info[cmdid].timeout = jiffies + timeout; 172c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch info[cmdid].aborted = 0; 173b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return cmdid; 174b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 175b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 176b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int alloc_cmdid_killable(struct nvme_queue *nvmeq, void *ctx, 177c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox nvme_completion_fn handler, unsigned timeout) 178b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 179b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int cmdid; 180b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox wait_event_killable(nvmeq->sq_full, 181e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox (cmdid = alloc_cmdid(nvmeq, ctx, handler, timeout)) >= 0); 182b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return (cmdid < 0) ? -EINTR : cmdid; 183b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 184b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 185c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox/* Special values must be less than 0x1000 */ 186c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox#define CMD_CTX_BASE ((void *)POISON_POINTER_DELTA) 187d2d8703481f60d67f49e3177196cbe474b11377cMatthew Wilcox#define CMD_CTX_CANCELLED (0x30C + CMD_CTX_BASE) 188d2d8703481f60d67f49e3177196cbe474b11377cMatthew Wilcox#define CMD_CTX_COMPLETED (0x310 + CMD_CTX_BASE) 189d2d8703481f60d67f49e3177196cbe474b11377cMatthew Wilcox#define CMD_CTX_INVALID (0x314 + CMD_CTX_BASE) 19000df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox#define CMD_CTX_FLUSH (0x318 + CMD_CTX_BASE) 191c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch#define CMD_CTX_ABORT (0x31C + CMD_CTX_BASE) 192be7b62754e097adc0cb16c25c9ee86ee20de62fbMatthew Wilcox 1935c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcoxstatic void special_completion(struct nvme_dev *dev, void *ctx, 194c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox struct nvme_completion *cqe) 195c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox{ 196c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox if (ctx == CMD_CTX_CANCELLED) 197c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox return; 198c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox if (ctx == CMD_CTX_FLUSH) 199c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox return; 200c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch if (ctx == CMD_CTX_ABORT) { 201c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch ++dev->abort_limit; 202c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch return; 203c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch } 204c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox if (ctx == CMD_CTX_COMPLETED) { 2055c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcox dev_warn(&dev->pci_dev->dev, 206c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox "completed id %d twice on queue %d\n", 207c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox cqe->command_id, le16_to_cpup(&cqe->sq_id)); 208c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox return; 209c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox } 210c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox if (ctx == CMD_CTX_INVALID) { 2115c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcox dev_warn(&dev->pci_dev->dev, 212c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox "invalid id %d completed on queue %d\n", 213c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox cqe->command_id, le16_to_cpup(&cqe->sq_id)); 214c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox return; 215c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox } 216c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox 2175c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcox dev_warn(&dev->pci_dev->dev, "Unknown special completion %p\n", ctx); 218c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox} 219c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox 2204d115420707afcabe77d2535e092356df6664b70Keith Buschstatic void async_completion(struct nvme_dev *dev, void *ctx, 2214d115420707afcabe77d2535e092356df6664b70Keith Busch struct nvme_completion *cqe) 2224d115420707afcabe77d2535e092356df6664b70Keith Busch{ 2234d115420707afcabe77d2535e092356df6664b70Keith Busch struct async_cmd_info *cmdinfo = ctx; 2244d115420707afcabe77d2535e092356df6664b70Keith Busch cmdinfo->result = le32_to_cpup(&cqe->result); 2254d115420707afcabe77d2535e092356df6664b70Keith Busch cmdinfo->status = le16_to_cpup(&cqe->status) >> 1; 2264d115420707afcabe77d2535e092356df6664b70Keith Busch queue_kthread_work(cmdinfo->worker, &cmdinfo->work); 2274d115420707afcabe77d2535e092356df6664b70Keith Busch} 2284d115420707afcabe77d2535e092356df6664b70Keith Busch 229184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox/* 230184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox * Called with local interrupts disabled and the q_lock held. May not sleep. 231184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox */ 232c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcoxstatic void *free_cmdid(struct nvme_queue *nvmeq, int cmdid, 233c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox nvme_completion_fn *fn) 234b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 235c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox void *ctx; 236e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox struct nvme_cmd_info *info = nvme_cmd_info(nvmeq); 237b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 238c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox if (cmdid >= nvmeq->q_depth) { 239c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox *fn = special_completion; 24048e3d39816416b3bf03dee3a796c0c04427c1a31Matthew Wilcox return CMD_CTX_INVALID; 241c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox } 242859361a228258edf4821d9f5635825033eca78e8Keith Busch if (fn) 243859361a228258edf4821d9f5635825033eca78e8Keith Busch *fn = info[cmdid].fn; 244c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox ctx = info[cmdid].ctx; 245c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox info[cmdid].fn = special_completion; 246e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox info[cmdid].ctx = CMD_CTX_COMPLETED; 247b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox clear_bit(cmdid, nvmeq->cmdid_data); 248b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox wake_up(&nvmeq->sq_full); 249c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox return ctx; 250b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 251b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 252c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcoxstatic void *cancel_cmdid(struct nvme_queue *nvmeq, int cmdid, 253c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox nvme_completion_fn *fn) 2543c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox{ 255c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox void *ctx; 256e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox struct nvme_cmd_info *info = nvme_cmd_info(nvmeq); 257c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox if (fn) 258c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox *fn = info[cmdid].fn; 259c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox ctx = info[cmdid].ctx; 260c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox info[cmdid].fn = special_completion; 261e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox info[cmdid].ctx = CMD_CTX_CANCELLED; 262c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox return ctx; 2633c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox} 2643c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox 2655d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermastruct nvme_queue *get_nvmeq(struct nvme_dev *dev) 266b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 267040a93b52a9eee8177ebaf2ba0ee0f9f518d1bf8Matthew Wilcox return dev->queues[get_cpu() + 1]; 268b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 269b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2705d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermavoid put_nvmeq(struct nvme_queue *nvmeq) 271b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 2721b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox put_cpu(); 273b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 274b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 275b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/** 276714a7a22884b74862540bc84955274d86b2f6040Matthew Wilcox * nvme_submit_cmd() - Copy a command into a queue and ring the doorbell 277b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * @nvmeq: The queue to use 278b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * @cmd: The command to send 279b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 280b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Safe to use from interrupt context 281b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 282b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_submit_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd) 283b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 284b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unsigned long flags; 285b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 tail; 286b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox spin_lock_irqsave(&nvmeq->q_lock, flags); 287b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox tail = nvmeq->sq_tail; 288b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memcpy(&nvmeq->sq_cmds[tail], cmd, sizeof(*cmd)); 289b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (++tail == nvmeq->q_depth) 290b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox tail = 0; 2917547881d0951384f9833ec3a80fac8f3f16f3b98Matthew Wilcox writel(tail, nvmeq->q_db); 292b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->sq_tail = tail; 293b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox spin_unlock_irqrestore(&nvmeq->q_lock, flags); 294b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 295b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 296b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 297b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 298eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcoxstatic __le64 **iod_list(struct nvme_iod *iod) 299e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews{ 300eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return ((void *)iod) + iod->offset; 301e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews} 302e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews 303eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox/* 304eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox * Will slightly overestimate the number of pages needed. This is OK 305eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox * as it only leads to a small amount of wasted memory for the lifetime of 306eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox * the I/O. 307eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox */ 308eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcoxstatic int nvme_npages(unsigned size) 309eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox{ 310eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox unsigned nprps = DIV_ROUND_UP(size + PAGE_SIZE, PAGE_SIZE); 311eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return DIV_ROUND_UP(8 * nprps, PAGE_SIZE - 8); 312eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox} 313b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 314eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcoxstatic struct nvme_iod * 315eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcoxnvme_alloc_iod(unsigned nseg, unsigned nbytes, gfp_t gfp) 316b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 317eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox struct nvme_iod *iod = kmalloc(sizeof(struct nvme_iod) + 318eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox sizeof(__le64 *) * nvme_npages(nbytes) + 319eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox sizeof(struct scatterlist) * nseg, gfp); 320eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox 321eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox if (iod) { 322eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->offset = offsetof(struct nvme_iod, sg[nseg]); 323eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->npages = -1; 324eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->length = nbytes; 3252b1960341576bf51c01b12fefeb1cc53820923e7Keith Busch iod->nents = 0; 3266198221fa0df0298513b35796f63f242ea97134eKeith Busch iod->start_time = jiffies; 327eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox } 328eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox 329eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return iod; 330b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 331b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 3325d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermavoid nvme_free_iod(struct nvme_dev *dev, struct nvme_iod *iod) 333b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 334eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox const int last_prp = PAGE_SIZE / 8 - 1; 335eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox int i; 336eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox __le64 **list = iod_list(iod); 337eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox dma_addr_t prp_dma = iod->first_dma; 338eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox 339eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox if (iod->npages == 0) 340eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox dma_pool_free(dev->prp_small_pool, list[0], prp_dma); 341eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox for (i = 0; i < iod->npages; i++) { 342eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox __le64 *prp_list = list[i]; 343eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox dma_addr_t next_prp_dma = le64_to_cpu(prp_list[last_prp]); 344eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox dma_pool_free(dev->prp_page_pool, prp_list, prp_dma); 345eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox prp_dma = next_prp_dma; 346eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox } 347eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox kfree(iod); 348b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 349b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 3506198221fa0df0298513b35796f63f242ea97134eKeith Buschstatic void nvme_start_io_acct(struct bio *bio) 3516198221fa0df0298513b35796f63f242ea97134eKeith Busch{ 3526198221fa0df0298513b35796f63f242ea97134eKeith Busch struct gendisk *disk = bio->bi_bdev->bd_disk; 3536198221fa0df0298513b35796f63f242ea97134eKeith Busch const int rw = bio_data_dir(bio); 3546198221fa0df0298513b35796f63f242ea97134eKeith Busch int cpu = part_stat_lock(); 3556198221fa0df0298513b35796f63f242ea97134eKeith Busch part_round_stats(cpu, &disk->part0); 3566198221fa0df0298513b35796f63f242ea97134eKeith Busch part_stat_inc(cpu, &disk->part0, ios[rw]); 3576198221fa0df0298513b35796f63f242ea97134eKeith Busch part_stat_add(cpu, &disk->part0, sectors[rw], bio_sectors(bio)); 3586198221fa0df0298513b35796f63f242ea97134eKeith Busch part_inc_in_flight(&disk->part0, rw); 3596198221fa0df0298513b35796f63f242ea97134eKeith Busch part_stat_unlock(); 3606198221fa0df0298513b35796f63f242ea97134eKeith Busch} 3616198221fa0df0298513b35796f63f242ea97134eKeith Busch 3626198221fa0df0298513b35796f63f242ea97134eKeith Buschstatic void nvme_end_io_acct(struct bio *bio, unsigned long start_time) 3636198221fa0df0298513b35796f63f242ea97134eKeith Busch{ 3646198221fa0df0298513b35796f63f242ea97134eKeith Busch struct gendisk *disk = bio->bi_bdev->bd_disk; 3656198221fa0df0298513b35796f63f242ea97134eKeith Busch const int rw = bio_data_dir(bio); 3666198221fa0df0298513b35796f63f242ea97134eKeith Busch unsigned long duration = jiffies - start_time; 3676198221fa0df0298513b35796f63f242ea97134eKeith Busch int cpu = part_stat_lock(); 3686198221fa0df0298513b35796f63f242ea97134eKeith Busch part_stat_add(cpu, &disk->part0, ticks[rw], duration); 3696198221fa0df0298513b35796f63f242ea97134eKeith Busch part_round_stats(cpu, &disk->part0); 3706198221fa0df0298513b35796f63f242ea97134eKeith Busch part_dec_in_flight(&disk->part0, rw); 3716198221fa0df0298513b35796f63f242ea97134eKeith Busch part_stat_unlock(); 3726198221fa0df0298513b35796f63f242ea97134eKeith Busch} 3736198221fa0df0298513b35796f63f242ea97134eKeith Busch 3745c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcoxstatic void bio_completion(struct nvme_dev *dev, void *ctx, 375b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_completion *cqe) 376b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 377eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox struct nvme_iod *iod = ctx; 378eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox struct bio *bio = iod->private; 379b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 status = le16_to_cpup(&cqe->status) >> 1; 380b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 3819e59d091b0eb04f223ed037348e3d9e36f30e72bKeith Busch if (iod->nents) { 3822b1960341576bf51c01b12fefeb1cc53820923e7Keith Busch dma_unmap_sg(&dev->pci_dev->dev, iod->sg, iod->nents, 383b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox bio_data_dir(bio) ? DMA_TO_DEVICE : DMA_FROM_DEVICE); 3849e59d091b0eb04f223ed037348e3d9e36f30e72bKeith Busch nvme_end_io_acct(bio, iod->start_time); 3859e59d091b0eb04f223ed037348e3d9e36f30e72bKeith Busch } 386eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox nvme_free_iod(dev, iod); 387427e97080196548557b288517537ab7eb48c309fKeith Busch if (status) 3881ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox bio_endio(bio, -EIO); 389427e97080196548557b288517537ab7eb48c309fKeith Busch else 3901ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox bio_endio(bio, 0); 391b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 392b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 393184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox/* length is in bytes. gfp flags indicates whether we may sleep. */ 3945d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermaint nvme_setup_prps(struct nvme_dev *dev, struct nvme_common_command *cmd, 3955d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma struct nvme_iod *iod, int total_len, gfp_t gfp) 396ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox{ 39799802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox struct dma_pool *pool; 398eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox int length = total_len; 399eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox struct scatterlist *sg = iod->sg; 400ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox int dma_len = sg_dma_len(sg); 401ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox u64 dma_addr = sg_dma_address(sg); 402ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox int offset = offset_in_page(dma_addr); 403e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews __le64 *prp_list; 404eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox __le64 **list = iod_list(iod); 405e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_addr_t prp_dma; 406eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox int nprps, i; 407ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox 408ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmd->prp1 = cpu_to_le64(dma_addr); 409ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox length -= (PAGE_SIZE - offset); 410ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox if (length <= 0) 411eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return total_len; 412ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox 413ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox dma_len -= (PAGE_SIZE - offset); 414ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox if (dma_len) { 415ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox dma_addr += (PAGE_SIZE - offset); 416ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox } else { 417ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox sg = sg_next(sg); 418ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox dma_addr = sg_dma_address(sg); 419ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox dma_len = sg_dma_len(sg); 420ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox } 421ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox 422ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox if (length <= PAGE_SIZE) { 423ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmd->prp2 = cpu_to_le64(dma_addr); 424eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return total_len; 425e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews } 426e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews 427e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews nprps = DIV_ROUND_UP(length, PAGE_SIZE); 42899802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox if (nprps <= (256 / 8)) { 42999802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox pool = dev->prp_small_pool; 430eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->npages = 0; 43199802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox } else { 43299802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox pool = dev->prp_page_pool; 433eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->npages = 1; 43499802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox } 43599802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox 436b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox prp_list = dma_pool_alloc(pool, gfp, &prp_dma); 437b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox if (!prp_list) { 438b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox cmd->prp2 = cpu_to_le64(dma_addr); 439eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->npages = -1; 440eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return (total_len - length) + PAGE_SIZE; 441b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox } 442eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox list[0] = prp_list; 443eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->first_dma = prp_dma; 444e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews cmd->prp2 = cpu_to_le64(prp_dma); 445e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews i = 0; 446e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews for (;;) { 4477523d834dd1573610078eb1ac0933f6490232f90Matthew Wilcox if (i == PAGE_SIZE / 8) { 448e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews __le64 *old_prp_list = prp_list; 449b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox prp_list = dma_pool_alloc(pool, gfp, &prp_dma); 450eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox if (!prp_list) 451eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return total_len - length; 452eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox list[iod->npages++] = prp_list; 4537523d834dd1573610078eb1ac0933f6490232f90Matthew Wilcox prp_list[0] = old_prp_list[i - 1]; 4547523d834dd1573610078eb1ac0933f6490232f90Matthew Wilcox old_prp_list[i - 1] = cpu_to_le64(prp_dma); 4557523d834dd1573610078eb1ac0933f6490232f90Matthew Wilcox i = 1; 456e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews } 457e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews prp_list[i++] = cpu_to_le64(dma_addr); 458e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_len -= PAGE_SIZE; 459e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_addr += PAGE_SIZE; 460e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews length -= PAGE_SIZE; 461e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews if (length <= 0) 462e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews break; 463e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews if (dma_len > 0) 464e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews continue; 465e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews BUG_ON(dma_len < 0); 466e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews sg = sg_next(sg); 467e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_addr = sg_dma_address(sg); 468e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_len = sg_dma_len(sg); 469ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox } 470ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox 471eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return total_len; 472ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox} 473ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox 474427e97080196548557b288517537ab7eb48c309fKeith Buschstruct nvme_bio_pair { 475427e97080196548557b288517537ab7eb48c309fKeith Busch struct bio b1, b2, *parent; 476427e97080196548557b288517537ab7eb48c309fKeith Busch struct bio_vec *bv1, *bv2; 477427e97080196548557b288517537ab7eb48c309fKeith Busch int err; 478427e97080196548557b288517537ab7eb48c309fKeith Busch atomic_t cnt; 479427e97080196548557b288517537ab7eb48c309fKeith Busch}; 480427e97080196548557b288517537ab7eb48c309fKeith Busch 481427e97080196548557b288517537ab7eb48c309fKeith Buschstatic void nvme_bio_pair_endio(struct bio *bio, int err) 482427e97080196548557b288517537ab7eb48c309fKeith Busch{ 483427e97080196548557b288517537ab7eb48c309fKeith Busch struct nvme_bio_pair *bp = bio->bi_private; 484427e97080196548557b288517537ab7eb48c309fKeith Busch 485427e97080196548557b288517537ab7eb48c309fKeith Busch if (err) 486427e97080196548557b288517537ab7eb48c309fKeith Busch bp->err = err; 487427e97080196548557b288517537ab7eb48c309fKeith Busch 488427e97080196548557b288517537ab7eb48c309fKeith Busch if (atomic_dec_and_test(&bp->cnt)) { 489427e97080196548557b288517537ab7eb48c309fKeith Busch bio_endio(bp->parent, bp->err); 4901b56749e541ad59068582f2a28297843e243b856Keith Busch kfree(bp->bv1); 4911b56749e541ad59068582f2a28297843e243b856Keith Busch kfree(bp->bv2); 492427e97080196548557b288517537ab7eb48c309fKeith Busch kfree(bp); 493427e97080196548557b288517537ab7eb48c309fKeith Busch } 494427e97080196548557b288517537ab7eb48c309fKeith Busch} 495427e97080196548557b288517537ab7eb48c309fKeith Busch 496427e97080196548557b288517537ab7eb48c309fKeith Buschstatic struct nvme_bio_pair *nvme_bio_split(struct bio *bio, int idx, 497427e97080196548557b288517537ab7eb48c309fKeith Busch int len, int offset) 498427e97080196548557b288517537ab7eb48c309fKeith Busch{ 499427e97080196548557b288517537ab7eb48c309fKeith Busch struct nvme_bio_pair *bp; 500427e97080196548557b288517537ab7eb48c309fKeith Busch 501427e97080196548557b288517537ab7eb48c309fKeith Busch BUG_ON(len > bio->bi_size); 502427e97080196548557b288517537ab7eb48c309fKeith Busch BUG_ON(idx > bio->bi_vcnt); 503427e97080196548557b288517537ab7eb48c309fKeith Busch 504427e97080196548557b288517537ab7eb48c309fKeith Busch bp = kmalloc(sizeof(*bp), GFP_ATOMIC); 505427e97080196548557b288517537ab7eb48c309fKeith Busch if (!bp) 506427e97080196548557b288517537ab7eb48c309fKeith Busch return NULL; 507427e97080196548557b288517537ab7eb48c309fKeith Busch bp->err = 0; 508427e97080196548557b288517537ab7eb48c309fKeith Busch 509427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b1 = *bio; 510427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b2 = *bio; 511427e97080196548557b288517537ab7eb48c309fKeith Busch 512427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b1.bi_size = len; 513427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b2.bi_size -= len; 514427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b1.bi_vcnt = idx; 515427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b2.bi_idx = idx; 516427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b2.bi_sector += len >> 9; 517427e97080196548557b288517537ab7eb48c309fKeith Busch 518427e97080196548557b288517537ab7eb48c309fKeith Busch if (offset) { 519427e97080196548557b288517537ab7eb48c309fKeith Busch bp->bv1 = kmalloc(bio->bi_max_vecs * sizeof(struct bio_vec), 520427e97080196548557b288517537ab7eb48c309fKeith Busch GFP_ATOMIC); 521427e97080196548557b288517537ab7eb48c309fKeith Busch if (!bp->bv1) 522427e97080196548557b288517537ab7eb48c309fKeith Busch goto split_fail_1; 523427e97080196548557b288517537ab7eb48c309fKeith Busch 524427e97080196548557b288517537ab7eb48c309fKeith Busch bp->bv2 = kmalloc(bio->bi_max_vecs * sizeof(struct bio_vec), 525427e97080196548557b288517537ab7eb48c309fKeith Busch GFP_ATOMIC); 526427e97080196548557b288517537ab7eb48c309fKeith Busch if (!bp->bv2) 527427e97080196548557b288517537ab7eb48c309fKeith Busch goto split_fail_2; 528427e97080196548557b288517537ab7eb48c309fKeith Busch 529427e97080196548557b288517537ab7eb48c309fKeith Busch memcpy(bp->bv1, bio->bi_io_vec, 530427e97080196548557b288517537ab7eb48c309fKeith Busch bio->bi_max_vecs * sizeof(struct bio_vec)); 531427e97080196548557b288517537ab7eb48c309fKeith Busch memcpy(bp->bv2, bio->bi_io_vec, 532427e97080196548557b288517537ab7eb48c309fKeith Busch bio->bi_max_vecs * sizeof(struct bio_vec)); 533427e97080196548557b288517537ab7eb48c309fKeith Busch 534427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b1.bi_io_vec = bp->bv1; 535427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b2.bi_io_vec = bp->bv2; 536427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b2.bi_io_vec[idx].bv_offset += offset; 537427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b2.bi_io_vec[idx].bv_len -= offset; 538427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b1.bi_io_vec[idx].bv_len = offset; 539427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b1.bi_vcnt++; 540427e97080196548557b288517537ab7eb48c309fKeith Busch } else 541427e97080196548557b288517537ab7eb48c309fKeith Busch bp->bv1 = bp->bv2 = NULL; 542427e97080196548557b288517537ab7eb48c309fKeith Busch 543427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b1.bi_private = bp; 544427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b2.bi_private = bp; 545427e97080196548557b288517537ab7eb48c309fKeith Busch 546427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b1.bi_end_io = nvme_bio_pair_endio; 547427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b2.bi_end_io = nvme_bio_pair_endio; 548427e97080196548557b288517537ab7eb48c309fKeith Busch 549427e97080196548557b288517537ab7eb48c309fKeith Busch bp->parent = bio; 550427e97080196548557b288517537ab7eb48c309fKeith Busch atomic_set(&bp->cnt, 2); 551427e97080196548557b288517537ab7eb48c309fKeith Busch 552427e97080196548557b288517537ab7eb48c309fKeith Busch return bp; 553427e97080196548557b288517537ab7eb48c309fKeith Busch 554427e97080196548557b288517537ab7eb48c309fKeith Busch split_fail_2: 555427e97080196548557b288517537ab7eb48c309fKeith Busch kfree(bp->bv1); 556427e97080196548557b288517537ab7eb48c309fKeith Busch split_fail_1: 557427e97080196548557b288517537ab7eb48c309fKeith Busch kfree(bp); 558427e97080196548557b288517537ab7eb48c309fKeith Busch return NULL; 559427e97080196548557b288517537ab7eb48c309fKeith Busch} 560427e97080196548557b288517537ab7eb48c309fKeith Busch 561427e97080196548557b288517537ab7eb48c309fKeith Buschstatic int nvme_split_and_submit(struct bio *bio, struct nvme_queue *nvmeq, 562427e97080196548557b288517537ab7eb48c309fKeith Busch int idx, int len, int offset) 563427e97080196548557b288517537ab7eb48c309fKeith Busch{ 564427e97080196548557b288517537ab7eb48c309fKeith Busch struct nvme_bio_pair *bp = nvme_bio_split(bio, idx, len, offset); 565427e97080196548557b288517537ab7eb48c309fKeith Busch if (!bp) 566427e97080196548557b288517537ab7eb48c309fKeith Busch return -ENOMEM; 567427e97080196548557b288517537ab7eb48c309fKeith Busch 568427e97080196548557b288517537ab7eb48c309fKeith Busch if (bio_list_empty(&nvmeq->sq_cong)) 569427e97080196548557b288517537ab7eb48c309fKeith Busch add_wait_queue(&nvmeq->sq_full, &nvmeq->sq_cong_wait); 570427e97080196548557b288517537ab7eb48c309fKeith Busch bio_list_add(&nvmeq->sq_cong, &bp->b1); 571427e97080196548557b288517537ab7eb48c309fKeith Busch bio_list_add(&nvmeq->sq_cong, &bp->b2); 572427e97080196548557b288517537ab7eb48c309fKeith Busch 573427e97080196548557b288517537ab7eb48c309fKeith Busch return 0; 574427e97080196548557b288517537ab7eb48c309fKeith Busch} 575427e97080196548557b288517537ab7eb48c309fKeith Busch 5761ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox/* NVMe scatterlists require no holes in the virtual address */ 5771ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox#define BIOVEC_NOT_VIRT_MERGEABLE(vec1, vec2) ((vec2)->bv_offset || \ 5781ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox (((vec1)->bv_offset + (vec1)->bv_len) % PAGE_SIZE)) 5791ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox 580427e97080196548557b288517537ab7eb48c309fKeith Buschstatic int nvme_map_bio(struct nvme_queue *nvmeq, struct nvme_iod *iod, 581b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct bio *bio, enum dma_data_direction dma_dir, int psegs) 582b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 583768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox struct bio_vec *bvec, *bvprv = NULL; 584768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox struct scatterlist *sg = NULL; 585159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch int i, length = 0, nsegs = 0, split_len = bio->bi_size; 586159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch 587159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch if (nvmeq->dev->stripe_size) 588159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch split_len = nvmeq->dev->stripe_size - 589159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch ((bio->bi_sector << 9) & (nvmeq->dev->stripe_size - 1)); 590b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 591eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox sg_init_table(iod->sg, psegs); 592b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox bio_for_each_segment(bvec, bio, i) { 593768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox if (bvprv && BIOVEC_PHYS_MERGEABLE(bvprv, bvec)) { 594768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox sg->length += bvec->bv_len; 595768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox } else { 5961ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox if (bvprv && BIOVEC_NOT_VIRT_MERGEABLE(bvprv, bvec)) 597427e97080196548557b288517537ab7eb48c309fKeith Busch return nvme_split_and_submit(bio, nvmeq, i, 598427e97080196548557b288517537ab7eb48c309fKeith Busch length, 0); 599427e97080196548557b288517537ab7eb48c309fKeith Busch 600eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox sg = sg ? sg + 1 : iod->sg; 601768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox sg_set_page(sg, bvec->bv_page, bvec->bv_len, 602768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox bvec->bv_offset); 603768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox nsegs++; 604768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox } 605159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch 606159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch if (split_len - length < bvec->bv_len) 607159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch return nvme_split_and_submit(bio, nvmeq, i, split_len, 608159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch split_len - length); 6091ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox length += bvec->bv_len; 610768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox bvprv = bvec; 611b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 612eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->nents = nsegs; 613768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox sg_mark_end(sg); 614427e97080196548557b288517537ab7eb48c309fKeith Busch if (dma_map_sg(nvmeq->q_dmadev, iod->sg, iod->nents, dma_dir) == 0) 6151ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox return -ENOMEM; 616427e97080196548557b288517537ab7eb48c309fKeith Busch 617159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch BUG_ON(length != bio->bi_size); 6181ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox return length; 619b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 620b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 6210e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch/* 6220e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch * We reuse the small pool to allocate the 16-byte range here as it is not 6230e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch * worth having a special pool for these or additional cases to handle freeing 6240e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch * the iod. 6250e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch */ 6260e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Buschstatic int nvme_submit_discard(struct nvme_queue *nvmeq, struct nvme_ns *ns, 6270e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch struct bio *bio, struct nvme_iod *iod, int cmdid) 6280e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch{ 6290e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch struct nvme_dsm_range *range; 6300e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch struct nvme_command *cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail]; 6310e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch 6320e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch range = dma_pool_alloc(nvmeq->dev->prp_small_pool, GFP_ATOMIC, 6330e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch &iod->first_dma); 6340e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch if (!range) 6350e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch return -ENOMEM; 6360e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch 6370e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch iod_list(iod)[0] = (__le64 *)range; 6380e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch iod->npages = 0; 6390e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch 6400e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch range->cattr = cpu_to_le32(0); 6410e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch range->nlb = cpu_to_le32(bio->bi_size >> ns->lba_shift); 642063cc6d5591ea9c0631b81ac5c7b829d99738b2fMatthew Wilcox range->slba = cpu_to_le64(nvme_block_nr(ns, bio->bi_sector)); 6430e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch 6440e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch memset(cmnd, 0, sizeof(*cmnd)); 6450e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch cmnd->dsm.opcode = nvme_cmd_dsm; 6460e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch cmnd->dsm.command_id = cmdid; 6470e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch cmnd->dsm.nsid = cpu_to_le32(ns->ns_id); 6480e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch cmnd->dsm.prp1 = cpu_to_le64(iod->first_dma); 6490e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch cmnd->dsm.nr = 0; 6500e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD); 6510e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch 6520e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch if (++nvmeq->sq_tail == nvmeq->q_depth) 6530e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch nvmeq->sq_tail = 0; 6540e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch writel(nvmeq->sq_tail, nvmeq->q_db); 6550e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch 6560e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch return 0; 6570e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch} 6580e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch 65900df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcoxstatic int nvme_submit_flush(struct nvme_queue *nvmeq, struct nvme_ns *ns, 66000df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox int cmdid) 66100df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox{ 66200df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox struct nvme_command *cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail]; 66300df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 66400df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox memset(cmnd, 0, sizeof(*cmnd)); 66500df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox cmnd->common.opcode = nvme_cmd_flush; 66600df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox cmnd->common.command_id = cmdid; 66700df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox cmnd->common.nsid = cpu_to_le32(ns->ns_id); 66800df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 66900df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox if (++nvmeq->sq_tail == nvmeq->q_depth) 67000df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox nvmeq->sq_tail = 0; 67100df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox writel(nvmeq->sq_tail, nvmeq->q_db); 67200df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 67300df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox return 0; 67400df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox} 67500df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 6765d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermaint nvme_submit_flush_data(struct nvme_queue *nvmeq, struct nvme_ns *ns) 67700df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox{ 67800df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox int cmdid = alloc_cmdid(nvmeq, (void *)CMD_CTX_FLUSH, 679ff976d724a74e4522e9ca2de1fb37ac4520f454fMatthew Wilcox special_completion, NVME_IO_TIMEOUT); 68000df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox if (unlikely(cmdid < 0)) 68100df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox return cmdid; 68200df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 68300df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox return nvme_submit_flush(nvmeq, ns, cmdid); 68400df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox} 68500df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 686184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox/* 687184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox * Called with local interrupts disabled and the q_lock held. May not sleep. 688184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox */ 689b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_submit_bio_queue(struct nvme_queue *nvmeq, struct nvme_ns *ns, 690b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct bio *bio) 691b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 692ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox struct nvme_command *cmnd; 693eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox struct nvme_iod *iod; 694b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox enum dma_data_direction dma_dir; 6951287dabd345f447bbe0f7a99fc95ab89bcfc0f5dWei Yongjun int cmdid, length, result; 696b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 control; 697b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 dsmgmt; 698b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int psegs = bio_phys_segments(ns->queue, bio); 699b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 70000df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox if ((bio->bi_rw & REQ_FLUSH) && psegs) { 70100df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox result = nvme_submit_flush_data(nvmeq, ns); 70200df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox if (result) 70300df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox return result; 70400df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox } 70500df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 7061287dabd345f447bbe0f7a99fc95ab89bcfc0f5dWei Yongjun result = -ENOMEM; 707eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod = nvme_alloc_iod(psegs, bio->bi_size, GFP_ATOMIC); 708eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox if (!iod) 709eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox goto nomem; 710eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->private = bio; 711b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 712eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox result = -EBUSY; 713ff976d724a74e4522e9ca2de1fb37ac4520f454fMatthew Wilcox cmdid = alloc_cmdid(nvmeq, iod, bio_completion, NVME_IO_TIMEOUT); 714b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (unlikely(cmdid < 0)) 715eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox goto free_iod; 716b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 7170e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch if (bio->bi_rw & REQ_DISCARD) { 7180e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch result = nvme_submit_discard(nvmeq, ns, bio, iod, cmdid); 7190e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch if (result) 7200e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch goto free_cmdid; 7210e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch return result; 7220e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch } 72300df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox if ((bio->bi_rw & REQ_FLUSH) && !psegs) 72400df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox return nvme_submit_flush(nvmeq, ns, cmdid); 72500df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 726b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox control = 0; 727b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (bio->bi_rw & REQ_FUA) 728b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox control |= NVME_RW_FUA; 729b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (bio->bi_rw & (REQ_FAILFAST_DEV | REQ_RAHEAD)) 730b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox control |= NVME_RW_LR; 731b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 732b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dsmgmt = 0; 733b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (bio->bi_rw & REQ_RAHEAD) 734b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH; 735b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 736ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail]; 737b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 738b8deb62cf271fa9381edc8cf52bcae2f0225c55aMatthew Wilcox memset(cmnd, 0, sizeof(*cmnd)); 739b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (bio_data_dir(bio)) { 740ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.opcode = nvme_cmd_write; 741b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_dir = DMA_TO_DEVICE; 742b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } else { 743ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.opcode = nvme_cmd_read; 744b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_dir = DMA_FROM_DEVICE; 745b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 746b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 747427e97080196548557b288517537ab7eb48c309fKeith Busch result = nvme_map_bio(nvmeq, iod, bio, dma_dir, psegs); 748427e97080196548557b288517537ab7eb48c309fKeith Busch if (result <= 0) 749859361a228258edf4821d9f5635825033eca78e8Keith Busch goto free_cmdid; 7501ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox length = result; 751b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 752ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.command_id = cmdid; 753ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.nsid = cpu_to_le32(ns->ns_id); 754eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox length = nvme_setup_prps(nvmeq->dev, &cmnd->common, iod, length, 755eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox GFP_ATOMIC); 756063cc6d5591ea9c0631b81ac5c7b829d99738b2fMatthew Wilcox cmnd->rw.slba = cpu_to_le64(nvme_block_nr(ns, bio->bi_sector)); 7571ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox cmnd->rw.length = cpu_to_le16((length >> ns->lba_shift) - 1); 758ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.control = cpu_to_le16(control); 759ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt); 760b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 7616198221fa0df0298513b35796f63f242ea97134eKeith Busch nvme_start_io_acct(bio); 762b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (++nvmeq->sq_tail == nvmeq->q_depth) 763b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->sq_tail = 0; 7647547881d0951384f9833ec3a80fac8f3f16f3b98Matthew Wilcox writel(nvmeq->sq_tail, nvmeq->q_db); 765b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 7661974b1ae8852324a75fb8cfecbc7b758fd5a2c3cMatthew Wilcox return 0; 7671974b1ae8852324a75fb8cfecbc7b758fd5a2c3cMatthew Wilcox 768859361a228258edf4821d9f5635825033eca78e8Keith Busch free_cmdid: 769859361a228258edf4821d9f5635825033eca78e8Keith Busch free_cmdid(nvmeq, cmdid, NULL); 770eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox free_iod: 771eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox nvme_free_iod(nvmeq->dev, iod); 772eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox nomem: 773eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox return result; 774b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 775b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 776e9539f47525ecee05c9f22c3565885f3e9492c52Matthew Wilcoxstatic int nvme_process_cq(struct nvme_queue *nvmeq) 777b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 778821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox u16 head, phase; 779b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 780b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox head = nvmeq->cq_head; 781821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox phase = nvmeq->cq_phase; 782b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 783b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox for (;;) { 784c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox void *ctx; 785c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox nvme_completion_fn fn; 786b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_completion cqe = nvmeq->cqes[head]; 787821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox if ((le16_to_cpu(cqe.status) & 1) != phase) 788b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox break; 789b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->sq_head = le16_to_cpu(cqe.sq_head); 790b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (++head == nvmeq->q_depth) { 791b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox head = 0; 792821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox phase = !phase; 793b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 794b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 795c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox ctx = free_cmdid(nvmeq, cqe.command_id, &fn); 7965c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcox fn(nvmeq->dev, ctx, &cqe); 797b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 798b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 799b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox /* If the controller ignores the cq head doorbell and continuously 800b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * writes to the queue, it is theoretically possible to wrap around 801b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * the queue twice and mistakenly return IRQ_NONE. Linux only 802b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * requires that 0.1% of your interrupts are handled, so this isn't 803b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * a big problem. 804b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 805821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox if (head == nvmeq->cq_head && phase == nvmeq->cq_phase) 806e9539f47525ecee05c9f22c3565885f3e9492c52Matthew Wilcox return 0; 807b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 808b80d5ccca3a012e91ca64a2a0b13049163a6a698Haiyan Hu writel(head, nvmeq->q_db + nvmeq->dev->db_stride); 809b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->cq_head = head; 810821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox nvmeq->cq_phase = phase; 811b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 812e9539f47525ecee05c9f22c3565885f3e9492c52Matthew Wilcox nvmeq->cqe_seen = 1; 813e9539f47525ecee05c9f22c3565885f3e9492c52Matthew Wilcox return 1; 814b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 815b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 8167d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcoxstatic void nvme_make_request(struct request_queue *q, struct bio *bio) 8177d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox{ 8187d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox struct nvme_ns *ns = q->queuedata; 8197d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox struct nvme_queue *nvmeq = get_nvmeq(ns->dev); 8207d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox int result = -EBUSY; 8217d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox 822cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch if (!nvmeq) { 823cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch put_nvmeq(NULL); 824cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch bio_endio(bio, -EIO); 825cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch return; 826cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch } 827cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch 8287d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox spin_lock_irq(&nvmeq->q_lock); 829224042742582c9938788b81165180c876e997a07Keith Busch if (!nvmeq->q_suspended && bio_list_empty(&nvmeq->sq_cong)) 8307d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox result = nvme_submit_bio_queue(nvmeq, ns, bio); 8317d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox if (unlikely(result)) { 8327d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox if (bio_list_empty(&nvmeq->sq_cong)) 8337d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox add_wait_queue(&nvmeq->sq_full, &nvmeq->sq_cong_wait); 8347d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox bio_list_add(&nvmeq->sq_cong, bio); 8357d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox } 8367d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox 8377d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox nvme_process_cq(nvmeq); 8387d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox spin_unlock_irq(&nvmeq->q_lock); 8397d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox put_nvmeq(nvmeq); 8407d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox} 8417d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox 842b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic irqreturn_t nvme_irq(int irq, void *data) 843b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 84458ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox irqreturn_t result; 84558ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox struct nvme_queue *nvmeq = data; 84658ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox spin_lock(&nvmeq->q_lock); 847e9539f47525ecee05c9f22c3565885f3e9492c52Matthew Wilcox nvme_process_cq(nvmeq); 848e9539f47525ecee05c9f22c3565885f3e9492c52Matthew Wilcox result = nvmeq->cqe_seen ? IRQ_HANDLED : IRQ_NONE; 849e9539f47525ecee05c9f22c3565885f3e9492c52Matthew Wilcox nvmeq->cqe_seen = 0; 85058ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox spin_unlock(&nvmeq->q_lock); 85158ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox return result; 85258ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox} 85358ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox 85458ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcoxstatic irqreturn_t nvme_irq_check(int irq, void *data) 85558ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox{ 85658ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox struct nvme_queue *nvmeq = data; 85758ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox struct nvme_completion cqe = nvmeq->cqes[nvmeq->cq_head]; 85858ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox if ((le16_to_cpu(cqe.status) & 1) != nvmeq->cq_phase) 85958ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox return IRQ_NONE; 86058ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox return IRQ_WAKE_THREAD; 86158ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox} 86258ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox 8633c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcoxstatic void nvme_abort_command(struct nvme_queue *nvmeq, int cmdid) 8643c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox{ 8653c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox spin_lock_irq(&nvmeq->q_lock); 866c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox cancel_cmdid(nvmeq, cmdid, NULL); 8673c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox spin_unlock_irq(&nvmeq->q_lock); 8683c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox} 8693c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox 870c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcoxstruct sync_cmd_info { 871c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox struct task_struct *task; 872c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox u32 result; 873c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox int status; 874c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox}; 875c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox 8765c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcoxstatic void sync_completion(struct nvme_dev *dev, void *ctx, 877c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox struct nvme_completion *cqe) 878c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox{ 879c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox struct sync_cmd_info *cmdinfo = ctx; 880c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox cmdinfo->result = le32_to_cpup(&cqe->result); 881c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox cmdinfo->status = le16_to_cpup(&cqe->status) >> 1; 882c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox wake_up_process(cmdinfo->task); 883c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox} 884c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox 885b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 886b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Returns 0 on success. If the result is negative, it's a Linux error code; 887b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * if the result is positive, it's an NVM Express status code 888b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 8895d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermaint nvme_submit_sync_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd, 8905d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma u32 *result, unsigned timeout) 891b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 892b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int cmdid; 893b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct sync_cmd_info cmdinfo; 894b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 895b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cmdinfo.task = current; 896b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cmdinfo.status = -EINTR; 897b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 898c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox cmdid = alloc_cmdid_killable(nvmeq, &cmdinfo, sync_completion, 899e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox timeout); 900b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (cmdid < 0) 901b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return cmdid; 902b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cmd->common.command_id = cmdid; 903b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 9043c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox set_current_state(TASK_KILLABLE); 9053c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox nvme_submit_cmd(nvmeq, cmd); 90678f8d2577bd79ce9d62f4d9e6d3b895bd1dd1d1dKeith Busch schedule_timeout(timeout); 907b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 9083c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox if (cmdinfo.status == -EINTR) { 9093c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox nvme_abort_command(nvmeq, cmdid); 9103c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox return -EINTR; 9113c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox } 9123c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox 913b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (result) 914b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox *result = cmdinfo.result; 915b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 916b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return cmdinfo.status; 917b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 918b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 9194d115420707afcabe77d2535e092356df6664b70Keith Buschstatic int nvme_submit_async_cmd(struct nvme_queue *nvmeq, 9204d115420707afcabe77d2535e092356df6664b70Keith Busch struct nvme_command *cmd, 9214d115420707afcabe77d2535e092356df6664b70Keith Busch struct async_cmd_info *cmdinfo, unsigned timeout) 9224d115420707afcabe77d2535e092356df6664b70Keith Busch{ 9234d115420707afcabe77d2535e092356df6664b70Keith Busch int cmdid; 9244d115420707afcabe77d2535e092356df6664b70Keith Busch 9254d115420707afcabe77d2535e092356df6664b70Keith Busch cmdid = alloc_cmdid_killable(nvmeq, cmdinfo, async_completion, timeout); 9264d115420707afcabe77d2535e092356df6664b70Keith Busch if (cmdid < 0) 9274d115420707afcabe77d2535e092356df6664b70Keith Busch return cmdid; 9284d115420707afcabe77d2535e092356df6664b70Keith Busch cmdinfo->status = -EINTR; 9294d115420707afcabe77d2535e092356df6664b70Keith Busch cmd->common.command_id = cmdid; 9304d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_submit_cmd(nvmeq, cmd); 9314d115420707afcabe77d2535e092356df6664b70Keith Busch return 0; 9324d115420707afcabe77d2535e092356df6664b70Keith Busch} 9334d115420707afcabe77d2535e092356df6664b70Keith Busch 9345d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermaint nvme_submit_admin_cmd(struct nvme_dev *dev, struct nvme_command *cmd, 935b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 *result) 936b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 937e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox return nvme_submit_sync_cmd(dev->queues[0], cmd, result, ADMIN_TIMEOUT); 938b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 939b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 9404d115420707afcabe77d2535e092356df6664b70Keith Buschstatic int nvme_submit_admin_cmd_async(struct nvme_dev *dev, 9414d115420707afcabe77d2535e092356df6664b70Keith Busch struct nvme_command *cmd, struct async_cmd_info *cmdinfo) 9424d115420707afcabe77d2535e092356df6664b70Keith Busch{ 9434d115420707afcabe77d2535e092356df6664b70Keith Busch return nvme_submit_async_cmd(dev->queues[0], cmd, cmdinfo, 9444d115420707afcabe77d2535e092356df6664b70Keith Busch ADMIN_TIMEOUT); 9454d115420707afcabe77d2535e092356df6664b70Keith Busch} 9464d115420707afcabe77d2535e092356df6664b70Keith Busch 947b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id) 948b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 949b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int status; 950b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_command c; 951b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 952b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset(&c, 0, sizeof(c)); 953b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.delete_queue.opcode = opcode; 954b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.delete_queue.qid = cpu_to_le16(id); 955b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 956b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox status = nvme_submit_admin_cmd(dev, &c, NULL); 957b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (status) 958b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -EIO; 959b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 960b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 961b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 962b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_alloc_cq(struct nvme_dev *dev, u16 qid, 963b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq) 964b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 965b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int status; 966b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_command c; 967b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int flags = NVME_QUEUE_PHYS_CONTIG | NVME_CQ_IRQ_ENABLED; 968b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 969b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset(&c, 0, sizeof(c)); 970b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.opcode = nvme_admin_create_cq; 971b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.prp1 = cpu_to_le64(nvmeq->cq_dma_addr); 972b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.cqid = cpu_to_le16(qid); 973b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1); 974b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.cq_flags = cpu_to_le16(flags); 975b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.irq_vector = cpu_to_le16(nvmeq->cq_vector); 976b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 977b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox status = nvme_submit_admin_cmd(dev, &c, NULL); 978b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (status) 979b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -EIO; 980b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 981b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 982b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 983b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_alloc_sq(struct nvme_dev *dev, u16 qid, 984b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq) 985b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 986b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int status; 987b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_command c; 988b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int flags = NVME_QUEUE_PHYS_CONTIG | NVME_SQ_PRIO_MEDIUM; 989b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 990b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset(&c, 0, sizeof(c)); 991b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.opcode = nvme_admin_create_sq; 992b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.prp1 = cpu_to_le64(nvmeq->sq_dma_addr); 993b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.sqid = cpu_to_le16(qid); 994b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1); 995b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.sq_flags = cpu_to_le16(flags); 996b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.cqid = cpu_to_le16(qid); 997b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 998b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox status = nvme_submit_admin_cmd(dev, &c, NULL); 999b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (status) 1000b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -EIO; 1001b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 1002b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1003b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1004b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_delete_cq(struct nvme_dev *dev, u16 cqid) 1005b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1006b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return adapter_delete_queue(dev, nvme_admin_delete_cq, cqid); 1007b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1008b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1009b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_delete_sq(struct nvme_dev *dev, u16 sqid) 1010b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1011b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return adapter_delete_queue(dev, nvme_admin_delete_sq, sqid); 1012b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1013b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 10145d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermaint nvme_identify(struct nvme_dev *dev, unsigned nsid, unsigned cns, 1015bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox dma_addr_t dma_addr) 1016bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox{ 1017bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox struct nvme_command c; 1018bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox 1019bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox memset(&c, 0, sizeof(c)); 1020bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox c.identify.opcode = nvme_admin_identify; 1021bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox c.identify.nsid = cpu_to_le32(nsid); 1022bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox c.identify.prp1 = cpu_to_le64(dma_addr); 1023bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox c.identify.cns = cpu_to_le32(cns); 1024bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox 1025bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox return nvme_submit_admin_cmd(dev, &c, NULL); 1026bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox} 1027bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox 10285d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermaint nvme_get_features(struct nvme_dev *dev, unsigned fid, unsigned nsid, 102908df1e05657fc6712e520e7c09cc6c86160ceb35Keith Busch dma_addr_t dma_addr, u32 *result) 1030bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox{ 1031bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox struct nvme_command c; 1032bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox 1033bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox memset(&c, 0, sizeof(c)); 1034bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox c.features.opcode = nvme_admin_get_features; 1035a42ceccef0c43b46ff6bc1b12a7c1076ef243df1Keith Busch c.features.nsid = cpu_to_le32(nsid); 1036bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox c.features.prp1 = cpu_to_le64(dma_addr); 1037bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox c.features.fid = cpu_to_le32(fid); 1038bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox 103908df1e05657fc6712e520e7c09cc6c86160ceb35Keith Busch return nvme_submit_admin_cmd(dev, &c, result); 1040df3481399042200792822b6243e36a95a557b57eMatthew Wilcox} 1041df3481399042200792822b6243e36a95a557b57eMatthew Wilcox 10425d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermaint nvme_set_features(struct nvme_dev *dev, unsigned fid, unsigned dword11, 10435d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma dma_addr_t dma_addr, u32 *result) 1044df3481399042200792822b6243e36a95a557b57eMatthew Wilcox{ 1045df3481399042200792822b6243e36a95a557b57eMatthew Wilcox struct nvme_command c; 1046df3481399042200792822b6243e36a95a557b57eMatthew Wilcox 1047df3481399042200792822b6243e36a95a557b57eMatthew Wilcox memset(&c, 0, sizeof(c)); 1048df3481399042200792822b6243e36a95a557b57eMatthew Wilcox c.features.opcode = nvme_admin_set_features; 1049df3481399042200792822b6243e36a95a557b57eMatthew Wilcox c.features.prp1 = cpu_to_le64(dma_addr); 1050df3481399042200792822b6243e36a95a557b57eMatthew Wilcox c.features.fid = cpu_to_le32(fid); 1051df3481399042200792822b6243e36a95a557b57eMatthew Wilcox c.features.dword11 = cpu_to_le32(dword11); 1052df3481399042200792822b6243e36a95a557b57eMatthew Wilcox 1053bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox return nvme_submit_admin_cmd(dev, &c, result); 1054bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox} 1055bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox 1056a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox/** 1057c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch * nvme_abort_cmd - Attempt aborting a command 1058c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch * @cmdid: Command id of a timed out IO 1059c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch * @queue: The queue with timed out IO 1060c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch * 1061c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch * Schedule controller reset if the command was already aborted once before and 1062c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch * still hasn't been returned to the driver, or if this is the admin queue. 1063c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch */ 1064c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Buschstatic void nvme_abort_cmd(int cmdid, struct nvme_queue *nvmeq) 1065c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch{ 1066c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch int a_cmdid; 1067c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch struct nvme_command cmd; 1068c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch struct nvme_dev *dev = nvmeq->dev; 1069c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch struct nvme_cmd_info *info = nvme_cmd_info(nvmeq); 1070c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch 1071c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch if (!nvmeq->qid || info[cmdid].aborted) { 1072c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch if (work_busy(&dev->reset_work)) 1073c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch return; 1074c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch list_del_init(&dev->node); 1075c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch dev_warn(&dev->pci_dev->dev, 1076c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch "I/O %d QID %d timeout, reset controller\n", cmdid, 1077c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch nvmeq->qid); 1078c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch INIT_WORK(&dev->reset_work, nvme_reset_failed_dev); 1079c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch queue_work(nvme_workq, &dev->reset_work); 1080c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch return; 1081c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch } 1082c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch 1083c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch if (!dev->abort_limit) 1084c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch return; 1085c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch 1086c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch a_cmdid = alloc_cmdid(dev->queues[0], CMD_CTX_ABORT, special_completion, 1087c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch ADMIN_TIMEOUT); 1088c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch if (a_cmdid < 0) 1089c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch return; 1090c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch 1091c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch memset(&cmd, 0, sizeof(cmd)); 1092c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch cmd.abort.opcode = nvme_admin_abort_cmd; 1093c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch cmd.abort.cid = cmdid; 1094c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch cmd.abort.sqid = cpu_to_le16(nvmeq->qid); 1095c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch cmd.abort.command_id = a_cmdid; 1096c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch 1097c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch --dev->abort_limit; 1098c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch info[cmdid].aborted = 1; 1099c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch info[cmdid].timeout = jiffies + ADMIN_TIMEOUT; 1100c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch 1101c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch dev_warn(nvmeq->q_dmadev, "Aborting I/O %d QID %d\n", cmdid, 1102c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch nvmeq->qid); 1103c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch nvme_submit_cmd(dev->queues[0], &cmd); 1104c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch} 1105c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch 1106c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch/** 1107a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox * nvme_cancel_ios - Cancel outstanding I/Os 1108a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox * @queue: The queue to cancel I/Os on 1109a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox * @timeout: True to only cancel I/Os which have timed out 1110a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox */ 1111a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcoxstatic void nvme_cancel_ios(struct nvme_queue *nvmeq, bool timeout) 1112a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox{ 1113a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox int depth = nvmeq->q_depth - 1; 1114a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox struct nvme_cmd_info *info = nvme_cmd_info(nvmeq); 1115a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox unsigned long now = jiffies; 1116a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox int cmdid; 1117a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox 1118a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox for_each_set_bit(cmdid, nvmeq->cmdid_data, depth) { 1119a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox void *ctx; 1120a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox nvme_completion_fn fn; 1121a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox static struct nvme_completion cqe = { 1122af2d9ca744af5e03390eeb3864d08ce75c860899Matthew Wilcox .status = cpu_to_le16(NVME_SC_ABORT_REQ << 1), 1123a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox }; 1124a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox 1125a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox if (timeout && !time_after(now, info[cmdid].timeout)) 1126a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox continue; 1127053ab702cc2702f25a97ead087ed344b864785b7Keith Busch if (info[cmdid].ctx == CMD_CTX_CANCELLED) 1128053ab702cc2702f25a97ead087ed344b864785b7Keith Busch continue; 1129c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch if (timeout && nvmeq->dev->initialized) { 1130c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch nvme_abort_cmd(cmdid, nvmeq); 1131c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch continue; 1132c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch } 1133c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch dev_warn(nvmeq->q_dmadev, "Cancelling I/O %d QID %d\n", cmdid, 1134c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch nvmeq->qid); 1135a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox ctx = cancel_cmdid(nvmeq, cmdid, &fn); 1136a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox fn(nvmeq->dev, ctx, &cqe); 1137a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox } 1138a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox} 1139a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox 1140224042742582c9938788b81165180c876e997a07Keith Buschstatic void nvme_free_queue(struct nvme_queue *nvmeq) 11419e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox{ 1142224042742582c9938788b81165180c876e997a07Keith Busch spin_lock_irq(&nvmeq->q_lock); 1143224042742582c9938788b81165180c876e997a07Keith Busch while (bio_list_peek(&nvmeq->sq_cong)) { 1144224042742582c9938788b81165180c876e997a07Keith Busch struct bio *bio = bio_list_pop(&nvmeq->sq_cong); 1145224042742582c9938788b81165180c876e997a07Keith Busch bio_endio(bio, -EIO); 1146224042742582c9938788b81165180c876e997a07Keith Busch } 1147224042742582c9938788b81165180c876e997a07Keith Busch spin_unlock_irq(&nvmeq->q_lock); 1148224042742582c9938788b81165180c876e997a07Keith Busch 11499e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth), 11509e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox (void *)nvmeq->cqes, nvmeq->cq_dma_addr); 11519e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth), 11529e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox nvmeq->sq_cmds, nvmeq->sq_dma_addr); 11539e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox kfree(nvmeq); 11549e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox} 11559e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox 1156a1a5ef9998b1379780790b878457b0e0f48b25dbKeith Buschstatic void nvme_free_queues(struct nvme_dev *dev, int lowest) 1157224042742582c9938788b81165180c876e997a07Keith Busch{ 1158224042742582c9938788b81165180c876e997a07Keith Busch int i; 1159224042742582c9938788b81165180c876e997a07Keith Busch 1160a1a5ef9998b1379780790b878457b0e0f48b25dbKeith Busch for (i = dev->queue_count - 1; i >= lowest; i--) { 1161224042742582c9938788b81165180c876e997a07Keith Busch nvme_free_queue(dev->queues[i]); 1162224042742582c9938788b81165180c876e997a07Keith Busch dev->queue_count--; 1163224042742582c9938788b81165180c876e997a07Keith Busch dev->queues[i] = NULL; 1164224042742582c9938788b81165180c876e997a07Keith Busch } 1165224042742582c9938788b81165180c876e997a07Keith Busch} 1166224042742582c9938788b81165180c876e997a07Keith Busch 11674d115420707afcabe77d2535e092356df6664b70Keith Busch/** 11684d115420707afcabe77d2535e092356df6664b70Keith Busch * nvme_suspend_queue - put queue into suspended state 11694d115420707afcabe77d2535e092356df6664b70Keith Busch * @nvmeq - queue to suspend 11704d115420707afcabe77d2535e092356df6664b70Keith Busch * 11714d115420707afcabe77d2535e092356df6664b70Keith Busch * Returns 1 if already suspended, 0 otherwise. 11724d115420707afcabe77d2535e092356df6664b70Keith Busch */ 11734d115420707afcabe77d2535e092356df6664b70Keith Buschstatic int nvme_suspend_queue(struct nvme_queue *nvmeq) 1174b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 11754d115420707afcabe77d2535e092356df6664b70Keith Busch int vector = nvmeq->dev->entry[nvmeq->cq_vector].vector; 1176b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1177a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox spin_lock_irq(&nvmeq->q_lock); 1178224042742582c9938788b81165180c876e997a07Keith Busch if (nvmeq->q_suspended) { 1179224042742582c9938788b81165180c876e997a07Keith Busch spin_unlock_irq(&nvmeq->q_lock); 11804d115420707afcabe77d2535e092356df6664b70Keith Busch return 1; 11813295874b6074d749516d6decd43afad7bf6e38ffKeith Busch } 1182224042742582c9938788b81165180c876e997a07Keith Busch nvmeq->q_suspended = 1; 1183a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox spin_unlock_irq(&nvmeq->q_lock); 1184a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox 1185aba2080f3f1639f9202f1a52993669844abcfb80Matthew Wilcox irq_set_affinity_hint(vector, NULL); 1186aba2080f3f1639f9202f1a52993669844abcfb80Matthew Wilcox free_irq(vector, nvmeq); 1187b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 11884d115420707afcabe77d2535e092356df6664b70Keith Busch return 0; 11894d115420707afcabe77d2535e092356df6664b70Keith Busch} 11904d115420707afcabe77d2535e092356df6664b70Keith Busch 11914d115420707afcabe77d2535e092356df6664b70Keith Buschstatic void nvme_clear_queue(struct nvme_queue *nvmeq) 11924d115420707afcabe77d2535e092356df6664b70Keith Busch{ 11934d115420707afcabe77d2535e092356df6664b70Keith Busch spin_lock_irq(&nvmeq->q_lock); 11944d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_process_cq(nvmeq); 11954d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_cancel_ios(nvmeq, false); 11964d115420707afcabe77d2535e092356df6664b70Keith Busch spin_unlock_irq(&nvmeq->q_lock); 11974d115420707afcabe77d2535e092356df6664b70Keith Busch} 11984d115420707afcabe77d2535e092356df6664b70Keith Busch 11994d115420707afcabe77d2535e092356df6664b70Keith Buschstatic void nvme_disable_queue(struct nvme_dev *dev, int qid) 12004d115420707afcabe77d2535e092356df6664b70Keith Busch{ 12014d115420707afcabe77d2535e092356df6664b70Keith Busch struct nvme_queue *nvmeq = dev->queues[qid]; 12024d115420707afcabe77d2535e092356df6664b70Keith Busch 12034d115420707afcabe77d2535e092356df6664b70Keith Busch if (!nvmeq) 12044d115420707afcabe77d2535e092356df6664b70Keith Busch return; 12054d115420707afcabe77d2535e092356df6664b70Keith Busch if (nvme_suspend_queue(nvmeq)) 12064d115420707afcabe77d2535e092356df6664b70Keith Busch return; 12074d115420707afcabe77d2535e092356df6664b70Keith Busch 12080e53d18051725da46cbccfb7874a6422d4d4f274Keith Busch /* Don't tell the adapter to delete the admin queue. 12090e53d18051725da46cbccfb7874a6422d4d4f274Keith Busch * Don't tell a removed adapter to delete IO queues. */ 12100e53d18051725da46cbccfb7874a6422d4d4f274Keith Busch if (qid && readl(&dev->bar->csts) != -1) { 1211b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox adapter_delete_sq(dev, qid); 1212b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox adapter_delete_cq(dev, qid); 1213b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 12144d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_clear_queue(nvmeq); 1215b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1216b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1217b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev, int qid, 1218b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int depth, int vector) 1219b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1220b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct device *dmadev = &dev->pci_dev->dev; 1221224042742582c9938788b81165180c876e997a07Keith Busch unsigned extra = nvme_queue_extra(depth); 1222b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq = kzalloc(sizeof(*nvmeq) + extra, GFP_KERNEL); 1223b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!nvmeq) 1224b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 1225b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1226b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->cqes = dma_alloc_coherent(dmadev, CQ_SIZE(depth), 1227b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox &nvmeq->cq_dma_addr, GFP_KERNEL); 1228b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!nvmeq->cqes) 1229b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto free_nvmeq; 1230b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset((void *)nvmeq->cqes, 0, CQ_SIZE(depth)); 1231b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1232b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->sq_cmds = dma_alloc_coherent(dmadev, SQ_SIZE(depth), 1233b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox &nvmeq->sq_dma_addr, GFP_KERNEL); 1234b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!nvmeq->sq_cmds) 1235b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto free_cqdma; 1236b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1237b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->q_dmadev = dmadev; 1238091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox nvmeq->dev = dev; 12393193f07bb7ae723f33ac8e8b9db317a4f68d7d18Matthew Wilcox snprintf(nvmeq->irqname, sizeof(nvmeq->irqname), "nvme%dq%d", 12403193f07bb7ae723f33ac8e8b9db317a4f68d7d18Matthew Wilcox dev->instance, qid); 1241b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox spin_lock_init(&nvmeq->q_lock); 1242b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->cq_head = 0; 1243821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox nvmeq->cq_phase = 1; 1244b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox init_waitqueue_head(&nvmeq->sq_full); 12451fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox init_waitqueue_entry(&nvmeq->sq_cong_wait, nvme_thread); 1246b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox bio_list_init(&nvmeq->sq_cong); 1247b80d5ccca3a012e91ca64a2a0b13049163a6a698Haiyan Hu nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride]; 1248b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->q_depth = depth; 1249b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->cq_vector = vector; 1250c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch nvmeq->qid = qid; 1251224042742582c9938788b81165180c876e997a07Keith Busch nvmeq->q_suspended = 1; 1252224042742582c9938788b81165180c876e997a07Keith Busch dev->queue_count++; 1253b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1254b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return nvmeq; 1255b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1256b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox free_cqdma: 125768b8eca5f882230e4991df0db0a3094f49420a90Keith Busch dma_free_coherent(dmadev, CQ_SIZE(depth), (void *)nvmeq->cqes, 1258b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->cq_dma_addr); 1259b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox free_nvmeq: 1260b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(nvmeq); 1261b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 1262b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1263b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 12643001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcoxstatic int queue_request_irq(struct nvme_dev *dev, struct nvme_queue *nvmeq, 12653001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox const char *name) 12663001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox{ 126758ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox if (use_threaded_interrupts) 126858ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox return request_threaded_irq(dev->entry[nvmeq->cq_vector].vector, 1269481e5bad84239b01415b888df2040c1860ae3cfdMichael Opdenacker nvme_irq_check, nvme_irq, IRQF_SHARED, 127058ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox name, nvmeq); 12713001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox return request_irq(dev->entry[nvmeq->cq_vector].vector, nvme_irq, 1272481e5bad84239b01415b888df2040c1860ae3cfdMichael Opdenacker IRQF_SHARED, name, nvmeq); 12733001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox} 12743001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox 1275224042742582c9938788b81165180c876e997a07Keith Buschstatic void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid) 1276b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1277224042742582c9938788b81165180c876e997a07Keith Busch struct nvme_dev *dev = nvmeq->dev; 1278224042742582c9938788b81165180c876e997a07Keith Busch unsigned extra = nvme_queue_extra(nvmeq->q_depth); 1279b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1280224042742582c9938788b81165180c876e997a07Keith Busch nvmeq->sq_tail = 0; 1281224042742582c9938788b81165180c876e997a07Keith Busch nvmeq->cq_head = 0; 1282224042742582c9938788b81165180c876e997a07Keith Busch nvmeq->cq_phase = 1; 1283b80d5ccca3a012e91ca64a2a0b13049163a6a698Haiyan Hu nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride]; 1284224042742582c9938788b81165180c876e997a07Keith Busch memset(nvmeq->cmdid_data, 0, extra); 1285224042742582c9938788b81165180c876e997a07Keith Busch memset((void *)nvmeq->cqes, 0, CQ_SIZE(nvmeq->q_depth)); 1286224042742582c9938788b81165180c876e997a07Keith Busch nvme_cancel_ios(nvmeq, false); 1287224042742582c9938788b81165180c876e997a07Keith Busch nvmeq->q_suspended = 0; 1288224042742582c9938788b81165180c876e997a07Keith Busch} 1289224042742582c9938788b81165180c876e997a07Keith Busch 1290224042742582c9938788b81165180c876e997a07Keith Buschstatic int nvme_create_queue(struct nvme_queue *nvmeq, int qid) 1291224042742582c9938788b81165180c876e997a07Keith Busch{ 1292224042742582c9938788b81165180c876e997a07Keith Busch struct nvme_dev *dev = nvmeq->dev; 1293224042742582c9938788b81165180c876e997a07Keith Busch int result; 12943f85d50b609e8a5ef151656210203a6e94c19538Matthew Wilcox 1295b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox result = adapter_alloc_cq(dev, qid, nvmeq); 1296b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (result < 0) 1297224042742582c9938788b81165180c876e997a07Keith Busch return result; 1298b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1299b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox result = adapter_alloc_sq(dev, qid, nvmeq); 1300b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (result < 0) 1301b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto release_cq; 1302b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 13033193f07bb7ae723f33ac8e8b9db317a4f68d7d18Matthew Wilcox result = queue_request_irq(dev, nvmeq, nvmeq->irqname); 1304b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (result < 0) 1305b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto release_sq; 1306b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 13070a8d44cb33377969337fa6c1961b631605d5f453Matthew Wilcox spin_lock_irq(&nvmeq->q_lock); 1308224042742582c9938788b81165180c876e997a07Keith Busch nvme_init_queue(nvmeq, qid); 13090a8d44cb33377969337fa6c1961b631605d5f453Matthew Wilcox spin_unlock_irq(&nvmeq->q_lock); 1310224042742582c9938788b81165180c876e997a07Keith Busch 1311224042742582c9938788b81165180c876e997a07Keith Busch return result; 1312b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1313b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox release_sq: 1314b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox adapter_delete_sq(dev, qid); 1315b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox release_cq: 1316b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox adapter_delete_cq(dev, qid); 1317224042742582c9938788b81165180c876e997a07Keith Busch return result; 1318b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1319b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1320ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcoxstatic int nvme_wait_ready(struct nvme_dev *dev, u64 cap, bool enabled) 1321ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox{ 1322ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox unsigned long timeout; 1323ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox u32 bit = enabled ? NVME_CSTS_RDY : 0; 1324ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox 1325ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox timeout = ((NVME_CAP_TIMEOUT(cap) + 1) * HZ / 2) + jiffies; 1326ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox 1327ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox while ((readl(&dev->bar->csts) & NVME_CSTS_RDY) != bit) { 1328ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox msleep(100); 1329ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox if (fatal_signal_pending(current)) 1330ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox return -EINTR; 1331ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox if (time_after(jiffies, timeout)) { 1332ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox dev_err(&dev->pci_dev->dev, 1333ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox "Device not ready; aborting initialisation\n"); 1334ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox return -ENODEV; 1335ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox } 1336ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox } 1337ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox 1338ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox return 0; 1339ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox} 1340ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox 1341ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox/* 1342ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox * If the device has been passed off to us in an enabled state, just clear 1343ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox * the enabled bit. The spec says we should set the 'shutdown notification 1344ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox * bits', but doing so may cause the device to complete commands to the 1345ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox * admin queue ... and we don't know what memory that might be pointing at! 1346ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox */ 1347ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcoxstatic int nvme_disable_ctrl(struct nvme_dev *dev, u64 cap) 1348ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox{ 134944af146a84fa4a8e136d824207dcd356958a112bMatthew Wilcox u32 cc = readl(&dev->bar->cc); 135044af146a84fa4a8e136d824207dcd356958a112bMatthew Wilcox 135144af146a84fa4a8e136d824207dcd356958a112bMatthew Wilcox if (cc & NVME_CC_ENABLE) 135244af146a84fa4a8e136d824207dcd356958a112bMatthew Wilcox writel(cc & ~NVME_CC_ENABLE, &dev->bar->cc); 1353ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox return nvme_wait_ready(dev, cap, false); 1354ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox} 1355ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox 1356ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcoxstatic int nvme_enable_ctrl(struct nvme_dev *dev, u64 cap) 1357ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox{ 1358ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox return nvme_wait_ready(dev, cap, true); 1359ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox} 1360ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox 13611894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Buschstatic int nvme_shutdown_ctrl(struct nvme_dev *dev) 13621894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch{ 13631894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch unsigned long timeout; 13641894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch u32 cc; 13651894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch 13661894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch cc = (readl(&dev->bar->cc) & ~NVME_CC_SHN_MASK) | NVME_CC_SHN_NORMAL; 13671894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch writel(cc, &dev->bar->cc); 13681894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch 13691894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch timeout = 2 * HZ + jiffies; 13701894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch while ((readl(&dev->bar->csts) & NVME_CSTS_SHST_MASK) != 13711894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch NVME_CSTS_SHST_CMPLT) { 13721894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch msleep(100); 13731894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch if (fatal_signal_pending(current)) 13741894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch return -EINTR; 13751894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch if (time_after(jiffies, timeout)) { 13761894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch dev_err(&dev->pci_dev->dev, 13771894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch "Device shutdown incomplete; abort shutdown\n"); 13781894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch return -ENODEV; 13791894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch } 13801894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch } 13811894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch 13821894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch return 0; 13831894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch} 13841894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch 13858d85fce77edfc22f1d6dbf78e3af723b4b556f3dGreg Kroah-Hartmanstatic int nvme_configure_admin_queue(struct nvme_dev *dev) 1386b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1387ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox int result; 1388b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 aqa; 1389ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox u64 cap = readq(&dev->bar->cap); 1390b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq; 1391b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1392ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox result = nvme_disable_ctrl(dev, cap); 1393ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox if (result < 0) 1394ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox return result; 1395b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1396cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch nvmeq = dev->queues[0]; 1397cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch if (!nvmeq) { 1398cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch nvmeq = nvme_alloc_queue(dev, 0, 64, 0); 1399cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch if (!nvmeq) 1400cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch return -ENOMEM; 1401cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch dev->queues[0] = nvmeq; 1402cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch } 1403b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1404b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox aqa = nvmeq->q_depth - 1; 1405b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox aqa |= aqa << 16; 1406b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1407b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->ctrl_config = NVME_CC_ENABLE | NVME_CC_CSS_NVM; 1408b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->ctrl_config |= (PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT; 1409b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->ctrl_config |= NVME_CC_ARB_RR | NVME_CC_SHN_NONE; 14107f53f9d2424533256ae86f7df5661a17de743de8Matthew Wilcox dev->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES; 1411b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1412b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox writel(aqa, &dev->bar->aqa); 1413b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox writeq(nvmeq->sq_dma_addr, &dev->bar->asq); 1414b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox writeq(nvmeq->cq_dma_addr, &dev->bar->acq); 1415b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox writel(dev->ctrl_config, &dev->bar->cc); 1416b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1417ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox result = nvme_enable_ctrl(dev, cap); 1418025c557a71bd06a9f6d32259f00e2218b15bf0a4Keith Busch if (result) 1419cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch return result; 14209e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox 14213193f07bb7ae723f33ac8e8b9db317a4f68d7d18Matthew Wilcox result = queue_request_irq(dev, nvmeq, nvmeq->irqname); 1422025c557a71bd06a9f6d32259f00e2218b15bf0a4Keith Busch if (result) 1423cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch return result; 1424025c557a71bd06a9f6d32259f00e2218b15bf0a4Keith Busch 14250a8d44cb33377969337fa6c1961b631605d5f453Matthew Wilcox spin_lock_irq(&nvmeq->q_lock); 1426224042742582c9938788b81165180c876e997a07Keith Busch nvme_init_queue(nvmeq, 0); 14270a8d44cb33377969337fa6c1961b631605d5f453Matthew Wilcox spin_unlock_irq(&nvmeq->q_lock); 1428b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return result; 1429b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1430b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 14315d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermastruct nvme_iod *nvme_map_user_pages(struct nvme_dev *dev, int write, 1432eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox unsigned long addr, unsigned length) 1433b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 143436c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox int i, err, count, nents, offset; 14357fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox struct scatterlist *sg; 14367fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox struct page **pages; 1437eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox struct nvme_iod *iod; 143836c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox 143936c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox if (addr & 3) 1440eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return ERR_PTR(-EINVAL); 14415460fc03105fbed01fe27aa572d9f65bb410a61dDan Carpenter if (!length || length > INT_MAX - PAGE_SIZE) 1442eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return ERR_PTR(-EINVAL); 14437fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox 144436c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox offset = offset_in_page(addr); 14457fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox count = DIV_ROUND_UP(offset + length, PAGE_SIZE); 14467fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox pages = kcalloc(count, sizeof(*pages), GFP_KERNEL); 144722fff826e715e9727d3c7a69f15e602a9801b673Dan Carpenter if (!pages) 144822fff826e715e9727d3c7a69f15e602a9801b673Dan Carpenter return ERR_PTR(-ENOMEM); 144936c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox 145036c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox err = get_user_pages_fast(addr, count, 1, pages); 145136c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox if (err < count) { 145236c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox count = err; 145336c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox err = -EFAULT; 145436c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox goto put_pages; 145536c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox } 14567fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox 1457eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod = nvme_alloc_iod(count, length, GFP_KERNEL); 1458eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox sg = iod->sg; 145936c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox sg_init_table(sg, count); 1460d0ba1e497bca83a3d353eb47c9658afc54d83228Matthew Wilcox for (i = 0; i < count; i++) { 1461d0ba1e497bca83a3d353eb47c9658afc54d83228Matthew Wilcox sg_set_page(&sg[i], pages[i], 14625460fc03105fbed01fe27aa572d9f65bb410a61dDan Carpenter min_t(unsigned, length, PAGE_SIZE - offset), 14635460fc03105fbed01fe27aa572d9f65bb410a61dDan Carpenter offset); 1464d0ba1e497bca83a3d353eb47c9658afc54d83228Matthew Wilcox length -= (PAGE_SIZE - offset); 1465d0ba1e497bca83a3d353eb47c9658afc54d83228Matthew Wilcox offset = 0; 14667fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox } 1467fe304c43c6d63e29ed4fc46a874d7a74313788c5Matthew Wilcox sg_mark_end(&sg[i - 1]); 14681c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox iod->nents = count; 14697fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox 14707fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox err = -ENOMEM; 14717fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox nents = dma_map_sg(&dev->pci_dev->dev, sg, count, 14727fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox write ? DMA_TO_DEVICE : DMA_FROM_DEVICE); 147336c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox if (!nents) 1474eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox goto free_iod; 1475b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 14767fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox kfree(pages); 1477eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return iod; 1478b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1479eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox free_iod: 1480eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox kfree(iod); 14817fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox put_pages: 14827fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox for (i = 0; i < count; i++) 14837fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox put_page(pages[i]); 14847fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox kfree(pages); 1485eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return ERR_PTR(err); 14867fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox} 1487b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 14885d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermavoid nvme_unmap_user_pages(struct nvme_dev *dev, int write, 14891c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox struct nvme_iod *iod) 14907fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox{ 14911c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox int i; 1492b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 14931c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox dma_unmap_sg(&dev->pci_dev->dev, iod->sg, iod->nents, 14941c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox write ? DMA_TO_DEVICE : DMA_FROM_DEVICE); 14957fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox 14961c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox for (i = 0; i < iod->nents; i++) 14971c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox put_page(sg_page(&iod->sg[i])); 14987fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox} 1499b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1500a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcoxstatic int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio) 1501a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox{ 1502a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox struct nvme_dev *dev = ns->dev; 1503a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox struct nvme_queue *nvmeq; 1504a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox struct nvme_user_io io; 1505a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox struct nvme_command c; 1506f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch unsigned length, meta_len; 1507f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch int status, i; 1508f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch struct nvme_iod *iod, *meta_iod = NULL; 1509f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch dma_addr_t meta_dma_addr; 1510f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch void *meta, *uninitialized_var(meta_mem); 1511a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox 1512a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox if (copy_from_user(&io, uio, sizeof(io))) 1513a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox return -EFAULT; 15146c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox length = (io.nblocks + 1) << ns->lba_shift; 1515f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_len = (io.nblocks + 1) * ns->ms; 1516f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1517f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch if (meta_len && ((io.metadata & 3) || !io.metadata)) 1518f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch return -EINVAL; 15196c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox 15206c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox switch (io.opcode) { 15216c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox case nvme_cmd_write: 15226c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox case nvme_cmd_read: 15236bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox case nvme_cmd_compare: 1524eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod = nvme_map_user_pages(dev, io.opcode & 1, io.addr, length); 15256413214c5d424fd5aae6567848340f962ad2ce0fMatthew Wilcox break; 15266c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox default: 15276bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox return -EINVAL; 15286c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox } 15296c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox 1530eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox if (IS_ERR(iod)) 1531eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return PTR_ERR(iod); 1532a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox 1533a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox memset(&c, 0, sizeof(c)); 1534a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.opcode = io.opcode; 1535a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.flags = io.flags; 15366c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox c.rw.nsid = cpu_to_le32(ns->ns_id); 1537a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.slba = cpu_to_le64(io.slba); 15386c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox c.rw.length = cpu_to_le16(io.nblocks); 1539a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.control = cpu_to_le16(io.control); 15401c9b52651dad0ff1fa71fc6205c86d972f25bcc0Matthew Wilcox c.rw.dsmgmt = cpu_to_le32(io.dsmgmt); 15411c9b52651dad0ff1fa71fc6205c86d972f25bcc0Matthew Wilcox c.rw.reftag = cpu_to_le32(io.reftag); 15421c9b52651dad0ff1fa71fc6205c86d972f25bcc0Matthew Wilcox c.rw.apptag = cpu_to_le16(io.apptag); 15431c9b52651dad0ff1fa71fc6205c86d972f25bcc0Matthew Wilcox c.rw.appmask = cpu_to_le16(io.appmask); 1544f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1545f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch if (meta_len) { 15461b56749e541ad59068582f2a28297843e243b856Keith Busch meta_iod = nvme_map_user_pages(dev, io.opcode & 1, io.metadata, 15471b56749e541ad59068582f2a28297843e243b856Keith Busch meta_len); 1548f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch if (IS_ERR(meta_iod)) { 1549f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch status = PTR_ERR(meta_iod); 1550f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_iod = NULL; 1551f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch goto unmap; 1552f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch } 1553f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1554f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_mem = dma_alloc_coherent(&dev->pci_dev->dev, meta_len, 1555f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch &meta_dma_addr, GFP_KERNEL); 1556f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch if (!meta_mem) { 1557f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch status = -ENOMEM; 1558f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch goto unmap; 1559f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch } 1560f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1561f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch if (io.opcode & 1) { 1562f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch int meta_offset = 0; 1563f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1564f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch for (i = 0; i < meta_iod->nents; i++) { 1565f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta = kmap_atomic(sg_page(&meta_iod->sg[i])) + 1566f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_iod->sg[i].offset; 1567f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch memcpy(meta_mem + meta_offset, meta, 1568f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_iod->sg[i].length); 1569f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch kunmap_atomic(meta); 1570f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_offset += meta_iod->sg[i].length; 1571f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch } 1572f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch } 1573f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1574f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch c.rw.metadata = cpu_to_le64(meta_dma_addr); 1575f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch } 1576f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1577eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox length = nvme_setup_prps(dev, &c.common, iod, length, GFP_KERNEL); 1578a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox 1579040a93b52a9eee8177ebaf2ba0ee0f9f518d1bf8Matthew Wilcox nvmeq = get_nvmeq(dev); 1580fa92282149842645931580225647238428374758Matthew Wilcox /* 1581fa92282149842645931580225647238428374758Matthew Wilcox * Since nvme_submit_sync_cmd sleeps, we can't keep preemption 1582b1ad37efcafe396ac3944853589688dd0ec3c64eMatthew Wilcox * disabled. We may be preempted at any point, and be rescheduled 1583b1ad37efcafe396ac3944853589688dd0ec3c64eMatthew Wilcox * to a different CPU. That will cause cacheline bouncing, but no 1584b1ad37efcafe396ac3944853589688dd0ec3c64eMatthew Wilcox * additional races since q_lock already protects against other CPUs. 1585b1ad37efcafe396ac3944853589688dd0ec3c64eMatthew Wilcox */ 1586a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox put_nvmeq(nvmeq); 1587b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox if (length != (io.nblocks + 1) << ns->lba_shift) 1588b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox status = -ENOMEM; 1589224042742582c9938788b81165180c876e997a07Keith Busch else if (!nvmeq || nvmeq->q_suspended) 1590224042742582c9938788b81165180c876e997a07Keith Busch status = -EBUSY; 1591b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox else 1592ff976d724a74e4522e9ca2de1fb37ac4520f454fMatthew Wilcox status = nvme_submit_sync_cmd(nvmeq, &c, NULL, NVME_IO_TIMEOUT); 1593a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox 1594f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch if (meta_len) { 1595f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch if (status == NVME_SC_SUCCESS && !(io.opcode & 1)) { 1596f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch int meta_offset = 0; 1597f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1598f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch for (i = 0; i < meta_iod->nents; i++) { 1599f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta = kmap_atomic(sg_page(&meta_iod->sg[i])) + 1600f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_iod->sg[i].offset; 1601f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch memcpy(meta, meta_mem + meta_offset, 1602f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_iod->sg[i].length); 1603f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch kunmap_atomic(meta); 1604f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_offset += meta_iod->sg[i].length; 1605f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch } 1606f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch } 1607f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1608f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch dma_free_coherent(&dev->pci_dev->dev, meta_len, meta_mem, 1609f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_dma_addr); 1610f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch } 1611f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1612f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch unmap: 16131c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox nvme_unmap_user_pages(dev, io.opcode & 1, iod); 1614eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox nvme_free_iod(dev, iod); 1615f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1616f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch if (meta_iod) { 1617f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch nvme_unmap_user_pages(dev, io.opcode & 1, meta_iod); 1618f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch nvme_free_iod(dev, meta_iod); 1619f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch } 1620f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1621a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox return status; 1622a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox} 1623a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox 162450af8baec46a99a9b81a4600c0374f83a5a590a9Keith Buschstatic int nvme_user_admin_cmd(struct nvme_dev *dev, 16256bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox struct nvme_admin_cmd __user *ucmd) 16266ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox{ 16276bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox struct nvme_admin_cmd cmd; 16286ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox struct nvme_command c; 1629eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox int status, length; 1630c7d36ab8fa04c213328119a9c0d66985fe204ee5Keith Busch struct nvme_iod *uninitialized_var(iod); 163194f370cab6e5ac514b658c6b2b3aa308cefc5c7aKeith Busch unsigned timeout; 16326ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox 16336bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox if (!capable(CAP_SYS_ADMIN)) 16346bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox return -EACCES; 16356bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox if (copy_from_user(&cmd, ucmd, sizeof(cmd))) 16366ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox return -EFAULT; 16376ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox 16386ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox memset(&c, 0, sizeof(c)); 16396bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.opcode = cmd.opcode; 16406bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.flags = cmd.flags; 16416bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.nsid = cpu_to_le32(cmd.nsid); 16426bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw2[0] = cpu_to_le32(cmd.cdw2); 16436bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw2[1] = cpu_to_le32(cmd.cdw3); 16446bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw10[0] = cpu_to_le32(cmd.cdw10); 16456bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw10[1] = cpu_to_le32(cmd.cdw11); 16466bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw10[2] = cpu_to_le32(cmd.cdw12); 16476bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw10[3] = cpu_to_le32(cmd.cdw13); 16486bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw10[4] = cpu_to_le32(cmd.cdw14); 16496bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw10[5] = cpu_to_le32(cmd.cdw15); 16506bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox 16516bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox length = cmd.data_len; 16526bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox if (cmd.data_len) { 1653497421880acecd0281d3182d534f3d28c927caecMatthew Wilcox iod = nvme_map_user_pages(dev, cmd.opcode & 1, cmd.addr, 1654497421880acecd0281d3182d534f3d28c927caecMatthew Wilcox length); 1655eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox if (IS_ERR(iod)) 1656eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return PTR_ERR(iod); 1657eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox length = nvme_setup_prps(dev, &c.common, iod, length, 1658eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox GFP_KERNEL); 16596bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox } 16606bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox 166194f370cab6e5ac514b658c6b2b3aa308cefc5c7aKeith Busch timeout = cmd.timeout_ms ? msecs_to_jiffies(cmd.timeout_ms) : 166294f370cab6e5ac514b658c6b2b3aa308cefc5c7aKeith Busch ADMIN_TIMEOUT; 16636bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox if (length != cmd.data_len) 1664b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox status = -ENOMEM; 1665b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox else 166694f370cab6e5ac514b658c6b2b3aa308cefc5c7aKeith Busch status = nvme_submit_sync_cmd(dev->queues[0], &c, &cmd.result, 166794f370cab6e5ac514b658c6b2b3aa308cefc5c7aKeith Busch timeout); 1668eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox 16696bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox if (cmd.data_len) { 16701c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox nvme_unmap_user_pages(dev, cmd.opcode & 1, iod); 1671eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox nvme_free_iod(dev, iod); 16726bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox } 1673f4f117f64baf8840d22266d518227b2a186d294bKeith Busch 1674cf90bc4830b858487fe4b9b9ecd0031e23ca3e83Chayan Biswas if ((status >= 0) && copy_to_user(&ucmd->result, &cmd.result, 1675f4f117f64baf8840d22266d518227b2a186d294bKeith Busch sizeof(cmd.result))) 1676f4f117f64baf8840d22266d518227b2a186d294bKeith Busch status = -EFAULT; 1677f4f117f64baf8840d22266d518227b2a186d294bKeith Busch 16786ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox return status; 16796ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox} 16806ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox 1681b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, 1682b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unsigned long arg) 1683b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1684b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_ns *ns = bdev->bd_disk->private_data; 1685b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1686b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox switch (cmd) { 16876bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox case NVME_IOCTL_ID: 1688c3bfe7176c035a0a2c70bc79180fb13a6c57142aMatthew Wilcox force_successful_syscall_return(); 16896bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox return ns->ns_id; 16906bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox case NVME_IOCTL_ADMIN_CMD: 169150af8baec46a99a9b81a4600c0374f83a5a590a9Keith Busch return nvme_user_admin_cmd(ns->dev, (void __user *)arg); 1692a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox case NVME_IOCTL_SUBMIT_IO: 1693a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox return nvme_submit_io(ns, (void __user *)arg); 16945d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma case SG_GET_VERSION_NUM: 16955d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma return nvme_sg_get_version_num((void __user *)arg); 16965d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma case SG_IO: 16975d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma return nvme_sg_io(ns, (void __user *)arg); 1698b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox default: 1699b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -ENOTTY; 1700b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 1701b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1702b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1703320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch#ifdef CONFIG_COMPAT 1704320a382746e0ab1304476ea7e986a8d416ab99dbKeith Buschstatic int nvme_compat_ioctl(struct block_device *bdev, fmode_t mode, 1705320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch unsigned int cmd, unsigned long arg) 1706320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch{ 1707320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch struct nvme_ns *ns = bdev->bd_disk->private_data; 1708320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch 1709320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch switch (cmd) { 1710320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch case SG_IO: 1711320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch return nvme_sg_io32(ns, arg); 1712320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch } 1713320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch return nvme_ioctl(bdev, mode, cmd, arg); 1714320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch} 1715320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch#else 1716320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch#define nvme_compat_ioctl NULL 1717320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch#endif 1718320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch 1719b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic const struct block_device_operations nvme_fops = { 1720b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .owner = THIS_MODULE, 1721b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .ioctl = nvme_ioctl, 1722320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch .compat_ioctl = nvme_compat_ioctl, 1723b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 1724b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 17251fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcoxstatic void nvme_resubmit_bios(struct nvme_queue *nvmeq) 17261fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox{ 17271fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox while (bio_list_peek(&nvmeq->sq_cong)) { 17281fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox struct bio *bio = bio_list_pop(&nvmeq->sq_cong); 17291fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox struct nvme_ns *ns = bio->bi_bdev->bd_disk->private_data; 1730427e97080196548557b288517537ab7eb48c309fKeith Busch 1731427e97080196548557b288517537ab7eb48c309fKeith Busch if (bio_list_empty(&nvmeq->sq_cong)) 1732427e97080196548557b288517537ab7eb48c309fKeith Busch remove_wait_queue(&nvmeq->sq_full, 1733427e97080196548557b288517537ab7eb48c309fKeith Busch &nvmeq->sq_cong_wait); 17341fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox if (nvme_submit_bio_queue(nvmeq, ns, bio)) { 1735427e97080196548557b288517537ab7eb48c309fKeith Busch if (bio_list_empty(&nvmeq->sq_cong)) 1736427e97080196548557b288517537ab7eb48c309fKeith Busch add_wait_queue(&nvmeq->sq_full, 1737427e97080196548557b288517537ab7eb48c309fKeith Busch &nvmeq->sq_cong_wait); 17381fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox bio_list_add_head(&nvmeq->sq_cong, bio); 17391fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox break; 17401fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox } 17411fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox } 17421fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox} 17431fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 17441fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcoxstatic int nvme_kthread(void *data) 17451fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox{ 1746d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch struct nvme_dev *dev, *next; 17471fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 17481fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox while (!kthread_should_stop()) { 1749564a232c059913d91b491e04c2b2d670b8f94615Arjan van de Ven set_current_state(TASK_INTERRUPTIBLE); 17501fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_lock(&dev_list_lock); 1751d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch list_for_each_entry_safe(dev, next, &dev_list, node) { 17521fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox int i; 1753d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch if (readl(&dev->bar->csts) & NVME_CSTS_CFS && 1754d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch dev->initialized) { 1755d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch if (work_busy(&dev->reset_work)) 1756d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch continue; 1757d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch list_del_init(&dev->node); 1758d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch dev_warn(&dev->pci_dev->dev, 1759d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch "Failed status, reset controller\n"); 1760d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch INIT_WORK(&dev->reset_work, 1761d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch nvme_reset_failed_dev); 1762d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch queue_work(nvme_workq, &dev->reset_work); 1763d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch continue; 1764d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch } 17651fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox for (i = 0; i < dev->queue_count; i++) { 17661fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox struct nvme_queue *nvmeq = dev->queues[i]; 1767740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox if (!nvmeq) 1768740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox continue; 17691fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_lock_irq(&nvmeq->q_lock); 1770224042742582c9938788b81165180c876e997a07Keith Busch if (nvmeq->q_suspended) 1771224042742582c9938788b81165180c876e997a07Keith Busch goto unlock; 1772bc57a0f7a44cfcf3e9873f6c6b8dcecdca486b1fMatthew Wilcox nvme_process_cq(nvmeq); 1773a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox nvme_cancel_ios(nvmeq, true); 17741fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox nvme_resubmit_bios(nvmeq); 1775224042742582c9938788b81165180c876e997a07Keith Busch unlock: 17761fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_unlock_irq(&nvmeq->q_lock); 17771fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox } 17781fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox } 17791fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_unlock(&dev_list_lock); 1780acb7aa0db09b8abd38abeb84334a8a27a52fbb1bArjan van de Ven schedule_timeout(round_jiffies_relative(HZ)); 17811fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox } 17821fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox return 0; 17831fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox} 17841fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 17850e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Buschstatic void nvme_config_discard(struct nvme_ns *ns) 17860e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch{ 17870e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch u32 logical_block_size = queue_logical_block_size(ns->queue); 17880e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch ns->queue->limits.discard_zeroes_data = 0; 17890e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch ns->queue->limits.discard_alignment = logical_block_size; 17900e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch ns->queue->limits.discard_granularity = logical_block_size; 17910e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch ns->queue->limits.max_discard_sectors = 0xffffffff; 17920e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, ns->queue); 17930e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch} 17940e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch 1795c3bfe7176c035a0a2c70bc79180fb13a6c57142aMatthew Wilcoxstatic struct nvme_ns *nvme_alloc_ns(struct nvme_dev *dev, unsigned nsid, 1796b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_id_ns *id, struct nvme_lba_range_type *rt) 1797b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1798b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_ns *ns; 1799b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct gendisk *disk; 1800b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int lbaf; 1801b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1802b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (rt->attributes & NVME_LBART_ATTRIB_HIDE) 1803b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 1804b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1805b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns = kzalloc(sizeof(*ns), GFP_KERNEL); 1806b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!ns) 1807b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 1808b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->queue = blk_alloc_queue(GFP_KERNEL); 1809b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!ns->queue) 1810b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto out_free_ns; 18114eeb9215a0d5c9494ca8b20158cc8ee82618840cMatthew Wilcox ns->queue->queue_flags = QUEUE_FLAG_DEFAULT; 18124eeb9215a0d5c9494ca8b20158cc8ee82618840cMatthew Wilcox queue_flag_set_unlocked(QUEUE_FLAG_NOMERGES, ns->queue); 18134eeb9215a0d5c9494ca8b20158cc8ee82618840cMatthew Wilcox queue_flag_set_unlocked(QUEUE_FLAG_NONROT, ns->queue); 1814b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox blk_queue_make_request(ns->queue, nvme_make_request); 1815b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->dev = dev; 1816b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->queue->queuedata = ns; 1817b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1818469071a37afc8a627b6b2ddf29db0a097d864845Matthew Wilcox disk = alloc_disk(0); 1819b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!disk) 1820b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto out_free_queue; 18215aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox ns->ns_id = nsid; 1822b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->disk = disk; 1823b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox lbaf = id->flbas & 0xf; 1824b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->lba_shift = id->lbaf[lbaf].ds; 1825f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch ns->ms = le16_to_cpu(id->lbaf[lbaf].ms); 1826e9ef46369f5107e634a93b7fc4e62a1f53343197Keith Busch blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift); 18278fc23e032debd682f5ba9fc524a5846c10d2c522Keith Busch if (dev->max_hw_sectors) 18288fc23e032debd682f5ba9fc524a5846c10d2c522Keith Busch blk_queue_max_hw_sectors(ns->queue, dev->max_hw_sectors); 1829b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1830b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk->major = nvme_major; 1831469071a37afc8a627b6b2ddf29db0a097d864845Matthew Wilcox disk->first_minor = 0; 1832b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk->fops = &nvme_fops; 1833b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk->private_data = ns; 1834b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk->queue = ns->queue; 1835388f037f4e7f0a24bac6b1a24f144f5d939f58cfMatthew Wilcox disk->driverfs_dev = &dev->pci_dev->dev; 1836469071a37afc8a627b6b2ddf29db0a097d864845Matthew Wilcox disk->flags = GENHD_FL_EXT_DEVT; 18375aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox sprintf(disk->disk_name, "nvme%dn%d", dev->instance, nsid); 1838b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox set_capacity(disk, le64_to_cpup(&id->nsze) << (ns->lba_shift - 9)); 1839b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 18400e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch if (dev->oncs & NVME_CTRL_ONCS_DSM) 18410e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch nvme_config_discard(ns); 18420e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch 1843b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return ns; 1844b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1845b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox out_free_queue: 1846b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox blk_cleanup_queue(ns->queue); 1847b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox out_free_ns: 1848b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(ns); 1849b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 1850b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1851b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1852b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void nvme_ns_free(struct nvme_ns *ns) 1853b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1854b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox put_disk(ns->disk); 1855b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox blk_cleanup_queue(ns->queue); 1856b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(ns); 1857b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1858b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1859b3b06812e199f248561ce7824a4a8a9cd573c05aMatthew Wilcoxstatic int set_queue_count(struct nvme_dev *dev, int count) 1860b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1861b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int status; 1862b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 result; 1863b3b06812e199f248561ce7824a4a8a9cd573c05aMatthew Wilcox u32 q_count = (count - 1) | ((count - 1) << 16); 1864b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1865df3481399042200792822b6243e36a95a557b57eMatthew Wilcox status = nvme_set_features(dev, NVME_FEAT_NUM_QUEUES, q_count, 0, 1866bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox &result); 1867b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (status) 18687e03b124065507e72008ef294c30001eca74a031Keith Busch return status < 0 ? -EIO : -EBUSY; 1869b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return min(result & 0xffff, result >> 16) + 1; 1870b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1871b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 18729d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Buschstatic size_t db_bar_size(struct nvme_dev *dev, unsigned nr_io_queues) 18739d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch{ 1874b80d5ccca3a012e91ca64a2a0b13049163a6a698Haiyan Hu return 4096 + ((nr_io_queues + 1) * 8 * dev->db_stride); 18759d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch} 18769d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch 18778d85fce77edfc22f1d6dbf78e3af723b4b556f3dGreg Kroah-Hartmanstatic int nvme_setup_io_queues(struct nvme_dev *dev) 1878b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 18793193f07bb7ae723f33ac8e8b9db317a4f68d7d18Matthew Wilcox struct nvme_queue *adminq = dev->queues[0]; 1880fa08a396647767abd24a9e7015cb177121d0cf15Ramachandra Rao Gajula struct pci_dev *pdev = dev->pci_dev; 18819d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch int result, cpu, i, vecs, nr_io_queues, size, q_depth; 1882b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1883b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox nr_io_queues = num_online_cpus(); 1884b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox result = set_queue_count(dev, nr_io_queues); 18851b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox if (result < 0) 18861b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox return result; 1887b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox if (result < nr_io_queues) 1888b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox nr_io_queues = result; 1889b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 18909d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch size = db_bar_size(dev, nr_io_queues); 18919d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch if (size > 8192) { 1892f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox iounmap(dev->bar); 18939d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch do { 18949d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch dev->bar = ioremap(pci_resource_start(pdev, 0), size); 18959d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch if (dev->bar) 18969d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch break; 18979d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch if (!--nr_io_queues) 18989d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch return -ENOMEM; 18999d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch size = db_bar_size(dev, nr_io_queues); 19009d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch } while (1); 1901f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox dev->dbs = ((void __iomem *)dev->bar) + 4096; 1902f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox dev->queues[0]->q_db = dev->dbs; 1903f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox } 1904f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox 19059d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch /* Deregister the admin queue's interrupt */ 19063193f07bb7ae723f33ac8e8b9db317a4f68d7d18Matthew Wilcox free_irq(dev->entry[0].vector, adminq); 19079d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch 1908063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox vecs = nr_io_queues; 1909063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox for (i = 0; i < vecs; i++) 19101b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox dev->entry[i].entry = i; 19111b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox for (;;) { 1912063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox result = pci_enable_msix(pdev, dev->entry, vecs); 1913063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox if (result <= 0) 19141b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox break; 1915063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox vecs = result; 19161b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox } 19171b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox 1918063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox if (result < 0) { 1919063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox vecs = nr_io_queues; 1920063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox if (vecs > 32) 1921063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox vecs = 32; 1922fa08a396647767abd24a9e7015cb177121d0cf15Ramachandra Rao Gajula for (;;) { 1923063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox result = pci_enable_msi_block(pdev, vecs); 1924fa08a396647767abd24a9e7015cb177121d0cf15Ramachandra Rao Gajula if (result == 0) { 1925063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox for (i = 0; i < vecs; i++) 1926fa08a396647767abd24a9e7015cb177121d0cf15Ramachandra Rao Gajula dev->entry[i].vector = i + pdev->irq; 1927fa08a396647767abd24a9e7015cb177121d0cf15Ramachandra Rao Gajula break; 1928063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox } else if (result < 0) { 1929063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox vecs = 1; 1930fa08a396647767abd24a9e7015cb177121d0cf15Ramachandra Rao Gajula break; 1931fa08a396647767abd24a9e7015cb177121d0cf15Ramachandra Rao Gajula } 1932063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox vecs = result; 1933fa08a396647767abd24a9e7015cb177121d0cf15Ramachandra Rao Gajula } 1934fa08a396647767abd24a9e7015cb177121d0cf15Ramachandra Rao Gajula } 1935fa08a396647767abd24a9e7015cb177121d0cf15Ramachandra Rao Gajula 1936063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox /* 1937063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox * Should investigate if there's a performance win from allocating 1938063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox * more queues than interrupt vectors; it might allow the submission 1939063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox * path to scale better, even if the receive path is limited by the 1940063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox * number of interrupts. 1941063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox */ 1942063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox nr_io_queues = vecs; 1943063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox 19443193f07bb7ae723f33ac8e8b9db317a4f68d7d18Matthew Wilcox result = queue_request_irq(dev, adminq, adminq->irqname); 19459d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch if (result) { 19463193f07bb7ae723f33ac8e8b9db317a4f68d7d18Matthew Wilcox adminq->q_suspended = 1; 1947224042742582c9938788b81165180c876e997a07Keith Busch goto free_queues; 19489d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch } 19491b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox 1950cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch /* Free previously allocated queues that are no longer usable */ 1951cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch spin_lock(&dev_list_lock); 1952cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch for (i = dev->queue_count - 1; i > nr_io_queues; i--) { 1953cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch struct nvme_queue *nvmeq = dev->queues[i]; 1954cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch 19550a8d44cb33377969337fa6c1961b631605d5f453Matthew Wilcox spin_lock_irq(&nvmeq->q_lock); 1956cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch nvme_cancel_ios(nvmeq, false); 19570a8d44cb33377969337fa6c1961b631605d5f453Matthew Wilcox spin_unlock_irq(&nvmeq->q_lock); 1958cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch 1959cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch nvme_free_queue(nvmeq); 1960cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch dev->queue_count--; 1961cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch dev->queues[i] = NULL; 1962cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch } 1963cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch spin_unlock(&dev_list_lock); 1964cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch 19651b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox cpu = cpumask_first(cpu_online_mask); 1966b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox for (i = 0; i < nr_io_queues; i++) { 19671b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox irq_set_affinity_hint(dev->entry[i].vector, get_cpu_mask(cpu)); 19681b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox cpu = cpumask_next(cpu, cpu_online_mask); 19691b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox } 19701b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox 1971a0cadb85b8b758608ae0759151e29de7581c6731Keith Busch q_depth = min_t(int, NVME_CAP_MQES(readq(&dev->bar->cap)) + 1, 1972a0cadb85b8b758608ae0759151e29de7581c6731Keith Busch NVME_Q_DEPTH); 1973cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch for (i = dev->queue_count - 1; i < nr_io_queues; i++) { 1974224042742582c9938788b81165180c876e997a07Keith Busch dev->queues[i + 1] = nvme_alloc_queue(dev, i + 1, q_depth, i); 1975224042742582c9938788b81165180c876e997a07Keith Busch if (!dev->queues[i + 1]) { 1976224042742582c9938788b81165180c876e997a07Keith Busch result = -ENOMEM; 1977224042742582c9938788b81165180c876e997a07Keith Busch goto free_queues; 1978224042742582c9938788b81165180c876e997a07Keith Busch } 19791b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox } 1980b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 19819ecdc946212f7cd592986b2c519b470404caa6b8Matthew Wilcox for (; i < num_possible_cpus(); i++) { 19829ecdc946212f7cd592986b2c519b470404caa6b8Matthew Wilcox int target = i % rounddown_pow_of_two(dev->queue_count - 1); 19839ecdc946212f7cd592986b2c519b470404caa6b8Matthew Wilcox dev->queues[i + 1] = dev->queues[target + 1]; 19849ecdc946212f7cd592986b2c519b470404caa6b8Matthew Wilcox } 19859ecdc946212f7cd592986b2c519b470404caa6b8Matthew Wilcox 1986224042742582c9938788b81165180c876e997a07Keith Busch for (i = 1; i < dev->queue_count; i++) { 1987224042742582c9938788b81165180c876e997a07Keith Busch result = nvme_create_queue(dev->queues[i], i); 1988224042742582c9938788b81165180c876e997a07Keith Busch if (result) { 1989224042742582c9938788b81165180c876e997a07Keith Busch for (--i; i > 0; i--) 1990224042742582c9938788b81165180c876e997a07Keith Busch nvme_disable_queue(dev, i); 1991224042742582c9938788b81165180c876e997a07Keith Busch goto free_queues; 1992224042742582c9938788b81165180c876e997a07Keith Busch } 1993224042742582c9938788b81165180c876e997a07Keith Busch } 1994b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1995224042742582c9938788b81165180c876e997a07Keith Busch return 0; 1996b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1997224042742582c9938788b81165180c876e997a07Keith Busch free_queues: 1998a1a5ef9998b1379780790b878457b0e0f48b25dbKeith Busch nvme_free_queues(dev, 1); 1999224042742582c9938788b81165180c876e997a07Keith Busch return result; 2000b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 2001b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2002422ef0c7c81934c3344e1623d2dcfda5fa2fab8dMatthew Wilcox/* 2003422ef0c7c81934c3344e1623d2dcfda5fa2fab8dMatthew Wilcox * Return: error value if an error occurred setting up the queues or calling 2004422ef0c7c81934c3344e1623d2dcfda5fa2fab8dMatthew Wilcox * Identify Device. 0 if these succeeded, even if adding some of the 2005422ef0c7c81934c3344e1623d2dcfda5fa2fab8dMatthew Wilcox * namespaces failed. At the moment, these failures are silent. TBD which 2006422ef0c7c81934c3344e1623d2dcfda5fa2fab8dMatthew Wilcox * failures should be reported. 2007422ef0c7c81934c3344e1623d2dcfda5fa2fab8dMatthew Wilcox */ 20088d85fce77edfc22f1d6dbf78e3af723b4b556f3dGreg Kroah-Hartmanstatic int nvme_dev_add(struct nvme_dev *dev) 2009b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 201068608c268bf265b039daeaafee6c902dfef32024Matthew Wilcox struct pci_dev *pdev = dev->pci_dev; 2011c3bfe7176c035a0a2c70bc79180fb13a6c57142aMatthew Wilcox int res; 2012c3bfe7176c035a0a2c70bc79180fb13a6c57142aMatthew Wilcox unsigned nn, i; 2013cbb6218fd4ae8f98ddf0d66f0826a7a3a9c88298Keith Busch struct nvme_ns *ns; 201451814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox struct nvme_id_ctrl *ctrl; 2015bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox struct nvme_id_ns *id_ns; 2016bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox void *mem; 2017b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_addr_t dma_addr; 2018159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch int shift = NVME_CAP_MPSMIN(readq(&dev->bar->cap)) + 12; 2019b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 202068608c268bf265b039daeaafee6c902dfef32024Matthew Wilcox mem = dma_alloc_coherent(&pdev->dev, 8192, &dma_addr, GFP_KERNEL); 2021a9ef4343afbe67a6abf83c0f0294e80db48e513aKeith Busch if (!mem) 2022a9ef4343afbe67a6abf83c0f0294e80db48e513aKeith Busch return -ENOMEM; 2023b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2024bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox res = nvme_identify(dev, 0, 1, dma_addr); 2025b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (res) { 2026b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox res = -EIO; 2027cbb6218fd4ae8f98ddf0d66f0826a7a3a9c88298Keith Busch goto out; 2028b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 2029b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2030bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox ctrl = mem; 203151814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox nn = le32_to_cpup(&ctrl->nn); 20320e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch dev->oncs = le16_to_cpup(&ctrl->oncs); 2033c30341dc3c436cf43508cd44cdfbb3810c38c195Keith Busch dev->abort_limit = ctrl->acl + 1; 203451814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox memcpy(dev->serial, ctrl->sn, sizeof(ctrl->sn)); 203551814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox memcpy(dev->model, ctrl->mn, sizeof(ctrl->mn)); 203651814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox memcpy(dev->firmware_rev, ctrl->fr, sizeof(ctrl->fr)); 2037159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch if (ctrl->mdts) 20388fc23e032debd682f5ba9fc524a5846c10d2c522Keith Busch dev->max_hw_sectors = 1 << (ctrl->mdts + shift - 9); 203968608c268bf265b039daeaafee6c902dfef32024Matthew Wilcox if ((pdev->vendor == PCI_VENDOR_ID_INTEL) && 204068608c268bf265b039daeaafee6c902dfef32024Matthew Wilcox (pdev->device == 0x0953) && ctrl->vs[3]) 2041159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch dev->stripe_size = 1 << (ctrl->vs[3] + shift); 2042b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2043bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox id_ns = mem; 20442b2c1896871838cdf549442e8ad0264be5fa74e3Matthew Wilcox for (i = 1; i <= nn; i++) { 2045bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox res = nvme_identify(dev, i, 0, dma_addr); 2046b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (res) 2047b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox continue; 2048b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2049bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox if (id_ns->ncap == 0) 2050b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox continue; 2051b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2052bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox res = nvme_get_features(dev, NVME_FEAT_LBA_RANGE, i, 205308df1e05657fc6712e520e7c09cc6c86160ceb35Keith Busch dma_addr + 4096, NULL); 2054b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (res) 2055122090366d1d5c6ec1bfb6dfdb3a6d121ff074aaKeith Busch memset(mem + 4096, 0, 4096); 2056b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2057bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox ns = nvme_alloc_ns(dev, i, mem, mem + 4096); 2058b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (ns) 2059b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox list_add_tail(&ns->list, &dev->namespaces); 2060b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 2061b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox list_for_each_entry(ns, &dev->namespaces, list) 2062b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox add_disk(ns->disk); 2063422ef0c7c81934c3344e1623d2dcfda5fa2fab8dMatthew Wilcox res = 0; 2064b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2065bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox out: 2066684f5c2025b067a23722e620d0b3b858d8dc5d01Matthew Wilcox dma_free_coherent(&dev->pci_dev->dev, 8192, mem, dma_addr); 2067b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return res; 2068b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 2069b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 20700877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Buschstatic int nvme_dev_map(struct nvme_dev *dev) 20710877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch{ 20720877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch int bars, result = -ENOMEM; 20730877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch struct pci_dev *pdev = dev->pci_dev; 20740877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch 20750877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch if (pci_enable_device_mem(pdev)) 20760877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch return result; 20770877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch 20780877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch dev->entry[0].vector = pdev->irq; 20790877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch pci_set_master(pdev); 20800877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch bars = pci_select_bars(pdev, IORESOURCE_MEM); 20810877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch if (pci_request_selected_regions(pdev, bars, "nvme")) 20820877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch goto disable_pci; 20830877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch 2084052d0efada8ccfccd924059b75f5fb119cd10f9aRussell King if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) && 2085052d0efada8ccfccd924059b75f5fb119cd10f9aRussell King dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32))) 2086052d0efada8ccfccd924059b75f5fb119cd10f9aRussell King goto disable; 20870877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch 20880877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch dev->bar = ioremap(pci_resource_start(pdev, 0), 8192); 20890877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch if (!dev->bar) 20900877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch goto disable; 20910e53d18051725da46cbccfb7874a6422d4d4f274Keith Busch if (readl(&dev->bar->csts) == -1) { 20920e53d18051725da46cbccfb7874a6422d4d4f274Keith Busch result = -ENODEV; 20930e53d18051725da46cbccfb7874a6422d4d4f274Keith Busch goto unmap; 20940e53d18051725da46cbccfb7874a6422d4d4f274Keith Busch } 2095b80d5ccca3a012e91ca64a2a0b13049163a6a698Haiyan Hu dev->db_stride = 1 << NVME_CAP_STRIDE(readq(&dev->bar->cap)); 20960877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch dev->dbs = ((void __iomem *)dev->bar) + 4096; 20970877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch 20980877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch return 0; 20990877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch 21000e53d18051725da46cbccfb7874a6422d4d4f274Keith Busch unmap: 21010e53d18051725da46cbccfb7874a6422d4d4f274Keith Busch iounmap(dev->bar); 21020e53d18051725da46cbccfb7874a6422d4d4f274Keith Busch dev->bar = NULL; 21030877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch disable: 21040877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch pci_release_regions(pdev); 21050877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch disable_pci: 21060877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch pci_disable_device(pdev); 21070877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch return result; 21080877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch} 21090877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch 21100877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Buschstatic void nvme_dev_unmap(struct nvme_dev *dev) 21110877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch{ 21120877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch if (dev->pci_dev->msi_enabled) 21130877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch pci_disable_msi(dev->pci_dev); 21140877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch else if (dev->pci_dev->msix_enabled) 21150877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch pci_disable_msix(dev->pci_dev); 21160877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch 21170877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch if (dev->bar) { 21180877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch iounmap(dev->bar); 21190877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch dev->bar = NULL; 21209a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch pci_release_regions(dev->pci_dev); 21210877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch } 21220877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch 21230877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch if (pci_is_enabled(dev->pci_dev)) 21240877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch pci_disable_device(dev->pci_dev); 21250877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch} 21260877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch 21274d115420707afcabe77d2535e092356df6664b70Keith Buschstruct nvme_delq_ctx { 21284d115420707afcabe77d2535e092356df6664b70Keith Busch struct task_struct *waiter; 21294d115420707afcabe77d2535e092356df6664b70Keith Busch struct kthread_worker *worker; 21304d115420707afcabe77d2535e092356df6664b70Keith Busch atomic_t refcount; 21314d115420707afcabe77d2535e092356df6664b70Keith Busch}; 21324d115420707afcabe77d2535e092356df6664b70Keith Busch 21334d115420707afcabe77d2535e092356df6664b70Keith Buschstatic void nvme_wait_dq(struct nvme_delq_ctx *dq, struct nvme_dev *dev) 21344d115420707afcabe77d2535e092356df6664b70Keith Busch{ 21354d115420707afcabe77d2535e092356df6664b70Keith Busch dq->waiter = current; 21364d115420707afcabe77d2535e092356df6664b70Keith Busch mb(); 21374d115420707afcabe77d2535e092356df6664b70Keith Busch 21384d115420707afcabe77d2535e092356df6664b70Keith Busch for (;;) { 21394d115420707afcabe77d2535e092356df6664b70Keith Busch set_current_state(TASK_KILLABLE); 21404d115420707afcabe77d2535e092356df6664b70Keith Busch if (!atomic_read(&dq->refcount)) 21414d115420707afcabe77d2535e092356df6664b70Keith Busch break; 21424d115420707afcabe77d2535e092356df6664b70Keith Busch if (!schedule_timeout(ADMIN_TIMEOUT) || 21434d115420707afcabe77d2535e092356df6664b70Keith Busch fatal_signal_pending(current)) { 21444d115420707afcabe77d2535e092356df6664b70Keith Busch set_current_state(TASK_RUNNING); 21454d115420707afcabe77d2535e092356df6664b70Keith Busch 21464d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_disable_ctrl(dev, readq(&dev->bar->cap)); 21474d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_disable_queue(dev, 0); 21484d115420707afcabe77d2535e092356df6664b70Keith Busch 21494d115420707afcabe77d2535e092356df6664b70Keith Busch send_sig(SIGKILL, dq->worker->task, 1); 21504d115420707afcabe77d2535e092356df6664b70Keith Busch flush_kthread_worker(dq->worker); 21514d115420707afcabe77d2535e092356df6664b70Keith Busch return; 21524d115420707afcabe77d2535e092356df6664b70Keith Busch } 21534d115420707afcabe77d2535e092356df6664b70Keith Busch } 21544d115420707afcabe77d2535e092356df6664b70Keith Busch set_current_state(TASK_RUNNING); 21554d115420707afcabe77d2535e092356df6664b70Keith Busch} 21564d115420707afcabe77d2535e092356df6664b70Keith Busch 21574d115420707afcabe77d2535e092356df6664b70Keith Buschstatic void nvme_put_dq(struct nvme_delq_ctx *dq) 21584d115420707afcabe77d2535e092356df6664b70Keith Busch{ 21594d115420707afcabe77d2535e092356df6664b70Keith Busch atomic_dec(&dq->refcount); 21604d115420707afcabe77d2535e092356df6664b70Keith Busch if (dq->waiter) 21614d115420707afcabe77d2535e092356df6664b70Keith Busch wake_up_process(dq->waiter); 21624d115420707afcabe77d2535e092356df6664b70Keith Busch} 21634d115420707afcabe77d2535e092356df6664b70Keith Busch 21644d115420707afcabe77d2535e092356df6664b70Keith Buschstatic struct nvme_delq_ctx *nvme_get_dq(struct nvme_delq_ctx *dq) 21654d115420707afcabe77d2535e092356df6664b70Keith Busch{ 21664d115420707afcabe77d2535e092356df6664b70Keith Busch atomic_inc(&dq->refcount); 21674d115420707afcabe77d2535e092356df6664b70Keith Busch return dq; 21684d115420707afcabe77d2535e092356df6664b70Keith Busch} 21694d115420707afcabe77d2535e092356df6664b70Keith Busch 21704d115420707afcabe77d2535e092356df6664b70Keith Buschstatic void nvme_del_queue_end(struct nvme_queue *nvmeq) 21714d115420707afcabe77d2535e092356df6664b70Keith Busch{ 21724d115420707afcabe77d2535e092356df6664b70Keith Busch struct nvme_delq_ctx *dq = nvmeq->cmdinfo.ctx; 21734d115420707afcabe77d2535e092356df6664b70Keith Busch 21744d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_clear_queue(nvmeq); 21754d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_put_dq(dq); 21764d115420707afcabe77d2535e092356df6664b70Keith Busch} 21774d115420707afcabe77d2535e092356df6664b70Keith Busch 21784d115420707afcabe77d2535e092356df6664b70Keith Buschstatic int adapter_async_del_queue(struct nvme_queue *nvmeq, u8 opcode, 21794d115420707afcabe77d2535e092356df6664b70Keith Busch kthread_work_func_t fn) 21804d115420707afcabe77d2535e092356df6664b70Keith Busch{ 21814d115420707afcabe77d2535e092356df6664b70Keith Busch struct nvme_command c; 21824d115420707afcabe77d2535e092356df6664b70Keith Busch 21834d115420707afcabe77d2535e092356df6664b70Keith Busch memset(&c, 0, sizeof(c)); 21844d115420707afcabe77d2535e092356df6664b70Keith Busch c.delete_queue.opcode = opcode; 21854d115420707afcabe77d2535e092356df6664b70Keith Busch c.delete_queue.qid = cpu_to_le16(nvmeq->qid); 21864d115420707afcabe77d2535e092356df6664b70Keith Busch 21874d115420707afcabe77d2535e092356df6664b70Keith Busch init_kthread_work(&nvmeq->cmdinfo.work, fn); 21884d115420707afcabe77d2535e092356df6664b70Keith Busch return nvme_submit_admin_cmd_async(nvmeq->dev, &c, &nvmeq->cmdinfo); 21894d115420707afcabe77d2535e092356df6664b70Keith Busch} 21904d115420707afcabe77d2535e092356df6664b70Keith Busch 21914d115420707afcabe77d2535e092356df6664b70Keith Buschstatic void nvme_del_cq_work_handler(struct kthread_work *work) 21924d115420707afcabe77d2535e092356df6664b70Keith Busch{ 21934d115420707afcabe77d2535e092356df6664b70Keith Busch struct nvme_queue *nvmeq = container_of(work, struct nvme_queue, 21944d115420707afcabe77d2535e092356df6664b70Keith Busch cmdinfo.work); 21954d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_del_queue_end(nvmeq); 21964d115420707afcabe77d2535e092356df6664b70Keith Busch} 21974d115420707afcabe77d2535e092356df6664b70Keith Busch 21984d115420707afcabe77d2535e092356df6664b70Keith Buschstatic int nvme_delete_cq(struct nvme_queue *nvmeq) 21994d115420707afcabe77d2535e092356df6664b70Keith Busch{ 22004d115420707afcabe77d2535e092356df6664b70Keith Busch return adapter_async_del_queue(nvmeq, nvme_admin_delete_cq, 22014d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_del_cq_work_handler); 22024d115420707afcabe77d2535e092356df6664b70Keith Busch} 22034d115420707afcabe77d2535e092356df6664b70Keith Busch 22044d115420707afcabe77d2535e092356df6664b70Keith Buschstatic void nvme_del_sq_work_handler(struct kthread_work *work) 22054d115420707afcabe77d2535e092356df6664b70Keith Busch{ 22064d115420707afcabe77d2535e092356df6664b70Keith Busch struct nvme_queue *nvmeq = container_of(work, struct nvme_queue, 22074d115420707afcabe77d2535e092356df6664b70Keith Busch cmdinfo.work); 22084d115420707afcabe77d2535e092356df6664b70Keith Busch int status = nvmeq->cmdinfo.status; 22094d115420707afcabe77d2535e092356df6664b70Keith Busch 22104d115420707afcabe77d2535e092356df6664b70Keith Busch if (!status) 22114d115420707afcabe77d2535e092356df6664b70Keith Busch status = nvme_delete_cq(nvmeq); 22124d115420707afcabe77d2535e092356df6664b70Keith Busch if (status) 22134d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_del_queue_end(nvmeq); 22144d115420707afcabe77d2535e092356df6664b70Keith Busch} 22154d115420707afcabe77d2535e092356df6664b70Keith Busch 22164d115420707afcabe77d2535e092356df6664b70Keith Buschstatic int nvme_delete_sq(struct nvme_queue *nvmeq) 22174d115420707afcabe77d2535e092356df6664b70Keith Busch{ 22184d115420707afcabe77d2535e092356df6664b70Keith Busch return adapter_async_del_queue(nvmeq, nvme_admin_delete_sq, 22194d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_del_sq_work_handler); 22204d115420707afcabe77d2535e092356df6664b70Keith Busch} 22214d115420707afcabe77d2535e092356df6664b70Keith Busch 22224d115420707afcabe77d2535e092356df6664b70Keith Buschstatic void nvme_del_queue_start(struct kthread_work *work) 22234d115420707afcabe77d2535e092356df6664b70Keith Busch{ 22244d115420707afcabe77d2535e092356df6664b70Keith Busch struct nvme_queue *nvmeq = container_of(work, struct nvme_queue, 22254d115420707afcabe77d2535e092356df6664b70Keith Busch cmdinfo.work); 22264d115420707afcabe77d2535e092356df6664b70Keith Busch allow_signal(SIGKILL); 22274d115420707afcabe77d2535e092356df6664b70Keith Busch if (nvme_delete_sq(nvmeq)) 22284d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_del_queue_end(nvmeq); 22294d115420707afcabe77d2535e092356df6664b70Keith Busch} 22304d115420707afcabe77d2535e092356df6664b70Keith Busch 22314d115420707afcabe77d2535e092356df6664b70Keith Buschstatic void nvme_disable_io_queues(struct nvme_dev *dev) 22324d115420707afcabe77d2535e092356df6664b70Keith Busch{ 22334d115420707afcabe77d2535e092356df6664b70Keith Busch int i; 22344d115420707afcabe77d2535e092356df6664b70Keith Busch DEFINE_KTHREAD_WORKER_ONSTACK(worker); 22354d115420707afcabe77d2535e092356df6664b70Keith Busch struct nvme_delq_ctx dq; 22364d115420707afcabe77d2535e092356df6664b70Keith Busch struct task_struct *kworker_task = kthread_run(kthread_worker_fn, 22374d115420707afcabe77d2535e092356df6664b70Keith Busch &worker, "nvme%d", dev->instance); 22384d115420707afcabe77d2535e092356df6664b70Keith Busch 22394d115420707afcabe77d2535e092356df6664b70Keith Busch if (IS_ERR(kworker_task)) { 22404d115420707afcabe77d2535e092356df6664b70Keith Busch dev_err(&dev->pci_dev->dev, 22414d115420707afcabe77d2535e092356df6664b70Keith Busch "Failed to create queue del task\n"); 22424d115420707afcabe77d2535e092356df6664b70Keith Busch for (i = dev->queue_count - 1; i > 0; i--) 22434d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_disable_queue(dev, i); 22444d115420707afcabe77d2535e092356df6664b70Keith Busch return; 22454d115420707afcabe77d2535e092356df6664b70Keith Busch } 22464d115420707afcabe77d2535e092356df6664b70Keith Busch 22474d115420707afcabe77d2535e092356df6664b70Keith Busch dq.waiter = NULL; 22484d115420707afcabe77d2535e092356df6664b70Keith Busch atomic_set(&dq.refcount, 0); 22494d115420707afcabe77d2535e092356df6664b70Keith Busch dq.worker = &worker; 22504d115420707afcabe77d2535e092356df6664b70Keith Busch for (i = dev->queue_count - 1; i > 0; i--) { 22514d115420707afcabe77d2535e092356df6664b70Keith Busch struct nvme_queue *nvmeq = dev->queues[i]; 22524d115420707afcabe77d2535e092356df6664b70Keith Busch 22534d115420707afcabe77d2535e092356df6664b70Keith Busch if (nvme_suspend_queue(nvmeq)) 22544d115420707afcabe77d2535e092356df6664b70Keith Busch continue; 22554d115420707afcabe77d2535e092356df6664b70Keith Busch nvmeq->cmdinfo.ctx = nvme_get_dq(&dq); 22564d115420707afcabe77d2535e092356df6664b70Keith Busch nvmeq->cmdinfo.worker = dq.worker; 22574d115420707afcabe77d2535e092356df6664b70Keith Busch init_kthread_work(&nvmeq->cmdinfo.work, nvme_del_queue_start); 22584d115420707afcabe77d2535e092356df6664b70Keith Busch queue_kthread_work(dq.worker, &nvmeq->cmdinfo.work); 22594d115420707afcabe77d2535e092356df6664b70Keith Busch } 22604d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_wait_dq(&dq, dev); 22614d115420707afcabe77d2535e092356df6664b70Keith Busch kthread_stop(kworker_task); 22624d115420707afcabe77d2535e092356df6664b70Keith Busch} 22634d115420707afcabe77d2535e092356df6664b70Keith Busch 2264f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Buschstatic void nvme_dev_shutdown(struct nvme_dev *dev) 2265b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 2266224042742582c9938788b81165180c876e997a07Keith Busch int i; 2267224042742582c9938788b81165180c876e997a07Keith Busch 2268d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch dev->initialized = 0; 2269b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 22701fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_lock(&dev_list_lock); 2271f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch list_del_init(&dev->node); 22721fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_unlock(&dev_list_lock); 22731fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 22744d115420707afcabe77d2535e092356df6664b70Keith Busch if (!dev->bar || (dev->bar && readl(&dev->bar->csts) == -1)) { 22754d115420707afcabe77d2535e092356df6664b70Keith Busch for (i = dev->queue_count - 1; i >= 0; i--) { 22764d115420707afcabe77d2535e092356df6664b70Keith Busch struct nvme_queue *nvmeq = dev->queues[i]; 22774d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_suspend_queue(nvmeq); 22784d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_clear_queue(nvmeq); 22794d115420707afcabe77d2535e092356df6664b70Keith Busch } 22804d115420707afcabe77d2535e092356df6664b70Keith Busch } else { 22814d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_disable_io_queues(dev); 22821894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch nvme_shutdown_ctrl(dev); 22834d115420707afcabe77d2535e092356df6664b70Keith Busch nvme_disable_queue(dev, 0); 22844d115420707afcabe77d2535e092356df6664b70Keith Busch } 2285f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch nvme_dev_unmap(dev); 2286f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch} 2287f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch 2288f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Buschstatic void nvme_dev_remove(struct nvme_dev *dev) 2289f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch{ 2290f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch struct nvme_ns *ns, *next; 2291f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch 2292b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox list_for_each_entry_safe(ns, next, &dev->namespaces, list) { 2293b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox list_del(&ns->list); 2294b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox del_gendisk(ns->disk); 2295b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_ns_free(ns); 2296b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 2297b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 2298b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2299091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcoxstatic int nvme_setup_prp_pools(struct nvme_dev *dev) 2300091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox{ 2301091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox struct device *dmadev = &dev->pci_dev->dev; 2302091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox dev->prp_page_pool = dma_pool_create("prp list page", dmadev, 2303091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox PAGE_SIZE, PAGE_SIZE, 0); 2304091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox if (!dev->prp_page_pool) 2305091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox return -ENOMEM; 2306091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox 230799802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox /* Optimisation for I/Os between 4k and 128k */ 230899802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox dev->prp_small_pool = dma_pool_create("prp list 256", dmadev, 230999802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox 256, 256, 0); 231099802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox if (!dev->prp_small_pool) { 231199802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox dma_pool_destroy(dev->prp_page_pool); 231299802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox return -ENOMEM; 231399802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox } 2314091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox return 0; 2315091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox} 2316091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox 2317091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcoxstatic void nvme_release_prp_pools(struct nvme_dev *dev) 2318091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox{ 2319091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox dma_pool_destroy(dev->prp_page_pool); 232099802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox dma_pool_destroy(dev->prp_small_pool); 2321091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox} 2322091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox 2323cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anhstatic DEFINE_IDA(nvme_instance_ida); 2324cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh 2325cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anhstatic int nvme_set_instance(struct nvme_dev *dev) 2326b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 2327cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh int instance, error; 2328cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh 2329cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh do { 2330cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh if (!ida_pre_get(&nvme_instance_ida, GFP_KERNEL)) 2331cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh return -ENODEV; 2332cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh 2333cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh spin_lock(&dev_list_lock); 2334cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh error = ida_get_new(&nvme_instance_ida, &instance); 2335cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh spin_unlock(&dev_list_lock); 2336cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh } while (error == -EAGAIN); 2337cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh 2338cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh if (error) 2339cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh return -ENODEV; 2340cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh 2341cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh dev->instance = instance; 2342cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh return 0; 2343b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 2344b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2345b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void nvme_release_instance(struct nvme_dev *dev) 2346b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 2347cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh spin_lock(&dev_list_lock); 2348cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh ida_remove(&nvme_instance_ida, dev->instance); 2349cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh spin_unlock(&dev_list_lock); 2350b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 2351b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 23525e82e952f04681c10f35e02ee0a4a43ec027137aKeith Buschstatic void nvme_free_dev(struct kref *kref) 23535e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch{ 23545e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch struct nvme_dev *dev = container_of(kref, struct nvme_dev, kref); 23555e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch kfree(dev->queues); 23565e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch kfree(dev->entry); 23575e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch kfree(dev); 23585e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch} 23595e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch 23605e82e952f04681c10f35e02ee0a4a43ec027137aKeith Buschstatic int nvme_dev_open(struct inode *inode, struct file *f) 23615e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch{ 23625e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch struct nvme_dev *dev = container_of(f->private_data, struct nvme_dev, 23635e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch miscdev); 23645e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch kref_get(&dev->kref); 23655e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch f->private_data = dev; 23665e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch return 0; 23675e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch} 23685e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch 23695e82e952f04681c10f35e02ee0a4a43ec027137aKeith Buschstatic int nvme_dev_release(struct inode *inode, struct file *f) 23705e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch{ 23715e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch struct nvme_dev *dev = f->private_data; 23725e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch kref_put(&dev->kref, nvme_free_dev); 23735e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch return 0; 23745e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch} 23755e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch 23765e82e952f04681c10f35e02ee0a4a43ec027137aKeith Buschstatic long nvme_dev_ioctl(struct file *f, unsigned int cmd, unsigned long arg) 23775e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch{ 23785e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch struct nvme_dev *dev = f->private_data; 23795e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch switch (cmd) { 23805e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch case NVME_IOCTL_ADMIN_CMD: 23815e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch return nvme_user_admin_cmd(dev, (void __user *)arg); 23825e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch default: 23835e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch return -ENOTTY; 23845e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch } 23855e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch} 23865e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch 23875e82e952f04681c10f35e02ee0a4a43ec027137aKeith Buschstatic const struct file_operations nvme_dev_fops = { 23885e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch .owner = THIS_MODULE, 23895e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch .open = nvme_dev_open, 23905e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch .release = nvme_dev_release, 23915e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch .unlocked_ioctl = nvme_dev_ioctl, 23925e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch .compat_ioctl = nvme_dev_ioctl, 23935e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch}; 23945e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch 2395f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Buschstatic int nvme_dev_start(struct nvme_dev *dev) 2396f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch{ 2397f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch int result; 2398f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch 2399f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch result = nvme_dev_map(dev); 2400f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch if (result) 2401f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch return result; 2402f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch 2403f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch result = nvme_configure_admin_queue(dev); 2404f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch if (result) 2405f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch goto unmap; 2406f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch 2407f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch spin_lock(&dev_list_lock); 2408f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch list_add(&dev->node, &dev_list); 2409f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch spin_unlock(&dev_list_lock); 2410f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch 2411f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch result = nvme_setup_io_queues(dev); 2412d82e8bfdef9afae83b894be49af4644d9ac3c359Keith Busch if (result && result != -EBUSY) 2413f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch goto disable; 2414f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch 2415d82e8bfdef9afae83b894be49af4644d9ac3c359Keith Busch return result; 2416f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch 2417f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch disable: 2418a1a5ef9998b1379780790b878457b0e0f48b25dbKeith Busch nvme_disable_queue(dev, 0); 2419f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch spin_lock(&dev_list_lock); 2420f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch list_del_init(&dev->node); 2421f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch spin_unlock(&dev_list_lock); 2422f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch unmap: 2423f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch nvme_dev_unmap(dev); 2424f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch return result; 2425f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch} 2426f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch 24279a6b94584de1a0467d85b435df9c744c5c45a270Keith Buschstatic int nvme_remove_dead_ctrl(void *arg) 24289a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch{ 24299a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch struct nvme_dev *dev = (struct nvme_dev *)arg; 24309a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch struct pci_dev *pdev = dev->pci_dev; 24319a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch 24329a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch if (pci_get_drvdata(pdev)) 24339a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch pci_stop_and_remove_bus_device(pdev); 24349a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch kref_put(&dev->kref, nvme_free_dev); 24359a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch return 0; 24369a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch} 24379a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch 24389a6b94584de1a0467d85b435df9c744c5c45a270Keith Buschstatic void nvme_remove_disks(struct work_struct *ws) 24399a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch{ 24409a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch int i; 24419a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch struct nvme_dev *dev = container_of(ws, struct nvme_dev, reset_work); 24429a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch 24439a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch nvme_dev_remove(dev); 24449a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch spin_lock(&dev_list_lock); 24459a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch for (i = dev->queue_count - 1; i > 0; i--) { 24469a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch BUG_ON(!dev->queues[i] || !dev->queues[i]->q_suspended); 24479a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch nvme_free_queue(dev->queues[i]); 24489a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch dev->queue_count--; 24499a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch dev->queues[i] = NULL; 24509a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch } 24519a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch spin_unlock(&dev_list_lock); 24529a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch} 24539a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch 24549a6b94584de1a0467d85b435df9c744c5c45a270Keith Buschstatic int nvme_dev_resume(struct nvme_dev *dev) 24559a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch{ 24569a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch int ret; 24579a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch 24589a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch ret = nvme_dev_start(dev); 24599a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch if (ret && ret != -EBUSY) 24609a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch return ret; 24619a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch if (ret == -EBUSY) { 24629a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch spin_lock(&dev_list_lock); 24639a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch INIT_WORK(&dev->reset_work, nvme_remove_disks); 24649a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch queue_work(nvme_workq, &dev->reset_work); 24659a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch spin_unlock(&dev_list_lock); 24669a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch } 2467d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch dev->initialized = 1; 24689a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch return 0; 24699a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch} 24709a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch 24719a6b94584de1a0467d85b435df9c744c5c45a270Keith Buschstatic void nvme_dev_reset(struct nvme_dev *dev) 24729a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch{ 24739a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch nvme_dev_shutdown(dev); 24749a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch if (nvme_dev_resume(dev)) { 24759a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch dev_err(&dev->pci_dev->dev, "Device failed to resume\n"); 24769a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch kref_get(&dev->kref); 24779a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch if (IS_ERR(kthread_run(nvme_remove_dead_ctrl, dev, "nvme%d", 24789a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch dev->instance))) { 24799a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch dev_err(&dev->pci_dev->dev, 24809a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch "Failed to start controller remove task\n"); 24819a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch kref_put(&dev->kref, nvme_free_dev); 24829a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch } 24839a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch } 24849a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch} 24859a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch 24869a6b94584de1a0467d85b435df9c744c5c45a270Keith Buschstatic void nvme_reset_failed_dev(struct work_struct *ws) 24879a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch{ 24889a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch struct nvme_dev *dev = container_of(ws, struct nvme_dev, reset_work); 24899a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch nvme_dev_reset(dev); 24909a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch} 24919a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch 24928d85fce77edfc22f1d6dbf78e3af723b4b556f3dGreg Kroah-Hartmanstatic int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id) 2493b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 24940877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch int result = -ENOMEM; 2495b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_dev *dev; 2496b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2497b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev = kzalloc(sizeof(*dev), GFP_KERNEL); 2498b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!dev) 2499b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -ENOMEM; 2500b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->entry = kcalloc(num_possible_cpus(), sizeof(*dev->entry), 2501b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox GFP_KERNEL); 2502b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!dev->entry) 2503b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto free; 25041b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox dev->queues = kcalloc(num_possible_cpus() + 1, sizeof(void *), 25051b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox GFP_KERNEL); 2506b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!dev->queues) 2507b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto free; 2508b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2509b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox INIT_LIST_HEAD(&dev->namespaces); 2510b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->pci_dev = pdev; 25119a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch pci_set_drvdata(pdev, dev); 2512cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh result = nvme_set_instance(dev); 2513cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh if (result) 25140877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch goto free; 2515b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2516091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox result = nvme_setup_prp_pools(dev); 2517091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox if (result) 25180877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch goto release; 2519091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox 2520f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch result = nvme_dev_start(dev); 2521d82e8bfdef9afae83b894be49af4644d9ac3c359Keith Busch if (result) { 2522d82e8bfdef9afae83b894be49af4644d9ac3c359Keith Busch if (result == -EBUSY) 2523d82e8bfdef9afae83b894be49af4644d9ac3c359Keith Busch goto create_cdev; 25240877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch goto release_pools; 2525d82e8bfdef9afae83b894be49af4644d9ac3c359Keith Busch } 2526b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2527740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox result = nvme_dev_add(dev); 2528d82e8bfdef9afae83b894be49af4644d9ac3c359Keith Busch if (result) 2529f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch goto shutdown; 2530740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox 2531d82e8bfdef9afae83b894be49af4644d9ac3c359Keith Busch create_cdev: 25325e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch scnprintf(dev->name, sizeof(dev->name), "nvme%d", dev->instance); 25335e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch dev->miscdev.minor = MISC_DYNAMIC_MINOR; 25345e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch dev->miscdev.parent = &pdev->dev; 25355e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch dev->miscdev.name = dev->name; 25365e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch dev->miscdev.fops = &nvme_dev_fops; 25375e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch result = misc_register(&dev->miscdev); 25385e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch if (result) 25395e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch goto remove; 25405e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch 2541d4b4ff8e28b474fac0fbfa9cfc40f88b9e41e380Keith Busch dev->initialized = 1; 25425e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch kref_init(&dev->kref); 2543b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 2544b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 25455e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch remove: 25465e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch nvme_dev_remove(dev); 2547f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch shutdown: 2548f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch nvme_dev_shutdown(dev); 25490877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch release_pools: 2550a1a5ef9998b1379780790b878457b0e0f48b25dbKeith Busch nvme_free_queues(dev, 0); 2551091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox nvme_release_prp_pools(dev); 25520877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch release: 25530877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch nvme_release_instance(dev); 2554b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox free: 2555b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(dev->queues); 2556b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(dev->entry); 2557b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(dev); 2558b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return result; 2559b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 2560b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 256109ece1424f52a2d3ad0ab715e96b0fb342af4f03Keith Buschstatic void nvme_shutdown(struct pci_dev *pdev) 256209ece1424f52a2d3ad0ab715e96b0fb342af4f03Keith Busch{ 256309ece1424f52a2d3ad0ab715e96b0fb342af4f03Keith Busch struct nvme_dev *dev = pci_get_drvdata(pdev); 256409ece1424f52a2d3ad0ab715e96b0fb342af4f03Keith Busch nvme_dev_shutdown(dev); 256509ece1424f52a2d3ad0ab715e96b0fb342af4f03Keith Busch} 256609ece1424f52a2d3ad0ab715e96b0fb342af4f03Keith Busch 25678d85fce77edfc22f1d6dbf78e3af723b4b556f3dGreg Kroah-Hartmanstatic void nvme_remove(struct pci_dev *pdev) 2568b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 2569b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_dev *dev = pci_get_drvdata(pdev); 25709a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch 25719a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch spin_lock(&dev_list_lock); 25729a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch list_del_init(&dev->node); 25739a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch spin_unlock(&dev_list_lock); 25749a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch 25759a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch pci_set_drvdata(pdev, NULL); 25769a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch flush_work(&dev->reset_work); 25775e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch misc_deregister(&dev->miscdev); 25789a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch nvme_dev_remove(dev); 25799a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch nvme_dev_shutdown(dev); 2580a1a5ef9998b1379780790b878457b0e0f48b25dbKeith Busch nvme_free_queues(dev, 0); 25819a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch nvme_release_instance(dev); 25829a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch nvme_release_prp_pools(dev); 25835e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch kref_put(&dev->kref, nvme_free_dev); 2584b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 2585b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2586b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* These functions are yet to be implemented */ 2587b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_error_detected NULL 2588b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_dump_registers NULL 2589b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_link_reset NULL 2590b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_slot_reset NULL 2591b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_error_resume NULL 2592cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch 2593cd63894630ab17a192bf97427d16dbec10710a6aKeith Buschstatic int nvme_suspend(struct device *dev) 2594cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch{ 2595cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch struct pci_dev *pdev = to_pci_dev(dev); 2596cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch struct nvme_dev *ndev = pci_get_drvdata(pdev); 2597cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch 2598cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch nvme_dev_shutdown(ndev); 2599cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch return 0; 2600cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch} 2601cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch 2602cd63894630ab17a192bf97427d16dbec10710a6aKeith Buschstatic int nvme_resume(struct device *dev) 2603cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch{ 2604cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch struct pci_dev *pdev = to_pci_dev(dev); 2605cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch struct nvme_dev *ndev = pci_get_drvdata(pdev); 2606cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch 26079a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch if (nvme_dev_resume(ndev) && !work_busy(&ndev->reset_work)) { 26089a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch INIT_WORK(&ndev->reset_work, nvme_reset_failed_dev); 26099a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch queue_work(nvme_workq, &ndev->reset_work); 26109a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch } 26119a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch return 0; 2612cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch} 2613cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch 2614cd63894630ab17a192bf97427d16dbec10710a6aKeith Buschstatic SIMPLE_DEV_PM_OPS(nvme_dev_pm_ops, nvme_suspend, nvme_resume); 2615b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 26161d3520357df99baf4ad89f86268ac96cd38092d9Stephen Hemmingerstatic const struct pci_error_handlers nvme_err_handler = { 2617b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .error_detected = nvme_error_detected, 2618b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .mmio_enabled = nvme_dump_registers, 2619b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .link_reset = nvme_link_reset, 2620b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .slot_reset = nvme_slot_reset, 2621b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .resume = nvme_error_resume, 2622b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 2623b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2624b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* Move to pci_ids.h later */ 2625b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define PCI_CLASS_STORAGE_EXPRESS 0x010802 2626b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2627b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic DEFINE_PCI_DEVICE_TABLE(nvme_id_table) = { 2628b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox { PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) }, 2629b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox { 0, } 2630b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 2631b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew WilcoxMODULE_DEVICE_TABLE(pci, nvme_id_table); 2632b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2633b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic struct pci_driver nvme_driver = { 2634b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .name = "nvme", 2635b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .id_table = nvme_id_table, 2636b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .probe = nvme_probe, 26378d85fce77edfc22f1d6dbf78e3af723b4b556f3dGreg Kroah-Hartman .remove = nvme_remove, 263809ece1424f52a2d3ad0ab715e96b0fb342af4f03Keith Busch .shutdown = nvme_shutdown, 2639cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch .driver = { 2640cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch .pm = &nvme_dev_pm_ops, 2641cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch }, 2642b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .err_handler = &nvme_err_handler, 2643b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 2644b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2645b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int __init nvme_init(void) 2646b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 26470ac13140d796eb1e2f8956aea97a6e5e4ebcf981Matthew Wilcox int result; 26481fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 26491fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox nvme_thread = kthread_run(nvme_kthread, NULL, "nvme"); 26501fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox if (IS_ERR(nvme_thread)) 26511fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox return PTR_ERR(nvme_thread); 2652b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 26539a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch result = -ENOMEM; 26549a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch nvme_workq = create_singlethread_workqueue("nvme"); 26559a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch if (!nvme_workq) 26569a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch goto kill_kthread; 26579a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch 26585c42ea1643a630060f9e71e06d3933d244970967Keith Busch result = register_blkdev(nvme_major, "nvme"); 26595c42ea1643a630060f9e71e06d3933d244970967Keith Busch if (result < 0) 26609a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch goto kill_workq; 26615c42ea1643a630060f9e71e06d3933d244970967Keith Busch else if (result > 0) 26620ac13140d796eb1e2f8956aea97a6e5e4ebcf981Matthew Wilcox nvme_major = result; 2663b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2664b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox result = pci_register_driver(&nvme_driver); 26651fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox if (result) 26661fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox goto unregister_blkdev; 26671fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox return 0; 2668b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 26691fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox unregister_blkdev: 2670b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unregister_blkdev(nvme_major, "nvme"); 26719a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch kill_workq: 26729a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch destroy_workqueue(nvme_workq); 26731fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox kill_kthread: 26741fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox kthread_stop(nvme_thread); 2675b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return result; 2676b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 2677b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2678b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void __exit nvme_exit(void) 2679b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 2680b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox pci_unregister_driver(&nvme_driver); 2681b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unregister_blkdev(nvme_major, "nvme"); 26829a6b94584de1a0467d85b435df9c744c5c45a270Keith Busch destroy_workqueue(nvme_workq); 26831fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox kthread_stop(nvme_thread); 2684b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 2685b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2686b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew WilcoxMODULE_AUTHOR("Matthew Wilcox <willy@linux.intel.com>"); 2687b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew WilcoxMODULE_LICENSE("GPL"); 2688366e8217e5ec6ce9f73aec19c46d983110fb4a98Matthew WilcoxMODULE_VERSION("0.8"); 2689b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxmodule_init(nvme_init); 2690b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxmodule_exit(nvme_exit); 2691