nvme-core.c revision 0a8d44cb33377969337fa6c1961b631605d5f453
1b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 2b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * NVM Express device driver 3b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Copyright (c) 2011, Intel Corporation. 4b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 5b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * This program is free software; you can redistribute it and/or modify it 6b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * under the terms and conditions of the GNU General Public License, 7b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * version 2, as published by the Free Software Foundation. 8b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 9b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * This program is distributed in the hope it will be useful, but WITHOUT 10b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * more details. 13b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 14b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * You should have received a copy of the GNU General Public License along with 15b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * this program; if not, write to the Free Software Foundation, Inc., 16b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. 17b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 18b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 19b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/nvme.h> 20b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/bio.h> 218de055350fbaa96b6563892c195a60be583faa9cMatthew Wilcox#include <linux/bitops.h> 22b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/blkdev.h> 23fd63e9ceeeae58cfe877c2d49d41c1bf7532303cMatthew Wilcox#include <linux/delay.h> 24b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/errno.h> 25b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/fs.h> 26b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/genhd.h> 275aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox#include <linux/idr.h> 28b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/init.h> 29b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/interrupt.h> 30b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/io.h> 31b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/kdev_t.h> 321fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox#include <linux/kthread.h> 33b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/kernel.h> 34b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/mm.h> 35b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/module.h> 36b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/moduleparam.h> 37b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/pci.h> 38be7b62754e097adc0cb16c25c9ee86ee20de62fbMatthew Wilcox#include <linux/poison.h> 39c3bfe7176c035a0a2c70bc79180fb13a6c57142aMatthew Wilcox#include <linux/ptrace.h> 40b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/sched.h> 41b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/slab.h> 42b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/types.h> 435d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma#include <scsi/sg.h> 44797a796a13df6b84a4791e57306737059b5b2384Hitoshi Mitake#include <asm-generic/io-64-nonatomic-lo-hi.h> 45797a796a13df6b84a4791e57306737059b5b2384Hitoshi Mitake 46b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define NVME_Q_DEPTH 1024 47b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define SQ_SIZE(depth) (depth * sizeof(struct nvme_command)) 48b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define CQ_SIZE(depth) (depth * sizeof(struct nvme_completion)) 49b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define NVME_MINORS 64 50e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox#define ADMIN_TIMEOUT (60 * HZ) 51b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 52b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_major; 53b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxmodule_param(nvme_major, int, 0); 54b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 5558ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcoxstatic int use_threaded_interrupts; 5658ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcoxmodule_param(use_threaded_interrupts, int, 0); 5758ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox 581fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcoxstatic DEFINE_SPINLOCK(dev_list_lock); 591fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcoxstatic LIST_HEAD(dev_list); 601fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcoxstatic struct task_struct *nvme_thread; 611fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 62b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 63b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * An NVM Express queue. Each device has at least two (one for admin 64b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * commands and one for I/O commands). 65b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 66b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstruct nvme_queue { 67b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct device *q_dmadev; 68091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox struct nvme_dev *dev; 69b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox spinlock_t q_lock; 70b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_command *sq_cmds; 71b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox volatile struct nvme_completion *cqes; 72b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_addr_t sq_dma_addr; 73b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_addr_t cq_dma_addr; 74b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox wait_queue_head_t sq_full; 751fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox wait_queue_t sq_cong_wait; 76b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct bio_list sq_cong; 77b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 __iomem *q_db; 78b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 q_depth; 79b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 cq_vector; 80b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 sq_head; 81b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 sq_tail; 82b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 cq_head; 83e9539f47525ecee05c9f22c3565885f3e9492c52Matthew Wilcox u8 cq_phase; 84e9539f47525ecee05c9f22c3565885f3e9492c52Matthew Wilcox u8 cqe_seen; 85224042742582c9938788b81165180c876e997a07Keith Busch u8 q_suspended; 86b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unsigned long cmdid_data[]; 87b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 88b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 89b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 90b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Check we didin't inadvertently grow the command struct 91b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 92b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic inline void _nvme_check_size(void) 93b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 94b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64); 95b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_create_cq) != 64); 96b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_create_sq) != 64); 97b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64); 98b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_features) != 64); 99f8ebf8409abfdaeeb8c847381629a2a8b8e3d816Vishal Verma BUILD_BUG_ON(sizeof(struct nvme_format_cmd) != 64); 100b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_command) != 64); 101b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != 4096); 102b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_id_ns) != 4096); 103b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64); 1046ecec74520d8a357726e6c12f99080dbe7b347ddKeith Busch BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512); 105b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 106b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1075c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcoxtypedef void (*nvme_completion_fn)(struct nvme_dev *, void *, 108c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox struct nvme_completion *); 109c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox 110e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcoxstruct nvme_cmd_info { 111c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox nvme_completion_fn fn; 112c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox void *ctx; 113e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox unsigned long timeout; 114e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox}; 115e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox 116e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcoxstatic struct nvme_cmd_info *nvme_cmd_info(struct nvme_queue *nvmeq) 117e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox{ 118e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox return (void *)&nvmeq->cmdid_data[BITS_TO_LONGS(nvmeq->q_depth)]; 119e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox} 120e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox 121224042742582c9938788b81165180c876e997a07Keith Buschstatic unsigned nvme_queue_extra(int depth) 122224042742582c9938788b81165180c876e997a07Keith Busch{ 123224042742582c9938788b81165180c876e997a07Keith Busch return DIV_ROUND_UP(depth, 8) + (depth * sizeof(struct nvme_cmd_info)); 124224042742582c9938788b81165180c876e997a07Keith Busch} 125224042742582c9938788b81165180c876e997a07Keith Busch 126b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/** 127714a7a22884b74862540bc84955274d86b2f6040Matthew Wilcox * alloc_cmdid() - Allocate a Command ID 128714a7a22884b74862540bc84955274d86b2f6040Matthew Wilcox * @nvmeq: The queue that will be used for this command 129714a7a22884b74862540bc84955274d86b2f6040Matthew Wilcox * @ctx: A pointer that will be passed to the handler 130c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox * @handler: The function to call on completion 131b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 132b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Allocate a Command ID for a queue. The data passed in will 133b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * be passed to the completion handler. This is implemented by using 134b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * the bottom two bits of the ctx pointer to store the handler ID. 135b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Passing in a pointer that's not 4-byte aligned will cause a BUG. 136b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * We can change this if it becomes a problem. 137184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox * 138184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox * May be called with local interrupts disabled and the q_lock held, 139184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox * or with interrupts enabled and no locks held. 140b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 141c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcoxstatic int alloc_cmdid(struct nvme_queue *nvmeq, void *ctx, 142c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox nvme_completion_fn handler, unsigned timeout) 143b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 144e6d15f79f997a98b3a69abbc462fc9041cc1a7b4Matthew Wilcox int depth = nvmeq->q_depth - 1; 145e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox struct nvme_cmd_info *info = nvme_cmd_info(nvmeq); 146b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int cmdid; 147b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 148b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox do { 149b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cmdid = find_first_zero_bit(nvmeq->cmdid_data, depth); 150b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (cmdid >= depth) 151b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -EBUSY; 152b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } while (test_and_set_bit(cmdid, nvmeq->cmdid_data)); 153b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 154c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox info[cmdid].fn = handler; 155c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox info[cmdid].ctx = ctx; 156e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox info[cmdid].timeout = jiffies + timeout; 157b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return cmdid; 158b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 159b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 160b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int alloc_cmdid_killable(struct nvme_queue *nvmeq, void *ctx, 161c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox nvme_completion_fn handler, unsigned timeout) 162b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 163b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int cmdid; 164b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox wait_event_killable(nvmeq->sq_full, 165e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox (cmdid = alloc_cmdid(nvmeq, ctx, handler, timeout)) >= 0); 166b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return (cmdid < 0) ? -EINTR : cmdid; 167b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 168b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 169c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox/* Special values must be less than 0x1000 */ 170c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox#define CMD_CTX_BASE ((void *)POISON_POINTER_DELTA) 171d2d8703481f60d67f49e3177196cbe474b11377cMatthew Wilcox#define CMD_CTX_CANCELLED (0x30C + CMD_CTX_BASE) 172d2d8703481f60d67f49e3177196cbe474b11377cMatthew Wilcox#define CMD_CTX_COMPLETED (0x310 + CMD_CTX_BASE) 173d2d8703481f60d67f49e3177196cbe474b11377cMatthew Wilcox#define CMD_CTX_INVALID (0x314 + CMD_CTX_BASE) 17400df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox#define CMD_CTX_FLUSH (0x318 + CMD_CTX_BASE) 175be7b62754e097adc0cb16c25c9ee86ee20de62fbMatthew Wilcox 1765c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcoxstatic void special_completion(struct nvme_dev *dev, void *ctx, 177c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox struct nvme_completion *cqe) 178c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox{ 179c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox if (ctx == CMD_CTX_CANCELLED) 180c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox return; 181c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox if (ctx == CMD_CTX_FLUSH) 182c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox return; 183c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox if (ctx == CMD_CTX_COMPLETED) { 1845c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcox dev_warn(&dev->pci_dev->dev, 185c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox "completed id %d twice on queue %d\n", 186c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox cqe->command_id, le16_to_cpup(&cqe->sq_id)); 187c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox return; 188c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox } 189c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox if (ctx == CMD_CTX_INVALID) { 1905c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcox dev_warn(&dev->pci_dev->dev, 191c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox "invalid id %d completed on queue %d\n", 192c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox cqe->command_id, le16_to_cpup(&cqe->sq_id)); 193c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox return; 194c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox } 195c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox 1965c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcox dev_warn(&dev->pci_dev->dev, "Unknown special completion %p\n", ctx); 197c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox} 198c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox 199184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox/* 200184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox * Called with local interrupts disabled and the q_lock held. May not sleep. 201184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox */ 202c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcoxstatic void *free_cmdid(struct nvme_queue *nvmeq, int cmdid, 203c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox nvme_completion_fn *fn) 204b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 205c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox void *ctx; 206e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox struct nvme_cmd_info *info = nvme_cmd_info(nvmeq); 207b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 208c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox if (cmdid >= nvmeq->q_depth) { 209c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox *fn = special_completion; 21048e3d39816416b3bf03dee3a796c0c04427c1a31Matthew Wilcox return CMD_CTX_INVALID; 211c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox } 212859361a228258edf4821d9f5635825033eca78e8Keith Busch if (fn) 213859361a228258edf4821d9f5635825033eca78e8Keith Busch *fn = info[cmdid].fn; 214c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox ctx = info[cmdid].ctx; 215c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox info[cmdid].fn = special_completion; 216e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox info[cmdid].ctx = CMD_CTX_COMPLETED; 217b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox clear_bit(cmdid, nvmeq->cmdid_data); 218b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox wake_up(&nvmeq->sq_full); 219c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox return ctx; 220b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 221b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 222c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcoxstatic void *cancel_cmdid(struct nvme_queue *nvmeq, int cmdid, 223c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox nvme_completion_fn *fn) 2243c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox{ 225c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox void *ctx; 226e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox struct nvme_cmd_info *info = nvme_cmd_info(nvmeq); 227c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox if (fn) 228c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox *fn = info[cmdid].fn; 229c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox ctx = info[cmdid].ctx; 230c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox info[cmdid].fn = special_completion; 231e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox info[cmdid].ctx = CMD_CTX_CANCELLED; 232c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox return ctx; 2333c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox} 2343c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox 2355d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermastruct nvme_queue *get_nvmeq(struct nvme_dev *dev) 236b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 237040a93b52a9eee8177ebaf2ba0ee0f9f518d1bf8Matthew Wilcox return dev->queues[get_cpu() + 1]; 238b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 239b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2405d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermavoid put_nvmeq(struct nvme_queue *nvmeq) 241b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 2421b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox put_cpu(); 243b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 244b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 245b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/** 246714a7a22884b74862540bc84955274d86b2f6040Matthew Wilcox * nvme_submit_cmd() - Copy a command into a queue and ring the doorbell 247b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * @nvmeq: The queue to use 248b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * @cmd: The command to send 249b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 250b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Safe to use from interrupt context 251b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 252b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_submit_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd) 253b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 254b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unsigned long flags; 255b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 tail; 256b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox spin_lock_irqsave(&nvmeq->q_lock, flags); 257b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox tail = nvmeq->sq_tail; 258b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memcpy(&nvmeq->sq_cmds[tail], cmd, sizeof(*cmd)); 259b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (++tail == nvmeq->q_depth) 260b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox tail = 0; 2617547881d0951384f9833ec3a80fac8f3f16f3b98Matthew Wilcox writel(tail, nvmeq->q_db); 262b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->sq_tail = tail; 263b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox spin_unlock_irqrestore(&nvmeq->q_lock, flags); 264b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 265b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 266b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 267b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 268eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcoxstatic __le64 **iod_list(struct nvme_iod *iod) 269e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews{ 270eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return ((void *)iod) + iod->offset; 271e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews} 272e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews 273eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox/* 274eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox * Will slightly overestimate the number of pages needed. This is OK 275eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox * as it only leads to a small amount of wasted memory for the lifetime of 276eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox * the I/O. 277eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox */ 278eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcoxstatic int nvme_npages(unsigned size) 279eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox{ 280eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox unsigned nprps = DIV_ROUND_UP(size + PAGE_SIZE, PAGE_SIZE); 281eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return DIV_ROUND_UP(8 * nprps, PAGE_SIZE - 8); 282eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox} 283b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 284eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcoxstatic struct nvme_iod * 285eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcoxnvme_alloc_iod(unsigned nseg, unsigned nbytes, gfp_t gfp) 286b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 287eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox struct nvme_iod *iod = kmalloc(sizeof(struct nvme_iod) + 288eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox sizeof(__le64 *) * nvme_npages(nbytes) + 289eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox sizeof(struct scatterlist) * nseg, gfp); 290eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox 291eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox if (iod) { 292eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->offset = offsetof(struct nvme_iod, sg[nseg]); 293eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->npages = -1; 294eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->length = nbytes; 2952b1960341576bf51c01b12fefeb1cc53820923e7Keith Busch iod->nents = 0; 2966198221fa0df0298513b35796f63f242ea97134eKeith Busch iod->start_time = jiffies; 297eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox } 298eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox 299eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return iod; 300b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 301b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 3025d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermavoid nvme_free_iod(struct nvme_dev *dev, struct nvme_iod *iod) 303b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 304eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox const int last_prp = PAGE_SIZE / 8 - 1; 305eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox int i; 306eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox __le64 **list = iod_list(iod); 307eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox dma_addr_t prp_dma = iod->first_dma; 308eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox 309eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox if (iod->npages == 0) 310eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox dma_pool_free(dev->prp_small_pool, list[0], prp_dma); 311eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox for (i = 0; i < iod->npages; i++) { 312eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox __le64 *prp_list = list[i]; 313eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox dma_addr_t next_prp_dma = le64_to_cpu(prp_list[last_prp]); 314eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox dma_pool_free(dev->prp_page_pool, prp_list, prp_dma); 315eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox prp_dma = next_prp_dma; 316eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox } 317eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox kfree(iod); 318b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 319b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 3206198221fa0df0298513b35796f63f242ea97134eKeith Buschstatic void nvme_start_io_acct(struct bio *bio) 3216198221fa0df0298513b35796f63f242ea97134eKeith Busch{ 3226198221fa0df0298513b35796f63f242ea97134eKeith Busch struct gendisk *disk = bio->bi_bdev->bd_disk; 3236198221fa0df0298513b35796f63f242ea97134eKeith Busch const int rw = bio_data_dir(bio); 3246198221fa0df0298513b35796f63f242ea97134eKeith Busch int cpu = part_stat_lock(); 3256198221fa0df0298513b35796f63f242ea97134eKeith Busch part_round_stats(cpu, &disk->part0); 3266198221fa0df0298513b35796f63f242ea97134eKeith Busch part_stat_inc(cpu, &disk->part0, ios[rw]); 3276198221fa0df0298513b35796f63f242ea97134eKeith Busch part_stat_add(cpu, &disk->part0, sectors[rw], bio_sectors(bio)); 3286198221fa0df0298513b35796f63f242ea97134eKeith Busch part_inc_in_flight(&disk->part0, rw); 3296198221fa0df0298513b35796f63f242ea97134eKeith Busch part_stat_unlock(); 3306198221fa0df0298513b35796f63f242ea97134eKeith Busch} 3316198221fa0df0298513b35796f63f242ea97134eKeith Busch 3326198221fa0df0298513b35796f63f242ea97134eKeith Buschstatic void nvme_end_io_acct(struct bio *bio, unsigned long start_time) 3336198221fa0df0298513b35796f63f242ea97134eKeith Busch{ 3346198221fa0df0298513b35796f63f242ea97134eKeith Busch struct gendisk *disk = bio->bi_bdev->bd_disk; 3356198221fa0df0298513b35796f63f242ea97134eKeith Busch const int rw = bio_data_dir(bio); 3366198221fa0df0298513b35796f63f242ea97134eKeith Busch unsigned long duration = jiffies - start_time; 3376198221fa0df0298513b35796f63f242ea97134eKeith Busch int cpu = part_stat_lock(); 3386198221fa0df0298513b35796f63f242ea97134eKeith Busch part_stat_add(cpu, &disk->part0, ticks[rw], duration); 3396198221fa0df0298513b35796f63f242ea97134eKeith Busch part_round_stats(cpu, &disk->part0); 3406198221fa0df0298513b35796f63f242ea97134eKeith Busch part_dec_in_flight(&disk->part0, rw); 3416198221fa0df0298513b35796f63f242ea97134eKeith Busch part_stat_unlock(); 3426198221fa0df0298513b35796f63f242ea97134eKeith Busch} 3436198221fa0df0298513b35796f63f242ea97134eKeith Busch 3445c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcoxstatic void bio_completion(struct nvme_dev *dev, void *ctx, 345b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_completion *cqe) 346b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 347eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox struct nvme_iod *iod = ctx; 348eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox struct bio *bio = iod->private; 349b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 status = le16_to_cpup(&cqe->status) >> 1; 350b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 3519e59d091b0eb04f223ed037348e3d9e36f30e72bKeith Busch if (iod->nents) { 3522b1960341576bf51c01b12fefeb1cc53820923e7Keith Busch dma_unmap_sg(&dev->pci_dev->dev, iod->sg, iod->nents, 353b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox bio_data_dir(bio) ? DMA_TO_DEVICE : DMA_FROM_DEVICE); 3549e59d091b0eb04f223ed037348e3d9e36f30e72bKeith Busch nvme_end_io_acct(bio, iod->start_time); 3559e59d091b0eb04f223ed037348e3d9e36f30e72bKeith Busch } 356eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox nvme_free_iod(dev, iod); 357427e97080196548557b288517537ab7eb48c309fKeith Busch if (status) 3581ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox bio_endio(bio, -EIO); 359427e97080196548557b288517537ab7eb48c309fKeith Busch else 3601ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox bio_endio(bio, 0); 361b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 362b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 363184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox/* length is in bytes. gfp flags indicates whether we may sleep. */ 3645d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermaint nvme_setup_prps(struct nvme_dev *dev, struct nvme_common_command *cmd, 3655d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma struct nvme_iod *iod, int total_len, gfp_t gfp) 366ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox{ 36799802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox struct dma_pool *pool; 368eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox int length = total_len; 369eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox struct scatterlist *sg = iod->sg; 370ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox int dma_len = sg_dma_len(sg); 371ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox u64 dma_addr = sg_dma_address(sg); 372ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox int offset = offset_in_page(dma_addr); 373e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews __le64 *prp_list; 374eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox __le64 **list = iod_list(iod); 375e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_addr_t prp_dma; 376eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox int nprps, i; 377ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox 378ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmd->prp1 = cpu_to_le64(dma_addr); 379ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox length -= (PAGE_SIZE - offset); 380ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox if (length <= 0) 381eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return total_len; 382ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox 383ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox dma_len -= (PAGE_SIZE - offset); 384ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox if (dma_len) { 385ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox dma_addr += (PAGE_SIZE - offset); 386ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox } else { 387ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox sg = sg_next(sg); 388ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox dma_addr = sg_dma_address(sg); 389ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox dma_len = sg_dma_len(sg); 390ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox } 391ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox 392ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox if (length <= PAGE_SIZE) { 393ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmd->prp2 = cpu_to_le64(dma_addr); 394eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return total_len; 395e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews } 396e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews 397e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews nprps = DIV_ROUND_UP(length, PAGE_SIZE); 39899802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox if (nprps <= (256 / 8)) { 39999802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox pool = dev->prp_small_pool; 400eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->npages = 0; 40199802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox } else { 40299802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox pool = dev->prp_page_pool; 403eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->npages = 1; 40499802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox } 40599802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox 406b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox prp_list = dma_pool_alloc(pool, gfp, &prp_dma); 407b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox if (!prp_list) { 408b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox cmd->prp2 = cpu_to_le64(dma_addr); 409eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->npages = -1; 410eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return (total_len - length) + PAGE_SIZE; 411b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox } 412eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox list[0] = prp_list; 413eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->first_dma = prp_dma; 414e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews cmd->prp2 = cpu_to_le64(prp_dma); 415e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews i = 0; 416e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews for (;;) { 4177523d834dd1573610078eb1ac0933f6490232f90Matthew Wilcox if (i == PAGE_SIZE / 8) { 418e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews __le64 *old_prp_list = prp_list; 419b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox prp_list = dma_pool_alloc(pool, gfp, &prp_dma); 420eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox if (!prp_list) 421eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return total_len - length; 422eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox list[iod->npages++] = prp_list; 4237523d834dd1573610078eb1ac0933f6490232f90Matthew Wilcox prp_list[0] = old_prp_list[i - 1]; 4247523d834dd1573610078eb1ac0933f6490232f90Matthew Wilcox old_prp_list[i - 1] = cpu_to_le64(prp_dma); 4257523d834dd1573610078eb1ac0933f6490232f90Matthew Wilcox i = 1; 426e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews } 427e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews prp_list[i++] = cpu_to_le64(dma_addr); 428e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_len -= PAGE_SIZE; 429e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_addr += PAGE_SIZE; 430e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews length -= PAGE_SIZE; 431e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews if (length <= 0) 432e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews break; 433e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews if (dma_len > 0) 434e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews continue; 435e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews BUG_ON(dma_len < 0); 436e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews sg = sg_next(sg); 437e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_addr = sg_dma_address(sg); 438e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_len = sg_dma_len(sg); 439ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox } 440ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox 441eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return total_len; 442ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox} 443ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox 444427e97080196548557b288517537ab7eb48c309fKeith Buschstruct nvme_bio_pair { 445427e97080196548557b288517537ab7eb48c309fKeith Busch struct bio b1, b2, *parent; 446427e97080196548557b288517537ab7eb48c309fKeith Busch struct bio_vec *bv1, *bv2; 447427e97080196548557b288517537ab7eb48c309fKeith Busch int err; 448427e97080196548557b288517537ab7eb48c309fKeith Busch atomic_t cnt; 449427e97080196548557b288517537ab7eb48c309fKeith Busch}; 450427e97080196548557b288517537ab7eb48c309fKeith Busch 451427e97080196548557b288517537ab7eb48c309fKeith Buschstatic void nvme_bio_pair_endio(struct bio *bio, int err) 452427e97080196548557b288517537ab7eb48c309fKeith Busch{ 453427e97080196548557b288517537ab7eb48c309fKeith Busch struct nvme_bio_pair *bp = bio->bi_private; 454427e97080196548557b288517537ab7eb48c309fKeith Busch 455427e97080196548557b288517537ab7eb48c309fKeith Busch if (err) 456427e97080196548557b288517537ab7eb48c309fKeith Busch bp->err = err; 457427e97080196548557b288517537ab7eb48c309fKeith Busch 458427e97080196548557b288517537ab7eb48c309fKeith Busch if (atomic_dec_and_test(&bp->cnt)) { 459427e97080196548557b288517537ab7eb48c309fKeith Busch bio_endio(bp->parent, bp->err); 4601b56749e541ad59068582f2a28297843e243b856Keith Busch kfree(bp->bv1); 4611b56749e541ad59068582f2a28297843e243b856Keith Busch kfree(bp->bv2); 462427e97080196548557b288517537ab7eb48c309fKeith Busch kfree(bp); 463427e97080196548557b288517537ab7eb48c309fKeith Busch } 464427e97080196548557b288517537ab7eb48c309fKeith Busch} 465427e97080196548557b288517537ab7eb48c309fKeith Busch 466427e97080196548557b288517537ab7eb48c309fKeith Buschstatic struct nvme_bio_pair *nvme_bio_split(struct bio *bio, int idx, 467427e97080196548557b288517537ab7eb48c309fKeith Busch int len, int offset) 468427e97080196548557b288517537ab7eb48c309fKeith Busch{ 469427e97080196548557b288517537ab7eb48c309fKeith Busch struct nvme_bio_pair *bp; 470427e97080196548557b288517537ab7eb48c309fKeith Busch 471427e97080196548557b288517537ab7eb48c309fKeith Busch BUG_ON(len > bio->bi_size); 472427e97080196548557b288517537ab7eb48c309fKeith Busch BUG_ON(idx > bio->bi_vcnt); 473427e97080196548557b288517537ab7eb48c309fKeith Busch 474427e97080196548557b288517537ab7eb48c309fKeith Busch bp = kmalloc(sizeof(*bp), GFP_ATOMIC); 475427e97080196548557b288517537ab7eb48c309fKeith Busch if (!bp) 476427e97080196548557b288517537ab7eb48c309fKeith Busch return NULL; 477427e97080196548557b288517537ab7eb48c309fKeith Busch bp->err = 0; 478427e97080196548557b288517537ab7eb48c309fKeith Busch 479427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b1 = *bio; 480427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b2 = *bio; 481427e97080196548557b288517537ab7eb48c309fKeith Busch 482427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b1.bi_size = len; 483427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b2.bi_size -= len; 484427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b1.bi_vcnt = idx; 485427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b2.bi_idx = idx; 486427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b2.bi_sector += len >> 9; 487427e97080196548557b288517537ab7eb48c309fKeith Busch 488427e97080196548557b288517537ab7eb48c309fKeith Busch if (offset) { 489427e97080196548557b288517537ab7eb48c309fKeith Busch bp->bv1 = kmalloc(bio->bi_max_vecs * sizeof(struct bio_vec), 490427e97080196548557b288517537ab7eb48c309fKeith Busch GFP_ATOMIC); 491427e97080196548557b288517537ab7eb48c309fKeith Busch if (!bp->bv1) 492427e97080196548557b288517537ab7eb48c309fKeith Busch goto split_fail_1; 493427e97080196548557b288517537ab7eb48c309fKeith Busch 494427e97080196548557b288517537ab7eb48c309fKeith Busch bp->bv2 = kmalloc(bio->bi_max_vecs * sizeof(struct bio_vec), 495427e97080196548557b288517537ab7eb48c309fKeith Busch GFP_ATOMIC); 496427e97080196548557b288517537ab7eb48c309fKeith Busch if (!bp->bv2) 497427e97080196548557b288517537ab7eb48c309fKeith Busch goto split_fail_2; 498427e97080196548557b288517537ab7eb48c309fKeith Busch 499427e97080196548557b288517537ab7eb48c309fKeith Busch memcpy(bp->bv1, bio->bi_io_vec, 500427e97080196548557b288517537ab7eb48c309fKeith Busch bio->bi_max_vecs * sizeof(struct bio_vec)); 501427e97080196548557b288517537ab7eb48c309fKeith Busch memcpy(bp->bv2, bio->bi_io_vec, 502427e97080196548557b288517537ab7eb48c309fKeith Busch bio->bi_max_vecs * sizeof(struct bio_vec)); 503427e97080196548557b288517537ab7eb48c309fKeith Busch 504427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b1.bi_io_vec = bp->bv1; 505427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b2.bi_io_vec = bp->bv2; 506427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b2.bi_io_vec[idx].bv_offset += offset; 507427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b2.bi_io_vec[idx].bv_len -= offset; 508427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b1.bi_io_vec[idx].bv_len = offset; 509427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b1.bi_vcnt++; 510427e97080196548557b288517537ab7eb48c309fKeith Busch } else 511427e97080196548557b288517537ab7eb48c309fKeith Busch bp->bv1 = bp->bv2 = NULL; 512427e97080196548557b288517537ab7eb48c309fKeith Busch 513427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b1.bi_private = bp; 514427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b2.bi_private = bp; 515427e97080196548557b288517537ab7eb48c309fKeith Busch 516427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b1.bi_end_io = nvme_bio_pair_endio; 517427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b2.bi_end_io = nvme_bio_pair_endio; 518427e97080196548557b288517537ab7eb48c309fKeith Busch 519427e97080196548557b288517537ab7eb48c309fKeith Busch bp->parent = bio; 520427e97080196548557b288517537ab7eb48c309fKeith Busch atomic_set(&bp->cnt, 2); 521427e97080196548557b288517537ab7eb48c309fKeith Busch 522427e97080196548557b288517537ab7eb48c309fKeith Busch return bp; 523427e97080196548557b288517537ab7eb48c309fKeith Busch 524427e97080196548557b288517537ab7eb48c309fKeith Busch split_fail_2: 525427e97080196548557b288517537ab7eb48c309fKeith Busch kfree(bp->bv1); 526427e97080196548557b288517537ab7eb48c309fKeith Busch split_fail_1: 527427e97080196548557b288517537ab7eb48c309fKeith Busch kfree(bp); 528427e97080196548557b288517537ab7eb48c309fKeith Busch return NULL; 529427e97080196548557b288517537ab7eb48c309fKeith Busch} 530427e97080196548557b288517537ab7eb48c309fKeith Busch 531427e97080196548557b288517537ab7eb48c309fKeith Buschstatic int nvme_split_and_submit(struct bio *bio, struct nvme_queue *nvmeq, 532427e97080196548557b288517537ab7eb48c309fKeith Busch int idx, int len, int offset) 533427e97080196548557b288517537ab7eb48c309fKeith Busch{ 534427e97080196548557b288517537ab7eb48c309fKeith Busch struct nvme_bio_pair *bp = nvme_bio_split(bio, idx, len, offset); 535427e97080196548557b288517537ab7eb48c309fKeith Busch if (!bp) 536427e97080196548557b288517537ab7eb48c309fKeith Busch return -ENOMEM; 537427e97080196548557b288517537ab7eb48c309fKeith Busch 538427e97080196548557b288517537ab7eb48c309fKeith Busch if (bio_list_empty(&nvmeq->sq_cong)) 539427e97080196548557b288517537ab7eb48c309fKeith Busch add_wait_queue(&nvmeq->sq_full, &nvmeq->sq_cong_wait); 540427e97080196548557b288517537ab7eb48c309fKeith Busch bio_list_add(&nvmeq->sq_cong, &bp->b1); 541427e97080196548557b288517537ab7eb48c309fKeith Busch bio_list_add(&nvmeq->sq_cong, &bp->b2); 542427e97080196548557b288517537ab7eb48c309fKeith Busch 543427e97080196548557b288517537ab7eb48c309fKeith Busch return 0; 544427e97080196548557b288517537ab7eb48c309fKeith Busch} 545427e97080196548557b288517537ab7eb48c309fKeith Busch 5461ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox/* NVMe scatterlists require no holes in the virtual address */ 5471ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox#define BIOVEC_NOT_VIRT_MERGEABLE(vec1, vec2) ((vec2)->bv_offset || \ 5481ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox (((vec1)->bv_offset + (vec1)->bv_len) % PAGE_SIZE)) 5491ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox 550427e97080196548557b288517537ab7eb48c309fKeith Buschstatic int nvme_map_bio(struct nvme_queue *nvmeq, struct nvme_iod *iod, 551b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct bio *bio, enum dma_data_direction dma_dir, int psegs) 552b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 553768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox struct bio_vec *bvec, *bvprv = NULL; 554768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox struct scatterlist *sg = NULL; 555159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch int i, length = 0, nsegs = 0, split_len = bio->bi_size; 556159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch 557159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch if (nvmeq->dev->stripe_size) 558159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch split_len = nvmeq->dev->stripe_size - 559159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch ((bio->bi_sector << 9) & (nvmeq->dev->stripe_size - 1)); 560b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 561eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox sg_init_table(iod->sg, psegs); 562b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox bio_for_each_segment(bvec, bio, i) { 563768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox if (bvprv && BIOVEC_PHYS_MERGEABLE(bvprv, bvec)) { 564768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox sg->length += bvec->bv_len; 565768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox } else { 5661ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox if (bvprv && BIOVEC_NOT_VIRT_MERGEABLE(bvprv, bvec)) 567427e97080196548557b288517537ab7eb48c309fKeith Busch return nvme_split_and_submit(bio, nvmeq, i, 568427e97080196548557b288517537ab7eb48c309fKeith Busch length, 0); 569427e97080196548557b288517537ab7eb48c309fKeith Busch 570eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox sg = sg ? sg + 1 : iod->sg; 571768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox sg_set_page(sg, bvec->bv_page, bvec->bv_len, 572768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox bvec->bv_offset); 573768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox nsegs++; 574768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox } 575159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch 576159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch if (split_len - length < bvec->bv_len) 577159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch return nvme_split_and_submit(bio, nvmeq, i, split_len, 578159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch split_len - length); 5791ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox length += bvec->bv_len; 580768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox bvprv = bvec; 581b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 582eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->nents = nsegs; 583768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox sg_mark_end(sg); 584427e97080196548557b288517537ab7eb48c309fKeith Busch if (dma_map_sg(nvmeq->q_dmadev, iod->sg, iod->nents, dma_dir) == 0) 5851ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox return -ENOMEM; 586427e97080196548557b288517537ab7eb48c309fKeith Busch 587159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch BUG_ON(length != bio->bi_size); 5881ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox return length; 589b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 590b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 5910e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch/* 5920e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch * We reuse the small pool to allocate the 16-byte range here as it is not 5930e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch * worth having a special pool for these or additional cases to handle freeing 5940e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch * the iod. 5950e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch */ 5960e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Buschstatic int nvme_submit_discard(struct nvme_queue *nvmeq, struct nvme_ns *ns, 5970e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch struct bio *bio, struct nvme_iod *iod, int cmdid) 5980e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch{ 5990e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch struct nvme_dsm_range *range; 6000e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch struct nvme_command *cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail]; 6010e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch 6020e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch range = dma_pool_alloc(nvmeq->dev->prp_small_pool, GFP_ATOMIC, 6030e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch &iod->first_dma); 6040e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch if (!range) 6050e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch return -ENOMEM; 6060e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch 6070e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch iod_list(iod)[0] = (__le64 *)range; 6080e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch iod->npages = 0; 6090e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch 6100e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch range->cattr = cpu_to_le32(0); 6110e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch range->nlb = cpu_to_le32(bio->bi_size >> ns->lba_shift); 612063cc6d5591ea9c0631b81ac5c7b829d99738b2fMatthew Wilcox range->slba = cpu_to_le64(nvme_block_nr(ns, bio->bi_sector)); 6130e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch 6140e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch memset(cmnd, 0, sizeof(*cmnd)); 6150e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch cmnd->dsm.opcode = nvme_cmd_dsm; 6160e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch cmnd->dsm.command_id = cmdid; 6170e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch cmnd->dsm.nsid = cpu_to_le32(ns->ns_id); 6180e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch cmnd->dsm.prp1 = cpu_to_le64(iod->first_dma); 6190e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch cmnd->dsm.nr = 0; 6200e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD); 6210e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch 6220e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch if (++nvmeq->sq_tail == nvmeq->q_depth) 6230e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch nvmeq->sq_tail = 0; 6240e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch writel(nvmeq->sq_tail, nvmeq->q_db); 6250e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch 6260e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch return 0; 6270e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch} 6280e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch 62900df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcoxstatic int nvme_submit_flush(struct nvme_queue *nvmeq, struct nvme_ns *ns, 63000df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox int cmdid) 63100df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox{ 63200df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox struct nvme_command *cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail]; 63300df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 63400df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox memset(cmnd, 0, sizeof(*cmnd)); 63500df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox cmnd->common.opcode = nvme_cmd_flush; 63600df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox cmnd->common.command_id = cmdid; 63700df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox cmnd->common.nsid = cpu_to_le32(ns->ns_id); 63800df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 63900df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox if (++nvmeq->sq_tail == nvmeq->q_depth) 64000df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox nvmeq->sq_tail = 0; 64100df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox writel(nvmeq->sq_tail, nvmeq->q_db); 64200df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 64300df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox return 0; 64400df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox} 64500df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 6465d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermaint nvme_submit_flush_data(struct nvme_queue *nvmeq, struct nvme_ns *ns) 64700df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox{ 64800df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox int cmdid = alloc_cmdid(nvmeq, (void *)CMD_CTX_FLUSH, 649ff976d724a74e4522e9ca2de1fb37ac4520f454fMatthew Wilcox special_completion, NVME_IO_TIMEOUT); 65000df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox if (unlikely(cmdid < 0)) 65100df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox return cmdid; 65200df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 65300df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox return nvme_submit_flush(nvmeq, ns, cmdid); 65400df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox} 65500df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 656184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox/* 657184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox * Called with local interrupts disabled and the q_lock held. May not sleep. 658184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox */ 659b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_submit_bio_queue(struct nvme_queue *nvmeq, struct nvme_ns *ns, 660b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct bio *bio) 661b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 662ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox struct nvme_command *cmnd; 663eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox struct nvme_iod *iod; 664b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox enum dma_data_direction dma_dir; 6651287dabd345f447bbe0f7a99fc95ab89bcfc0f5dWei Yongjun int cmdid, length, result; 666b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 control; 667b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 dsmgmt; 668b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int psegs = bio_phys_segments(ns->queue, bio); 669b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 67000df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox if ((bio->bi_rw & REQ_FLUSH) && psegs) { 67100df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox result = nvme_submit_flush_data(nvmeq, ns); 67200df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox if (result) 67300df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox return result; 67400df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox } 67500df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 6761287dabd345f447bbe0f7a99fc95ab89bcfc0f5dWei Yongjun result = -ENOMEM; 677eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod = nvme_alloc_iod(psegs, bio->bi_size, GFP_ATOMIC); 678eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox if (!iod) 679eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox goto nomem; 680eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->private = bio; 681b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 682eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox result = -EBUSY; 683ff976d724a74e4522e9ca2de1fb37ac4520f454fMatthew Wilcox cmdid = alloc_cmdid(nvmeq, iod, bio_completion, NVME_IO_TIMEOUT); 684b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (unlikely(cmdid < 0)) 685eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox goto free_iod; 686b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 6870e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch if (bio->bi_rw & REQ_DISCARD) { 6880e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch result = nvme_submit_discard(nvmeq, ns, bio, iod, cmdid); 6890e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch if (result) 6900e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch goto free_cmdid; 6910e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch return result; 6920e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch } 69300df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox if ((bio->bi_rw & REQ_FLUSH) && !psegs) 69400df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox return nvme_submit_flush(nvmeq, ns, cmdid); 69500df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 696b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox control = 0; 697b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (bio->bi_rw & REQ_FUA) 698b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox control |= NVME_RW_FUA; 699b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (bio->bi_rw & (REQ_FAILFAST_DEV | REQ_RAHEAD)) 700b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox control |= NVME_RW_LR; 701b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 702b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dsmgmt = 0; 703b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (bio->bi_rw & REQ_RAHEAD) 704b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH; 705b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 706ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail]; 707b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 708b8deb62cf271fa9381edc8cf52bcae2f0225c55aMatthew Wilcox memset(cmnd, 0, sizeof(*cmnd)); 709b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (bio_data_dir(bio)) { 710ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.opcode = nvme_cmd_write; 711b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_dir = DMA_TO_DEVICE; 712b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } else { 713ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.opcode = nvme_cmd_read; 714b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_dir = DMA_FROM_DEVICE; 715b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 716b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 717427e97080196548557b288517537ab7eb48c309fKeith Busch result = nvme_map_bio(nvmeq, iod, bio, dma_dir, psegs); 718427e97080196548557b288517537ab7eb48c309fKeith Busch if (result <= 0) 719859361a228258edf4821d9f5635825033eca78e8Keith Busch goto free_cmdid; 7201ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox length = result; 721b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 722ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.command_id = cmdid; 723ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.nsid = cpu_to_le32(ns->ns_id); 724eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox length = nvme_setup_prps(nvmeq->dev, &cmnd->common, iod, length, 725eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox GFP_ATOMIC); 726063cc6d5591ea9c0631b81ac5c7b829d99738b2fMatthew Wilcox cmnd->rw.slba = cpu_to_le64(nvme_block_nr(ns, bio->bi_sector)); 7271ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox cmnd->rw.length = cpu_to_le16((length >> ns->lba_shift) - 1); 728ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.control = cpu_to_le16(control); 729ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt); 730b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 7316198221fa0df0298513b35796f63f242ea97134eKeith Busch nvme_start_io_acct(bio); 732b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (++nvmeq->sq_tail == nvmeq->q_depth) 733b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->sq_tail = 0; 7347547881d0951384f9833ec3a80fac8f3f16f3b98Matthew Wilcox writel(nvmeq->sq_tail, nvmeq->q_db); 735b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 7361974b1ae8852324a75fb8cfecbc7b758fd5a2c3cMatthew Wilcox return 0; 7371974b1ae8852324a75fb8cfecbc7b758fd5a2c3cMatthew Wilcox 738859361a228258edf4821d9f5635825033eca78e8Keith Busch free_cmdid: 739859361a228258edf4821d9f5635825033eca78e8Keith Busch free_cmdid(nvmeq, cmdid, NULL); 740eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox free_iod: 741eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox nvme_free_iod(nvmeq->dev, iod); 742eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox nomem: 743eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox return result; 744b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 745b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 746e9539f47525ecee05c9f22c3565885f3e9492c52Matthew Wilcoxstatic int nvme_process_cq(struct nvme_queue *nvmeq) 747b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 748821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox u16 head, phase; 749b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 750b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox head = nvmeq->cq_head; 751821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox phase = nvmeq->cq_phase; 752b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 753b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox for (;;) { 754c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox void *ctx; 755c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox nvme_completion_fn fn; 756b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_completion cqe = nvmeq->cqes[head]; 757821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox if ((le16_to_cpu(cqe.status) & 1) != phase) 758b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox break; 759b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->sq_head = le16_to_cpu(cqe.sq_head); 760b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (++head == nvmeq->q_depth) { 761b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox head = 0; 762821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox phase = !phase; 763b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 764b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 765c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox ctx = free_cmdid(nvmeq, cqe.command_id, &fn); 7665c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcox fn(nvmeq->dev, ctx, &cqe); 767b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 768b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 769b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox /* If the controller ignores the cq head doorbell and continuously 770b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * writes to the queue, it is theoretically possible to wrap around 771b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * the queue twice and mistakenly return IRQ_NONE. Linux only 772b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * requires that 0.1% of your interrupts are handled, so this isn't 773b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * a big problem. 774b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 775821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox if (head == nvmeq->cq_head && phase == nvmeq->cq_phase) 776e9539f47525ecee05c9f22c3565885f3e9492c52Matthew Wilcox return 0; 777b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 778b80d5ccca3a012e91ca64a2a0b13049163a6a698Haiyan Hu writel(head, nvmeq->q_db + nvmeq->dev->db_stride); 779b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->cq_head = head; 780821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox nvmeq->cq_phase = phase; 781b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 782e9539f47525ecee05c9f22c3565885f3e9492c52Matthew Wilcox nvmeq->cqe_seen = 1; 783e9539f47525ecee05c9f22c3565885f3e9492c52Matthew Wilcox return 1; 784b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 785b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 7867d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcoxstatic void nvme_make_request(struct request_queue *q, struct bio *bio) 7877d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox{ 7887d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox struct nvme_ns *ns = q->queuedata; 7897d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox struct nvme_queue *nvmeq = get_nvmeq(ns->dev); 7907d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox int result = -EBUSY; 7917d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox 792cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch if (!nvmeq) { 793cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch put_nvmeq(NULL); 794cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch bio_endio(bio, -EIO); 795cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch return; 796cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch } 797cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch 7987d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox spin_lock_irq(&nvmeq->q_lock); 799224042742582c9938788b81165180c876e997a07Keith Busch if (!nvmeq->q_suspended && bio_list_empty(&nvmeq->sq_cong)) 8007d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox result = nvme_submit_bio_queue(nvmeq, ns, bio); 8017d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox if (unlikely(result)) { 8027d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox if (bio_list_empty(&nvmeq->sq_cong)) 8037d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox add_wait_queue(&nvmeq->sq_full, &nvmeq->sq_cong_wait); 8047d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox bio_list_add(&nvmeq->sq_cong, bio); 8057d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox } 8067d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox 8077d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox nvme_process_cq(nvmeq); 8087d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox spin_unlock_irq(&nvmeq->q_lock); 8097d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox put_nvmeq(nvmeq); 8107d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox} 8117d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox 812b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic irqreturn_t nvme_irq(int irq, void *data) 813b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 81458ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox irqreturn_t result; 81558ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox struct nvme_queue *nvmeq = data; 81658ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox spin_lock(&nvmeq->q_lock); 817e9539f47525ecee05c9f22c3565885f3e9492c52Matthew Wilcox nvme_process_cq(nvmeq); 818e9539f47525ecee05c9f22c3565885f3e9492c52Matthew Wilcox result = nvmeq->cqe_seen ? IRQ_HANDLED : IRQ_NONE; 819e9539f47525ecee05c9f22c3565885f3e9492c52Matthew Wilcox nvmeq->cqe_seen = 0; 82058ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox spin_unlock(&nvmeq->q_lock); 82158ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox return result; 82258ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox} 82358ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox 82458ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcoxstatic irqreturn_t nvme_irq_check(int irq, void *data) 82558ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox{ 82658ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox struct nvme_queue *nvmeq = data; 82758ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox struct nvme_completion cqe = nvmeq->cqes[nvmeq->cq_head]; 82858ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox if ((le16_to_cpu(cqe.status) & 1) != nvmeq->cq_phase) 82958ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox return IRQ_NONE; 83058ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox return IRQ_WAKE_THREAD; 83158ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox} 83258ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox 8333c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcoxstatic void nvme_abort_command(struct nvme_queue *nvmeq, int cmdid) 8343c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox{ 8353c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox spin_lock_irq(&nvmeq->q_lock); 836c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox cancel_cmdid(nvmeq, cmdid, NULL); 8373c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox spin_unlock_irq(&nvmeq->q_lock); 8383c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox} 8393c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox 840c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcoxstruct sync_cmd_info { 841c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox struct task_struct *task; 842c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox u32 result; 843c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox int status; 844c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox}; 845c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox 8465c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcoxstatic void sync_completion(struct nvme_dev *dev, void *ctx, 847c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox struct nvme_completion *cqe) 848c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox{ 849c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox struct sync_cmd_info *cmdinfo = ctx; 850c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox cmdinfo->result = le32_to_cpup(&cqe->result); 851c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox cmdinfo->status = le16_to_cpup(&cqe->status) >> 1; 852c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox wake_up_process(cmdinfo->task); 853c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox} 854c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox 855b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 856b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Returns 0 on success. If the result is negative, it's a Linux error code; 857b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * if the result is positive, it's an NVM Express status code 858b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 8595d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermaint nvme_submit_sync_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd, 8605d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma u32 *result, unsigned timeout) 861b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 862b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int cmdid; 863b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct sync_cmd_info cmdinfo; 864b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 865b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cmdinfo.task = current; 866b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cmdinfo.status = -EINTR; 867b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 868c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox cmdid = alloc_cmdid_killable(nvmeq, &cmdinfo, sync_completion, 869e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox timeout); 870b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (cmdid < 0) 871b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return cmdid; 872b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cmd->common.command_id = cmdid; 873b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 8743c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox set_current_state(TASK_KILLABLE); 8753c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox nvme_submit_cmd(nvmeq, cmd); 87678f8d2577bd79ce9d62f4d9e6d3b895bd1dd1d1dKeith Busch schedule_timeout(timeout); 877b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 8783c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox if (cmdinfo.status == -EINTR) { 8793c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox nvme_abort_command(nvmeq, cmdid); 8803c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox return -EINTR; 8813c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox } 8823c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox 883b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (result) 884b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox *result = cmdinfo.result; 885b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 886b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return cmdinfo.status; 887b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 888b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 8895d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermaint nvme_submit_admin_cmd(struct nvme_dev *dev, struct nvme_command *cmd, 890b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 *result) 891b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 892e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox return nvme_submit_sync_cmd(dev->queues[0], cmd, result, ADMIN_TIMEOUT); 893b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 894b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 895b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id) 896b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 897b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int status; 898b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_command c; 899b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 900b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset(&c, 0, sizeof(c)); 901b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.delete_queue.opcode = opcode; 902b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.delete_queue.qid = cpu_to_le16(id); 903b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 904b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox status = nvme_submit_admin_cmd(dev, &c, NULL); 905b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (status) 906b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -EIO; 907b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 908b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 909b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 910b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_alloc_cq(struct nvme_dev *dev, u16 qid, 911b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq) 912b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 913b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int status; 914b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_command c; 915b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int flags = NVME_QUEUE_PHYS_CONTIG | NVME_CQ_IRQ_ENABLED; 916b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 917b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset(&c, 0, sizeof(c)); 918b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.opcode = nvme_admin_create_cq; 919b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.prp1 = cpu_to_le64(nvmeq->cq_dma_addr); 920b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.cqid = cpu_to_le16(qid); 921b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1); 922b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.cq_flags = cpu_to_le16(flags); 923b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.irq_vector = cpu_to_le16(nvmeq->cq_vector); 924b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 925b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox status = nvme_submit_admin_cmd(dev, &c, NULL); 926b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (status) 927b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -EIO; 928b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 929b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 930b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 931b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_alloc_sq(struct nvme_dev *dev, u16 qid, 932b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq) 933b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 934b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int status; 935b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_command c; 936b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int flags = NVME_QUEUE_PHYS_CONTIG | NVME_SQ_PRIO_MEDIUM; 937b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 938b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset(&c, 0, sizeof(c)); 939b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.opcode = nvme_admin_create_sq; 940b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.prp1 = cpu_to_le64(nvmeq->sq_dma_addr); 941b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.sqid = cpu_to_le16(qid); 942b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1); 943b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.sq_flags = cpu_to_le16(flags); 944b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.cqid = cpu_to_le16(qid); 945b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 946b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox status = nvme_submit_admin_cmd(dev, &c, NULL); 947b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (status) 948b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -EIO; 949b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 950b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 951b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 952b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_delete_cq(struct nvme_dev *dev, u16 cqid) 953b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 954b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return adapter_delete_queue(dev, nvme_admin_delete_cq, cqid); 955b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 956b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 957b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_delete_sq(struct nvme_dev *dev, u16 sqid) 958b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 959b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return adapter_delete_queue(dev, nvme_admin_delete_sq, sqid); 960b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 961b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 9625d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermaint nvme_identify(struct nvme_dev *dev, unsigned nsid, unsigned cns, 963bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox dma_addr_t dma_addr) 964bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox{ 965bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox struct nvme_command c; 966bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox 967bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox memset(&c, 0, sizeof(c)); 968bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox c.identify.opcode = nvme_admin_identify; 969bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox c.identify.nsid = cpu_to_le32(nsid); 970bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox c.identify.prp1 = cpu_to_le64(dma_addr); 971bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox c.identify.cns = cpu_to_le32(cns); 972bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox 973bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox return nvme_submit_admin_cmd(dev, &c, NULL); 974bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox} 975bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox 9765d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermaint nvme_get_features(struct nvme_dev *dev, unsigned fid, unsigned nsid, 97708df1e05657fc6712e520e7c09cc6c86160ceb35Keith Busch dma_addr_t dma_addr, u32 *result) 978bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox{ 979bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox struct nvme_command c; 980bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox 981bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox memset(&c, 0, sizeof(c)); 982bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox c.features.opcode = nvme_admin_get_features; 983a42ceccef0c43b46ff6bc1b12a7c1076ef243df1Keith Busch c.features.nsid = cpu_to_le32(nsid); 984bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox c.features.prp1 = cpu_to_le64(dma_addr); 985bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox c.features.fid = cpu_to_le32(fid); 986bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox 98708df1e05657fc6712e520e7c09cc6c86160ceb35Keith Busch return nvme_submit_admin_cmd(dev, &c, result); 988df3481399042200792822b6243e36a95a557b57eMatthew Wilcox} 989df3481399042200792822b6243e36a95a557b57eMatthew Wilcox 9905d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermaint nvme_set_features(struct nvme_dev *dev, unsigned fid, unsigned dword11, 9915d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma dma_addr_t dma_addr, u32 *result) 992df3481399042200792822b6243e36a95a557b57eMatthew Wilcox{ 993df3481399042200792822b6243e36a95a557b57eMatthew Wilcox struct nvme_command c; 994df3481399042200792822b6243e36a95a557b57eMatthew Wilcox 995df3481399042200792822b6243e36a95a557b57eMatthew Wilcox memset(&c, 0, sizeof(c)); 996df3481399042200792822b6243e36a95a557b57eMatthew Wilcox c.features.opcode = nvme_admin_set_features; 997df3481399042200792822b6243e36a95a557b57eMatthew Wilcox c.features.prp1 = cpu_to_le64(dma_addr); 998df3481399042200792822b6243e36a95a557b57eMatthew Wilcox c.features.fid = cpu_to_le32(fid); 999df3481399042200792822b6243e36a95a557b57eMatthew Wilcox c.features.dword11 = cpu_to_le32(dword11); 1000df3481399042200792822b6243e36a95a557b57eMatthew Wilcox 1001bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox return nvme_submit_admin_cmd(dev, &c, result); 1002bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox} 1003bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox 1004a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox/** 1005a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox * nvme_cancel_ios - Cancel outstanding I/Os 1006a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox * @queue: The queue to cancel I/Os on 1007a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox * @timeout: True to only cancel I/Os which have timed out 1008a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox */ 1009a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcoxstatic void nvme_cancel_ios(struct nvme_queue *nvmeq, bool timeout) 1010a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox{ 1011a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox int depth = nvmeq->q_depth - 1; 1012a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox struct nvme_cmd_info *info = nvme_cmd_info(nvmeq); 1013a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox unsigned long now = jiffies; 1014a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox int cmdid; 1015a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox 1016a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox for_each_set_bit(cmdid, nvmeq->cmdid_data, depth) { 1017a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox void *ctx; 1018a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox nvme_completion_fn fn; 1019a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox static struct nvme_completion cqe = { 1020af2d9ca744af5e03390eeb3864d08ce75c860899Matthew Wilcox .status = cpu_to_le16(NVME_SC_ABORT_REQ << 1), 1021a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox }; 1022a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox 1023a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox if (timeout && !time_after(now, info[cmdid].timeout)) 1024a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox continue; 1025053ab702cc2702f25a97ead087ed344b864785b7Keith Busch if (info[cmdid].ctx == CMD_CTX_CANCELLED) 1026053ab702cc2702f25a97ead087ed344b864785b7Keith Busch continue; 1027a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox dev_warn(nvmeq->q_dmadev, "Cancelling I/O %d\n", cmdid); 1028a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox ctx = cancel_cmdid(nvmeq, cmdid, &fn); 1029a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox fn(nvmeq->dev, ctx, &cqe); 1030a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox } 1031a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox} 1032a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox 1033224042742582c9938788b81165180c876e997a07Keith Buschstatic void nvme_free_queue(struct nvme_queue *nvmeq) 10349e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox{ 1035224042742582c9938788b81165180c876e997a07Keith Busch spin_lock_irq(&nvmeq->q_lock); 1036224042742582c9938788b81165180c876e997a07Keith Busch while (bio_list_peek(&nvmeq->sq_cong)) { 1037224042742582c9938788b81165180c876e997a07Keith Busch struct bio *bio = bio_list_pop(&nvmeq->sq_cong); 1038224042742582c9938788b81165180c876e997a07Keith Busch bio_endio(bio, -EIO); 1039224042742582c9938788b81165180c876e997a07Keith Busch } 1040224042742582c9938788b81165180c876e997a07Keith Busch spin_unlock_irq(&nvmeq->q_lock); 1041224042742582c9938788b81165180c876e997a07Keith Busch 10429e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth), 10439e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox (void *)nvmeq->cqes, nvmeq->cq_dma_addr); 10449e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth), 10459e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox nvmeq->sq_cmds, nvmeq->sq_dma_addr); 10469e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox kfree(nvmeq); 10479e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox} 10489e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox 1049224042742582c9938788b81165180c876e997a07Keith Buschstatic void nvme_free_queues(struct nvme_dev *dev) 1050224042742582c9938788b81165180c876e997a07Keith Busch{ 1051224042742582c9938788b81165180c876e997a07Keith Busch int i; 1052224042742582c9938788b81165180c876e997a07Keith Busch 1053224042742582c9938788b81165180c876e997a07Keith Busch for (i = dev->queue_count - 1; i >= 0; i--) { 1054224042742582c9938788b81165180c876e997a07Keith Busch nvme_free_queue(dev->queues[i]); 1055224042742582c9938788b81165180c876e997a07Keith Busch dev->queue_count--; 1056224042742582c9938788b81165180c876e997a07Keith Busch dev->queues[i] = NULL; 1057224042742582c9938788b81165180c876e997a07Keith Busch } 1058224042742582c9938788b81165180c876e997a07Keith Busch} 1059224042742582c9938788b81165180c876e997a07Keith Busch 1060224042742582c9938788b81165180c876e997a07Keith Buschstatic void nvme_disable_queue(struct nvme_dev *dev, int qid) 1061b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1062b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq = dev->queues[qid]; 1063aba2080f3f1639f9202f1a52993669844abcfb80Matthew Wilcox int vector = dev->entry[nvmeq->cq_vector].vector; 1064b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1065a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox spin_lock_irq(&nvmeq->q_lock); 1066224042742582c9938788b81165180c876e997a07Keith Busch if (nvmeq->q_suspended) { 1067224042742582c9938788b81165180c876e997a07Keith Busch spin_unlock_irq(&nvmeq->q_lock); 1068224042742582c9938788b81165180c876e997a07Keith Busch return; 10693295874b6074d749516d6decd43afad7bf6e38ffKeith Busch } 1070224042742582c9938788b81165180c876e997a07Keith Busch nvmeq->q_suspended = 1; 1071a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox spin_unlock_irq(&nvmeq->q_lock); 1072a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox 1073aba2080f3f1639f9202f1a52993669844abcfb80Matthew Wilcox irq_set_affinity_hint(vector, NULL); 1074aba2080f3f1639f9202f1a52993669844abcfb80Matthew Wilcox free_irq(vector, nvmeq); 1075b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1076b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox /* Don't tell the adapter to delete the admin queue */ 1077b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (qid) { 1078b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox adapter_delete_sq(dev, qid); 1079b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox adapter_delete_cq(dev, qid); 1080b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 1081b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1082224042742582c9938788b81165180c876e997a07Keith Busch spin_lock_irq(&nvmeq->q_lock); 1083224042742582c9938788b81165180c876e997a07Keith Busch nvme_process_cq(nvmeq); 1084224042742582c9938788b81165180c876e997a07Keith Busch nvme_cancel_ios(nvmeq, false); 1085224042742582c9938788b81165180c876e997a07Keith Busch spin_unlock_irq(&nvmeq->q_lock); 1086b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1087b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1088b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev, int qid, 1089b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int depth, int vector) 1090b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1091b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct device *dmadev = &dev->pci_dev->dev; 1092224042742582c9938788b81165180c876e997a07Keith Busch unsigned extra = nvme_queue_extra(depth); 1093b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq = kzalloc(sizeof(*nvmeq) + extra, GFP_KERNEL); 1094b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!nvmeq) 1095b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 1096b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1097b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->cqes = dma_alloc_coherent(dmadev, CQ_SIZE(depth), 1098b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox &nvmeq->cq_dma_addr, GFP_KERNEL); 1099b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!nvmeq->cqes) 1100b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto free_nvmeq; 1101b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset((void *)nvmeq->cqes, 0, CQ_SIZE(depth)); 1102b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1103b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->sq_cmds = dma_alloc_coherent(dmadev, SQ_SIZE(depth), 1104b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox &nvmeq->sq_dma_addr, GFP_KERNEL); 1105b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!nvmeq->sq_cmds) 1106b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto free_cqdma; 1107b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1108b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->q_dmadev = dmadev; 1109091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox nvmeq->dev = dev; 1110b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox spin_lock_init(&nvmeq->q_lock); 1111b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->cq_head = 0; 1112821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox nvmeq->cq_phase = 1; 1113b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox init_waitqueue_head(&nvmeq->sq_full); 11141fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox init_waitqueue_entry(&nvmeq->sq_cong_wait, nvme_thread); 1115b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox bio_list_init(&nvmeq->sq_cong); 1116b80d5ccca3a012e91ca64a2a0b13049163a6a698Haiyan Hu nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride]; 1117b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->q_depth = depth; 1118b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->cq_vector = vector; 1119224042742582c9938788b81165180c876e997a07Keith Busch nvmeq->q_suspended = 1; 1120224042742582c9938788b81165180c876e997a07Keith Busch dev->queue_count++; 1121b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1122b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return nvmeq; 1123b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1124b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox free_cqdma: 112568b8eca5f882230e4991df0db0a3094f49420a90Keith Busch dma_free_coherent(dmadev, CQ_SIZE(depth), (void *)nvmeq->cqes, 1126b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->cq_dma_addr); 1127b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox free_nvmeq: 1128b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(nvmeq); 1129b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 1130b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1131b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 11323001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcoxstatic int queue_request_irq(struct nvme_dev *dev, struct nvme_queue *nvmeq, 11333001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox const char *name) 11343001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox{ 113558ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox if (use_threaded_interrupts) 113658ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox return request_threaded_irq(dev->entry[nvmeq->cq_vector].vector, 1137481e5bad84239b01415b888df2040c1860ae3cfdMichael Opdenacker nvme_irq_check, nvme_irq, IRQF_SHARED, 113858ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox name, nvmeq); 11393001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox return request_irq(dev->entry[nvmeq->cq_vector].vector, nvme_irq, 1140481e5bad84239b01415b888df2040c1860ae3cfdMichael Opdenacker IRQF_SHARED, name, nvmeq); 11413001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox} 11423001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox 1143224042742582c9938788b81165180c876e997a07Keith Buschstatic void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid) 1144b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1145224042742582c9938788b81165180c876e997a07Keith Busch struct nvme_dev *dev = nvmeq->dev; 1146224042742582c9938788b81165180c876e997a07Keith Busch unsigned extra = nvme_queue_extra(nvmeq->q_depth); 1147b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1148224042742582c9938788b81165180c876e997a07Keith Busch nvmeq->sq_tail = 0; 1149224042742582c9938788b81165180c876e997a07Keith Busch nvmeq->cq_head = 0; 1150224042742582c9938788b81165180c876e997a07Keith Busch nvmeq->cq_phase = 1; 1151b80d5ccca3a012e91ca64a2a0b13049163a6a698Haiyan Hu nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride]; 1152224042742582c9938788b81165180c876e997a07Keith Busch memset(nvmeq->cmdid_data, 0, extra); 1153224042742582c9938788b81165180c876e997a07Keith Busch memset((void *)nvmeq->cqes, 0, CQ_SIZE(nvmeq->q_depth)); 1154224042742582c9938788b81165180c876e997a07Keith Busch nvme_cancel_ios(nvmeq, false); 1155224042742582c9938788b81165180c876e997a07Keith Busch nvmeq->q_suspended = 0; 1156224042742582c9938788b81165180c876e997a07Keith Busch} 1157224042742582c9938788b81165180c876e997a07Keith Busch 1158224042742582c9938788b81165180c876e997a07Keith Buschstatic int nvme_create_queue(struct nvme_queue *nvmeq, int qid) 1159224042742582c9938788b81165180c876e997a07Keith Busch{ 1160224042742582c9938788b81165180c876e997a07Keith Busch struct nvme_dev *dev = nvmeq->dev; 1161224042742582c9938788b81165180c876e997a07Keith Busch int result; 11623f85d50b609e8a5ef151656210203a6e94c19538Matthew Wilcox 1163b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox result = adapter_alloc_cq(dev, qid, nvmeq); 1164b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (result < 0) 1165224042742582c9938788b81165180c876e997a07Keith Busch return result; 1166b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1167b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox result = adapter_alloc_sq(dev, qid, nvmeq); 1168b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (result < 0) 1169b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto release_cq; 1170b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 11713001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox result = queue_request_irq(dev, nvmeq, "nvme"); 1172b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (result < 0) 1173b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto release_sq; 1174b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 11750a8d44cb33377969337fa6c1961b631605d5f453Matthew Wilcox spin_lock_irq(&nvmeq->q_lock); 1176224042742582c9938788b81165180c876e997a07Keith Busch nvme_init_queue(nvmeq, qid); 11770a8d44cb33377969337fa6c1961b631605d5f453Matthew Wilcox spin_unlock_irq(&nvmeq->q_lock); 1178224042742582c9938788b81165180c876e997a07Keith Busch 1179224042742582c9938788b81165180c876e997a07Keith Busch return result; 1180b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1181b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox release_sq: 1182b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox adapter_delete_sq(dev, qid); 1183b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox release_cq: 1184b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox adapter_delete_cq(dev, qid); 1185224042742582c9938788b81165180c876e997a07Keith Busch return result; 1186b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1187b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1188ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcoxstatic int nvme_wait_ready(struct nvme_dev *dev, u64 cap, bool enabled) 1189ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox{ 1190ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox unsigned long timeout; 1191ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox u32 bit = enabled ? NVME_CSTS_RDY : 0; 1192ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox 1193ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox timeout = ((NVME_CAP_TIMEOUT(cap) + 1) * HZ / 2) + jiffies; 1194ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox 1195ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox while ((readl(&dev->bar->csts) & NVME_CSTS_RDY) != bit) { 1196ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox msleep(100); 1197ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox if (fatal_signal_pending(current)) 1198ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox return -EINTR; 1199ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox if (time_after(jiffies, timeout)) { 1200ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox dev_err(&dev->pci_dev->dev, 1201ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox "Device not ready; aborting initialisation\n"); 1202ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox return -ENODEV; 1203ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox } 1204ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox } 1205ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox 1206ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox return 0; 1207ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox} 1208ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox 1209ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox/* 1210ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox * If the device has been passed off to us in an enabled state, just clear 1211ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox * the enabled bit. The spec says we should set the 'shutdown notification 1212ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox * bits', but doing so may cause the device to complete commands to the 1213ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox * admin queue ... and we don't know what memory that might be pointing at! 1214ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox */ 1215ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcoxstatic int nvme_disable_ctrl(struct nvme_dev *dev, u64 cap) 1216ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox{ 121744af146a84fa4a8e136d824207dcd356958a112bMatthew Wilcox u32 cc = readl(&dev->bar->cc); 121844af146a84fa4a8e136d824207dcd356958a112bMatthew Wilcox 121944af146a84fa4a8e136d824207dcd356958a112bMatthew Wilcox if (cc & NVME_CC_ENABLE) 122044af146a84fa4a8e136d824207dcd356958a112bMatthew Wilcox writel(cc & ~NVME_CC_ENABLE, &dev->bar->cc); 1221ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox return nvme_wait_ready(dev, cap, false); 1222ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox} 1223ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox 1224ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcoxstatic int nvme_enable_ctrl(struct nvme_dev *dev, u64 cap) 1225ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox{ 1226ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox return nvme_wait_ready(dev, cap, true); 1227ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox} 1228ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox 12291894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Buschstatic int nvme_shutdown_ctrl(struct nvme_dev *dev) 12301894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch{ 12311894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch unsigned long timeout; 12321894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch u32 cc; 12331894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch 12341894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch cc = (readl(&dev->bar->cc) & ~NVME_CC_SHN_MASK) | NVME_CC_SHN_NORMAL; 12351894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch writel(cc, &dev->bar->cc); 12361894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch 12371894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch timeout = 2 * HZ + jiffies; 12381894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch while ((readl(&dev->bar->csts) & NVME_CSTS_SHST_MASK) != 12391894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch NVME_CSTS_SHST_CMPLT) { 12401894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch msleep(100); 12411894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch if (fatal_signal_pending(current)) 12421894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch return -EINTR; 12431894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch if (time_after(jiffies, timeout)) { 12441894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch dev_err(&dev->pci_dev->dev, 12451894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch "Device shutdown incomplete; abort shutdown\n"); 12461894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch return -ENODEV; 12471894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch } 12481894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch } 12491894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch 12501894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch return 0; 12511894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch} 12521894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch 12538d85fce77edfc22f1d6dbf78e3af723b4b556f3dGreg Kroah-Hartmanstatic int nvme_configure_admin_queue(struct nvme_dev *dev) 1254b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1255ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox int result; 1256b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 aqa; 1257ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox u64 cap = readq(&dev->bar->cap); 1258b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq; 1259b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1260ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox result = nvme_disable_ctrl(dev, cap); 1261ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox if (result < 0) 1262ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox return result; 1263b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1264cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch nvmeq = dev->queues[0]; 1265cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch if (!nvmeq) { 1266cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch nvmeq = nvme_alloc_queue(dev, 0, 64, 0); 1267cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch if (!nvmeq) 1268cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch return -ENOMEM; 1269cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch dev->queues[0] = nvmeq; 1270cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch } 1271b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1272b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox aqa = nvmeq->q_depth - 1; 1273b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox aqa |= aqa << 16; 1274b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1275b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->ctrl_config = NVME_CC_ENABLE | NVME_CC_CSS_NVM; 1276b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->ctrl_config |= (PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT; 1277b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->ctrl_config |= NVME_CC_ARB_RR | NVME_CC_SHN_NONE; 12787f53f9d2424533256ae86f7df5661a17de743de8Matthew Wilcox dev->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES; 1279b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1280b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox writel(aqa, &dev->bar->aqa); 1281b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox writeq(nvmeq->sq_dma_addr, &dev->bar->asq); 1282b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox writeq(nvmeq->cq_dma_addr, &dev->bar->acq); 1283b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox writel(dev->ctrl_config, &dev->bar->cc); 1284b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1285ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox result = nvme_enable_ctrl(dev, cap); 1286025c557a71bd06a9f6d32259f00e2218b15bf0a4Keith Busch if (result) 1287cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch return result; 12889e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox 12893001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox result = queue_request_irq(dev, nvmeq, "nvme admin"); 1290025c557a71bd06a9f6d32259f00e2218b15bf0a4Keith Busch if (result) 1291cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch return result; 1292025c557a71bd06a9f6d32259f00e2218b15bf0a4Keith Busch 12930a8d44cb33377969337fa6c1961b631605d5f453Matthew Wilcox spin_lock_irq(&nvmeq->q_lock); 1294224042742582c9938788b81165180c876e997a07Keith Busch nvme_init_queue(nvmeq, 0); 12950a8d44cb33377969337fa6c1961b631605d5f453Matthew Wilcox spin_unlock_irq(&nvmeq->q_lock); 1296b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return result; 1297b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1298b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 12995d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermastruct nvme_iod *nvme_map_user_pages(struct nvme_dev *dev, int write, 1300eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox unsigned long addr, unsigned length) 1301b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 130236c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox int i, err, count, nents, offset; 13037fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox struct scatterlist *sg; 13047fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox struct page **pages; 1305eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox struct nvme_iod *iod; 130636c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox 130736c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox if (addr & 3) 1308eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return ERR_PTR(-EINVAL); 13095460fc03105fbed01fe27aa572d9f65bb410a61dDan Carpenter if (!length || length > INT_MAX - PAGE_SIZE) 1310eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return ERR_PTR(-EINVAL); 13117fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox 131236c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox offset = offset_in_page(addr); 13137fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox count = DIV_ROUND_UP(offset + length, PAGE_SIZE); 13147fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox pages = kcalloc(count, sizeof(*pages), GFP_KERNEL); 131522fff826e715e9727d3c7a69f15e602a9801b673Dan Carpenter if (!pages) 131622fff826e715e9727d3c7a69f15e602a9801b673Dan Carpenter return ERR_PTR(-ENOMEM); 131736c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox 131836c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox err = get_user_pages_fast(addr, count, 1, pages); 131936c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox if (err < count) { 132036c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox count = err; 132136c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox err = -EFAULT; 132236c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox goto put_pages; 132336c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox } 13247fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox 1325eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod = nvme_alloc_iod(count, length, GFP_KERNEL); 1326eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox sg = iod->sg; 132736c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox sg_init_table(sg, count); 1328d0ba1e497bca83a3d353eb47c9658afc54d83228Matthew Wilcox for (i = 0; i < count; i++) { 1329d0ba1e497bca83a3d353eb47c9658afc54d83228Matthew Wilcox sg_set_page(&sg[i], pages[i], 13305460fc03105fbed01fe27aa572d9f65bb410a61dDan Carpenter min_t(unsigned, length, PAGE_SIZE - offset), 13315460fc03105fbed01fe27aa572d9f65bb410a61dDan Carpenter offset); 1332d0ba1e497bca83a3d353eb47c9658afc54d83228Matthew Wilcox length -= (PAGE_SIZE - offset); 1333d0ba1e497bca83a3d353eb47c9658afc54d83228Matthew Wilcox offset = 0; 13347fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox } 1335fe304c43c6d63e29ed4fc46a874d7a74313788c5Matthew Wilcox sg_mark_end(&sg[i - 1]); 13361c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox iod->nents = count; 13377fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox 13387fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox err = -ENOMEM; 13397fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox nents = dma_map_sg(&dev->pci_dev->dev, sg, count, 13407fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox write ? DMA_TO_DEVICE : DMA_FROM_DEVICE); 134136c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox if (!nents) 1342eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox goto free_iod; 1343b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 13447fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox kfree(pages); 1345eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return iod; 1346b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1347eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox free_iod: 1348eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox kfree(iod); 13497fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox put_pages: 13507fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox for (i = 0; i < count; i++) 13517fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox put_page(pages[i]); 13527fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox kfree(pages); 1353eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return ERR_PTR(err); 13547fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox} 1355b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 13565d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermavoid nvme_unmap_user_pages(struct nvme_dev *dev, int write, 13571c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox struct nvme_iod *iod) 13587fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox{ 13591c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox int i; 1360b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 13611c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox dma_unmap_sg(&dev->pci_dev->dev, iod->sg, iod->nents, 13621c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox write ? DMA_TO_DEVICE : DMA_FROM_DEVICE); 13637fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox 13641c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox for (i = 0; i < iod->nents; i++) 13651c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox put_page(sg_page(&iod->sg[i])); 13667fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox} 1367b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1368a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcoxstatic int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio) 1369a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox{ 1370a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox struct nvme_dev *dev = ns->dev; 1371a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox struct nvme_queue *nvmeq; 1372a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox struct nvme_user_io io; 1373a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox struct nvme_command c; 1374f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch unsigned length, meta_len; 1375f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch int status, i; 1376f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch struct nvme_iod *iod, *meta_iod = NULL; 1377f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch dma_addr_t meta_dma_addr; 1378f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch void *meta, *uninitialized_var(meta_mem); 1379a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox 1380a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox if (copy_from_user(&io, uio, sizeof(io))) 1381a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox return -EFAULT; 13826c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox length = (io.nblocks + 1) << ns->lba_shift; 1383f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_len = (io.nblocks + 1) * ns->ms; 1384f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1385f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch if (meta_len && ((io.metadata & 3) || !io.metadata)) 1386f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch return -EINVAL; 13876c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox 13886c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox switch (io.opcode) { 13896c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox case nvme_cmd_write: 13906c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox case nvme_cmd_read: 13916bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox case nvme_cmd_compare: 1392eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod = nvme_map_user_pages(dev, io.opcode & 1, io.addr, length); 13936413214c5d424fd5aae6567848340f962ad2ce0fMatthew Wilcox break; 13946c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox default: 13956bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox return -EINVAL; 13966c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox } 13976c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox 1398eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox if (IS_ERR(iod)) 1399eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return PTR_ERR(iod); 1400a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox 1401a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox memset(&c, 0, sizeof(c)); 1402a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.opcode = io.opcode; 1403a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.flags = io.flags; 14046c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox c.rw.nsid = cpu_to_le32(ns->ns_id); 1405a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.slba = cpu_to_le64(io.slba); 14066c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox c.rw.length = cpu_to_le16(io.nblocks); 1407a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.control = cpu_to_le16(io.control); 14081c9b52651dad0ff1fa71fc6205c86d972f25bcc0Matthew Wilcox c.rw.dsmgmt = cpu_to_le32(io.dsmgmt); 14091c9b52651dad0ff1fa71fc6205c86d972f25bcc0Matthew Wilcox c.rw.reftag = cpu_to_le32(io.reftag); 14101c9b52651dad0ff1fa71fc6205c86d972f25bcc0Matthew Wilcox c.rw.apptag = cpu_to_le16(io.apptag); 14111c9b52651dad0ff1fa71fc6205c86d972f25bcc0Matthew Wilcox c.rw.appmask = cpu_to_le16(io.appmask); 1412f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1413f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch if (meta_len) { 14141b56749e541ad59068582f2a28297843e243b856Keith Busch meta_iod = nvme_map_user_pages(dev, io.opcode & 1, io.metadata, 14151b56749e541ad59068582f2a28297843e243b856Keith Busch meta_len); 1416f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch if (IS_ERR(meta_iod)) { 1417f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch status = PTR_ERR(meta_iod); 1418f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_iod = NULL; 1419f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch goto unmap; 1420f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch } 1421f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1422f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_mem = dma_alloc_coherent(&dev->pci_dev->dev, meta_len, 1423f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch &meta_dma_addr, GFP_KERNEL); 1424f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch if (!meta_mem) { 1425f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch status = -ENOMEM; 1426f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch goto unmap; 1427f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch } 1428f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1429f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch if (io.opcode & 1) { 1430f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch int meta_offset = 0; 1431f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1432f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch for (i = 0; i < meta_iod->nents; i++) { 1433f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta = kmap_atomic(sg_page(&meta_iod->sg[i])) + 1434f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_iod->sg[i].offset; 1435f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch memcpy(meta_mem + meta_offset, meta, 1436f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_iod->sg[i].length); 1437f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch kunmap_atomic(meta); 1438f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_offset += meta_iod->sg[i].length; 1439f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch } 1440f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch } 1441f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1442f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch c.rw.metadata = cpu_to_le64(meta_dma_addr); 1443f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch } 1444f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1445eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox length = nvme_setup_prps(dev, &c.common, iod, length, GFP_KERNEL); 1446a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox 1447040a93b52a9eee8177ebaf2ba0ee0f9f518d1bf8Matthew Wilcox nvmeq = get_nvmeq(dev); 1448fa92282149842645931580225647238428374758Matthew Wilcox /* 1449fa92282149842645931580225647238428374758Matthew Wilcox * Since nvme_submit_sync_cmd sleeps, we can't keep preemption 1450b1ad37efcafe396ac3944853589688dd0ec3c64eMatthew Wilcox * disabled. We may be preempted at any point, and be rescheduled 1451b1ad37efcafe396ac3944853589688dd0ec3c64eMatthew Wilcox * to a different CPU. That will cause cacheline bouncing, but no 1452b1ad37efcafe396ac3944853589688dd0ec3c64eMatthew Wilcox * additional races since q_lock already protects against other CPUs. 1453b1ad37efcafe396ac3944853589688dd0ec3c64eMatthew Wilcox */ 1454a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox put_nvmeq(nvmeq); 1455b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox if (length != (io.nblocks + 1) << ns->lba_shift) 1456b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox status = -ENOMEM; 1457224042742582c9938788b81165180c876e997a07Keith Busch else if (!nvmeq || nvmeq->q_suspended) 1458224042742582c9938788b81165180c876e997a07Keith Busch status = -EBUSY; 1459b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox else 1460ff976d724a74e4522e9ca2de1fb37ac4520f454fMatthew Wilcox status = nvme_submit_sync_cmd(nvmeq, &c, NULL, NVME_IO_TIMEOUT); 1461a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox 1462f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch if (meta_len) { 1463f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch if (status == NVME_SC_SUCCESS && !(io.opcode & 1)) { 1464f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch int meta_offset = 0; 1465f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1466f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch for (i = 0; i < meta_iod->nents; i++) { 1467f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta = kmap_atomic(sg_page(&meta_iod->sg[i])) + 1468f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_iod->sg[i].offset; 1469f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch memcpy(meta, meta_mem + meta_offset, 1470f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_iod->sg[i].length); 1471f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch kunmap_atomic(meta); 1472f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_offset += meta_iod->sg[i].length; 1473f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch } 1474f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch } 1475f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1476f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch dma_free_coherent(&dev->pci_dev->dev, meta_len, meta_mem, 1477f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_dma_addr); 1478f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch } 1479f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1480f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch unmap: 14811c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox nvme_unmap_user_pages(dev, io.opcode & 1, iod); 1482eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox nvme_free_iod(dev, iod); 1483f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1484f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch if (meta_iod) { 1485f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch nvme_unmap_user_pages(dev, io.opcode & 1, meta_iod); 1486f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch nvme_free_iod(dev, meta_iod); 1487f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch } 1488f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1489a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox return status; 1490a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox} 1491a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox 149250af8baec46a99a9b81a4600c0374f83a5a590a9Keith Buschstatic int nvme_user_admin_cmd(struct nvme_dev *dev, 14936bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox struct nvme_admin_cmd __user *ucmd) 14946ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox{ 14956bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox struct nvme_admin_cmd cmd; 14966ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox struct nvme_command c; 1497eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox int status, length; 1498c7d36ab8fa04c213328119a9c0d66985fe204ee5Keith Busch struct nvme_iod *uninitialized_var(iod); 149994f370cab6e5ac514b658c6b2b3aa308cefc5c7aKeith Busch unsigned timeout; 15006ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox 15016bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox if (!capable(CAP_SYS_ADMIN)) 15026bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox return -EACCES; 15036bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox if (copy_from_user(&cmd, ucmd, sizeof(cmd))) 15046ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox return -EFAULT; 15056ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox 15066ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox memset(&c, 0, sizeof(c)); 15076bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.opcode = cmd.opcode; 15086bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.flags = cmd.flags; 15096bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.nsid = cpu_to_le32(cmd.nsid); 15106bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw2[0] = cpu_to_le32(cmd.cdw2); 15116bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw2[1] = cpu_to_le32(cmd.cdw3); 15126bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw10[0] = cpu_to_le32(cmd.cdw10); 15136bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw10[1] = cpu_to_le32(cmd.cdw11); 15146bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw10[2] = cpu_to_le32(cmd.cdw12); 15156bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw10[3] = cpu_to_le32(cmd.cdw13); 15166bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw10[4] = cpu_to_le32(cmd.cdw14); 15176bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw10[5] = cpu_to_le32(cmd.cdw15); 15186bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox 15196bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox length = cmd.data_len; 15206bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox if (cmd.data_len) { 1521497421880acecd0281d3182d534f3d28c927caecMatthew Wilcox iod = nvme_map_user_pages(dev, cmd.opcode & 1, cmd.addr, 1522497421880acecd0281d3182d534f3d28c927caecMatthew Wilcox length); 1523eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox if (IS_ERR(iod)) 1524eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return PTR_ERR(iod); 1525eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox length = nvme_setup_prps(dev, &c.common, iod, length, 1526eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox GFP_KERNEL); 15276bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox } 15286bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox 152994f370cab6e5ac514b658c6b2b3aa308cefc5c7aKeith Busch timeout = cmd.timeout_ms ? msecs_to_jiffies(cmd.timeout_ms) : 153094f370cab6e5ac514b658c6b2b3aa308cefc5c7aKeith Busch ADMIN_TIMEOUT; 15316bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox if (length != cmd.data_len) 1532b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox status = -ENOMEM; 1533b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox else 153494f370cab6e5ac514b658c6b2b3aa308cefc5c7aKeith Busch status = nvme_submit_sync_cmd(dev->queues[0], &c, &cmd.result, 153594f370cab6e5ac514b658c6b2b3aa308cefc5c7aKeith Busch timeout); 1536eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox 15376bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox if (cmd.data_len) { 15381c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox nvme_unmap_user_pages(dev, cmd.opcode & 1, iod); 1539eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox nvme_free_iod(dev, iod); 15406bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox } 1541f4f117f64baf8840d22266d518227b2a186d294bKeith Busch 1542cf90bc4830b858487fe4b9b9ecd0031e23ca3e83Chayan Biswas if ((status >= 0) && copy_to_user(&ucmd->result, &cmd.result, 1543f4f117f64baf8840d22266d518227b2a186d294bKeith Busch sizeof(cmd.result))) 1544f4f117f64baf8840d22266d518227b2a186d294bKeith Busch status = -EFAULT; 1545f4f117f64baf8840d22266d518227b2a186d294bKeith Busch 15466ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox return status; 15476ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox} 15486ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox 1549b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, 1550b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unsigned long arg) 1551b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1552b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_ns *ns = bdev->bd_disk->private_data; 1553b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1554b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox switch (cmd) { 15556bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox case NVME_IOCTL_ID: 1556c3bfe7176c035a0a2c70bc79180fb13a6c57142aMatthew Wilcox force_successful_syscall_return(); 15576bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox return ns->ns_id; 15586bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox case NVME_IOCTL_ADMIN_CMD: 155950af8baec46a99a9b81a4600c0374f83a5a590a9Keith Busch return nvme_user_admin_cmd(ns->dev, (void __user *)arg); 1560a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox case NVME_IOCTL_SUBMIT_IO: 1561a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox return nvme_submit_io(ns, (void __user *)arg); 15625d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma case SG_GET_VERSION_NUM: 15635d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma return nvme_sg_get_version_num((void __user *)arg); 15645d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma case SG_IO: 15655d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma return nvme_sg_io(ns, (void __user *)arg); 1566b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox default: 1567b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -ENOTTY; 1568b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 1569b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1570b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1571320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch#ifdef CONFIG_COMPAT 1572320a382746e0ab1304476ea7e986a8d416ab99dbKeith Buschstatic int nvme_compat_ioctl(struct block_device *bdev, fmode_t mode, 1573320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch unsigned int cmd, unsigned long arg) 1574320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch{ 1575320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch struct nvme_ns *ns = bdev->bd_disk->private_data; 1576320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch 1577320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch switch (cmd) { 1578320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch case SG_IO: 1579320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch return nvme_sg_io32(ns, arg); 1580320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch } 1581320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch return nvme_ioctl(bdev, mode, cmd, arg); 1582320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch} 1583320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch#else 1584320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch#define nvme_compat_ioctl NULL 1585320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch#endif 1586320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch 1587b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic const struct block_device_operations nvme_fops = { 1588b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .owner = THIS_MODULE, 1589b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .ioctl = nvme_ioctl, 1590320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch .compat_ioctl = nvme_compat_ioctl, 1591b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 1592b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 15931fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcoxstatic void nvme_resubmit_bios(struct nvme_queue *nvmeq) 15941fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox{ 15951fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox while (bio_list_peek(&nvmeq->sq_cong)) { 15961fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox struct bio *bio = bio_list_pop(&nvmeq->sq_cong); 15971fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox struct nvme_ns *ns = bio->bi_bdev->bd_disk->private_data; 1598427e97080196548557b288517537ab7eb48c309fKeith Busch 1599427e97080196548557b288517537ab7eb48c309fKeith Busch if (bio_list_empty(&nvmeq->sq_cong)) 1600427e97080196548557b288517537ab7eb48c309fKeith Busch remove_wait_queue(&nvmeq->sq_full, 1601427e97080196548557b288517537ab7eb48c309fKeith Busch &nvmeq->sq_cong_wait); 16021fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox if (nvme_submit_bio_queue(nvmeq, ns, bio)) { 1603427e97080196548557b288517537ab7eb48c309fKeith Busch if (bio_list_empty(&nvmeq->sq_cong)) 1604427e97080196548557b288517537ab7eb48c309fKeith Busch add_wait_queue(&nvmeq->sq_full, 1605427e97080196548557b288517537ab7eb48c309fKeith Busch &nvmeq->sq_cong_wait); 16061fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox bio_list_add_head(&nvmeq->sq_cong, bio); 16071fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox break; 16081fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox } 16091fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox } 16101fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox} 16111fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 16121fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcoxstatic int nvme_kthread(void *data) 16131fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox{ 16141fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox struct nvme_dev *dev; 16151fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 16161fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox while (!kthread_should_stop()) { 1617564a232c059913d91b491e04c2b2d670b8f94615Arjan van de Ven set_current_state(TASK_INTERRUPTIBLE); 16181fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_lock(&dev_list_lock); 16191fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox list_for_each_entry(dev, &dev_list, node) { 16201fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox int i; 16211fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox for (i = 0; i < dev->queue_count; i++) { 16221fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox struct nvme_queue *nvmeq = dev->queues[i]; 1623740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox if (!nvmeq) 1624740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox continue; 16251fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_lock_irq(&nvmeq->q_lock); 1626224042742582c9938788b81165180c876e997a07Keith Busch if (nvmeq->q_suspended) 1627224042742582c9938788b81165180c876e997a07Keith Busch goto unlock; 1628bc57a0f7a44cfcf3e9873f6c6b8dcecdca486b1fMatthew Wilcox nvme_process_cq(nvmeq); 1629a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox nvme_cancel_ios(nvmeq, true); 16301fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox nvme_resubmit_bios(nvmeq); 1631224042742582c9938788b81165180c876e997a07Keith Busch unlock: 16321fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_unlock_irq(&nvmeq->q_lock); 16331fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox } 16341fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox } 16351fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_unlock(&dev_list_lock); 1636acb7aa0db09b8abd38abeb84334a8a27a52fbb1bArjan van de Ven schedule_timeout(round_jiffies_relative(HZ)); 16371fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox } 16381fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox return 0; 16391fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox} 16401fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 16415aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcoxstatic DEFINE_IDA(nvme_index_ida); 16425aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox 16435aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcoxstatic int nvme_get_ns_idx(void) 16445aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox{ 16455aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox int index, error; 16465aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox 16475aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox do { 16485aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox if (!ida_pre_get(&nvme_index_ida, GFP_KERNEL)) 16495aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox return -1; 16505aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox 16515aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox spin_lock(&dev_list_lock); 16525aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox error = ida_get_new(&nvme_index_ida, &index); 16535aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox spin_unlock(&dev_list_lock); 16545aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox } while (error == -EAGAIN); 16555aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox 16565aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox if (error) 16575aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox index = -1; 16585aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox return index; 16595aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox} 16605aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox 16615aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcoxstatic void nvme_put_ns_idx(int index) 16625aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox{ 16635aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox spin_lock(&dev_list_lock); 16645aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox ida_remove(&nvme_index_ida, index); 16655aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox spin_unlock(&dev_list_lock); 16665aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox} 16675aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox 16680e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Buschstatic void nvme_config_discard(struct nvme_ns *ns) 16690e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch{ 16700e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch u32 logical_block_size = queue_logical_block_size(ns->queue); 16710e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch ns->queue->limits.discard_zeroes_data = 0; 16720e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch ns->queue->limits.discard_alignment = logical_block_size; 16730e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch ns->queue->limits.discard_granularity = logical_block_size; 16740e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch ns->queue->limits.max_discard_sectors = 0xffffffff; 16750e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, ns->queue); 16760e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch} 16770e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch 1678c3bfe7176c035a0a2c70bc79180fb13a6c57142aMatthew Wilcoxstatic struct nvme_ns *nvme_alloc_ns(struct nvme_dev *dev, unsigned nsid, 1679b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_id_ns *id, struct nvme_lba_range_type *rt) 1680b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1681b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_ns *ns; 1682b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct gendisk *disk; 1683b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int lbaf; 1684b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1685b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (rt->attributes & NVME_LBART_ATTRIB_HIDE) 1686b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 1687b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1688b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns = kzalloc(sizeof(*ns), GFP_KERNEL); 1689b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!ns) 1690b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 1691b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->queue = blk_alloc_queue(GFP_KERNEL); 1692b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!ns->queue) 1693b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto out_free_ns; 16944eeb9215a0d5c9494ca8b20158cc8ee82618840cMatthew Wilcox ns->queue->queue_flags = QUEUE_FLAG_DEFAULT; 16954eeb9215a0d5c9494ca8b20158cc8ee82618840cMatthew Wilcox queue_flag_set_unlocked(QUEUE_FLAG_NOMERGES, ns->queue); 16964eeb9215a0d5c9494ca8b20158cc8ee82618840cMatthew Wilcox queue_flag_set_unlocked(QUEUE_FLAG_NONROT, ns->queue); 1697b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox blk_queue_make_request(ns->queue, nvme_make_request); 1698b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->dev = dev; 1699b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->queue->queuedata = ns; 1700b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1701b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk = alloc_disk(NVME_MINORS); 1702b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!disk) 1703b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto out_free_queue; 17045aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox ns->ns_id = nsid; 1705b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->disk = disk; 1706b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox lbaf = id->flbas & 0xf; 1707b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->lba_shift = id->lbaf[lbaf].ds; 1708f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch ns->ms = le16_to_cpu(id->lbaf[lbaf].ms); 1709e9ef46369f5107e634a93b7fc4e62a1f53343197Keith Busch blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift); 17108fc23e032debd682f5ba9fc524a5846c10d2c522Keith Busch if (dev->max_hw_sectors) 17118fc23e032debd682f5ba9fc524a5846c10d2c522Keith Busch blk_queue_max_hw_sectors(ns->queue, dev->max_hw_sectors); 1712b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1713b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk->major = nvme_major; 1714b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk->minors = NVME_MINORS; 17155aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox disk->first_minor = NVME_MINORS * nvme_get_ns_idx(); 1716b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk->fops = &nvme_fops; 1717b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk->private_data = ns; 1718b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk->queue = ns->queue; 1719388f037f4e7f0a24bac6b1a24f144f5d939f58cfMatthew Wilcox disk->driverfs_dev = &dev->pci_dev->dev; 17205aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox sprintf(disk->disk_name, "nvme%dn%d", dev->instance, nsid); 1721b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox set_capacity(disk, le64_to_cpup(&id->nsze) << (ns->lba_shift - 9)); 1722b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 17230e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch if (dev->oncs & NVME_CTRL_ONCS_DSM) 17240e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch nvme_config_discard(ns); 17250e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch 1726b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return ns; 1727b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1728b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox out_free_queue: 1729b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox blk_cleanup_queue(ns->queue); 1730b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox out_free_ns: 1731b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(ns); 1732b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 1733b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1734b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1735b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void nvme_ns_free(struct nvme_ns *ns) 1736b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 17375aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox int index = ns->disk->first_minor / NVME_MINORS; 1738b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox put_disk(ns->disk); 17395aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox nvme_put_ns_idx(index); 1740b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox blk_cleanup_queue(ns->queue); 1741b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(ns); 1742b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1743b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1744b3b06812e199f248561ce7824a4a8a9cd573c05aMatthew Wilcoxstatic int set_queue_count(struct nvme_dev *dev, int count) 1745b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1746b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int status; 1747b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 result; 1748b3b06812e199f248561ce7824a4a8a9cd573c05aMatthew Wilcox u32 q_count = (count - 1) | ((count - 1) << 16); 1749b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1750df3481399042200792822b6243e36a95a557b57eMatthew Wilcox status = nvme_set_features(dev, NVME_FEAT_NUM_QUEUES, q_count, 0, 1751bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox &result); 1752b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (status) 17537e03b124065507e72008ef294c30001eca74a031Keith Busch return status < 0 ? -EIO : -EBUSY; 1754b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return min(result & 0xffff, result >> 16) + 1; 1755b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1756b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 17579d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Buschstatic size_t db_bar_size(struct nvme_dev *dev, unsigned nr_io_queues) 17589d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch{ 1759b80d5ccca3a012e91ca64a2a0b13049163a6a698Haiyan Hu return 4096 + ((nr_io_queues + 1) * 8 * dev->db_stride); 17609d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch} 17619d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch 17628d85fce77edfc22f1d6dbf78e3af723b4b556f3dGreg Kroah-Hartmanstatic int nvme_setup_io_queues(struct nvme_dev *dev) 1763b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1764fa08a396647767abd24a9e7015cb177121d0cf15Ramachandra Rao Gajula struct pci_dev *pdev = dev->pci_dev; 17659d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch int result, cpu, i, vecs, nr_io_queues, size, q_depth; 1766b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1767b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox nr_io_queues = num_online_cpus(); 1768b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox result = set_queue_count(dev, nr_io_queues); 17691b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox if (result < 0) 17701b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox return result; 1771b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox if (result < nr_io_queues) 1772b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox nr_io_queues = result; 1773b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 17749d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch size = db_bar_size(dev, nr_io_queues); 17759d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch if (size > 8192) { 1776f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox iounmap(dev->bar); 17779d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch do { 17789d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch dev->bar = ioremap(pci_resource_start(pdev, 0), size); 17799d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch if (dev->bar) 17809d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch break; 17819d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch if (!--nr_io_queues) 17829d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch return -ENOMEM; 17839d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch size = db_bar_size(dev, nr_io_queues); 17849d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch } while (1); 1785f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox dev->dbs = ((void __iomem *)dev->bar) + 4096; 1786f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox dev->queues[0]->q_db = dev->dbs; 1787f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox } 1788f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox 17899d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch /* Deregister the admin queue's interrupt */ 17909d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch free_irq(dev->entry[0].vector, dev->queues[0]); 17919d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch 1792063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox vecs = nr_io_queues; 1793063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox for (i = 0; i < vecs; i++) 17941b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox dev->entry[i].entry = i; 17951b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox for (;;) { 1796063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox result = pci_enable_msix(pdev, dev->entry, vecs); 1797063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox if (result <= 0) 17981b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox break; 1799063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox vecs = result; 18001b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox } 18011b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox 1802063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox if (result < 0) { 1803063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox vecs = nr_io_queues; 1804063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox if (vecs > 32) 1805063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox vecs = 32; 1806fa08a396647767abd24a9e7015cb177121d0cf15Ramachandra Rao Gajula for (;;) { 1807063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox result = pci_enable_msi_block(pdev, vecs); 1808fa08a396647767abd24a9e7015cb177121d0cf15Ramachandra Rao Gajula if (result == 0) { 1809063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox for (i = 0; i < vecs; i++) 1810fa08a396647767abd24a9e7015cb177121d0cf15Ramachandra Rao Gajula dev->entry[i].vector = i + pdev->irq; 1811fa08a396647767abd24a9e7015cb177121d0cf15Ramachandra Rao Gajula break; 1812063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox } else if (result < 0) { 1813063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox vecs = 1; 1814fa08a396647767abd24a9e7015cb177121d0cf15Ramachandra Rao Gajula break; 1815fa08a396647767abd24a9e7015cb177121d0cf15Ramachandra Rao Gajula } 1816063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox vecs = result; 1817fa08a396647767abd24a9e7015cb177121d0cf15Ramachandra Rao Gajula } 1818fa08a396647767abd24a9e7015cb177121d0cf15Ramachandra Rao Gajula } 1819fa08a396647767abd24a9e7015cb177121d0cf15Ramachandra Rao Gajula 1820063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox /* 1821063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox * Should investigate if there's a performance win from allocating 1822063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox * more queues than interrupt vectors; it might allow the submission 1823063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox * path to scale better, even if the receive path is limited by the 1824063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox * number of interrupts. 1825063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox */ 1826063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox nr_io_queues = vecs; 1827063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox 18281b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox result = queue_request_irq(dev, dev->queues[0], "nvme admin"); 18299d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch if (result) { 18309d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch dev->queues[0]->q_suspended = 1; 1831224042742582c9938788b81165180c876e997a07Keith Busch goto free_queues; 18329d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch } 18331b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox 1834cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch /* Free previously allocated queues that are no longer usable */ 1835cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch spin_lock(&dev_list_lock); 1836cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch for (i = dev->queue_count - 1; i > nr_io_queues; i--) { 1837cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch struct nvme_queue *nvmeq = dev->queues[i]; 1838cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch 18390a8d44cb33377969337fa6c1961b631605d5f453Matthew Wilcox spin_lock_irq(&nvmeq->q_lock); 1840cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch nvme_cancel_ios(nvmeq, false); 18410a8d44cb33377969337fa6c1961b631605d5f453Matthew Wilcox spin_unlock_irq(&nvmeq->q_lock); 1842cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch 1843cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch nvme_free_queue(nvmeq); 1844cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch dev->queue_count--; 1845cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch dev->queues[i] = NULL; 1846cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch } 1847cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch spin_unlock(&dev_list_lock); 1848cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch 18491b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox cpu = cpumask_first(cpu_online_mask); 1850b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox for (i = 0; i < nr_io_queues; i++) { 18511b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox irq_set_affinity_hint(dev->entry[i].vector, get_cpu_mask(cpu)); 18521b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox cpu = cpumask_next(cpu, cpu_online_mask); 18531b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox } 18541b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox 1855a0cadb85b8b758608ae0759151e29de7581c6731Keith Busch q_depth = min_t(int, NVME_CAP_MQES(readq(&dev->bar->cap)) + 1, 1856a0cadb85b8b758608ae0759151e29de7581c6731Keith Busch NVME_Q_DEPTH); 1857cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch for (i = dev->queue_count - 1; i < nr_io_queues; i++) { 1858224042742582c9938788b81165180c876e997a07Keith Busch dev->queues[i + 1] = nvme_alloc_queue(dev, i + 1, q_depth, i); 1859224042742582c9938788b81165180c876e997a07Keith Busch if (!dev->queues[i + 1]) { 1860224042742582c9938788b81165180c876e997a07Keith Busch result = -ENOMEM; 1861224042742582c9938788b81165180c876e997a07Keith Busch goto free_queues; 1862224042742582c9938788b81165180c876e997a07Keith Busch } 18631b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox } 1864b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 18659ecdc946212f7cd592986b2c519b470404caa6b8Matthew Wilcox for (; i < num_possible_cpus(); i++) { 18669ecdc946212f7cd592986b2c519b470404caa6b8Matthew Wilcox int target = i % rounddown_pow_of_two(dev->queue_count - 1); 18679ecdc946212f7cd592986b2c519b470404caa6b8Matthew Wilcox dev->queues[i + 1] = dev->queues[target + 1]; 18689ecdc946212f7cd592986b2c519b470404caa6b8Matthew Wilcox } 18699ecdc946212f7cd592986b2c519b470404caa6b8Matthew Wilcox 1870224042742582c9938788b81165180c876e997a07Keith Busch for (i = 1; i < dev->queue_count; i++) { 1871224042742582c9938788b81165180c876e997a07Keith Busch result = nvme_create_queue(dev->queues[i], i); 1872224042742582c9938788b81165180c876e997a07Keith Busch if (result) { 1873224042742582c9938788b81165180c876e997a07Keith Busch for (--i; i > 0; i--) 1874224042742582c9938788b81165180c876e997a07Keith Busch nvme_disable_queue(dev, i); 1875224042742582c9938788b81165180c876e997a07Keith Busch goto free_queues; 1876224042742582c9938788b81165180c876e997a07Keith Busch } 1877224042742582c9938788b81165180c876e997a07Keith Busch } 1878b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1879224042742582c9938788b81165180c876e997a07Keith Busch return 0; 1880b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1881224042742582c9938788b81165180c876e997a07Keith Busch free_queues: 1882224042742582c9938788b81165180c876e997a07Keith Busch nvme_free_queues(dev); 1883224042742582c9938788b81165180c876e997a07Keith Busch return result; 1884b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1885b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1886422ef0c7c81934c3344e1623d2dcfda5fa2fab8dMatthew Wilcox/* 1887422ef0c7c81934c3344e1623d2dcfda5fa2fab8dMatthew Wilcox * Return: error value if an error occurred setting up the queues or calling 1888422ef0c7c81934c3344e1623d2dcfda5fa2fab8dMatthew Wilcox * Identify Device. 0 if these succeeded, even if adding some of the 1889422ef0c7c81934c3344e1623d2dcfda5fa2fab8dMatthew Wilcox * namespaces failed. At the moment, these failures are silent. TBD which 1890422ef0c7c81934c3344e1623d2dcfda5fa2fab8dMatthew Wilcox * failures should be reported. 1891422ef0c7c81934c3344e1623d2dcfda5fa2fab8dMatthew Wilcox */ 18928d85fce77edfc22f1d6dbf78e3af723b4b556f3dGreg Kroah-Hartmanstatic int nvme_dev_add(struct nvme_dev *dev) 1893b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1894c3bfe7176c035a0a2c70bc79180fb13a6c57142aMatthew Wilcox int res; 1895c3bfe7176c035a0a2c70bc79180fb13a6c57142aMatthew Wilcox unsigned nn, i; 1896cbb6218fd4ae8f98ddf0d66f0826a7a3a9c88298Keith Busch struct nvme_ns *ns; 189751814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox struct nvme_id_ctrl *ctrl; 1898bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox struct nvme_id_ns *id_ns; 1899bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox void *mem; 1900b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_addr_t dma_addr; 1901159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch int shift = NVME_CAP_MPSMIN(readq(&dev->bar->cap)) + 12; 1902b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1903bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox mem = dma_alloc_coherent(&dev->pci_dev->dev, 8192, &dma_addr, 1904b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox GFP_KERNEL); 1905a9ef4343afbe67a6abf83c0f0294e80db48e513aKeith Busch if (!mem) 1906a9ef4343afbe67a6abf83c0f0294e80db48e513aKeith Busch return -ENOMEM; 1907b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1908bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox res = nvme_identify(dev, 0, 1, dma_addr); 1909b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (res) { 1910b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox res = -EIO; 1911cbb6218fd4ae8f98ddf0d66f0826a7a3a9c88298Keith Busch goto out; 1912b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 1913b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1914bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox ctrl = mem; 191551814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox nn = le32_to_cpup(&ctrl->nn); 19160e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch dev->oncs = le16_to_cpup(&ctrl->oncs); 191751814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox memcpy(dev->serial, ctrl->sn, sizeof(ctrl->sn)); 191851814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox memcpy(dev->model, ctrl->mn, sizeof(ctrl->mn)); 191951814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox memcpy(dev->firmware_rev, ctrl->fr, sizeof(ctrl->fr)); 1920159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch if (ctrl->mdts) 19218fc23e032debd682f5ba9fc524a5846c10d2c522Keith Busch dev->max_hw_sectors = 1 << (ctrl->mdts + shift - 9); 1922159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch if ((dev->pci_dev->vendor == PCI_VENDOR_ID_INTEL) && 1923159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch (dev->pci_dev->device == 0x0953) && ctrl->vs[3]) 1924159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch dev->stripe_size = 1 << (ctrl->vs[3] + shift); 1925b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1926bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox id_ns = mem; 19272b2c1896871838cdf549442e8ad0264be5fa74e3Matthew Wilcox for (i = 1; i <= nn; i++) { 1928bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox res = nvme_identify(dev, i, 0, dma_addr); 1929b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (res) 1930b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox continue; 1931b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1932bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox if (id_ns->ncap == 0) 1933b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox continue; 1934b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1935bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox res = nvme_get_features(dev, NVME_FEAT_LBA_RANGE, i, 193608df1e05657fc6712e520e7c09cc6c86160ceb35Keith Busch dma_addr + 4096, NULL); 1937b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (res) 1938122090366d1d5c6ec1bfb6dfdb3a6d121ff074aaKeith Busch memset(mem + 4096, 0, 4096); 1939b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1940bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox ns = nvme_alloc_ns(dev, i, mem, mem + 4096); 1941b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (ns) 1942b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox list_add_tail(&ns->list, &dev->namespaces); 1943b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 1944b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox list_for_each_entry(ns, &dev->namespaces, list) 1945b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox add_disk(ns->disk); 1946422ef0c7c81934c3344e1623d2dcfda5fa2fab8dMatthew Wilcox res = 0; 1947b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1948bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox out: 1949684f5c2025b067a23722e620d0b3b858d8dc5d01Matthew Wilcox dma_free_coherent(&dev->pci_dev->dev, 8192, mem, dma_addr); 1950b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return res; 1951b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1952b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 19530877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Buschstatic int nvme_dev_map(struct nvme_dev *dev) 19540877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch{ 19550877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch int bars, result = -ENOMEM; 19560877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch struct pci_dev *pdev = dev->pci_dev; 19570877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch 19580877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch if (pci_enable_device_mem(pdev)) 19590877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch return result; 19600877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch 19610877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch dev->entry[0].vector = pdev->irq; 19620877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch pci_set_master(pdev); 19630877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch bars = pci_select_bars(pdev, IORESOURCE_MEM); 19640877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch if (pci_request_selected_regions(pdev, bars, "nvme")) 19650877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch goto disable_pci; 19660877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch 1967052d0efada8ccfccd924059b75f5fb119cd10f9aRussell King if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) && 1968052d0efada8ccfccd924059b75f5fb119cd10f9aRussell King dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32))) 1969052d0efada8ccfccd924059b75f5fb119cd10f9aRussell King goto disable; 19700877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch 19710877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch pci_set_drvdata(pdev, dev); 19720877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch dev->bar = ioremap(pci_resource_start(pdev, 0), 8192); 19730877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch if (!dev->bar) 19740877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch goto disable; 19750877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch 1976b80d5ccca3a012e91ca64a2a0b13049163a6a698Haiyan Hu dev->db_stride = 1 << NVME_CAP_STRIDE(readq(&dev->bar->cap)); 19770877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch dev->dbs = ((void __iomem *)dev->bar) + 4096; 19780877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch 19790877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch return 0; 19800877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch 19810877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch disable: 19820877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch pci_release_regions(pdev); 19830877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch disable_pci: 19840877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch pci_disable_device(pdev); 19850877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch return result; 19860877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch} 19870877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch 19880877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Buschstatic void nvme_dev_unmap(struct nvme_dev *dev) 19890877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch{ 19900877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch if (dev->pci_dev->msi_enabled) 19910877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch pci_disable_msi(dev->pci_dev); 19920877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch else if (dev->pci_dev->msix_enabled) 19930877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch pci_disable_msix(dev->pci_dev); 19940877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch 19950877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch if (dev->bar) { 19960877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch iounmap(dev->bar); 19970877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch dev->bar = NULL; 19980877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch } 19990877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch 20000877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch pci_release_regions(dev->pci_dev); 20010877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch if (pci_is_enabled(dev->pci_dev)) 20020877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch pci_disable_device(dev->pci_dev); 20030877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch} 20040877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch 2005f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Buschstatic void nvme_dev_shutdown(struct nvme_dev *dev) 2006b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 2007224042742582c9938788b81165180c876e997a07Keith Busch int i; 2008224042742582c9938788b81165180c876e997a07Keith Busch 2009224042742582c9938788b81165180c876e997a07Keith Busch for (i = dev->queue_count - 1; i >= 0; i--) 2010224042742582c9938788b81165180c876e997a07Keith Busch nvme_disable_queue(dev, i); 2011b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 20121fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_lock(&dev_list_lock); 2013f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch list_del_init(&dev->node); 20141fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_unlock(&dev_list_lock); 20151fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 20161894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch if (dev->bar) 20171894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch nvme_shutdown_ctrl(dev); 2018f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch nvme_dev_unmap(dev); 2019f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch} 2020f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch 2021f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Buschstatic void nvme_dev_remove(struct nvme_dev *dev) 2022f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch{ 2023f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch struct nvme_ns *ns, *next; 2024f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch 2025b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox list_for_each_entry_safe(ns, next, &dev->namespaces, list) { 2026b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox list_del(&ns->list); 2027b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox del_gendisk(ns->disk); 2028b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_ns_free(ns); 2029b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 2030b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 2031b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2032091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcoxstatic int nvme_setup_prp_pools(struct nvme_dev *dev) 2033091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox{ 2034091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox struct device *dmadev = &dev->pci_dev->dev; 2035091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox dev->prp_page_pool = dma_pool_create("prp list page", dmadev, 2036091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox PAGE_SIZE, PAGE_SIZE, 0); 2037091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox if (!dev->prp_page_pool) 2038091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox return -ENOMEM; 2039091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox 204099802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox /* Optimisation for I/Os between 4k and 128k */ 204199802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox dev->prp_small_pool = dma_pool_create("prp list 256", dmadev, 204299802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox 256, 256, 0); 204399802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox if (!dev->prp_small_pool) { 204499802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox dma_pool_destroy(dev->prp_page_pool); 204599802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox return -ENOMEM; 204699802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox } 2047091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox return 0; 2048091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox} 2049091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox 2050091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcoxstatic void nvme_release_prp_pools(struct nvme_dev *dev) 2051091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox{ 2052091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox dma_pool_destroy(dev->prp_page_pool); 205399802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox dma_pool_destroy(dev->prp_small_pool); 2054091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox} 2055091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox 2056cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anhstatic DEFINE_IDA(nvme_instance_ida); 2057cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh 2058cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anhstatic int nvme_set_instance(struct nvme_dev *dev) 2059b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 2060cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh int instance, error; 2061cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh 2062cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh do { 2063cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh if (!ida_pre_get(&nvme_instance_ida, GFP_KERNEL)) 2064cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh return -ENODEV; 2065cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh 2066cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh spin_lock(&dev_list_lock); 2067cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh error = ida_get_new(&nvme_instance_ida, &instance); 2068cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh spin_unlock(&dev_list_lock); 2069cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh } while (error == -EAGAIN); 2070cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh 2071cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh if (error) 2072cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh return -ENODEV; 2073cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh 2074cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh dev->instance = instance; 2075cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh return 0; 2076b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 2077b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2078b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void nvme_release_instance(struct nvme_dev *dev) 2079b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 2080cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh spin_lock(&dev_list_lock); 2081cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh ida_remove(&nvme_instance_ida, dev->instance); 2082cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh spin_unlock(&dev_list_lock); 2083b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 2084b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 20855e82e952f04681c10f35e02ee0a4a43ec027137aKeith Buschstatic void nvme_free_dev(struct kref *kref) 20865e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch{ 20875e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch struct nvme_dev *dev = container_of(kref, struct nvme_dev, kref); 20885e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch nvme_dev_remove(dev); 2089f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch nvme_dev_shutdown(dev); 2090f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch nvme_free_queues(dev); 20915e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch nvme_release_instance(dev); 20925e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch nvme_release_prp_pools(dev); 20935e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch kfree(dev->queues); 20945e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch kfree(dev->entry); 20955e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch kfree(dev); 20965e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch} 20975e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch 20985e82e952f04681c10f35e02ee0a4a43ec027137aKeith Buschstatic int nvme_dev_open(struct inode *inode, struct file *f) 20995e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch{ 21005e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch struct nvme_dev *dev = container_of(f->private_data, struct nvme_dev, 21015e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch miscdev); 21025e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch kref_get(&dev->kref); 21035e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch f->private_data = dev; 21045e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch return 0; 21055e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch} 21065e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch 21075e82e952f04681c10f35e02ee0a4a43ec027137aKeith Buschstatic int nvme_dev_release(struct inode *inode, struct file *f) 21085e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch{ 21095e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch struct nvme_dev *dev = f->private_data; 21105e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch kref_put(&dev->kref, nvme_free_dev); 21115e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch return 0; 21125e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch} 21135e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch 21145e82e952f04681c10f35e02ee0a4a43ec027137aKeith Buschstatic long nvme_dev_ioctl(struct file *f, unsigned int cmd, unsigned long arg) 21155e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch{ 21165e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch struct nvme_dev *dev = f->private_data; 21175e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch switch (cmd) { 21185e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch case NVME_IOCTL_ADMIN_CMD: 21195e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch return nvme_user_admin_cmd(dev, (void __user *)arg); 21205e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch default: 21215e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch return -ENOTTY; 21225e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch } 21235e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch} 21245e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch 21255e82e952f04681c10f35e02ee0a4a43ec027137aKeith Buschstatic const struct file_operations nvme_dev_fops = { 21265e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch .owner = THIS_MODULE, 21275e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch .open = nvme_dev_open, 21285e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch .release = nvme_dev_release, 21295e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch .unlocked_ioctl = nvme_dev_ioctl, 21305e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch .compat_ioctl = nvme_dev_ioctl, 21315e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch}; 21325e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch 2133f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Buschstatic int nvme_dev_start(struct nvme_dev *dev) 2134f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch{ 2135f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch int result; 2136f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch 2137f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch result = nvme_dev_map(dev); 2138f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch if (result) 2139f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch return result; 2140f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch 2141f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch result = nvme_configure_admin_queue(dev); 2142f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch if (result) 2143f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch goto unmap; 2144f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch 2145f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch spin_lock(&dev_list_lock); 2146f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch list_add(&dev->node, &dev_list); 2147f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch spin_unlock(&dev_list_lock); 2148f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch 2149f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch result = nvme_setup_io_queues(dev); 2150d82e8bfdef9afae83b894be49af4644d9ac3c359Keith Busch if (result && result != -EBUSY) 2151f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch goto disable; 2152f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch 2153d82e8bfdef9afae83b894be49af4644d9ac3c359Keith Busch return result; 2154f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch 2155f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch disable: 2156f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch spin_lock(&dev_list_lock); 2157f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch list_del_init(&dev->node); 2158f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch spin_unlock(&dev_list_lock); 2159f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch unmap: 2160f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch nvme_dev_unmap(dev); 2161f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch return result; 2162f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch} 2163f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch 21648d85fce77edfc22f1d6dbf78e3af723b4b556f3dGreg Kroah-Hartmanstatic int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id) 2165b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 21660877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch int result = -ENOMEM; 2167b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_dev *dev; 2168b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2169b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev = kzalloc(sizeof(*dev), GFP_KERNEL); 2170b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!dev) 2171b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -ENOMEM; 2172b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->entry = kcalloc(num_possible_cpus(), sizeof(*dev->entry), 2173b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox GFP_KERNEL); 2174b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!dev->entry) 2175b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto free; 21761b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox dev->queues = kcalloc(num_possible_cpus() + 1, sizeof(void *), 21771b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox GFP_KERNEL); 2178b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!dev->queues) 2179b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto free; 2180b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2181b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox INIT_LIST_HEAD(&dev->namespaces); 2182b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->pci_dev = pdev; 2183052d0efada8ccfccd924059b75f5fb119cd10f9aRussell King 2184cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh result = nvme_set_instance(dev); 2185cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh if (result) 21860877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch goto free; 2187b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2188091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox result = nvme_setup_prp_pools(dev); 2189091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox if (result) 21900877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch goto release; 2191091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox 2192f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch result = nvme_dev_start(dev); 2193d82e8bfdef9afae83b894be49af4644d9ac3c359Keith Busch if (result) { 2194d82e8bfdef9afae83b894be49af4644d9ac3c359Keith Busch if (result == -EBUSY) 2195d82e8bfdef9afae83b894be49af4644d9ac3c359Keith Busch goto create_cdev; 21960877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch goto release_pools; 2197d82e8bfdef9afae83b894be49af4644d9ac3c359Keith Busch } 2198b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2199740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox result = nvme_dev_add(dev); 2200d82e8bfdef9afae83b894be49af4644d9ac3c359Keith Busch if (result) 2201f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch goto shutdown; 2202740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox 2203d82e8bfdef9afae83b894be49af4644d9ac3c359Keith Busch create_cdev: 22045e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch scnprintf(dev->name, sizeof(dev->name), "nvme%d", dev->instance); 22055e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch dev->miscdev.minor = MISC_DYNAMIC_MINOR; 22065e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch dev->miscdev.parent = &pdev->dev; 22075e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch dev->miscdev.name = dev->name; 22085e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch dev->miscdev.fops = &nvme_dev_fops; 22095e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch result = misc_register(&dev->miscdev); 22105e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch if (result) 22115e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch goto remove; 22125e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch 22135e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch kref_init(&dev->kref); 2214b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 2215b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 22165e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch remove: 22175e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch nvme_dev_remove(dev); 2218f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch shutdown: 2219f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch nvme_dev_shutdown(dev); 22200877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch release_pools: 2221f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch nvme_free_queues(dev); 2222091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox nvme_release_prp_pools(dev); 22230877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch release: 22240877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch nvme_release_instance(dev); 2225b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox free: 2226b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(dev->queues); 2227b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(dev->entry); 2228b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(dev); 2229b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return result; 2230b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 2231b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 22328d85fce77edfc22f1d6dbf78e3af723b4b556f3dGreg Kroah-Hartmanstatic void nvme_remove(struct pci_dev *pdev) 2233b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 2234b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_dev *dev = pci_get_drvdata(pdev); 22355e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch misc_deregister(&dev->miscdev); 22365e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch kref_put(&dev->kref, nvme_free_dev); 2237b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 2238b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2239b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* These functions are yet to be implemented */ 2240b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_error_detected NULL 2241b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_dump_registers NULL 2242b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_link_reset NULL 2243b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_slot_reset NULL 2244b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_error_resume NULL 2245cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch 2246cd63894630ab17a192bf97427d16dbec10710a6aKeith Buschstatic int nvme_suspend(struct device *dev) 2247cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch{ 2248cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch struct pci_dev *pdev = to_pci_dev(dev); 2249cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch struct nvme_dev *ndev = pci_get_drvdata(pdev); 2250cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch 2251cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch nvme_dev_shutdown(ndev); 2252cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch return 0; 2253cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch} 2254cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch 2255cd63894630ab17a192bf97427d16dbec10710a6aKeith Buschstatic int nvme_resume(struct device *dev) 2256cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch{ 2257cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch struct pci_dev *pdev = to_pci_dev(dev); 2258cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch struct nvme_dev *ndev = pci_get_drvdata(pdev); 2259cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch int ret; 2260cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch 2261cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch ret = nvme_dev_start(ndev); 2262cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch /* XXX: should remove gendisks if resume fails */ 2263cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch if (ret) 2264cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch nvme_free_queues(ndev); 2265cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch return ret; 2266cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch} 2267cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch 2268cd63894630ab17a192bf97427d16dbec10710a6aKeith Buschstatic SIMPLE_DEV_PM_OPS(nvme_dev_pm_ops, nvme_suspend, nvme_resume); 2269b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 22701d3520357df99baf4ad89f86268ac96cd38092d9Stephen Hemmingerstatic const struct pci_error_handlers nvme_err_handler = { 2271b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .error_detected = nvme_error_detected, 2272b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .mmio_enabled = nvme_dump_registers, 2273b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .link_reset = nvme_link_reset, 2274b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .slot_reset = nvme_slot_reset, 2275b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .resume = nvme_error_resume, 2276b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 2277b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2278b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* Move to pci_ids.h later */ 2279b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define PCI_CLASS_STORAGE_EXPRESS 0x010802 2280b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2281b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic DEFINE_PCI_DEVICE_TABLE(nvme_id_table) = { 2282b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox { PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) }, 2283b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox { 0, } 2284b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 2285b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew WilcoxMODULE_DEVICE_TABLE(pci, nvme_id_table); 2286b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2287b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic struct pci_driver nvme_driver = { 2288b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .name = "nvme", 2289b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .id_table = nvme_id_table, 2290b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .probe = nvme_probe, 22918d85fce77edfc22f1d6dbf78e3af723b4b556f3dGreg Kroah-Hartman .remove = nvme_remove, 2292cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch .driver = { 2293cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch .pm = &nvme_dev_pm_ops, 2294cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch }, 2295b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .err_handler = &nvme_err_handler, 2296b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 2297b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2298b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int __init nvme_init(void) 2299b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 23000ac13140d796eb1e2f8956aea97a6e5e4ebcf981Matthew Wilcox int result; 23011fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 23021fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox nvme_thread = kthread_run(nvme_kthread, NULL, "nvme"); 23031fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox if (IS_ERR(nvme_thread)) 23041fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox return PTR_ERR(nvme_thread); 2305b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 23065c42ea1643a630060f9e71e06d3933d244970967Keith Busch result = register_blkdev(nvme_major, "nvme"); 23075c42ea1643a630060f9e71e06d3933d244970967Keith Busch if (result < 0) 23081fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox goto kill_kthread; 23095c42ea1643a630060f9e71e06d3933d244970967Keith Busch else if (result > 0) 23100ac13140d796eb1e2f8956aea97a6e5e4ebcf981Matthew Wilcox nvme_major = result; 2311b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2312b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox result = pci_register_driver(&nvme_driver); 23131fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox if (result) 23141fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox goto unregister_blkdev; 23151fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox return 0; 2316b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 23171fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox unregister_blkdev: 2318b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unregister_blkdev(nvme_major, "nvme"); 23191fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox kill_kthread: 23201fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox kthread_stop(nvme_thread); 2321b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return result; 2322b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 2323b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2324b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void __exit nvme_exit(void) 2325b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 2326b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox pci_unregister_driver(&nvme_driver); 2327b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unregister_blkdev(nvme_major, "nvme"); 23281fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox kthread_stop(nvme_thread); 2329b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 2330b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2331b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew WilcoxMODULE_AUTHOR("Matthew Wilcox <willy@linux.intel.com>"); 2332b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew WilcoxMODULE_LICENSE("GPL"); 2333366e8217e5ec6ce9f73aec19c46d983110fb4a98Matthew WilcoxMODULE_VERSION("0.8"); 2334b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxmodule_init(nvme_init); 2335b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxmodule_exit(nvme_exit); 2336