nvme-core.c revision 184d2944cb3b92a2e8e1733c59d1e531ad6e924a
1b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 2b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * NVM Express device driver 3b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Copyright (c) 2011, Intel Corporation. 4b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 5b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * This program is free software; you can redistribute it and/or modify it 6b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * under the terms and conditions of the GNU General Public License, 7b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * version 2, as published by the Free Software Foundation. 8b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 9b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * This program is distributed in the hope it will be useful, but WITHOUT 10b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * more details. 13b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 14b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * You should have received a copy of the GNU General Public License along with 15b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * this program; if not, write to the Free Software Foundation, Inc., 16b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. 17b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 18b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 19b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/nvme.h> 20b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/bio.h> 218de055350fbaa96b6563892c195a60be583faa9cMatthew Wilcox#include <linux/bitops.h> 22b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/blkdev.h> 23fd63e9ceeeae58cfe877c2d49d41c1bf7532303cMatthew Wilcox#include <linux/delay.h> 24b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/errno.h> 25b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/fs.h> 26b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/genhd.h> 275aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox#include <linux/idr.h> 28b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/init.h> 29b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/interrupt.h> 30b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/io.h> 31b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/kdev_t.h> 321fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox#include <linux/kthread.h> 33b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/kernel.h> 34b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/mm.h> 35b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/module.h> 36b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/moduleparam.h> 37b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/pci.h> 38be7b62754e097adc0cb16c25c9ee86ee20de62fbMatthew Wilcox#include <linux/poison.h> 39b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/sched.h> 40b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/slab.h> 41b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/types.h> 42b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/version.h> 43b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 44b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define NVME_Q_DEPTH 1024 45b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define SQ_SIZE(depth) (depth * sizeof(struct nvme_command)) 46b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define CQ_SIZE(depth) (depth * sizeof(struct nvme_completion)) 47b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define NVME_MINORS 64 48e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox#define IO_TIMEOUT (5 * HZ) 49e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox#define ADMIN_TIMEOUT (60 * HZ) 50b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 51b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_major; 52b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxmodule_param(nvme_major, int, 0); 53b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 5458ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcoxstatic int use_threaded_interrupts; 5558ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcoxmodule_param(use_threaded_interrupts, int, 0); 5658ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox 571fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcoxstatic DEFINE_SPINLOCK(dev_list_lock); 581fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcoxstatic LIST_HEAD(dev_list); 591fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcoxstatic struct task_struct *nvme_thread; 601fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 61b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 62b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Represents an NVM Express device. Each nvme_dev is a PCI function. 63b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 64b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstruct nvme_dev { 651fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox struct list_head node; 66b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue **queues; 67b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 __iomem *dbs; 68b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct pci_dev *pci_dev; 69091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox struct dma_pool *prp_page_pool; 7099802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox struct dma_pool *prp_small_pool; 71b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int instance; 72b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int queue_count; 73b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 ctrl_config; 74b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct msix_entry *entry; 75b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_bar __iomem *bar; 76b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct list_head namespaces; 7751814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox char serial[20]; 7851814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox char model[40]; 7951814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox char firmware_rev[8]; 80b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 81b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 82b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 83b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * An NVM Express namespace is equivalent to a SCSI LUN 84b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 85b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstruct nvme_ns { 86b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct list_head list; 87b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 88b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_dev *dev; 89b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct request_queue *queue; 90b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct gendisk *disk; 91b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 92b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int ns_id; 93b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int lba_shift; 94b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 95b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 96b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 97b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * An NVM Express queue. Each device has at least two (one for admin 98b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * commands and one for I/O commands). 99b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 100b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstruct nvme_queue { 101b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct device *q_dmadev; 102091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox struct nvme_dev *dev; 103b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox spinlock_t q_lock; 104b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_command *sq_cmds; 105b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox volatile struct nvme_completion *cqes; 106b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_addr_t sq_dma_addr; 107b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_addr_t cq_dma_addr; 108b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox wait_queue_head_t sq_full; 1091fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox wait_queue_t sq_cong_wait; 110b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct bio_list sq_cong; 111b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 __iomem *q_db; 112b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 q_depth; 113b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 cq_vector; 114b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 sq_head; 115b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 sq_tail; 116b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 cq_head; 117821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox u16 cq_phase; 118b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unsigned long cmdid_data[]; 119b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 120b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 121b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 122b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Check we didin't inadvertently grow the command struct 123b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 124b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic inline void _nvme_check_size(void) 125b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 126b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64); 127b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_create_cq) != 64); 128b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_create_sq) != 64); 129b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64); 130b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_features) != 64); 131b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_command) != 64); 132b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != 4096); 133b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_id_ns) != 4096); 134b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64); 135b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 136b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 137e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcoxstruct nvme_cmd_info { 138e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox unsigned long ctx; 139e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox unsigned long timeout; 140e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox}; 141e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox 142e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcoxstatic struct nvme_cmd_info *nvme_cmd_info(struct nvme_queue *nvmeq) 143e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox{ 144e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox return (void *)&nvmeq->cmdid_data[BITS_TO_LONGS(nvmeq->q_depth)]; 145e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox} 146e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox 147b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/** 148714a7a22884b74862540bc84955274d86b2f6040Matthew Wilcox * alloc_cmdid() - Allocate a Command ID 149714a7a22884b74862540bc84955274d86b2f6040Matthew Wilcox * @nvmeq: The queue that will be used for this command 150714a7a22884b74862540bc84955274d86b2f6040Matthew Wilcox * @ctx: A pointer that will be passed to the handler 151714a7a22884b74862540bc84955274d86b2f6040Matthew Wilcox * @handler: The ID of the handler to call 152b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 153b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Allocate a Command ID for a queue. The data passed in will 154b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * be passed to the completion handler. This is implemented by using 155b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * the bottom two bits of the ctx pointer to store the handler ID. 156b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Passing in a pointer that's not 4-byte aligned will cause a BUG. 157b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * We can change this if it becomes a problem. 158184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox * 159184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox * May be called with local interrupts disabled and the q_lock held, 160184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox * or with interrupts enabled and no locks held. 161b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 162e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcoxstatic int alloc_cmdid(struct nvme_queue *nvmeq, void *ctx, int handler, 163e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox unsigned timeout) 164b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 165e6d15f79f997a98b3a69abbc462fc9041cc1a7b4Matthew Wilcox int depth = nvmeq->q_depth - 1; 166e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox struct nvme_cmd_info *info = nvme_cmd_info(nvmeq); 167b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int cmdid; 168b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 169b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUG_ON((unsigned long)ctx & 3); 170b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 171b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox do { 172b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cmdid = find_first_zero_bit(nvmeq->cmdid_data, depth); 173b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (cmdid >= depth) 174b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -EBUSY; 175b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } while (test_and_set_bit(cmdid, nvmeq->cmdid_data)); 176b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 177e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox info[cmdid].ctx = (unsigned long)ctx | handler; 178e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox info[cmdid].timeout = jiffies + timeout; 179b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return cmdid; 180b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 181b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 182b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int alloc_cmdid_killable(struct nvme_queue *nvmeq, void *ctx, 183e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox int handler, unsigned timeout) 184b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 185b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int cmdid; 186b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox wait_event_killable(nvmeq->sq_full, 187e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox (cmdid = alloc_cmdid(nvmeq, ctx, handler, timeout)) >= 0); 188b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return (cmdid < 0) ? -EINTR : cmdid; 189b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 190b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 191fa92282149842645931580225647238428374758Matthew Wilcox/* 192fa92282149842645931580225647238428374758Matthew Wilcox * If you need more than four handlers, you'll need to change how 193be7b62754e097adc0cb16c25c9ee86ee20de62fbMatthew Wilcox * alloc_cmdid and nvme_process_cq work. Consider using a special 194be7b62754e097adc0cb16c25c9ee86ee20de62fbMatthew Wilcox * CMD_CTX value instead, if that works for your situation. 195b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 196b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxenum { 197b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox sync_completion_id = 0, 198b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox bio_completion_id, 199b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 200b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 20100df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox/* Special values must be a multiple of 4, and less than 0x1000 */ 202be7b62754e097adc0cb16c25c9ee86ee20de62fbMatthew Wilcox#define CMD_CTX_BASE (POISON_POINTER_DELTA + sync_completion_id) 203d2d8703481f60d67f49e3177196cbe474b11377cMatthew Wilcox#define CMD_CTX_CANCELLED (0x30C + CMD_CTX_BASE) 204d2d8703481f60d67f49e3177196cbe474b11377cMatthew Wilcox#define CMD_CTX_COMPLETED (0x310 + CMD_CTX_BASE) 205d2d8703481f60d67f49e3177196cbe474b11377cMatthew Wilcox#define CMD_CTX_INVALID (0x314 + CMD_CTX_BASE) 20600df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox#define CMD_CTX_FLUSH (0x318 + CMD_CTX_BASE) 207be7b62754e097adc0cb16c25c9ee86ee20de62fbMatthew Wilcox 208184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox/* 209184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox * Called with local interrupts disabled and the q_lock held. May not sleep. 210184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox */ 211b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic unsigned long free_cmdid(struct nvme_queue *nvmeq, int cmdid) 212b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 213b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unsigned long data; 214e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox struct nvme_cmd_info *info = nvme_cmd_info(nvmeq); 215b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 216e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox if (cmdid >= nvmeq->q_depth) 21748e3d39816416b3bf03dee3a796c0c04427c1a31Matthew Wilcox return CMD_CTX_INVALID; 218e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox data = info[cmdid].ctx; 219e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox info[cmdid].ctx = CMD_CTX_COMPLETED; 220b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox clear_bit(cmdid, nvmeq->cmdid_data); 221b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox wake_up(&nvmeq->sq_full); 222b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return data; 223b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 224b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 22521075bdee0a6f56058920d889df4ae561bfed754Matthew Wilcoxstatic unsigned long cancel_cmdid(struct nvme_queue *nvmeq, int cmdid) 2263c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox{ 22721075bdee0a6f56058920d889df4ae561bfed754Matthew Wilcox unsigned long data; 228e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox struct nvme_cmd_info *info = nvme_cmd_info(nvmeq); 22921075bdee0a6f56058920d889df4ae561bfed754Matthew Wilcox data = info[cmdid].ctx; 230e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox info[cmdid].ctx = CMD_CTX_CANCELLED; 23121075bdee0a6f56058920d889df4ae561bfed754Matthew Wilcox return data; 2323c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox} 2333c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox 234b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic struct nvme_queue *get_nvmeq(struct nvme_ns *ns) 235b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 2369ecdc946212f7cd592986b2c519b470404caa6b8Matthew Wilcox return ns->dev->queues[get_cpu() + 1]; 237b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 238b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 239b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void put_nvmeq(struct nvme_queue *nvmeq) 240b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 2411b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox put_cpu(); 242b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 243b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 244b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/** 245714a7a22884b74862540bc84955274d86b2f6040Matthew Wilcox * nvme_submit_cmd() - Copy a command into a queue and ring the doorbell 246b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * @nvmeq: The queue to use 247b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * @cmd: The command to send 248b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 249b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Safe to use from interrupt context 250b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 251b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_submit_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd) 252b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 253b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unsigned long flags; 254b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 tail; 255b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox spin_lock_irqsave(&nvmeq->q_lock, flags); 256b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox tail = nvmeq->sq_tail; 257b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memcpy(&nvmeq->sq_cmds[tail], cmd, sizeof(*cmd)); 258b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (++tail == nvmeq->q_depth) 259b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox tail = 0; 2607547881d0951384f9833ec3a80fac8f3f16f3b98Matthew Wilcox writel(tail, nvmeq->q_db); 261b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->sq_tail = tail; 262b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox spin_unlock_irqrestore(&nvmeq->q_lock, flags); 263b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 264b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 265b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 266b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 267e025344c56e08b155f43ea09647969286c78377cShane Michael Matthewsstruct nvme_prps { 268e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews int npages; 269e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_addr_t first_dma; 270e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews __le64 *list[0]; 271e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews}; 272e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews 273d567760c409f981d35fc755b51d5bf56a99a467bMatthew Wilcoxstatic void nvme_free_prps(struct nvme_dev *dev, struct nvme_prps *prps) 274e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews{ 275e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews const int last_prp = PAGE_SIZE / 8 - 1; 276e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews int i; 277e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_addr_t prp_dma; 278e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews 279e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews if (!prps) 280e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews return; 281e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews 282e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews prp_dma = prps->first_dma; 28399802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox 28499802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox if (prps->npages == 0) 28599802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox dma_pool_free(dev->prp_small_pool, prps->list[0], prp_dma); 286e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews for (i = 0; i < prps->npages; i++) { 287e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews __le64 *prp_list = prps->list[i]; 288e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_addr_t next_prp_dma = le64_to_cpu(prp_list[last_prp]); 289091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox dma_pool_free(dev->prp_page_pool, prp_list, prp_dma); 290e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews prp_dma = next_prp_dma; 291e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews } 292e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews kfree(prps); 293e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews} 294e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews 295d534df3c730af9073a9ddc076d9fd65cbdca22b3Matthew Wilcoxstruct nvme_bio { 296b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct bio *bio; 297b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int nents; 298e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews struct nvme_prps *prps; 299b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct scatterlist sg[0]; 300b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 301b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 302b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* XXX: use a mempool */ 303d534df3c730af9073a9ddc076d9fd65cbdca22b3Matthew Wilcoxstatic struct nvme_bio *alloc_nbio(unsigned nseg, gfp_t gfp) 304b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 305d534df3c730af9073a9ddc076d9fd65cbdca22b3Matthew Wilcox return kzalloc(sizeof(struct nvme_bio) + 306b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox sizeof(struct scatterlist) * nseg, gfp); 307b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 308b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 309d534df3c730af9073a9ddc076d9fd65cbdca22b3Matthew Wilcoxstatic void free_nbio(struct nvme_queue *nvmeq, struct nvme_bio *nbio) 310b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 311d567760c409f981d35fc755b51d5bf56a99a467bMatthew Wilcox nvme_free_prps(nvmeq->dev, nbio->prps); 312d534df3c730af9073a9ddc076d9fd65cbdca22b3Matthew Wilcox kfree(nbio); 313b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 314b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 315b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void bio_completion(struct nvme_queue *nvmeq, void *ctx, 316b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_completion *cqe) 317b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 318d534df3c730af9073a9ddc076d9fd65cbdca22b3Matthew Wilcox struct nvme_bio *nbio = ctx; 319d534df3c730af9073a9ddc076d9fd65cbdca22b3Matthew Wilcox struct bio *bio = nbio->bio; 320b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 status = le16_to_cpup(&cqe->status) >> 1; 321b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 322d534df3c730af9073a9ddc076d9fd65cbdca22b3Matthew Wilcox dma_unmap_sg(nvmeq->q_dmadev, nbio->sg, nbio->nents, 323b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox bio_data_dir(bio) ? DMA_TO_DEVICE : DMA_FROM_DEVICE); 324d534df3c730af9073a9ddc076d9fd65cbdca22b3Matthew Wilcox free_nbio(nvmeq, nbio); 32509a58f536436efed02ead722e835cb4ce7674afcMatthew Wilcox if (status) { 3261ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox bio_endio(bio, -EIO); 32709a58f536436efed02ead722e835cb4ce7674afcMatthew Wilcox } else if (bio->bi_vcnt > bio->bi_idx) { 3281ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox bio_list_add(&nvmeq->sq_cong, bio); 3291ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox wake_up_process(nvme_thread); 3301ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox } else { 3311ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox bio_endio(bio, 0); 3321ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox } 333b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 334b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 335184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox/* length is in bytes. gfp flags indicates whether we may sleep. */ 336d567760c409f981d35fc755b51d5bf56a99a467bMatthew Wilcoxstatic struct nvme_prps *nvme_setup_prps(struct nvme_dev *dev, 337e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews struct nvme_common_command *cmd, 338b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox struct scatterlist *sg, int *len, 339b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox gfp_t gfp) 340ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox{ 34199802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox struct dma_pool *pool; 342b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox int length = *len; 343ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox int dma_len = sg_dma_len(sg); 344ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox u64 dma_addr = sg_dma_address(sg); 345ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox int offset = offset_in_page(dma_addr); 346e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews __le64 *prp_list; 347e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_addr_t prp_dma; 348e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews int nprps, npages, i, prp_page; 349e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews struct nvme_prps *prps = NULL; 350ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox 351ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmd->prp1 = cpu_to_le64(dma_addr); 352ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox length -= (PAGE_SIZE - offset); 353ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox if (length <= 0) 354e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews return prps; 355ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox 356ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox dma_len -= (PAGE_SIZE - offset); 357ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox if (dma_len) { 358ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox dma_addr += (PAGE_SIZE - offset); 359ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox } else { 360ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox sg = sg_next(sg); 361ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox dma_addr = sg_dma_address(sg); 362ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox dma_len = sg_dma_len(sg); 363ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox } 364ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox 365ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox if (length <= PAGE_SIZE) { 366ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmd->prp2 = cpu_to_le64(dma_addr); 367e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews return prps; 368e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews } 369e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews 370e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews nprps = DIV_ROUND_UP(length, PAGE_SIZE); 371e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews npages = DIV_ROUND_UP(8 * nprps, PAGE_SIZE); 372b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox prps = kmalloc(sizeof(*prps) + sizeof(__le64 *) * npages, gfp); 373b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox if (!prps) { 374b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox cmd->prp2 = cpu_to_le64(dma_addr); 375b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox *len = (*len - length) + PAGE_SIZE; 376b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox return prps; 377b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox } 378e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews prp_page = 0; 37999802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox if (nprps <= (256 / 8)) { 38099802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox pool = dev->prp_small_pool; 38199802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox prps->npages = 0; 38299802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox } else { 38399802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox pool = dev->prp_page_pool; 38499802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox prps->npages = npages; 38599802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox } 38699802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox 387b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox prp_list = dma_pool_alloc(pool, gfp, &prp_dma); 388b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox if (!prp_list) { 389b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox cmd->prp2 = cpu_to_le64(dma_addr); 390b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox *len = (*len - length) + PAGE_SIZE; 391b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox kfree(prps); 392b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox return NULL; 393b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox } 394e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews prps->list[prp_page++] = prp_list; 395e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews prps->first_dma = prp_dma; 396e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews cmd->prp2 = cpu_to_le64(prp_dma); 397e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews i = 0; 398e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews for (;;) { 3997523d834dd1573610078eb1ac0933f6490232f90Matthew Wilcox if (i == PAGE_SIZE / 8) { 400e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews __le64 *old_prp_list = prp_list; 401b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox prp_list = dma_pool_alloc(pool, gfp, &prp_dma); 402b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox if (!prp_list) { 403b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox *len = (*len - length); 404b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox return prps; 405b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox } 406e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews prps->list[prp_page++] = prp_list; 4077523d834dd1573610078eb1ac0933f6490232f90Matthew Wilcox prp_list[0] = old_prp_list[i - 1]; 4087523d834dd1573610078eb1ac0933f6490232f90Matthew Wilcox old_prp_list[i - 1] = cpu_to_le64(prp_dma); 4097523d834dd1573610078eb1ac0933f6490232f90Matthew Wilcox i = 1; 410e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews } 411e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews prp_list[i++] = cpu_to_le64(dma_addr); 412e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_len -= PAGE_SIZE; 413e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_addr += PAGE_SIZE; 414e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews length -= PAGE_SIZE; 415e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews if (length <= 0) 416e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews break; 417e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews if (dma_len > 0) 418e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews continue; 419e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews BUG_ON(dma_len < 0); 420e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews sg = sg_next(sg); 421e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_addr = sg_dma_address(sg); 422e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_len = sg_dma_len(sg); 423ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox } 424ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox 425e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews return prps; 426ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox} 427ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox 4281ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox/* NVMe scatterlists require no holes in the virtual address */ 4291ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox#define BIOVEC_NOT_VIRT_MERGEABLE(vec1, vec2) ((vec2)->bv_offset || \ 4301ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox (((vec1)->bv_offset + (vec1)->bv_len) % PAGE_SIZE)) 4311ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox 432d534df3c730af9073a9ddc076d9fd65cbdca22b3Matthew Wilcoxstatic int nvme_map_bio(struct device *dev, struct nvme_bio *nbio, 433b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct bio *bio, enum dma_data_direction dma_dir, int psegs) 434b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 435768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox struct bio_vec *bvec, *bvprv = NULL; 436768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox struct scatterlist *sg = NULL; 4371ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox int i, old_idx, length = 0, nsegs = 0; 438b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 439768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox sg_init_table(nbio->sg, psegs); 4401ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox old_idx = bio->bi_idx; 441b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox bio_for_each_segment(bvec, bio, i) { 442768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox if (bvprv && BIOVEC_PHYS_MERGEABLE(bvprv, bvec)) { 443768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox sg->length += bvec->bv_len; 444768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox } else { 4451ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox if (bvprv && BIOVEC_NOT_VIRT_MERGEABLE(bvprv, bvec)) 4461ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox break; 447768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox sg = sg ? sg + 1 : nbio->sg; 448768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox sg_set_page(sg, bvec->bv_page, bvec->bv_len, 449768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox bvec->bv_offset); 450768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox nsegs++; 451768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox } 4521ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox length += bvec->bv_len; 453768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox bvprv = bvec; 454b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 4551ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox bio->bi_idx = i; 456d534df3c730af9073a9ddc076d9fd65cbdca22b3Matthew Wilcox nbio->nents = nsegs; 457768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox sg_mark_end(sg); 4581ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox if (dma_map_sg(dev, nbio->sg, nbio->nents, dma_dir) == 0) { 4591ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox bio->bi_idx = old_idx; 4601ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox return -ENOMEM; 4611ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox } 4621ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox return length; 463b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 464b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 46500df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcoxstatic int nvme_submit_flush(struct nvme_queue *nvmeq, struct nvme_ns *ns, 46600df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox int cmdid) 46700df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox{ 46800df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox struct nvme_command *cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail]; 46900df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 47000df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox memset(cmnd, 0, sizeof(*cmnd)); 47100df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox cmnd->common.opcode = nvme_cmd_flush; 47200df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox cmnd->common.command_id = cmdid; 47300df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox cmnd->common.nsid = cpu_to_le32(ns->ns_id); 47400df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 47500df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox if (++nvmeq->sq_tail == nvmeq->q_depth) 47600df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox nvmeq->sq_tail = 0; 47700df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox writel(nvmeq->sq_tail, nvmeq->q_db); 47800df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 47900df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox return 0; 48000df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox} 48100df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 48200df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcoxstatic int nvme_submit_flush_data(struct nvme_queue *nvmeq, struct nvme_ns *ns) 48300df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox{ 48400df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox int cmdid = alloc_cmdid(nvmeq, (void *)CMD_CTX_FLUSH, 48500df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox sync_completion_id, IO_TIMEOUT); 48600df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox if (unlikely(cmdid < 0)) 48700df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox return cmdid; 48800df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 48900df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox return nvme_submit_flush(nvmeq, ns, cmdid); 49000df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox} 49100df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 492184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox/* 493184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox * Called with local interrupts disabled and the q_lock held. May not sleep. 494184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox */ 495b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_submit_bio_queue(struct nvme_queue *nvmeq, struct nvme_ns *ns, 496b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct bio *bio) 497b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 498ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox struct nvme_command *cmnd; 499d534df3c730af9073a9ddc076d9fd65cbdca22b3Matthew Wilcox struct nvme_bio *nbio; 500b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox enum dma_data_direction dma_dir; 5011ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox int cmdid, length, result = -ENOMEM; 502b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 control; 503b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 dsmgmt; 504b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int psegs = bio_phys_segments(ns->queue, bio); 505b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 50600df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox if ((bio->bi_rw & REQ_FLUSH) && psegs) { 50700df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox result = nvme_submit_flush_data(nvmeq, ns); 50800df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox if (result) 50900df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox return result; 51000df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox } 51100df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 512eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox nbio = alloc_nbio(psegs, GFP_ATOMIC); 513d534df3c730af9073a9ddc076d9fd65cbdca22b3Matthew Wilcox if (!nbio) 514eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox goto nomem; 515d534df3c730af9073a9ddc076d9fd65cbdca22b3Matthew Wilcox nbio->bio = bio; 516b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 517eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox result = -EBUSY; 518d534df3c730af9073a9ddc076d9fd65cbdca22b3Matthew Wilcox cmdid = alloc_cmdid(nvmeq, nbio, bio_completion_id, IO_TIMEOUT); 519b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (unlikely(cmdid < 0)) 520d534df3c730af9073a9ddc076d9fd65cbdca22b3Matthew Wilcox goto free_nbio; 521b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 52200df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox if ((bio->bi_rw & REQ_FLUSH) && !psegs) 52300df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox return nvme_submit_flush(nvmeq, ns, cmdid); 52400df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 525b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox control = 0; 526b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (bio->bi_rw & REQ_FUA) 527b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox control |= NVME_RW_FUA; 528b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (bio->bi_rw & (REQ_FAILFAST_DEV | REQ_RAHEAD)) 529b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox control |= NVME_RW_LR; 530b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 531b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dsmgmt = 0; 532b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (bio->bi_rw & REQ_RAHEAD) 533b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH; 534b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 535ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail]; 536b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 537b8deb62cf271fa9381edc8cf52bcae2f0225c55aMatthew Wilcox memset(cmnd, 0, sizeof(*cmnd)); 538b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (bio_data_dir(bio)) { 539ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.opcode = nvme_cmd_write; 540b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_dir = DMA_TO_DEVICE; 541b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } else { 542ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.opcode = nvme_cmd_read; 543b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_dir = DMA_FROM_DEVICE; 544b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 545b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 5461ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox result = nvme_map_bio(nvmeq->q_dmadev, nbio, bio, dma_dir, psegs); 5471ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox if (result < 0) 548eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox goto free_nbio; 5491ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox length = result; 550b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 551ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.command_id = cmdid; 552ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.nsid = cpu_to_le32(ns->ns_id); 553d567760c409f981d35fc755b51d5bf56a99a467bMatthew Wilcox nbio->prps = nvme_setup_prps(nvmeq->dev, &cmnd->common, nbio->sg, 554b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox &length, GFP_ATOMIC); 555ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.slba = cpu_to_le64(bio->bi_sector >> (ns->lba_shift - 9)); 5561ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox cmnd->rw.length = cpu_to_le16((length >> ns->lba_shift) - 1); 557ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.control = cpu_to_le16(control); 558ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt); 559b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 560d8ee9d69f275769aaad40ef7c944565ff8d2d24fMatthew Wilcox bio->bi_sector += length >> 9; 561d8ee9d69f275769aaad40ef7c944565ff8d2d24fMatthew Wilcox 562b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (++nvmeq->sq_tail == nvmeq->q_depth) 563b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->sq_tail = 0; 5647547881d0951384f9833ec3a80fac8f3f16f3b98Matthew Wilcox writel(nvmeq->sq_tail, nvmeq->q_db); 565b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 5661974b1ae8852324a75fb8cfecbc7b758fd5a2c3cMatthew Wilcox return 0; 5671974b1ae8852324a75fb8cfecbc7b758fd5a2c3cMatthew Wilcox 568d534df3c730af9073a9ddc076d9fd65cbdca22b3Matthew Wilcox free_nbio: 569d534df3c730af9073a9ddc076d9fd65cbdca22b3Matthew Wilcox free_nbio(nvmeq, nbio); 570eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox nomem: 571eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox return result; 572b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 573b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 574b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 575b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * NB: return value of non-zero would mean that we were a stacking driver. 576b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * make_request must always succeed. 577b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 578b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_make_request(struct request_queue *q, struct bio *bio) 579b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 580b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_ns *ns = q->queuedata; 581b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq = get_nvmeq(ns); 582eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox int result = -EBUSY; 583eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox 584eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox spin_lock_irq(&nvmeq->q_lock); 585eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox if (bio_list_empty(&nvmeq->sq_cong)) 586eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox result = nvme_submit_bio_queue(nvmeq, ns, bio); 587eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox if (unlikely(result)) { 588eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox if (bio_list_empty(&nvmeq->sq_cong)) 589eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox add_wait_queue(&nvmeq->sq_full, &nvmeq->sq_cong_wait); 590b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox bio_list_add(&nvmeq->sq_cong, bio); 591b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 592eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox 593eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox spin_unlock_irq(&nvmeq->q_lock); 594b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox put_nvmeq(nvmeq); 595b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 596b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 597b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 598b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 599b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstruct sync_cmd_info { 600b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct task_struct *task; 601b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 result; 602b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int status; 603b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 604b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 605b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void sync_completion(struct nvme_queue *nvmeq, void *ctx, 606b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_completion *cqe) 607b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 608b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct sync_cmd_info *cmdinfo = ctx; 609c42705592be2a539f3027b6f3907de8e8f9591a8Matthew Wilcox if (unlikely((unsigned long)cmdinfo == CMD_CTX_CANCELLED)) 610be7b62754e097adc0cb16c25c9ee86ee20de62fbMatthew Wilcox return; 61100df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox if ((unsigned long)cmdinfo == CMD_CTX_FLUSH) 61200df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox return; 613b36235df01ec4141b4e589571d6789076c346d88Matthew Wilcox if (unlikely((unsigned long)cmdinfo == CMD_CTX_COMPLETED)) { 614b36235df01ec4141b4e589571d6789076c346d88Matthew Wilcox dev_warn(nvmeq->q_dmadev, 615b36235df01ec4141b4e589571d6789076c346d88Matthew Wilcox "completed id %d twice on queue %d\n", 616b36235df01ec4141b4e589571d6789076c346d88Matthew Wilcox cqe->command_id, le16_to_cpup(&cqe->sq_id)); 617b36235df01ec4141b4e589571d6789076c346d88Matthew Wilcox return; 618b36235df01ec4141b4e589571d6789076c346d88Matthew Wilcox } 61948e3d39816416b3bf03dee3a796c0c04427c1a31Matthew Wilcox if (unlikely((unsigned long)cmdinfo == CMD_CTX_INVALID)) { 62048e3d39816416b3bf03dee3a796c0c04427c1a31Matthew Wilcox dev_warn(nvmeq->q_dmadev, 62148e3d39816416b3bf03dee3a796c0c04427c1a31Matthew Wilcox "invalid id %d completed on queue %d\n", 62248e3d39816416b3bf03dee3a796c0c04427c1a31Matthew Wilcox cqe->command_id, le16_to_cpup(&cqe->sq_id)); 62348e3d39816416b3bf03dee3a796c0c04427c1a31Matthew Wilcox return; 62448e3d39816416b3bf03dee3a796c0c04427c1a31Matthew Wilcox } 625b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cmdinfo->result = le32_to_cpup(&cqe->result); 626b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cmdinfo->status = le16_to_cpup(&cqe->status) >> 1; 627b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox wake_up_process(cmdinfo->task); 628b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 629b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 630b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxtypedef void (*completion_fn)(struct nvme_queue *, void *, 631b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_completion *); 632b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 6338de055350fbaa96b6563892c195a60be583faa9cMatthew Wilcoxstatic const completion_fn nvme_completions[4] = { 6348de055350fbaa96b6563892c195a60be583faa9cMatthew Wilcox [sync_completion_id] = sync_completion, 6358de055350fbaa96b6563892c195a60be583faa9cMatthew Wilcox [bio_completion_id] = bio_completion, 6368de055350fbaa96b6563892c195a60be583faa9cMatthew Wilcox}; 6378de055350fbaa96b6563892c195a60be583faa9cMatthew Wilcox 638b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic irqreturn_t nvme_process_cq(struct nvme_queue *nvmeq) 639b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 640821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox u16 head, phase; 641b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 642b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox head = nvmeq->cq_head; 643821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox phase = nvmeq->cq_phase; 644b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 645b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox for (;;) { 646b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unsigned long data; 647b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox void *ptr; 648b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unsigned char handler; 649b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_completion cqe = nvmeq->cqes[head]; 650821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox if ((le16_to_cpu(cqe.status) & 1) != phase) 651b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox break; 652b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->sq_head = le16_to_cpu(cqe.sq_head); 653b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (++head == nvmeq->q_depth) { 654b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox head = 0; 655821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox phase = !phase; 656b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 657b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 658b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox data = free_cmdid(nvmeq, cqe.command_id); 659b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox handler = data & 3; 660b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ptr = (void *)(data & ~3UL); 6618de055350fbaa96b6563892c195a60be583faa9cMatthew Wilcox nvme_completions[handler](nvmeq, ptr, &cqe); 662b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 663b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 664b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox /* If the controller ignores the cq head doorbell and continuously 665b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * writes to the queue, it is theoretically possible to wrap around 666b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * the queue twice and mistakenly return IRQ_NONE. Linux only 667b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * requires that 0.1% of your interrupts are handled, so this isn't 668b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * a big problem. 669b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 670821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox if (head == nvmeq->cq_head && phase == nvmeq->cq_phase) 671b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return IRQ_NONE; 672b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 673b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox writel(head, nvmeq->q_db + 1); 674b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->cq_head = head; 675821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox nvmeq->cq_phase = phase; 676b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 677b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return IRQ_HANDLED; 678b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 679b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 680b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic irqreturn_t nvme_irq(int irq, void *data) 681b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 68258ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox irqreturn_t result; 68358ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox struct nvme_queue *nvmeq = data; 68458ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox spin_lock(&nvmeq->q_lock); 68558ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox result = nvme_process_cq(nvmeq); 68658ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox spin_unlock(&nvmeq->q_lock); 68758ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox return result; 68858ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox} 68958ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox 69058ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcoxstatic irqreturn_t nvme_irq_check(int irq, void *data) 69158ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox{ 69258ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox struct nvme_queue *nvmeq = data; 69358ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox struct nvme_completion cqe = nvmeq->cqes[nvmeq->cq_head]; 69458ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox if ((le16_to_cpu(cqe.status) & 1) != nvmeq->cq_phase) 69558ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox return IRQ_NONE; 69658ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox return IRQ_WAKE_THREAD; 69758ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox} 69858ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox 6993c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcoxstatic void nvme_abort_command(struct nvme_queue *nvmeq, int cmdid) 7003c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox{ 7013c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox spin_lock_irq(&nvmeq->q_lock); 70221075bdee0a6f56058920d889df4ae561bfed754Matthew Wilcox cancel_cmdid(nvmeq, cmdid); 7033c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox spin_unlock_irq(&nvmeq->q_lock); 7043c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox} 7053c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox 706b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 707b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Returns 0 on success. If the result is negative, it's a Linux error code; 708b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * if the result is positive, it's an NVM Express status code 709b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 7103c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcoxstatic int nvme_submit_sync_cmd(struct nvme_queue *nvmeq, 711e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox struct nvme_command *cmd, u32 *result, unsigned timeout) 712b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 713b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int cmdid; 714b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct sync_cmd_info cmdinfo; 715b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 716b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cmdinfo.task = current; 717b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cmdinfo.status = -EINTR; 718b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 719e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox cmdid = alloc_cmdid_killable(nvmeq, &cmdinfo, sync_completion_id, 720e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox timeout); 721b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (cmdid < 0) 722b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return cmdid; 723b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cmd->common.command_id = cmdid; 724b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 7253c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox set_current_state(TASK_KILLABLE); 7263c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox nvme_submit_cmd(nvmeq, cmd); 727b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox schedule(); 728b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 7293c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox if (cmdinfo.status == -EINTR) { 7303c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox nvme_abort_command(nvmeq, cmdid); 7313c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox return -EINTR; 7323c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox } 7333c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox 734b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (result) 735b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox *result = cmdinfo.result; 736b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 737b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return cmdinfo.status; 738b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 739b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 740b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_submit_admin_cmd(struct nvme_dev *dev, struct nvme_command *cmd, 741b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 *result) 742b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 743e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox return nvme_submit_sync_cmd(dev->queues[0], cmd, result, ADMIN_TIMEOUT); 744b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 745b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 746b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id) 747b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 748b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int status; 749b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_command c; 750b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 751b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset(&c, 0, sizeof(c)); 752b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.delete_queue.opcode = opcode; 753b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.delete_queue.qid = cpu_to_le16(id); 754b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 755b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox status = nvme_submit_admin_cmd(dev, &c, NULL); 756b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (status) 757b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -EIO; 758b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 759b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 760b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 761b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_alloc_cq(struct nvme_dev *dev, u16 qid, 762b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq) 763b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 764b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int status; 765b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_command c; 766b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int flags = NVME_QUEUE_PHYS_CONTIG | NVME_CQ_IRQ_ENABLED; 767b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 768b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset(&c, 0, sizeof(c)); 769b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.opcode = nvme_admin_create_cq; 770b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.prp1 = cpu_to_le64(nvmeq->cq_dma_addr); 771b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.cqid = cpu_to_le16(qid); 772b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1); 773b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.cq_flags = cpu_to_le16(flags); 774b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.irq_vector = cpu_to_le16(nvmeq->cq_vector); 775b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 776b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox status = nvme_submit_admin_cmd(dev, &c, NULL); 777b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (status) 778b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -EIO; 779b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 780b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 781b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 782b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_alloc_sq(struct nvme_dev *dev, u16 qid, 783b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq) 784b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 785b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int status; 786b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_command c; 787b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int flags = NVME_QUEUE_PHYS_CONTIG | NVME_SQ_PRIO_MEDIUM; 788b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 789b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset(&c, 0, sizeof(c)); 790b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.opcode = nvme_admin_create_sq; 791b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.prp1 = cpu_to_le64(nvmeq->sq_dma_addr); 792b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.sqid = cpu_to_le16(qid); 793b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1); 794b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.sq_flags = cpu_to_le16(flags); 795b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.cqid = cpu_to_le16(qid); 796b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 797b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox status = nvme_submit_admin_cmd(dev, &c, NULL); 798b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (status) 799b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -EIO; 800b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 801b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 802b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 803b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_delete_cq(struct nvme_dev *dev, u16 cqid) 804b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 805b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return adapter_delete_queue(dev, nvme_admin_delete_cq, cqid); 806b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 807b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 808b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_delete_sq(struct nvme_dev *dev, u16 sqid) 809b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 810b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return adapter_delete_queue(dev, nvme_admin_delete_sq, sqid); 811b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 812b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 813b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void nvme_free_queue(struct nvme_dev *dev, int qid) 814b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 815b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq = dev->queues[qid]; 816aba2080f3f1639f9202f1a52993669844abcfb80Matthew Wilcox int vector = dev->entry[nvmeq->cq_vector].vector; 817b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 818aba2080f3f1639f9202f1a52993669844abcfb80Matthew Wilcox irq_set_affinity_hint(vector, NULL); 819aba2080f3f1639f9202f1a52993669844abcfb80Matthew Wilcox free_irq(vector, nvmeq); 820b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 821b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox /* Don't tell the adapter to delete the admin queue */ 822b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (qid) { 823b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox adapter_delete_sq(dev, qid); 824b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox adapter_delete_cq(dev, qid); 825b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 826b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 827b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth), 828b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox (void *)nvmeq->cqes, nvmeq->cq_dma_addr); 829b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth), 830b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->sq_cmds, nvmeq->sq_dma_addr); 831b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(nvmeq); 832b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 833b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 834b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev, int qid, 835b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int depth, int vector) 836b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 837b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct device *dmadev = &dev->pci_dev->dev; 838e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox unsigned extra = (depth / 8) + (depth * sizeof(struct nvme_cmd_info)); 839b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq = kzalloc(sizeof(*nvmeq) + extra, GFP_KERNEL); 840b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!nvmeq) 841b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 842b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 843b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->cqes = dma_alloc_coherent(dmadev, CQ_SIZE(depth), 844b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox &nvmeq->cq_dma_addr, GFP_KERNEL); 845b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!nvmeq->cqes) 846b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto free_nvmeq; 847b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset((void *)nvmeq->cqes, 0, CQ_SIZE(depth)); 848b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 849b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->sq_cmds = dma_alloc_coherent(dmadev, SQ_SIZE(depth), 850b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox &nvmeq->sq_dma_addr, GFP_KERNEL); 851b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!nvmeq->sq_cmds) 852b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto free_cqdma; 853b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 854b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->q_dmadev = dmadev; 855091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox nvmeq->dev = dev; 856b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox spin_lock_init(&nvmeq->q_lock); 857b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->cq_head = 0; 858821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox nvmeq->cq_phase = 1; 859b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox init_waitqueue_head(&nvmeq->sq_full); 8601fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox init_waitqueue_entry(&nvmeq->sq_cong_wait, nvme_thread); 861b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox bio_list_init(&nvmeq->sq_cong); 862b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->q_db = &dev->dbs[qid * 2]; 863b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->q_depth = depth; 864b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->cq_vector = vector; 865b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 866b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return nvmeq; 867b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 868b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox free_cqdma: 869b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_free_coherent(dmadev, CQ_SIZE(nvmeq->q_depth), (void *)nvmeq->cqes, 870b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->cq_dma_addr); 871b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox free_nvmeq: 872b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(nvmeq); 873b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 874b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 875b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 8763001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcoxstatic int queue_request_irq(struct nvme_dev *dev, struct nvme_queue *nvmeq, 8773001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox const char *name) 8783001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox{ 87958ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox if (use_threaded_interrupts) 88058ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox return request_threaded_irq(dev->entry[nvmeq->cq_vector].vector, 881ec6ce618d65b5ce1bef83a5509255107a0feac44Matthew Wilcox nvme_irq_check, nvme_irq, 88258ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox IRQF_DISABLED | IRQF_SHARED, 88358ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox name, nvmeq); 8843001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox return request_irq(dev->entry[nvmeq->cq_vector].vector, nvme_irq, 8853001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox IRQF_DISABLED | IRQF_SHARED, name, nvmeq); 8863001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox} 8873001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox 888b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic __devinit struct nvme_queue *nvme_create_queue(struct nvme_dev *dev, 889b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int qid, int cq_size, int vector) 890b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 891b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int result; 892b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq = nvme_alloc_queue(dev, qid, cq_size, vector); 893b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 8943f85d50b609e8a5ef151656210203a6e94c19538Matthew Wilcox if (!nvmeq) 8953f85d50b609e8a5ef151656210203a6e94c19538Matthew Wilcox return NULL; 8963f85d50b609e8a5ef151656210203a6e94c19538Matthew Wilcox 897b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox result = adapter_alloc_cq(dev, qid, nvmeq); 898b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (result < 0) 899b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto free_nvmeq; 900b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 901b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox result = adapter_alloc_sq(dev, qid, nvmeq); 902b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (result < 0) 903b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto release_cq; 904b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 9053001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox result = queue_request_irq(dev, nvmeq, "nvme"); 906b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (result < 0) 907b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto release_sq; 908b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 909b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return nvmeq; 910b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 911b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox release_sq: 912b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox adapter_delete_sq(dev, qid); 913b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox release_cq: 914b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox adapter_delete_cq(dev, qid); 915b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox free_nvmeq: 916b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth), 917b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox (void *)nvmeq->cqes, nvmeq->cq_dma_addr); 918b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth), 919b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->sq_cmds, nvmeq->sq_dma_addr); 920b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(nvmeq); 921b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 922b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 923b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 924b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int __devinit nvme_configure_admin_queue(struct nvme_dev *dev) 925b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 926b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int result; 927b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 aqa; 92822605f96810d073eb74051d0295b6577d6a6a563Matthew Wilcox u64 cap; 92922605f96810d073eb74051d0295b6577d6a6a563Matthew Wilcox unsigned long timeout; 930b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq; 931b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 932b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->dbs = ((void __iomem *)dev->bar) + 4096; 933b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 934b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq = nvme_alloc_queue(dev, 0, 64, 0); 9353f85d50b609e8a5ef151656210203a6e94c19538Matthew Wilcox if (!nvmeq) 9363f85d50b609e8a5ef151656210203a6e94c19538Matthew Wilcox return -ENOMEM; 937b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 938b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox aqa = nvmeq->q_depth - 1; 939b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox aqa |= aqa << 16; 940b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 941b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->ctrl_config = NVME_CC_ENABLE | NVME_CC_CSS_NVM; 942b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->ctrl_config |= (PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT; 943b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->ctrl_config |= NVME_CC_ARB_RR | NVME_CC_SHN_NONE; 9447f53f9d2424533256ae86f7df5661a17de743de8Matthew Wilcox dev->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES; 945b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 9465911f20039ce59d7e7834f0c42151cf759b6f786Shane Michael Matthews writel(0, &dev->bar->cc); 947b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox writel(aqa, &dev->bar->aqa); 948b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox writeq(nvmeq->sq_dma_addr, &dev->bar->asq); 949b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox writeq(nvmeq->cq_dma_addr, &dev->bar->acq); 950b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox writel(dev->ctrl_config, &dev->bar->cc); 951b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 95222605f96810d073eb74051d0295b6577d6a6a563Matthew Wilcox cap = readq(&dev->bar->cap); 95322605f96810d073eb74051d0295b6577d6a6a563Matthew Wilcox timeout = ((NVME_CAP_TIMEOUT(cap) + 1) * HZ / 2) + jiffies; 95422605f96810d073eb74051d0295b6577d6a6a563Matthew Wilcox 955b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox while (!(readl(&dev->bar->csts) & NVME_CSTS_RDY)) { 956b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox msleep(100); 957b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (fatal_signal_pending(current)) 958b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -EINTR; 95922605f96810d073eb74051d0295b6577d6a6a563Matthew Wilcox if (time_after(jiffies, timeout)) { 96022605f96810d073eb74051d0295b6577d6a6a563Matthew Wilcox dev_err(&dev->pci_dev->dev, 96122605f96810d073eb74051d0295b6577d6a6a563Matthew Wilcox "Device not ready; aborting initialisation\n"); 96222605f96810d073eb74051d0295b6577d6a6a563Matthew Wilcox return -ENODEV; 96322605f96810d073eb74051d0295b6577d6a6a563Matthew Wilcox } 964b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 965b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 9663001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox result = queue_request_irq(dev, nvmeq, "nvme admin"); 967b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->queues[0] = nvmeq; 968b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return result; 969b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 970b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 9717fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcoxstatic int nvme_map_user_pages(struct nvme_dev *dev, int write, 9727fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox unsigned long addr, unsigned length, 9737fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox struct scatterlist **sgp) 974b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 97536c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox int i, err, count, nents, offset; 9767fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox struct scatterlist *sg; 9777fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox struct page **pages; 97836c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox 97936c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox if (addr & 3) 98036c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox return -EINVAL; 9817fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox if (!length) 9827fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox return -EINVAL; 9837fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox 98436c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox offset = offset_in_page(addr); 9857fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox count = DIV_ROUND_UP(offset + length, PAGE_SIZE); 9867fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox pages = kcalloc(count, sizeof(*pages), GFP_KERNEL); 98736c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox 98836c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox err = get_user_pages_fast(addr, count, 1, pages); 98936c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox if (err < count) { 99036c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox count = err; 99136c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox err = -EFAULT; 99236c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox goto put_pages; 99336c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox } 9947fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox 9957fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox sg = kcalloc(count, sizeof(*sg), GFP_KERNEL); 99636c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox sg_init_table(sg, count); 997ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox sg_set_page(&sg[0], pages[0], PAGE_SIZE - offset, offset); 9987fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox length -= (PAGE_SIZE - offset); 9997fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox for (i = 1; i < count; i++) { 10007fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox sg_set_page(&sg[i], pages[i], min_t(int, length, PAGE_SIZE), 0); 10017fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox length -= PAGE_SIZE; 10027fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox } 10037fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox 10047fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox err = -ENOMEM; 10057fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox nents = dma_map_sg(&dev->pci_dev->dev, sg, count, 10067fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox write ? DMA_TO_DEVICE : DMA_FROM_DEVICE); 100736c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox if (!nents) 100836c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox goto put_pages; 1009b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 10107fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox kfree(pages); 10117fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox *sgp = sg; 10127fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox return nents; 1013b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 10147fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox put_pages: 10157fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox for (i = 0; i < count; i++) 10167fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox put_page(pages[i]); 10177fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox kfree(pages); 10187fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox return err; 10197fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox} 1020b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 10217fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcoxstatic void nvme_unmap_user_pages(struct nvme_dev *dev, int write, 10227fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox unsigned long addr, int length, 10237fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox struct scatterlist *sg, int nents) 10247fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox{ 10257fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox int i, count; 1026b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 10277fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox count = DIV_ROUND_UP(offset_in_page(addr) + length, PAGE_SIZE); 102836c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox dma_unmap_sg(&dev->pci_dev->dev, sg, nents, DMA_FROM_DEVICE); 10297fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox 103036c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox for (i = 0; i < count; i++) 10317fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox put_page(sg_page(&sg[i])); 10327fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox} 1033b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 10347fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcoxstatic int nvme_submit_user_admin_command(struct nvme_dev *dev, 10357fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox unsigned long addr, unsigned length, 10367fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox struct nvme_command *cmd) 10377fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox{ 1038b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox int err, nents, tmplen = length; 10397fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox struct scatterlist *sg; 1040e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews struct nvme_prps *prps; 10417fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox 10427fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox nents = nvme_map_user_pages(dev, 0, addr, length, &sg); 10437fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox if (nents < 0) 10447fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox return nents; 1045b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox prps = nvme_setup_prps(dev, &cmd->common, sg, &tmplen, GFP_KERNEL); 1046b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox if (tmplen != length) 1047b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox err = -ENOMEM; 1048b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox else 1049b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox err = nvme_submit_admin_cmd(dev, cmd, NULL); 10507fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox nvme_unmap_user_pages(dev, 0, addr, length, sg, nents); 1051d567760c409f981d35fc755b51d5bf56a99a467bMatthew Wilcox nvme_free_prps(dev, prps); 10527fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox return err ? -EIO : 0; 1053b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1054b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1055bd38c5557cf482fc195e2264b32ea62eed60730aMatthew Wilcoxstatic int nvme_identify(struct nvme_ns *ns, unsigned long addr, int cns) 1056b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1057b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_command c; 1058b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1059bd38c5557cf482fc195e2264b32ea62eed60730aMatthew Wilcox memset(&c, 0, sizeof(c)); 1060bd38c5557cf482fc195e2264b32ea62eed60730aMatthew Wilcox c.identify.opcode = nvme_admin_identify; 1061bd38c5557cf482fc195e2264b32ea62eed60730aMatthew Wilcox c.identify.nsid = cns ? 0 : cpu_to_le32(ns->ns_id); 1062bd38c5557cf482fc195e2264b32ea62eed60730aMatthew Wilcox c.identify.cns = cpu_to_le32(cns); 1063bd38c5557cf482fc195e2264b32ea62eed60730aMatthew Wilcox 1064bd38c5557cf482fc195e2264b32ea62eed60730aMatthew Wilcox return nvme_submit_user_admin_command(ns->dev, addr, 4096, &c); 1065bd38c5557cf482fc195e2264b32ea62eed60730aMatthew Wilcox} 1066bd38c5557cf482fc195e2264b32ea62eed60730aMatthew Wilcox 1067bd38c5557cf482fc195e2264b32ea62eed60730aMatthew Wilcoxstatic int nvme_get_range_type(struct nvme_ns *ns, unsigned long addr) 1068bd38c5557cf482fc195e2264b32ea62eed60730aMatthew Wilcox{ 1069bd38c5557cf482fc195e2264b32ea62eed60730aMatthew Wilcox struct nvme_command c; 1070b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1071b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset(&c, 0, sizeof(c)); 1072b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.features.opcode = nvme_admin_get_features; 1073b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.features.nsid = cpu_to_le32(ns->ns_id); 1074b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.features.fid = cpu_to_le32(NVME_FEAT_LBA_RANGE); 1075b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1076bd38c5557cf482fc195e2264b32ea62eed60730aMatthew Wilcox return nvme_submit_user_admin_command(ns->dev, addr, 4096, &c); 1077b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1078b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1079a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcoxstatic int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio) 1080a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox{ 1081a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox struct nvme_dev *dev = ns->dev; 1082a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox struct nvme_queue *nvmeq; 1083a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox struct nvme_user_io io; 1084a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox struct nvme_command c; 1085a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox unsigned length; 1086a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox int nents, status; 1087a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox struct scatterlist *sg; 1088e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews struct nvme_prps *prps; 1089a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox 1090a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox if (copy_from_user(&io, uio, sizeof(io))) 1091a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox return -EFAULT; 10926c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox length = (io.nblocks + 1) << ns->lba_shift; 10936c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox 10946c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox switch (io.opcode) { 10956c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox case nvme_cmd_write: 10966c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox case nvme_cmd_read: 10976c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox nents = nvme_map_user_pages(dev, io.opcode & 1, io.addr, 10986c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox length, &sg); 10996c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox default: 11006c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox return -EFAULT; 11016c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox } 11026c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox 1103a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox if (nents < 0) 1104a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox return nents; 1105a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox 1106a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox memset(&c, 0, sizeof(c)); 1107a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.opcode = io.opcode; 1108a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.flags = io.flags; 11096c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox c.rw.nsid = cpu_to_le32(ns->ns_id); 1110a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.slba = cpu_to_le64(io.slba); 11116c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox c.rw.length = cpu_to_le16(io.nblocks); 1112a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.control = cpu_to_le16(io.control); 1113a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.dsmgmt = cpu_to_le16(io.dsmgmt); 11146c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox c.rw.reftag = io.reftag; 11156c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox c.rw.apptag = io.apptag; 11166c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox c.rw.appmask = io.appmask; 1117a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox /* XXX: metadata */ 1118b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox prps = nvme_setup_prps(dev, &c.common, sg, &length, GFP_KERNEL); 1119a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox 1120d567760c409f981d35fc755b51d5bf56a99a467bMatthew Wilcox nvmeq = get_nvmeq(ns); 1121fa92282149842645931580225647238428374758Matthew Wilcox /* 1122fa92282149842645931580225647238428374758Matthew Wilcox * Since nvme_submit_sync_cmd sleeps, we can't keep preemption 1123b1ad37efcafe396ac3944853589688dd0ec3c64eMatthew Wilcox * disabled. We may be preempted at any point, and be rescheduled 1124b1ad37efcafe396ac3944853589688dd0ec3c64eMatthew Wilcox * to a different CPU. That will cause cacheline bouncing, but no 1125b1ad37efcafe396ac3944853589688dd0ec3c64eMatthew Wilcox * additional races since q_lock already protects against other CPUs. 1126b1ad37efcafe396ac3944853589688dd0ec3c64eMatthew Wilcox */ 1127a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox put_nvmeq(nvmeq); 1128b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox if (length != (io.nblocks + 1) << ns->lba_shift) 1129b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox status = -ENOMEM; 1130b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox else 1131b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox status = nvme_submit_sync_cmd(nvmeq, &c, NULL, IO_TIMEOUT); 1132a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox 1133a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox nvme_unmap_user_pages(dev, io.opcode & 1, io.addr, length, sg, nents); 1134d567760c409f981d35fc755b51d5bf56a99a467bMatthew Wilcox nvme_free_prps(dev, prps); 1135a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox return status; 1136a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox} 1137a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox 11386ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcoxstatic int nvme_download_firmware(struct nvme_ns *ns, 11396ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox struct nvme_dlfw __user *udlfw) 11406ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox{ 11416ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox struct nvme_dev *dev = ns->dev; 11426ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox struct nvme_dlfw dlfw; 11436ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox struct nvme_command c; 1144b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox int nents, status, length; 11456ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox struct scatterlist *sg; 1146e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews struct nvme_prps *prps; 11476ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox 11486ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox if (copy_from_user(&dlfw, udlfw, sizeof(dlfw))) 11496ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox return -EFAULT; 11506ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox if (dlfw.length >= (1 << 30)) 11516ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox return -EINVAL; 1152b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox length = dlfw.length * 4; 11536ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox 1154b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox nents = nvme_map_user_pages(dev, 1, dlfw.addr, length, &sg); 11556ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox if (nents < 0) 11566ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox return nents; 11576ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox 11586ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox memset(&c, 0, sizeof(c)); 11596ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox c.dlfw.opcode = nvme_admin_download_fw; 11606ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox c.dlfw.numd = cpu_to_le32(dlfw.length); 11616ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox c.dlfw.offset = cpu_to_le32(dlfw.offset); 1162b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox prps = nvme_setup_prps(dev, &c.common, sg, &length, GFP_KERNEL); 1163b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox if (length != dlfw.length * 4) 1164b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox status = -ENOMEM; 1165b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox else 1166b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox status = nvme_submit_admin_cmd(dev, &c, NULL); 11676ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox nvme_unmap_user_pages(dev, 0, dlfw.addr, dlfw.length * 4, sg, nents); 1168d567760c409f981d35fc755b51d5bf56a99a467bMatthew Wilcox nvme_free_prps(dev, prps); 11696ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox return status; 11706ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox} 11716ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox 11726ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcoxstatic int nvme_activate_firmware(struct nvme_ns *ns, unsigned long arg) 11736ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox{ 11746ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox struct nvme_dev *dev = ns->dev; 11756ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox struct nvme_command c; 11766ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox 11776ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox memset(&c, 0, sizeof(c)); 11786ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox c.common.opcode = nvme_admin_activate_fw; 11796ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox c.common.rsvd10[0] = cpu_to_le32(arg); 11806ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox 11816ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox return nvme_submit_admin_cmd(dev, &c, NULL); 11826ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox} 11836ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox 1184b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, 1185b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unsigned long arg) 1186b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1187b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_ns *ns = bdev->bd_disk->private_data; 1188b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1189b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox switch (cmd) { 1190b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox case NVME_IOCTL_IDENTIFY_NS: 119136c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox return nvme_identify(ns, arg, 0); 1192b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox case NVME_IOCTL_IDENTIFY_CTRL: 119336c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox return nvme_identify(ns, arg, 1); 1194b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox case NVME_IOCTL_GET_RANGE_TYPE: 1195bd38c5557cf482fc195e2264b32ea62eed60730aMatthew Wilcox return nvme_get_range_type(ns, arg); 1196a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox case NVME_IOCTL_SUBMIT_IO: 1197a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox return nvme_submit_io(ns, (void __user *)arg); 11986ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox case NVME_IOCTL_DOWNLOAD_FW: 11996ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox return nvme_download_firmware(ns, (void __user *)arg); 12006ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox case NVME_IOCTL_ACTIVATE_FW: 12016ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox return nvme_activate_firmware(ns, arg); 1202b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox default: 1203b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -ENOTTY; 1204b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 1205b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1206b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1207b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic const struct block_device_operations nvme_fops = { 1208b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .owner = THIS_MODULE, 1209b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .ioctl = nvme_ioctl, 12104948168280b269a514045766ddd872cfac5968e1Matthew Wilcox .compat_ioctl = nvme_ioctl, 1211b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 1212b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 12138de055350fbaa96b6563892c195a60be583faa9cMatthew Wilcoxstatic void nvme_timeout_ios(struct nvme_queue *nvmeq) 12148de055350fbaa96b6563892c195a60be583faa9cMatthew Wilcox{ 12158de055350fbaa96b6563892c195a60be583faa9cMatthew Wilcox int depth = nvmeq->q_depth - 1; 12168de055350fbaa96b6563892c195a60be583faa9cMatthew Wilcox struct nvme_cmd_info *info = nvme_cmd_info(nvmeq); 12178de055350fbaa96b6563892c195a60be583faa9cMatthew Wilcox unsigned long now = jiffies; 12188de055350fbaa96b6563892c195a60be583faa9cMatthew Wilcox int cmdid; 12198de055350fbaa96b6563892c195a60be583faa9cMatthew Wilcox 12208de055350fbaa96b6563892c195a60be583faa9cMatthew Wilcox for_each_set_bit(cmdid, nvmeq->cmdid_data, depth) { 12218de055350fbaa96b6563892c195a60be583faa9cMatthew Wilcox unsigned long data; 12228de055350fbaa96b6563892c195a60be583faa9cMatthew Wilcox void *ptr; 12238de055350fbaa96b6563892c195a60be583faa9cMatthew Wilcox unsigned char handler; 12248de055350fbaa96b6563892c195a60be583faa9cMatthew Wilcox static struct nvme_completion cqe = { .status = cpu_to_le16(NVME_SC_ABORT_REQ) << 1, }; 12258de055350fbaa96b6563892c195a60be583faa9cMatthew Wilcox 12268de055350fbaa96b6563892c195a60be583faa9cMatthew Wilcox if (!time_after(now, info[cmdid].timeout)) 12278de055350fbaa96b6563892c195a60be583faa9cMatthew Wilcox continue; 12288de055350fbaa96b6563892c195a60be583faa9cMatthew Wilcox dev_warn(nvmeq->q_dmadev, "Timing out I/O %d\n", cmdid); 12298de055350fbaa96b6563892c195a60be583faa9cMatthew Wilcox data = cancel_cmdid(nvmeq, cmdid); 12308de055350fbaa96b6563892c195a60be583faa9cMatthew Wilcox handler = data & 3; 12318de055350fbaa96b6563892c195a60be583faa9cMatthew Wilcox ptr = (void *)(data & ~3UL); 12328de055350fbaa96b6563892c195a60be583faa9cMatthew Wilcox nvme_completions[handler](nvmeq, ptr, &cqe); 12338de055350fbaa96b6563892c195a60be583faa9cMatthew Wilcox } 12348de055350fbaa96b6563892c195a60be583faa9cMatthew Wilcox} 12358de055350fbaa96b6563892c195a60be583faa9cMatthew Wilcox 12361fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcoxstatic void nvme_resubmit_bios(struct nvme_queue *nvmeq) 12371fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox{ 12381fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox while (bio_list_peek(&nvmeq->sq_cong)) { 12391fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox struct bio *bio = bio_list_pop(&nvmeq->sq_cong); 12401fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox struct nvme_ns *ns = bio->bi_bdev->bd_disk->private_data; 12411fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox if (nvme_submit_bio_queue(nvmeq, ns, bio)) { 12421fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox bio_list_add_head(&nvmeq->sq_cong, bio); 12431fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox break; 12441fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox } 12453cb967c03926edd2c414082f4cc0feb7b372edaeMatthew Wilcox if (bio_list_empty(&nvmeq->sq_cong)) 12463cb967c03926edd2c414082f4cc0feb7b372edaeMatthew Wilcox remove_wait_queue(&nvmeq->sq_full, 12473cb967c03926edd2c414082f4cc0feb7b372edaeMatthew Wilcox &nvmeq->sq_cong_wait); 12481fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox } 12491fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox} 12501fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 12511fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcoxstatic int nvme_kthread(void *data) 12521fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox{ 12531fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox struct nvme_dev *dev; 12541fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 12551fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox while (!kthread_should_stop()) { 12561fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox __set_current_state(TASK_RUNNING); 12571fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_lock(&dev_list_lock); 12581fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox list_for_each_entry(dev, &dev_list, node) { 12591fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox int i; 12601fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox for (i = 0; i < dev->queue_count; i++) { 12611fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox struct nvme_queue *nvmeq = dev->queues[i]; 1262740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox if (!nvmeq) 1263740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox continue; 12641fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_lock_irq(&nvmeq->q_lock); 12651fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox if (nvme_process_cq(nvmeq)) 12661fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox printk("process_cq did something\n"); 12678de055350fbaa96b6563892c195a60be583faa9cMatthew Wilcox nvme_timeout_ios(nvmeq); 12681fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox nvme_resubmit_bios(nvmeq); 12691fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_unlock_irq(&nvmeq->q_lock); 12701fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox } 12711fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox } 12721fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_unlock(&dev_list_lock); 12731fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox set_current_state(TASK_INTERRUPTIBLE); 12741fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox schedule_timeout(HZ); 12751fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox } 12761fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox return 0; 12771fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox} 12781fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 12795aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcoxstatic DEFINE_IDA(nvme_index_ida); 12805aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox 12815aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcoxstatic int nvme_get_ns_idx(void) 12825aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox{ 12835aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox int index, error; 12845aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox 12855aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox do { 12865aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox if (!ida_pre_get(&nvme_index_ida, GFP_KERNEL)) 12875aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox return -1; 12885aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox 12895aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox spin_lock(&dev_list_lock); 12905aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox error = ida_get_new(&nvme_index_ida, &index); 12915aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox spin_unlock(&dev_list_lock); 12925aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox } while (error == -EAGAIN); 12935aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox 12945aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox if (error) 12955aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox index = -1; 12965aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox return index; 12975aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox} 12985aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox 12995aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcoxstatic void nvme_put_ns_idx(int index) 13005aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox{ 13015aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox spin_lock(&dev_list_lock); 13025aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox ida_remove(&nvme_index_ida, index); 13035aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox spin_unlock(&dev_list_lock); 13045aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox} 13055aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox 13065aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcoxstatic struct nvme_ns *nvme_alloc_ns(struct nvme_dev *dev, int nsid, 1307b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_id_ns *id, struct nvme_lba_range_type *rt) 1308b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1309b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_ns *ns; 1310b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct gendisk *disk; 1311b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int lbaf; 1312b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1313b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (rt->attributes & NVME_LBART_ATTRIB_HIDE) 1314b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 1315b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1316b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns = kzalloc(sizeof(*ns), GFP_KERNEL); 1317b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!ns) 1318b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 1319b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->queue = blk_alloc_queue(GFP_KERNEL); 1320b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!ns->queue) 1321b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto out_free_ns; 1322b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->queue->queue_flags = QUEUE_FLAG_DEFAULT | QUEUE_FLAG_NOMERGES | 1323b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox QUEUE_FLAG_NONROT | QUEUE_FLAG_DISCARD; 1324b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox blk_queue_make_request(ns->queue, nvme_make_request); 1325b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->dev = dev; 1326b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->queue->queuedata = ns; 1327b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1328b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk = alloc_disk(NVME_MINORS); 1329b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!disk) 1330b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto out_free_queue; 13315aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox ns->ns_id = nsid; 1332b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->disk = disk; 1333b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox lbaf = id->flbas & 0xf; 1334b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->lba_shift = id->lbaf[lbaf].ds; 1335b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1336b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk->major = nvme_major; 1337b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk->minors = NVME_MINORS; 13385aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox disk->first_minor = NVME_MINORS * nvme_get_ns_idx(); 1339b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk->fops = &nvme_fops; 1340b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk->private_data = ns; 1341b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk->queue = ns->queue; 1342388f037f4e7f0a24bac6b1a24f144f5d939f58cfMatthew Wilcox disk->driverfs_dev = &dev->pci_dev->dev; 13435aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox sprintf(disk->disk_name, "nvme%dn%d", dev->instance, nsid); 1344b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox set_capacity(disk, le64_to_cpup(&id->nsze) << (ns->lba_shift - 9)); 1345b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1346b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return ns; 1347b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1348b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox out_free_queue: 1349b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox blk_cleanup_queue(ns->queue); 1350b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox out_free_ns: 1351b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(ns); 1352b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 1353b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1354b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1355b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void nvme_ns_free(struct nvme_ns *ns) 1356b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 13575aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox int index = ns->disk->first_minor / NVME_MINORS; 1358b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox put_disk(ns->disk); 13595aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox nvme_put_ns_idx(index); 1360b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox blk_cleanup_queue(ns->queue); 1361b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(ns); 1362b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1363b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1364b3b06812e199f248561ce7824a4a8a9cd573c05aMatthew Wilcoxstatic int set_queue_count(struct nvme_dev *dev, int count) 1365b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1366b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int status; 1367b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 result; 1368b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_command c; 1369b3b06812e199f248561ce7824a4a8a9cd573c05aMatthew Wilcox u32 q_count = (count - 1) | ((count - 1) << 16); 1370b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1371b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset(&c, 0, sizeof(c)); 1372b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.features.opcode = nvme_admin_get_features; 1373b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.features.fid = cpu_to_le32(NVME_FEAT_NUM_QUEUES); 1374b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.features.dword11 = cpu_to_le32(q_count); 1375b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1376b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox status = nvme_submit_admin_cmd(dev, &c, &result); 1377b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (status) 1378b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -EIO; 1379b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return min(result & 0xffff, result >> 16) + 1; 1380b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1381b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1382b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int __devinit nvme_setup_io_queues(struct nvme_dev *dev) 1383b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1384b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox int result, cpu, i, nr_io_queues; 1385b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1386b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox nr_io_queues = num_online_cpus(); 1387b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox result = set_queue_count(dev, nr_io_queues); 13881b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox if (result < 0) 13891b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox return result; 1390b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox if (result < nr_io_queues) 1391b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox nr_io_queues = result; 1392b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 13931b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox /* Deregister the admin queue's interrupt */ 13941b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox free_irq(dev->entry[0].vector, dev->queues[0]); 13951b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox 1396b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox for (i = 0; i < nr_io_queues; i++) 13971b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox dev->entry[i].entry = i; 13981b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox for (;;) { 1399b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox result = pci_enable_msix(dev->pci_dev, dev->entry, 1400b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox nr_io_queues); 14011b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox if (result == 0) { 14021b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox break; 14031b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox } else if (result > 0) { 1404b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox nr_io_queues = result; 14051b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox continue; 14061b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox } else { 1407b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox nr_io_queues = 1; 14081b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox break; 14091b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox } 14101b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox } 14111b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox 14121b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox result = queue_request_irq(dev, dev->queues[0], "nvme admin"); 14131b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox /* XXX: handle failure here */ 14141b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox 14151b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox cpu = cpumask_first(cpu_online_mask); 1416b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox for (i = 0; i < nr_io_queues; i++) { 14171b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox irq_set_affinity_hint(dev->entry[i].vector, get_cpu_mask(cpu)); 14181b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox cpu = cpumask_next(cpu, cpu_online_mask); 14191b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox } 14201b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox 1421b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox for (i = 0; i < nr_io_queues; i++) { 14221b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox dev->queues[i + 1] = nvme_create_queue(dev, i + 1, 14231b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox NVME_Q_DEPTH, i); 14241b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox if (!dev->queues[i + 1]) 14251b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox return -ENOMEM; 14261b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox dev->queue_count++; 14271b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox } 1428b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 14299ecdc946212f7cd592986b2c519b470404caa6b8Matthew Wilcox for (; i < num_possible_cpus(); i++) { 14309ecdc946212f7cd592986b2c519b470404caa6b8Matthew Wilcox int target = i % rounddown_pow_of_two(dev->queue_count - 1); 14319ecdc946212f7cd592986b2c519b470404caa6b8Matthew Wilcox dev->queues[i + 1] = dev->queues[target + 1]; 14329ecdc946212f7cd592986b2c519b470404caa6b8Matthew Wilcox } 14339ecdc946212f7cd592986b2c519b470404caa6b8Matthew Wilcox 1434b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 1435b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1436b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1437b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void nvme_free_queues(struct nvme_dev *dev) 1438b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1439b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int i; 1440b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1441b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox for (i = dev->queue_count - 1; i >= 0; i--) 1442b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_free_queue(dev, i); 1443b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1444b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1445b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int __devinit nvme_dev_add(struct nvme_dev *dev) 1446b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1447b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int res, nn, i; 1448b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_ns *ns, *next; 144951814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox struct nvme_id_ctrl *ctrl; 1450b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox void *id; 1451b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_addr_t dma_addr; 1452b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_command cid, crt; 1453b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1454b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox res = nvme_setup_io_queues(dev); 1455b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (res) 1456b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return res; 1457b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1458b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox /* XXX: Switch to a SG list once prp2 works */ 1459b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox id = dma_alloc_coherent(&dev->pci_dev->dev, 8192, &dma_addr, 1460b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox GFP_KERNEL); 1461b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1462b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset(&cid, 0, sizeof(cid)); 1463b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cid.identify.opcode = nvme_admin_identify; 1464b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cid.identify.nsid = 0; 1465b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cid.identify.prp1 = cpu_to_le64(dma_addr); 1466b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cid.identify.cns = cpu_to_le32(1); 1467b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1468b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox res = nvme_submit_admin_cmd(dev, &cid, NULL); 1469b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (res) { 1470b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox res = -EIO; 1471b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto out_free; 1472b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 1473b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 147451814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox ctrl = id; 147551814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox nn = le32_to_cpup(&ctrl->nn); 147651814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox memcpy(dev->serial, ctrl->sn, sizeof(ctrl->sn)); 147751814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox memcpy(dev->model, ctrl->mn, sizeof(ctrl->mn)); 147851814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox memcpy(dev->firmware_rev, ctrl->fr, sizeof(ctrl->fr)); 1479b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1480b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cid.identify.cns = 0; 1481b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset(&crt, 0, sizeof(crt)); 1482b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox crt.features.opcode = nvme_admin_get_features; 1483b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox crt.features.prp1 = cpu_to_le64(dma_addr + 4096); 1484b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox crt.features.fid = cpu_to_le32(NVME_FEAT_LBA_RANGE); 1485b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1486ac88c36a385b848cb9efcb877fdfc4153a60bcabMatthew Wilcox for (i = 0; i <= nn; i++) { 1487b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cid.identify.nsid = cpu_to_le32(i); 1488b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox res = nvme_submit_admin_cmd(dev, &cid, NULL); 1489b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (res) 1490b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox continue; 1491b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1492b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (((struct nvme_id_ns *)id)->ncap == 0) 1493b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox continue; 1494b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1495b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox crt.features.nsid = cpu_to_le32(i); 1496b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox res = nvme_submit_admin_cmd(dev, &crt, NULL); 1497b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (res) 1498b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox continue; 1499b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1500b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns = nvme_alloc_ns(dev, i, id, id + 4096); 1501b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (ns) 1502b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox list_add_tail(&ns->list, &dev->namespaces); 1503b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 1504b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox list_for_each_entry(ns, &dev->namespaces, list) 1505b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox add_disk(ns->disk); 1506b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1507b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_free_coherent(&dev->pci_dev->dev, 4096, id, dma_addr); 1508b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 1509b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1510b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox out_free: 1511b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox list_for_each_entry_safe(ns, next, &dev->namespaces, list) { 1512b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox list_del(&ns->list); 1513b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_ns_free(ns); 1514b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 1515b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1516b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_free_coherent(&dev->pci_dev->dev, 4096, id, dma_addr); 1517b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return res; 1518b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1519b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1520b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_dev_remove(struct nvme_dev *dev) 1521b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1522b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_ns *ns, *next; 1523b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 15241fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_lock(&dev_list_lock); 15251fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox list_del(&dev->node); 15261fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_unlock(&dev_list_lock); 15271fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 1528b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox /* TODO: wait all I/O finished or cancel them */ 1529b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1530b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox list_for_each_entry_safe(ns, next, &dev->namespaces, list) { 1531b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox list_del(&ns->list); 1532b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox del_gendisk(ns->disk); 1533b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_ns_free(ns); 1534b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 1535b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1536b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_free_queues(dev); 1537b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1538b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 1539b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1540b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1541091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcoxstatic int nvme_setup_prp_pools(struct nvme_dev *dev) 1542091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox{ 1543091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox struct device *dmadev = &dev->pci_dev->dev; 1544091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox dev->prp_page_pool = dma_pool_create("prp list page", dmadev, 1545091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox PAGE_SIZE, PAGE_SIZE, 0); 1546091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox if (!dev->prp_page_pool) 1547091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox return -ENOMEM; 1548091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox 154999802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox /* Optimisation for I/Os between 4k and 128k */ 155099802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox dev->prp_small_pool = dma_pool_create("prp list 256", dmadev, 155199802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox 256, 256, 0); 155299802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox if (!dev->prp_small_pool) { 155399802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox dma_pool_destroy(dev->prp_page_pool); 155499802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox return -ENOMEM; 155599802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox } 1556091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox return 0; 1557091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox} 1558091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox 1559091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcoxstatic void nvme_release_prp_pools(struct nvme_dev *dev) 1560091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox{ 1561091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox dma_pool_destroy(dev->prp_page_pool); 156299802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox dma_pool_destroy(dev->prp_small_pool); 1563091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox} 1564091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox 1565b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* XXX: Use an ida or something to let remove / add work correctly */ 1566b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void nvme_set_instance(struct nvme_dev *dev) 1567b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1568b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox static int instance; 1569b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->instance = instance++; 1570b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1571b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1572b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void nvme_release_instance(struct nvme_dev *dev) 1573b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1574b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1575b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1576b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int __devinit nvme_probe(struct pci_dev *pdev, 1577b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox const struct pci_device_id *id) 1578b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1579574e8b95bc3780e10e9b5e9d51074d503dd3d5d9Matthew Wilcox int bars, result = -ENOMEM; 1580b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_dev *dev; 1581b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1582b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev = kzalloc(sizeof(*dev), GFP_KERNEL); 1583b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!dev) 1584b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -ENOMEM; 1585b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->entry = kcalloc(num_possible_cpus(), sizeof(*dev->entry), 1586b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox GFP_KERNEL); 1587b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!dev->entry) 1588b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto free; 15891b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox dev->queues = kcalloc(num_possible_cpus() + 1, sizeof(void *), 15901b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox GFP_KERNEL); 1591b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!dev->queues) 1592b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto free; 1593b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 15940ee5a7d7cb9309bd393a25c395f19fb12a842602Shane Michael Matthews if (pci_enable_device_mem(pdev)) 15950ee5a7d7cb9309bd393a25c395f19fb12a842602Shane Michael Matthews goto free; 1596f64d3365a3e5cb46e69db7e2c82a7cb9a5bed1b8Matthew Wilcox pci_set_master(pdev); 1597574e8b95bc3780e10e9b5e9d51074d503dd3d5d9Matthew Wilcox bars = pci_select_bars(pdev, IORESOURCE_MEM); 1598574e8b95bc3780e10e9b5e9d51074d503dd3d5d9Matthew Wilcox if (pci_request_selected_regions(pdev, bars, "nvme")) 1599574e8b95bc3780e10e9b5e9d51074d503dd3d5d9Matthew Wilcox goto disable; 16000ee5a7d7cb9309bd393a25c395f19fb12a842602Shane Michael Matthews 1601b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox INIT_LIST_HEAD(&dev->namespaces); 1602b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->pci_dev = pdev; 1603b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox pci_set_drvdata(pdev, dev); 16042930353f9f2b9e4629e935acd970cb73c1171229Matthew Wilcox dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)); 16052930353f9f2b9e4629e935acd970cb73c1171229Matthew Wilcox dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64)); 1606b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_set_instance(dev); 160753c9577e9ca68a633c6e9df2b54eaecacfa77f62Matthew Wilcox dev->entry[0].vector = pdev->irq; 1608b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1609091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox result = nvme_setup_prp_pools(dev); 1610091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox if (result) 1611091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox goto disable_msix; 1612091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox 1613b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->bar = ioremap(pci_resource_start(pdev, 0), 8192); 1614b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!dev->bar) { 1615b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox result = -ENOMEM; 1616574e8b95bc3780e10e9b5e9d51074d503dd3d5d9Matthew Wilcox goto disable_msix; 1617b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 1618b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1619b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox result = nvme_configure_admin_queue(dev); 1620b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (result) 1621b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto unmap; 1622b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->queue_count++; 1623b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 16241fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_lock(&dev_list_lock); 16251fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox list_add(&dev->node, &dev_list); 16261fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_unlock(&dev_list_lock); 16271fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 1628740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox result = nvme_dev_add(dev); 1629740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox if (result) 1630740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox goto delete; 1631740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox 1632b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 1633b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1634b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox delete: 1635740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox spin_lock(&dev_list_lock); 1636740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox list_del(&dev->node); 1637740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox spin_unlock(&dev_list_lock); 1638740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox 1639b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_free_queues(dev); 1640b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unmap: 1641b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox iounmap(dev->bar); 1642574e8b95bc3780e10e9b5e9d51074d503dd3d5d9Matthew Wilcox disable_msix: 1643b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox pci_disable_msix(pdev); 1644b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_release_instance(dev); 1645091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox nvme_release_prp_pools(dev); 1646574e8b95bc3780e10e9b5e9d51074d503dd3d5d9Matthew Wilcox disable: 16470ee5a7d7cb9309bd393a25c395f19fb12a842602Shane Michael Matthews pci_disable_device(pdev); 1648574e8b95bc3780e10e9b5e9d51074d503dd3d5d9Matthew Wilcox pci_release_regions(pdev); 1649b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox free: 1650b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(dev->queues); 1651b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(dev->entry); 1652b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(dev); 1653b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return result; 1654b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1655b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1656b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void __devexit nvme_remove(struct pci_dev *pdev) 1657b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1658b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_dev *dev = pci_get_drvdata(pdev); 1659b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_dev_remove(dev); 1660b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox pci_disable_msix(pdev); 1661b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox iounmap(dev->bar); 1662b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_release_instance(dev); 1663091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox nvme_release_prp_pools(dev); 16640ee5a7d7cb9309bd393a25c395f19fb12a842602Shane Michael Matthews pci_disable_device(pdev); 1665574e8b95bc3780e10e9b5e9d51074d503dd3d5d9Matthew Wilcox pci_release_regions(pdev); 1666b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(dev->queues); 1667b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(dev->entry); 1668b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(dev); 1669b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1670b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1671b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* These functions are yet to be implemented */ 1672b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_error_detected NULL 1673b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_dump_registers NULL 1674b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_link_reset NULL 1675b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_slot_reset NULL 1676b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_error_resume NULL 1677b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_suspend NULL 1678b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_resume NULL 1679b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1680b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic struct pci_error_handlers nvme_err_handler = { 1681b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .error_detected = nvme_error_detected, 1682b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .mmio_enabled = nvme_dump_registers, 1683b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .link_reset = nvme_link_reset, 1684b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .slot_reset = nvme_slot_reset, 1685b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .resume = nvme_error_resume, 1686b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 1687b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1688b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* Move to pci_ids.h later */ 1689b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define PCI_CLASS_STORAGE_EXPRESS 0x010802 1690b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1691b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic DEFINE_PCI_DEVICE_TABLE(nvme_id_table) = { 1692b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox { PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) }, 1693b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox { 0, } 1694b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 1695b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew WilcoxMODULE_DEVICE_TABLE(pci, nvme_id_table); 1696b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1697b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic struct pci_driver nvme_driver = { 1698b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .name = "nvme", 1699b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .id_table = nvme_id_table, 1700b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .probe = nvme_probe, 1701b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .remove = __devexit_p(nvme_remove), 1702b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .suspend = nvme_suspend, 1703b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .resume = nvme_resume, 1704b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .err_handler = &nvme_err_handler, 1705b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 1706b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1707b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int __init nvme_init(void) 1708b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 17091fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox int result = -EBUSY; 17101fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 17111fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox nvme_thread = kthread_run(nvme_kthread, NULL, "nvme"); 17121fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox if (IS_ERR(nvme_thread)) 17131fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox return PTR_ERR(nvme_thread); 1714b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1715b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_major = register_blkdev(nvme_major, "nvme"); 1716b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (nvme_major <= 0) 17171fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox goto kill_kthread; 1718b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1719b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox result = pci_register_driver(&nvme_driver); 17201fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox if (result) 17211fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox goto unregister_blkdev; 17221fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox return 0; 1723b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 17241fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox unregister_blkdev: 1725b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unregister_blkdev(nvme_major, "nvme"); 17261fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox kill_kthread: 17271fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox kthread_stop(nvme_thread); 1728b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return result; 1729b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1730b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1731b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void __exit nvme_exit(void) 1732b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1733b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox pci_unregister_driver(&nvme_driver); 1734b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unregister_blkdev(nvme_major, "nvme"); 17351fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox kthread_stop(nvme_thread); 1736b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1737b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1738b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew WilcoxMODULE_AUTHOR("Matthew Wilcox <willy@linux.intel.com>"); 1739b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew WilcoxMODULE_LICENSE("GPL"); 17408ef700678f65e2eef1c3a94cdedb79d757608392Matthew WilcoxMODULE_VERSION("0.5"); 1741b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxmodule_init(nvme_init); 1742b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxmodule_exit(nvme_exit); 1743