nvme-core.c revision b348b7d54368c87811907a8e88f0d96713c43009
1b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 2b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * NVM Express device driver 3b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Copyright (c) 2011, Intel Corporation. 4b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 5b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * This program is free software; you can redistribute it and/or modify it 6b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * under the terms and conditions of the GNU General Public License, 7b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * version 2, as published by the Free Software Foundation. 8b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 9b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * This program is distributed in the hope it will be useful, but WITHOUT 10b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * more details. 13b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 14b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * You should have received a copy of the GNU General Public License along with 15b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * this program; if not, write to the Free Software Foundation, Inc., 16b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. 17b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 18b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 19b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/nvme.h> 20b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/bio.h> 21b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/blkdev.h> 22b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/errno.h> 23b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/fs.h> 24b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/genhd.h> 25b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/init.h> 26b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/interrupt.h> 27b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/io.h> 28b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/kdev_t.h> 291fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox#include <linux/kthread.h> 30b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/kernel.h> 31b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/mm.h> 32b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/module.h> 33b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/moduleparam.h> 34b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/pci.h> 35be7b62754e097adc0cb16c25c9ee86ee20de62fbMatthew Wilcox#include <linux/poison.h> 36b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/sched.h> 37b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/slab.h> 38b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/types.h> 39b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/version.h> 40b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 41b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define NVME_Q_DEPTH 1024 42b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define SQ_SIZE(depth) (depth * sizeof(struct nvme_command)) 43b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define CQ_SIZE(depth) (depth * sizeof(struct nvme_completion)) 44b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define NVME_MINORS 64 45e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox#define IO_TIMEOUT (5 * HZ) 46e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox#define ADMIN_TIMEOUT (60 * HZ) 47b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 48b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_major; 49b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxmodule_param(nvme_major, int, 0); 50b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 5158ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcoxstatic int use_threaded_interrupts; 5258ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcoxmodule_param(use_threaded_interrupts, int, 0); 5358ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox 541fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcoxstatic DEFINE_SPINLOCK(dev_list_lock); 551fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcoxstatic LIST_HEAD(dev_list); 561fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcoxstatic struct task_struct *nvme_thread; 571fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 58b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 59b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Represents an NVM Express device. Each nvme_dev is a PCI function. 60b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 61b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstruct nvme_dev { 621fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox struct list_head node; 63b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue **queues; 64b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 __iomem *dbs; 65b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct pci_dev *pci_dev; 66091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox struct dma_pool *prp_page_pool; 6799802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox struct dma_pool *prp_small_pool; 68b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int instance; 69b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int queue_count; 70b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 ctrl_config; 71b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct msix_entry *entry; 72b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_bar __iomem *bar; 73b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct list_head namespaces; 7451814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox char serial[20]; 7551814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox char model[40]; 7651814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox char firmware_rev[8]; 77b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 78b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 79b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 80b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * An NVM Express namespace is equivalent to a SCSI LUN 81b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 82b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstruct nvme_ns { 83b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct list_head list; 84b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 85b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_dev *dev; 86b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct request_queue *queue; 87b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct gendisk *disk; 88b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 89b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int ns_id; 90b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int lba_shift; 91b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 92b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 93b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 94b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * An NVM Express queue. Each device has at least two (one for admin 95b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * commands and one for I/O commands). 96b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 97b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstruct nvme_queue { 98b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct device *q_dmadev; 99091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox struct nvme_dev *dev; 100b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox spinlock_t q_lock; 101b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_command *sq_cmds; 102b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox volatile struct nvme_completion *cqes; 103b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_addr_t sq_dma_addr; 104b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_addr_t cq_dma_addr; 105b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox wait_queue_head_t sq_full; 1061fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox wait_queue_t sq_cong_wait; 107b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct bio_list sq_cong; 108b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 __iomem *q_db; 109b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 q_depth; 110b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 cq_vector; 111b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 sq_head; 112b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 sq_tail; 113b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 cq_head; 114821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox u16 cq_phase; 115b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unsigned long cmdid_data[]; 116b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 117b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 118b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 119b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Check we didin't inadvertently grow the command struct 120b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 121b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic inline void _nvme_check_size(void) 122b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 123b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64); 124b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_create_cq) != 64); 125b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_create_sq) != 64); 126b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64); 127b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_features) != 64); 128b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_command) != 64); 129b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != 4096); 130b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_id_ns) != 4096); 131b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64); 132b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 133b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 134e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcoxstruct nvme_cmd_info { 135e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox unsigned long ctx; 136e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox unsigned long timeout; 137e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox}; 138e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox 139e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcoxstatic struct nvme_cmd_info *nvme_cmd_info(struct nvme_queue *nvmeq) 140e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox{ 141e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox return (void *)&nvmeq->cmdid_data[BITS_TO_LONGS(nvmeq->q_depth)]; 142e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox} 143e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox 144b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/** 145b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * alloc_cmdid - Allocate a Command ID 146b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * @param nvmeq The queue that will be used for this command 147b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * @param ctx A pointer that will be passed to the handler 148b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * @param handler The ID of the handler to call 149b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 150b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Allocate a Command ID for a queue. The data passed in will 151b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * be passed to the completion handler. This is implemented by using 152b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * the bottom two bits of the ctx pointer to store the handler ID. 153b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Passing in a pointer that's not 4-byte aligned will cause a BUG. 154b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * We can change this if it becomes a problem. 155b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 156e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcoxstatic int alloc_cmdid(struct nvme_queue *nvmeq, void *ctx, int handler, 157e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox unsigned timeout) 158b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 159b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int depth = nvmeq->q_depth; 160e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox struct nvme_cmd_info *info = nvme_cmd_info(nvmeq); 161b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int cmdid; 162b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 163b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUG_ON((unsigned long)ctx & 3); 164b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 165b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox do { 166b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cmdid = find_first_zero_bit(nvmeq->cmdid_data, depth); 167b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (cmdid >= depth) 168b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -EBUSY; 169b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } while (test_and_set_bit(cmdid, nvmeq->cmdid_data)); 170b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 171e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox info[cmdid].ctx = (unsigned long)ctx | handler; 172e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox info[cmdid].timeout = jiffies + timeout; 173b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return cmdid; 174b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 175b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 176b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int alloc_cmdid_killable(struct nvme_queue *nvmeq, void *ctx, 177e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox int handler, unsigned timeout) 178b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 179b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int cmdid; 180b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox wait_event_killable(nvmeq->sq_full, 181e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox (cmdid = alloc_cmdid(nvmeq, ctx, handler, timeout)) >= 0); 182b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return (cmdid < 0) ? -EINTR : cmdid; 183b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 184b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 185b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* If you need more than four handlers, you'll need to change how 186be7b62754e097adc0cb16c25c9ee86ee20de62fbMatthew Wilcox * alloc_cmdid and nvme_process_cq work. Consider using a special 187be7b62754e097adc0cb16c25c9ee86ee20de62fbMatthew Wilcox * CMD_CTX value instead, if that works for your situation. 188b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 189b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxenum { 190b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox sync_completion_id = 0, 191b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox bio_completion_id, 192b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 193b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 194be7b62754e097adc0cb16c25c9ee86ee20de62fbMatthew Wilcox#define CMD_CTX_BASE (POISON_POINTER_DELTA + sync_completion_id) 195d2d8703481f60d67f49e3177196cbe474b11377cMatthew Wilcox#define CMD_CTX_CANCELLED (0x30C + CMD_CTX_BASE) 196d2d8703481f60d67f49e3177196cbe474b11377cMatthew Wilcox#define CMD_CTX_COMPLETED (0x310 + CMD_CTX_BASE) 197d2d8703481f60d67f49e3177196cbe474b11377cMatthew Wilcox#define CMD_CTX_INVALID (0x314 + CMD_CTX_BASE) 198be7b62754e097adc0cb16c25c9ee86ee20de62fbMatthew Wilcox 199b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic unsigned long free_cmdid(struct nvme_queue *nvmeq, int cmdid) 200b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 201b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unsigned long data; 202e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox struct nvme_cmd_info *info = nvme_cmd_info(nvmeq); 203b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 204e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox if (cmdid >= nvmeq->q_depth) 20548e3d39816416b3bf03dee3a796c0c04427c1a31Matthew Wilcox return CMD_CTX_INVALID; 206e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox data = info[cmdid].ctx; 207e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox info[cmdid].ctx = CMD_CTX_COMPLETED; 208b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox clear_bit(cmdid, nvmeq->cmdid_data); 209b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox wake_up(&nvmeq->sq_full); 210b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return data; 211b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 212b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 213be7b62754e097adc0cb16c25c9ee86ee20de62fbMatthew Wilcoxstatic void cancel_cmdid_data(struct nvme_queue *nvmeq, int cmdid) 2143c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox{ 215e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox struct nvme_cmd_info *info = nvme_cmd_info(nvmeq); 216e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox info[cmdid].ctx = CMD_CTX_CANCELLED; 2173c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox} 2183c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox 219b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic struct nvme_queue *get_nvmeq(struct nvme_ns *ns) 220b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 2211b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox int qid, cpu = get_cpu(); 2221b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox if (cpu < ns->dev->queue_count) 2231b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox qid = cpu + 1; 2241b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox else 2251b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox qid = (cpu % rounddown_pow_of_two(ns->dev->queue_count)) + 1; 2261b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox return ns->dev->queues[qid]; 227b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 228b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 229b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void put_nvmeq(struct nvme_queue *nvmeq) 230b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 2311b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox put_cpu(); 232b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 233b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 234b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/** 235b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * nvme_submit_cmd: Copy a command into a queue and ring the doorbell 236b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * @nvmeq: The queue to use 237b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * @cmd: The command to send 238b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 239b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Safe to use from interrupt context 240b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 241b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_submit_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd) 242b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 243b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unsigned long flags; 244b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 tail; 245b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox /* XXX: Need to check tail isn't going to overrun head */ 246b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox spin_lock_irqsave(&nvmeq->q_lock, flags); 247b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox tail = nvmeq->sq_tail; 248b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memcpy(&nvmeq->sq_cmds[tail], cmd, sizeof(*cmd)); 249b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox writel(tail, nvmeq->q_db); 250b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (++tail == nvmeq->q_depth) 251b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox tail = 0; 252b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->sq_tail = tail; 253b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox spin_unlock_irqrestore(&nvmeq->q_lock, flags); 254b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 255b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 256b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 257b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 258e025344c56e08b155f43ea09647969286c78377cShane Michael Matthewsstruct nvme_prps { 259e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews int npages; 260e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_addr_t first_dma; 261e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews __le64 *list[0]; 262e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews}; 263e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews 264d567760c409f981d35fc755b51d5bf56a99a467bMatthew Wilcoxstatic void nvme_free_prps(struct nvme_dev *dev, struct nvme_prps *prps) 265e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews{ 266e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews const int last_prp = PAGE_SIZE / 8 - 1; 267e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews int i; 268e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_addr_t prp_dma; 269e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews 270e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews if (!prps) 271e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews return; 272e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews 273e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews prp_dma = prps->first_dma; 27499802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox 27599802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox if (prps->npages == 0) 27699802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox dma_pool_free(dev->prp_small_pool, prps->list[0], prp_dma); 277e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews for (i = 0; i < prps->npages; i++) { 278e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews __le64 *prp_list = prps->list[i]; 279e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_addr_t next_prp_dma = le64_to_cpu(prp_list[last_prp]); 280091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox dma_pool_free(dev->prp_page_pool, prp_list, prp_dma); 281e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews prp_dma = next_prp_dma; 282e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews } 283e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews kfree(prps); 284e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews} 285e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews 286d534df3c730af9073a9ddc076d9fd65cbdca22b3Matthew Wilcoxstruct nvme_bio { 287b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct bio *bio; 288b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int nents; 289e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews struct nvme_prps *prps; 290b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct scatterlist sg[0]; 291b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 292b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 293b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* XXX: use a mempool */ 294d534df3c730af9073a9ddc076d9fd65cbdca22b3Matthew Wilcoxstatic struct nvme_bio *alloc_nbio(unsigned nseg, gfp_t gfp) 295b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 296d534df3c730af9073a9ddc076d9fd65cbdca22b3Matthew Wilcox return kzalloc(sizeof(struct nvme_bio) + 297b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox sizeof(struct scatterlist) * nseg, gfp); 298b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 299b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 300d534df3c730af9073a9ddc076d9fd65cbdca22b3Matthew Wilcoxstatic void free_nbio(struct nvme_queue *nvmeq, struct nvme_bio *nbio) 301b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 302d567760c409f981d35fc755b51d5bf56a99a467bMatthew Wilcox nvme_free_prps(nvmeq->dev, nbio->prps); 303d534df3c730af9073a9ddc076d9fd65cbdca22b3Matthew Wilcox kfree(nbio); 304b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 305b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 306b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void bio_completion(struct nvme_queue *nvmeq, void *ctx, 307b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_completion *cqe) 308b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 309d534df3c730af9073a9ddc076d9fd65cbdca22b3Matthew Wilcox struct nvme_bio *nbio = ctx; 310d534df3c730af9073a9ddc076d9fd65cbdca22b3Matthew Wilcox struct bio *bio = nbio->bio; 311b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 status = le16_to_cpup(&cqe->status) >> 1; 312b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 313d534df3c730af9073a9ddc076d9fd65cbdca22b3Matthew Wilcox dma_unmap_sg(nvmeq->q_dmadev, nbio->sg, nbio->nents, 314b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox bio_data_dir(bio) ? DMA_TO_DEVICE : DMA_FROM_DEVICE); 315d534df3c730af9073a9ddc076d9fd65cbdca22b3Matthew Wilcox free_nbio(nvmeq, nbio); 316b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox bio_endio(bio, status ? -EIO : 0); 317b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 318b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 319ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox/* length is in bytes */ 320d567760c409f981d35fc755b51d5bf56a99a467bMatthew Wilcoxstatic struct nvme_prps *nvme_setup_prps(struct nvme_dev *dev, 321e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews struct nvme_common_command *cmd, 322ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox struct scatterlist *sg, int length) 323ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox{ 32499802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox struct dma_pool *pool; 325ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox int dma_len = sg_dma_len(sg); 326ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox u64 dma_addr = sg_dma_address(sg); 327ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox int offset = offset_in_page(dma_addr); 328e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews __le64 *prp_list; 329e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_addr_t prp_dma; 330e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews int nprps, npages, i, prp_page; 331e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews struct nvme_prps *prps = NULL; 332ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox 333ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmd->prp1 = cpu_to_le64(dma_addr); 334ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox length -= (PAGE_SIZE - offset); 335ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox if (length <= 0) 336e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews return prps; 337ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox 338ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox dma_len -= (PAGE_SIZE - offset); 339ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox if (dma_len) { 340ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox dma_addr += (PAGE_SIZE - offset); 341ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox } else { 342ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox sg = sg_next(sg); 343ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox dma_addr = sg_dma_address(sg); 344ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox dma_len = sg_dma_len(sg); 345ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox } 346ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox 347ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox if (length <= PAGE_SIZE) { 348ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmd->prp2 = cpu_to_le64(dma_addr); 349e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews return prps; 350e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews } 351e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews 352e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews nprps = DIV_ROUND_UP(length, PAGE_SIZE); 353e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews npages = DIV_ROUND_UP(8 * nprps, PAGE_SIZE); 354e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews prps = kmalloc(sizeof(*prps) + sizeof(__le64 *) * npages, GFP_ATOMIC); 355e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews prp_page = 0; 35699802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox if (nprps <= (256 / 8)) { 35799802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox pool = dev->prp_small_pool; 35899802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox prps->npages = 0; 35999802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox } else { 36099802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox pool = dev->prp_page_pool; 36199802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox prps->npages = npages; 36299802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox } 36399802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox 36499802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma); 365e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews prps->list[prp_page++] = prp_list; 366e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews prps->first_dma = prp_dma; 367e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews cmd->prp2 = cpu_to_le64(prp_dma); 368e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews i = 0; 369e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews for (;;) { 370e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews if (i == PAGE_SIZE / 8 - 1) { 371e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews __le64 *old_prp_list = prp_list; 37299802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma); 373e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews prps->list[prp_page++] = prp_list; 374e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews old_prp_list[i] = cpu_to_le64(prp_dma); 375e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews i = 0; 376e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews } 377e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews prp_list[i++] = cpu_to_le64(dma_addr); 378e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_len -= PAGE_SIZE; 379e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_addr += PAGE_SIZE; 380e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews length -= PAGE_SIZE; 381e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews if (length <= 0) 382e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews break; 383e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews if (dma_len > 0) 384e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews continue; 385e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews BUG_ON(dma_len < 0); 386e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews sg = sg_next(sg); 387e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_addr = sg_dma_address(sg); 388e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_len = sg_dma_len(sg); 389ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox } 390ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox 391e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews return prps; 392ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox} 393ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox 394d534df3c730af9073a9ddc076d9fd65cbdca22b3Matthew Wilcoxstatic int nvme_map_bio(struct device *dev, struct nvme_bio *nbio, 395b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct bio *bio, enum dma_data_direction dma_dir, int psegs) 396b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 397768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox struct bio_vec *bvec, *bvprv = NULL; 398768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox struct scatterlist *sg = NULL; 399768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox int i, nsegs = 0; 400b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 401768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox sg_init_table(nbio->sg, psegs); 402b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox bio_for_each_segment(bvec, bio, i) { 403768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox if (bvprv && BIOVEC_PHYS_MERGEABLE(bvprv, bvec)) { 404768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox sg->length += bvec->bv_len; 405768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox } else { 406768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox /* Check bvprv && offset == 0 */ 407768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox sg = sg ? sg + 1 : nbio->sg; 408768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox sg_set_page(sg, bvec->bv_page, bvec->bv_len, 409768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox bvec->bv_offset); 410768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox nsegs++; 411768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox } 412768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox bvprv = bvec; 413b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 414d534df3c730af9073a9ddc076d9fd65cbdca22b3Matthew Wilcox nbio->nents = nsegs; 415768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox sg_mark_end(sg); 416d534df3c730af9073a9ddc076d9fd65cbdca22b3Matthew Wilcox return dma_map_sg(dev, nbio->sg, nbio->nents, dma_dir); 417b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 418b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 419b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_submit_bio_queue(struct nvme_queue *nvmeq, struct nvme_ns *ns, 420b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct bio *bio) 421b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 422ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox struct nvme_command *cmnd; 423d534df3c730af9073a9ddc076d9fd65cbdca22b3Matthew Wilcox struct nvme_bio *nbio; 424b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox enum dma_data_direction dma_dir; 425eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox int cmdid, result = -ENOMEM; 426b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 control; 427b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 dsmgmt; 428b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int psegs = bio_phys_segments(ns->queue, bio); 429b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 430eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox nbio = alloc_nbio(psegs, GFP_ATOMIC); 431d534df3c730af9073a9ddc076d9fd65cbdca22b3Matthew Wilcox if (!nbio) 432eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox goto nomem; 433d534df3c730af9073a9ddc076d9fd65cbdca22b3Matthew Wilcox nbio->bio = bio; 434b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 435eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox result = -EBUSY; 436d534df3c730af9073a9ddc076d9fd65cbdca22b3Matthew Wilcox cmdid = alloc_cmdid(nvmeq, nbio, bio_completion_id, IO_TIMEOUT); 437b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (unlikely(cmdid < 0)) 438d534df3c730af9073a9ddc076d9fd65cbdca22b3Matthew Wilcox goto free_nbio; 439b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 440b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox control = 0; 441b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (bio->bi_rw & REQ_FUA) 442b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox control |= NVME_RW_FUA; 443b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (bio->bi_rw & (REQ_FAILFAST_DEV | REQ_RAHEAD)) 444b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox control |= NVME_RW_LR; 445b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 446b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dsmgmt = 0; 447b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (bio->bi_rw & REQ_RAHEAD) 448b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH; 449b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 450ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail]; 451b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 452b8deb62cf271fa9381edc8cf52bcae2f0225c55aMatthew Wilcox memset(cmnd, 0, sizeof(*cmnd)); 453b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (bio_data_dir(bio)) { 454ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.opcode = nvme_cmd_write; 455b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_dir = DMA_TO_DEVICE; 456b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } else { 457ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.opcode = nvme_cmd_read; 458b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_dir = DMA_FROM_DEVICE; 459b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 460b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 461eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox result = -ENOMEM; 4621974b1ae8852324a75fb8cfecbc7b758fd5a2c3cMatthew Wilcox if (nvme_map_bio(nvmeq->q_dmadev, nbio, bio, dma_dir, psegs) == 0) 463eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox goto free_nbio; 464b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 465ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.command_id = cmdid; 466ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.nsid = cpu_to_le32(ns->ns_id); 467d567760c409f981d35fc755b51d5bf56a99a467bMatthew Wilcox nbio->prps = nvme_setup_prps(nvmeq->dev, &cmnd->common, nbio->sg, 468e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews bio->bi_size); 469ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.slba = cpu_to_le64(bio->bi_sector >> (ns->lba_shift - 9)); 470ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.length = cpu_to_le16((bio->bi_size >> ns->lba_shift) - 1); 471ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.control = cpu_to_le16(control); 472ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt); 473b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 474b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox writel(nvmeq->sq_tail, nvmeq->q_db); 475b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (++nvmeq->sq_tail == nvmeq->q_depth) 476b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->sq_tail = 0; 477b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 4781974b1ae8852324a75fb8cfecbc7b758fd5a2c3cMatthew Wilcox return 0; 4791974b1ae8852324a75fb8cfecbc7b758fd5a2c3cMatthew Wilcox 480d534df3c730af9073a9ddc076d9fd65cbdca22b3Matthew Wilcox free_nbio: 481d534df3c730af9073a9ddc076d9fd65cbdca22b3Matthew Wilcox free_nbio(nvmeq, nbio); 482eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox nomem: 483eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox return result; 484b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 485b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 486b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 487b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * NB: return value of non-zero would mean that we were a stacking driver. 488b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * make_request must always succeed. 489b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 490b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_make_request(struct request_queue *q, struct bio *bio) 491b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 492b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_ns *ns = q->queuedata; 493b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq = get_nvmeq(ns); 494eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox int result = -EBUSY; 495eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox 496eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox spin_lock_irq(&nvmeq->q_lock); 497eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox if (bio_list_empty(&nvmeq->sq_cong)) 498eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox result = nvme_submit_bio_queue(nvmeq, ns, bio); 499eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox if (unlikely(result)) { 500eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox if (bio_list_empty(&nvmeq->sq_cong)) 501eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox add_wait_queue(&nvmeq->sq_full, &nvmeq->sq_cong_wait); 502b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox bio_list_add(&nvmeq->sq_cong, bio); 503b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 504eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox 505eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox spin_unlock_irq(&nvmeq->q_lock); 506b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox put_nvmeq(nvmeq); 507b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 508b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 509b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 510b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 511b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstruct sync_cmd_info { 512b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct task_struct *task; 513b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 result; 514b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int status; 515b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 516b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 517b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void sync_completion(struct nvme_queue *nvmeq, void *ctx, 518b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_completion *cqe) 519b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 520b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct sync_cmd_info *cmdinfo = ctx; 521be7b62754e097adc0cb16c25c9ee86ee20de62fbMatthew Wilcox if ((unsigned long)cmdinfo == CMD_CTX_CANCELLED) 522be7b62754e097adc0cb16c25c9ee86ee20de62fbMatthew Wilcox return; 523b36235df01ec4141b4e589571d6789076c346d88Matthew Wilcox if (unlikely((unsigned long)cmdinfo == CMD_CTX_COMPLETED)) { 524b36235df01ec4141b4e589571d6789076c346d88Matthew Wilcox dev_warn(nvmeq->q_dmadev, 525b36235df01ec4141b4e589571d6789076c346d88Matthew Wilcox "completed id %d twice on queue %d\n", 526b36235df01ec4141b4e589571d6789076c346d88Matthew Wilcox cqe->command_id, le16_to_cpup(&cqe->sq_id)); 527b36235df01ec4141b4e589571d6789076c346d88Matthew Wilcox return; 528b36235df01ec4141b4e589571d6789076c346d88Matthew Wilcox } 52948e3d39816416b3bf03dee3a796c0c04427c1a31Matthew Wilcox if (unlikely((unsigned long)cmdinfo == CMD_CTX_INVALID)) { 53048e3d39816416b3bf03dee3a796c0c04427c1a31Matthew Wilcox dev_warn(nvmeq->q_dmadev, 53148e3d39816416b3bf03dee3a796c0c04427c1a31Matthew Wilcox "invalid id %d completed on queue %d\n", 53248e3d39816416b3bf03dee3a796c0c04427c1a31Matthew Wilcox cqe->command_id, le16_to_cpup(&cqe->sq_id)); 53348e3d39816416b3bf03dee3a796c0c04427c1a31Matthew Wilcox return; 53448e3d39816416b3bf03dee3a796c0c04427c1a31Matthew Wilcox } 535b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cmdinfo->result = le32_to_cpup(&cqe->result); 536b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cmdinfo->status = le16_to_cpup(&cqe->status) >> 1; 537b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox wake_up_process(cmdinfo->task); 538b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 539b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 540b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxtypedef void (*completion_fn)(struct nvme_queue *, void *, 541b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_completion *); 542b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 543b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic irqreturn_t nvme_process_cq(struct nvme_queue *nvmeq) 544b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 545821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox u16 head, phase; 546b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 547b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox static const completion_fn completions[4] = { 548b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox [sync_completion_id] = sync_completion, 549b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox [bio_completion_id] = bio_completion, 550b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox }; 551b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 552b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox head = nvmeq->cq_head; 553821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox phase = nvmeq->cq_phase; 554b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 555b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox for (;;) { 556b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unsigned long data; 557b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox void *ptr; 558b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unsigned char handler; 559b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_completion cqe = nvmeq->cqes[head]; 560821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox if ((le16_to_cpu(cqe.status) & 1) != phase) 561b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox break; 562b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->sq_head = le16_to_cpu(cqe.sq_head); 563b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (++head == nvmeq->q_depth) { 564b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox head = 0; 565821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox phase = !phase; 566b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 567b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 568b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox data = free_cmdid(nvmeq, cqe.command_id); 569b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox handler = data & 3; 570b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ptr = (void *)(data & ~3UL); 571b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox completions[handler](nvmeq, ptr, &cqe); 572b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 573b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 574b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox /* If the controller ignores the cq head doorbell and continuously 575b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * writes to the queue, it is theoretically possible to wrap around 576b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * the queue twice and mistakenly return IRQ_NONE. Linux only 577b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * requires that 0.1% of your interrupts are handled, so this isn't 578b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * a big problem. 579b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 580821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox if (head == nvmeq->cq_head && phase == nvmeq->cq_phase) 581b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return IRQ_NONE; 582b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 583b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox writel(head, nvmeq->q_db + 1); 584b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->cq_head = head; 585821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox nvmeq->cq_phase = phase; 586b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 587b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return IRQ_HANDLED; 588b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 589b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 590b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic irqreturn_t nvme_irq(int irq, void *data) 591b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 59258ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox irqreturn_t result; 59358ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox struct nvme_queue *nvmeq = data; 59458ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox spin_lock(&nvmeq->q_lock); 59558ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox result = nvme_process_cq(nvmeq); 59658ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox spin_unlock(&nvmeq->q_lock); 59758ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox return result; 59858ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox} 59958ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox 60058ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcoxstatic irqreturn_t nvme_irq_check(int irq, void *data) 60158ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox{ 60258ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox struct nvme_queue *nvmeq = data; 60358ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox struct nvme_completion cqe = nvmeq->cqes[nvmeq->cq_head]; 60458ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox if ((le16_to_cpu(cqe.status) & 1) != nvmeq->cq_phase) 60558ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox return IRQ_NONE; 60658ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox return IRQ_WAKE_THREAD; 60758ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox} 60858ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox 6093c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcoxstatic void nvme_abort_command(struct nvme_queue *nvmeq, int cmdid) 6103c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox{ 6113c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox spin_lock_irq(&nvmeq->q_lock); 612be7b62754e097adc0cb16c25c9ee86ee20de62fbMatthew Wilcox cancel_cmdid_data(nvmeq, cmdid); 6133c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox spin_unlock_irq(&nvmeq->q_lock); 6143c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox} 6153c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox 616b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 617b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Returns 0 on success. If the result is negative, it's a Linux error code; 618b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * if the result is positive, it's an NVM Express status code 619b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 6203c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcoxstatic int nvme_submit_sync_cmd(struct nvme_queue *nvmeq, 621e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox struct nvme_command *cmd, u32 *result, unsigned timeout) 622b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 623b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int cmdid; 624b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct sync_cmd_info cmdinfo; 625b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 626b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cmdinfo.task = current; 627b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cmdinfo.status = -EINTR; 628b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 629e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox cmdid = alloc_cmdid_killable(nvmeq, &cmdinfo, sync_completion_id, 630e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox timeout); 631b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (cmdid < 0) 632b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return cmdid; 633b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cmd->common.command_id = cmdid; 634b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 6353c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox set_current_state(TASK_KILLABLE); 6363c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox nvme_submit_cmd(nvmeq, cmd); 637b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox schedule(); 638b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 6393c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox if (cmdinfo.status == -EINTR) { 6403c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox nvme_abort_command(nvmeq, cmdid); 6413c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox return -EINTR; 6423c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox } 6433c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox 644b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (result) 645b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox *result = cmdinfo.result; 646b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 647b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return cmdinfo.status; 648b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 649b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 650b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_submit_admin_cmd(struct nvme_dev *dev, struct nvme_command *cmd, 651b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 *result) 652b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 653e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox return nvme_submit_sync_cmd(dev->queues[0], cmd, result, ADMIN_TIMEOUT); 654b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 655b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 656b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id) 657b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 658b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int status; 659b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_command c; 660b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 661b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset(&c, 0, sizeof(c)); 662b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.delete_queue.opcode = opcode; 663b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.delete_queue.qid = cpu_to_le16(id); 664b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 665b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox status = nvme_submit_admin_cmd(dev, &c, NULL); 666b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (status) 667b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -EIO; 668b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 669b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 670b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 671b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_alloc_cq(struct nvme_dev *dev, u16 qid, 672b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq) 673b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 674b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int status; 675b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_command c; 676b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int flags = NVME_QUEUE_PHYS_CONTIG | NVME_CQ_IRQ_ENABLED; 677b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 678b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset(&c, 0, sizeof(c)); 679b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.opcode = nvme_admin_create_cq; 680b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.prp1 = cpu_to_le64(nvmeq->cq_dma_addr); 681b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.cqid = cpu_to_le16(qid); 682b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1); 683b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.cq_flags = cpu_to_le16(flags); 684b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.irq_vector = cpu_to_le16(nvmeq->cq_vector); 685b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 686b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox status = nvme_submit_admin_cmd(dev, &c, NULL); 687b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (status) 688b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -EIO; 689b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 690b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 691b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 692b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_alloc_sq(struct nvme_dev *dev, u16 qid, 693b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq) 694b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 695b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int status; 696b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_command c; 697b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int flags = NVME_QUEUE_PHYS_CONTIG | NVME_SQ_PRIO_MEDIUM; 698b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 699b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset(&c, 0, sizeof(c)); 700b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.opcode = nvme_admin_create_sq; 701b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.prp1 = cpu_to_le64(nvmeq->sq_dma_addr); 702b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.sqid = cpu_to_le16(qid); 703b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1); 704b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.sq_flags = cpu_to_le16(flags); 705b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.cqid = cpu_to_le16(qid); 706b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 707b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox status = nvme_submit_admin_cmd(dev, &c, NULL); 708b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (status) 709b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -EIO; 710b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 711b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 712b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 713b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_delete_cq(struct nvme_dev *dev, u16 cqid) 714b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 715b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return adapter_delete_queue(dev, nvme_admin_delete_cq, cqid); 716b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 717b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 718b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_delete_sq(struct nvme_dev *dev, u16 sqid) 719b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 720b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return adapter_delete_queue(dev, nvme_admin_delete_sq, sqid); 721b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 722b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 723b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void nvme_free_queue(struct nvme_dev *dev, int qid) 724b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 725b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq = dev->queues[qid]; 726b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 727b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox free_irq(dev->entry[nvmeq->cq_vector].vector, nvmeq); 728b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 729b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox /* Don't tell the adapter to delete the admin queue */ 730b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (qid) { 731b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox adapter_delete_sq(dev, qid); 732b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox adapter_delete_cq(dev, qid); 733b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 734b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 735b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth), 736b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox (void *)nvmeq->cqes, nvmeq->cq_dma_addr); 737b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth), 738b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->sq_cmds, nvmeq->sq_dma_addr); 739b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(nvmeq); 740b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 741b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 742b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev, int qid, 743b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int depth, int vector) 744b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 745b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct device *dmadev = &dev->pci_dev->dev; 746e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox unsigned extra = (depth / 8) + (depth * sizeof(struct nvme_cmd_info)); 747b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq = kzalloc(sizeof(*nvmeq) + extra, GFP_KERNEL); 748b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!nvmeq) 749b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 750b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 751b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->cqes = dma_alloc_coherent(dmadev, CQ_SIZE(depth), 752b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox &nvmeq->cq_dma_addr, GFP_KERNEL); 753b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!nvmeq->cqes) 754b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto free_nvmeq; 755b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset((void *)nvmeq->cqes, 0, CQ_SIZE(depth)); 756b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 757b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->sq_cmds = dma_alloc_coherent(dmadev, SQ_SIZE(depth), 758b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox &nvmeq->sq_dma_addr, GFP_KERNEL); 759b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!nvmeq->sq_cmds) 760b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto free_cqdma; 761b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 762b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->q_dmadev = dmadev; 763091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox nvmeq->dev = dev; 764b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox spin_lock_init(&nvmeq->q_lock); 765b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->cq_head = 0; 766821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox nvmeq->cq_phase = 1; 767b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox init_waitqueue_head(&nvmeq->sq_full); 7681fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox init_waitqueue_entry(&nvmeq->sq_cong_wait, nvme_thread); 769b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox bio_list_init(&nvmeq->sq_cong); 770b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->q_db = &dev->dbs[qid * 2]; 771b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->q_depth = depth; 772b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->cq_vector = vector; 773b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 774b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return nvmeq; 775b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 776b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox free_cqdma: 777b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_free_coherent(dmadev, CQ_SIZE(nvmeq->q_depth), (void *)nvmeq->cqes, 778b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->cq_dma_addr); 779b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox free_nvmeq: 780b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(nvmeq); 781b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 782b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 783b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 7843001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcoxstatic int queue_request_irq(struct nvme_dev *dev, struct nvme_queue *nvmeq, 7853001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox const char *name) 7863001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox{ 78758ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox if (use_threaded_interrupts) 78858ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox return request_threaded_irq(dev->entry[nvmeq->cq_vector].vector, 789ec6ce618d65b5ce1bef83a5509255107a0feac44Matthew Wilcox nvme_irq_check, nvme_irq, 79058ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox IRQF_DISABLED | IRQF_SHARED, 79158ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox name, nvmeq); 7923001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox return request_irq(dev->entry[nvmeq->cq_vector].vector, nvme_irq, 7933001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox IRQF_DISABLED | IRQF_SHARED, name, nvmeq); 7943001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox} 7953001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox 796b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic __devinit struct nvme_queue *nvme_create_queue(struct nvme_dev *dev, 797b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int qid, int cq_size, int vector) 798b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 799b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int result; 800b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq = nvme_alloc_queue(dev, qid, cq_size, vector); 801b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 8023f85d50b609e8a5ef151656210203a6e94c19538Matthew Wilcox if (!nvmeq) 8033f85d50b609e8a5ef151656210203a6e94c19538Matthew Wilcox return NULL; 8043f85d50b609e8a5ef151656210203a6e94c19538Matthew Wilcox 805b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox result = adapter_alloc_cq(dev, qid, nvmeq); 806b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (result < 0) 807b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto free_nvmeq; 808b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 809b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox result = adapter_alloc_sq(dev, qid, nvmeq); 810b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (result < 0) 811b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto release_cq; 812b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 8133001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox result = queue_request_irq(dev, nvmeq, "nvme"); 814b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (result < 0) 815b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto release_sq; 816b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 817b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return nvmeq; 818b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 819b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox release_sq: 820b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox adapter_delete_sq(dev, qid); 821b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox release_cq: 822b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox adapter_delete_cq(dev, qid); 823b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox free_nvmeq: 824b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth), 825b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox (void *)nvmeq->cqes, nvmeq->cq_dma_addr); 826b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth), 827b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->sq_cmds, nvmeq->sq_dma_addr); 828b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(nvmeq); 829b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 830b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 831b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 832b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int __devinit nvme_configure_admin_queue(struct nvme_dev *dev) 833b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 834b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int result; 835b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 aqa; 836b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq; 837b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 838b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->dbs = ((void __iomem *)dev->bar) + 4096; 839b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 840b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq = nvme_alloc_queue(dev, 0, 64, 0); 8413f85d50b609e8a5ef151656210203a6e94c19538Matthew Wilcox if (!nvmeq) 8423f85d50b609e8a5ef151656210203a6e94c19538Matthew Wilcox return -ENOMEM; 843b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 844b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox aqa = nvmeq->q_depth - 1; 845b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox aqa |= aqa << 16; 846b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 847b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->ctrl_config = NVME_CC_ENABLE | NVME_CC_CSS_NVM; 848b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->ctrl_config |= (PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT; 849b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->ctrl_config |= NVME_CC_ARB_RR | NVME_CC_SHN_NONE; 850b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 8515911f20039ce59d7e7834f0c42151cf759b6f786Shane Michael Matthews writel(0, &dev->bar->cc); 852b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox writel(aqa, &dev->bar->aqa); 853b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox writeq(nvmeq->sq_dma_addr, &dev->bar->asq); 854b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox writeq(nvmeq->cq_dma_addr, &dev->bar->acq); 855b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox writel(dev->ctrl_config, &dev->bar->cc); 856b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 857b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox while (!(readl(&dev->bar->csts) & NVME_CSTS_RDY)) { 858b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox msleep(100); 859b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (fatal_signal_pending(current)) 860b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -EINTR; 861b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 862b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 8633001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox result = queue_request_irq(dev, nvmeq, "nvme admin"); 864b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->queues[0] = nvmeq; 865b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return result; 866b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 867b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 8687fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcoxstatic int nvme_map_user_pages(struct nvme_dev *dev, int write, 8697fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox unsigned long addr, unsigned length, 8707fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox struct scatterlist **sgp) 871b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 87236c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox int i, err, count, nents, offset; 8737fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox struct scatterlist *sg; 8747fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox struct page **pages; 87536c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox 87636c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox if (addr & 3) 87736c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox return -EINVAL; 8787fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox if (!length) 8797fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox return -EINVAL; 8807fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox 88136c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox offset = offset_in_page(addr); 8827fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox count = DIV_ROUND_UP(offset + length, PAGE_SIZE); 8837fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox pages = kcalloc(count, sizeof(*pages), GFP_KERNEL); 88436c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox 88536c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox err = get_user_pages_fast(addr, count, 1, pages); 88636c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox if (err < count) { 88736c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox count = err; 88836c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox err = -EFAULT; 88936c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox goto put_pages; 89036c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox } 8917fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox 8927fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox sg = kcalloc(count, sizeof(*sg), GFP_KERNEL); 89336c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox sg_init_table(sg, count); 894ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox sg_set_page(&sg[0], pages[0], PAGE_SIZE - offset, offset); 8957fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox length -= (PAGE_SIZE - offset); 8967fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox for (i = 1; i < count; i++) { 8977fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox sg_set_page(&sg[i], pages[i], min_t(int, length, PAGE_SIZE), 0); 8987fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox length -= PAGE_SIZE; 8997fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox } 9007fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox 9017fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox err = -ENOMEM; 9027fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox nents = dma_map_sg(&dev->pci_dev->dev, sg, count, 9037fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox write ? DMA_TO_DEVICE : DMA_FROM_DEVICE); 90436c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox if (!nents) 90536c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox goto put_pages; 906b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 9077fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox kfree(pages); 9087fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox *sgp = sg; 9097fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox return nents; 910b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 9117fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox put_pages: 9127fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox for (i = 0; i < count; i++) 9137fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox put_page(pages[i]); 9147fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox kfree(pages); 9157fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox return err; 9167fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox} 917b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 9187fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcoxstatic void nvme_unmap_user_pages(struct nvme_dev *dev, int write, 9197fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox unsigned long addr, int length, 9207fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox struct scatterlist *sg, int nents) 9217fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox{ 9227fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox int i, count; 923b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 9247fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox count = DIV_ROUND_UP(offset_in_page(addr) + length, PAGE_SIZE); 92536c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox dma_unmap_sg(&dev->pci_dev->dev, sg, nents, DMA_FROM_DEVICE); 9267fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox 92736c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox for (i = 0; i < count; i++) 9287fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox put_page(sg_page(&sg[i])); 9297fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox} 930b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 9317fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcoxstatic int nvme_submit_user_admin_command(struct nvme_dev *dev, 9327fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox unsigned long addr, unsigned length, 9337fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox struct nvme_command *cmd) 9347fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox{ 9357fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox int err, nents; 9367fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox struct scatterlist *sg; 937e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews struct nvme_prps *prps; 9387fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox 9397fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox nents = nvme_map_user_pages(dev, 0, addr, length, &sg); 9407fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox if (nents < 0) 9417fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox return nents; 942d567760c409f981d35fc755b51d5bf56a99a467bMatthew Wilcox prps = nvme_setup_prps(dev, &cmd->common, sg, length); 9437fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox err = nvme_submit_admin_cmd(dev, cmd, NULL); 9447fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox nvme_unmap_user_pages(dev, 0, addr, length, sg, nents); 945d567760c409f981d35fc755b51d5bf56a99a467bMatthew Wilcox nvme_free_prps(dev, prps); 9467fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox return err ? -EIO : 0; 947b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 948b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 949bd38c5557cf482fc195e2264b32ea62eed60730aMatthew Wilcoxstatic int nvme_identify(struct nvme_ns *ns, unsigned long addr, int cns) 950b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 951b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_command c; 952b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 953bd38c5557cf482fc195e2264b32ea62eed60730aMatthew Wilcox memset(&c, 0, sizeof(c)); 954bd38c5557cf482fc195e2264b32ea62eed60730aMatthew Wilcox c.identify.opcode = nvme_admin_identify; 955bd38c5557cf482fc195e2264b32ea62eed60730aMatthew Wilcox c.identify.nsid = cns ? 0 : cpu_to_le32(ns->ns_id); 956bd38c5557cf482fc195e2264b32ea62eed60730aMatthew Wilcox c.identify.cns = cpu_to_le32(cns); 957bd38c5557cf482fc195e2264b32ea62eed60730aMatthew Wilcox 958bd38c5557cf482fc195e2264b32ea62eed60730aMatthew Wilcox return nvme_submit_user_admin_command(ns->dev, addr, 4096, &c); 959bd38c5557cf482fc195e2264b32ea62eed60730aMatthew Wilcox} 960bd38c5557cf482fc195e2264b32ea62eed60730aMatthew Wilcox 961bd38c5557cf482fc195e2264b32ea62eed60730aMatthew Wilcoxstatic int nvme_get_range_type(struct nvme_ns *ns, unsigned long addr) 962bd38c5557cf482fc195e2264b32ea62eed60730aMatthew Wilcox{ 963bd38c5557cf482fc195e2264b32ea62eed60730aMatthew Wilcox struct nvme_command c; 964b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 965b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset(&c, 0, sizeof(c)); 966b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.features.opcode = nvme_admin_get_features; 967b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.features.nsid = cpu_to_le32(ns->ns_id); 968b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.features.fid = cpu_to_le32(NVME_FEAT_LBA_RANGE); 969b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 970bd38c5557cf482fc195e2264b32ea62eed60730aMatthew Wilcox return nvme_submit_user_admin_command(ns->dev, addr, 4096, &c); 971b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 972b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 973a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcoxstatic int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio) 974a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox{ 975a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox struct nvme_dev *dev = ns->dev; 976a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox struct nvme_queue *nvmeq; 977a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox struct nvme_user_io io; 978a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox struct nvme_command c; 979a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox unsigned length; 980a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox u32 result; 981a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox int nents, status; 982a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox struct scatterlist *sg; 983e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews struct nvme_prps *prps; 984a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox 985a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox if (copy_from_user(&io, uio, sizeof(io))) 986a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox return -EFAULT; 987a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox length = io.nblocks << io.block_shift; 988a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox nents = nvme_map_user_pages(dev, io.opcode & 1, io.addr, length, &sg); 989a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox if (nents < 0) 990a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox return nents; 991a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox 992a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox memset(&c, 0, sizeof(c)); 993a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.opcode = io.opcode; 994a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.flags = io.flags; 995a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.nsid = cpu_to_le32(io.nsid); 996a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.slba = cpu_to_le64(io.slba); 997a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.length = cpu_to_le16(io.nblocks - 1); 998a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.control = cpu_to_le16(io.control); 999a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.dsmgmt = cpu_to_le16(io.dsmgmt); 1000a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.reftag = cpu_to_le32(io.reftag); /* XXX: endian? */ 1001a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.apptag = cpu_to_le16(io.apptag); 1002a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.appmask = cpu_to_le16(io.appmask); 1003a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox /* XXX: metadata */ 1004d567760c409f981d35fc755b51d5bf56a99a467bMatthew Wilcox prps = nvme_setup_prps(dev, &c.common, sg, length); 1005a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox 1006d567760c409f981d35fc755b51d5bf56a99a467bMatthew Wilcox nvmeq = get_nvmeq(ns); 1007b1ad37efcafe396ac3944853589688dd0ec3c64eMatthew Wilcox /* Since nvme_submit_sync_cmd sleeps, we can't keep preemption 1008b1ad37efcafe396ac3944853589688dd0ec3c64eMatthew Wilcox * disabled. We may be preempted at any point, and be rescheduled 1009b1ad37efcafe396ac3944853589688dd0ec3c64eMatthew Wilcox * to a different CPU. That will cause cacheline bouncing, but no 1010b1ad37efcafe396ac3944853589688dd0ec3c64eMatthew Wilcox * additional races since q_lock already protects against other CPUs. 1011b1ad37efcafe396ac3944853589688dd0ec3c64eMatthew Wilcox */ 1012a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox put_nvmeq(nvmeq); 1013e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox status = nvme_submit_sync_cmd(nvmeq, &c, &result, IO_TIMEOUT); 1014a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox 1015a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox nvme_unmap_user_pages(dev, io.opcode & 1, io.addr, length, sg, nents); 1016d567760c409f981d35fc755b51d5bf56a99a467bMatthew Wilcox nvme_free_prps(dev, prps); 1017a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox put_user(result, &uio->result); 1018a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox return status; 1019a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox} 1020a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox 10216ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcoxstatic int nvme_download_firmware(struct nvme_ns *ns, 10226ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox struct nvme_dlfw __user *udlfw) 10236ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox{ 10246ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox struct nvme_dev *dev = ns->dev; 10256ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox struct nvme_dlfw dlfw; 10266ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox struct nvme_command c; 10276ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox int nents, status; 10286ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox struct scatterlist *sg; 1029e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews struct nvme_prps *prps; 10306ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox 10316ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox if (copy_from_user(&dlfw, udlfw, sizeof(dlfw))) 10326ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox return -EFAULT; 10336ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox if (dlfw.length >= (1 << 30)) 10346ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox return -EINVAL; 10356ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox 10366ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox nents = nvme_map_user_pages(dev, 1, dlfw.addr, dlfw.length * 4, &sg); 10376ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox if (nents < 0) 10386ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox return nents; 10396ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox 10406ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox memset(&c, 0, sizeof(c)); 10416ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox c.dlfw.opcode = nvme_admin_download_fw; 10426ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox c.dlfw.numd = cpu_to_le32(dlfw.length); 10436ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox c.dlfw.offset = cpu_to_le32(dlfw.offset); 1044d567760c409f981d35fc755b51d5bf56a99a467bMatthew Wilcox prps = nvme_setup_prps(dev, &c.common, sg, dlfw.length * 4); 10456ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox 10466ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox status = nvme_submit_admin_cmd(dev, &c, NULL); 10476ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox nvme_unmap_user_pages(dev, 0, dlfw.addr, dlfw.length * 4, sg, nents); 1048d567760c409f981d35fc755b51d5bf56a99a467bMatthew Wilcox nvme_free_prps(dev, prps); 10496ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox return status; 10506ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox} 10516ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox 10526ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcoxstatic int nvme_activate_firmware(struct nvme_ns *ns, unsigned long arg) 10536ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox{ 10546ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox struct nvme_dev *dev = ns->dev; 10556ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox struct nvme_command c; 10566ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox 10576ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox memset(&c, 0, sizeof(c)); 10586ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox c.common.opcode = nvme_admin_activate_fw; 10596ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox c.common.rsvd10[0] = cpu_to_le32(arg); 10606ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox 10616ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox return nvme_submit_admin_cmd(dev, &c, NULL); 10626ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox} 10636ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox 1064b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, 1065b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unsigned long arg) 1066b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1067b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_ns *ns = bdev->bd_disk->private_data; 1068b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1069b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox switch (cmd) { 1070b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox case NVME_IOCTL_IDENTIFY_NS: 107136c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox return nvme_identify(ns, arg, 0); 1072b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox case NVME_IOCTL_IDENTIFY_CTRL: 107336c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox return nvme_identify(ns, arg, 1); 1074b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox case NVME_IOCTL_GET_RANGE_TYPE: 1075bd38c5557cf482fc195e2264b32ea62eed60730aMatthew Wilcox return nvme_get_range_type(ns, arg); 1076a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox case NVME_IOCTL_SUBMIT_IO: 1077a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox return nvme_submit_io(ns, (void __user *)arg); 10786ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox case NVME_IOCTL_DOWNLOAD_FW: 10796ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox return nvme_download_firmware(ns, (void __user *)arg); 10806ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox case NVME_IOCTL_ACTIVATE_FW: 10816ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox return nvme_activate_firmware(ns, arg); 1082b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox default: 1083b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -ENOTTY; 1084b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 1085b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1086b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1087b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic const struct block_device_operations nvme_fops = { 1088b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .owner = THIS_MODULE, 1089b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .ioctl = nvme_ioctl, 1090b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 1091b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 10921fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcoxstatic void nvme_resubmit_bios(struct nvme_queue *nvmeq) 10931fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox{ 10941fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox while (bio_list_peek(&nvmeq->sq_cong)) { 10951fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox struct bio *bio = bio_list_pop(&nvmeq->sq_cong); 10961fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox struct nvme_ns *ns = bio->bi_bdev->bd_disk->private_data; 10971fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox if (nvme_submit_bio_queue(nvmeq, ns, bio)) { 10981fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox bio_list_add_head(&nvmeq->sq_cong, bio); 10991fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox break; 11001fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox } 11011fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox } 11021fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox} 11031fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 11041fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcoxstatic int nvme_kthread(void *data) 11051fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox{ 11061fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox struct nvme_dev *dev; 11071fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 11081fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox while (!kthread_should_stop()) { 11091fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox __set_current_state(TASK_RUNNING); 11101fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_lock(&dev_list_lock); 11111fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox list_for_each_entry(dev, &dev_list, node) { 11121fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox int i; 11131fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox for (i = 0; i < dev->queue_count; i++) { 11141fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox struct nvme_queue *nvmeq = dev->queues[i]; 11151fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_lock_irq(&nvmeq->q_lock); 11161fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox if (nvme_process_cq(nvmeq)) 11171fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox printk("process_cq did something\n"); 11181fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox nvme_resubmit_bios(nvmeq); 11191fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_unlock_irq(&nvmeq->q_lock); 11201fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox } 11211fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox } 11221fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_unlock(&dev_list_lock); 11231fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox set_current_state(TASK_INTERRUPTIBLE); 11241fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox schedule_timeout(HZ); 11251fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox } 11261fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox return 0; 11271fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox} 11281fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 1129b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic struct nvme_ns *nvme_alloc_ns(struct nvme_dev *dev, int index, 1130b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_id_ns *id, struct nvme_lba_range_type *rt) 1131b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1132b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_ns *ns; 1133b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct gendisk *disk; 1134b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int lbaf; 1135b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1136b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (rt->attributes & NVME_LBART_ATTRIB_HIDE) 1137b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 1138b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1139b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns = kzalloc(sizeof(*ns), GFP_KERNEL); 1140b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!ns) 1141b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 1142b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->queue = blk_alloc_queue(GFP_KERNEL); 1143b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!ns->queue) 1144b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto out_free_ns; 1145b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->queue->queue_flags = QUEUE_FLAG_DEFAULT | QUEUE_FLAG_NOMERGES | 1146b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox QUEUE_FLAG_NONROT | QUEUE_FLAG_DISCARD; 1147b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox blk_queue_make_request(ns->queue, nvme_make_request); 1148b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->dev = dev; 1149b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->queue->queuedata = ns; 1150b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1151b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk = alloc_disk(NVME_MINORS); 1152b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!disk) 1153b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto out_free_queue; 1154b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->ns_id = index; 1155b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->disk = disk; 1156b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox lbaf = id->flbas & 0xf; 1157b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->lba_shift = id->lbaf[lbaf].ds; 1158b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1159b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk->major = nvme_major; 1160b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk->minors = NVME_MINORS; 1161b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk->first_minor = NVME_MINORS * index; 1162b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk->fops = &nvme_fops; 1163b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk->private_data = ns; 1164b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk->queue = ns->queue; 1165388f037f4e7f0a24bac6b1a24f144f5d939f58cfMatthew Wilcox disk->driverfs_dev = &dev->pci_dev->dev; 1166b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox sprintf(disk->disk_name, "nvme%dn%d", dev->instance, index); 1167b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox set_capacity(disk, le64_to_cpup(&id->nsze) << (ns->lba_shift - 9)); 1168b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1169b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return ns; 1170b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1171b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox out_free_queue: 1172b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox blk_cleanup_queue(ns->queue); 1173b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox out_free_ns: 1174b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(ns); 1175b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 1176b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1177b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1178b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void nvme_ns_free(struct nvme_ns *ns) 1179b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1180b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox put_disk(ns->disk); 1181b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox blk_cleanup_queue(ns->queue); 1182b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(ns); 1183b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1184b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1185b3b06812e199f248561ce7824a4a8a9cd573c05aMatthew Wilcoxstatic int set_queue_count(struct nvme_dev *dev, int count) 1186b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1187b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int status; 1188b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 result; 1189b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_command c; 1190b3b06812e199f248561ce7824a4a8a9cd573c05aMatthew Wilcox u32 q_count = (count - 1) | ((count - 1) << 16); 1191b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1192b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset(&c, 0, sizeof(c)); 1193b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.features.opcode = nvme_admin_get_features; 1194b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.features.fid = cpu_to_le32(NVME_FEAT_NUM_QUEUES); 1195b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.features.dword11 = cpu_to_le32(q_count); 1196b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1197b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox status = nvme_submit_admin_cmd(dev, &c, &result); 1198b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (status) 1199b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -EIO; 1200b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return min(result & 0xffff, result >> 16) + 1; 1201b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1202b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1203b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int __devinit nvme_setup_io_queues(struct nvme_dev *dev) 1204b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1205b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox int result, cpu, i, nr_io_queues; 1206b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1207b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox nr_io_queues = num_online_cpus(); 1208b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox result = set_queue_count(dev, nr_io_queues); 12091b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox if (result < 0) 12101b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox return result; 1211b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox if (result < nr_io_queues) 1212b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox nr_io_queues = result; 1213b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 12141b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox /* Deregister the admin queue's interrupt */ 12151b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox free_irq(dev->entry[0].vector, dev->queues[0]); 12161b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox 1217b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox for (i = 0; i < nr_io_queues; i++) 12181b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox dev->entry[i].entry = i; 12191b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox for (;;) { 1220b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox result = pci_enable_msix(dev->pci_dev, dev->entry, 1221b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox nr_io_queues); 12221b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox if (result == 0) { 12231b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox break; 12241b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox } else if (result > 0) { 1225b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox nr_io_queues = result; 12261b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox continue; 12271b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox } else { 1228b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox nr_io_queues = 1; 12291b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox break; 12301b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox } 12311b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox } 12321b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox 12331b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox result = queue_request_irq(dev, dev->queues[0], "nvme admin"); 12341b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox /* XXX: handle failure here */ 12351b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox 12361b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox cpu = cpumask_first(cpu_online_mask); 1237b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox for (i = 0; i < nr_io_queues; i++) { 12381b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox irq_set_affinity_hint(dev->entry[i].vector, get_cpu_mask(cpu)); 12391b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox cpu = cpumask_next(cpu, cpu_online_mask); 12401b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox } 12411b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox 1242b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox for (i = 0; i < nr_io_queues; i++) { 12431b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox dev->queues[i + 1] = nvme_create_queue(dev, i + 1, 12441b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox NVME_Q_DEPTH, i); 12451b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox if (!dev->queues[i + 1]) 12461b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox return -ENOMEM; 12471b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox dev->queue_count++; 12481b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox } 1249b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1250b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 1251b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1252b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1253b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void nvme_free_queues(struct nvme_dev *dev) 1254b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1255b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int i; 1256b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1257b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox for (i = dev->queue_count - 1; i >= 0; i--) 1258b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_free_queue(dev, i); 1259b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1260b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1261b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int __devinit nvme_dev_add(struct nvme_dev *dev) 1262b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1263b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int res, nn, i; 1264b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_ns *ns, *next; 126551814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox struct nvme_id_ctrl *ctrl; 1266b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox void *id; 1267b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_addr_t dma_addr; 1268b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_command cid, crt; 1269b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1270b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox res = nvme_setup_io_queues(dev); 1271b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (res) 1272b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return res; 1273b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1274b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox /* XXX: Switch to a SG list once prp2 works */ 1275b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox id = dma_alloc_coherent(&dev->pci_dev->dev, 8192, &dma_addr, 1276b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox GFP_KERNEL); 1277b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1278b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset(&cid, 0, sizeof(cid)); 1279b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cid.identify.opcode = nvme_admin_identify; 1280b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cid.identify.nsid = 0; 1281b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cid.identify.prp1 = cpu_to_le64(dma_addr); 1282b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cid.identify.cns = cpu_to_le32(1); 1283b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1284b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox res = nvme_submit_admin_cmd(dev, &cid, NULL); 1285b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (res) { 1286b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox res = -EIO; 1287b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto out_free; 1288b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 1289b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 129051814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox ctrl = id; 129151814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox nn = le32_to_cpup(&ctrl->nn); 129251814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox memcpy(dev->serial, ctrl->sn, sizeof(ctrl->sn)); 129351814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox memcpy(dev->model, ctrl->mn, sizeof(ctrl->mn)); 129451814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox memcpy(dev->firmware_rev, ctrl->fr, sizeof(ctrl->fr)); 1295b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1296b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cid.identify.cns = 0; 1297b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset(&crt, 0, sizeof(crt)); 1298b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox crt.features.opcode = nvme_admin_get_features; 1299b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox crt.features.prp1 = cpu_to_le64(dma_addr + 4096); 1300b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox crt.features.fid = cpu_to_le32(NVME_FEAT_LBA_RANGE); 1301b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1302b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox for (i = 0; i < nn; i++) { 1303b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cid.identify.nsid = cpu_to_le32(i); 1304b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox res = nvme_submit_admin_cmd(dev, &cid, NULL); 1305b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (res) 1306b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox continue; 1307b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1308b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (((struct nvme_id_ns *)id)->ncap == 0) 1309b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox continue; 1310b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1311b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox crt.features.nsid = cpu_to_le32(i); 1312b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox res = nvme_submit_admin_cmd(dev, &crt, NULL); 1313b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (res) 1314b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox continue; 1315b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1316b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns = nvme_alloc_ns(dev, i, id, id + 4096); 1317b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (ns) 1318b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox list_add_tail(&ns->list, &dev->namespaces); 1319b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 1320b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox list_for_each_entry(ns, &dev->namespaces, list) 1321b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox add_disk(ns->disk); 1322b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1323b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_free_coherent(&dev->pci_dev->dev, 4096, id, dma_addr); 1324b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 1325b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1326b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox out_free: 1327b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox list_for_each_entry_safe(ns, next, &dev->namespaces, list) { 1328b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox list_del(&ns->list); 1329b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_ns_free(ns); 1330b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 1331b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1332b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_free_coherent(&dev->pci_dev->dev, 4096, id, dma_addr); 1333b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return res; 1334b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1335b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1336b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_dev_remove(struct nvme_dev *dev) 1337b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1338b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_ns *ns, *next; 1339b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 13401fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_lock(&dev_list_lock); 13411fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox list_del(&dev->node); 13421fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_unlock(&dev_list_lock); 13431fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 1344b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox /* TODO: wait all I/O finished or cancel them */ 1345b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1346b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox list_for_each_entry_safe(ns, next, &dev->namespaces, list) { 1347b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox list_del(&ns->list); 1348b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox del_gendisk(ns->disk); 1349b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_ns_free(ns); 1350b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 1351b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1352b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_free_queues(dev); 1353b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1354b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 1355b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1356b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1357091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcoxstatic int nvme_setup_prp_pools(struct nvme_dev *dev) 1358091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox{ 1359091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox struct device *dmadev = &dev->pci_dev->dev; 1360091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox dev->prp_page_pool = dma_pool_create("prp list page", dmadev, 1361091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox PAGE_SIZE, PAGE_SIZE, 0); 1362091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox if (!dev->prp_page_pool) 1363091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox return -ENOMEM; 1364091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox 136599802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox /* Optimisation for I/Os between 4k and 128k */ 136699802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox dev->prp_small_pool = dma_pool_create("prp list 256", dmadev, 136799802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox 256, 256, 0); 136899802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox if (!dev->prp_small_pool) { 136999802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox dma_pool_destroy(dev->prp_page_pool); 137099802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox return -ENOMEM; 137199802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox } 1372091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox return 0; 1373091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox} 1374091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox 1375091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcoxstatic void nvme_release_prp_pools(struct nvme_dev *dev) 1376091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox{ 1377091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox dma_pool_destroy(dev->prp_page_pool); 137899802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox dma_pool_destroy(dev->prp_small_pool); 1379091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox} 1380091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox 1381b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* XXX: Use an ida or something to let remove / add work correctly */ 1382b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void nvme_set_instance(struct nvme_dev *dev) 1383b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1384b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox static int instance; 1385b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->instance = instance++; 1386b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1387b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1388b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void nvme_release_instance(struct nvme_dev *dev) 1389b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1390b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1391b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1392b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int __devinit nvme_probe(struct pci_dev *pdev, 1393b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox const struct pci_device_id *id) 1394b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1395574e8b95bc3780e10e9b5e9d51074d503dd3d5d9Matthew Wilcox int bars, result = -ENOMEM; 1396b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_dev *dev; 1397b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1398b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev = kzalloc(sizeof(*dev), GFP_KERNEL); 1399b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!dev) 1400b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -ENOMEM; 1401b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->entry = kcalloc(num_possible_cpus(), sizeof(*dev->entry), 1402b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox GFP_KERNEL); 1403b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!dev->entry) 1404b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto free; 14051b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox dev->queues = kcalloc(num_possible_cpus() + 1, sizeof(void *), 14061b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox GFP_KERNEL); 1407b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!dev->queues) 1408b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto free; 1409b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 14100ee5a7d7cb9309bd393a25c395f19fb12a842602Shane Michael Matthews if (pci_enable_device_mem(pdev)) 14110ee5a7d7cb9309bd393a25c395f19fb12a842602Shane Michael Matthews goto free; 1412f64d3365a3e5cb46e69db7e2c82a7cb9a5bed1b8Matthew Wilcox pci_set_master(pdev); 1413574e8b95bc3780e10e9b5e9d51074d503dd3d5d9Matthew Wilcox bars = pci_select_bars(pdev, IORESOURCE_MEM); 1414574e8b95bc3780e10e9b5e9d51074d503dd3d5d9Matthew Wilcox if (pci_request_selected_regions(pdev, bars, "nvme")) 1415574e8b95bc3780e10e9b5e9d51074d503dd3d5d9Matthew Wilcox goto disable; 14160ee5a7d7cb9309bd393a25c395f19fb12a842602Shane Michael Matthews 1417b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox INIT_LIST_HEAD(&dev->namespaces); 1418b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->pci_dev = pdev; 1419b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox pci_set_drvdata(pdev, dev); 14202930353f9f2b9e4629e935acd970cb73c1171229Matthew Wilcox dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)); 14212930353f9f2b9e4629e935acd970cb73c1171229Matthew Wilcox dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64)); 1422b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_set_instance(dev); 142353c9577e9ca68a633c6e9df2b54eaecacfa77f62Matthew Wilcox dev->entry[0].vector = pdev->irq; 1424b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1425091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox result = nvme_setup_prp_pools(dev); 1426091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox if (result) 1427091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox goto disable_msix; 1428091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox 1429b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->bar = ioremap(pci_resource_start(pdev, 0), 8192); 1430b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!dev->bar) { 1431b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox result = -ENOMEM; 1432574e8b95bc3780e10e9b5e9d51074d503dd3d5d9Matthew Wilcox goto disable_msix; 1433b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 1434b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1435b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox result = nvme_configure_admin_queue(dev); 1436b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (result) 1437b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto unmap; 1438b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->queue_count++; 1439b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1440b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox result = nvme_dev_add(dev); 1441b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (result) 1442b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto delete; 14431fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 14441fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_lock(&dev_list_lock); 14451fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox list_add(&dev->node, &dev_list); 14461fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_unlock(&dev_list_lock); 14471fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 1448b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 1449b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1450b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox delete: 1451b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_free_queues(dev); 1452b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unmap: 1453b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox iounmap(dev->bar); 1454574e8b95bc3780e10e9b5e9d51074d503dd3d5d9Matthew Wilcox disable_msix: 1455b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox pci_disable_msix(pdev); 1456b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_release_instance(dev); 1457091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox nvme_release_prp_pools(dev); 1458574e8b95bc3780e10e9b5e9d51074d503dd3d5d9Matthew Wilcox disable: 14590ee5a7d7cb9309bd393a25c395f19fb12a842602Shane Michael Matthews pci_disable_device(pdev); 1460574e8b95bc3780e10e9b5e9d51074d503dd3d5d9Matthew Wilcox pci_release_regions(pdev); 1461b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox free: 1462b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(dev->queues); 1463b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(dev->entry); 1464b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(dev); 1465b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return result; 1466b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1467b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1468b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void __devexit nvme_remove(struct pci_dev *pdev) 1469b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1470b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_dev *dev = pci_get_drvdata(pdev); 1471b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_dev_remove(dev); 1472b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox pci_disable_msix(pdev); 1473b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox iounmap(dev->bar); 1474b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_release_instance(dev); 1475091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox nvme_release_prp_pools(dev); 14760ee5a7d7cb9309bd393a25c395f19fb12a842602Shane Michael Matthews pci_disable_device(pdev); 1477574e8b95bc3780e10e9b5e9d51074d503dd3d5d9Matthew Wilcox pci_release_regions(pdev); 1478b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(dev->queues); 1479b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(dev->entry); 1480b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(dev); 1481b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1482b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1483b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* These functions are yet to be implemented */ 1484b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_error_detected NULL 1485b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_dump_registers NULL 1486b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_link_reset NULL 1487b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_slot_reset NULL 1488b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_error_resume NULL 1489b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_suspend NULL 1490b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_resume NULL 1491b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1492b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic struct pci_error_handlers nvme_err_handler = { 1493b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .error_detected = nvme_error_detected, 1494b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .mmio_enabled = nvme_dump_registers, 1495b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .link_reset = nvme_link_reset, 1496b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .slot_reset = nvme_slot_reset, 1497b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .resume = nvme_error_resume, 1498b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 1499b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1500b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* Move to pci_ids.h later */ 1501b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define PCI_CLASS_STORAGE_EXPRESS 0x010802 1502b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1503b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic DEFINE_PCI_DEVICE_TABLE(nvme_id_table) = { 1504b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox { PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) }, 1505b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox { 0, } 1506b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 1507b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew WilcoxMODULE_DEVICE_TABLE(pci, nvme_id_table); 1508b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1509b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic struct pci_driver nvme_driver = { 1510b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .name = "nvme", 1511b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .id_table = nvme_id_table, 1512b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .probe = nvme_probe, 1513b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .remove = __devexit_p(nvme_remove), 1514b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .suspend = nvme_suspend, 1515b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .resume = nvme_resume, 1516b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .err_handler = &nvme_err_handler, 1517b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 1518b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1519b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int __init nvme_init(void) 1520b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 15211fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox int result = -EBUSY; 15221fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 15231fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox nvme_thread = kthread_run(nvme_kthread, NULL, "nvme"); 15241fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox if (IS_ERR(nvme_thread)) 15251fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox return PTR_ERR(nvme_thread); 1526b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1527b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_major = register_blkdev(nvme_major, "nvme"); 1528b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (nvme_major <= 0) 15291fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox goto kill_kthread; 1530b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1531b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox result = pci_register_driver(&nvme_driver); 15321fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox if (result) 15331fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox goto unregister_blkdev; 15341fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox return 0; 1535b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 15361fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox unregister_blkdev: 1537b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unregister_blkdev(nvme_major, "nvme"); 15381fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox kill_kthread: 15391fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox kthread_stop(nvme_thread); 1540b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return result; 1541b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1542b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1543b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void __exit nvme_exit(void) 1544b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1545b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox pci_unregister_driver(&nvme_driver); 1546b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unregister_blkdev(nvme_major, "nvme"); 15471fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox kthread_stop(nvme_thread); 1548b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1549b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1550b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew WilcoxMODULE_AUTHOR("Matthew Wilcox <willy@linux.intel.com>"); 1551b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew WilcoxMODULE_LICENSE("GPL"); 1552ad8a5df97cb060aa4d817af25587c99e2d2fda97Matthew WilcoxMODULE_VERSION("0.3"); 1553b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxmodule_init(nvme_init); 1554b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxmodule_exit(nvme_exit); 1555