nvme-core.c revision 08df1e05657fc6712e520e7c09cc6c86160ceb35
1b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 2b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * NVM Express device driver 3b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Copyright (c) 2011, Intel Corporation. 4b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 5b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * This program is free software; you can redistribute it and/or modify it 6b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * under the terms and conditions of the GNU General Public License, 7b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * version 2, as published by the Free Software Foundation. 8b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 9b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * This program is distributed in the hope it will be useful, but WITHOUT 10b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * more details. 13b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 14b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * You should have received a copy of the GNU General Public License along with 15b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * this program; if not, write to the Free Software Foundation, Inc., 16b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. 17b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 18b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 19b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/nvme.h> 20b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/bio.h> 218de055350fbaa96b6563892c195a60be583faa9cMatthew Wilcox#include <linux/bitops.h> 22b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/blkdev.h> 23fd63e9ceeeae58cfe877c2d49d41c1bf7532303cMatthew Wilcox#include <linux/delay.h> 24b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/errno.h> 25b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/fs.h> 26b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/genhd.h> 275aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox#include <linux/idr.h> 28b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/init.h> 29b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/interrupt.h> 30b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/io.h> 31b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/kdev_t.h> 321fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox#include <linux/kthread.h> 33b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/kernel.h> 34b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/mm.h> 35b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/module.h> 36b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/moduleparam.h> 37b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/pci.h> 38be7b62754e097adc0cb16c25c9ee86ee20de62fbMatthew Wilcox#include <linux/poison.h> 39b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/sched.h> 40b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/slab.h> 41b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/types.h> 42b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 43797a796a13df6b84a4791e57306737059b5b2384Hitoshi Mitake#include <asm-generic/io-64-nonatomic-lo-hi.h> 44797a796a13df6b84a4791e57306737059b5b2384Hitoshi Mitake 45b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define NVME_Q_DEPTH 1024 46b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define SQ_SIZE(depth) (depth * sizeof(struct nvme_command)) 47b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define CQ_SIZE(depth) (depth * sizeof(struct nvme_completion)) 48b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define NVME_MINORS 64 49ff976d724a74e4522e9ca2de1fb37ac4520f454fMatthew Wilcox#define NVME_IO_TIMEOUT (5 * HZ) 50e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox#define ADMIN_TIMEOUT (60 * HZ) 51b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 52b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_major; 53b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxmodule_param(nvme_major, int, 0); 54b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 5558ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcoxstatic int use_threaded_interrupts; 5658ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcoxmodule_param(use_threaded_interrupts, int, 0); 5758ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox 581fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcoxstatic DEFINE_SPINLOCK(dev_list_lock); 591fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcoxstatic LIST_HEAD(dev_list); 601fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcoxstatic struct task_struct *nvme_thread; 611fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 62b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 63b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Represents an NVM Express device. Each nvme_dev is a PCI function. 64b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 65b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstruct nvme_dev { 661fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox struct list_head node; 67b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue **queues; 68b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 __iomem *dbs; 69b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct pci_dev *pci_dev; 70091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox struct dma_pool *prp_page_pool; 7199802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox struct dma_pool *prp_small_pool; 72b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int instance; 73b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int queue_count; 74f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox int db_stride; 75b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 ctrl_config; 76b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct msix_entry *entry; 77b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_bar __iomem *bar; 78b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct list_head namespaces; 7951814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox char serial[20]; 8051814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox char model[40]; 8151814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox char firmware_rev[8]; 828fc23e032debd682f5ba9fc524a5846c10d2c522Keith Busch u32 max_hw_sectors; 83b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 84b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 85b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 86b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * An NVM Express namespace is equivalent to a SCSI LUN 87b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 88b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstruct nvme_ns { 89b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct list_head list; 90b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 91b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_dev *dev; 92b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct request_queue *queue; 93b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct gendisk *disk; 94b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 95b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int ns_id; 96b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int lba_shift; 97b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 98b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 99b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 100b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * An NVM Express queue. Each device has at least two (one for admin 101b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * commands and one for I/O commands). 102b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 103b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstruct nvme_queue { 104b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct device *q_dmadev; 105091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox struct nvme_dev *dev; 106b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox spinlock_t q_lock; 107b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_command *sq_cmds; 108b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox volatile struct nvme_completion *cqes; 109b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_addr_t sq_dma_addr; 110b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_addr_t cq_dma_addr; 111b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox wait_queue_head_t sq_full; 1121fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox wait_queue_t sq_cong_wait; 113b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct bio_list sq_cong; 114b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 __iomem *q_db; 115b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 q_depth; 116b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 cq_vector; 117b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 sq_head; 118b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 sq_tail; 119b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 cq_head; 120821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox u16 cq_phase; 121b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unsigned long cmdid_data[]; 122b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 123b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 124b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 125b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Check we didin't inadvertently grow the command struct 126b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 127b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic inline void _nvme_check_size(void) 128b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 129b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64); 130b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_create_cq) != 64); 131b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_create_sq) != 64); 132b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64); 133b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_features) != 64); 134b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_command) != 64); 135b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != 4096); 136b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_id_ns) != 4096); 137b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64); 138b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 139b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1405c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcoxtypedef void (*nvme_completion_fn)(struct nvme_dev *, void *, 141c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox struct nvme_completion *); 142c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox 143e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcoxstruct nvme_cmd_info { 144c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox nvme_completion_fn fn; 145c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox void *ctx; 146e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox unsigned long timeout; 147e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox}; 148e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox 149e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcoxstatic struct nvme_cmd_info *nvme_cmd_info(struct nvme_queue *nvmeq) 150e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox{ 151e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox return (void *)&nvmeq->cmdid_data[BITS_TO_LONGS(nvmeq->q_depth)]; 152e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox} 153e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox 154b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/** 155714a7a22884b74862540bc84955274d86b2f6040Matthew Wilcox * alloc_cmdid() - Allocate a Command ID 156714a7a22884b74862540bc84955274d86b2f6040Matthew Wilcox * @nvmeq: The queue that will be used for this command 157714a7a22884b74862540bc84955274d86b2f6040Matthew Wilcox * @ctx: A pointer that will be passed to the handler 158c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox * @handler: The function to call on completion 159b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 160b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Allocate a Command ID for a queue. The data passed in will 161b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * be passed to the completion handler. This is implemented by using 162b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * the bottom two bits of the ctx pointer to store the handler ID. 163b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Passing in a pointer that's not 4-byte aligned will cause a BUG. 164b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * We can change this if it becomes a problem. 165184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox * 166184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox * May be called with local interrupts disabled and the q_lock held, 167184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox * or with interrupts enabled and no locks held. 168b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 169c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcoxstatic int alloc_cmdid(struct nvme_queue *nvmeq, void *ctx, 170c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox nvme_completion_fn handler, unsigned timeout) 171b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 172e6d15f79f997a98b3a69abbc462fc9041cc1a7b4Matthew Wilcox int depth = nvmeq->q_depth - 1; 173e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox struct nvme_cmd_info *info = nvme_cmd_info(nvmeq); 174b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int cmdid; 175b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 176b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox do { 177b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cmdid = find_first_zero_bit(nvmeq->cmdid_data, depth); 178b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (cmdid >= depth) 179b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -EBUSY; 180b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } while (test_and_set_bit(cmdid, nvmeq->cmdid_data)); 181b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 182c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox info[cmdid].fn = handler; 183c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox info[cmdid].ctx = ctx; 184e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox info[cmdid].timeout = jiffies + timeout; 185b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return cmdid; 186b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 187b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 188b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int alloc_cmdid_killable(struct nvme_queue *nvmeq, void *ctx, 189c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox nvme_completion_fn handler, unsigned timeout) 190b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 191b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int cmdid; 192b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox wait_event_killable(nvmeq->sq_full, 193e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox (cmdid = alloc_cmdid(nvmeq, ctx, handler, timeout)) >= 0); 194b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return (cmdid < 0) ? -EINTR : cmdid; 195b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 196b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 197c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox/* Special values must be less than 0x1000 */ 198c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox#define CMD_CTX_BASE ((void *)POISON_POINTER_DELTA) 199d2d8703481f60d67f49e3177196cbe474b11377cMatthew Wilcox#define CMD_CTX_CANCELLED (0x30C + CMD_CTX_BASE) 200d2d8703481f60d67f49e3177196cbe474b11377cMatthew Wilcox#define CMD_CTX_COMPLETED (0x310 + CMD_CTX_BASE) 201d2d8703481f60d67f49e3177196cbe474b11377cMatthew Wilcox#define CMD_CTX_INVALID (0x314 + CMD_CTX_BASE) 20200df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox#define CMD_CTX_FLUSH (0x318 + CMD_CTX_BASE) 203be7b62754e097adc0cb16c25c9ee86ee20de62fbMatthew Wilcox 2045c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcoxstatic void special_completion(struct nvme_dev *dev, void *ctx, 205c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox struct nvme_completion *cqe) 206c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox{ 207c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox if (ctx == CMD_CTX_CANCELLED) 208c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox return; 209c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox if (ctx == CMD_CTX_FLUSH) 210c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox return; 211c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox if (ctx == CMD_CTX_COMPLETED) { 2125c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcox dev_warn(&dev->pci_dev->dev, 213c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox "completed id %d twice on queue %d\n", 214c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox cqe->command_id, le16_to_cpup(&cqe->sq_id)); 215c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox return; 216c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox } 217c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox if (ctx == CMD_CTX_INVALID) { 2185c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcox dev_warn(&dev->pci_dev->dev, 219c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox "invalid id %d completed on queue %d\n", 220c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox cqe->command_id, le16_to_cpup(&cqe->sq_id)); 221c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox return; 222c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox } 223c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox 2245c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcox dev_warn(&dev->pci_dev->dev, "Unknown special completion %p\n", ctx); 225c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox} 226c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox 227184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox/* 228184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox * Called with local interrupts disabled and the q_lock held. May not sleep. 229184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox */ 230c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcoxstatic void *free_cmdid(struct nvme_queue *nvmeq, int cmdid, 231c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox nvme_completion_fn *fn) 232b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 233c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox void *ctx; 234e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox struct nvme_cmd_info *info = nvme_cmd_info(nvmeq); 235b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 236c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox if (cmdid >= nvmeq->q_depth) { 237c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox *fn = special_completion; 23848e3d39816416b3bf03dee3a796c0c04427c1a31Matthew Wilcox return CMD_CTX_INVALID; 239c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox } 240859361a228258edf4821d9f5635825033eca78e8Keith Busch if (fn) 241859361a228258edf4821d9f5635825033eca78e8Keith Busch *fn = info[cmdid].fn; 242c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox ctx = info[cmdid].ctx; 243c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox info[cmdid].fn = special_completion; 244e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox info[cmdid].ctx = CMD_CTX_COMPLETED; 245b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox clear_bit(cmdid, nvmeq->cmdid_data); 246b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox wake_up(&nvmeq->sq_full); 247c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox return ctx; 248b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 249b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 250c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcoxstatic void *cancel_cmdid(struct nvme_queue *nvmeq, int cmdid, 251c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox nvme_completion_fn *fn) 2523c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox{ 253c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox void *ctx; 254e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox struct nvme_cmd_info *info = nvme_cmd_info(nvmeq); 255c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox if (fn) 256c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox *fn = info[cmdid].fn; 257c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox ctx = info[cmdid].ctx; 258c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox info[cmdid].fn = special_completion; 259e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox info[cmdid].ctx = CMD_CTX_CANCELLED; 260c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox return ctx; 2613c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox} 2623c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox 263040a93b52a9eee8177ebaf2ba0ee0f9f518d1bf8Matthew Wilcoxstatic struct nvme_queue *get_nvmeq(struct nvme_dev *dev) 264b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 265040a93b52a9eee8177ebaf2ba0ee0f9f518d1bf8Matthew Wilcox return dev->queues[get_cpu() + 1]; 266b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 267b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 268b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void put_nvmeq(struct nvme_queue *nvmeq) 269b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 2701b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox put_cpu(); 271b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 272b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 273b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/** 274714a7a22884b74862540bc84955274d86b2f6040Matthew Wilcox * nvme_submit_cmd() - Copy a command into a queue and ring the doorbell 275b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * @nvmeq: The queue to use 276b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * @cmd: The command to send 277b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 278b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Safe to use from interrupt context 279b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 280b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_submit_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd) 281b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 282b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unsigned long flags; 283b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 tail; 284b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox spin_lock_irqsave(&nvmeq->q_lock, flags); 285b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox tail = nvmeq->sq_tail; 286b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memcpy(&nvmeq->sq_cmds[tail], cmd, sizeof(*cmd)); 287b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (++tail == nvmeq->q_depth) 288b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox tail = 0; 2897547881d0951384f9833ec3a80fac8f3f16f3b98Matthew Wilcox writel(tail, nvmeq->q_db); 290b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->sq_tail = tail; 291b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox spin_unlock_irqrestore(&nvmeq->q_lock, flags); 292b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 293b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 294b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 295b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 296eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox/* 297eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox * The nvme_iod describes the data in an I/O, including the list of PRP 298eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox * entries. You can't see it in this data structure because C doesn't let 299eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox * me express that. Use nvme_alloc_iod to ensure there's enough space 300eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox * allocated to store the PRP list. 301eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox */ 302eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcoxstruct nvme_iod { 303eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox void *private; /* For the use of the submitter of the I/O */ 304eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox int npages; /* In the PRP list. 0 means small pool in use */ 305eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox int offset; /* Of PRP list */ 306eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox int nents; /* Used in scatterlist */ 307eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox int length; /* Of data, in bytes */ 308e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_addr_t first_dma; 309eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox struct scatterlist sg[0]; 310e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews}; 311e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews 312eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcoxstatic __le64 **iod_list(struct nvme_iod *iod) 313e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews{ 314eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return ((void *)iod) + iod->offset; 315e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews} 316e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews 317eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox/* 318eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox * Will slightly overestimate the number of pages needed. This is OK 319eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox * as it only leads to a small amount of wasted memory for the lifetime of 320eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox * the I/O. 321eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox */ 322eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcoxstatic int nvme_npages(unsigned size) 323eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox{ 324eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox unsigned nprps = DIV_ROUND_UP(size + PAGE_SIZE, PAGE_SIZE); 325eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return DIV_ROUND_UP(8 * nprps, PAGE_SIZE - 8); 326eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox} 327b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 328eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcoxstatic struct nvme_iod * 329eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcoxnvme_alloc_iod(unsigned nseg, unsigned nbytes, gfp_t gfp) 330b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 331eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox struct nvme_iod *iod = kmalloc(sizeof(struct nvme_iod) + 332eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox sizeof(__le64 *) * nvme_npages(nbytes) + 333eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox sizeof(struct scatterlist) * nseg, gfp); 334eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox 335eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox if (iod) { 336eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->offset = offsetof(struct nvme_iod, sg[nseg]); 337eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->npages = -1; 338eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->length = nbytes; 339eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox } 340eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox 341eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return iod; 342b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 343b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 344eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcoxstatic void nvme_free_iod(struct nvme_dev *dev, struct nvme_iod *iod) 345b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 346eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox const int last_prp = PAGE_SIZE / 8 - 1; 347eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox int i; 348eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox __le64 **list = iod_list(iod); 349eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox dma_addr_t prp_dma = iod->first_dma; 350eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox 351eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox if (iod->npages == 0) 352eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox dma_pool_free(dev->prp_small_pool, list[0], prp_dma); 353eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox for (i = 0; i < iod->npages; i++) { 354eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox __le64 *prp_list = list[i]; 355eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox dma_addr_t next_prp_dma = le64_to_cpu(prp_list[last_prp]); 356eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox dma_pool_free(dev->prp_page_pool, prp_list, prp_dma); 357eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox prp_dma = next_prp_dma; 358eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox } 359eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox kfree(iod); 360b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 361b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 3625c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcoxstatic void requeue_bio(struct nvme_dev *dev, struct bio *bio) 3635c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcox{ 3645c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcox struct nvme_queue *nvmeq = get_nvmeq(dev); 3655c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcox if (bio_list_empty(&nvmeq->sq_cong)) 3665c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcox add_wait_queue(&nvmeq->sq_full, &nvmeq->sq_cong_wait); 3675c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcox bio_list_add(&nvmeq->sq_cong, bio); 3685c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcox put_nvmeq(nvmeq); 3695c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcox wake_up_process(nvme_thread); 3705c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcox} 3715c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcox 3725c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcoxstatic void bio_completion(struct nvme_dev *dev, void *ctx, 373b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_completion *cqe) 374b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 375eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox struct nvme_iod *iod = ctx; 376eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox struct bio *bio = iod->private; 377b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 status = le16_to_cpup(&cqe->status) >> 1; 378b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 379eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox dma_unmap_sg(&dev->pci_dev->dev, iod->sg, iod->nents, 380b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox bio_data_dir(bio) ? DMA_TO_DEVICE : DMA_FROM_DEVICE); 381eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox nvme_free_iod(dev, iod); 38209a58f536436efed02ead722e835cb4ce7674afcMatthew Wilcox if (status) { 3831ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox bio_endio(bio, -EIO); 38409a58f536436efed02ead722e835cb4ce7674afcMatthew Wilcox } else if (bio->bi_vcnt > bio->bi_idx) { 3855c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcox requeue_bio(dev, bio); 3861ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox } else { 3871ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox bio_endio(bio, 0); 3881ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox } 389b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 390b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 391184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox/* length is in bytes. gfp flags indicates whether we may sleep. */ 392eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcoxstatic int nvme_setup_prps(struct nvme_dev *dev, 393eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox struct nvme_common_command *cmd, struct nvme_iod *iod, 394eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox int total_len, gfp_t gfp) 395ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox{ 39699802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox struct dma_pool *pool; 397eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox int length = total_len; 398eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox struct scatterlist *sg = iod->sg; 399ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox int dma_len = sg_dma_len(sg); 400ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox u64 dma_addr = sg_dma_address(sg); 401ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox int offset = offset_in_page(dma_addr); 402e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews __le64 *prp_list; 403eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox __le64 **list = iod_list(iod); 404e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_addr_t prp_dma; 405eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox int nprps, i; 406ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox 407ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmd->prp1 = cpu_to_le64(dma_addr); 408ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox length -= (PAGE_SIZE - offset); 409ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox if (length <= 0) 410eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return total_len; 411ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox 412ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox dma_len -= (PAGE_SIZE - offset); 413ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox if (dma_len) { 414ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox dma_addr += (PAGE_SIZE - offset); 415ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox } else { 416ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox sg = sg_next(sg); 417ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox dma_addr = sg_dma_address(sg); 418ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox dma_len = sg_dma_len(sg); 419ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox } 420ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox 421ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox if (length <= PAGE_SIZE) { 422ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmd->prp2 = cpu_to_le64(dma_addr); 423eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return total_len; 424e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews } 425e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews 426e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews nprps = DIV_ROUND_UP(length, PAGE_SIZE); 42799802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox if (nprps <= (256 / 8)) { 42899802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox pool = dev->prp_small_pool; 429eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->npages = 0; 43099802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox } else { 43199802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox pool = dev->prp_page_pool; 432eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->npages = 1; 43399802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox } 43499802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox 435b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox prp_list = dma_pool_alloc(pool, gfp, &prp_dma); 436b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox if (!prp_list) { 437b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox cmd->prp2 = cpu_to_le64(dma_addr); 438eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->npages = -1; 439eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return (total_len - length) + PAGE_SIZE; 440b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox } 441eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox list[0] = prp_list; 442eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->first_dma = prp_dma; 443e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews cmd->prp2 = cpu_to_le64(prp_dma); 444e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews i = 0; 445e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews for (;;) { 4467523d834dd1573610078eb1ac0933f6490232f90Matthew Wilcox if (i == PAGE_SIZE / 8) { 447e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews __le64 *old_prp_list = prp_list; 448b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox prp_list = dma_pool_alloc(pool, gfp, &prp_dma); 449eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox if (!prp_list) 450eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return total_len - length; 451eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox list[iod->npages++] = prp_list; 4527523d834dd1573610078eb1ac0933f6490232f90Matthew Wilcox prp_list[0] = old_prp_list[i - 1]; 4537523d834dd1573610078eb1ac0933f6490232f90Matthew Wilcox old_prp_list[i - 1] = cpu_to_le64(prp_dma); 4547523d834dd1573610078eb1ac0933f6490232f90Matthew Wilcox i = 1; 455e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews } 456e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews prp_list[i++] = cpu_to_le64(dma_addr); 457e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_len -= PAGE_SIZE; 458e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_addr += PAGE_SIZE; 459e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews length -= PAGE_SIZE; 460e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews if (length <= 0) 461e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews break; 462e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews if (dma_len > 0) 463e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews continue; 464e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews BUG_ON(dma_len < 0); 465e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews sg = sg_next(sg); 466e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_addr = sg_dma_address(sg); 467e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_len = sg_dma_len(sg); 468ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox } 469ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox 470eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return total_len; 471ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox} 472ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox 4731ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox/* NVMe scatterlists require no holes in the virtual address */ 4741ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox#define BIOVEC_NOT_VIRT_MERGEABLE(vec1, vec2) ((vec2)->bv_offset || \ 4751ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox (((vec1)->bv_offset + (vec1)->bv_len) % PAGE_SIZE)) 4761ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox 477eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcoxstatic int nvme_map_bio(struct device *dev, struct nvme_iod *iod, 478b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct bio *bio, enum dma_data_direction dma_dir, int psegs) 479b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 480768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox struct bio_vec *bvec, *bvprv = NULL; 481768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox struct scatterlist *sg = NULL; 4821ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox int i, old_idx, length = 0, nsegs = 0; 483b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 484eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox sg_init_table(iod->sg, psegs); 4851ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox old_idx = bio->bi_idx; 486b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox bio_for_each_segment(bvec, bio, i) { 487768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox if (bvprv && BIOVEC_PHYS_MERGEABLE(bvprv, bvec)) { 488768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox sg->length += bvec->bv_len; 489768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox } else { 4901ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox if (bvprv && BIOVEC_NOT_VIRT_MERGEABLE(bvprv, bvec)) 4911ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox break; 492eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox sg = sg ? sg + 1 : iod->sg; 493768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox sg_set_page(sg, bvec->bv_page, bvec->bv_len, 494768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox bvec->bv_offset); 495768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox nsegs++; 496768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox } 4971ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox length += bvec->bv_len; 498768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox bvprv = bvec; 499b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 5001ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox bio->bi_idx = i; 501eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->nents = nsegs; 502768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox sg_mark_end(sg); 503eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox if (dma_map_sg(dev, iod->sg, iod->nents, dma_dir) == 0) { 5041ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox bio->bi_idx = old_idx; 5051ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox return -ENOMEM; 5061ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox } 5071ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox return length; 508b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 509b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 51000df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcoxstatic int nvme_submit_flush(struct nvme_queue *nvmeq, struct nvme_ns *ns, 51100df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox int cmdid) 51200df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox{ 51300df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox struct nvme_command *cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail]; 51400df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 51500df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox memset(cmnd, 0, sizeof(*cmnd)); 51600df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox cmnd->common.opcode = nvme_cmd_flush; 51700df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox cmnd->common.command_id = cmdid; 51800df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox cmnd->common.nsid = cpu_to_le32(ns->ns_id); 51900df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 52000df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox if (++nvmeq->sq_tail == nvmeq->q_depth) 52100df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox nvmeq->sq_tail = 0; 52200df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox writel(nvmeq->sq_tail, nvmeq->q_db); 52300df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 52400df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox return 0; 52500df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox} 52600df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 52700df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcoxstatic int nvme_submit_flush_data(struct nvme_queue *nvmeq, struct nvme_ns *ns) 52800df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox{ 52900df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox int cmdid = alloc_cmdid(nvmeq, (void *)CMD_CTX_FLUSH, 530ff976d724a74e4522e9ca2de1fb37ac4520f454fMatthew Wilcox special_completion, NVME_IO_TIMEOUT); 53100df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox if (unlikely(cmdid < 0)) 53200df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox return cmdid; 53300df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 53400df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox return nvme_submit_flush(nvmeq, ns, cmdid); 53500df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox} 53600df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 537184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox/* 538184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox * Called with local interrupts disabled and the q_lock held. May not sleep. 539184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox */ 540b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_submit_bio_queue(struct nvme_queue *nvmeq, struct nvme_ns *ns, 541b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct bio *bio) 542b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 543ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox struct nvme_command *cmnd; 544eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox struct nvme_iod *iod; 545b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox enum dma_data_direction dma_dir; 5461ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox int cmdid, length, result = -ENOMEM; 547b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 control; 548b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 dsmgmt; 549b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int psegs = bio_phys_segments(ns->queue, bio); 550b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 55100df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox if ((bio->bi_rw & REQ_FLUSH) && psegs) { 55200df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox result = nvme_submit_flush_data(nvmeq, ns); 55300df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox if (result) 55400df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox return result; 55500df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox } 55600df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 557eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod = nvme_alloc_iod(psegs, bio->bi_size, GFP_ATOMIC); 558eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox if (!iod) 559eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox goto nomem; 560eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->private = bio; 561b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 562eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox result = -EBUSY; 563ff976d724a74e4522e9ca2de1fb37ac4520f454fMatthew Wilcox cmdid = alloc_cmdid(nvmeq, iod, bio_completion, NVME_IO_TIMEOUT); 564b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (unlikely(cmdid < 0)) 565eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox goto free_iod; 566b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 56700df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox if ((bio->bi_rw & REQ_FLUSH) && !psegs) 56800df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox return nvme_submit_flush(nvmeq, ns, cmdid); 56900df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 570b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox control = 0; 571b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (bio->bi_rw & REQ_FUA) 572b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox control |= NVME_RW_FUA; 573b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (bio->bi_rw & (REQ_FAILFAST_DEV | REQ_RAHEAD)) 574b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox control |= NVME_RW_LR; 575b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 576b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dsmgmt = 0; 577b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (bio->bi_rw & REQ_RAHEAD) 578b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH; 579b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 580ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail]; 581b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 582b8deb62cf271fa9381edc8cf52bcae2f0225c55aMatthew Wilcox memset(cmnd, 0, sizeof(*cmnd)); 583b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (bio_data_dir(bio)) { 584ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.opcode = nvme_cmd_write; 585b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_dir = DMA_TO_DEVICE; 586b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } else { 587ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.opcode = nvme_cmd_read; 588b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_dir = DMA_FROM_DEVICE; 589b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 590b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 591eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox result = nvme_map_bio(nvmeq->q_dmadev, iod, bio, dma_dir, psegs); 5921ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox if (result < 0) 593859361a228258edf4821d9f5635825033eca78e8Keith Busch goto free_cmdid; 5941ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox length = result; 595b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 596ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.command_id = cmdid; 597ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.nsid = cpu_to_le32(ns->ns_id); 598eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox length = nvme_setup_prps(nvmeq->dev, &cmnd->common, iod, length, 599eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox GFP_ATOMIC); 600ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.slba = cpu_to_le64(bio->bi_sector >> (ns->lba_shift - 9)); 6011ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox cmnd->rw.length = cpu_to_le16((length >> ns->lba_shift) - 1); 602ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.control = cpu_to_le16(control); 603ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt); 604b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 605d8ee9d69f275769aaad40ef7c944565ff8d2d24fMatthew Wilcox bio->bi_sector += length >> 9; 606d8ee9d69f275769aaad40ef7c944565ff8d2d24fMatthew Wilcox 607b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (++nvmeq->sq_tail == nvmeq->q_depth) 608b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->sq_tail = 0; 6097547881d0951384f9833ec3a80fac8f3f16f3b98Matthew Wilcox writel(nvmeq->sq_tail, nvmeq->q_db); 610b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 6111974b1ae8852324a75fb8cfecbc7b758fd5a2c3cMatthew Wilcox return 0; 6121974b1ae8852324a75fb8cfecbc7b758fd5a2c3cMatthew Wilcox 613859361a228258edf4821d9f5635825033eca78e8Keith Busch free_cmdid: 614859361a228258edf4821d9f5635825033eca78e8Keith Busch free_cmdid(nvmeq, cmdid, NULL); 615eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox free_iod: 616eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox nvme_free_iod(nvmeq->dev, iod); 617eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox nomem: 618eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox return result; 619b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 620b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 62193c3d65b28bab6da520c2add9cb387a0303f8b2dLinus Torvaldsstatic void nvme_make_request(struct request_queue *q, struct bio *bio) 622b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 623b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_ns *ns = q->queuedata; 624040a93b52a9eee8177ebaf2ba0ee0f9f518d1bf8Matthew Wilcox struct nvme_queue *nvmeq = get_nvmeq(ns->dev); 625eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox int result = -EBUSY; 626eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox 627eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox spin_lock_irq(&nvmeq->q_lock); 628eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox if (bio_list_empty(&nvmeq->sq_cong)) 629eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox result = nvme_submit_bio_queue(nvmeq, ns, bio); 630eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox if (unlikely(result)) { 631eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox if (bio_list_empty(&nvmeq->sq_cong)) 632eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox add_wait_queue(&nvmeq->sq_full, &nvmeq->sq_cong_wait); 633b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox bio_list_add(&nvmeq->sq_cong, bio); 634b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 635eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox 636eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox spin_unlock_irq(&nvmeq->q_lock); 637b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox put_nvmeq(nvmeq); 638b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 639b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 640b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic irqreturn_t nvme_process_cq(struct nvme_queue *nvmeq) 641b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 642821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox u16 head, phase; 643b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 644b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox head = nvmeq->cq_head; 645821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox phase = nvmeq->cq_phase; 646b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 647b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox for (;;) { 648c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox void *ctx; 649c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox nvme_completion_fn fn; 650b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_completion cqe = nvmeq->cqes[head]; 651821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox if ((le16_to_cpu(cqe.status) & 1) != phase) 652b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox break; 653b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->sq_head = le16_to_cpu(cqe.sq_head); 654b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (++head == nvmeq->q_depth) { 655b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox head = 0; 656821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox phase = !phase; 657b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 658b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 659c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox ctx = free_cmdid(nvmeq, cqe.command_id, &fn); 6605c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcox fn(nvmeq->dev, ctx, &cqe); 661b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 662b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 663b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox /* If the controller ignores the cq head doorbell and continuously 664b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * writes to the queue, it is theoretically possible to wrap around 665b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * the queue twice and mistakenly return IRQ_NONE. Linux only 666b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * requires that 0.1% of your interrupts are handled, so this isn't 667b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * a big problem. 668b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 669821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox if (head == nvmeq->cq_head && phase == nvmeq->cq_phase) 670b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return IRQ_NONE; 671b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 672f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox writel(head, nvmeq->q_db + (1 << nvmeq->dev->db_stride)); 673b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->cq_head = head; 674821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox nvmeq->cq_phase = phase; 675b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 676b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return IRQ_HANDLED; 677b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 678b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 679b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic irqreturn_t nvme_irq(int irq, void *data) 680b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 68158ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox irqreturn_t result; 68258ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox struct nvme_queue *nvmeq = data; 68358ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox spin_lock(&nvmeq->q_lock); 68458ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox result = nvme_process_cq(nvmeq); 68558ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox spin_unlock(&nvmeq->q_lock); 68658ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox return result; 68758ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox} 68858ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox 68958ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcoxstatic irqreturn_t nvme_irq_check(int irq, void *data) 69058ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox{ 69158ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox struct nvme_queue *nvmeq = data; 69258ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox struct nvme_completion cqe = nvmeq->cqes[nvmeq->cq_head]; 69358ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox if ((le16_to_cpu(cqe.status) & 1) != nvmeq->cq_phase) 69458ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox return IRQ_NONE; 69558ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox return IRQ_WAKE_THREAD; 69658ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox} 69758ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox 6983c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcoxstatic void nvme_abort_command(struct nvme_queue *nvmeq, int cmdid) 6993c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox{ 7003c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox spin_lock_irq(&nvmeq->q_lock); 701c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox cancel_cmdid(nvmeq, cmdid, NULL); 7023c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox spin_unlock_irq(&nvmeq->q_lock); 7033c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox} 7043c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox 705c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcoxstruct sync_cmd_info { 706c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox struct task_struct *task; 707c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox u32 result; 708c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox int status; 709c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox}; 710c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox 7115c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcoxstatic void sync_completion(struct nvme_dev *dev, void *ctx, 712c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox struct nvme_completion *cqe) 713c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox{ 714c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox struct sync_cmd_info *cmdinfo = ctx; 715c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox cmdinfo->result = le32_to_cpup(&cqe->result); 716c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox cmdinfo->status = le16_to_cpup(&cqe->status) >> 1; 717c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox wake_up_process(cmdinfo->task); 718c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox} 719c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox 720b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 721b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Returns 0 on success. If the result is negative, it's a Linux error code; 722b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * if the result is positive, it's an NVM Express status code 723b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 7243c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcoxstatic int nvme_submit_sync_cmd(struct nvme_queue *nvmeq, 725e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox struct nvme_command *cmd, u32 *result, unsigned timeout) 726b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 727b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int cmdid; 728b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct sync_cmd_info cmdinfo; 729b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 730b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cmdinfo.task = current; 731b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cmdinfo.status = -EINTR; 732b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 733c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox cmdid = alloc_cmdid_killable(nvmeq, &cmdinfo, sync_completion, 734e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox timeout); 735b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (cmdid < 0) 736b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return cmdid; 737b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cmd->common.command_id = cmdid; 738b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 7393c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox set_current_state(TASK_KILLABLE); 7403c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox nvme_submit_cmd(nvmeq, cmd); 741b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox schedule(); 742b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 7433c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox if (cmdinfo.status == -EINTR) { 7443c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox nvme_abort_command(nvmeq, cmdid); 7453c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox return -EINTR; 7463c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox } 7473c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox 748b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (result) 749b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox *result = cmdinfo.result; 750b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 751b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return cmdinfo.status; 752b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 753b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 754b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_submit_admin_cmd(struct nvme_dev *dev, struct nvme_command *cmd, 755b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 *result) 756b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 757e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox return nvme_submit_sync_cmd(dev->queues[0], cmd, result, ADMIN_TIMEOUT); 758b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 759b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 760b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id) 761b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 762b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int status; 763b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_command c; 764b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 765b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset(&c, 0, sizeof(c)); 766b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.delete_queue.opcode = opcode; 767b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.delete_queue.qid = cpu_to_le16(id); 768b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 769b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox status = nvme_submit_admin_cmd(dev, &c, NULL); 770b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (status) 771b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -EIO; 772b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 773b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 774b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 775b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_alloc_cq(struct nvme_dev *dev, u16 qid, 776b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq) 777b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 778b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int status; 779b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_command c; 780b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int flags = NVME_QUEUE_PHYS_CONTIG | NVME_CQ_IRQ_ENABLED; 781b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 782b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset(&c, 0, sizeof(c)); 783b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.opcode = nvme_admin_create_cq; 784b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.prp1 = cpu_to_le64(nvmeq->cq_dma_addr); 785b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.cqid = cpu_to_le16(qid); 786b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1); 787b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.cq_flags = cpu_to_le16(flags); 788b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.irq_vector = cpu_to_le16(nvmeq->cq_vector); 789b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 790b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox status = nvme_submit_admin_cmd(dev, &c, NULL); 791b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (status) 792b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -EIO; 793b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 794b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 795b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 796b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_alloc_sq(struct nvme_dev *dev, u16 qid, 797b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq) 798b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 799b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int status; 800b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_command c; 801b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int flags = NVME_QUEUE_PHYS_CONTIG | NVME_SQ_PRIO_MEDIUM; 802b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 803b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset(&c, 0, sizeof(c)); 804b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.opcode = nvme_admin_create_sq; 805b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.prp1 = cpu_to_le64(nvmeq->sq_dma_addr); 806b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.sqid = cpu_to_le16(qid); 807b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1); 808b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.sq_flags = cpu_to_le16(flags); 809b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.cqid = cpu_to_le16(qid); 810b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 811b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox status = nvme_submit_admin_cmd(dev, &c, NULL); 812b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (status) 813b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -EIO; 814b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 815b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 816b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 817b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_delete_cq(struct nvme_dev *dev, u16 cqid) 818b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 819b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return adapter_delete_queue(dev, nvme_admin_delete_cq, cqid); 820b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 821b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 822b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_delete_sq(struct nvme_dev *dev, u16 sqid) 823b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 824b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return adapter_delete_queue(dev, nvme_admin_delete_sq, sqid); 825b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 826b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 827bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcoxstatic int nvme_identify(struct nvme_dev *dev, unsigned nsid, unsigned cns, 828bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox dma_addr_t dma_addr) 829bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox{ 830bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox struct nvme_command c; 831bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox 832bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox memset(&c, 0, sizeof(c)); 833bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox c.identify.opcode = nvme_admin_identify; 834bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox c.identify.nsid = cpu_to_le32(nsid); 835bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox c.identify.prp1 = cpu_to_le64(dma_addr); 836bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox c.identify.cns = cpu_to_le32(cns); 837bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox 838bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox return nvme_submit_admin_cmd(dev, &c, NULL); 839bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox} 840bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox 84108df1e05657fc6712e520e7c09cc6c86160ceb35Keith Buschstatic int nvme_get_features(struct nvme_dev *dev, unsigned fid, unsigned nsid, 84208df1e05657fc6712e520e7c09cc6c86160ceb35Keith Busch dma_addr_t dma_addr, u32 *result) 843bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox{ 844bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox struct nvme_command c; 845bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox 846bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox memset(&c, 0, sizeof(c)); 847bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox c.features.opcode = nvme_admin_get_features; 848a42ceccef0c43b46ff6bc1b12a7c1076ef243df1Keith Busch c.features.nsid = cpu_to_le32(nsid); 849bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox c.features.prp1 = cpu_to_le64(dma_addr); 850bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox c.features.fid = cpu_to_le32(fid); 851bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox 85208df1e05657fc6712e520e7c09cc6c86160ceb35Keith Busch return nvme_submit_admin_cmd(dev, &c, result); 853df3481399042200792822b6243e36a95a557b57eMatthew Wilcox} 854df3481399042200792822b6243e36a95a557b57eMatthew Wilcox 855df3481399042200792822b6243e36a95a557b57eMatthew Wilcoxstatic int nvme_set_features(struct nvme_dev *dev, unsigned fid, 856df3481399042200792822b6243e36a95a557b57eMatthew Wilcox unsigned dword11, dma_addr_t dma_addr, u32 *result) 857df3481399042200792822b6243e36a95a557b57eMatthew Wilcox{ 858df3481399042200792822b6243e36a95a557b57eMatthew Wilcox struct nvme_command c; 859df3481399042200792822b6243e36a95a557b57eMatthew Wilcox 860df3481399042200792822b6243e36a95a557b57eMatthew Wilcox memset(&c, 0, sizeof(c)); 861df3481399042200792822b6243e36a95a557b57eMatthew Wilcox c.features.opcode = nvme_admin_set_features; 862df3481399042200792822b6243e36a95a557b57eMatthew Wilcox c.features.prp1 = cpu_to_le64(dma_addr); 863df3481399042200792822b6243e36a95a557b57eMatthew Wilcox c.features.fid = cpu_to_le32(fid); 864df3481399042200792822b6243e36a95a557b57eMatthew Wilcox c.features.dword11 = cpu_to_le32(dword11); 865df3481399042200792822b6243e36a95a557b57eMatthew Wilcox 866bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox return nvme_submit_admin_cmd(dev, &c, result); 867bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox} 868bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox 869a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox/** 870a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox * nvme_cancel_ios - Cancel outstanding I/Os 871a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox * @queue: The queue to cancel I/Os on 872a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox * @timeout: True to only cancel I/Os which have timed out 873a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox */ 874a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcoxstatic void nvme_cancel_ios(struct nvme_queue *nvmeq, bool timeout) 875a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox{ 876a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox int depth = nvmeq->q_depth - 1; 877a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox struct nvme_cmd_info *info = nvme_cmd_info(nvmeq); 878a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox unsigned long now = jiffies; 879a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox int cmdid; 880a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox 881a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox for_each_set_bit(cmdid, nvmeq->cmdid_data, depth) { 882a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox void *ctx; 883a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox nvme_completion_fn fn; 884a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox static struct nvme_completion cqe = { 885a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox .status = cpu_to_le16(NVME_SC_ABORT_REQ) << 1, 886a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox }; 887a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox 888a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox if (timeout && !time_after(now, info[cmdid].timeout)) 889a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox continue; 890a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox dev_warn(nvmeq->q_dmadev, "Cancelling I/O %d\n", cmdid); 891a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox ctx = cancel_cmdid(nvmeq, cmdid, &fn); 892a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox fn(nvmeq->dev, ctx, &cqe); 893a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox } 894a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox} 895a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox 8969e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcoxstatic void nvme_free_queue_mem(struct nvme_queue *nvmeq) 8979e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox{ 8989e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth), 8999e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox (void *)nvmeq->cqes, nvmeq->cq_dma_addr); 9009e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth), 9019e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox nvmeq->sq_cmds, nvmeq->sq_dma_addr); 9029e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox kfree(nvmeq); 9039e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox} 9049e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox 905b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void nvme_free_queue(struct nvme_dev *dev, int qid) 906b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 907b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq = dev->queues[qid]; 908aba2080f3f1639f9202f1a52993669844abcfb80Matthew Wilcox int vector = dev->entry[nvmeq->cq_vector].vector; 909b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 910a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox spin_lock_irq(&nvmeq->q_lock); 911a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox nvme_cancel_ios(nvmeq, false); 9123295874b6074d749516d6decd43afad7bf6e38ffKeith Busch while (bio_list_peek(&nvmeq->sq_cong)) { 9133295874b6074d749516d6decd43afad7bf6e38ffKeith Busch struct bio *bio = bio_list_pop(&nvmeq->sq_cong); 9143295874b6074d749516d6decd43afad7bf6e38ffKeith Busch bio_endio(bio, -EIO); 9153295874b6074d749516d6decd43afad7bf6e38ffKeith Busch } 916a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox spin_unlock_irq(&nvmeq->q_lock); 917a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox 918aba2080f3f1639f9202f1a52993669844abcfb80Matthew Wilcox irq_set_affinity_hint(vector, NULL); 919aba2080f3f1639f9202f1a52993669844abcfb80Matthew Wilcox free_irq(vector, nvmeq); 920b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 921b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox /* Don't tell the adapter to delete the admin queue */ 922b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (qid) { 923b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox adapter_delete_sq(dev, qid); 924b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox adapter_delete_cq(dev, qid); 925b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 926b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 9279e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox nvme_free_queue_mem(nvmeq); 928b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 929b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 930b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev, int qid, 931b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int depth, int vector) 932b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 933b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct device *dmadev = &dev->pci_dev->dev; 934a0cadb85b8b758608ae0759151e29de7581c6731Keith Busch unsigned extra = DIV_ROUND_UP(depth, 8) + (depth * 935a0cadb85b8b758608ae0759151e29de7581c6731Keith Busch sizeof(struct nvme_cmd_info)); 936b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq = kzalloc(sizeof(*nvmeq) + extra, GFP_KERNEL); 937b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!nvmeq) 938b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 939b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 940b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->cqes = dma_alloc_coherent(dmadev, CQ_SIZE(depth), 941b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox &nvmeq->cq_dma_addr, GFP_KERNEL); 942b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!nvmeq->cqes) 943b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto free_nvmeq; 944b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset((void *)nvmeq->cqes, 0, CQ_SIZE(depth)); 945b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 946b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->sq_cmds = dma_alloc_coherent(dmadev, SQ_SIZE(depth), 947b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox &nvmeq->sq_dma_addr, GFP_KERNEL); 948b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!nvmeq->sq_cmds) 949b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto free_cqdma; 950b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 951b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->q_dmadev = dmadev; 952091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox nvmeq->dev = dev; 953b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox spin_lock_init(&nvmeq->q_lock); 954b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->cq_head = 0; 955821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox nvmeq->cq_phase = 1; 956b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox init_waitqueue_head(&nvmeq->sq_full); 9571fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox init_waitqueue_entry(&nvmeq->sq_cong_wait, nvme_thread); 958b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox bio_list_init(&nvmeq->sq_cong); 959f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox nvmeq->q_db = &dev->dbs[qid << (dev->db_stride + 1)]; 960b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->q_depth = depth; 961b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->cq_vector = vector; 962b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 963b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return nvmeq; 964b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 965b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox free_cqdma: 966b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_free_coherent(dmadev, CQ_SIZE(nvmeq->q_depth), (void *)nvmeq->cqes, 967b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->cq_dma_addr); 968b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox free_nvmeq: 969b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(nvmeq); 970b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 971b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 972b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 9733001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcoxstatic int queue_request_irq(struct nvme_dev *dev, struct nvme_queue *nvmeq, 9743001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox const char *name) 9753001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox{ 97658ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox if (use_threaded_interrupts) 97758ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox return request_threaded_irq(dev->entry[nvmeq->cq_vector].vector, 978ec6ce618d65b5ce1bef83a5509255107a0feac44Matthew Wilcox nvme_irq_check, nvme_irq, 97958ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox IRQF_DISABLED | IRQF_SHARED, 98058ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox name, nvmeq); 9813001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox return request_irq(dev->entry[nvmeq->cq_vector].vector, nvme_irq, 9823001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox IRQF_DISABLED | IRQF_SHARED, name, nvmeq); 9833001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox} 9843001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox 985b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic __devinit struct nvme_queue *nvme_create_queue(struct nvme_dev *dev, 986b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int qid, int cq_size, int vector) 987b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 988b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int result; 989b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq = nvme_alloc_queue(dev, qid, cq_size, vector); 990b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 9913f85d50b609e8a5ef151656210203a6e94c19538Matthew Wilcox if (!nvmeq) 9926f0f54499f2edf7e25410cdd99e6f030f3485fd1Matthew Wilcox return ERR_PTR(-ENOMEM); 9933f85d50b609e8a5ef151656210203a6e94c19538Matthew Wilcox 994b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox result = adapter_alloc_cq(dev, qid, nvmeq); 995b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (result < 0) 996b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto free_nvmeq; 997b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 998b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox result = adapter_alloc_sq(dev, qid, nvmeq); 999b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (result < 0) 1000b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto release_cq; 1001b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 10023001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox result = queue_request_irq(dev, nvmeq, "nvme"); 1003b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (result < 0) 1004b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto release_sq; 1005b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1006b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return nvmeq; 1007b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1008b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox release_sq: 1009b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox adapter_delete_sq(dev, qid); 1010b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox release_cq: 1011b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox adapter_delete_cq(dev, qid); 1012b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox free_nvmeq: 1013b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth), 1014b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox (void *)nvmeq->cqes, nvmeq->cq_dma_addr); 1015b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth), 1016b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->sq_cmds, nvmeq->sq_dma_addr); 1017b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(nvmeq); 10186f0f54499f2edf7e25410cdd99e6f030f3485fd1Matthew Wilcox return ERR_PTR(result); 1019b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1020b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1021b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int __devinit nvme_configure_admin_queue(struct nvme_dev *dev) 1022b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 10239e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox int result = 0; 1024b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 aqa; 102522605f96810d073eb74051d0295b6577d6a6a563Matthew Wilcox u64 cap; 102622605f96810d073eb74051d0295b6577d6a6a563Matthew Wilcox unsigned long timeout; 1027b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq; 1028b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1029b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->dbs = ((void __iomem *)dev->bar) + 4096; 1030b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1031b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq = nvme_alloc_queue(dev, 0, 64, 0); 10323f85d50b609e8a5ef151656210203a6e94c19538Matthew Wilcox if (!nvmeq) 10333f85d50b609e8a5ef151656210203a6e94c19538Matthew Wilcox return -ENOMEM; 1034b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1035b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox aqa = nvmeq->q_depth - 1; 1036b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox aqa |= aqa << 16; 1037b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1038b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->ctrl_config = NVME_CC_ENABLE | NVME_CC_CSS_NVM; 1039b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->ctrl_config |= (PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT; 1040b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->ctrl_config |= NVME_CC_ARB_RR | NVME_CC_SHN_NONE; 10417f53f9d2424533256ae86f7df5661a17de743de8Matthew Wilcox dev->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES; 1042b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 10435911f20039ce59d7e7834f0c42151cf759b6f786Shane Michael Matthews writel(0, &dev->bar->cc); 1044b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox writel(aqa, &dev->bar->aqa); 1045b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox writeq(nvmeq->sq_dma_addr, &dev->bar->asq); 1046b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox writeq(nvmeq->cq_dma_addr, &dev->bar->acq); 1047b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox writel(dev->ctrl_config, &dev->bar->cc); 1048b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 104922605f96810d073eb74051d0295b6577d6a6a563Matthew Wilcox cap = readq(&dev->bar->cap); 105022605f96810d073eb74051d0295b6577d6a6a563Matthew Wilcox timeout = ((NVME_CAP_TIMEOUT(cap) + 1) * HZ / 2) + jiffies; 1051f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox dev->db_stride = NVME_CAP_STRIDE(cap); 105222605f96810d073eb74051d0295b6577d6a6a563Matthew Wilcox 10539e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox while (!result && !(readl(&dev->bar->csts) & NVME_CSTS_RDY)) { 1054b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox msleep(100); 1055b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (fatal_signal_pending(current)) 10569e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox result = -EINTR; 105722605f96810d073eb74051d0295b6577d6a6a563Matthew Wilcox if (time_after(jiffies, timeout)) { 105822605f96810d073eb74051d0295b6577d6a6a563Matthew Wilcox dev_err(&dev->pci_dev->dev, 105922605f96810d073eb74051d0295b6577d6a6a563Matthew Wilcox "Device not ready; aborting initialisation\n"); 10609e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox result = -ENODEV; 106122605f96810d073eb74051d0295b6577d6a6a563Matthew Wilcox } 1062b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 1063b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 10649e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox if (result) { 10659e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox nvme_free_queue_mem(nvmeq); 10669e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox return result; 10679e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox } 10689e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox 10693001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox result = queue_request_irq(dev, nvmeq, "nvme admin"); 1070b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->queues[0] = nvmeq; 1071b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return result; 1072b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1073b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1074eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcoxstatic struct nvme_iod *nvme_map_user_pages(struct nvme_dev *dev, int write, 1075eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox unsigned long addr, unsigned length) 1076b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 107736c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox int i, err, count, nents, offset; 10787fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox struct scatterlist *sg; 10797fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox struct page **pages; 1080eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox struct nvme_iod *iod; 108136c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox 108236c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox if (addr & 3) 1083eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return ERR_PTR(-EINVAL); 10847fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox if (!length) 1085eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return ERR_PTR(-EINVAL); 10867fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox 108736c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox offset = offset_in_page(addr); 10887fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox count = DIV_ROUND_UP(offset + length, PAGE_SIZE); 10897fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox pages = kcalloc(count, sizeof(*pages), GFP_KERNEL); 109022fff826e715e9727d3c7a69f15e602a9801b673Dan Carpenter if (!pages) 109122fff826e715e9727d3c7a69f15e602a9801b673Dan Carpenter return ERR_PTR(-ENOMEM); 109236c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox 109336c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox err = get_user_pages_fast(addr, count, 1, pages); 109436c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox if (err < count) { 109536c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox count = err; 109636c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox err = -EFAULT; 109736c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox goto put_pages; 109836c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox } 10997fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox 1100eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod = nvme_alloc_iod(count, length, GFP_KERNEL); 1101eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox sg = iod->sg; 110236c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox sg_init_table(sg, count); 1103d0ba1e497bca83a3d353eb47c9658afc54d83228Matthew Wilcox for (i = 0; i < count; i++) { 1104d0ba1e497bca83a3d353eb47c9658afc54d83228Matthew Wilcox sg_set_page(&sg[i], pages[i], 1105d0ba1e497bca83a3d353eb47c9658afc54d83228Matthew Wilcox min_t(int, length, PAGE_SIZE - offset), offset); 1106d0ba1e497bca83a3d353eb47c9658afc54d83228Matthew Wilcox length -= (PAGE_SIZE - offset); 1107d0ba1e497bca83a3d353eb47c9658afc54d83228Matthew Wilcox offset = 0; 11087fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox } 1109fe304c43c6d63e29ed4fc46a874d7a74313788c5Matthew Wilcox sg_mark_end(&sg[i - 1]); 11101c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox iod->nents = count; 11117fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox 11127fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox err = -ENOMEM; 11137fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox nents = dma_map_sg(&dev->pci_dev->dev, sg, count, 11147fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox write ? DMA_TO_DEVICE : DMA_FROM_DEVICE); 111536c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox if (!nents) 1116eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox goto free_iod; 1117b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 11187fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox kfree(pages); 1119eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return iod; 1120b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1121eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox free_iod: 1122eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox kfree(iod); 11237fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox put_pages: 11247fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox for (i = 0; i < count; i++) 11257fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox put_page(pages[i]); 11267fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox kfree(pages); 1127eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return ERR_PTR(err); 11287fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox} 1129b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 11307fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcoxstatic void nvme_unmap_user_pages(struct nvme_dev *dev, int write, 11311c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox struct nvme_iod *iod) 11327fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox{ 11331c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox int i; 1134b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 11351c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox dma_unmap_sg(&dev->pci_dev->dev, iod->sg, iod->nents, 11361c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox write ? DMA_TO_DEVICE : DMA_FROM_DEVICE); 11377fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox 11381c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox for (i = 0; i < iod->nents; i++) 11391c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox put_page(sg_page(&iod->sg[i])); 11407fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox} 1141b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1142a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcoxstatic int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio) 1143a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox{ 1144a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox struct nvme_dev *dev = ns->dev; 1145a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox struct nvme_queue *nvmeq; 1146a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox struct nvme_user_io io; 1147a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox struct nvme_command c; 1148a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox unsigned length; 1149eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox int status; 1150eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox struct nvme_iod *iod; 1151a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox 1152a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox if (copy_from_user(&io, uio, sizeof(io))) 1153a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox return -EFAULT; 11546c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox length = (io.nblocks + 1) << ns->lba_shift; 11556c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox 11566c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox switch (io.opcode) { 11576c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox case nvme_cmd_write: 11586c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox case nvme_cmd_read: 11596bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox case nvme_cmd_compare: 1160eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod = nvme_map_user_pages(dev, io.opcode & 1, io.addr, length); 11616413214c5d424fd5aae6567848340f962ad2ce0fMatthew Wilcox break; 11626c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox default: 11636bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox return -EINVAL; 11646c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox } 11656c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox 1166eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox if (IS_ERR(iod)) 1167eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return PTR_ERR(iod); 1168a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox 1169a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox memset(&c, 0, sizeof(c)); 1170a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.opcode = io.opcode; 1171a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.flags = io.flags; 11726c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox c.rw.nsid = cpu_to_le32(ns->ns_id); 1173a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.slba = cpu_to_le64(io.slba); 11746c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox c.rw.length = cpu_to_le16(io.nblocks); 1175a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.control = cpu_to_le16(io.control); 1176a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.dsmgmt = cpu_to_le16(io.dsmgmt); 11776c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox c.rw.reftag = io.reftag; 11786c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox c.rw.apptag = io.apptag; 11796c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox c.rw.appmask = io.appmask; 1180a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox /* XXX: metadata */ 1181eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox length = nvme_setup_prps(dev, &c.common, iod, length, GFP_KERNEL); 1182a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox 1183040a93b52a9eee8177ebaf2ba0ee0f9f518d1bf8Matthew Wilcox nvmeq = get_nvmeq(dev); 1184fa92282149842645931580225647238428374758Matthew Wilcox /* 1185fa92282149842645931580225647238428374758Matthew Wilcox * Since nvme_submit_sync_cmd sleeps, we can't keep preemption 1186b1ad37efcafe396ac3944853589688dd0ec3c64eMatthew Wilcox * disabled. We may be preempted at any point, and be rescheduled 1187b1ad37efcafe396ac3944853589688dd0ec3c64eMatthew Wilcox * to a different CPU. That will cause cacheline bouncing, but no 1188b1ad37efcafe396ac3944853589688dd0ec3c64eMatthew Wilcox * additional races since q_lock already protects against other CPUs. 1189b1ad37efcafe396ac3944853589688dd0ec3c64eMatthew Wilcox */ 1190a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox put_nvmeq(nvmeq); 1191b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox if (length != (io.nblocks + 1) << ns->lba_shift) 1192b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox status = -ENOMEM; 1193b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox else 1194ff976d724a74e4522e9ca2de1fb37ac4520f454fMatthew Wilcox status = nvme_submit_sync_cmd(nvmeq, &c, NULL, NVME_IO_TIMEOUT); 1195a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox 11961c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox nvme_unmap_user_pages(dev, io.opcode & 1, iod); 1197eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox nvme_free_iod(dev, iod); 1198a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox return status; 1199a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox} 1200a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox 120150af8baec46a99a9b81a4600c0374f83a5a590a9Keith Buschstatic int nvme_user_admin_cmd(struct nvme_dev *dev, 12026bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox struct nvme_admin_cmd __user *ucmd) 12036ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox{ 12046bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox struct nvme_admin_cmd cmd; 12056ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox struct nvme_command c; 1206eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox int status, length; 1207c7d36ab8fa04c213328119a9c0d66985fe204ee5Keith Busch struct nvme_iod *uninitialized_var(iod); 12086ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox 12096bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox if (!capable(CAP_SYS_ADMIN)) 12106bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox return -EACCES; 12116bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox if (copy_from_user(&cmd, ucmd, sizeof(cmd))) 12126ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox return -EFAULT; 12136ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox 12146ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox memset(&c, 0, sizeof(c)); 12156bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.opcode = cmd.opcode; 12166bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.flags = cmd.flags; 12176bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.nsid = cpu_to_le32(cmd.nsid); 12186bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw2[0] = cpu_to_le32(cmd.cdw2); 12196bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw2[1] = cpu_to_le32(cmd.cdw3); 12206bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw10[0] = cpu_to_le32(cmd.cdw10); 12216bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw10[1] = cpu_to_le32(cmd.cdw11); 12226bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw10[2] = cpu_to_le32(cmd.cdw12); 12236bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw10[3] = cpu_to_le32(cmd.cdw13); 12246bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw10[4] = cpu_to_le32(cmd.cdw14); 12256bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw10[5] = cpu_to_le32(cmd.cdw15); 12266bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox 12276bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox length = cmd.data_len; 12286bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox if (cmd.data_len) { 1229497421880acecd0281d3182d534f3d28c927caecMatthew Wilcox iod = nvme_map_user_pages(dev, cmd.opcode & 1, cmd.addr, 1230497421880acecd0281d3182d534f3d28c927caecMatthew Wilcox length); 1231eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox if (IS_ERR(iod)) 1232eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return PTR_ERR(iod); 1233eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox length = nvme_setup_prps(dev, &c.common, iod, length, 1234eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox GFP_KERNEL); 12356bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox } 12366bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox 12376bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox if (length != cmd.data_len) 1238b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox status = -ENOMEM; 1239b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox else 1240f4f117f64baf8840d22266d518227b2a186d294bKeith Busch status = nvme_submit_admin_cmd(dev, &c, &cmd.result); 1241eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox 12426bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox if (cmd.data_len) { 12431c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox nvme_unmap_user_pages(dev, cmd.opcode & 1, iod); 1244eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox nvme_free_iod(dev, iod); 12456bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox } 1246f4f117f64baf8840d22266d518227b2a186d294bKeith Busch 1247f4f117f64baf8840d22266d518227b2a186d294bKeith Busch if (!status && copy_to_user(&ucmd->result, &cmd.result, 1248f4f117f64baf8840d22266d518227b2a186d294bKeith Busch sizeof(cmd.result))) 1249f4f117f64baf8840d22266d518227b2a186d294bKeith Busch status = -EFAULT; 1250f4f117f64baf8840d22266d518227b2a186d294bKeith Busch 12516ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox return status; 12526ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox} 12536ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox 1254b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, 1255b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unsigned long arg) 1256b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1257b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_ns *ns = bdev->bd_disk->private_data; 1258b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1259b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox switch (cmd) { 12606bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox case NVME_IOCTL_ID: 12616bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox return ns->ns_id; 12626bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox case NVME_IOCTL_ADMIN_CMD: 126350af8baec46a99a9b81a4600c0374f83a5a590a9Keith Busch return nvme_user_admin_cmd(ns->dev, (void __user *)arg); 1264a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox case NVME_IOCTL_SUBMIT_IO: 1265a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox return nvme_submit_io(ns, (void __user *)arg); 1266b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox default: 1267b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -ENOTTY; 1268b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 1269b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1270b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1271b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic const struct block_device_operations nvme_fops = { 1272b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .owner = THIS_MODULE, 1273b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .ioctl = nvme_ioctl, 12744948168280b269a514045766ddd872cfac5968e1Matthew Wilcox .compat_ioctl = nvme_ioctl, 1275b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 1276b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 12771fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcoxstatic void nvme_resubmit_bios(struct nvme_queue *nvmeq) 12781fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox{ 12791fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox while (bio_list_peek(&nvmeq->sq_cong)) { 12801fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox struct bio *bio = bio_list_pop(&nvmeq->sq_cong); 12811fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox struct nvme_ns *ns = bio->bi_bdev->bd_disk->private_data; 12821fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox if (nvme_submit_bio_queue(nvmeq, ns, bio)) { 12831fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox bio_list_add_head(&nvmeq->sq_cong, bio); 12841fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox break; 12851fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox } 12863cb967c03926edd2c414082f4cc0feb7b372edaeMatthew Wilcox if (bio_list_empty(&nvmeq->sq_cong)) 12873cb967c03926edd2c414082f4cc0feb7b372edaeMatthew Wilcox remove_wait_queue(&nvmeq->sq_full, 12883cb967c03926edd2c414082f4cc0feb7b372edaeMatthew Wilcox &nvmeq->sq_cong_wait); 12891fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox } 12901fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox} 12911fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 12921fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcoxstatic int nvme_kthread(void *data) 12931fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox{ 12941fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox struct nvme_dev *dev; 12951fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 12961fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox while (!kthread_should_stop()) { 12971fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox __set_current_state(TASK_RUNNING); 12981fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_lock(&dev_list_lock); 12991fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox list_for_each_entry(dev, &dev_list, node) { 13001fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox int i; 13011fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox for (i = 0; i < dev->queue_count; i++) { 13021fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox struct nvme_queue *nvmeq = dev->queues[i]; 1303740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox if (!nvmeq) 1304740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox continue; 13051fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_lock_irq(&nvmeq->q_lock); 13061fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox if (nvme_process_cq(nvmeq)) 13071fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox printk("process_cq did something\n"); 1308a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox nvme_cancel_ios(nvmeq, true); 13091fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox nvme_resubmit_bios(nvmeq); 13101fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_unlock_irq(&nvmeq->q_lock); 13111fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox } 13121fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox } 13131fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_unlock(&dev_list_lock); 13141fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox set_current_state(TASK_INTERRUPTIBLE); 13151fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox schedule_timeout(HZ); 13161fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox } 13171fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox return 0; 13181fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox} 13191fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 13205aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcoxstatic DEFINE_IDA(nvme_index_ida); 13215aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox 13225aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcoxstatic int nvme_get_ns_idx(void) 13235aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox{ 13245aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox int index, error; 13255aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox 13265aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox do { 13275aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox if (!ida_pre_get(&nvme_index_ida, GFP_KERNEL)) 13285aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox return -1; 13295aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox 13305aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox spin_lock(&dev_list_lock); 13315aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox error = ida_get_new(&nvme_index_ida, &index); 13325aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox spin_unlock(&dev_list_lock); 13335aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox } while (error == -EAGAIN); 13345aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox 13355aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox if (error) 13365aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox index = -1; 13375aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox return index; 13385aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox} 13395aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox 13405aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcoxstatic void nvme_put_ns_idx(int index) 13415aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox{ 13425aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox spin_lock(&dev_list_lock); 13435aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox ida_remove(&nvme_index_ida, index); 13445aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox spin_unlock(&dev_list_lock); 13455aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox} 13465aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox 13475aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcoxstatic struct nvme_ns *nvme_alloc_ns(struct nvme_dev *dev, int nsid, 1348b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_id_ns *id, struct nvme_lba_range_type *rt) 1349b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1350b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_ns *ns; 1351b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct gendisk *disk; 1352b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int lbaf; 1353b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1354b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (rt->attributes & NVME_LBART_ATTRIB_HIDE) 1355b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 1356b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1357b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns = kzalloc(sizeof(*ns), GFP_KERNEL); 1358b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!ns) 1359b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 1360b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->queue = blk_alloc_queue(GFP_KERNEL); 1361b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!ns->queue) 1362b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto out_free_ns; 13634eeb9215a0d5c9494ca8b20158cc8ee82618840cMatthew Wilcox ns->queue->queue_flags = QUEUE_FLAG_DEFAULT; 13644eeb9215a0d5c9494ca8b20158cc8ee82618840cMatthew Wilcox queue_flag_set_unlocked(QUEUE_FLAG_NOMERGES, ns->queue); 13654eeb9215a0d5c9494ca8b20158cc8ee82618840cMatthew Wilcox queue_flag_set_unlocked(QUEUE_FLAG_NONROT, ns->queue); 13664eeb9215a0d5c9494ca8b20158cc8ee82618840cMatthew Wilcox/* queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, ns->queue); */ 1367b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox blk_queue_make_request(ns->queue, nvme_make_request); 1368b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->dev = dev; 1369b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->queue->queuedata = ns; 1370b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1371b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk = alloc_disk(NVME_MINORS); 1372b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!disk) 1373b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto out_free_queue; 13745aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox ns->ns_id = nsid; 1375b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->disk = disk; 1376b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox lbaf = id->flbas & 0xf; 1377b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->lba_shift = id->lbaf[lbaf].ds; 1378e9ef46369f5107e634a93b7fc4e62a1f53343197Keith Busch blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift); 13798fc23e032debd682f5ba9fc524a5846c10d2c522Keith Busch if (dev->max_hw_sectors) 13808fc23e032debd682f5ba9fc524a5846c10d2c522Keith Busch blk_queue_max_hw_sectors(ns->queue, dev->max_hw_sectors); 1381b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1382b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk->major = nvme_major; 1383b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk->minors = NVME_MINORS; 13845aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox disk->first_minor = NVME_MINORS * nvme_get_ns_idx(); 1385b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk->fops = &nvme_fops; 1386b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk->private_data = ns; 1387b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk->queue = ns->queue; 1388388f037f4e7f0a24bac6b1a24f144f5d939f58cfMatthew Wilcox disk->driverfs_dev = &dev->pci_dev->dev; 13895aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox sprintf(disk->disk_name, "nvme%dn%d", dev->instance, nsid); 1390b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox set_capacity(disk, le64_to_cpup(&id->nsze) << (ns->lba_shift - 9)); 1391b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1392b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return ns; 1393b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1394b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox out_free_queue: 1395b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox blk_cleanup_queue(ns->queue); 1396b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox out_free_ns: 1397b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(ns); 1398b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 1399b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1400b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1401b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void nvme_ns_free(struct nvme_ns *ns) 1402b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 14035aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox int index = ns->disk->first_minor / NVME_MINORS; 1404b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox put_disk(ns->disk); 14055aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox nvme_put_ns_idx(index); 1406b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox blk_cleanup_queue(ns->queue); 1407b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(ns); 1408b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1409b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1410b3b06812e199f248561ce7824a4a8a9cd573c05aMatthew Wilcoxstatic int set_queue_count(struct nvme_dev *dev, int count) 1411b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1412b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int status; 1413b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 result; 1414b3b06812e199f248561ce7824a4a8a9cd573c05aMatthew Wilcox u32 q_count = (count - 1) | ((count - 1) << 16); 1415b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1416df3481399042200792822b6243e36a95a557b57eMatthew Wilcox status = nvme_set_features(dev, NVME_FEAT_NUM_QUEUES, q_count, 0, 1417bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox &result); 1418b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (status) 1419b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -EIO; 1420b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return min(result & 0xffff, result >> 16) + 1; 1421b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1422b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1423b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int __devinit nvme_setup_io_queues(struct nvme_dev *dev) 1424b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1425a0cadb85b8b758608ae0759151e29de7581c6731Keith Busch int result, cpu, i, nr_io_queues, db_bar_size, q_depth; 1426b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1427b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox nr_io_queues = num_online_cpus(); 1428b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox result = set_queue_count(dev, nr_io_queues); 14291b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox if (result < 0) 14301b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox return result; 1431b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox if (result < nr_io_queues) 1432b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox nr_io_queues = result; 1433b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 14341b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox /* Deregister the admin queue's interrupt */ 14351b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox free_irq(dev->entry[0].vector, dev->queues[0]); 14361b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox 1437f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox db_bar_size = 4096 + ((nr_io_queues + 1) << (dev->db_stride + 3)); 1438f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox if (db_bar_size > 8192) { 1439f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox iounmap(dev->bar); 1440f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox dev->bar = ioremap(pci_resource_start(dev->pci_dev, 0), 1441f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox db_bar_size); 1442f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox dev->dbs = ((void __iomem *)dev->bar) + 4096; 1443f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox dev->queues[0]->q_db = dev->dbs; 1444f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox } 1445f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox 1446b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox for (i = 0; i < nr_io_queues; i++) 14471b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox dev->entry[i].entry = i; 14481b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox for (;;) { 1449b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox result = pci_enable_msix(dev->pci_dev, dev->entry, 1450b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox nr_io_queues); 14511b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox if (result == 0) { 14521b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox break; 14531b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox } else if (result > 0) { 1454b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox nr_io_queues = result; 14551b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox continue; 14561b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox } else { 1457b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox nr_io_queues = 1; 14581b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox break; 14591b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox } 14601b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox } 14611b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox 14621b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox result = queue_request_irq(dev, dev->queues[0], "nvme admin"); 14631b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox /* XXX: handle failure here */ 14641b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox 14651b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox cpu = cpumask_first(cpu_online_mask); 1466b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox for (i = 0; i < nr_io_queues; i++) { 14671b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox irq_set_affinity_hint(dev->entry[i].vector, get_cpu_mask(cpu)); 14681b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox cpu = cpumask_next(cpu, cpu_online_mask); 14691b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox } 14701b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox 1471a0cadb85b8b758608ae0759151e29de7581c6731Keith Busch q_depth = min_t(int, NVME_CAP_MQES(readq(&dev->bar->cap)) + 1, 1472a0cadb85b8b758608ae0759151e29de7581c6731Keith Busch NVME_Q_DEPTH); 1473b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox for (i = 0; i < nr_io_queues; i++) { 1474a0cadb85b8b758608ae0759151e29de7581c6731Keith Busch dev->queues[i + 1] = nvme_create_queue(dev, i + 1, q_depth, i); 14756f0f54499f2edf7e25410cdd99e6f030f3485fd1Matthew Wilcox if (IS_ERR(dev->queues[i + 1])) 14766f0f54499f2edf7e25410cdd99e6f030f3485fd1Matthew Wilcox return PTR_ERR(dev->queues[i + 1]); 14771b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox dev->queue_count++; 14781b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox } 1479b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 14809ecdc946212f7cd592986b2c519b470404caa6b8Matthew Wilcox for (; i < num_possible_cpus(); i++) { 14819ecdc946212f7cd592986b2c519b470404caa6b8Matthew Wilcox int target = i % rounddown_pow_of_two(dev->queue_count - 1); 14829ecdc946212f7cd592986b2c519b470404caa6b8Matthew Wilcox dev->queues[i + 1] = dev->queues[target + 1]; 14839ecdc946212f7cd592986b2c519b470404caa6b8Matthew Wilcox } 14849ecdc946212f7cd592986b2c519b470404caa6b8Matthew Wilcox 1485b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 1486b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1487b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1488b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void nvme_free_queues(struct nvme_dev *dev) 1489b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1490b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int i; 1491b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1492b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox for (i = dev->queue_count - 1; i >= 0; i--) 1493b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_free_queue(dev, i); 1494b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1495b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1496b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int __devinit nvme_dev_add(struct nvme_dev *dev) 1497b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1498b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int res, nn, i; 1499b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_ns *ns, *next; 150051814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox struct nvme_id_ctrl *ctrl; 1501bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox struct nvme_id_ns *id_ns; 1502bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox void *mem; 1503b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_addr_t dma_addr; 1504b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1505b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox res = nvme_setup_io_queues(dev); 1506b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (res) 1507b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return res; 1508b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1509bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox mem = dma_alloc_coherent(&dev->pci_dev->dev, 8192, &dma_addr, 1510b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox GFP_KERNEL); 1511b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1512bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox res = nvme_identify(dev, 0, 1, dma_addr); 1513b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (res) { 1514b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox res = -EIO; 1515b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto out_free; 1516b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 1517b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1518bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox ctrl = mem; 151951814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox nn = le32_to_cpup(&ctrl->nn); 152051814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox memcpy(dev->serial, ctrl->sn, sizeof(ctrl->sn)); 152151814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox memcpy(dev->model, ctrl->mn, sizeof(ctrl->mn)); 152251814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox memcpy(dev->firmware_rev, ctrl->fr, sizeof(ctrl->fr)); 15238fc23e032debd682f5ba9fc524a5846c10d2c522Keith Busch if (ctrl->mdts) { 15248fc23e032debd682f5ba9fc524a5846c10d2c522Keith Busch int shift = NVME_CAP_MPSMIN(readq(&dev->bar->cap)) + 12; 15258fc23e032debd682f5ba9fc524a5846c10d2c522Keith Busch dev->max_hw_sectors = 1 << (ctrl->mdts + shift - 9); 15268fc23e032debd682f5ba9fc524a5846c10d2c522Keith Busch } 1527b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1528bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox id_ns = mem; 15292b2c1896871838cdf549442e8ad0264be5fa74e3Matthew Wilcox for (i = 1; i <= nn; i++) { 1530bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox res = nvme_identify(dev, i, 0, dma_addr); 1531b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (res) 1532b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox continue; 1533b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1534bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox if (id_ns->ncap == 0) 1535b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox continue; 1536b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1537bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox res = nvme_get_features(dev, NVME_FEAT_LBA_RANGE, i, 153808df1e05657fc6712e520e7c09cc6c86160ceb35Keith Busch dma_addr + 4096, NULL); 1539b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (res) 1540b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox continue; 1541b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1542bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox ns = nvme_alloc_ns(dev, i, mem, mem + 4096); 1543b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (ns) 1544b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox list_add_tail(&ns->list, &dev->namespaces); 1545b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 1546b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox list_for_each_entry(ns, &dev->namespaces, list) 1547b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox add_disk(ns->disk); 1548b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1549bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox goto out; 1550b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1551b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox out_free: 1552b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox list_for_each_entry_safe(ns, next, &dev->namespaces, list) { 1553b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox list_del(&ns->list); 1554b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_ns_free(ns); 1555b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 1556b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1557bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox out: 1558684f5c2025b067a23722e620d0b3b858d8dc5d01Matthew Wilcox dma_free_coherent(&dev->pci_dev->dev, 8192, mem, dma_addr); 1559b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return res; 1560b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1561b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1562b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_dev_remove(struct nvme_dev *dev) 1563b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1564b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_ns *ns, *next; 1565b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 15661fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_lock(&dev_list_lock); 15671fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox list_del(&dev->node); 15681fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_unlock(&dev_list_lock); 15691fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 1570b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox list_for_each_entry_safe(ns, next, &dev->namespaces, list) { 1571b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox list_del(&ns->list); 1572b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox del_gendisk(ns->disk); 1573b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_ns_free(ns); 1574b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 1575b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1576b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_free_queues(dev); 1577b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1578b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 1579b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1580b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1581091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcoxstatic int nvme_setup_prp_pools(struct nvme_dev *dev) 1582091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox{ 1583091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox struct device *dmadev = &dev->pci_dev->dev; 1584091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox dev->prp_page_pool = dma_pool_create("prp list page", dmadev, 1585091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox PAGE_SIZE, PAGE_SIZE, 0); 1586091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox if (!dev->prp_page_pool) 1587091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox return -ENOMEM; 1588091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox 158999802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox /* Optimisation for I/Os between 4k and 128k */ 159099802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox dev->prp_small_pool = dma_pool_create("prp list 256", dmadev, 159199802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox 256, 256, 0); 159299802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox if (!dev->prp_small_pool) { 159399802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox dma_pool_destroy(dev->prp_page_pool); 159499802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox return -ENOMEM; 159599802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox } 1596091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox return 0; 1597091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox} 1598091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox 1599091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcoxstatic void nvme_release_prp_pools(struct nvme_dev *dev) 1600091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox{ 1601091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox dma_pool_destroy(dev->prp_page_pool); 160299802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox dma_pool_destroy(dev->prp_small_pool); 1603091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox} 1604091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox 1605cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anhstatic DEFINE_IDA(nvme_instance_ida); 1606cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh 1607cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anhstatic int nvme_set_instance(struct nvme_dev *dev) 1608b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1609cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh int instance, error; 1610cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh 1611cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh do { 1612cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh if (!ida_pre_get(&nvme_instance_ida, GFP_KERNEL)) 1613cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh return -ENODEV; 1614cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh 1615cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh spin_lock(&dev_list_lock); 1616cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh error = ida_get_new(&nvme_instance_ida, &instance); 1617cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh spin_unlock(&dev_list_lock); 1618cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh } while (error == -EAGAIN); 1619cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh 1620cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh if (error) 1621cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh return -ENODEV; 1622cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh 1623cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh dev->instance = instance; 1624cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh return 0; 1625b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1626b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1627b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void nvme_release_instance(struct nvme_dev *dev) 1628b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1629cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh spin_lock(&dev_list_lock); 1630cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh ida_remove(&nvme_instance_ida, dev->instance); 1631cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh spin_unlock(&dev_list_lock); 1632b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1633b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1634b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int __devinit nvme_probe(struct pci_dev *pdev, 1635b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox const struct pci_device_id *id) 1636b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1637574e8b95bc3780e10e9b5e9d51074d503dd3d5d9Matthew Wilcox int bars, result = -ENOMEM; 1638b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_dev *dev; 1639b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1640b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev = kzalloc(sizeof(*dev), GFP_KERNEL); 1641b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!dev) 1642b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -ENOMEM; 1643b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->entry = kcalloc(num_possible_cpus(), sizeof(*dev->entry), 1644b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox GFP_KERNEL); 1645b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!dev->entry) 1646b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto free; 16471b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox dev->queues = kcalloc(num_possible_cpus() + 1, sizeof(void *), 16481b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox GFP_KERNEL); 1649b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!dev->queues) 1650b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto free; 1651b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 16520ee5a7d7cb9309bd393a25c395f19fb12a842602Shane Michael Matthews if (pci_enable_device_mem(pdev)) 16530ee5a7d7cb9309bd393a25c395f19fb12a842602Shane Michael Matthews goto free; 1654f64d3365a3e5cb46e69db7e2c82a7cb9a5bed1b8Matthew Wilcox pci_set_master(pdev); 1655574e8b95bc3780e10e9b5e9d51074d503dd3d5d9Matthew Wilcox bars = pci_select_bars(pdev, IORESOURCE_MEM); 1656574e8b95bc3780e10e9b5e9d51074d503dd3d5d9Matthew Wilcox if (pci_request_selected_regions(pdev, bars, "nvme")) 1657574e8b95bc3780e10e9b5e9d51074d503dd3d5d9Matthew Wilcox goto disable; 16580ee5a7d7cb9309bd393a25c395f19fb12a842602Shane Michael Matthews 1659b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox INIT_LIST_HEAD(&dev->namespaces); 1660b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->pci_dev = pdev; 1661b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox pci_set_drvdata(pdev, dev); 16622930353f9f2b9e4629e935acd970cb73c1171229Matthew Wilcox dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)); 16632930353f9f2b9e4629e935acd970cb73c1171229Matthew Wilcox dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64)); 1664cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh result = nvme_set_instance(dev); 1665cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh if (result) 1666cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh goto disable; 1667cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh 166853c9577e9ca68a633c6e9df2b54eaecacfa77f62Matthew Wilcox dev->entry[0].vector = pdev->irq; 1669b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1670091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox result = nvme_setup_prp_pools(dev); 1671091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox if (result) 1672091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox goto disable_msix; 1673091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox 1674b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->bar = ioremap(pci_resource_start(pdev, 0), 8192); 1675b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!dev->bar) { 1676b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox result = -ENOMEM; 1677574e8b95bc3780e10e9b5e9d51074d503dd3d5d9Matthew Wilcox goto disable_msix; 1678b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 1679b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1680b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox result = nvme_configure_admin_queue(dev); 1681b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (result) 1682b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto unmap; 1683b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->queue_count++; 1684b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 16851fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_lock(&dev_list_lock); 16861fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox list_add(&dev->node, &dev_list); 16871fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_unlock(&dev_list_lock); 16881fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 1689740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox result = nvme_dev_add(dev); 1690740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox if (result) 1691740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox goto delete; 1692740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox 1693b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 1694b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1695b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox delete: 1696740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox spin_lock(&dev_list_lock); 1697740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox list_del(&dev->node); 1698740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox spin_unlock(&dev_list_lock); 1699740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox 1700b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_free_queues(dev); 1701b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unmap: 1702b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox iounmap(dev->bar); 1703574e8b95bc3780e10e9b5e9d51074d503dd3d5d9Matthew Wilcox disable_msix: 1704b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox pci_disable_msix(pdev); 1705b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_release_instance(dev); 1706091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox nvme_release_prp_pools(dev); 1707574e8b95bc3780e10e9b5e9d51074d503dd3d5d9Matthew Wilcox disable: 17080ee5a7d7cb9309bd393a25c395f19fb12a842602Shane Michael Matthews pci_disable_device(pdev); 1709574e8b95bc3780e10e9b5e9d51074d503dd3d5d9Matthew Wilcox pci_release_regions(pdev); 1710b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox free: 1711b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(dev->queues); 1712b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(dev->entry); 1713b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(dev); 1714b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return result; 1715b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1716b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1717b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void __devexit nvme_remove(struct pci_dev *pdev) 1718b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1719b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_dev *dev = pci_get_drvdata(pdev); 1720b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_dev_remove(dev); 1721b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox pci_disable_msix(pdev); 1722b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox iounmap(dev->bar); 1723b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_release_instance(dev); 1724091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox nvme_release_prp_pools(dev); 17250ee5a7d7cb9309bd393a25c395f19fb12a842602Shane Michael Matthews pci_disable_device(pdev); 1726574e8b95bc3780e10e9b5e9d51074d503dd3d5d9Matthew Wilcox pci_release_regions(pdev); 1727b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(dev->queues); 1728b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(dev->entry); 1729b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(dev); 1730b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1731b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1732b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* These functions are yet to be implemented */ 1733b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_error_detected NULL 1734b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_dump_registers NULL 1735b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_link_reset NULL 1736b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_slot_reset NULL 1737b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_error_resume NULL 1738b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_suspend NULL 1739b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_resume NULL 1740b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 17411d3520357df99baf4ad89f86268ac96cd38092d9Stephen Hemmingerstatic const struct pci_error_handlers nvme_err_handler = { 1742b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .error_detected = nvme_error_detected, 1743b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .mmio_enabled = nvme_dump_registers, 1744b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .link_reset = nvme_link_reset, 1745b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .slot_reset = nvme_slot_reset, 1746b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .resume = nvme_error_resume, 1747b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 1748b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1749b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* Move to pci_ids.h later */ 1750b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define PCI_CLASS_STORAGE_EXPRESS 0x010802 1751b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1752b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic DEFINE_PCI_DEVICE_TABLE(nvme_id_table) = { 1753b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox { PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) }, 1754b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox { 0, } 1755b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 1756b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew WilcoxMODULE_DEVICE_TABLE(pci, nvme_id_table); 1757b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1758b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic struct pci_driver nvme_driver = { 1759b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .name = "nvme", 1760b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .id_table = nvme_id_table, 1761b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .probe = nvme_probe, 1762b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .remove = __devexit_p(nvme_remove), 1763b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .suspend = nvme_suspend, 1764b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .resume = nvme_resume, 1765b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .err_handler = &nvme_err_handler, 1766b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 1767b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1768b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int __init nvme_init(void) 1769b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 17700ac13140d796eb1e2f8956aea97a6e5e4ebcf981Matthew Wilcox int result; 17711fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 17721fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox nvme_thread = kthread_run(nvme_kthread, NULL, "nvme"); 17731fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox if (IS_ERR(nvme_thread)) 17741fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox return PTR_ERR(nvme_thread); 1775b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 17765c42ea1643a630060f9e71e06d3933d244970967Keith Busch result = register_blkdev(nvme_major, "nvme"); 17775c42ea1643a630060f9e71e06d3933d244970967Keith Busch if (result < 0) 17781fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox goto kill_kthread; 17795c42ea1643a630060f9e71e06d3933d244970967Keith Busch else if (result > 0) 17800ac13140d796eb1e2f8956aea97a6e5e4ebcf981Matthew Wilcox nvme_major = result; 1781b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1782b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox result = pci_register_driver(&nvme_driver); 17831fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox if (result) 17841fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox goto unregister_blkdev; 17851fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox return 0; 1786b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 17871fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox unregister_blkdev: 1788b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unregister_blkdev(nvme_major, "nvme"); 17891fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox kill_kthread: 17901fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox kthread_stop(nvme_thread); 1791b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return result; 1792b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1793b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1794b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void __exit nvme_exit(void) 1795b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1796b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox pci_unregister_driver(&nvme_driver); 1797b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unregister_blkdev(nvme_major, "nvme"); 17981fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox kthread_stop(nvme_thread); 1799b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1800b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1801b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew WilcoxMODULE_AUTHOR("Matthew Wilcox <willy@linux.intel.com>"); 1802b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew WilcoxMODULE_LICENSE("GPL"); 1803366e8217e5ec6ce9f73aec19c46d983110fb4a98Matthew WilcoxMODULE_VERSION("0.8"); 1804b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxmodule_init(nvme_init); 1805b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxmodule_exit(nvme_exit); 1806