nvme-core.c revision f410c680b5344f1cb63ff011c6ae3d4963f45c30
1b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 2b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * NVM Express device driver 3b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Copyright (c) 2011, Intel Corporation. 4b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 5b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * This program is free software; you can redistribute it and/or modify it 6b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * under the terms and conditions of the GNU General Public License, 7b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * version 2, as published by the Free Software Foundation. 8b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 9b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * This program is distributed in the hope it will be useful, but WITHOUT 10b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * more details. 13b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 14b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * You should have received a copy of the GNU General Public License along with 15b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * this program; if not, write to the Free Software Foundation, Inc., 16b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. 17b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 18b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 19b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/nvme.h> 20b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/bio.h> 218de055350fbaa96b6563892c195a60be583faa9cMatthew Wilcox#include <linux/bitops.h> 22b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/blkdev.h> 23fd63e9ceeeae58cfe877c2d49d41c1bf7532303cMatthew Wilcox#include <linux/delay.h> 24b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/errno.h> 25b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/fs.h> 26b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/genhd.h> 275aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox#include <linux/idr.h> 28b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/init.h> 29b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/interrupt.h> 30b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/io.h> 31b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/kdev_t.h> 321fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox#include <linux/kthread.h> 33b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/kernel.h> 34b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/mm.h> 35b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/module.h> 36b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/moduleparam.h> 37b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/pci.h> 38be7b62754e097adc0cb16c25c9ee86ee20de62fbMatthew Wilcox#include <linux/poison.h> 39b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/sched.h> 40b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/slab.h> 41b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/types.h> 425d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma#include <scsi/sg.h> 43797a796a13df6b84a4791e57306737059b5b2384Hitoshi Mitake#include <asm-generic/io-64-nonatomic-lo-hi.h> 44797a796a13df6b84a4791e57306737059b5b2384Hitoshi Mitake 45b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define NVME_Q_DEPTH 1024 46b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define SQ_SIZE(depth) (depth * sizeof(struct nvme_command)) 47b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define CQ_SIZE(depth) (depth * sizeof(struct nvme_completion)) 48b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define NVME_MINORS 64 49e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox#define ADMIN_TIMEOUT (60 * HZ) 50b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 51b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_major; 52b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxmodule_param(nvme_major, int, 0); 53b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 5458ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcoxstatic int use_threaded_interrupts; 5558ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcoxmodule_param(use_threaded_interrupts, int, 0); 5658ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox 571fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcoxstatic DEFINE_SPINLOCK(dev_list_lock); 581fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcoxstatic LIST_HEAD(dev_list); 591fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcoxstatic struct task_struct *nvme_thread; 601fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 61b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 62b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * An NVM Express queue. Each device has at least two (one for admin 63b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * commands and one for I/O commands). 64b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 65b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstruct nvme_queue { 66b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct device *q_dmadev; 67091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox struct nvme_dev *dev; 68b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox spinlock_t q_lock; 69b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_command *sq_cmds; 70b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox volatile struct nvme_completion *cqes; 71b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_addr_t sq_dma_addr; 72b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_addr_t cq_dma_addr; 73b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox wait_queue_head_t sq_full; 741fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox wait_queue_t sq_cong_wait; 75b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct bio_list sq_cong; 76b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 __iomem *q_db; 77b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 q_depth; 78b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 cq_vector; 79b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 sq_head; 80b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 sq_tail; 81b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 cq_head; 82821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox u16 cq_phase; 83b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unsigned long cmdid_data[]; 84b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 85b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 86b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 87b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Check we didin't inadvertently grow the command struct 88b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 89b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic inline void _nvme_check_size(void) 90b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 91b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64); 92b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_create_cq) != 64); 93b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_create_sq) != 64); 94b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64); 95b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_features) != 64); 96f8ebf8409abfdaeeb8c847381629a2a8b8e3d816Vishal Verma BUILD_BUG_ON(sizeof(struct nvme_format_cmd) != 64); 97b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_command) != 64); 98b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != 4096); 99b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_id_ns) != 4096); 100b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64); 1016ecec74520d8a357726e6c12f99080dbe7b347ddKeith Busch BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512); 102b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 103b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1045c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcoxtypedef void (*nvme_completion_fn)(struct nvme_dev *, void *, 105c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox struct nvme_completion *); 106c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox 107e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcoxstruct nvme_cmd_info { 108c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox nvme_completion_fn fn; 109c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox void *ctx; 110e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox unsigned long timeout; 111e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox}; 112e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox 113e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcoxstatic struct nvme_cmd_info *nvme_cmd_info(struct nvme_queue *nvmeq) 114e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox{ 115e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox return (void *)&nvmeq->cmdid_data[BITS_TO_LONGS(nvmeq->q_depth)]; 116e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox} 117e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox 118b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/** 119714a7a22884b74862540bc84955274d86b2f6040Matthew Wilcox * alloc_cmdid() - Allocate a Command ID 120714a7a22884b74862540bc84955274d86b2f6040Matthew Wilcox * @nvmeq: The queue that will be used for this command 121714a7a22884b74862540bc84955274d86b2f6040Matthew Wilcox * @ctx: A pointer that will be passed to the handler 122c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox * @handler: The function to call on completion 123b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 124b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Allocate a Command ID for a queue. The data passed in will 125b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * be passed to the completion handler. This is implemented by using 126b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * the bottom two bits of the ctx pointer to store the handler ID. 127b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Passing in a pointer that's not 4-byte aligned will cause a BUG. 128b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * We can change this if it becomes a problem. 129184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox * 130184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox * May be called with local interrupts disabled and the q_lock held, 131184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox * or with interrupts enabled and no locks held. 132b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 133c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcoxstatic int alloc_cmdid(struct nvme_queue *nvmeq, void *ctx, 134c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox nvme_completion_fn handler, unsigned timeout) 135b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 136e6d15f79f997a98b3a69abbc462fc9041cc1a7b4Matthew Wilcox int depth = nvmeq->q_depth - 1; 137e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox struct nvme_cmd_info *info = nvme_cmd_info(nvmeq); 138b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int cmdid; 139b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 140b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox do { 141b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cmdid = find_first_zero_bit(nvmeq->cmdid_data, depth); 142b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (cmdid >= depth) 143b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -EBUSY; 144b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } while (test_and_set_bit(cmdid, nvmeq->cmdid_data)); 145b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 146c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox info[cmdid].fn = handler; 147c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox info[cmdid].ctx = ctx; 148e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox info[cmdid].timeout = jiffies + timeout; 149b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return cmdid; 150b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 151b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 152b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int alloc_cmdid_killable(struct nvme_queue *nvmeq, void *ctx, 153c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox nvme_completion_fn handler, unsigned timeout) 154b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 155b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int cmdid; 156b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox wait_event_killable(nvmeq->sq_full, 157e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox (cmdid = alloc_cmdid(nvmeq, ctx, handler, timeout)) >= 0); 158b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return (cmdid < 0) ? -EINTR : cmdid; 159b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 160b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 161c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox/* Special values must be less than 0x1000 */ 162c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox#define CMD_CTX_BASE ((void *)POISON_POINTER_DELTA) 163d2d8703481f60d67f49e3177196cbe474b11377cMatthew Wilcox#define CMD_CTX_CANCELLED (0x30C + CMD_CTX_BASE) 164d2d8703481f60d67f49e3177196cbe474b11377cMatthew Wilcox#define CMD_CTX_COMPLETED (0x310 + CMD_CTX_BASE) 165d2d8703481f60d67f49e3177196cbe474b11377cMatthew Wilcox#define CMD_CTX_INVALID (0x314 + CMD_CTX_BASE) 16600df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox#define CMD_CTX_FLUSH (0x318 + CMD_CTX_BASE) 167be7b62754e097adc0cb16c25c9ee86ee20de62fbMatthew Wilcox 1685c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcoxstatic void special_completion(struct nvme_dev *dev, void *ctx, 169c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox struct nvme_completion *cqe) 170c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox{ 171c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox if (ctx == CMD_CTX_CANCELLED) 172c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox return; 173c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox if (ctx == CMD_CTX_FLUSH) 174c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox return; 175c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox if (ctx == CMD_CTX_COMPLETED) { 1765c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcox dev_warn(&dev->pci_dev->dev, 177c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox "completed id %d twice on queue %d\n", 178c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox cqe->command_id, le16_to_cpup(&cqe->sq_id)); 179c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox return; 180c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox } 181c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox if (ctx == CMD_CTX_INVALID) { 1825c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcox dev_warn(&dev->pci_dev->dev, 183c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox "invalid id %d completed on queue %d\n", 184c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox cqe->command_id, le16_to_cpup(&cqe->sq_id)); 185c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox return; 186c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox } 187c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox 1885c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcox dev_warn(&dev->pci_dev->dev, "Unknown special completion %p\n", ctx); 189c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox} 190c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox 191184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox/* 192184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox * Called with local interrupts disabled and the q_lock held. May not sleep. 193184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox */ 194c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcoxstatic void *free_cmdid(struct nvme_queue *nvmeq, int cmdid, 195c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox nvme_completion_fn *fn) 196b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 197c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox void *ctx; 198e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox struct nvme_cmd_info *info = nvme_cmd_info(nvmeq); 199b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 200c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox if (cmdid >= nvmeq->q_depth) { 201c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox *fn = special_completion; 20248e3d39816416b3bf03dee3a796c0c04427c1a31Matthew Wilcox return CMD_CTX_INVALID; 203c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox } 204859361a228258edf4821d9f5635825033eca78e8Keith Busch if (fn) 205859361a228258edf4821d9f5635825033eca78e8Keith Busch *fn = info[cmdid].fn; 206c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox ctx = info[cmdid].ctx; 207c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox info[cmdid].fn = special_completion; 208e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox info[cmdid].ctx = CMD_CTX_COMPLETED; 209b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox clear_bit(cmdid, nvmeq->cmdid_data); 210b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox wake_up(&nvmeq->sq_full); 211c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox return ctx; 212b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 213b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 214c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcoxstatic void *cancel_cmdid(struct nvme_queue *nvmeq, int cmdid, 215c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox nvme_completion_fn *fn) 2163c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox{ 217c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox void *ctx; 218e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox struct nvme_cmd_info *info = nvme_cmd_info(nvmeq); 219c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox if (fn) 220c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox *fn = info[cmdid].fn; 221c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox ctx = info[cmdid].ctx; 222c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox info[cmdid].fn = special_completion; 223e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox info[cmdid].ctx = CMD_CTX_CANCELLED; 224c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox return ctx; 2253c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox} 2263c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox 2275d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermastruct nvme_queue *get_nvmeq(struct nvme_dev *dev) 228b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 229040a93b52a9eee8177ebaf2ba0ee0f9f518d1bf8Matthew Wilcox return dev->queues[get_cpu() + 1]; 230b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 231b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2325d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermavoid put_nvmeq(struct nvme_queue *nvmeq) 233b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 2341b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox put_cpu(); 235b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 236b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 237b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/** 238714a7a22884b74862540bc84955274d86b2f6040Matthew Wilcox * nvme_submit_cmd() - Copy a command into a queue and ring the doorbell 239b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * @nvmeq: The queue to use 240b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * @cmd: The command to send 241b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 242b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Safe to use from interrupt context 243b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 244b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_submit_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd) 245b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 246b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unsigned long flags; 247b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 tail; 248b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox spin_lock_irqsave(&nvmeq->q_lock, flags); 249b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox tail = nvmeq->sq_tail; 250b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memcpy(&nvmeq->sq_cmds[tail], cmd, sizeof(*cmd)); 251b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (++tail == nvmeq->q_depth) 252b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox tail = 0; 2537547881d0951384f9833ec3a80fac8f3f16f3b98Matthew Wilcox writel(tail, nvmeq->q_db); 254b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->sq_tail = tail; 255b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox spin_unlock_irqrestore(&nvmeq->q_lock, flags); 256b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 257b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 258b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 259b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 260eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcoxstatic __le64 **iod_list(struct nvme_iod *iod) 261e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews{ 262eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return ((void *)iod) + iod->offset; 263e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews} 264e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews 265eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox/* 266eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox * Will slightly overestimate the number of pages needed. This is OK 267eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox * as it only leads to a small amount of wasted memory for the lifetime of 268eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox * the I/O. 269eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox */ 270eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcoxstatic int nvme_npages(unsigned size) 271eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox{ 272eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox unsigned nprps = DIV_ROUND_UP(size + PAGE_SIZE, PAGE_SIZE); 273eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return DIV_ROUND_UP(8 * nprps, PAGE_SIZE - 8); 274eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox} 275b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 276eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcoxstatic struct nvme_iod * 277eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcoxnvme_alloc_iod(unsigned nseg, unsigned nbytes, gfp_t gfp) 278b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 279eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox struct nvme_iod *iod = kmalloc(sizeof(struct nvme_iod) + 280eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox sizeof(__le64 *) * nvme_npages(nbytes) + 281eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox sizeof(struct scatterlist) * nseg, gfp); 282eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox 283eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox if (iod) { 284eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->offset = offsetof(struct nvme_iod, sg[nseg]); 285eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->npages = -1; 286eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->length = nbytes; 2872b1960341576bf51c01b12fefeb1cc53820923e7Keith Busch iod->nents = 0; 288eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox } 289eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox 290eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return iod; 291b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 292b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2935d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermavoid nvme_free_iod(struct nvme_dev *dev, struct nvme_iod *iod) 294b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 295eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox const int last_prp = PAGE_SIZE / 8 - 1; 296eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox int i; 297eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox __le64 **list = iod_list(iod); 298eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox dma_addr_t prp_dma = iod->first_dma; 299eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox 300eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox if (iod->npages == 0) 301eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox dma_pool_free(dev->prp_small_pool, list[0], prp_dma); 302eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox for (i = 0; i < iod->npages; i++) { 303eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox __le64 *prp_list = list[i]; 304eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox dma_addr_t next_prp_dma = le64_to_cpu(prp_list[last_prp]); 305eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox dma_pool_free(dev->prp_page_pool, prp_list, prp_dma); 306eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox prp_dma = next_prp_dma; 307eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox } 308eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox kfree(iod); 309b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 310b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 3115c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcoxstatic void bio_completion(struct nvme_dev *dev, void *ctx, 312b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_completion *cqe) 313b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 314eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox struct nvme_iod *iod = ctx; 315eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox struct bio *bio = iod->private; 316b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 status = le16_to_cpup(&cqe->status) >> 1; 317b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 3182b1960341576bf51c01b12fefeb1cc53820923e7Keith Busch if (iod->nents) 3192b1960341576bf51c01b12fefeb1cc53820923e7Keith Busch dma_unmap_sg(&dev->pci_dev->dev, iod->sg, iod->nents, 320b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox bio_data_dir(bio) ? DMA_TO_DEVICE : DMA_FROM_DEVICE); 321eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox nvme_free_iod(dev, iod); 322427e97080196548557b288517537ab7eb48c309fKeith Busch if (status) 3231ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox bio_endio(bio, -EIO); 324427e97080196548557b288517537ab7eb48c309fKeith Busch else 3251ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox bio_endio(bio, 0); 326b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 327b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 328184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox/* length is in bytes. gfp flags indicates whether we may sleep. */ 3295d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermaint nvme_setup_prps(struct nvme_dev *dev, struct nvme_common_command *cmd, 3305d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma struct nvme_iod *iod, int total_len, gfp_t gfp) 331ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox{ 33299802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox struct dma_pool *pool; 333eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox int length = total_len; 334eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox struct scatterlist *sg = iod->sg; 335ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox int dma_len = sg_dma_len(sg); 336ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox u64 dma_addr = sg_dma_address(sg); 337ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox int offset = offset_in_page(dma_addr); 338e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews __le64 *prp_list; 339eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox __le64 **list = iod_list(iod); 340e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_addr_t prp_dma; 341eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox int nprps, i; 342ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox 343ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmd->prp1 = cpu_to_le64(dma_addr); 344ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox length -= (PAGE_SIZE - offset); 345ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox if (length <= 0) 346eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return total_len; 347ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox 348ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox dma_len -= (PAGE_SIZE - offset); 349ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox if (dma_len) { 350ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox dma_addr += (PAGE_SIZE - offset); 351ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox } else { 352ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox sg = sg_next(sg); 353ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox dma_addr = sg_dma_address(sg); 354ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox dma_len = sg_dma_len(sg); 355ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox } 356ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox 357ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox if (length <= PAGE_SIZE) { 358ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmd->prp2 = cpu_to_le64(dma_addr); 359eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return total_len; 360e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews } 361e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews 362e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews nprps = DIV_ROUND_UP(length, PAGE_SIZE); 36399802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox if (nprps <= (256 / 8)) { 36499802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox pool = dev->prp_small_pool; 365eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->npages = 0; 36699802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox } else { 36799802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox pool = dev->prp_page_pool; 368eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->npages = 1; 36999802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox } 37099802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox 371b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox prp_list = dma_pool_alloc(pool, gfp, &prp_dma); 372b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox if (!prp_list) { 373b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox cmd->prp2 = cpu_to_le64(dma_addr); 374eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->npages = -1; 375eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return (total_len - length) + PAGE_SIZE; 376b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox } 377eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox list[0] = prp_list; 378eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->first_dma = prp_dma; 379e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews cmd->prp2 = cpu_to_le64(prp_dma); 380e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews i = 0; 381e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews for (;;) { 3827523d834dd1573610078eb1ac0933f6490232f90Matthew Wilcox if (i == PAGE_SIZE / 8) { 383e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews __le64 *old_prp_list = prp_list; 384b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox prp_list = dma_pool_alloc(pool, gfp, &prp_dma); 385eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox if (!prp_list) 386eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return total_len - length; 387eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox list[iod->npages++] = prp_list; 3887523d834dd1573610078eb1ac0933f6490232f90Matthew Wilcox prp_list[0] = old_prp_list[i - 1]; 3897523d834dd1573610078eb1ac0933f6490232f90Matthew Wilcox old_prp_list[i - 1] = cpu_to_le64(prp_dma); 3907523d834dd1573610078eb1ac0933f6490232f90Matthew Wilcox i = 1; 391e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews } 392e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews prp_list[i++] = cpu_to_le64(dma_addr); 393e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_len -= PAGE_SIZE; 394e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_addr += PAGE_SIZE; 395e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews length -= PAGE_SIZE; 396e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews if (length <= 0) 397e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews break; 398e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews if (dma_len > 0) 399e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews continue; 400e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews BUG_ON(dma_len < 0); 401e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews sg = sg_next(sg); 402e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_addr = sg_dma_address(sg); 403e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews dma_len = sg_dma_len(sg); 404ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox } 405ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox 406eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return total_len; 407ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox} 408ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox 409427e97080196548557b288517537ab7eb48c309fKeith Buschstruct nvme_bio_pair { 410427e97080196548557b288517537ab7eb48c309fKeith Busch struct bio b1, b2, *parent; 411427e97080196548557b288517537ab7eb48c309fKeith Busch struct bio_vec *bv1, *bv2; 412427e97080196548557b288517537ab7eb48c309fKeith Busch int err; 413427e97080196548557b288517537ab7eb48c309fKeith Busch atomic_t cnt; 414427e97080196548557b288517537ab7eb48c309fKeith Busch}; 415427e97080196548557b288517537ab7eb48c309fKeith Busch 416427e97080196548557b288517537ab7eb48c309fKeith Buschstatic void nvme_bio_pair_endio(struct bio *bio, int err) 417427e97080196548557b288517537ab7eb48c309fKeith Busch{ 418427e97080196548557b288517537ab7eb48c309fKeith Busch struct nvme_bio_pair *bp = bio->bi_private; 419427e97080196548557b288517537ab7eb48c309fKeith Busch 420427e97080196548557b288517537ab7eb48c309fKeith Busch if (err) 421427e97080196548557b288517537ab7eb48c309fKeith Busch bp->err = err; 422427e97080196548557b288517537ab7eb48c309fKeith Busch 423427e97080196548557b288517537ab7eb48c309fKeith Busch if (atomic_dec_and_test(&bp->cnt)) { 424427e97080196548557b288517537ab7eb48c309fKeith Busch bio_endio(bp->parent, bp->err); 425427e97080196548557b288517537ab7eb48c309fKeith Busch if (bp->bv1) 426427e97080196548557b288517537ab7eb48c309fKeith Busch kfree(bp->bv1); 427427e97080196548557b288517537ab7eb48c309fKeith Busch if (bp->bv2) 428427e97080196548557b288517537ab7eb48c309fKeith Busch kfree(bp->bv2); 429427e97080196548557b288517537ab7eb48c309fKeith Busch kfree(bp); 430427e97080196548557b288517537ab7eb48c309fKeith Busch } 431427e97080196548557b288517537ab7eb48c309fKeith Busch} 432427e97080196548557b288517537ab7eb48c309fKeith Busch 433427e97080196548557b288517537ab7eb48c309fKeith Buschstatic struct nvme_bio_pair *nvme_bio_split(struct bio *bio, int idx, 434427e97080196548557b288517537ab7eb48c309fKeith Busch int len, int offset) 435427e97080196548557b288517537ab7eb48c309fKeith Busch{ 436427e97080196548557b288517537ab7eb48c309fKeith Busch struct nvme_bio_pair *bp; 437427e97080196548557b288517537ab7eb48c309fKeith Busch 438427e97080196548557b288517537ab7eb48c309fKeith Busch BUG_ON(len > bio->bi_size); 439427e97080196548557b288517537ab7eb48c309fKeith Busch BUG_ON(idx > bio->bi_vcnt); 440427e97080196548557b288517537ab7eb48c309fKeith Busch 441427e97080196548557b288517537ab7eb48c309fKeith Busch bp = kmalloc(sizeof(*bp), GFP_ATOMIC); 442427e97080196548557b288517537ab7eb48c309fKeith Busch if (!bp) 443427e97080196548557b288517537ab7eb48c309fKeith Busch return NULL; 444427e97080196548557b288517537ab7eb48c309fKeith Busch bp->err = 0; 445427e97080196548557b288517537ab7eb48c309fKeith Busch 446427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b1 = *bio; 447427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b2 = *bio; 448427e97080196548557b288517537ab7eb48c309fKeith Busch 449427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b1.bi_size = len; 450427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b2.bi_size -= len; 451427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b1.bi_vcnt = idx; 452427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b2.bi_idx = idx; 453427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b2.bi_sector += len >> 9; 454427e97080196548557b288517537ab7eb48c309fKeith Busch 455427e97080196548557b288517537ab7eb48c309fKeith Busch if (offset) { 456427e97080196548557b288517537ab7eb48c309fKeith Busch bp->bv1 = kmalloc(bio->bi_max_vecs * sizeof(struct bio_vec), 457427e97080196548557b288517537ab7eb48c309fKeith Busch GFP_ATOMIC); 458427e97080196548557b288517537ab7eb48c309fKeith Busch if (!bp->bv1) 459427e97080196548557b288517537ab7eb48c309fKeith Busch goto split_fail_1; 460427e97080196548557b288517537ab7eb48c309fKeith Busch 461427e97080196548557b288517537ab7eb48c309fKeith Busch bp->bv2 = kmalloc(bio->bi_max_vecs * sizeof(struct bio_vec), 462427e97080196548557b288517537ab7eb48c309fKeith Busch GFP_ATOMIC); 463427e97080196548557b288517537ab7eb48c309fKeith Busch if (!bp->bv2) 464427e97080196548557b288517537ab7eb48c309fKeith Busch goto split_fail_2; 465427e97080196548557b288517537ab7eb48c309fKeith Busch 466427e97080196548557b288517537ab7eb48c309fKeith Busch memcpy(bp->bv1, bio->bi_io_vec, 467427e97080196548557b288517537ab7eb48c309fKeith Busch bio->bi_max_vecs * sizeof(struct bio_vec)); 468427e97080196548557b288517537ab7eb48c309fKeith Busch memcpy(bp->bv2, bio->bi_io_vec, 469427e97080196548557b288517537ab7eb48c309fKeith Busch bio->bi_max_vecs * sizeof(struct bio_vec)); 470427e97080196548557b288517537ab7eb48c309fKeith Busch 471427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b1.bi_io_vec = bp->bv1; 472427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b2.bi_io_vec = bp->bv2; 473427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b2.bi_io_vec[idx].bv_offset += offset; 474427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b2.bi_io_vec[idx].bv_len -= offset; 475427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b1.bi_io_vec[idx].bv_len = offset; 476427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b1.bi_vcnt++; 477427e97080196548557b288517537ab7eb48c309fKeith Busch } else 478427e97080196548557b288517537ab7eb48c309fKeith Busch bp->bv1 = bp->bv2 = NULL; 479427e97080196548557b288517537ab7eb48c309fKeith Busch 480427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b1.bi_private = bp; 481427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b2.bi_private = bp; 482427e97080196548557b288517537ab7eb48c309fKeith Busch 483427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b1.bi_end_io = nvme_bio_pair_endio; 484427e97080196548557b288517537ab7eb48c309fKeith Busch bp->b2.bi_end_io = nvme_bio_pair_endio; 485427e97080196548557b288517537ab7eb48c309fKeith Busch 486427e97080196548557b288517537ab7eb48c309fKeith Busch bp->parent = bio; 487427e97080196548557b288517537ab7eb48c309fKeith Busch atomic_set(&bp->cnt, 2); 488427e97080196548557b288517537ab7eb48c309fKeith Busch 489427e97080196548557b288517537ab7eb48c309fKeith Busch return bp; 490427e97080196548557b288517537ab7eb48c309fKeith Busch 491427e97080196548557b288517537ab7eb48c309fKeith Busch split_fail_2: 492427e97080196548557b288517537ab7eb48c309fKeith Busch kfree(bp->bv1); 493427e97080196548557b288517537ab7eb48c309fKeith Busch split_fail_1: 494427e97080196548557b288517537ab7eb48c309fKeith Busch kfree(bp); 495427e97080196548557b288517537ab7eb48c309fKeith Busch return NULL; 496427e97080196548557b288517537ab7eb48c309fKeith Busch} 497427e97080196548557b288517537ab7eb48c309fKeith Busch 498427e97080196548557b288517537ab7eb48c309fKeith Buschstatic int nvme_split_and_submit(struct bio *bio, struct nvme_queue *nvmeq, 499427e97080196548557b288517537ab7eb48c309fKeith Busch int idx, int len, int offset) 500427e97080196548557b288517537ab7eb48c309fKeith Busch{ 501427e97080196548557b288517537ab7eb48c309fKeith Busch struct nvme_bio_pair *bp = nvme_bio_split(bio, idx, len, offset); 502427e97080196548557b288517537ab7eb48c309fKeith Busch if (!bp) 503427e97080196548557b288517537ab7eb48c309fKeith Busch return -ENOMEM; 504427e97080196548557b288517537ab7eb48c309fKeith Busch 505427e97080196548557b288517537ab7eb48c309fKeith Busch if (bio_list_empty(&nvmeq->sq_cong)) 506427e97080196548557b288517537ab7eb48c309fKeith Busch add_wait_queue(&nvmeq->sq_full, &nvmeq->sq_cong_wait); 507427e97080196548557b288517537ab7eb48c309fKeith Busch bio_list_add(&nvmeq->sq_cong, &bp->b1); 508427e97080196548557b288517537ab7eb48c309fKeith Busch bio_list_add(&nvmeq->sq_cong, &bp->b2); 509427e97080196548557b288517537ab7eb48c309fKeith Busch 510427e97080196548557b288517537ab7eb48c309fKeith Busch return 0; 511427e97080196548557b288517537ab7eb48c309fKeith Busch} 512427e97080196548557b288517537ab7eb48c309fKeith Busch 5131ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox/* NVMe scatterlists require no holes in the virtual address */ 5141ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox#define BIOVEC_NOT_VIRT_MERGEABLE(vec1, vec2) ((vec2)->bv_offset || \ 5151ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox (((vec1)->bv_offset + (vec1)->bv_len) % PAGE_SIZE)) 5161ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox 517427e97080196548557b288517537ab7eb48c309fKeith Buschstatic int nvme_map_bio(struct nvme_queue *nvmeq, struct nvme_iod *iod, 518b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct bio *bio, enum dma_data_direction dma_dir, int psegs) 519b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 520768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox struct bio_vec *bvec, *bvprv = NULL; 521768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox struct scatterlist *sg = NULL; 522159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch int i, length = 0, nsegs = 0, split_len = bio->bi_size; 523159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch 524159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch if (nvmeq->dev->stripe_size) 525159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch split_len = nvmeq->dev->stripe_size - 526159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch ((bio->bi_sector << 9) & (nvmeq->dev->stripe_size - 1)); 527b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 528eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox sg_init_table(iod->sg, psegs); 529b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox bio_for_each_segment(bvec, bio, i) { 530768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox if (bvprv && BIOVEC_PHYS_MERGEABLE(bvprv, bvec)) { 531768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox sg->length += bvec->bv_len; 532768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox } else { 5331ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox if (bvprv && BIOVEC_NOT_VIRT_MERGEABLE(bvprv, bvec)) 534427e97080196548557b288517537ab7eb48c309fKeith Busch return nvme_split_and_submit(bio, nvmeq, i, 535427e97080196548557b288517537ab7eb48c309fKeith Busch length, 0); 536427e97080196548557b288517537ab7eb48c309fKeith Busch 537eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox sg = sg ? sg + 1 : iod->sg; 538768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox sg_set_page(sg, bvec->bv_page, bvec->bv_len, 539768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox bvec->bv_offset); 540768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox nsegs++; 541768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox } 542159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch 543159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch if (split_len - length < bvec->bv_len) 544159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch return nvme_split_and_submit(bio, nvmeq, i, split_len, 545159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch split_len - length); 5461ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox length += bvec->bv_len; 547768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox bvprv = bvec; 548b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 549eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->nents = nsegs; 550768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox sg_mark_end(sg); 551427e97080196548557b288517537ab7eb48c309fKeith Busch if (dma_map_sg(nvmeq->q_dmadev, iod->sg, iod->nents, dma_dir) == 0) 5521ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox return -ENOMEM; 553427e97080196548557b288517537ab7eb48c309fKeith Busch 554159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch BUG_ON(length != bio->bi_size); 5551ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox return length; 556b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 557b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 5580e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch/* 5590e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch * We reuse the small pool to allocate the 16-byte range here as it is not 5600e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch * worth having a special pool for these or additional cases to handle freeing 5610e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch * the iod. 5620e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch */ 5630e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Buschstatic int nvme_submit_discard(struct nvme_queue *nvmeq, struct nvme_ns *ns, 5640e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch struct bio *bio, struct nvme_iod *iod, int cmdid) 5650e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch{ 5660e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch struct nvme_dsm_range *range; 5670e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch struct nvme_command *cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail]; 5680e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch 5690e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch range = dma_pool_alloc(nvmeq->dev->prp_small_pool, GFP_ATOMIC, 5700e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch &iod->first_dma); 5710e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch if (!range) 5720e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch return -ENOMEM; 5730e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch 5740e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch iod_list(iod)[0] = (__le64 *)range; 5750e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch iod->npages = 0; 5760e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch 5770e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch range->cattr = cpu_to_le32(0); 5780e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch range->nlb = cpu_to_le32(bio->bi_size >> ns->lba_shift); 579063cc6d5591ea9c0631b81ac5c7b829d99738b2fMatthew Wilcox range->slba = cpu_to_le64(nvme_block_nr(ns, bio->bi_sector)); 5800e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch 5810e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch memset(cmnd, 0, sizeof(*cmnd)); 5820e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch cmnd->dsm.opcode = nvme_cmd_dsm; 5830e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch cmnd->dsm.command_id = cmdid; 5840e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch cmnd->dsm.nsid = cpu_to_le32(ns->ns_id); 5850e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch cmnd->dsm.prp1 = cpu_to_le64(iod->first_dma); 5860e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch cmnd->dsm.nr = 0; 5870e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD); 5880e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch 5890e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch if (++nvmeq->sq_tail == nvmeq->q_depth) 5900e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch nvmeq->sq_tail = 0; 5910e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch writel(nvmeq->sq_tail, nvmeq->q_db); 5920e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch 5930e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch return 0; 5940e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch} 5950e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch 59600df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcoxstatic int nvme_submit_flush(struct nvme_queue *nvmeq, struct nvme_ns *ns, 59700df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox int cmdid) 59800df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox{ 59900df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox struct nvme_command *cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail]; 60000df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 60100df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox memset(cmnd, 0, sizeof(*cmnd)); 60200df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox cmnd->common.opcode = nvme_cmd_flush; 60300df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox cmnd->common.command_id = cmdid; 60400df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox cmnd->common.nsid = cpu_to_le32(ns->ns_id); 60500df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 60600df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox if (++nvmeq->sq_tail == nvmeq->q_depth) 60700df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox nvmeq->sq_tail = 0; 60800df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox writel(nvmeq->sq_tail, nvmeq->q_db); 60900df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 61000df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox return 0; 61100df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox} 61200df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 6135d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermaint nvme_submit_flush_data(struct nvme_queue *nvmeq, struct nvme_ns *ns) 61400df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox{ 61500df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox int cmdid = alloc_cmdid(nvmeq, (void *)CMD_CTX_FLUSH, 616ff976d724a74e4522e9ca2de1fb37ac4520f454fMatthew Wilcox special_completion, NVME_IO_TIMEOUT); 61700df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox if (unlikely(cmdid < 0)) 61800df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox return cmdid; 61900df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 62000df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox return nvme_submit_flush(nvmeq, ns, cmdid); 62100df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox} 62200df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 623184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox/* 624184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox * Called with local interrupts disabled and the q_lock held. May not sleep. 625184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox */ 626b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_submit_bio_queue(struct nvme_queue *nvmeq, struct nvme_ns *ns, 627b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct bio *bio) 628b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 629ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox struct nvme_command *cmnd; 630eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox struct nvme_iod *iod; 631b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox enum dma_data_direction dma_dir; 6321ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox int cmdid, length, result = -ENOMEM; 633b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 control; 634b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 dsmgmt; 635b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int psegs = bio_phys_segments(ns->queue, bio); 636b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 63700df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox if ((bio->bi_rw & REQ_FLUSH) && psegs) { 63800df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox result = nvme_submit_flush_data(nvmeq, ns); 63900df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox if (result) 64000df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox return result; 64100df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox } 64200df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 643eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod = nvme_alloc_iod(psegs, bio->bi_size, GFP_ATOMIC); 644eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox if (!iod) 645eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox goto nomem; 646eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod->private = bio; 647b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 648eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox result = -EBUSY; 649ff976d724a74e4522e9ca2de1fb37ac4520f454fMatthew Wilcox cmdid = alloc_cmdid(nvmeq, iod, bio_completion, NVME_IO_TIMEOUT); 650b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (unlikely(cmdid < 0)) 651eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox goto free_iod; 652b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 6530e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch if (bio->bi_rw & REQ_DISCARD) { 6540e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch result = nvme_submit_discard(nvmeq, ns, bio, iod, cmdid); 6550e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch if (result) 6560e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch goto free_cmdid; 6570e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch return result; 6580e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch } 65900df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox if ((bio->bi_rw & REQ_FLUSH) && !psegs) 66000df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox return nvme_submit_flush(nvmeq, ns, cmdid); 66100df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox 662b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox control = 0; 663b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (bio->bi_rw & REQ_FUA) 664b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox control |= NVME_RW_FUA; 665b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (bio->bi_rw & (REQ_FAILFAST_DEV | REQ_RAHEAD)) 666b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox control |= NVME_RW_LR; 667b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 668b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dsmgmt = 0; 669b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (bio->bi_rw & REQ_RAHEAD) 670b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH; 671b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 672ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail]; 673b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 674b8deb62cf271fa9381edc8cf52bcae2f0225c55aMatthew Wilcox memset(cmnd, 0, sizeof(*cmnd)); 675b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (bio_data_dir(bio)) { 676ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.opcode = nvme_cmd_write; 677b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_dir = DMA_TO_DEVICE; 678b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } else { 679ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.opcode = nvme_cmd_read; 680b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_dir = DMA_FROM_DEVICE; 681b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 682b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 683427e97080196548557b288517537ab7eb48c309fKeith Busch result = nvme_map_bio(nvmeq, iod, bio, dma_dir, psegs); 684427e97080196548557b288517537ab7eb48c309fKeith Busch if (result <= 0) 685859361a228258edf4821d9f5635825033eca78e8Keith Busch goto free_cmdid; 6861ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox length = result; 687b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 688ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.command_id = cmdid; 689ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.nsid = cpu_to_le32(ns->ns_id); 690eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox length = nvme_setup_prps(nvmeq->dev, &cmnd->common, iod, length, 691eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox GFP_ATOMIC); 692063cc6d5591ea9c0631b81ac5c7b829d99738b2fMatthew Wilcox cmnd->rw.slba = cpu_to_le64(nvme_block_nr(ns, bio->bi_sector)); 6931ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox cmnd->rw.length = cpu_to_le16((length >> ns->lba_shift) - 1); 694ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.control = cpu_to_le16(control); 695ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt); 696b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 697b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (++nvmeq->sq_tail == nvmeq->q_depth) 698b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->sq_tail = 0; 6997547881d0951384f9833ec3a80fac8f3f16f3b98Matthew Wilcox writel(nvmeq->sq_tail, nvmeq->q_db); 700b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 7011974b1ae8852324a75fb8cfecbc7b758fd5a2c3cMatthew Wilcox return 0; 7021974b1ae8852324a75fb8cfecbc7b758fd5a2c3cMatthew Wilcox 703859361a228258edf4821d9f5635825033eca78e8Keith Busch free_cmdid: 704859361a228258edf4821d9f5635825033eca78e8Keith Busch free_cmdid(nvmeq, cmdid, NULL); 705eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox free_iod: 706eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox nvme_free_iod(nvmeq->dev, iod); 707eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox nomem: 708eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox return result; 709b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 710b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 71193c3d65b28bab6da520c2add9cb387a0303f8b2dLinus Torvaldsstatic void nvme_make_request(struct request_queue *q, struct bio *bio) 712b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 713b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_ns *ns = q->queuedata; 714040a93b52a9eee8177ebaf2ba0ee0f9f518d1bf8Matthew Wilcox struct nvme_queue *nvmeq = get_nvmeq(ns->dev); 715eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox int result = -EBUSY; 716eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox 717eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox spin_lock_irq(&nvmeq->q_lock); 718eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox if (bio_list_empty(&nvmeq->sq_cong)) 719eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox result = nvme_submit_bio_queue(nvmeq, ns, bio); 720eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox if (unlikely(result)) { 721eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox if (bio_list_empty(&nvmeq->sq_cong)) 722eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox add_wait_queue(&nvmeq->sq_full, &nvmeq->sq_cong_wait); 723b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox bio_list_add(&nvmeq->sq_cong, bio); 724b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 725eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox 726eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox spin_unlock_irq(&nvmeq->q_lock); 727b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox put_nvmeq(nvmeq); 728b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 729b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 730b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic irqreturn_t nvme_process_cq(struct nvme_queue *nvmeq) 731b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 732821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox u16 head, phase; 733b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 734b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox head = nvmeq->cq_head; 735821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox phase = nvmeq->cq_phase; 736b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 737b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox for (;;) { 738c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox void *ctx; 739c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox nvme_completion_fn fn; 740b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_completion cqe = nvmeq->cqes[head]; 741821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox if ((le16_to_cpu(cqe.status) & 1) != phase) 742b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox break; 743b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->sq_head = le16_to_cpu(cqe.sq_head); 744b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (++head == nvmeq->q_depth) { 745b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox head = 0; 746821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox phase = !phase; 747b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 748b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 749c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox ctx = free_cmdid(nvmeq, cqe.command_id, &fn); 7505c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcox fn(nvmeq->dev, ctx, &cqe); 751b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 752b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 753b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox /* If the controller ignores the cq head doorbell and continuously 754b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * writes to the queue, it is theoretically possible to wrap around 755b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * the queue twice and mistakenly return IRQ_NONE. Linux only 756b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * requires that 0.1% of your interrupts are handled, so this isn't 757b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * a big problem. 758b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 759821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox if (head == nvmeq->cq_head && phase == nvmeq->cq_phase) 760b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return IRQ_NONE; 761b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 762f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox writel(head, nvmeq->q_db + (1 << nvmeq->dev->db_stride)); 763b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->cq_head = head; 764821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox nvmeq->cq_phase = phase; 765b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 766b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return IRQ_HANDLED; 767b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 768b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 769b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic irqreturn_t nvme_irq(int irq, void *data) 770b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 77158ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox irqreturn_t result; 77258ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox struct nvme_queue *nvmeq = data; 77358ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox spin_lock(&nvmeq->q_lock); 77458ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox result = nvme_process_cq(nvmeq); 77558ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox spin_unlock(&nvmeq->q_lock); 77658ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox return result; 77758ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox} 77858ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox 77958ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcoxstatic irqreturn_t nvme_irq_check(int irq, void *data) 78058ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox{ 78158ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox struct nvme_queue *nvmeq = data; 78258ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox struct nvme_completion cqe = nvmeq->cqes[nvmeq->cq_head]; 78358ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox if ((le16_to_cpu(cqe.status) & 1) != nvmeq->cq_phase) 78458ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox return IRQ_NONE; 78558ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox return IRQ_WAKE_THREAD; 78658ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox} 78758ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox 7883c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcoxstatic void nvme_abort_command(struct nvme_queue *nvmeq, int cmdid) 7893c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox{ 7903c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox spin_lock_irq(&nvmeq->q_lock); 791c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox cancel_cmdid(nvmeq, cmdid, NULL); 7923c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox spin_unlock_irq(&nvmeq->q_lock); 7933c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox} 7943c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox 795c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcoxstruct sync_cmd_info { 796c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox struct task_struct *task; 797c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox u32 result; 798c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox int status; 799c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox}; 800c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox 8015c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcoxstatic void sync_completion(struct nvme_dev *dev, void *ctx, 802c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox struct nvme_completion *cqe) 803c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox{ 804c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox struct sync_cmd_info *cmdinfo = ctx; 805c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox cmdinfo->result = le32_to_cpup(&cqe->result); 806c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox cmdinfo->status = le16_to_cpup(&cqe->status) >> 1; 807c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox wake_up_process(cmdinfo->task); 808c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox} 809c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox 810b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 811b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Returns 0 on success. If the result is negative, it's a Linux error code; 812b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * if the result is positive, it's an NVM Express status code 813b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 8145d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermaint nvme_submit_sync_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd, 8155d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma u32 *result, unsigned timeout) 816b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 817b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int cmdid; 818b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct sync_cmd_info cmdinfo; 819b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 820b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cmdinfo.task = current; 821b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cmdinfo.status = -EINTR; 822b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 823c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox cmdid = alloc_cmdid_killable(nvmeq, &cmdinfo, sync_completion, 824e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox timeout); 825b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (cmdid < 0) 826b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return cmdid; 827b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cmd->common.command_id = cmdid; 828b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 8293c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox set_current_state(TASK_KILLABLE); 8303c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox nvme_submit_cmd(nvmeq, cmd); 831b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox schedule(); 832b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 8333c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox if (cmdinfo.status == -EINTR) { 8343c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox nvme_abort_command(nvmeq, cmdid); 8353c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox return -EINTR; 8363c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox } 8373c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox 838b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (result) 839b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox *result = cmdinfo.result; 840b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 841b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return cmdinfo.status; 842b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 843b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 8445d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermaint nvme_submit_admin_cmd(struct nvme_dev *dev, struct nvme_command *cmd, 845b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 *result) 846b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 847e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox return nvme_submit_sync_cmd(dev->queues[0], cmd, result, ADMIN_TIMEOUT); 848b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 849b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 850b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id) 851b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 852b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int status; 853b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_command c; 854b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 855b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset(&c, 0, sizeof(c)); 856b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.delete_queue.opcode = opcode; 857b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.delete_queue.qid = cpu_to_le16(id); 858b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 859b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox status = nvme_submit_admin_cmd(dev, &c, NULL); 860b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (status) 861b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -EIO; 862b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 863b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 864b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 865b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_alloc_cq(struct nvme_dev *dev, u16 qid, 866b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq) 867b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 868b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int status; 869b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_command c; 870b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int flags = NVME_QUEUE_PHYS_CONTIG | NVME_CQ_IRQ_ENABLED; 871b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 872b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset(&c, 0, sizeof(c)); 873b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.opcode = nvme_admin_create_cq; 874b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.prp1 = cpu_to_le64(nvmeq->cq_dma_addr); 875b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.cqid = cpu_to_le16(qid); 876b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1); 877b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.cq_flags = cpu_to_le16(flags); 878b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.irq_vector = cpu_to_le16(nvmeq->cq_vector); 879b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 880b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox status = nvme_submit_admin_cmd(dev, &c, NULL); 881b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (status) 882b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -EIO; 883b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 884b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 885b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 886b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_alloc_sq(struct nvme_dev *dev, u16 qid, 887b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq) 888b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 889b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int status; 890b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_command c; 891b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int flags = NVME_QUEUE_PHYS_CONTIG | NVME_SQ_PRIO_MEDIUM; 892b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 893b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset(&c, 0, sizeof(c)); 894b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.opcode = nvme_admin_create_sq; 895b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.prp1 = cpu_to_le64(nvmeq->sq_dma_addr); 896b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.sqid = cpu_to_le16(qid); 897b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1); 898b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.sq_flags = cpu_to_le16(flags); 899b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.cqid = cpu_to_le16(qid); 900b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 901b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox status = nvme_submit_admin_cmd(dev, &c, NULL); 902b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (status) 903b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -EIO; 904b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 905b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 906b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 907b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_delete_cq(struct nvme_dev *dev, u16 cqid) 908b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 909b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return adapter_delete_queue(dev, nvme_admin_delete_cq, cqid); 910b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 911b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 912b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_delete_sq(struct nvme_dev *dev, u16 sqid) 913b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 914b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return adapter_delete_queue(dev, nvme_admin_delete_sq, sqid); 915b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 916b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 9175d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermaint nvme_identify(struct nvme_dev *dev, unsigned nsid, unsigned cns, 918bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox dma_addr_t dma_addr) 919bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox{ 920bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox struct nvme_command c; 921bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox 922bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox memset(&c, 0, sizeof(c)); 923bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox c.identify.opcode = nvme_admin_identify; 924bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox c.identify.nsid = cpu_to_le32(nsid); 925bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox c.identify.prp1 = cpu_to_le64(dma_addr); 926bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox c.identify.cns = cpu_to_le32(cns); 927bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox 928bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox return nvme_submit_admin_cmd(dev, &c, NULL); 929bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox} 930bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox 9315d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermaint nvme_get_features(struct nvme_dev *dev, unsigned fid, unsigned nsid, 93208df1e05657fc6712e520e7c09cc6c86160ceb35Keith Busch dma_addr_t dma_addr, u32 *result) 933bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox{ 934bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox struct nvme_command c; 935bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox 936bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox memset(&c, 0, sizeof(c)); 937bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox c.features.opcode = nvme_admin_get_features; 938a42ceccef0c43b46ff6bc1b12a7c1076ef243df1Keith Busch c.features.nsid = cpu_to_le32(nsid); 939bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox c.features.prp1 = cpu_to_le64(dma_addr); 940bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox c.features.fid = cpu_to_le32(fid); 941bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox 94208df1e05657fc6712e520e7c09cc6c86160ceb35Keith Busch return nvme_submit_admin_cmd(dev, &c, result); 943df3481399042200792822b6243e36a95a557b57eMatthew Wilcox} 944df3481399042200792822b6243e36a95a557b57eMatthew Wilcox 9455d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermaint nvme_set_features(struct nvme_dev *dev, unsigned fid, unsigned dword11, 9465d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma dma_addr_t dma_addr, u32 *result) 947df3481399042200792822b6243e36a95a557b57eMatthew Wilcox{ 948df3481399042200792822b6243e36a95a557b57eMatthew Wilcox struct nvme_command c; 949df3481399042200792822b6243e36a95a557b57eMatthew Wilcox 950df3481399042200792822b6243e36a95a557b57eMatthew Wilcox memset(&c, 0, sizeof(c)); 951df3481399042200792822b6243e36a95a557b57eMatthew Wilcox c.features.opcode = nvme_admin_set_features; 952df3481399042200792822b6243e36a95a557b57eMatthew Wilcox c.features.prp1 = cpu_to_le64(dma_addr); 953df3481399042200792822b6243e36a95a557b57eMatthew Wilcox c.features.fid = cpu_to_le32(fid); 954df3481399042200792822b6243e36a95a557b57eMatthew Wilcox c.features.dword11 = cpu_to_le32(dword11); 955df3481399042200792822b6243e36a95a557b57eMatthew Wilcox 956bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox return nvme_submit_admin_cmd(dev, &c, result); 957bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox} 958bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox 959a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox/** 960a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox * nvme_cancel_ios - Cancel outstanding I/Os 961a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox * @queue: The queue to cancel I/Os on 962a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox * @timeout: True to only cancel I/Os which have timed out 963a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox */ 964a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcoxstatic void nvme_cancel_ios(struct nvme_queue *nvmeq, bool timeout) 965a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox{ 966a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox int depth = nvmeq->q_depth - 1; 967a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox struct nvme_cmd_info *info = nvme_cmd_info(nvmeq); 968a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox unsigned long now = jiffies; 969a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox int cmdid; 970a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox 971a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox for_each_set_bit(cmdid, nvmeq->cmdid_data, depth) { 972a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox void *ctx; 973a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox nvme_completion_fn fn; 974a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox static struct nvme_completion cqe = { 975af2d9ca744af5e03390eeb3864d08ce75c860899Matthew Wilcox .status = cpu_to_le16(NVME_SC_ABORT_REQ << 1), 976a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox }; 977a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox 978a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox if (timeout && !time_after(now, info[cmdid].timeout)) 979a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox continue; 980a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox dev_warn(nvmeq->q_dmadev, "Cancelling I/O %d\n", cmdid); 981a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox ctx = cancel_cmdid(nvmeq, cmdid, &fn); 982a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox fn(nvmeq->dev, ctx, &cqe); 983a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox } 984a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox} 985a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox 9869e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcoxstatic void nvme_free_queue_mem(struct nvme_queue *nvmeq) 9879e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox{ 9889e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth), 9899e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox (void *)nvmeq->cqes, nvmeq->cq_dma_addr); 9909e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth), 9919e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox nvmeq->sq_cmds, nvmeq->sq_dma_addr); 9929e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox kfree(nvmeq); 9939e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox} 9949e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox 995b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void nvme_free_queue(struct nvme_dev *dev, int qid) 996b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 997b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq = dev->queues[qid]; 998aba2080f3f1639f9202f1a52993669844abcfb80Matthew Wilcox int vector = dev->entry[nvmeq->cq_vector].vector; 999b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1000a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox spin_lock_irq(&nvmeq->q_lock); 1001a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox nvme_cancel_ios(nvmeq, false); 10023295874b6074d749516d6decd43afad7bf6e38ffKeith Busch while (bio_list_peek(&nvmeq->sq_cong)) { 10033295874b6074d749516d6decd43afad7bf6e38ffKeith Busch struct bio *bio = bio_list_pop(&nvmeq->sq_cong); 10043295874b6074d749516d6decd43afad7bf6e38ffKeith Busch bio_endio(bio, -EIO); 10053295874b6074d749516d6decd43afad7bf6e38ffKeith Busch } 1006a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox spin_unlock_irq(&nvmeq->q_lock); 1007a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox 1008aba2080f3f1639f9202f1a52993669844abcfb80Matthew Wilcox irq_set_affinity_hint(vector, NULL); 1009aba2080f3f1639f9202f1a52993669844abcfb80Matthew Wilcox free_irq(vector, nvmeq); 1010b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1011b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox /* Don't tell the adapter to delete the admin queue */ 1012b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (qid) { 1013b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox adapter_delete_sq(dev, qid); 1014b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox adapter_delete_cq(dev, qid); 1015b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 1016b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 10179e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox nvme_free_queue_mem(nvmeq); 1018b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1019b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1020b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev, int qid, 1021b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int depth, int vector) 1022b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1023b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct device *dmadev = &dev->pci_dev->dev; 1024a0cadb85b8b758608ae0759151e29de7581c6731Keith Busch unsigned extra = DIV_ROUND_UP(depth, 8) + (depth * 1025a0cadb85b8b758608ae0759151e29de7581c6731Keith Busch sizeof(struct nvme_cmd_info)); 1026b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq = kzalloc(sizeof(*nvmeq) + extra, GFP_KERNEL); 1027b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!nvmeq) 1028b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 1029b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1030b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->cqes = dma_alloc_coherent(dmadev, CQ_SIZE(depth), 1031b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox &nvmeq->cq_dma_addr, GFP_KERNEL); 1032b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!nvmeq->cqes) 1033b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto free_nvmeq; 1034b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset((void *)nvmeq->cqes, 0, CQ_SIZE(depth)); 1035b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1036b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->sq_cmds = dma_alloc_coherent(dmadev, SQ_SIZE(depth), 1037b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox &nvmeq->sq_dma_addr, GFP_KERNEL); 1038b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!nvmeq->sq_cmds) 1039b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto free_cqdma; 1040b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1041b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->q_dmadev = dmadev; 1042091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox nvmeq->dev = dev; 1043b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox spin_lock_init(&nvmeq->q_lock); 1044b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->cq_head = 0; 1045821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox nvmeq->cq_phase = 1; 1046b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox init_waitqueue_head(&nvmeq->sq_full); 10471fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox init_waitqueue_entry(&nvmeq->sq_cong_wait, nvme_thread); 1048b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox bio_list_init(&nvmeq->sq_cong); 1049f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox nvmeq->q_db = &dev->dbs[qid << (dev->db_stride + 1)]; 1050b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->q_depth = depth; 1051b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->cq_vector = vector; 1052b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1053b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return nvmeq; 1054b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1055b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox free_cqdma: 105668b8eca5f882230e4991df0db0a3094f49420a90Keith Busch dma_free_coherent(dmadev, CQ_SIZE(depth), (void *)nvmeq->cqes, 1057b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->cq_dma_addr); 1058b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox free_nvmeq: 1059b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(nvmeq); 1060b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 1061b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1062b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 10633001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcoxstatic int queue_request_irq(struct nvme_dev *dev, struct nvme_queue *nvmeq, 10643001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox const char *name) 10653001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox{ 106658ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox if (use_threaded_interrupts) 106758ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox return request_threaded_irq(dev->entry[nvmeq->cq_vector].vector, 1068ec6ce618d65b5ce1bef83a5509255107a0feac44Matthew Wilcox nvme_irq_check, nvme_irq, 106958ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox IRQF_DISABLED | IRQF_SHARED, 107058ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox name, nvmeq); 10713001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox return request_irq(dev->entry[nvmeq->cq_vector].vector, nvme_irq, 10723001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox IRQF_DISABLED | IRQF_SHARED, name, nvmeq); 10733001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox} 10743001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox 10758d85fce77edfc22f1d6dbf78e3af723b4b556f3dGreg Kroah-Hartmanstatic struct nvme_queue *nvme_create_queue(struct nvme_dev *dev, int qid, 10768d85fce77edfc22f1d6dbf78e3af723b4b556f3dGreg Kroah-Hartman int cq_size, int vector) 1077b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1078b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int result; 1079b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq = nvme_alloc_queue(dev, qid, cq_size, vector); 1080b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 10813f85d50b609e8a5ef151656210203a6e94c19538Matthew Wilcox if (!nvmeq) 10826f0f54499f2edf7e25410cdd99e6f030f3485fd1Matthew Wilcox return ERR_PTR(-ENOMEM); 10833f85d50b609e8a5ef151656210203a6e94c19538Matthew Wilcox 1084b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox result = adapter_alloc_cq(dev, qid, nvmeq); 1085b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (result < 0) 1086b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto free_nvmeq; 1087b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1088b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox result = adapter_alloc_sq(dev, qid, nvmeq); 1089b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (result < 0) 1090b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto release_cq; 1091b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 10923001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox result = queue_request_irq(dev, nvmeq, "nvme"); 1093b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (result < 0) 1094b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto release_sq; 1095b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1096b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return nvmeq; 1097b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1098b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox release_sq: 1099b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox adapter_delete_sq(dev, qid); 1100b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox release_cq: 1101b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox adapter_delete_cq(dev, qid); 1102b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox free_nvmeq: 1103b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth), 1104b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox (void *)nvmeq->cqes, nvmeq->cq_dma_addr); 1105b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth), 1106b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->sq_cmds, nvmeq->sq_dma_addr); 1107b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(nvmeq); 11086f0f54499f2edf7e25410cdd99e6f030f3485fd1Matthew Wilcox return ERR_PTR(result); 1109b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1110b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 11118d85fce77edfc22f1d6dbf78e3af723b4b556f3dGreg Kroah-Hartmanstatic int nvme_configure_admin_queue(struct nvme_dev *dev) 1112b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 11139e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox int result = 0; 1114b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 aqa; 111522605f96810d073eb74051d0295b6577d6a6a563Matthew Wilcox u64 cap; 111622605f96810d073eb74051d0295b6577d6a6a563Matthew Wilcox unsigned long timeout; 1117b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq; 1118b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1119b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->dbs = ((void __iomem *)dev->bar) + 4096; 1120b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1121b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq = nvme_alloc_queue(dev, 0, 64, 0); 11223f85d50b609e8a5ef151656210203a6e94c19538Matthew Wilcox if (!nvmeq) 11233f85d50b609e8a5ef151656210203a6e94c19538Matthew Wilcox return -ENOMEM; 1124b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1125b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox aqa = nvmeq->q_depth - 1; 1126b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox aqa |= aqa << 16; 1127b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1128b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->ctrl_config = NVME_CC_ENABLE | NVME_CC_CSS_NVM; 1129b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->ctrl_config |= (PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT; 1130b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->ctrl_config |= NVME_CC_ARB_RR | NVME_CC_SHN_NONE; 11317f53f9d2424533256ae86f7df5661a17de743de8Matthew Wilcox dev->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES; 1132b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 11335911f20039ce59d7e7834f0c42151cf759b6f786Shane Michael Matthews writel(0, &dev->bar->cc); 1134b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox writel(aqa, &dev->bar->aqa); 1135b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox writeq(nvmeq->sq_dma_addr, &dev->bar->asq); 1136b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox writeq(nvmeq->cq_dma_addr, &dev->bar->acq); 1137b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox writel(dev->ctrl_config, &dev->bar->cc); 1138b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 113922605f96810d073eb74051d0295b6577d6a6a563Matthew Wilcox cap = readq(&dev->bar->cap); 114022605f96810d073eb74051d0295b6577d6a6a563Matthew Wilcox timeout = ((NVME_CAP_TIMEOUT(cap) + 1) * HZ / 2) + jiffies; 1141f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox dev->db_stride = NVME_CAP_STRIDE(cap); 114222605f96810d073eb74051d0295b6577d6a6a563Matthew Wilcox 11439e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox while (!result && !(readl(&dev->bar->csts) & NVME_CSTS_RDY)) { 1144b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox msleep(100); 1145b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (fatal_signal_pending(current)) 11469e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox result = -EINTR; 114722605f96810d073eb74051d0295b6577d6a6a563Matthew Wilcox if (time_after(jiffies, timeout)) { 114822605f96810d073eb74051d0295b6577d6a6a563Matthew Wilcox dev_err(&dev->pci_dev->dev, 114922605f96810d073eb74051d0295b6577d6a6a563Matthew Wilcox "Device not ready; aborting initialisation\n"); 11509e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox result = -ENODEV; 115122605f96810d073eb74051d0295b6577d6a6a563Matthew Wilcox } 1152b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 1153b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1154025c557a71bd06a9f6d32259f00e2218b15bf0a4Keith Busch if (result) 1155025c557a71bd06a9f6d32259f00e2218b15bf0a4Keith Busch goto free_q; 11569e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox 11573001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox result = queue_request_irq(dev, nvmeq, "nvme admin"); 1158025c557a71bd06a9f6d32259f00e2218b15bf0a4Keith Busch if (result) 1159025c557a71bd06a9f6d32259f00e2218b15bf0a4Keith Busch goto free_q; 1160025c557a71bd06a9f6d32259f00e2218b15bf0a4Keith Busch 1161b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->queues[0] = nvmeq; 1162b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return result; 1163025c557a71bd06a9f6d32259f00e2218b15bf0a4Keith Busch 1164025c557a71bd06a9f6d32259f00e2218b15bf0a4Keith Busch free_q: 1165025c557a71bd06a9f6d32259f00e2218b15bf0a4Keith Busch nvme_free_queue_mem(nvmeq); 1166025c557a71bd06a9f6d32259f00e2218b15bf0a4Keith Busch return result; 1167b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1168b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 11695d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermastruct nvme_iod *nvme_map_user_pages(struct nvme_dev *dev, int write, 1170eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox unsigned long addr, unsigned length) 1171b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 117236c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox int i, err, count, nents, offset; 11737fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox struct scatterlist *sg; 11747fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox struct page **pages; 1175eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox struct nvme_iod *iod; 117636c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox 117736c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox if (addr & 3) 1178eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return ERR_PTR(-EINVAL); 11797fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox if (!length) 1180eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return ERR_PTR(-EINVAL); 11817fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox 118236c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox offset = offset_in_page(addr); 11837fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox count = DIV_ROUND_UP(offset + length, PAGE_SIZE); 11847fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox pages = kcalloc(count, sizeof(*pages), GFP_KERNEL); 118522fff826e715e9727d3c7a69f15e602a9801b673Dan Carpenter if (!pages) 118622fff826e715e9727d3c7a69f15e602a9801b673Dan Carpenter return ERR_PTR(-ENOMEM); 118736c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox 118836c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox err = get_user_pages_fast(addr, count, 1, pages); 118936c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox if (err < count) { 119036c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox count = err; 119136c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox err = -EFAULT; 119236c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox goto put_pages; 119336c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox } 11947fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox 1195eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod = nvme_alloc_iod(count, length, GFP_KERNEL); 1196eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox sg = iod->sg; 119736c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox sg_init_table(sg, count); 1198d0ba1e497bca83a3d353eb47c9658afc54d83228Matthew Wilcox for (i = 0; i < count; i++) { 1199d0ba1e497bca83a3d353eb47c9658afc54d83228Matthew Wilcox sg_set_page(&sg[i], pages[i], 1200d0ba1e497bca83a3d353eb47c9658afc54d83228Matthew Wilcox min_t(int, length, PAGE_SIZE - offset), offset); 1201d0ba1e497bca83a3d353eb47c9658afc54d83228Matthew Wilcox length -= (PAGE_SIZE - offset); 1202d0ba1e497bca83a3d353eb47c9658afc54d83228Matthew Wilcox offset = 0; 12037fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox } 1204fe304c43c6d63e29ed4fc46a874d7a74313788c5Matthew Wilcox sg_mark_end(&sg[i - 1]); 12051c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox iod->nents = count; 12067fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox 12077fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox err = -ENOMEM; 12087fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox nents = dma_map_sg(&dev->pci_dev->dev, sg, count, 12097fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox write ? DMA_TO_DEVICE : DMA_FROM_DEVICE); 121036c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox if (!nents) 1211eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox goto free_iod; 1212b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 12137fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox kfree(pages); 1214eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return iod; 1215b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1216eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox free_iod: 1217eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox kfree(iod); 12187fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox put_pages: 12197fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox for (i = 0; i < count; i++) 12207fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox put_page(pages[i]); 12217fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox kfree(pages); 1222eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return ERR_PTR(err); 12237fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox} 1224b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 12255d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermavoid nvme_unmap_user_pages(struct nvme_dev *dev, int write, 12261c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox struct nvme_iod *iod) 12277fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox{ 12281c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox int i; 1229b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 12301c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox dma_unmap_sg(&dev->pci_dev->dev, iod->sg, iod->nents, 12311c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox write ? DMA_TO_DEVICE : DMA_FROM_DEVICE); 12327fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox 12331c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox for (i = 0; i < iod->nents; i++) 12341c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox put_page(sg_page(&iod->sg[i])); 12357fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox} 1236b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1237a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcoxstatic int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio) 1238a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox{ 1239a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox struct nvme_dev *dev = ns->dev; 1240a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox struct nvme_queue *nvmeq; 1241a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox struct nvme_user_io io; 1242a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox struct nvme_command c; 1243f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch unsigned length, meta_len; 1244f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch int status, i; 1245f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch struct nvme_iod *iod, *meta_iod = NULL; 1246f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch dma_addr_t meta_dma_addr; 1247f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch void *meta, *uninitialized_var(meta_mem); 1248a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox 1249a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox if (copy_from_user(&io, uio, sizeof(io))) 1250a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox return -EFAULT; 12516c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox length = (io.nblocks + 1) << ns->lba_shift; 1252f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_len = (io.nblocks + 1) * ns->ms; 1253f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1254f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch if (meta_len && ((io.metadata & 3) || !io.metadata)) 1255f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch return -EINVAL; 12566c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox 12576c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox switch (io.opcode) { 12586c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox case nvme_cmd_write: 12596c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox case nvme_cmd_read: 12606bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox case nvme_cmd_compare: 1261eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox iod = nvme_map_user_pages(dev, io.opcode & 1, io.addr, length); 12626413214c5d424fd5aae6567848340f962ad2ce0fMatthew Wilcox break; 12636c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox default: 12646bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox return -EINVAL; 12656c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox } 12666c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox 1267eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox if (IS_ERR(iod)) 1268eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return PTR_ERR(iod); 1269a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox 1270a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox memset(&c, 0, sizeof(c)); 1271a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.opcode = io.opcode; 1272a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.flags = io.flags; 12736c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox c.rw.nsid = cpu_to_le32(ns->ns_id); 1274a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.slba = cpu_to_le64(io.slba); 12756c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox c.rw.length = cpu_to_le16(io.nblocks); 1276a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.control = cpu_to_le16(io.control); 12771c9b52651dad0ff1fa71fc6205c86d972f25bcc0Matthew Wilcox c.rw.dsmgmt = cpu_to_le32(io.dsmgmt); 12781c9b52651dad0ff1fa71fc6205c86d972f25bcc0Matthew Wilcox c.rw.reftag = cpu_to_le32(io.reftag); 12791c9b52651dad0ff1fa71fc6205c86d972f25bcc0Matthew Wilcox c.rw.apptag = cpu_to_le16(io.apptag); 12801c9b52651dad0ff1fa71fc6205c86d972f25bcc0Matthew Wilcox c.rw.appmask = cpu_to_le16(io.appmask); 1281f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1282f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch if (meta_len) { 1283f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_iod = nvme_map_user_pages(dev, io.opcode & 1, io.metadata, meta_len); 1284f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch if (IS_ERR(meta_iod)) { 1285f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch status = PTR_ERR(meta_iod); 1286f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_iod = NULL; 1287f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch goto unmap; 1288f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch } 1289f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1290f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_mem = dma_alloc_coherent(&dev->pci_dev->dev, meta_len, 1291f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch &meta_dma_addr, GFP_KERNEL); 1292f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch if (!meta_mem) { 1293f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch status = -ENOMEM; 1294f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch goto unmap; 1295f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch } 1296f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1297f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch if (io.opcode & 1) { 1298f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch int meta_offset = 0; 1299f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1300f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch for (i = 0; i < meta_iod->nents; i++) { 1301f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta = kmap_atomic(sg_page(&meta_iod->sg[i])) + 1302f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_iod->sg[i].offset; 1303f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch memcpy(meta_mem + meta_offset, meta, 1304f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_iod->sg[i].length); 1305f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch kunmap_atomic(meta); 1306f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_offset += meta_iod->sg[i].length; 1307f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch } 1308f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch } 1309f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1310f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch c.rw.metadata = cpu_to_le64(meta_dma_addr); 1311f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch } 1312f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1313eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox length = nvme_setup_prps(dev, &c.common, iod, length, GFP_KERNEL); 1314a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox 1315040a93b52a9eee8177ebaf2ba0ee0f9f518d1bf8Matthew Wilcox nvmeq = get_nvmeq(dev); 1316fa92282149842645931580225647238428374758Matthew Wilcox /* 1317fa92282149842645931580225647238428374758Matthew Wilcox * Since nvme_submit_sync_cmd sleeps, we can't keep preemption 1318b1ad37efcafe396ac3944853589688dd0ec3c64eMatthew Wilcox * disabled. We may be preempted at any point, and be rescheduled 1319b1ad37efcafe396ac3944853589688dd0ec3c64eMatthew Wilcox * to a different CPU. That will cause cacheline bouncing, but no 1320b1ad37efcafe396ac3944853589688dd0ec3c64eMatthew Wilcox * additional races since q_lock already protects against other CPUs. 1321b1ad37efcafe396ac3944853589688dd0ec3c64eMatthew Wilcox */ 1322a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox put_nvmeq(nvmeq); 1323b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox if (length != (io.nblocks + 1) << ns->lba_shift) 1324b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox status = -ENOMEM; 1325b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox else 1326ff976d724a74e4522e9ca2de1fb37ac4520f454fMatthew Wilcox status = nvme_submit_sync_cmd(nvmeq, &c, NULL, NVME_IO_TIMEOUT); 1327a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox 1328f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch if (meta_len) { 1329f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch if (status == NVME_SC_SUCCESS && !(io.opcode & 1)) { 1330f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch int meta_offset = 0; 1331f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1332f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch for (i = 0; i < meta_iod->nents; i++) { 1333f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta = kmap_atomic(sg_page(&meta_iod->sg[i])) + 1334f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_iod->sg[i].offset; 1335f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch memcpy(meta, meta_mem + meta_offset, 1336f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_iod->sg[i].length); 1337f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch kunmap_atomic(meta); 1338f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_offset += meta_iod->sg[i].length; 1339f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch } 1340f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch } 1341f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1342f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch dma_free_coherent(&dev->pci_dev->dev, meta_len, meta_mem, 1343f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch meta_dma_addr); 1344f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch } 1345f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1346f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch unmap: 13471c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox nvme_unmap_user_pages(dev, io.opcode & 1, iod); 1348eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox nvme_free_iod(dev, iod); 1349f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1350f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch if (meta_iod) { 1351f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch nvme_unmap_user_pages(dev, io.opcode & 1, meta_iod); 1352f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch nvme_free_iod(dev, meta_iod); 1353f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch } 1354f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch 1355a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox return status; 1356a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox} 1357a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox 135850af8baec46a99a9b81a4600c0374f83a5a590a9Keith Buschstatic int nvme_user_admin_cmd(struct nvme_dev *dev, 13596bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox struct nvme_admin_cmd __user *ucmd) 13606ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox{ 13616bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox struct nvme_admin_cmd cmd; 13626ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox struct nvme_command c; 1363eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox int status, length; 1364c7d36ab8fa04c213328119a9c0d66985fe204ee5Keith Busch struct nvme_iod *uninitialized_var(iod); 13656ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox 13666bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox if (!capable(CAP_SYS_ADMIN)) 13676bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox return -EACCES; 13686bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox if (copy_from_user(&cmd, ucmd, sizeof(cmd))) 13696ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox return -EFAULT; 13706ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox 13716ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox memset(&c, 0, sizeof(c)); 13726bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.opcode = cmd.opcode; 13736bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.flags = cmd.flags; 13746bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.nsid = cpu_to_le32(cmd.nsid); 13756bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw2[0] = cpu_to_le32(cmd.cdw2); 13766bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw2[1] = cpu_to_le32(cmd.cdw3); 13776bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw10[0] = cpu_to_le32(cmd.cdw10); 13786bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw10[1] = cpu_to_le32(cmd.cdw11); 13796bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw10[2] = cpu_to_le32(cmd.cdw12); 13806bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw10[3] = cpu_to_le32(cmd.cdw13); 13816bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw10[4] = cpu_to_le32(cmd.cdw14); 13826bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox c.common.cdw10[5] = cpu_to_le32(cmd.cdw15); 13836bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox 13846bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox length = cmd.data_len; 13856bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox if (cmd.data_len) { 1386497421880acecd0281d3182d534f3d28c927caecMatthew Wilcox iod = nvme_map_user_pages(dev, cmd.opcode & 1, cmd.addr, 1387497421880acecd0281d3182d534f3d28c927caecMatthew Wilcox length); 1388eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox if (IS_ERR(iod)) 1389eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox return PTR_ERR(iod); 1390eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox length = nvme_setup_prps(dev, &c.common, iod, length, 1391eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox GFP_KERNEL); 13926bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox } 13936bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox 13946bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox if (length != cmd.data_len) 1395b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox status = -ENOMEM; 1396b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox else 1397f4f117f64baf8840d22266d518227b2a186d294bKeith Busch status = nvme_submit_admin_cmd(dev, &c, &cmd.result); 1398eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox 13996bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox if (cmd.data_len) { 14001c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox nvme_unmap_user_pages(dev, cmd.opcode & 1, iod); 1401eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox nvme_free_iod(dev, iod); 14026bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox } 1403f4f117f64baf8840d22266d518227b2a186d294bKeith Busch 1404f4f117f64baf8840d22266d518227b2a186d294bKeith Busch if (!status && copy_to_user(&ucmd->result, &cmd.result, 1405f4f117f64baf8840d22266d518227b2a186d294bKeith Busch sizeof(cmd.result))) 1406f4f117f64baf8840d22266d518227b2a186d294bKeith Busch status = -EFAULT; 1407f4f117f64baf8840d22266d518227b2a186d294bKeith Busch 14086ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox return status; 14096ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox} 14106ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox 1411b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, 1412b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unsigned long arg) 1413b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1414b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_ns *ns = bdev->bd_disk->private_data; 1415b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1416b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox switch (cmd) { 14176bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox case NVME_IOCTL_ID: 14186bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox return ns->ns_id; 14196bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox case NVME_IOCTL_ADMIN_CMD: 142050af8baec46a99a9b81a4600c0374f83a5a590a9Keith Busch return nvme_user_admin_cmd(ns->dev, (void __user *)arg); 1421a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox case NVME_IOCTL_SUBMIT_IO: 1422a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox return nvme_submit_io(ns, (void __user *)arg); 14235d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma case SG_GET_VERSION_NUM: 14245d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma return nvme_sg_get_version_num((void __user *)arg); 14255d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma case SG_IO: 14265d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma return nvme_sg_io(ns, (void __user *)arg); 1427b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox default: 1428b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -ENOTTY; 1429b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 1430b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1431b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1432b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic const struct block_device_operations nvme_fops = { 1433b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .owner = THIS_MODULE, 1434b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .ioctl = nvme_ioctl, 14354948168280b269a514045766ddd872cfac5968e1Matthew Wilcox .compat_ioctl = nvme_ioctl, 1436b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 1437b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 14381fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcoxstatic void nvme_resubmit_bios(struct nvme_queue *nvmeq) 14391fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox{ 14401fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox while (bio_list_peek(&nvmeq->sq_cong)) { 14411fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox struct bio *bio = bio_list_pop(&nvmeq->sq_cong); 14421fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox struct nvme_ns *ns = bio->bi_bdev->bd_disk->private_data; 1443427e97080196548557b288517537ab7eb48c309fKeith Busch 1444427e97080196548557b288517537ab7eb48c309fKeith Busch if (bio_list_empty(&nvmeq->sq_cong)) 1445427e97080196548557b288517537ab7eb48c309fKeith Busch remove_wait_queue(&nvmeq->sq_full, 1446427e97080196548557b288517537ab7eb48c309fKeith Busch &nvmeq->sq_cong_wait); 14471fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox if (nvme_submit_bio_queue(nvmeq, ns, bio)) { 1448427e97080196548557b288517537ab7eb48c309fKeith Busch if (bio_list_empty(&nvmeq->sq_cong)) 1449427e97080196548557b288517537ab7eb48c309fKeith Busch add_wait_queue(&nvmeq->sq_full, 1450427e97080196548557b288517537ab7eb48c309fKeith Busch &nvmeq->sq_cong_wait); 14511fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox bio_list_add_head(&nvmeq->sq_cong, bio); 14521fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox break; 14531fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox } 14541fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox } 14551fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox} 14561fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 14571fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcoxstatic int nvme_kthread(void *data) 14581fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox{ 14591fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox struct nvme_dev *dev; 14601fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 14611fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox while (!kthread_should_stop()) { 1462564a232c059913d91b491e04c2b2d670b8f94615Arjan van de Ven set_current_state(TASK_INTERRUPTIBLE); 14631fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_lock(&dev_list_lock); 14641fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox list_for_each_entry(dev, &dev_list, node) { 14651fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox int i; 14661fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox for (i = 0; i < dev->queue_count; i++) { 14671fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox struct nvme_queue *nvmeq = dev->queues[i]; 1468740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox if (!nvmeq) 1469740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox continue; 14701fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_lock_irq(&nvmeq->q_lock); 14711fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox if (nvme_process_cq(nvmeq)) 14721fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox printk("process_cq did something\n"); 1473a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox nvme_cancel_ios(nvmeq, true); 14741fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox nvme_resubmit_bios(nvmeq); 14751fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_unlock_irq(&nvmeq->q_lock); 14761fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox } 14771fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox } 14781fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_unlock(&dev_list_lock); 1479acb7aa0db09b8abd38abeb84334a8a27a52fbb1bArjan van de Ven schedule_timeout(round_jiffies_relative(HZ)); 14801fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox } 14811fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox return 0; 14821fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox} 14831fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 14845aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcoxstatic DEFINE_IDA(nvme_index_ida); 14855aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox 14865aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcoxstatic int nvme_get_ns_idx(void) 14875aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox{ 14885aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox int index, error; 14895aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox 14905aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox do { 14915aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox if (!ida_pre_get(&nvme_index_ida, GFP_KERNEL)) 14925aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox return -1; 14935aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox 14945aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox spin_lock(&dev_list_lock); 14955aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox error = ida_get_new(&nvme_index_ida, &index); 14965aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox spin_unlock(&dev_list_lock); 14975aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox } while (error == -EAGAIN); 14985aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox 14995aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox if (error) 15005aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox index = -1; 15015aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox return index; 15025aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox} 15035aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox 15045aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcoxstatic void nvme_put_ns_idx(int index) 15055aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox{ 15065aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox spin_lock(&dev_list_lock); 15075aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox ida_remove(&nvme_index_ida, index); 15085aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox spin_unlock(&dev_list_lock); 15095aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox} 15105aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox 15110e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Buschstatic void nvme_config_discard(struct nvme_ns *ns) 15120e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch{ 15130e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch u32 logical_block_size = queue_logical_block_size(ns->queue); 15140e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch ns->queue->limits.discard_zeroes_data = 0; 15150e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch ns->queue->limits.discard_alignment = logical_block_size; 15160e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch ns->queue->limits.discard_granularity = logical_block_size; 15170e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch ns->queue->limits.max_discard_sectors = 0xffffffff; 15180e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, ns->queue); 15190e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch} 15200e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch 15215aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcoxstatic struct nvme_ns *nvme_alloc_ns(struct nvme_dev *dev, int nsid, 1522b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_id_ns *id, struct nvme_lba_range_type *rt) 1523b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1524b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_ns *ns; 1525b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct gendisk *disk; 1526b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int lbaf; 1527b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1528b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (rt->attributes & NVME_LBART_ATTRIB_HIDE) 1529b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 1530b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1531b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns = kzalloc(sizeof(*ns), GFP_KERNEL); 1532b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!ns) 1533b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 1534b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->queue = blk_alloc_queue(GFP_KERNEL); 1535b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!ns->queue) 1536b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto out_free_ns; 15374eeb9215a0d5c9494ca8b20158cc8ee82618840cMatthew Wilcox ns->queue->queue_flags = QUEUE_FLAG_DEFAULT; 15384eeb9215a0d5c9494ca8b20158cc8ee82618840cMatthew Wilcox queue_flag_set_unlocked(QUEUE_FLAG_NOMERGES, ns->queue); 15394eeb9215a0d5c9494ca8b20158cc8ee82618840cMatthew Wilcox queue_flag_set_unlocked(QUEUE_FLAG_NONROT, ns->queue); 1540b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox blk_queue_make_request(ns->queue, nvme_make_request); 1541b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->dev = dev; 1542b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->queue->queuedata = ns; 1543b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1544b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk = alloc_disk(NVME_MINORS); 1545b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!disk) 1546b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto out_free_queue; 15475aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox ns->ns_id = nsid; 1548b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->disk = disk; 1549b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox lbaf = id->flbas & 0xf; 1550b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->lba_shift = id->lbaf[lbaf].ds; 1551f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch ns->ms = le16_to_cpu(id->lbaf[lbaf].ms); 1552e9ef46369f5107e634a93b7fc4e62a1f53343197Keith Busch blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift); 15538fc23e032debd682f5ba9fc524a5846c10d2c522Keith Busch if (dev->max_hw_sectors) 15548fc23e032debd682f5ba9fc524a5846c10d2c522Keith Busch blk_queue_max_hw_sectors(ns->queue, dev->max_hw_sectors); 1555b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1556b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk->major = nvme_major; 1557b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk->minors = NVME_MINORS; 15585aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox disk->first_minor = NVME_MINORS * nvme_get_ns_idx(); 1559b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk->fops = &nvme_fops; 1560b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk->private_data = ns; 1561b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk->queue = ns->queue; 1562388f037f4e7f0a24bac6b1a24f144f5d939f58cfMatthew Wilcox disk->driverfs_dev = &dev->pci_dev->dev; 15635aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox sprintf(disk->disk_name, "nvme%dn%d", dev->instance, nsid); 1564b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox set_capacity(disk, le64_to_cpup(&id->nsze) << (ns->lba_shift - 9)); 1565b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 15660e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch if (dev->oncs & NVME_CTRL_ONCS_DSM) 15670e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch nvme_config_discard(ns); 15680e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch 1569b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return ns; 1570b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1571b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox out_free_queue: 1572b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox blk_cleanup_queue(ns->queue); 1573b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox out_free_ns: 1574b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(ns); 1575b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 1576b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1577b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1578b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void nvme_ns_free(struct nvme_ns *ns) 1579b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 15805aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox int index = ns->disk->first_minor / NVME_MINORS; 1581b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox put_disk(ns->disk); 15825aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox nvme_put_ns_idx(index); 1583b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox blk_cleanup_queue(ns->queue); 1584b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(ns); 1585b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1586b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1587b3b06812e199f248561ce7824a4a8a9cd573c05aMatthew Wilcoxstatic int set_queue_count(struct nvme_dev *dev, int count) 1588b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1589b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int status; 1590b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 result; 1591b3b06812e199f248561ce7824a4a8a9cd573c05aMatthew Wilcox u32 q_count = (count - 1) | ((count - 1) << 16); 1592b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1593df3481399042200792822b6243e36a95a557b57eMatthew Wilcox status = nvme_set_features(dev, NVME_FEAT_NUM_QUEUES, q_count, 0, 1594bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox &result); 1595b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (status) 1596b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -EIO; 1597b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return min(result & 0xffff, result >> 16) + 1; 1598b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1599b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 16008d85fce77edfc22f1d6dbf78e3af723b4b556f3dGreg Kroah-Hartmanstatic int nvme_setup_io_queues(struct nvme_dev *dev) 1601b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1602a0cadb85b8b758608ae0759151e29de7581c6731Keith Busch int result, cpu, i, nr_io_queues, db_bar_size, q_depth; 1603b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1604b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox nr_io_queues = num_online_cpus(); 1605b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox result = set_queue_count(dev, nr_io_queues); 16061b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox if (result < 0) 16071b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox return result; 1608b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox if (result < nr_io_queues) 1609b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox nr_io_queues = result; 1610b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 16111b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox /* Deregister the admin queue's interrupt */ 16121b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox free_irq(dev->entry[0].vector, dev->queues[0]); 16131b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox 1614f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox db_bar_size = 4096 + ((nr_io_queues + 1) << (dev->db_stride + 3)); 1615f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox if (db_bar_size > 8192) { 1616f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox iounmap(dev->bar); 1617f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox dev->bar = ioremap(pci_resource_start(dev->pci_dev, 0), 1618f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox db_bar_size); 1619f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox dev->dbs = ((void __iomem *)dev->bar) + 4096; 1620f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox dev->queues[0]->q_db = dev->dbs; 1621f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox } 1622f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox 1623b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox for (i = 0; i < nr_io_queues; i++) 16241b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox dev->entry[i].entry = i; 16251b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox for (;;) { 1626b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox result = pci_enable_msix(dev->pci_dev, dev->entry, 1627b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox nr_io_queues); 16281b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox if (result == 0) { 16291b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox break; 16301b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox } else if (result > 0) { 1631b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox nr_io_queues = result; 16321b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox continue; 16331b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox } else { 1634b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox nr_io_queues = 1; 16351b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox break; 16361b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox } 16371b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox } 16381b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox 16391b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox result = queue_request_irq(dev, dev->queues[0], "nvme admin"); 16401b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox /* XXX: handle failure here */ 16411b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox 16421b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox cpu = cpumask_first(cpu_online_mask); 1643b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox for (i = 0; i < nr_io_queues; i++) { 16441b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox irq_set_affinity_hint(dev->entry[i].vector, get_cpu_mask(cpu)); 16451b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox cpu = cpumask_next(cpu, cpu_online_mask); 16461b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox } 16471b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox 1648a0cadb85b8b758608ae0759151e29de7581c6731Keith Busch q_depth = min_t(int, NVME_CAP_MQES(readq(&dev->bar->cap)) + 1, 1649a0cadb85b8b758608ae0759151e29de7581c6731Keith Busch NVME_Q_DEPTH); 1650b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox for (i = 0; i < nr_io_queues; i++) { 1651a0cadb85b8b758608ae0759151e29de7581c6731Keith Busch dev->queues[i + 1] = nvme_create_queue(dev, i + 1, q_depth, i); 16526f0f54499f2edf7e25410cdd99e6f030f3485fd1Matthew Wilcox if (IS_ERR(dev->queues[i + 1])) 16536f0f54499f2edf7e25410cdd99e6f030f3485fd1Matthew Wilcox return PTR_ERR(dev->queues[i + 1]); 16541b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox dev->queue_count++; 16551b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox } 1656b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 16579ecdc946212f7cd592986b2c519b470404caa6b8Matthew Wilcox for (; i < num_possible_cpus(); i++) { 16589ecdc946212f7cd592986b2c519b470404caa6b8Matthew Wilcox int target = i % rounddown_pow_of_two(dev->queue_count - 1); 16599ecdc946212f7cd592986b2c519b470404caa6b8Matthew Wilcox dev->queues[i + 1] = dev->queues[target + 1]; 16609ecdc946212f7cd592986b2c519b470404caa6b8Matthew Wilcox } 16619ecdc946212f7cd592986b2c519b470404caa6b8Matthew Wilcox 1662b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 1663b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1664b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1665b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void nvme_free_queues(struct nvme_dev *dev) 1666b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1667b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int i; 1668b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1669b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox for (i = dev->queue_count - 1; i >= 0; i--) 1670b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_free_queue(dev, i); 1671b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1672b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1673422ef0c7c81934c3344e1623d2dcfda5fa2fab8dMatthew Wilcox/* 1674422ef0c7c81934c3344e1623d2dcfda5fa2fab8dMatthew Wilcox * Return: error value if an error occurred setting up the queues or calling 1675422ef0c7c81934c3344e1623d2dcfda5fa2fab8dMatthew Wilcox * Identify Device. 0 if these succeeded, even if adding some of the 1676422ef0c7c81934c3344e1623d2dcfda5fa2fab8dMatthew Wilcox * namespaces failed. At the moment, these failures are silent. TBD which 1677422ef0c7c81934c3344e1623d2dcfda5fa2fab8dMatthew Wilcox * failures should be reported. 1678422ef0c7c81934c3344e1623d2dcfda5fa2fab8dMatthew Wilcox */ 16798d85fce77edfc22f1d6dbf78e3af723b4b556f3dGreg Kroah-Hartmanstatic int nvme_dev_add(struct nvme_dev *dev) 1680b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1681b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int res, nn, i; 1682cbb6218fd4ae8f98ddf0d66f0826a7a3a9c88298Keith Busch struct nvme_ns *ns; 168351814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox struct nvme_id_ctrl *ctrl; 1684bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox struct nvme_id_ns *id_ns; 1685bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox void *mem; 1686b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_addr_t dma_addr; 1687159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch int shift = NVME_CAP_MPSMIN(readq(&dev->bar->cap)) + 12; 1688b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1689b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox res = nvme_setup_io_queues(dev); 1690b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (res) 1691b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return res; 1692b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1693bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox mem = dma_alloc_coherent(&dev->pci_dev->dev, 8192, &dma_addr, 1694b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox GFP_KERNEL); 1695a9ef4343afbe67a6abf83c0f0294e80db48e513aKeith Busch if (!mem) 1696a9ef4343afbe67a6abf83c0f0294e80db48e513aKeith Busch return -ENOMEM; 1697b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1698bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox res = nvme_identify(dev, 0, 1, dma_addr); 1699b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (res) { 1700b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox res = -EIO; 1701cbb6218fd4ae8f98ddf0d66f0826a7a3a9c88298Keith Busch goto out; 1702b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 1703b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1704bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox ctrl = mem; 170551814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox nn = le32_to_cpup(&ctrl->nn); 17060e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch dev->oncs = le16_to_cpup(&ctrl->oncs); 170751814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox memcpy(dev->serial, ctrl->sn, sizeof(ctrl->sn)); 170851814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox memcpy(dev->model, ctrl->mn, sizeof(ctrl->mn)); 170951814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox memcpy(dev->firmware_rev, ctrl->fr, sizeof(ctrl->fr)); 1710159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch if (ctrl->mdts) 17118fc23e032debd682f5ba9fc524a5846c10d2c522Keith Busch dev->max_hw_sectors = 1 << (ctrl->mdts + shift - 9); 1712159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch if ((dev->pci_dev->vendor == PCI_VENDOR_ID_INTEL) && 1713159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch (dev->pci_dev->device == 0x0953) && ctrl->vs[3]) 1714159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch dev->stripe_size = 1 << (ctrl->vs[3] + shift); 1715b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1716bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox id_ns = mem; 17172b2c1896871838cdf549442e8ad0264be5fa74e3Matthew Wilcox for (i = 1; i <= nn; i++) { 1718bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox res = nvme_identify(dev, i, 0, dma_addr); 1719b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (res) 1720b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox continue; 1721b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1722bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox if (id_ns->ncap == 0) 1723b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox continue; 1724b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1725bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox res = nvme_get_features(dev, NVME_FEAT_LBA_RANGE, i, 172608df1e05657fc6712e520e7c09cc6c86160ceb35Keith Busch dma_addr + 4096, NULL); 1727b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (res) 1728122090366d1d5c6ec1bfb6dfdb3a6d121ff074aaKeith Busch memset(mem + 4096, 0, 4096); 1729b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1730bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox ns = nvme_alloc_ns(dev, i, mem, mem + 4096); 1731b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (ns) 1732b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox list_add_tail(&ns->list, &dev->namespaces); 1733b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 1734b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox list_for_each_entry(ns, &dev->namespaces, list) 1735b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox add_disk(ns->disk); 1736422ef0c7c81934c3344e1623d2dcfda5fa2fab8dMatthew Wilcox res = 0; 1737b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1738bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox out: 1739684f5c2025b067a23722e620d0b3b858d8dc5d01Matthew Wilcox dma_free_coherent(&dev->pci_dev->dev, 8192, mem, dma_addr); 1740b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return res; 1741b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1742b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1743b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_dev_remove(struct nvme_dev *dev) 1744b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1745b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_ns *ns, *next; 1746b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 17471fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_lock(&dev_list_lock); 17481fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox list_del(&dev->node); 17491fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_unlock(&dev_list_lock); 17501fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 1751b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox list_for_each_entry_safe(ns, next, &dev->namespaces, list) { 1752b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox list_del(&ns->list); 1753b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox del_gendisk(ns->disk); 1754b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_ns_free(ns); 1755b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 1756b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1757b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_free_queues(dev); 1758b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1759b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 1760b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1761b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1762091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcoxstatic int nvme_setup_prp_pools(struct nvme_dev *dev) 1763091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox{ 1764091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox struct device *dmadev = &dev->pci_dev->dev; 1765091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox dev->prp_page_pool = dma_pool_create("prp list page", dmadev, 1766091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox PAGE_SIZE, PAGE_SIZE, 0); 1767091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox if (!dev->prp_page_pool) 1768091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox return -ENOMEM; 1769091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox 177099802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox /* Optimisation for I/Os between 4k and 128k */ 177199802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox dev->prp_small_pool = dma_pool_create("prp list 256", dmadev, 177299802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox 256, 256, 0); 177399802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox if (!dev->prp_small_pool) { 177499802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox dma_pool_destroy(dev->prp_page_pool); 177599802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox return -ENOMEM; 177699802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox } 1777091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox return 0; 1778091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox} 1779091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox 1780091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcoxstatic void nvme_release_prp_pools(struct nvme_dev *dev) 1781091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox{ 1782091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox dma_pool_destroy(dev->prp_page_pool); 178399802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox dma_pool_destroy(dev->prp_small_pool); 1784091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox} 1785091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox 1786cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anhstatic DEFINE_IDA(nvme_instance_ida); 1787cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh 1788cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anhstatic int nvme_set_instance(struct nvme_dev *dev) 1789b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1790cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh int instance, error; 1791cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh 1792cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh do { 1793cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh if (!ida_pre_get(&nvme_instance_ida, GFP_KERNEL)) 1794cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh return -ENODEV; 1795cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh 1796cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh spin_lock(&dev_list_lock); 1797cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh error = ida_get_new(&nvme_instance_ida, &instance); 1798cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh spin_unlock(&dev_list_lock); 1799cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh } while (error == -EAGAIN); 1800cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh 1801cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh if (error) 1802cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh return -ENODEV; 1803cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh 1804cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh dev->instance = instance; 1805cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh return 0; 1806b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1807b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1808b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void nvme_release_instance(struct nvme_dev *dev) 1809b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1810cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh spin_lock(&dev_list_lock); 1811cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh ida_remove(&nvme_instance_ida, dev->instance); 1812cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh spin_unlock(&dev_list_lock); 1813b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1814b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 18155e82e952f04681c10f35e02ee0a4a43ec027137aKeith Buschstatic void nvme_free_dev(struct kref *kref) 18165e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch{ 18175e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch struct nvme_dev *dev = container_of(kref, struct nvme_dev, kref); 18185e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch nvme_dev_remove(dev); 18195e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch pci_disable_msix(dev->pci_dev); 18205e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch iounmap(dev->bar); 18215e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch nvme_release_instance(dev); 18225e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch nvme_release_prp_pools(dev); 18235e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch pci_disable_device(dev->pci_dev); 18245e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch pci_release_regions(dev->pci_dev); 18255e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch kfree(dev->queues); 18265e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch kfree(dev->entry); 18275e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch kfree(dev); 18285e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch} 18295e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch 18305e82e952f04681c10f35e02ee0a4a43ec027137aKeith Buschstatic int nvme_dev_open(struct inode *inode, struct file *f) 18315e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch{ 18325e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch struct nvme_dev *dev = container_of(f->private_data, struct nvme_dev, 18335e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch miscdev); 18345e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch kref_get(&dev->kref); 18355e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch f->private_data = dev; 18365e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch return 0; 18375e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch} 18385e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch 18395e82e952f04681c10f35e02ee0a4a43ec027137aKeith Buschstatic int nvme_dev_release(struct inode *inode, struct file *f) 18405e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch{ 18415e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch struct nvme_dev *dev = f->private_data; 18425e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch kref_put(&dev->kref, nvme_free_dev); 18435e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch return 0; 18445e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch} 18455e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch 18465e82e952f04681c10f35e02ee0a4a43ec027137aKeith Buschstatic long nvme_dev_ioctl(struct file *f, unsigned int cmd, unsigned long arg) 18475e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch{ 18485e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch struct nvme_dev *dev = f->private_data; 18495e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch switch (cmd) { 18505e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch case NVME_IOCTL_ADMIN_CMD: 18515e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch return nvme_user_admin_cmd(dev, (void __user *)arg); 18525e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch default: 18535e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch return -ENOTTY; 18545e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch } 18555e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch} 18565e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch 18575e82e952f04681c10f35e02ee0a4a43ec027137aKeith Buschstatic const struct file_operations nvme_dev_fops = { 18585e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch .owner = THIS_MODULE, 18595e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch .open = nvme_dev_open, 18605e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch .release = nvme_dev_release, 18615e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch .unlocked_ioctl = nvme_dev_ioctl, 18625e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch .compat_ioctl = nvme_dev_ioctl, 18635e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch}; 18645e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch 18658d85fce77edfc22f1d6dbf78e3af723b4b556f3dGreg Kroah-Hartmanstatic int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id) 1866b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1867574e8b95bc3780e10e9b5e9d51074d503dd3d5d9Matthew Wilcox int bars, result = -ENOMEM; 1868b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_dev *dev; 1869b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1870b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev = kzalloc(sizeof(*dev), GFP_KERNEL); 1871b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!dev) 1872b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -ENOMEM; 1873b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->entry = kcalloc(num_possible_cpus(), sizeof(*dev->entry), 1874b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox GFP_KERNEL); 1875b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!dev->entry) 1876b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto free; 18771b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox dev->queues = kcalloc(num_possible_cpus() + 1, sizeof(void *), 18781b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox GFP_KERNEL); 1879b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!dev->queues) 1880b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto free; 1881b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 18820ee5a7d7cb9309bd393a25c395f19fb12a842602Shane Michael Matthews if (pci_enable_device_mem(pdev)) 18830ee5a7d7cb9309bd393a25c395f19fb12a842602Shane Michael Matthews goto free; 1884f64d3365a3e5cb46e69db7e2c82a7cb9a5bed1b8Matthew Wilcox pci_set_master(pdev); 1885574e8b95bc3780e10e9b5e9d51074d503dd3d5d9Matthew Wilcox bars = pci_select_bars(pdev, IORESOURCE_MEM); 1886574e8b95bc3780e10e9b5e9d51074d503dd3d5d9Matthew Wilcox if (pci_request_selected_regions(pdev, bars, "nvme")) 1887574e8b95bc3780e10e9b5e9d51074d503dd3d5d9Matthew Wilcox goto disable; 18880ee5a7d7cb9309bd393a25c395f19fb12a842602Shane Michael Matthews 1889b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox INIT_LIST_HEAD(&dev->namespaces); 1890b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->pci_dev = pdev; 1891b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox pci_set_drvdata(pdev, dev); 18922930353f9f2b9e4629e935acd970cb73c1171229Matthew Wilcox dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)); 18932930353f9f2b9e4629e935acd970cb73c1171229Matthew Wilcox dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64)); 1894cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh result = nvme_set_instance(dev); 1895cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh if (result) 1896cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh goto disable; 1897cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh 189853c9577e9ca68a633c6e9df2b54eaecacfa77f62Matthew Wilcox dev->entry[0].vector = pdev->irq; 1899b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1900091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox result = nvme_setup_prp_pools(dev); 1901091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox if (result) 1902091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox goto disable_msix; 1903091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox 1904b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->bar = ioremap(pci_resource_start(pdev, 0), 8192); 1905b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!dev->bar) { 1906b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox result = -ENOMEM; 1907574e8b95bc3780e10e9b5e9d51074d503dd3d5d9Matthew Wilcox goto disable_msix; 1908b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 1909b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1910b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox result = nvme_configure_admin_queue(dev); 1911b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (result) 1912b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto unmap; 1913b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->queue_count++; 1914b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 19151fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_lock(&dev_list_lock); 19161fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox list_add(&dev->node, &dev_list); 19171fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox spin_unlock(&dev_list_lock); 19181fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 1919740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox result = nvme_dev_add(dev); 1920740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox if (result) 1921740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox goto delete; 1922740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox 19235e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch scnprintf(dev->name, sizeof(dev->name), "nvme%d", dev->instance); 19245e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch dev->miscdev.minor = MISC_DYNAMIC_MINOR; 19255e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch dev->miscdev.parent = &pdev->dev; 19265e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch dev->miscdev.name = dev->name; 19275e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch dev->miscdev.fops = &nvme_dev_fops; 19285e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch result = misc_register(&dev->miscdev); 19295e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch if (result) 19305e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch goto remove; 19315e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch 19325e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch kref_init(&dev->kref); 1933b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 1934b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 19355e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch remove: 19365e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch nvme_dev_remove(dev); 1937b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox delete: 1938740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox spin_lock(&dev_list_lock); 1939740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox list_del(&dev->node); 1940740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox spin_unlock(&dev_list_lock); 1941740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox 1942b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_free_queues(dev); 1943b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unmap: 1944b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox iounmap(dev->bar); 1945574e8b95bc3780e10e9b5e9d51074d503dd3d5d9Matthew Wilcox disable_msix: 1946b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox pci_disable_msix(pdev); 1947b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_release_instance(dev); 1948091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox nvme_release_prp_pools(dev); 1949574e8b95bc3780e10e9b5e9d51074d503dd3d5d9Matthew Wilcox disable: 19500ee5a7d7cb9309bd393a25c395f19fb12a842602Shane Michael Matthews pci_disable_device(pdev); 1951574e8b95bc3780e10e9b5e9d51074d503dd3d5d9Matthew Wilcox pci_release_regions(pdev); 1952b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox free: 1953b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(dev->queues); 1954b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(dev->entry); 1955b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(dev); 1956b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return result; 1957b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1958b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 19598d85fce77edfc22f1d6dbf78e3af723b4b556f3dGreg Kroah-Hartmanstatic void nvme_remove(struct pci_dev *pdev) 1960b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1961b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_dev *dev = pci_get_drvdata(pdev); 19625e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch misc_deregister(&dev->miscdev); 19635e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch kref_put(&dev->kref, nvme_free_dev); 1964b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1965b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1966b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* These functions are yet to be implemented */ 1967b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_error_detected NULL 1968b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_dump_registers NULL 1969b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_link_reset NULL 1970b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_slot_reset NULL 1971b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_error_resume NULL 1972b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_suspend NULL 1973b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_resume NULL 1974b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 19751d3520357df99baf4ad89f86268ac96cd38092d9Stephen Hemmingerstatic const struct pci_error_handlers nvme_err_handler = { 1976b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .error_detected = nvme_error_detected, 1977b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .mmio_enabled = nvme_dump_registers, 1978b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .link_reset = nvme_link_reset, 1979b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .slot_reset = nvme_slot_reset, 1980b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .resume = nvme_error_resume, 1981b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 1982b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1983b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* Move to pci_ids.h later */ 1984b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define PCI_CLASS_STORAGE_EXPRESS 0x010802 1985b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1986b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic DEFINE_PCI_DEVICE_TABLE(nvme_id_table) = { 1987b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox { PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) }, 1988b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox { 0, } 1989b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 1990b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew WilcoxMODULE_DEVICE_TABLE(pci, nvme_id_table); 1991b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1992b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic struct pci_driver nvme_driver = { 1993b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .name = "nvme", 1994b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .id_table = nvme_id_table, 1995b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .probe = nvme_probe, 19968d85fce77edfc22f1d6dbf78e3af723b4b556f3dGreg Kroah-Hartman .remove = nvme_remove, 1997b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .suspend = nvme_suspend, 1998b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .resume = nvme_resume, 1999b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .err_handler = &nvme_err_handler, 2000b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 2001b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2002b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int __init nvme_init(void) 2003b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 20040ac13140d796eb1e2f8956aea97a6e5e4ebcf981Matthew Wilcox int result; 20051fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox 20061fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox nvme_thread = kthread_run(nvme_kthread, NULL, "nvme"); 20071fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox if (IS_ERR(nvme_thread)) 20081fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox return PTR_ERR(nvme_thread); 2009b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 20105c42ea1643a630060f9e71e06d3933d244970967Keith Busch result = register_blkdev(nvme_major, "nvme"); 20115c42ea1643a630060f9e71e06d3933d244970967Keith Busch if (result < 0) 20121fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox goto kill_kthread; 20135c42ea1643a630060f9e71e06d3933d244970967Keith Busch else if (result > 0) 20140ac13140d796eb1e2f8956aea97a6e5e4ebcf981Matthew Wilcox nvme_major = result; 2015b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2016b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox result = pci_register_driver(&nvme_driver); 20171fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox if (result) 20181fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox goto unregister_blkdev; 20191fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox return 0; 2020b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 20211fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox unregister_blkdev: 2022b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unregister_blkdev(nvme_major, "nvme"); 20231fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox kill_kthread: 20241fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox kthread_stop(nvme_thread); 2025b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return result; 2026b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 2027b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2028b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void __exit nvme_exit(void) 2029b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 2030b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox pci_unregister_driver(&nvme_driver); 2031b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unregister_blkdev(nvme_major, "nvme"); 20321fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox kthread_stop(nvme_thread); 2033b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 2034b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 2035b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew WilcoxMODULE_AUTHOR("Matthew Wilcox <willy@linux.intel.com>"); 2036b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew WilcoxMODULE_LICENSE("GPL"); 2037366e8217e5ec6ce9f73aec19c46d983110fb4a98Matthew WilcoxMODULE_VERSION("0.8"); 2038b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxmodule_init(nvme_init); 2039b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxmodule_exit(nvme_exit); 2040