nvme-core.c revision 388f037f4e7f0a24bac6b1a24f144f5d939f58cf
1b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 2b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * NVM Express device driver 3b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Copyright (c) 2011, Intel Corporation. 4b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 5b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * This program is free software; you can redistribute it and/or modify it 6b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * under the terms and conditions of the GNU General Public License, 7b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * version 2, as published by the Free Software Foundation. 8b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 9b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * This program is distributed in the hope it will be useful, but WITHOUT 10b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * more details. 13b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 14b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * You should have received a copy of the GNU General Public License along with 15b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * this program; if not, write to the Free Software Foundation, Inc., 16b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. 17b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 18b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 19b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/nvme.h> 20b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/bio.h> 21b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/blkdev.h> 22b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/errno.h> 23b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/fs.h> 24b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/genhd.h> 25b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/init.h> 26b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/interrupt.h> 27b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/io.h> 28b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/kdev_t.h> 29b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/kernel.h> 30b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/mm.h> 31b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/module.h> 32b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/moduleparam.h> 33b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/pci.h> 34b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/sched.h> 35b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/slab.h> 36b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/types.h> 37b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/version.h> 38b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 39b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define NVME_Q_DEPTH 1024 40b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define SQ_SIZE(depth) (depth * sizeof(struct nvme_command)) 41b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define CQ_SIZE(depth) (depth * sizeof(struct nvme_completion)) 42b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define NVME_MINORS 64 43b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 44b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_major; 45b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxmodule_param(nvme_major, int, 0); 46b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 47b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 48b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Represents an NVM Express device. Each nvme_dev is a PCI function. 49b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 50b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstruct nvme_dev { 51b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue **queues; 52b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 __iomem *dbs; 53b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct pci_dev *pci_dev; 54b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int instance; 55b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int queue_count; 56b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 ctrl_config; 57b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct msix_entry *entry; 58b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_bar __iomem *bar; 59b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct list_head namespaces; 6051814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox char serial[20]; 6151814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox char model[40]; 6251814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox char firmware_rev[8]; 63b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 64b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 65b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 66b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * An NVM Express namespace is equivalent to a SCSI LUN 67b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 68b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstruct nvme_ns { 69b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct list_head list; 70b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 71b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_dev *dev; 72b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct request_queue *queue; 73b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct gendisk *disk; 74b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 75b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int ns_id; 76b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int lba_shift; 77b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 78b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 79b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 80b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * An NVM Express queue. Each device has at least two (one for admin 81b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * commands and one for I/O commands). 82b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 83b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstruct nvme_queue { 84b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct device *q_dmadev; 85b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox spinlock_t q_lock; 86b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_command *sq_cmds; 87b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox volatile struct nvme_completion *cqes; 88b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_addr_t sq_dma_addr; 89b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_addr_t cq_dma_addr; 90b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox wait_queue_head_t sq_full; 91b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct bio_list sq_cong; 92b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 __iomem *q_db; 93b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 q_depth; 94b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 cq_vector; 95b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 sq_head; 96b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 sq_tail; 97b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 cq_head; 98821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox u16 cq_phase; 99b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unsigned long cmdid_data[]; 100b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 101b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 102b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 103b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Check we didin't inadvertently grow the command struct 104b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 105b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic inline void _nvme_check_size(void) 106b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 107b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64); 108b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_create_cq) != 64); 109b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_create_sq) != 64); 110b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64); 111b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_features) != 64); 112b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_command) != 64); 113b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != 4096); 114b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_id_ns) != 4096); 115b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64); 116b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 117b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 118b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/** 119b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * alloc_cmdid - Allocate a Command ID 120b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * @param nvmeq The queue that will be used for this command 121b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * @param ctx A pointer that will be passed to the handler 122b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * @param handler The ID of the handler to call 123b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 124b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Allocate a Command ID for a queue. The data passed in will 125b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * be passed to the completion handler. This is implemented by using 126b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * the bottom two bits of the ctx pointer to store the handler ID. 127b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Passing in a pointer that's not 4-byte aligned will cause a BUG. 128b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * We can change this if it becomes a problem. 129b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 130b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int alloc_cmdid(struct nvme_queue *nvmeq, void *ctx, int handler) 131b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 132b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int depth = nvmeq->q_depth; 133b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unsigned long data = (unsigned long)ctx | handler; 134b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int cmdid; 135b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 136b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox BUG_ON((unsigned long)ctx & 3); 137b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 138b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox do { 139b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cmdid = find_first_zero_bit(nvmeq->cmdid_data, depth); 140b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (cmdid >= depth) 141b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -EBUSY; 142b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } while (test_and_set_bit(cmdid, nvmeq->cmdid_data)); 143b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 144b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->cmdid_data[cmdid + BITS_TO_LONGS(depth)] = data; 145b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return cmdid; 146b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 147b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 148b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int alloc_cmdid_killable(struct nvme_queue *nvmeq, void *ctx, 149b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int handler) 150b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 151b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int cmdid; 152b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox wait_event_killable(nvmeq->sq_full, 153b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox (cmdid = alloc_cmdid(nvmeq, ctx, handler)) >= 0); 154b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return (cmdid < 0) ? -EINTR : cmdid; 155b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 156b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 157b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* If you need more than four handlers, you'll need to change how 158b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * alloc_cmdid and nvme_process_cq work 159b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 160b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxenum { 161b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox sync_completion_id = 0, 162b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox bio_completion_id, 163b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 164b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 165b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic unsigned long free_cmdid(struct nvme_queue *nvmeq, int cmdid) 166b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 167b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unsigned long data; 168b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 169b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox data = nvmeq->cmdid_data[cmdid + BITS_TO_LONGS(nvmeq->q_depth)]; 170b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox clear_bit(cmdid, nvmeq->cmdid_data); 171b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox wake_up(&nvmeq->sq_full); 172b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return data; 173b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 174b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 175b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic struct nvme_queue *get_nvmeq(struct nvme_ns *ns) 176b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1771b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox int qid, cpu = get_cpu(); 1781b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox if (cpu < ns->dev->queue_count) 1791b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox qid = cpu + 1; 1801b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox else 1811b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox qid = (cpu % rounddown_pow_of_two(ns->dev->queue_count)) + 1; 1821b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox return ns->dev->queues[qid]; 183b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 184b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 185b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void put_nvmeq(struct nvme_queue *nvmeq) 186b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1871b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox put_cpu(); 188b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 189b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 190b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/** 191b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * nvme_submit_cmd: Copy a command into a queue and ring the doorbell 192b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * @nvmeq: The queue to use 193b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * @cmd: The command to send 194b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 195b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Safe to use from interrupt context 196b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 197b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_submit_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd) 198b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 199b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unsigned long flags; 200b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 tail; 201b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox /* XXX: Need to check tail isn't going to overrun head */ 202b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox spin_lock_irqsave(&nvmeq->q_lock, flags); 203b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox tail = nvmeq->sq_tail; 204b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memcpy(&nvmeq->sq_cmds[tail], cmd, sizeof(*cmd)); 205b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox writel(tail, nvmeq->q_db); 206b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (++tail == nvmeq->q_depth) 207b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox tail = 0; 208b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->sq_tail = tail; 209b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox spin_unlock_irqrestore(&nvmeq->q_lock, flags); 210b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 211b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 212b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 213b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 214b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstruct nvme_req_info { 215b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct bio *bio; 216b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int nents; 217b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct scatterlist sg[0]; 218b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 219b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 220b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* XXX: use a mempool */ 221b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic struct nvme_req_info *alloc_info(unsigned nseg, gfp_t gfp) 222b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 223b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return kmalloc(sizeof(struct nvme_req_info) + 224b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox sizeof(struct scatterlist) * nseg, gfp); 225b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 226b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 227b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void free_info(struct nvme_req_info *info) 228b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 229b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(info); 230b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 231b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 232b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void bio_completion(struct nvme_queue *nvmeq, void *ctx, 233b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_completion *cqe) 234b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 235b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_req_info *info = ctx; 236b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct bio *bio = info->bio; 237b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 status = le16_to_cpup(&cqe->status) >> 1; 238b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 239b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_unmap_sg(nvmeq->q_dmadev, info->sg, info->nents, 240b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox bio_data_dir(bio) ? DMA_TO_DEVICE : DMA_FROM_DEVICE); 241b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox free_info(info); 242b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox bio_endio(bio, status ? -EIO : 0); 243b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 244b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 245ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox/* length is in bytes */ 246ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcoxstatic void nvme_setup_prps(struct nvme_common_command *cmd, 247ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox struct scatterlist *sg, int length) 248ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox{ 249ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox int dma_len = sg_dma_len(sg); 250ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox u64 dma_addr = sg_dma_address(sg); 251ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox int offset = offset_in_page(dma_addr); 252ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox 253ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmd->prp1 = cpu_to_le64(dma_addr); 254ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox length -= (PAGE_SIZE - offset); 255ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox if (length <= 0) 256ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox return; 257ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox 258ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox dma_len -= (PAGE_SIZE - offset); 259ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox if (dma_len) { 260ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox dma_addr += (PAGE_SIZE - offset); 261ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox } else { 262ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox sg = sg_next(sg); 263ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox dma_addr = sg_dma_address(sg); 264ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox dma_len = sg_dma_len(sg); 265ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox } 266ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox 267ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox if (length <= PAGE_SIZE) { 268ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmd->prp2 = cpu_to_le64(dma_addr); 269ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox return; 270ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox } 271ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox 272ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox /* XXX: support PRP lists */ 273ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox} 274ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox 275b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_map_bio(struct device *dev, struct nvme_req_info *info, 276b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct bio *bio, enum dma_data_direction dma_dir, int psegs) 277b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 278b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct bio_vec *bvec; 279b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct scatterlist *sg = info->sg; 280b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int i, nsegs; 281b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 282b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox sg_init_table(sg, psegs); 283b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox bio_for_each_segment(bvec, bio, i) { 284b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox sg_set_page(sg, bvec->bv_page, bvec->bv_len, bvec->bv_offset); 285b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox /* XXX: handle non-mergable here */ 286b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nsegs++; 287b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 288b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox info->nents = nsegs; 289b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 290b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return dma_map_sg(dev, info->sg, info->nents, dma_dir); 291b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 292b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 293b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_submit_bio_queue(struct nvme_queue *nvmeq, struct nvme_ns *ns, 294b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct bio *bio) 295b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 296ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox struct nvme_command *cmnd; 297b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_req_info *info; 298b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox enum dma_data_direction dma_dir; 299b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int cmdid; 300b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u16 control; 301b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 dsmgmt; 302b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unsigned long flags; 303b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int psegs = bio_phys_segments(ns->queue, bio); 304b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 305b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox info = alloc_info(psegs, GFP_NOIO); 306b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!info) 307b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto congestion; 308b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox info->bio = bio; 309b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 310b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cmdid = alloc_cmdid(nvmeq, info, bio_completion_id); 311b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (unlikely(cmdid < 0)) 312b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto free_info; 313b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 314b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox control = 0; 315b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (bio->bi_rw & REQ_FUA) 316b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox control |= NVME_RW_FUA; 317b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (bio->bi_rw & (REQ_FAILFAST_DEV | REQ_RAHEAD)) 318b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox control |= NVME_RW_LR; 319b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 320b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dsmgmt = 0; 321b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (bio->bi_rw & REQ_RAHEAD) 322b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH; 323b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 324b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox spin_lock_irqsave(&nvmeq->q_lock, flags); 325ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail]; 326b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 327b8deb62cf271fa9381edc8cf52bcae2f0225c55aMatthew Wilcox memset(cmnd, 0, sizeof(*cmnd)); 328b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (bio_data_dir(bio)) { 329ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.opcode = nvme_cmd_write; 330b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_dir = DMA_TO_DEVICE; 331b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } else { 332ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.opcode = nvme_cmd_read; 333b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_dir = DMA_FROM_DEVICE; 334b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 335b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 336b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_map_bio(nvmeq->q_dmadev, info, bio, dma_dir, psegs); 337b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 338ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.flags = 1; 339ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.command_id = cmdid; 340ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.nsid = cpu_to_le32(ns->ns_id); 341ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox nvme_setup_prps(&cmnd->common, info->sg, bio->bi_size); 342ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.slba = cpu_to_le64(bio->bi_sector >> (ns->lba_shift - 9)); 343ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.length = cpu_to_le16((bio->bi_size >> ns->lba_shift) - 1); 344ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.control = cpu_to_le16(control); 345ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt); 346b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 347b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox writel(nvmeq->sq_tail, nvmeq->q_db); 348b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (++nvmeq->sq_tail == nvmeq->q_depth) 349b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->sq_tail = 0; 350b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 351b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox spin_unlock_irqrestore(&nvmeq->q_lock, flags); 352b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 353b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 354b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 355b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox free_info: 356b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox free_info(info); 357b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox congestion: 358b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -EBUSY; 359b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 360b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 361b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 362b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * NB: return value of non-zero would mean that we were a stacking driver. 363b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * make_request must always succeed. 364b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 365b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_make_request(struct request_queue *q, struct bio *bio) 366b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 367b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_ns *ns = q->queuedata; 368b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq = get_nvmeq(ns); 369b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 370b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (nvme_submit_bio_queue(nvmeq, ns, bio)) { 371b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox blk_set_queue_congested(q, rw_is_sync(bio->bi_rw)); 372b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox bio_list_add(&nvmeq->sq_cong, bio); 373b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 374b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox put_nvmeq(nvmeq); 375b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 376b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 377b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 378b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 379b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstruct sync_cmd_info { 380b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct task_struct *task; 381b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 result; 382b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int status; 383b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 384b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 385b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void sync_completion(struct nvme_queue *nvmeq, void *ctx, 386b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_completion *cqe) 387b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 388b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct sync_cmd_info *cmdinfo = ctx; 389b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cmdinfo->result = le32_to_cpup(&cqe->result); 390b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cmdinfo->status = le16_to_cpup(&cqe->status) >> 1; 391b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox wake_up_process(cmdinfo->task); 392b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 393b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 394b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxtypedef void (*completion_fn)(struct nvme_queue *, void *, 395b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_completion *); 396b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 397b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic irqreturn_t nvme_process_cq(struct nvme_queue *nvmeq) 398b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 399821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox u16 head, phase; 400b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 401b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox static const completion_fn completions[4] = { 402b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox [sync_completion_id] = sync_completion, 403b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox [bio_completion_id] = bio_completion, 404b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox }; 405b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 406b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox head = nvmeq->cq_head; 407821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox phase = nvmeq->cq_phase; 408b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 409b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox for (;;) { 410b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unsigned long data; 411b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox void *ptr; 412b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unsigned char handler; 413b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_completion cqe = nvmeq->cqes[head]; 414821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox if ((le16_to_cpu(cqe.status) & 1) != phase) 415b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox break; 416b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->sq_head = le16_to_cpu(cqe.sq_head); 417b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (++head == nvmeq->q_depth) { 418b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox head = 0; 419821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox phase = !phase; 420b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 421b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 422b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox data = free_cmdid(nvmeq, cqe.command_id); 423b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox handler = data & 3; 424b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ptr = (void *)(data & ~3UL); 425b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox completions[handler](nvmeq, ptr, &cqe); 426b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 427b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 428b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox /* If the controller ignores the cq head doorbell and continuously 429b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * writes to the queue, it is theoretically possible to wrap around 430b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * the queue twice and mistakenly return IRQ_NONE. Linux only 431b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * requires that 0.1% of your interrupts are handled, so this isn't 432b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * a big problem. 433b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 434821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox if (head == nvmeq->cq_head && phase == nvmeq->cq_phase) 435b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return IRQ_NONE; 436b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 437b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox writel(head, nvmeq->q_db + 1); 438b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->cq_head = head; 439821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox nvmeq->cq_phase = phase; 440b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 441b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return IRQ_HANDLED; 442b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 443b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 444b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic irqreturn_t nvme_irq(int irq, void *data) 445b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 446b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return nvme_process_cq(data); 447b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 448b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 449b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* 450b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Returns 0 on success. If the result is negative, it's a Linux error code; 451b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * if the result is positive, it's an NVM Express status code 452b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */ 453b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_submit_sync_cmd(struct nvme_queue *q, struct nvme_command *cmd, 454b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 *result) 455b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 456b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int cmdid; 457b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct sync_cmd_info cmdinfo; 458b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 459b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cmdinfo.task = current; 460b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cmdinfo.status = -EINTR; 461b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 462b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cmdid = alloc_cmdid_killable(q, &cmdinfo, sync_completion_id); 463b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (cmdid < 0) 464b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return cmdid; 465b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cmd->common.command_id = cmdid; 466b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 467b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox set_current_state(TASK_UNINTERRUPTIBLE); 468b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_submit_cmd(q, cmd); 469b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox schedule(); 470b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 471b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (result) 472b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox *result = cmdinfo.result; 473b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 474b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return cmdinfo.status; 475b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 476b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 477b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_submit_admin_cmd(struct nvme_dev *dev, struct nvme_command *cmd, 478b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 *result) 479b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 480b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return nvme_submit_sync_cmd(dev->queues[0], cmd, result); 481b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 482b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 483b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id) 484b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 485b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int status; 486b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_command c; 487b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 488b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset(&c, 0, sizeof(c)); 489b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.delete_queue.opcode = opcode; 490b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.delete_queue.qid = cpu_to_le16(id); 491b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 492b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox status = nvme_submit_admin_cmd(dev, &c, NULL); 493b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (status) 494b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -EIO; 495b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 496b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 497b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 498b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_alloc_cq(struct nvme_dev *dev, u16 qid, 499b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq) 500b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 501b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int status; 502b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_command c; 503b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int flags = NVME_QUEUE_PHYS_CONTIG | NVME_CQ_IRQ_ENABLED; 504b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 505b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset(&c, 0, sizeof(c)); 506b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.opcode = nvme_admin_create_cq; 507b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.prp1 = cpu_to_le64(nvmeq->cq_dma_addr); 508b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.cqid = cpu_to_le16(qid); 509b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1); 510b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.cq_flags = cpu_to_le16(flags); 511b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_cq.irq_vector = cpu_to_le16(nvmeq->cq_vector); 512b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 513b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox status = nvme_submit_admin_cmd(dev, &c, NULL); 514b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (status) 515b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -EIO; 516b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 517b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 518b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 519b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_alloc_sq(struct nvme_dev *dev, u16 qid, 520b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq) 521b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 522b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int status; 523b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_command c; 524b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int flags = NVME_QUEUE_PHYS_CONTIG | NVME_SQ_PRIO_MEDIUM; 525b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 526b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset(&c, 0, sizeof(c)); 527b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.opcode = nvme_admin_create_sq; 528b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.prp1 = cpu_to_le64(nvmeq->sq_dma_addr); 529b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.sqid = cpu_to_le16(qid); 530b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1); 531b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.sq_flags = cpu_to_le16(flags); 532b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.create_sq.cqid = cpu_to_le16(qid); 533b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 534b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox status = nvme_submit_admin_cmd(dev, &c, NULL); 535b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (status) 536b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -EIO; 537b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 538b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 539b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 540b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_delete_cq(struct nvme_dev *dev, u16 cqid) 541b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 542b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return adapter_delete_queue(dev, nvme_admin_delete_cq, cqid); 543b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 544b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 545b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_delete_sq(struct nvme_dev *dev, u16 sqid) 546b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 547b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return adapter_delete_queue(dev, nvme_admin_delete_sq, sqid); 548b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 549b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 550b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void nvme_free_queue(struct nvme_dev *dev, int qid) 551b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 552b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq = dev->queues[qid]; 553b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 554b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox free_irq(dev->entry[nvmeq->cq_vector].vector, nvmeq); 555b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 556b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox /* Don't tell the adapter to delete the admin queue */ 557b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (qid) { 558b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox adapter_delete_sq(dev, qid); 559b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox adapter_delete_cq(dev, qid); 560b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 561b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 562b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth), 563b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox (void *)nvmeq->cqes, nvmeq->cq_dma_addr); 564b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth), 565b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->sq_cmds, nvmeq->sq_dma_addr); 566b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(nvmeq); 567b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 568b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 569b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev, int qid, 570b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int depth, int vector) 571b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 572b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct device *dmadev = &dev->pci_dev->dev; 573b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unsigned extra = (depth + BITS_TO_LONGS(depth)) * sizeof(long); 574b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq = kzalloc(sizeof(*nvmeq) + extra, GFP_KERNEL); 575b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!nvmeq) 576b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 577b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 578b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->cqes = dma_alloc_coherent(dmadev, CQ_SIZE(depth), 579b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox &nvmeq->cq_dma_addr, GFP_KERNEL); 580b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!nvmeq->cqes) 581b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto free_nvmeq; 582b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset((void *)nvmeq->cqes, 0, CQ_SIZE(depth)); 583b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 584b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->sq_cmds = dma_alloc_coherent(dmadev, SQ_SIZE(depth), 585b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox &nvmeq->sq_dma_addr, GFP_KERNEL); 586b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!nvmeq->sq_cmds) 587b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto free_cqdma; 588b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 589b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->q_dmadev = dmadev; 590b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox spin_lock_init(&nvmeq->q_lock); 591b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->cq_head = 0; 592821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox nvmeq->cq_phase = 1; 593b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox init_waitqueue_head(&nvmeq->sq_full); 594b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox bio_list_init(&nvmeq->sq_cong); 595b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->q_db = &dev->dbs[qid * 2]; 596b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->q_depth = depth; 597b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->cq_vector = vector; 598b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 599b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return nvmeq; 600b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 601b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox free_cqdma: 602b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_free_coherent(dmadev, CQ_SIZE(nvmeq->q_depth), (void *)nvmeq->cqes, 603b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->cq_dma_addr); 604b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox free_nvmeq: 605b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(nvmeq); 606b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 607b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 608b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 6093001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcoxstatic int queue_request_irq(struct nvme_dev *dev, struct nvme_queue *nvmeq, 6103001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox const char *name) 6113001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox{ 6123001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox return request_irq(dev->entry[nvmeq->cq_vector].vector, nvme_irq, 6133001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox IRQF_DISABLED | IRQF_SHARED, name, nvmeq); 6143001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox} 6153001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox 616b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic __devinit struct nvme_queue *nvme_create_queue(struct nvme_dev *dev, 617b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int qid, int cq_size, int vector) 618b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 619b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int result; 620b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq = nvme_alloc_queue(dev, qid, cq_size, vector); 621b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 6223f85d50b609e8a5ef151656210203a6e94c19538Matthew Wilcox if (!nvmeq) 6233f85d50b609e8a5ef151656210203a6e94c19538Matthew Wilcox return NULL; 6243f85d50b609e8a5ef151656210203a6e94c19538Matthew Wilcox 625b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox result = adapter_alloc_cq(dev, qid, nvmeq); 626b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (result < 0) 627b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto free_nvmeq; 628b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 629b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox result = adapter_alloc_sq(dev, qid, nvmeq); 630b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (result < 0) 631b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto release_cq; 632b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 6333001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox result = queue_request_irq(dev, nvmeq, "nvme"); 634b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (result < 0) 635b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto release_sq; 636b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 637b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return nvmeq; 638b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 639b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox release_sq: 640b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox adapter_delete_sq(dev, qid); 641b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox release_cq: 642b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox adapter_delete_cq(dev, qid); 643b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox free_nvmeq: 644b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth), 645b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox (void *)nvmeq->cqes, nvmeq->cq_dma_addr); 646b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth), 647b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq->sq_cmds, nvmeq->sq_dma_addr); 648b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(nvmeq); 649b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 650b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 651b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 652b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int __devinit nvme_configure_admin_queue(struct nvme_dev *dev) 653b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 654b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int result; 655b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 aqa; 656b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_queue *nvmeq; 657b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 658b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->dbs = ((void __iomem *)dev->bar) + 4096; 659b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 660b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvmeq = nvme_alloc_queue(dev, 0, 64, 0); 6613f85d50b609e8a5ef151656210203a6e94c19538Matthew Wilcox if (!nvmeq) 6623f85d50b609e8a5ef151656210203a6e94c19538Matthew Wilcox return -ENOMEM; 663b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 664b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox aqa = nvmeq->q_depth - 1; 665b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox aqa |= aqa << 16; 666b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 667b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->ctrl_config = NVME_CC_ENABLE | NVME_CC_CSS_NVM; 668b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->ctrl_config |= (PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT; 669b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->ctrl_config |= NVME_CC_ARB_RR | NVME_CC_SHN_NONE; 670b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 6715911f20039ce59d7e7834f0c42151cf759b6f786Shane Michael Matthews writel(0, &dev->bar->cc); 672b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox writel(aqa, &dev->bar->aqa); 673b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox writeq(nvmeq->sq_dma_addr, &dev->bar->asq); 674b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox writeq(nvmeq->cq_dma_addr, &dev->bar->acq); 675b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox writel(dev->ctrl_config, &dev->bar->cc); 676b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 677b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox while (!(readl(&dev->bar->csts) & NVME_CSTS_RDY)) { 678b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox msleep(100); 679b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (fatal_signal_pending(current)) 680b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -EINTR; 681b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 682b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 6833001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox result = queue_request_irq(dev, nvmeq, "nvme admin"); 684b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->queues[0] = nvmeq; 685b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return result; 686b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 687b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 6887fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcoxstatic int nvme_map_user_pages(struct nvme_dev *dev, int write, 6897fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox unsigned long addr, unsigned length, 6907fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox struct scatterlist **sgp) 691b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 69236c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox int i, err, count, nents, offset; 6937fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox struct scatterlist *sg; 6947fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox struct page **pages; 69536c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox 69636c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox if (addr & 3) 69736c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox return -EINVAL; 6987fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox if (!length) 6997fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox return -EINVAL; 7007fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox 70136c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox offset = offset_in_page(addr); 7027fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox count = DIV_ROUND_UP(offset + length, PAGE_SIZE); 7037fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox pages = kcalloc(count, sizeof(*pages), GFP_KERNEL); 70436c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox 70536c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox err = get_user_pages_fast(addr, count, 1, pages); 70636c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox if (err < count) { 70736c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox count = err; 70836c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox err = -EFAULT; 70936c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox goto put_pages; 71036c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox } 7117fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox 7127fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox sg = kcalloc(count, sizeof(*sg), GFP_KERNEL); 71336c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox sg_init_table(sg, count); 714ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox sg_set_page(&sg[0], pages[0], PAGE_SIZE - offset, offset); 7157fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox length -= (PAGE_SIZE - offset); 7167fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox for (i = 1; i < count; i++) { 7177fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox sg_set_page(&sg[i], pages[i], min_t(int, length, PAGE_SIZE), 0); 7187fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox length -= PAGE_SIZE; 7197fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox } 7207fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox 7217fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox err = -ENOMEM; 7227fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox nents = dma_map_sg(&dev->pci_dev->dev, sg, count, 7237fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox write ? DMA_TO_DEVICE : DMA_FROM_DEVICE); 72436c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox if (!nents) 72536c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox goto put_pages; 726b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 7277fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox kfree(pages); 7287fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox *sgp = sg; 7297fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox return nents; 730b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 7317fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox put_pages: 7327fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox for (i = 0; i < count; i++) 7337fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox put_page(pages[i]); 7347fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox kfree(pages); 7357fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox return err; 7367fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox} 737b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 7387fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcoxstatic void nvme_unmap_user_pages(struct nvme_dev *dev, int write, 7397fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox unsigned long addr, int length, 7407fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox struct scatterlist *sg, int nents) 7417fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox{ 7427fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox int i, count; 743b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 7447fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox count = DIV_ROUND_UP(offset_in_page(addr) + length, PAGE_SIZE); 74536c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox dma_unmap_sg(&dev->pci_dev->dev, sg, nents, DMA_FROM_DEVICE); 7467fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox 74736c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox for (i = 0; i < count; i++) 7487fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox put_page(sg_page(&sg[i])); 7497fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox} 750b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 7517fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcoxstatic int nvme_submit_user_admin_command(struct nvme_dev *dev, 7527fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox unsigned long addr, unsigned length, 7537fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox struct nvme_command *cmd) 7547fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox{ 7557fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox int err, nents; 7567fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox struct scatterlist *sg; 7577fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox 7587fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox nents = nvme_map_user_pages(dev, 0, addr, length, &sg); 7597fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox if (nents < 0) 7607fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox return nents; 7617fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox nvme_setup_prps(&cmd->common, sg, length); 7627fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox err = nvme_submit_admin_cmd(dev, cmd, NULL); 7637fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox nvme_unmap_user_pages(dev, 0, addr, length, sg, nents); 7647fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox return err ? -EIO : 0; 765b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 766b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 767bd38c5557cf482fc195e2264b32ea62eed60730aMatthew Wilcoxstatic int nvme_identify(struct nvme_ns *ns, unsigned long addr, int cns) 768b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 769b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_command c; 770b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 771bd38c5557cf482fc195e2264b32ea62eed60730aMatthew Wilcox memset(&c, 0, sizeof(c)); 772bd38c5557cf482fc195e2264b32ea62eed60730aMatthew Wilcox c.identify.opcode = nvme_admin_identify; 773bd38c5557cf482fc195e2264b32ea62eed60730aMatthew Wilcox c.identify.nsid = cns ? 0 : cpu_to_le32(ns->ns_id); 774bd38c5557cf482fc195e2264b32ea62eed60730aMatthew Wilcox c.identify.cns = cpu_to_le32(cns); 775bd38c5557cf482fc195e2264b32ea62eed60730aMatthew Wilcox 776bd38c5557cf482fc195e2264b32ea62eed60730aMatthew Wilcox return nvme_submit_user_admin_command(ns->dev, addr, 4096, &c); 777bd38c5557cf482fc195e2264b32ea62eed60730aMatthew Wilcox} 778bd38c5557cf482fc195e2264b32ea62eed60730aMatthew Wilcox 779bd38c5557cf482fc195e2264b32ea62eed60730aMatthew Wilcoxstatic int nvme_get_range_type(struct nvme_ns *ns, unsigned long addr) 780bd38c5557cf482fc195e2264b32ea62eed60730aMatthew Wilcox{ 781bd38c5557cf482fc195e2264b32ea62eed60730aMatthew Wilcox struct nvme_command c; 782b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 783b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset(&c, 0, sizeof(c)); 784b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.features.opcode = nvme_admin_get_features; 785b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.features.nsid = cpu_to_le32(ns->ns_id); 786b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.features.fid = cpu_to_le32(NVME_FEAT_LBA_RANGE); 787b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 788bd38c5557cf482fc195e2264b32ea62eed60730aMatthew Wilcox return nvme_submit_user_admin_command(ns->dev, addr, 4096, &c); 789b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 790b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 791a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcoxstatic int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio) 792a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox{ 793a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox struct nvme_dev *dev = ns->dev; 794a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox struct nvme_queue *nvmeq; 795a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox struct nvme_user_io io; 796a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox struct nvme_command c; 797a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox unsigned length; 798a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox u32 result; 799a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox int nents, status; 800a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox struct scatterlist *sg; 801a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox 802a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox if (copy_from_user(&io, uio, sizeof(io))) 803a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox return -EFAULT; 804a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox length = io.nblocks << io.block_shift; 805a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox nents = nvme_map_user_pages(dev, io.opcode & 1, io.addr, length, &sg); 806a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox if (nents < 0) 807a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox return nents; 808a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox 809a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox memset(&c, 0, sizeof(c)); 810a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.opcode = io.opcode; 811a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.flags = io.flags; 812a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.nsid = cpu_to_le32(io.nsid); 813a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.slba = cpu_to_le64(io.slba); 814a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.length = cpu_to_le16(io.nblocks - 1); 815a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.control = cpu_to_le16(io.control); 816a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.dsmgmt = cpu_to_le16(io.dsmgmt); 817a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.reftag = cpu_to_le32(io.reftag); /* XXX: endian? */ 818a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.apptag = cpu_to_le16(io.apptag); 819a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox c.rw.appmask = cpu_to_le16(io.appmask); 820a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox /* XXX: metadata */ 821a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox nvme_setup_prps(&c.common, sg, length); 822a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox 823a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox nvmeq = get_nvmeq(ns); 824a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox status = nvme_submit_sync_cmd(nvmeq, &c, &result); 825a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox put_nvmeq(nvmeq); 826a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox 827a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox nvme_unmap_user_pages(dev, io.opcode & 1, io.addr, length, sg, nents); 828a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox put_user(result, &uio->result); 829a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox return status; 830a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox} 831a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox 832b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, 833b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unsigned long arg) 834b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 835b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_ns *ns = bdev->bd_disk->private_data; 836b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 837b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox switch (cmd) { 838b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox case NVME_IOCTL_IDENTIFY_NS: 83936c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox return nvme_identify(ns, arg, 0); 840b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox case NVME_IOCTL_IDENTIFY_CTRL: 84136c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox return nvme_identify(ns, arg, 1); 842b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox case NVME_IOCTL_GET_RANGE_TYPE: 843bd38c5557cf482fc195e2264b32ea62eed60730aMatthew Wilcox return nvme_get_range_type(ns, arg); 844a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox case NVME_IOCTL_SUBMIT_IO: 845a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox return nvme_submit_io(ns, (void __user *)arg); 846b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox default: 847b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -ENOTTY; 848b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 849b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 850b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 851b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic const struct block_device_operations nvme_fops = { 852b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .owner = THIS_MODULE, 853b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .ioctl = nvme_ioctl, 854b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 855b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 856b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic struct nvme_ns *nvme_alloc_ns(struct nvme_dev *dev, int index, 857b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_id_ns *id, struct nvme_lba_range_type *rt) 858b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 859b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_ns *ns; 860b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct gendisk *disk; 861b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int lbaf; 862b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 863b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (rt->attributes & NVME_LBART_ATTRIB_HIDE) 864b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 865b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 866b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns = kzalloc(sizeof(*ns), GFP_KERNEL); 867b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!ns) 868b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 869b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->queue = blk_alloc_queue(GFP_KERNEL); 870b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!ns->queue) 871b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto out_free_ns; 872b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->queue->queue_flags = QUEUE_FLAG_DEFAULT | QUEUE_FLAG_NOMERGES | 873b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox QUEUE_FLAG_NONROT | QUEUE_FLAG_DISCARD; 874b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox blk_queue_make_request(ns->queue, nvme_make_request); 875b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->dev = dev; 876b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->queue->queuedata = ns; 877b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 878b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk = alloc_disk(NVME_MINORS); 879b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!disk) 880b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto out_free_queue; 881b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->ns_id = index; 882b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->disk = disk; 883b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox lbaf = id->flbas & 0xf; 884b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns->lba_shift = id->lbaf[lbaf].ds; 885b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 886b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk->major = nvme_major; 887b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk->minors = NVME_MINORS; 888b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk->first_minor = NVME_MINORS * index; 889b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk->fops = &nvme_fops; 890b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk->private_data = ns; 891b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox disk->queue = ns->queue; 892388f037f4e7f0a24bac6b1a24f144f5d939f58cfMatthew Wilcox disk->driverfs_dev = &dev->pci_dev->dev; 893b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox sprintf(disk->disk_name, "nvme%dn%d", dev->instance, index); 894b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox set_capacity(disk, le64_to_cpup(&id->nsze) << (ns->lba_shift - 9)); 895b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 896b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return ns; 897b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 898b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox out_free_queue: 899b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox blk_cleanup_queue(ns->queue); 900b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox out_free_ns: 901b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(ns); 902b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return NULL; 903b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 904b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 905b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void nvme_ns_free(struct nvme_ns *ns) 906b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 907b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox put_disk(ns->disk); 908b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox blk_cleanup_queue(ns->queue); 909b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(ns); 910b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 911b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 912b3b06812e199f248561ce7824a4a8a9cd573c05aMatthew Wilcoxstatic int set_queue_count(struct nvme_dev *dev, int count) 913b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 914b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int status; 915b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox u32 result; 916b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_command c; 917b3b06812e199f248561ce7824a4a8a9cd573c05aMatthew Wilcox u32 q_count = (count - 1) | ((count - 1) << 16); 918b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 919b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset(&c, 0, sizeof(c)); 920b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.features.opcode = nvme_admin_get_features; 921b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.features.fid = cpu_to_le32(NVME_FEAT_NUM_QUEUES); 922b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox c.features.dword11 = cpu_to_le32(q_count); 923b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 924b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox status = nvme_submit_admin_cmd(dev, &c, &result); 925b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (status) 926b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -EIO; 927b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return min(result & 0xffff, result >> 16) + 1; 928b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 929b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 930b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int __devinit nvme_setup_io_queues(struct nvme_dev *dev) 931b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 9321b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox int result, cpu, i, nr_queues; 933b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 9341b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox nr_queues = num_online_cpus(); 9351b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox result = set_queue_count(dev, nr_queues); 9361b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox if (result < 0) 9371b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox return result; 9381b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox if (result < nr_queues) 9391b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox nr_queues = result; 940b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 9411b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox /* Deregister the admin queue's interrupt */ 9421b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox free_irq(dev->entry[0].vector, dev->queues[0]); 9431b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox 9441b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox for (i = 0; i < nr_queues; i++) 9451b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox dev->entry[i].entry = i; 9461b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox for (;;) { 9471b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox result = pci_enable_msix(dev->pci_dev, dev->entry, nr_queues); 9481b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox if (result == 0) { 9491b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox break; 9501b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox } else if (result > 0) { 9511b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox nr_queues = result; 9521b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox continue; 9531b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox } else { 9541b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox nr_queues = 1; 9551b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox break; 9561b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox } 9571b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox } 9581b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox 9591b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox result = queue_request_irq(dev, dev->queues[0], "nvme admin"); 9601b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox /* XXX: handle failure here */ 9611b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox 9621b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox cpu = cpumask_first(cpu_online_mask); 9631b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox for (i = 0; i < nr_queues; i++) { 9641b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox irq_set_affinity_hint(dev->entry[i].vector, get_cpu_mask(cpu)); 9651b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox cpu = cpumask_next(cpu, cpu_online_mask); 9661b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox } 9671b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox 9681b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox for (i = 0; i < nr_queues; i++) { 9691b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox dev->queues[i + 1] = nvme_create_queue(dev, i + 1, 9701b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox NVME_Q_DEPTH, i); 9711b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox if (!dev->queues[i + 1]) 9721b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox return -ENOMEM; 9731b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox dev->queue_count++; 9741b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox } 975b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 976b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 977b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 978b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 979b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void nvme_free_queues(struct nvme_dev *dev) 980b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 981b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int i; 982b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 983b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox for (i = dev->queue_count - 1; i >= 0; i--) 984b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_free_queue(dev, i); 985b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 986b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 987b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int __devinit nvme_dev_add(struct nvme_dev *dev) 988b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 989b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int res, nn, i; 990b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_ns *ns, *next; 99151814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox struct nvme_id_ctrl *ctrl; 992b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox void *id; 993b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_addr_t dma_addr; 994b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_command cid, crt; 995b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 996b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox res = nvme_setup_io_queues(dev); 997b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (res) 998b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return res; 999b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1000b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox /* XXX: Switch to a SG list once prp2 works */ 1001b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox id = dma_alloc_coherent(&dev->pci_dev->dev, 8192, &dma_addr, 1002b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox GFP_KERNEL); 1003b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1004b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset(&cid, 0, sizeof(cid)); 1005b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cid.identify.opcode = nvme_admin_identify; 1006b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cid.identify.nsid = 0; 1007b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cid.identify.prp1 = cpu_to_le64(dma_addr); 1008b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cid.identify.cns = cpu_to_le32(1); 1009b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1010b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox res = nvme_submit_admin_cmd(dev, &cid, NULL); 1011b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (res) { 1012b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox res = -EIO; 1013b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto out_free; 1014b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 1015b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 101651814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox ctrl = id; 101751814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox nn = le32_to_cpup(&ctrl->nn); 101851814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox memcpy(dev->serial, ctrl->sn, sizeof(ctrl->sn)); 101951814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox memcpy(dev->model, ctrl->mn, sizeof(ctrl->mn)); 102051814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox memcpy(dev->firmware_rev, ctrl->fr, sizeof(ctrl->fr)); 1021b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1022b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cid.identify.cns = 0; 1023b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox memset(&crt, 0, sizeof(crt)); 1024b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox crt.features.opcode = nvme_admin_get_features; 1025b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox crt.features.prp1 = cpu_to_le64(dma_addr + 4096); 1026b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox crt.features.fid = cpu_to_le32(NVME_FEAT_LBA_RANGE); 1027b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1028b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox for (i = 0; i < nn; i++) { 1029b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox cid.identify.nsid = cpu_to_le32(i); 1030b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox res = nvme_submit_admin_cmd(dev, &cid, NULL); 1031b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (res) 1032b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox continue; 1033b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1034b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (((struct nvme_id_ns *)id)->ncap == 0) 1035b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox continue; 1036b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1037b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox crt.features.nsid = cpu_to_le32(i); 1038b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox res = nvme_submit_admin_cmd(dev, &crt, NULL); 1039b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (res) 1040b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox continue; 1041b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1042b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox ns = nvme_alloc_ns(dev, i, id, id + 4096); 1043b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (ns) 1044b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox list_add_tail(&ns->list, &dev->namespaces); 1045b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 1046b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox list_for_each_entry(ns, &dev->namespaces, list) 1047b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox add_disk(ns->disk); 1048b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1049b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_free_coherent(&dev->pci_dev->dev, 4096, id, dma_addr); 1050b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 1051b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1052b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox out_free: 1053b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox list_for_each_entry_safe(ns, next, &dev->namespaces, list) { 1054b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox list_del(&ns->list); 1055b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_ns_free(ns); 1056b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 1057b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1058b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dma_free_coherent(&dev->pci_dev->dev, 4096, id, dma_addr); 1059b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return res; 1060b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1061b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1062b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_dev_remove(struct nvme_dev *dev) 1063b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1064b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_ns *ns, *next; 1065b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1066b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox /* TODO: wait all I/O finished or cancel them */ 1067b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1068b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox list_for_each_entry_safe(ns, next, &dev->namespaces, list) { 1069b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox list_del(&ns->list); 1070b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox del_gendisk(ns->disk); 1071b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_ns_free(ns); 1072b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 1073b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1074b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_free_queues(dev); 1075b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1076b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 1077b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1078b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1079b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* XXX: Use an ida or something to let remove / add work correctly */ 1080b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void nvme_set_instance(struct nvme_dev *dev) 1081b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1082b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox static int instance; 1083b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->instance = instance++; 1084b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1085b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1086b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void nvme_release_instance(struct nvme_dev *dev) 1087b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1088b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1089b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1090b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int __devinit nvme_probe(struct pci_dev *pdev, 1091b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox const struct pci_device_id *id) 1092b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1093574e8b95bc3780e10e9b5e9d51074d503dd3d5d9Matthew Wilcox int bars, result = -ENOMEM; 1094b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_dev *dev; 1095b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1096b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev = kzalloc(sizeof(*dev), GFP_KERNEL); 1097b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!dev) 1098b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -ENOMEM; 1099b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->entry = kcalloc(num_possible_cpus(), sizeof(*dev->entry), 1100b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox GFP_KERNEL); 1101b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!dev->entry) 1102b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto free; 11031b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox dev->queues = kcalloc(num_possible_cpus() + 1, sizeof(void *), 11041b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox GFP_KERNEL); 1105b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!dev->queues) 1106b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto free; 1107b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 11080ee5a7d7cb9309bd393a25c395f19fb12a842602Shane Michael Matthews if (pci_enable_device_mem(pdev)) 11090ee5a7d7cb9309bd393a25c395f19fb12a842602Shane Michael Matthews goto free; 1110f64d3365a3e5cb46e69db7e2c82a7cb9a5bed1b8Matthew Wilcox pci_set_master(pdev); 1111574e8b95bc3780e10e9b5e9d51074d503dd3d5d9Matthew Wilcox bars = pci_select_bars(pdev, IORESOURCE_MEM); 1112574e8b95bc3780e10e9b5e9d51074d503dd3d5d9Matthew Wilcox if (pci_request_selected_regions(pdev, bars, "nvme")) 1113574e8b95bc3780e10e9b5e9d51074d503dd3d5d9Matthew Wilcox goto disable; 11140ee5a7d7cb9309bd393a25c395f19fb12a842602Shane Michael Matthews 1115b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox INIT_LIST_HEAD(&dev->namespaces); 1116b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->pci_dev = pdev; 1117b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox pci_set_drvdata(pdev, dev); 11182930353f9f2b9e4629e935acd970cb73c1171229Matthew Wilcox dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)); 11192930353f9f2b9e4629e935acd970cb73c1171229Matthew Wilcox dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64)); 1120b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_set_instance(dev); 112153c9577e9ca68a633c6e9df2b54eaecacfa77f62Matthew Wilcox dev->entry[0].vector = pdev->irq; 1122b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1123b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->bar = ioremap(pci_resource_start(pdev, 0), 8192); 1124b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!dev->bar) { 1125b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox result = -ENOMEM; 1126574e8b95bc3780e10e9b5e9d51074d503dd3d5d9Matthew Wilcox goto disable_msix; 1127b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox } 1128b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1129b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox result = nvme_configure_admin_queue(dev); 1130b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (result) 1131b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto unmap; 1132b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox dev->queue_count++; 1133b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1134b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox result = nvme_dev_add(dev); 1135b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (result) 1136b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox goto delete; 1137b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 1138b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1139b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox delete: 1140b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_free_queues(dev); 1141b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unmap: 1142b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox iounmap(dev->bar); 1143574e8b95bc3780e10e9b5e9d51074d503dd3d5d9Matthew Wilcox disable_msix: 1144b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox pci_disable_msix(pdev); 1145b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_release_instance(dev); 1146574e8b95bc3780e10e9b5e9d51074d503dd3d5d9Matthew Wilcox disable: 11470ee5a7d7cb9309bd393a25c395f19fb12a842602Shane Michael Matthews pci_disable_device(pdev); 1148574e8b95bc3780e10e9b5e9d51074d503dd3d5d9Matthew Wilcox pci_release_regions(pdev); 1149b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox free: 1150b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(dev->queues); 1151b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(dev->entry); 1152b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(dev); 1153b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return result; 1154b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1155b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1156b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void __devexit nvme_remove(struct pci_dev *pdev) 1157b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1158b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox struct nvme_dev *dev = pci_get_drvdata(pdev); 1159b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_dev_remove(dev); 1160b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox pci_disable_msix(pdev); 1161b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox iounmap(dev->bar); 1162b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_release_instance(dev); 11630ee5a7d7cb9309bd393a25c395f19fb12a842602Shane Michael Matthews pci_disable_device(pdev); 1164574e8b95bc3780e10e9b5e9d51074d503dd3d5d9Matthew Wilcox pci_release_regions(pdev); 1165b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(dev->queues); 1166b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(dev->entry); 1167b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox kfree(dev); 1168b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1169b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1170b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* These functions are yet to be implemented */ 1171b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_error_detected NULL 1172b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_dump_registers NULL 1173b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_link_reset NULL 1174b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_slot_reset NULL 1175b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_error_resume NULL 1176b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_suspend NULL 1177b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_resume NULL 1178b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1179b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic struct pci_error_handlers nvme_err_handler = { 1180b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .error_detected = nvme_error_detected, 1181b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .mmio_enabled = nvme_dump_registers, 1182b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .link_reset = nvme_link_reset, 1183b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .slot_reset = nvme_slot_reset, 1184b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .resume = nvme_error_resume, 1185b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 1186b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1187b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* Move to pci_ids.h later */ 1188b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define PCI_CLASS_STORAGE_EXPRESS 0x010802 1189b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1190b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic DEFINE_PCI_DEVICE_TABLE(nvme_id_table) = { 1191b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox { PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) }, 1192b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox { 0, } 1193b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 1194b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew WilcoxMODULE_DEVICE_TABLE(pci, nvme_id_table); 1195b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1196b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic struct pci_driver nvme_driver = { 1197b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .name = "nvme", 1198b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .id_table = nvme_id_table, 1199b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .probe = nvme_probe, 1200b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .remove = __devexit_p(nvme_remove), 1201b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .suspend = nvme_suspend, 1202b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .resume = nvme_resume, 1203b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox .err_handler = &nvme_err_handler, 1204b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}; 1205b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1206b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int __init nvme_init(void) 1207b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1208b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox int result; 1209b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1210b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox nvme_major = register_blkdev(nvme_major, "nvme"); 1211b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (nvme_major <= 0) 1212b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return -EBUSY; 1213b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1214b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox result = pci_register_driver(&nvme_driver); 1215b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox if (!result) 1216b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return 0; 1217b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1218b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unregister_blkdev(nvme_major, "nvme"); 1219b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox return result; 1220b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1221b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1222b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void __exit nvme_exit(void) 1223b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{ 1224b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox pci_unregister_driver(&nvme_driver); 1225b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox unregister_blkdev(nvme_major, "nvme"); 1226b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox} 1227b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox 1228b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew WilcoxMODULE_AUTHOR("Matthew Wilcox <willy@linux.intel.com>"); 1229b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew WilcoxMODULE_LICENSE("GPL"); 1230b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew WilcoxMODULE_VERSION("0.1"); 1231b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxmodule_init(nvme_init); 1232b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxmodule_exit(nvme_exit); 1233