nvme-core.c revision 0a8d44cb33377969337fa6c1961b631605d5f453
1b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/*
2b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * NVM Express device driver
3b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Copyright (c) 2011, Intel Corporation.
4b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox *
5b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * This program is free software; you can redistribute it and/or modify it
6b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * under the terms and conditions of the GNU General Public License,
7b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * version 2, as published by the Free Software Foundation.
8b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox *
9b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * This program is distributed in the hope it will be useful, but WITHOUT
10b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
12b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * more details.
13b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox *
14b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * You should have received a copy of the GNU General Public License along with
15b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * this program; if not, write to the Free Software Foundation, Inc.,
16b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
17b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */
18b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
19b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/nvme.h>
20b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/bio.h>
218de055350fbaa96b6563892c195a60be583faa9cMatthew Wilcox#include <linux/bitops.h>
22b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/blkdev.h>
23fd63e9ceeeae58cfe877c2d49d41c1bf7532303cMatthew Wilcox#include <linux/delay.h>
24b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/errno.h>
25b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/fs.h>
26b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/genhd.h>
275aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox#include <linux/idr.h>
28b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/init.h>
29b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/interrupt.h>
30b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/io.h>
31b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/kdev_t.h>
321fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox#include <linux/kthread.h>
33b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/kernel.h>
34b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/mm.h>
35b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/module.h>
36b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/moduleparam.h>
37b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/pci.h>
38be7b62754e097adc0cb16c25c9ee86ee20de62fbMatthew Wilcox#include <linux/poison.h>
39c3bfe7176c035a0a2c70bc79180fb13a6c57142aMatthew Wilcox#include <linux/ptrace.h>
40b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/sched.h>
41b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/slab.h>
42b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#include <linux/types.h>
435d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma#include <scsi/sg.h>
44797a796a13df6b84a4791e57306737059b5b2384Hitoshi Mitake#include <asm-generic/io-64-nonatomic-lo-hi.h>
45797a796a13df6b84a4791e57306737059b5b2384Hitoshi Mitake
46b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define NVME_Q_DEPTH 1024
47b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define SQ_SIZE(depth)		(depth * sizeof(struct nvme_command))
48b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define CQ_SIZE(depth)		(depth * sizeof(struct nvme_completion))
49b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define NVME_MINORS 64
50e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox#define ADMIN_TIMEOUT	(60 * HZ)
51b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
52b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_major;
53b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxmodule_param(nvme_major, int, 0);
54b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
5558ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcoxstatic int use_threaded_interrupts;
5658ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcoxmodule_param(use_threaded_interrupts, int, 0);
5758ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox
581fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcoxstatic DEFINE_SPINLOCK(dev_list_lock);
591fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcoxstatic LIST_HEAD(dev_list);
601fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcoxstatic struct task_struct *nvme_thread;
611fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox
62b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/*
63b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * An NVM Express queue.  Each device has at least two (one for admin
64b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * commands and one for I/O commands).
65b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */
66b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstruct nvme_queue {
67b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	struct device *q_dmadev;
68091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox	struct nvme_dev *dev;
69b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	spinlock_t q_lock;
70b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	struct nvme_command *sq_cmds;
71b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	volatile struct nvme_completion *cqes;
72b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	dma_addr_t sq_dma_addr;
73b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	dma_addr_t cq_dma_addr;
74b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	wait_queue_head_t sq_full;
751fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox	wait_queue_t sq_cong_wait;
76b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	struct bio_list sq_cong;
77b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	u32 __iomem *q_db;
78b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	u16 q_depth;
79b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	u16 cq_vector;
80b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	u16 sq_head;
81b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	u16 sq_tail;
82b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	u16 cq_head;
83e9539f47525ecee05c9f22c3565885f3e9492c52Matthew Wilcox	u8 cq_phase;
84e9539f47525ecee05c9f22c3565885f3e9492c52Matthew Wilcox	u8 cqe_seen;
85224042742582c9938788b81165180c876e997a07Keith Busch	u8 q_suspended;
86b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	unsigned long cmdid_data[];
87b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox};
88b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
89b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/*
90b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Check we didin't inadvertently grow the command struct
91b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */
92b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic inline void _nvme_check_size(void)
93b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{
94b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64);
95b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	BUILD_BUG_ON(sizeof(struct nvme_create_cq) != 64);
96b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	BUILD_BUG_ON(sizeof(struct nvme_create_sq) != 64);
97b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64);
98b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	BUILD_BUG_ON(sizeof(struct nvme_features) != 64);
99f8ebf8409abfdaeeb8c847381629a2a8b8e3d816Vishal Verma	BUILD_BUG_ON(sizeof(struct nvme_format_cmd) != 64);
100b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	BUILD_BUG_ON(sizeof(struct nvme_command) != 64);
101b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != 4096);
102b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	BUILD_BUG_ON(sizeof(struct nvme_id_ns) != 4096);
103b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64);
1046ecec74520d8a357726e6c12f99080dbe7b347ddKeith Busch	BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512);
105b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}
106b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
1075c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcoxtypedef void (*nvme_completion_fn)(struct nvme_dev *, void *,
108c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox						struct nvme_completion *);
109c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox
110e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcoxstruct nvme_cmd_info {
111c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox	nvme_completion_fn fn;
112c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox	void *ctx;
113e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox	unsigned long timeout;
114e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox};
115e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox
116e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcoxstatic struct nvme_cmd_info *nvme_cmd_info(struct nvme_queue *nvmeq)
117e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox{
118e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox	return (void *)&nvmeq->cmdid_data[BITS_TO_LONGS(nvmeq->q_depth)];
119e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox}
120e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox
121224042742582c9938788b81165180c876e997a07Keith Buschstatic unsigned nvme_queue_extra(int depth)
122224042742582c9938788b81165180c876e997a07Keith Busch{
123224042742582c9938788b81165180c876e997a07Keith Busch	return DIV_ROUND_UP(depth, 8) + (depth * sizeof(struct nvme_cmd_info));
124224042742582c9938788b81165180c876e997a07Keith Busch}
125224042742582c9938788b81165180c876e997a07Keith Busch
126b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/**
127714a7a22884b74862540bc84955274d86b2f6040Matthew Wilcox * alloc_cmdid() - Allocate a Command ID
128714a7a22884b74862540bc84955274d86b2f6040Matthew Wilcox * @nvmeq: The queue that will be used for this command
129714a7a22884b74862540bc84955274d86b2f6040Matthew Wilcox * @ctx: A pointer that will be passed to the handler
130c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox * @handler: The function to call on completion
131b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox *
132b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Allocate a Command ID for a queue.  The data passed in will
133b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * be passed to the completion handler.  This is implemented by using
134b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * the bottom two bits of the ctx pointer to store the handler ID.
135b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Passing in a pointer that's not 4-byte aligned will cause a BUG.
136b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * We can change this if it becomes a problem.
137184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox *
138184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox * May be called with local interrupts disabled and the q_lock held,
139184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox * or with interrupts enabled and no locks held.
140b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */
141c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcoxstatic int alloc_cmdid(struct nvme_queue *nvmeq, void *ctx,
142c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox				nvme_completion_fn handler, unsigned timeout)
143b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{
144e6d15f79f997a98b3a69abbc462fc9041cc1a7b4Matthew Wilcox	int depth = nvmeq->q_depth - 1;
145e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox	struct nvme_cmd_info *info = nvme_cmd_info(nvmeq);
146b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	int cmdid;
147b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
148b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	do {
149b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox		cmdid = find_first_zero_bit(nvmeq->cmdid_data, depth);
150b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox		if (cmdid >= depth)
151b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox			return -EBUSY;
152b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	} while (test_and_set_bit(cmdid, nvmeq->cmdid_data));
153b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
154c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox	info[cmdid].fn = handler;
155c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox	info[cmdid].ctx = ctx;
156e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox	info[cmdid].timeout = jiffies + timeout;
157b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	return cmdid;
158b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}
159b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
160b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int alloc_cmdid_killable(struct nvme_queue *nvmeq, void *ctx,
161c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox				nvme_completion_fn handler, unsigned timeout)
162b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{
163b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	int cmdid;
164b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	wait_event_killable(nvmeq->sq_full,
165e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox		(cmdid = alloc_cmdid(nvmeq, ctx, handler, timeout)) >= 0);
166b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	return (cmdid < 0) ? -EINTR : cmdid;
167b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}
168b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
169c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox/* Special values must be less than 0x1000 */
170c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox#define CMD_CTX_BASE		((void *)POISON_POINTER_DELTA)
171d2d8703481f60d67f49e3177196cbe474b11377cMatthew Wilcox#define CMD_CTX_CANCELLED	(0x30C + CMD_CTX_BASE)
172d2d8703481f60d67f49e3177196cbe474b11377cMatthew Wilcox#define CMD_CTX_COMPLETED	(0x310 + CMD_CTX_BASE)
173d2d8703481f60d67f49e3177196cbe474b11377cMatthew Wilcox#define CMD_CTX_INVALID		(0x314 + CMD_CTX_BASE)
17400df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox#define CMD_CTX_FLUSH		(0x318 + CMD_CTX_BASE)
175be7b62754e097adc0cb16c25c9ee86ee20de62fbMatthew Wilcox
1765c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcoxstatic void special_completion(struct nvme_dev *dev, void *ctx,
177c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox						struct nvme_completion *cqe)
178c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox{
179c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox	if (ctx == CMD_CTX_CANCELLED)
180c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox		return;
181c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox	if (ctx == CMD_CTX_FLUSH)
182c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox		return;
183c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox	if (ctx == CMD_CTX_COMPLETED) {
1845c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcox		dev_warn(&dev->pci_dev->dev,
185c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox				"completed id %d twice on queue %d\n",
186c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox				cqe->command_id, le16_to_cpup(&cqe->sq_id));
187c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox		return;
188c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox	}
189c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox	if (ctx == CMD_CTX_INVALID) {
1905c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcox		dev_warn(&dev->pci_dev->dev,
191c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox				"invalid id %d completed on queue %d\n",
192c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox				cqe->command_id, le16_to_cpup(&cqe->sq_id));
193c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox		return;
194c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox	}
195c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox
1965c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcox	dev_warn(&dev->pci_dev->dev, "Unknown special completion %p\n", ctx);
197c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox}
198c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox
199184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox/*
200184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox * Called with local interrupts disabled and the q_lock held.  May not sleep.
201184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox */
202c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcoxstatic void *free_cmdid(struct nvme_queue *nvmeq, int cmdid,
203c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox						nvme_completion_fn *fn)
204b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{
205c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox	void *ctx;
206e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox	struct nvme_cmd_info *info = nvme_cmd_info(nvmeq);
207b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
208c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox	if (cmdid >= nvmeq->q_depth) {
209c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox		*fn = special_completion;
21048e3d39816416b3bf03dee3a796c0c04427c1a31Matthew Wilcox		return CMD_CTX_INVALID;
211c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox	}
212859361a228258edf4821d9f5635825033eca78e8Keith Busch	if (fn)
213859361a228258edf4821d9f5635825033eca78e8Keith Busch		*fn = info[cmdid].fn;
214c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox	ctx = info[cmdid].ctx;
215c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox	info[cmdid].fn = special_completion;
216e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox	info[cmdid].ctx = CMD_CTX_COMPLETED;
217b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	clear_bit(cmdid, nvmeq->cmdid_data);
218b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	wake_up(&nvmeq->sq_full);
219c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox	return ctx;
220b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}
221b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
222c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcoxstatic void *cancel_cmdid(struct nvme_queue *nvmeq, int cmdid,
223c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox						nvme_completion_fn *fn)
2243c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox{
225c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox	void *ctx;
226e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox	struct nvme_cmd_info *info = nvme_cmd_info(nvmeq);
227c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox	if (fn)
228c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox		*fn = info[cmdid].fn;
229c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox	ctx = info[cmdid].ctx;
230c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox	info[cmdid].fn = special_completion;
231e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox	info[cmdid].ctx = CMD_CTX_CANCELLED;
232c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox	return ctx;
2333c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox}
2343c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox
2355d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermastruct nvme_queue *get_nvmeq(struct nvme_dev *dev)
236b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{
237040a93b52a9eee8177ebaf2ba0ee0f9f518d1bf8Matthew Wilcox	return dev->queues[get_cpu() + 1];
238b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}
239b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
2405d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermavoid put_nvmeq(struct nvme_queue *nvmeq)
241b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{
2421b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox	put_cpu();
243b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}
244b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
245b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/**
246714a7a22884b74862540bc84955274d86b2f6040Matthew Wilcox * nvme_submit_cmd() - Copy a command into a queue and ring the doorbell
247b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * @nvmeq: The queue to use
248b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * @cmd: The command to send
249b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox *
250b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Safe to use from interrupt context
251b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */
252b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_submit_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd)
253b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{
254b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	unsigned long flags;
255b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	u16 tail;
256b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	spin_lock_irqsave(&nvmeq->q_lock, flags);
257b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	tail = nvmeq->sq_tail;
258b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	memcpy(&nvmeq->sq_cmds[tail], cmd, sizeof(*cmd));
259b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	if (++tail == nvmeq->q_depth)
260b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox		tail = 0;
2617547881d0951384f9833ec3a80fac8f3f16f3b98Matthew Wilcox	writel(tail, nvmeq->q_db);
262b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	nvmeq->sq_tail = tail;
263b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	spin_unlock_irqrestore(&nvmeq->q_lock, flags);
264b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
265b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	return 0;
266b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}
267b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
268eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcoxstatic __le64 **iod_list(struct nvme_iod *iod)
269e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews{
270eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox	return ((void *)iod) + iod->offset;
271e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews}
272e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews
273eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox/*
274eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox * Will slightly overestimate the number of pages needed.  This is OK
275eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox * as it only leads to a small amount of wasted memory for the lifetime of
276eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox * the I/O.
277eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox */
278eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcoxstatic int nvme_npages(unsigned size)
279eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox{
280eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox	unsigned nprps = DIV_ROUND_UP(size + PAGE_SIZE, PAGE_SIZE);
281eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox	return DIV_ROUND_UP(8 * nprps, PAGE_SIZE - 8);
282eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox}
283b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
284eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcoxstatic struct nvme_iod *
285eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcoxnvme_alloc_iod(unsigned nseg, unsigned nbytes, gfp_t gfp)
286b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{
287eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox	struct nvme_iod *iod = kmalloc(sizeof(struct nvme_iod) +
288eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox				sizeof(__le64 *) * nvme_npages(nbytes) +
289eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox				sizeof(struct scatterlist) * nseg, gfp);
290eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox
291eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox	if (iod) {
292eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox		iod->offset = offsetof(struct nvme_iod, sg[nseg]);
293eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox		iod->npages = -1;
294eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox		iod->length = nbytes;
2952b1960341576bf51c01b12fefeb1cc53820923e7Keith Busch		iod->nents = 0;
2966198221fa0df0298513b35796f63f242ea97134eKeith Busch		iod->start_time = jiffies;
297eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox	}
298eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox
299eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox	return iod;
300b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}
301b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
3025d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermavoid nvme_free_iod(struct nvme_dev *dev, struct nvme_iod *iod)
303b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{
304eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox	const int last_prp = PAGE_SIZE / 8 - 1;
305eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox	int i;
306eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox	__le64 **list = iod_list(iod);
307eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox	dma_addr_t prp_dma = iod->first_dma;
308eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox
309eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox	if (iod->npages == 0)
310eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox		dma_pool_free(dev->prp_small_pool, list[0], prp_dma);
311eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox	for (i = 0; i < iod->npages; i++) {
312eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox		__le64 *prp_list = list[i];
313eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox		dma_addr_t next_prp_dma = le64_to_cpu(prp_list[last_prp]);
314eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox		dma_pool_free(dev->prp_page_pool, prp_list, prp_dma);
315eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox		prp_dma = next_prp_dma;
316eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox	}
317eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox	kfree(iod);
318b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}
319b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
3206198221fa0df0298513b35796f63f242ea97134eKeith Buschstatic void nvme_start_io_acct(struct bio *bio)
3216198221fa0df0298513b35796f63f242ea97134eKeith Busch{
3226198221fa0df0298513b35796f63f242ea97134eKeith Busch	struct gendisk *disk = bio->bi_bdev->bd_disk;
3236198221fa0df0298513b35796f63f242ea97134eKeith Busch	const int rw = bio_data_dir(bio);
3246198221fa0df0298513b35796f63f242ea97134eKeith Busch	int cpu = part_stat_lock();
3256198221fa0df0298513b35796f63f242ea97134eKeith Busch	part_round_stats(cpu, &disk->part0);
3266198221fa0df0298513b35796f63f242ea97134eKeith Busch	part_stat_inc(cpu, &disk->part0, ios[rw]);
3276198221fa0df0298513b35796f63f242ea97134eKeith Busch	part_stat_add(cpu, &disk->part0, sectors[rw], bio_sectors(bio));
3286198221fa0df0298513b35796f63f242ea97134eKeith Busch	part_inc_in_flight(&disk->part0, rw);
3296198221fa0df0298513b35796f63f242ea97134eKeith Busch	part_stat_unlock();
3306198221fa0df0298513b35796f63f242ea97134eKeith Busch}
3316198221fa0df0298513b35796f63f242ea97134eKeith Busch
3326198221fa0df0298513b35796f63f242ea97134eKeith Buschstatic void nvme_end_io_acct(struct bio *bio, unsigned long start_time)
3336198221fa0df0298513b35796f63f242ea97134eKeith Busch{
3346198221fa0df0298513b35796f63f242ea97134eKeith Busch	struct gendisk *disk = bio->bi_bdev->bd_disk;
3356198221fa0df0298513b35796f63f242ea97134eKeith Busch	const int rw = bio_data_dir(bio);
3366198221fa0df0298513b35796f63f242ea97134eKeith Busch	unsigned long duration = jiffies - start_time;
3376198221fa0df0298513b35796f63f242ea97134eKeith Busch	int cpu = part_stat_lock();
3386198221fa0df0298513b35796f63f242ea97134eKeith Busch	part_stat_add(cpu, &disk->part0, ticks[rw], duration);
3396198221fa0df0298513b35796f63f242ea97134eKeith Busch	part_round_stats(cpu, &disk->part0);
3406198221fa0df0298513b35796f63f242ea97134eKeith Busch	part_dec_in_flight(&disk->part0, rw);
3416198221fa0df0298513b35796f63f242ea97134eKeith Busch	part_stat_unlock();
3426198221fa0df0298513b35796f63f242ea97134eKeith Busch}
3436198221fa0df0298513b35796f63f242ea97134eKeith Busch
3445c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcoxstatic void bio_completion(struct nvme_dev *dev, void *ctx,
345b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox						struct nvme_completion *cqe)
346b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{
347eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox	struct nvme_iod *iod = ctx;
348eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox	struct bio *bio = iod->private;
349b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	u16 status = le16_to_cpup(&cqe->status) >> 1;
350b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
3519e59d091b0eb04f223ed037348e3d9e36f30e72bKeith Busch	if (iod->nents) {
3522b1960341576bf51c01b12fefeb1cc53820923e7Keith Busch		dma_unmap_sg(&dev->pci_dev->dev, iod->sg, iod->nents,
353b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox			bio_data_dir(bio) ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
3549e59d091b0eb04f223ed037348e3d9e36f30e72bKeith Busch		nvme_end_io_acct(bio, iod->start_time);
3559e59d091b0eb04f223ed037348e3d9e36f30e72bKeith Busch	}
356eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox	nvme_free_iod(dev, iod);
357427e97080196548557b288517537ab7eb48c309fKeith Busch	if (status)
3581ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox		bio_endio(bio, -EIO);
359427e97080196548557b288517537ab7eb48c309fKeith Busch	else
3601ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox		bio_endio(bio, 0);
361b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}
362b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
363184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox/* length is in bytes.  gfp flags indicates whether we may sleep. */
3645d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermaint nvme_setup_prps(struct nvme_dev *dev, struct nvme_common_command *cmd,
3655d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma			struct nvme_iod *iod, int total_len, gfp_t gfp)
366ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox{
36799802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox	struct dma_pool *pool;
368eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox	int length = total_len;
369eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox	struct scatterlist *sg = iod->sg;
370ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox	int dma_len = sg_dma_len(sg);
371ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox	u64 dma_addr = sg_dma_address(sg);
372ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox	int offset = offset_in_page(dma_addr);
373e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews	__le64 *prp_list;
374eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox	__le64 **list = iod_list(iod);
375e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews	dma_addr_t prp_dma;
376eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox	int nprps, i;
377ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox
378ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox	cmd->prp1 = cpu_to_le64(dma_addr);
379ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox	length -= (PAGE_SIZE - offset);
380ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox	if (length <= 0)
381eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox		return total_len;
382ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox
383ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox	dma_len -= (PAGE_SIZE - offset);
384ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox	if (dma_len) {
385ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox		dma_addr += (PAGE_SIZE - offset);
386ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox	} else {
387ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox		sg = sg_next(sg);
388ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox		dma_addr = sg_dma_address(sg);
389ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox		dma_len = sg_dma_len(sg);
390ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox	}
391ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox
392ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox	if (length <= PAGE_SIZE) {
393ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox		cmd->prp2 = cpu_to_le64(dma_addr);
394eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox		return total_len;
395e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews	}
396e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews
397e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews	nprps = DIV_ROUND_UP(length, PAGE_SIZE);
39899802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox	if (nprps <= (256 / 8)) {
39999802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox		pool = dev->prp_small_pool;
400eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox		iod->npages = 0;
40199802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox	} else {
40299802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox		pool = dev->prp_page_pool;
403eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox		iod->npages = 1;
40499802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox	}
40599802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox
406b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox	prp_list = dma_pool_alloc(pool, gfp, &prp_dma);
407b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox	if (!prp_list) {
408b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox		cmd->prp2 = cpu_to_le64(dma_addr);
409eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox		iod->npages = -1;
410eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox		return (total_len - length) + PAGE_SIZE;
411b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox	}
412eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox	list[0] = prp_list;
413eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox	iod->first_dma = prp_dma;
414e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews	cmd->prp2 = cpu_to_le64(prp_dma);
415e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews	i = 0;
416e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews	for (;;) {
4177523d834dd1573610078eb1ac0933f6490232f90Matthew Wilcox		if (i == PAGE_SIZE / 8) {
418e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews			__le64 *old_prp_list = prp_list;
419b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox			prp_list = dma_pool_alloc(pool, gfp, &prp_dma);
420eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox			if (!prp_list)
421eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox				return total_len - length;
422eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox			list[iod->npages++] = prp_list;
4237523d834dd1573610078eb1ac0933f6490232f90Matthew Wilcox			prp_list[0] = old_prp_list[i - 1];
4247523d834dd1573610078eb1ac0933f6490232f90Matthew Wilcox			old_prp_list[i - 1] = cpu_to_le64(prp_dma);
4257523d834dd1573610078eb1ac0933f6490232f90Matthew Wilcox			i = 1;
426e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews		}
427e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews		prp_list[i++] = cpu_to_le64(dma_addr);
428e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews		dma_len -= PAGE_SIZE;
429e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews		dma_addr += PAGE_SIZE;
430e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews		length -= PAGE_SIZE;
431e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews		if (length <= 0)
432e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews			break;
433e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews		if (dma_len > 0)
434e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews			continue;
435e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews		BUG_ON(dma_len < 0);
436e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews		sg = sg_next(sg);
437e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews		dma_addr = sg_dma_address(sg);
438e025344c56e08b155f43ea09647969286c78377cShane Michael Matthews		dma_len = sg_dma_len(sg);
439ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox	}
440ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox
441eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox	return total_len;
442ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox}
443ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox
444427e97080196548557b288517537ab7eb48c309fKeith Buschstruct nvme_bio_pair {
445427e97080196548557b288517537ab7eb48c309fKeith Busch	struct bio b1, b2, *parent;
446427e97080196548557b288517537ab7eb48c309fKeith Busch	struct bio_vec *bv1, *bv2;
447427e97080196548557b288517537ab7eb48c309fKeith Busch	int err;
448427e97080196548557b288517537ab7eb48c309fKeith Busch	atomic_t cnt;
449427e97080196548557b288517537ab7eb48c309fKeith Busch};
450427e97080196548557b288517537ab7eb48c309fKeith Busch
451427e97080196548557b288517537ab7eb48c309fKeith Buschstatic void nvme_bio_pair_endio(struct bio *bio, int err)
452427e97080196548557b288517537ab7eb48c309fKeith Busch{
453427e97080196548557b288517537ab7eb48c309fKeith Busch	struct nvme_bio_pair *bp = bio->bi_private;
454427e97080196548557b288517537ab7eb48c309fKeith Busch
455427e97080196548557b288517537ab7eb48c309fKeith Busch	if (err)
456427e97080196548557b288517537ab7eb48c309fKeith Busch		bp->err = err;
457427e97080196548557b288517537ab7eb48c309fKeith Busch
458427e97080196548557b288517537ab7eb48c309fKeith Busch	if (atomic_dec_and_test(&bp->cnt)) {
459427e97080196548557b288517537ab7eb48c309fKeith Busch		bio_endio(bp->parent, bp->err);
4601b56749e541ad59068582f2a28297843e243b856Keith Busch		kfree(bp->bv1);
4611b56749e541ad59068582f2a28297843e243b856Keith Busch		kfree(bp->bv2);
462427e97080196548557b288517537ab7eb48c309fKeith Busch		kfree(bp);
463427e97080196548557b288517537ab7eb48c309fKeith Busch	}
464427e97080196548557b288517537ab7eb48c309fKeith Busch}
465427e97080196548557b288517537ab7eb48c309fKeith Busch
466427e97080196548557b288517537ab7eb48c309fKeith Buschstatic struct nvme_bio_pair *nvme_bio_split(struct bio *bio, int idx,
467427e97080196548557b288517537ab7eb48c309fKeith Busch							int len, int offset)
468427e97080196548557b288517537ab7eb48c309fKeith Busch{
469427e97080196548557b288517537ab7eb48c309fKeith Busch	struct nvme_bio_pair *bp;
470427e97080196548557b288517537ab7eb48c309fKeith Busch
471427e97080196548557b288517537ab7eb48c309fKeith Busch	BUG_ON(len > bio->bi_size);
472427e97080196548557b288517537ab7eb48c309fKeith Busch	BUG_ON(idx > bio->bi_vcnt);
473427e97080196548557b288517537ab7eb48c309fKeith Busch
474427e97080196548557b288517537ab7eb48c309fKeith Busch	bp = kmalloc(sizeof(*bp), GFP_ATOMIC);
475427e97080196548557b288517537ab7eb48c309fKeith Busch	if (!bp)
476427e97080196548557b288517537ab7eb48c309fKeith Busch		return NULL;
477427e97080196548557b288517537ab7eb48c309fKeith Busch	bp->err = 0;
478427e97080196548557b288517537ab7eb48c309fKeith Busch
479427e97080196548557b288517537ab7eb48c309fKeith Busch	bp->b1 = *bio;
480427e97080196548557b288517537ab7eb48c309fKeith Busch	bp->b2 = *bio;
481427e97080196548557b288517537ab7eb48c309fKeith Busch
482427e97080196548557b288517537ab7eb48c309fKeith Busch	bp->b1.bi_size = len;
483427e97080196548557b288517537ab7eb48c309fKeith Busch	bp->b2.bi_size -= len;
484427e97080196548557b288517537ab7eb48c309fKeith Busch	bp->b1.bi_vcnt = idx;
485427e97080196548557b288517537ab7eb48c309fKeith Busch	bp->b2.bi_idx = idx;
486427e97080196548557b288517537ab7eb48c309fKeith Busch	bp->b2.bi_sector += len >> 9;
487427e97080196548557b288517537ab7eb48c309fKeith Busch
488427e97080196548557b288517537ab7eb48c309fKeith Busch	if (offset) {
489427e97080196548557b288517537ab7eb48c309fKeith Busch		bp->bv1 = kmalloc(bio->bi_max_vecs * sizeof(struct bio_vec),
490427e97080196548557b288517537ab7eb48c309fKeith Busch								GFP_ATOMIC);
491427e97080196548557b288517537ab7eb48c309fKeith Busch		if (!bp->bv1)
492427e97080196548557b288517537ab7eb48c309fKeith Busch			goto split_fail_1;
493427e97080196548557b288517537ab7eb48c309fKeith Busch
494427e97080196548557b288517537ab7eb48c309fKeith Busch		bp->bv2 = kmalloc(bio->bi_max_vecs * sizeof(struct bio_vec),
495427e97080196548557b288517537ab7eb48c309fKeith Busch								GFP_ATOMIC);
496427e97080196548557b288517537ab7eb48c309fKeith Busch		if (!bp->bv2)
497427e97080196548557b288517537ab7eb48c309fKeith Busch			goto split_fail_2;
498427e97080196548557b288517537ab7eb48c309fKeith Busch
499427e97080196548557b288517537ab7eb48c309fKeith Busch		memcpy(bp->bv1, bio->bi_io_vec,
500427e97080196548557b288517537ab7eb48c309fKeith Busch			bio->bi_max_vecs * sizeof(struct bio_vec));
501427e97080196548557b288517537ab7eb48c309fKeith Busch		memcpy(bp->bv2, bio->bi_io_vec,
502427e97080196548557b288517537ab7eb48c309fKeith Busch			bio->bi_max_vecs * sizeof(struct bio_vec));
503427e97080196548557b288517537ab7eb48c309fKeith Busch
504427e97080196548557b288517537ab7eb48c309fKeith Busch		bp->b1.bi_io_vec = bp->bv1;
505427e97080196548557b288517537ab7eb48c309fKeith Busch		bp->b2.bi_io_vec = bp->bv2;
506427e97080196548557b288517537ab7eb48c309fKeith Busch		bp->b2.bi_io_vec[idx].bv_offset += offset;
507427e97080196548557b288517537ab7eb48c309fKeith Busch		bp->b2.bi_io_vec[idx].bv_len -= offset;
508427e97080196548557b288517537ab7eb48c309fKeith Busch		bp->b1.bi_io_vec[idx].bv_len = offset;
509427e97080196548557b288517537ab7eb48c309fKeith Busch		bp->b1.bi_vcnt++;
510427e97080196548557b288517537ab7eb48c309fKeith Busch	} else
511427e97080196548557b288517537ab7eb48c309fKeith Busch		bp->bv1 = bp->bv2 = NULL;
512427e97080196548557b288517537ab7eb48c309fKeith Busch
513427e97080196548557b288517537ab7eb48c309fKeith Busch	bp->b1.bi_private = bp;
514427e97080196548557b288517537ab7eb48c309fKeith Busch	bp->b2.bi_private = bp;
515427e97080196548557b288517537ab7eb48c309fKeith Busch
516427e97080196548557b288517537ab7eb48c309fKeith Busch	bp->b1.bi_end_io = nvme_bio_pair_endio;
517427e97080196548557b288517537ab7eb48c309fKeith Busch	bp->b2.bi_end_io = nvme_bio_pair_endio;
518427e97080196548557b288517537ab7eb48c309fKeith Busch
519427e97080196548557b288517537ab7eb48c309fKeith Busch	bp->parent = bio;
520427e97080196548557b288517537ab7eb48c309fKeith Busch	atomic_set(&bp->cnt, 2);
521427e97080196548557b288517537ab7eb48c309fKeith Busch
522427e97080196548557b288517537ab7eb48c309fKeith Busch	return bp;
523427e97080196548557b288517537ab7eb48c309fKeith Busch
524427e97080196548557b288517537ab7eb48c309fKeith Busch split_fail_2:
525427e97080196548557b288517537ab7eb48c309fKeith Busch	kfree(bp->bv1);
526427e97080196548557b288517537ab7eb48c309fKeith Busch split_fail_1:
527427e97080196548557b288517537ab7eb48c309fKeith Busch	kfree(bp);
528427e97080196548557b288517537ab7eb48c309fKeith Busch	return NULL;
529427e97080196548557b288517537ab7eb48c309fKeith Busch}
530427e97080196548557b288517537ab7eb48c309fKeith Busch
531427e97080196548557b288517537ab7eb48c309fKeith Buschstatic int nvme_split_and_submit(struct bio *bio, struct nvme_queue *nvmeq,
532427e97080196548557b288517537ab7eb48c309fKeith Busch						int idx, int len, int offset)
533427e97080196548557b288517537ab7eb48c309fKeith Busch{
534427e97080196548557b288517537ab7eb48c309fKeith Busch	struct nvme_bio_pair *bp = nvme_bio_split(bio, idx, len, offset);
535427e97080196548557b288517537ab7eb48c309fKeith Busch	if (!bp)
536427e97080196548557b288517537ab7eb48c309fKeith Busch		return -ENOMEM;
537427e97080196548557b288517537ab7eb48c309fKeith Busch
538427e97080196548557b288517537ab7eb48c309fKeith Busch	if (bio_list_empty(&nvmeq->sq_cong))
539427e97080196548557b288517537ab7eb48c309fKeith Busch		add_wait_queue(&nvmeq->sq_full, &nvmeq->sq_cong_wait);
540427e97080196548557b288517537ab7eb48c309fKeith Busch	bio_list_add(&nvmeq->sq_cong, &bp->b1);
541427e97080196548557b288517537ab7eb48c309fKeith Busch	bio_list_add(&nvmeq->sq_cong, &bp->b2);
542427e97080196548557b288517537ab7eb48c309fKeith Busch
543427e97080196548557b288517537ab7eb48c309fKeith Busch	return 0;
544427e97080196548557b288517537ab7eb48c309fKeith Busch}
545427e97080196548557b288517537ab7eb48c309fKeith Busch
5461ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox/* NVMe scatterlists require no holes in the virtual address */
5471ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox#define BIOVEC_NOT_VIRT_MERGEABLE(vec1, vec2)	((vec2)->bv_offset || \
5481ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox			(((vec1)->bv_offset + (vec1)->bv_len) % PAGE_SIZE))
5491ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox
550427e97080196548557b288517537ab7eb48c309fKeith Buschstatic int nvme_map_bio(struct nvme_queue *nvmeq, struct nvme_iod *iod,
551b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox		struct bio *bio, enum dma_data_direction dma_dir, int psegs)
552b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{
553768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox	struct bio_vec *bvec, *bvprv = NULL;
554768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox	struct scatterlist *sg = NULL;
555159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch	int i, length = 0, nsegs = 0, split_len = bio->bi_size;
556159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch
557159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch	if (nvmeq->dev->stripe_size)
558159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch		split_len = nvmeq->dev->stripe_size -
559159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch			((bio->bi_sector << 9) & (nvmeq->dev->stripe_size - 1));
560b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
561eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox	sg_init_table(iod->sg, psegs);
562b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	bio_for_each_segment(bvec, bio, i) {
563768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox		if (bvprv && BIOVEC_PHYS_MERGEABLE(bvprv, bvec)) {
564768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox			sg->length += bvec->bv_len;
565768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox		} else {
5661ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox			if (bvprv && BIOVEC_NOT_VIRT_MERGEABLE(bvprv, bvec))
567427e97080196548557b288517537ab7eb48c309fKeith Busch				return nvme_split_and_submit(bio, nvmeq, i,
568427e97080196548557b288517537ab7eb48c309fKeith Busch								length, 0);
569427e97080196548557b288517537ab7eb48c309fKeith Busch
570eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox			sg = sg ? sg + 1 : iod->sg;
571768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox			sg_set_page(sg, bvec->bv_page, bvec->bv_len,
572768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox							bvec->bv_offset);
573768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox			nsegs++;
574768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox		}
575159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch
576159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch		if (split_len - length < bvec->bv_len)
577159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch			return nvme_split_and_submit(bio, nvmeq, i, split_len,
578159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch							split_len - length);
5791ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox		length += bvec->bv_len;
580768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox		bvprv = bvec;
581b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	}
582eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox	iod->nents = nsegs;
583768308400f5b4ce665a072eb976a851978b7706eMatthew Wilcox	sg_mark_end(sg);
584427e97080196548557b288517537ab7eb48c309fKeith Busch	if (dma_map_sg(nvmeq->q_dmadev, iod->sg, iod->nents, dma_dir) == 0)
5851ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox		return -ENOMEM;
586427e97080196548557b288517537ab7eb48c309fKeith Busch
587159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch	BUG_ON(length != bio->bi_size);
5881ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox	return length;
589b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}
590b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
5910e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch/*
5920e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch * We reuse the small pool to allocate the 16-byte range here as it is not
5930e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch * worth having a special pool for these or additional cases to handle freeing
5940e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch * the iod.
5950e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch */
5960e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Buschstatic int nvme_submit_discard(struct nvme_queue *nvmeq, struct nvme_ns *ns,
5970e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch		struct bio *bio, struct nvme_iod *iod, int cmdid)
5980e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch{
5990e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch	struct nvme_dsm_range *range;
6000e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch	struct nvme_command *cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail];
6010e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch
6020e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch	range = dma_pool_alloc(nvmeq->dev->prp_small_pool, GFP_ATOMIC,
6030e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch							&iod->first_dma);
6040e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch	if (!range)
6050e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch		return -ENOMEM;
6060e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch
6070e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch	iod_list(iod)[0] = (__le64 *)range;
6080e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch	iod->npages = 0;
6090e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch
6100e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch	range->cattr = cpu_to_le32(0);
6110e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch	range->nlb = cpu_to_le32(bio->bi_size >> ns->lba_shift);
612063cc6d5591ea9c0631b81ac5c7b829d99738b2fMatthew Wilcox	range->slba = cpu_to_le64(nvme_block_nr(ns, bio->bi_sector));
6130e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch
6140e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch	memset(cmnd, 0, sizeof(*cmnd));
6150e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch	cmnd->dsm.opcode = nvme_cmd_dsm;
6160e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch	cmnd->dsm.command_id = cmdid;
6170e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch	cmnd->dsm.nsid = cpu_to_le32(ns->ns_id);
6180e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch	cmnd->dsm.prp1 = cpu_to_le64(iod->first_dma);
6190e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch	cmnd->dsm.nr = 0;
6200e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch	cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD);
6210e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch
6220e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch	if (++nvmeq->sq_tail == nvmeq->q_depth)
6230e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch		nvmeq->sq_tail = 0;
6240e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch	writel(nvmeq->sq_tail, nvmeq->q_db);
6250e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch
6260e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch	return 0;
6270e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch}
6280e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch
62900df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcoxstatic int nvme_submit_flush(struct nvme_queue *nvmeq, struct nvme_ns *ns,
63000df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox								int cmdid)
63100df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox{
63200df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox	struct nvme_command *cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail];
63300df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox
63400df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox	memset(cmnd, 0, sizeof(*cmnd));
63500df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox	cmnd->common.opcode = nvme_cmd_flush;
63600df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox	cmnd->common.command_id = cmdid;
63700df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox	cmnd->common.nsid = cpu_to_le32(ns->ns_id);
63800df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox
63900df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox	if (++nvmeq->sq_tail == nvmeq->q_depth)
64000df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox		nvmeq->sq_tail = 0;
64100df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox	writel(nvmeq->sq_tail, nvmeq->q_db);
64200df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox
64300df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox	return 0;
64400df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox}
64500df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox
6465d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermaint nvme_submit_flush_data(struct nvme_queue *nvmeq, struct nvme_ns *ns)
64700df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox{
64800df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox	int cmdid = alloc_cmdid(nvmeq, (void *)CMD_CTX_FLUSH,
649ff976d724a74e4522e9ca2de1fb37ac4520f454fMatthew Wilcox					special_completion, NVME_IO_TIMEOUT);
65000df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox	if (unlikely(cmdid < 0))
65100df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox		return cmdid;
65200df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox
65300df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox	return nvme_submit_flush(nvmeq, ns, cmdid);
65400df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox}
65500df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox
656184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox/*
657184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox * Called with local interrupts disabled and the q_lock held.  May not sleep.
658184d2944cb3b92a2e8e1733c59d1e531ad6e924aMatthew Wilcox */
659b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_submit_bio_queue(struct nvme_queue *nvmeq, struct nvme_ns *ns,
660b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox								struct bio *bio)
661b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{
662ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox	struct nvme_command *cmnd;
663eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox	struct nvme_iod *iod;
664b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	enum dma_data_direction dma_dir;
6651287dabd345f447bbe0f7a99fc95ab89bcfc0f5dWei Yongjun	int cmdid, length, result;
666b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	u16 control;
667b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	u32 dsmgmt;
668b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	int psegs = bio_phys_segments(ns->queue, bio);
669b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
67000df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox	if ((bio->bi_rw & REQ_FLUSH) && psegs) {
67100df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox		result = nvme_submit_flush_data(nvmeq, ns);
67200df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox		if (result)
67300df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox			return result;
67400df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox	}
67500df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox
6761287dabd345f447bbe0f7a99fc95ab89bcfc0f5dWei Yongjun	result = -ENOMEM;
677eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox	iod = nvme_alloc_iod(psegs, bio->bi_size, GFP_ATOMIC);
678eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox	if (!iod)
679eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox		goto nomem;
680eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox	iod->private = bio;
681b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
682eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox	result = -EBUSY;
683ff976d724a74e4522e9ca2de1fb37ac4520f454fMatthew Wilcox	cmdid = alloc_cmdid(nvmeq, iod, bio_completion, NVME_IO_TIMEOUT);
684b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	if (unlikely(cmdid < 0))
685eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox		goto free_iod;
686b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
6870e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch	if (bio->bi_rw & REQ_DISCARD) {
6880e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch		result = nvme_submit_discard(nvmeq, ns, bio, iod, cmdid);
6890e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch		if (result)
6900e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch			goto free_cmdid;
6910e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch		return result;
6920e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch	}
69300df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox	if ((bio->bi_rw & REQ_FLUSH) && !psegs)
69400df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox		return nvme_submit_flush(nvmeq, ns, cmdid);
69500df5cb4eb927078850086f8becc3286a69ea12eMatthew Wilcox
696b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	control = 0;
697b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	if (bio->bi_rw & REQ_FUA)
698b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox		control |= NVME_RW_FUA;
699b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	if (bio->bi_rw & (REQ_FAILFAST_DEV | REQ_RAHEAD))
700b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox		control |= NVME_RW_LR;
701b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
702b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	dsmgmt = 0;
703b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	if (bio->bi_rw & REQ_RAHEAD)
704b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox		dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH;
705b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
706ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox	cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail];
707b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
708b8deb62cf271fa9381edc8cf52bcae2f0225c55aMatthew Wilcox	memset(cmnd, 0, sizeof(*cmnd));
709b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	if (bio_data_dir(bio)) {
710ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox		cmnd->rw.opcode = nvme_cmd_write;
711b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox		dma_dir = DMA_TO_DEVICE;
712b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	} else {
713ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox		cmnd->rw.opcode = nvme_cmd_read;
714b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox		dma_dir = DMA_FROM_DEVICE;
715b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	}
716b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
717427e97080196548557b288517537ab7eb48c309fKeith Busch	result = nvme_map_bio(nvmeq, iod, bio, dma_dir, psegs);
718427e97080196548557b288517537ab7eb48c309fKeith Busch	if (result <= 0)
719859361a228258edf4821d9f5635825033eca78e8Keith Busch		goto free_cmdid;
7201ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox	length = result;
721b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
722ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox	cmnd->rw.command_id = cmdid;
723ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox	cmnd->rw.nsid = cpu_to_le32(ns->ns_id);
724eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox	length = nvme_setup_prps(nvmeq->dev, &cmnd->common, iod, length,
725eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox								GFP_ATOMIC);
726063cc6d5591ea9c0631b81ac5c7b829d99738b2fMatthew Wilcox	cmnd->rw.slba = cpu_to_le64(nvme_block_nr(ns, bio->bi_sector));
7271ad2f8932a72bf375361727949ced2cb4e8cfcefMatthew Wilcox	cmnd->rw.length = cpu_to_le16((length >> ns->lba_shift) - 1);
728ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox	cmnd->rw.control = cpu_to_le16(control);
729ff22b54fda2078fc3cd1bcdcb7a5ce5d08fd6591Matthew Wilcox	cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt);
730b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
7316198221fa0df0298513b35796f63f242ea97134eKeith Busch	nvme_start_io_acct(bio);
732b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	if (++nvmeq->sq_tail == nvmeq->q_depth)
733b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox		nvmeq->sq_tail = 0;
7347547881d0951384f9833ec3a80fac8f3f16f3b98Matthew Wilcox	writel(nvmeq->sq_tail, nvmeq->q_db);
735b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
7361974b1ae8852324a75fb8cfecbc7b758fd5a2c3cMatthew Wilcox	return 0;
7371974b1ae8852324a75fb8cfecbc7b758fd5a2c3cMatthew Wilcox
738859361a228258edf4821d9f5635825033eca78e8Keith Busch free_cmdid:
739859361a228258edf4821d9f5635825033eca78e8Keith Busch	free_cmdid(nvmeq, cmdid, NULL);
740eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox free_iod:
741eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox	nvme_free_iod(nvmeq->dev, iod);
742eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox nomem:
743eeee322647a67c20d9277c5e02c42b2126ea74bcMatthew Wilcox	return result;
744b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}
745b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
746e9539f47525ecee05c9f22c3565885f3e9492c52Matthew Wilcoxstatic int nvme_process_cq(struct nvme_queue *nvmeq)
747b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{
748821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox	u16 head, phase;
749b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
750b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	head = nvmeq->cq_head;
751821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox	phase = nvmeq->cq_phase;
752b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
753b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	for (;;) {
754c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox		void *ctx;
755c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox		nvme_completion_fn fn;
756b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox		struct nvme_completion cqe = nvmeq->cqes[head];
757821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox		if ((le16_to_cpu(cqe.status) & 1) != phase)
758b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox			break;
759b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox		nvmeq->sq_head = le16_to_cpu(cqe.sq_head);
760b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox		if (++head == nvmeq->q_depth) {
761b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox			head = 0;
762821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox			phase = !phase;
763b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox		}
764b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
765c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox		ctx = free_cmdid(nvmeq, cqe.command_id, &fn);
7665c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcox		fn(nvmeq->dev, ctx, &cqe);
767b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	}
768b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
769b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	/* If the controller ignores the cq head doorbell and continuously
770b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	 * writes to the queue, it is theoretically possible to wrap around
771b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	 * the queue twice and mistakenly return IRQ_NONE.  Linux only
772b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	 * requires that 0.1% of your interrupts are handled, so this isn't
773b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	 * a big problem.
774b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	 */
775821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox	if (head == nvmeq->cq_head && phase == nvmeq->cq_phase)
776e9539f47525ecee05c9f22c3565885f3e9492c52Matthew Wilcox		return 0;
777b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
778b80d5ccca3a012e91ca64a2a0b13049163a6a698Haiyan Hu	writel(head, nvmeq->q_db + nvmeq->dev->db_stride);
779b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	nvmeq->cq_head = head;
780821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox	nvmeq->cq_phase = phase;
781b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
782e9539f47525ecee05c9f22c3565885f3e9492c52Matthew Wilcox	nvmeq->cqe_seen = 1;
783e9539f47525ecee05c9f22c3565885f3e9492c52Matthew Wilcox	return 1;
784b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}
785b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
7867d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcoxstatic void nvme_make_request(struct request_queue *q, struct bio *bio)
7877d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox{
7887d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox	struct nvme_ns *ns = q->queuedata;
7897d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox	struct nvme_queue *nvmeq = get_nvmeq(ns->dev);
7907d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox	int result = -EBUSY;
7917d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox
792cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch	if (!nvmeq) {
793cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch		put_nvmeq(NULL);
794cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch		bio_endio(bio, -EIO);
795cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch		return;
796cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch	}
797cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch
7987d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox	spin_lock_irq(&nvmeq->q_lock);
799224042742582c9938788b81165180c876e997a07Keith Busch	if (!nvmeq->q_suspended && bio_list_empty(&nvmeq->sq_cong))
8007d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox		result = nvme_submit_bio_queue(nvmeq, ns, bio);
8017d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox	if (unlikely(result)) {
8027d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox		if (bio_list_empty(&nvmeq->sq_cong))
8037d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox			add_wait_queue(&nvmeq->sq_full, &nvmeq->sq_cong_wait);
8047d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox		bio_list_add(&nvmeq->sq_cong, bio);
8057d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox	}
8067d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox
8077d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox	nvme_process_cq(nvmeq);
8087d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox	spin_unlock_irq(&nvmeq->q_lock);
8097d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox	put_nvmeq(nvmeq);
8107d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox}
8117d8224574cbd2326a6be00f319f5f7597abec3f6Matthew Wilcox
812b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic irqreturn_t nvme_irq(int irq, void *data)
813b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{
81458ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox	irqreturn_t result;
81558ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox	struct nvme_queue *nvmeq = data;
81658ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox	spin_lock(&nvmeq->q_lock);
817e9539f47525ecee05c9f22c3565885f3e9492c52Matthew Wilcox	nvme_process_cq(nvmeq);
818e9539f47525ecee05c9f22c3565885f3e9492c52Matthew Wilcox	result = nvmeq->cqe_seen ? IRQ_HANDLED : IRQ_NONE;
819e9539f47525ecee05c9f22c3565885f3e9492c52Matthew Wilcox	nvmeq->cqe_seen = 0;
82058ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox	spin_unlock(&nvmeq->q_lock);
82158ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox	return result;
82258ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox}
82358ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox
82458ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcoxstatic irqreturn_t nvme_irq_check(int irq, void *data)
82558ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox{
82658ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox	struct nvme_queue *nvmeq = data;
82758ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox	struct nvme_completion cqe = nvmeq->cqes[nvmeq->cq_head];
82858ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox	if ((le16_to_cpu(cqe.status) & 1) != nvmeq->cq_phase)
82958ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox		return IRQ_NONE;
83058ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox	return IRQ_WAKE_THREAD;
83158ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox}
83258ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox
8333c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcoxstatic void nvme_abort_command(struct nvme_queue *nvmeq, int cmdid)
8343c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox{
8353c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox	spin_lock_irq(&nvmeq->q_lock);
836c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox	cancel_cmdid(nvmeq, cmdid, NULL);
8373c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox	spin_unlock_irq(&nvmeq->q_lock);
8383c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox}
8393c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox
840c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcoxstruct sync_cmd_info {
841c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox	struct task_struct *task;
842c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox	u32 result;
843c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox	int status;
844c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox};
845c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox
8465c1281a3bf5655ec1b90db495da3a2b77826ba88Matthew Wilcoxstatic void sync_completion(struct nvme_dev *dev, void *ctx,
847c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox						struct nvme_completion *cqe)
848c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox{
849c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox	struct sync_cmd_info *cmdinfo = ctx;
850c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox	cmdinfo->result = le32_to_cpup(&cqe->result);
851c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox	cmdinfo->status = le16_to_cpup(&cqe->status) >> 1;
852c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox	wake_up_process(cmdinfo->task);
853c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox}
854c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox
855b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/*
856b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * Returns 0 on success.  If the result is negative, it's a Linux error code;
857b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox * if the result is positive, it's an NVM Express status code
858b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox */
8595d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermaint nvme_submit_sync_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd,
8605d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma						u32 *result, unsigned timeout)
861b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{
862b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	int cmdid;
863b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	struct sync_cmd_info cmdinfo;
864b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
865b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	cmdinfo.task = current;
866b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	cmdinfo.status = -EINTR;
867b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
868c2f5b65020869215814df03c3941dac9436f99fbMatthew Wilcox	cmdid = alloc_cmdid_killable(nvmeq, &cmdinfo, sync_completion,
869e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox								timeout);
870b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	if (cmdid < 0)
871b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox		return cmdid;
872b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	cmd->common.command_id = cmdid;
873b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
8743c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox	set_current_state(TASK_KILLABLE);
8753c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox	nvme_submit_cmd(nvmeq, cmd);
87678f8d2577bd79ce9d62f4d9e6d3b895bd1dd1d1dKeith Busch	schedule_timeout(timeout);
877b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
8783c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox	if (cmdinfo.status == -EINTR) {
8793c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox		nvme_abort_command(nvmeq, cmdid);
8803c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox		return -EINTR;
8813c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox	}
8823c0cf138d7789feb3f335f6f1d24ad8fc8b3a23fMatthew Wilcox
883b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	if (result)
884b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox		*result = cmdinfo.result;
885b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
886b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	return cmdinfo.status;
887b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}
888b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
8895d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermaint nvme_submit_admin_cmd(struct nvme_dev *dev, struct nvme_command *cmd,
890b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox								u32 *result)
891b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{
892e85248e516c550382ba33ca325c272a0ca397e44Matthew Wilcox	return nvme_submit_sync_cmd(dev->queues[0], cmd, result, ADMIN_TIMEOUT);
893b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}
894b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
895b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id)
896b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{
897b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	int status;
898b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	struct nvme_command c;
899b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
900b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	memset(&c, 0, sizeof(c));
901b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	c.delete_queue.opcode = opcode;
902b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	c.delete_queue.qid = cpu_to_le16(id);
903b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
904b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	status = nvme_submit_admin_cmd(dev, &c, NULL);
905b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	if (status)
906b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox		return -EIO;
907b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	return 0;
908b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}
909b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
910b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_alloc_cq(struct nvme_dev *dev, u16 qid,
911b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox						struct nvme_queue *nvmeq)
912b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{
913b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	int status;
914b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	struct nvme_command c;
915b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	int flags = NVME_QUEUE_PHYS_CONTIG | NVME_CQ_IRQ_ENABLED;
916b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
917b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	memset(&c, 0, sizeof(c));
918b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	c.create_cq.opcode = nvme_admin_create_cq;
919b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	c.create_cq.prp1 = cpu_to_le64(nvmeq->cq_dma_addr);
920b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	c.create_cq.cqid = cpu_to_le16(qid);
921b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
922b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	c.create_cq.cq_flags = cpu_to_le16(flags);
923b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	c.create_cq.irq_vector = cpu_to_le16(nvmeq->cq_vector);
924b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
925b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	status = nvme_submit_admin_cmd(dev, &c, NULL);
926b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	if (status)
927b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox		return -EIO;
928b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	return 0;
929b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}
930b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
931b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_alloc_sq(struct nvme_dev *dev, u16 qid,
932b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox						struct nvme_queue *nvmeq)
933b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{
934b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	int status;
935b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	struct nvme_command c;
936b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	int flags = NVME_QUEUE_PHYS_CONTIG | NVME_SQ_PRIO_MEDIUM;
937b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
938b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	memset(&c, 0, sizeof(c));
939b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	c.create_sq.opcode = nvme_admin_create_sq;
940b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	c.create_sq.prp1 = cpu_to_le64(nvmeq->sq_dma_addr);
941b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	c.create_sq.sqid = cpu_to_le16(qid);
942b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
943b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	c.create_sq.sq_flags = cpu_to_le16(flags);
944b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	c.create_sq.cqid = cpu_to_le16(qid);
945b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
946b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	status = nvme_submit_admin_cmd(dev, &c, NULL);
947b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	if (status)
948b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox		return -EIO;
949b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	return 0;
950b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}
951b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
952b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_delete_cq(struct nvme_dev *dev, u16 cqid)
953b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{
954b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	return adapter_delete_queue(dev, nvme_admin_delete_cq, cqid);
955b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}
956b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
957b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int adapter_delete_sq(struct nvme_dev *dev, u16 sqid)
958b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{
959b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	return adapter_delete_queue(dev, nvme_admin_delete_sq, sqid);
960b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}
961b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
9625d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermaint nvme_identify(struct nvme_dev *dev, unsigned nsid, unsigned cns,
963bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox							dma_addr_t dma_addr)
964bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox{
965bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox	struct nvme_command c;
966bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox
967bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox	memset(&c, 0, sizeof(c));
968bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox	c.identify.opcode = nvme_admin_identify;
969bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox	c.identify.nsid = cpu_to_le32(nsid);
970bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox	c.identify.prp1 = cpu_to_le64(dma_addr);
971bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox	c.identify.cns = cpu_to_le32(cns);
972bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox
973bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox	return nvme_submit_admin_cmd(dev, &c, NULL);
974bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox}
975bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox
9765d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermaint nvme_get_features(struct nvme_dev *dev, unsigned fid, unsigned nsid,
97708df1e05657fc6712e520e7c09cc6c86160ceb35Keith Busch					dma_addr_t dma_addr, u32 *result)
978bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox{
979bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox	struct nvme_command c;
980bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox
981bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox	memset(&c, 0, sizeof(c));
982bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox	c.features.opcode = nvme_admin_get_features;
983a42ceccef0c43b46ff6bc1b12a7c1076ef243df1Keith Busch	c.features.nsid = cpu_to_le32(nsid);
984bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox	c.features.prp1 = cpu_to_le64(dma_addr);
985bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox	c.features.fid = cpu_to_le32(fid);
986bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox
98708df1e05657fc6712e520e7c09cc6c86160ceb35Keith Busch	return nvme_submit_admin_cmd(dev, &c, result);
988df3481399042200792822b6243e36a95a557b57eMatthew Wilcox}
989df3481399042200792822b6243e36a95a557b57eMatthew Wilcox
9905d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermaint nvme_set_features(struct nvme_dev *dev, unsigned fid, unsigned dword11,
9915d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma					dma_addr_t dma_addr, u32 *result)
992df3481399042200792822b6243e36a95a557b57eMatthew Wilcox{
993df3481399042200792822b6243e36a95a557b57eMatthew Wilcox	struct nvme_command c;
994df3481399042200792822b6243e36a95a557b57eMatthew Wilcox
995df3481399042200792822b6243e36a95a557b57eMatthew Wilcox	memset(&c, 0, sizeof(c));
996df3481399042200792822b6243e36a95a557b57eMatthew Wilcox	c.features.opcode = nvme_admin_set_features;
997df3481399042200792822b6243e36a95a557b57eMatthew Wilcox	c.features.prp1 = cpu_to_le64(dma_addr);
998df3481399042200792822b6243e36a95a557b57eMatthew Wilcox	c.features.fid = cpu_to_le32(fid);
999df3481399042200792822b6243e36a95a557b57eMatthew Wilcox	c.features.dword11 = cpu_to_le32(dword11);
1000df3481399042200792822b6243e36a95a557b57eMatthew Wilcox
1001bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox	return nvme_submit_admin_cmd(dev, &c, result);
1002bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox}
1003bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox
1004a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox/**
1005a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox * nvme_cancel_ios - Cancel outstanding I/Os
1006a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox * @queue: The queue to cancel I/Os on
1007a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox * @timeout: True to only cancel I/Os which have timed out
1008a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox */
1009a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcoxstatic void nvme_cancel_ios(struct nvme_queue *nvmeq, bool timeout)
1010a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox{
1011a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox	int depth = nvmeq->q_depth - 1;
1012a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox	struct nvme_cmd_info *info = nvme_cmd_info(nvmeq);
1013a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox	unsigned long now = jiffies;
1014a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox	int cmdid;
1015a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox
1016a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox	for_each_set_bit(cmdid, nvmeq->cmdid_data, depth) {
1017a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox		void *ctx;
1018a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox		nvme_completion_fn fn;
1019a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox		static struct nvme_completion cqe = {
1020af2d9ca744af5e03390eeb3864d08ce75c860899Matthew Wilcox			.status = cpu_to_le16(NVME_SC_ABORT_REQ << 1),
1021a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox		};
1022a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox
1023a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox		if (timeout && !time_after(now, info[cmdid].timeout))
1024a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox			continue;
1025053ab702cc2702f25a97ead087ed344b864785b7Keith Busch		if (info[cmdid].ctx == CMD_CTX_CANCELLED)
1026053ab702cc2702f25a97ead087ed344b864785b7Keith Busch			continue;
1027a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox		dev_warn(nvmeq->q_dmadev, "Cancelling I/O %d\n", cmdid);
1028a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox		ctx = cancel_cmdid(nvmeq, cmdid, &fn);
1029a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox		fn(nvmeq->dev, ctx, &cqe);
1030a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox	}
1031a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox}
1032a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox
1033224042742582c9938788b81165180c876e997a07Keith Buschstatic void nvme_free_queue(struct nvme_queue *nvmeq)
10349e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox{
1035224042742582c9938788b81165180c876e997a07Keith Busch	spin_lock_irq(&nvmeq->q_lock);
1036224042742582c9938788b81165180c876e997a07Keith Busch	while (bio_list_peek(&nvmeq->sq_cong)) {
1037224042742582c9938788b81165180c876e997a07Keith Busch		struct bio *bio = bio_list_pop(&nvmeq->sq_cong);
1038224042742582c9938788b81165180c876e997a07Keith Busch		bio_endio(bio, -EIO);
1039224042742582c9938788b81165180c876e997a07Keith Busch	}
1040224042742582c9938788b81165180c876e997a07Keith Busch	spin_unlock_irq(&nvmeq->q_lock);
1041224042742582c9938788b81165180c876e997a07Keith Busch
10429e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox	dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth),
10439e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox				(void *)nvmeq->cqes, nvmeq->cq_dma_addr);
10449e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox	dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth),
10459e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox					nvmeq->sq_cmds, nvmeq->sq_dma_addr);
10469e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox	kfree(nvmeq);
10479e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox}
10489e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox
1049224042742582c9938788b81165180c876e997a07Keith Buschstatic void nvme_free_queues(struct nvme_dev *dev)
1050224042742582c9938788b81165180c876e997a07Keith Busch{
1051224042742582c9938788b81165180c876e997a07Keith Busch	int i;
1052224042742582c9938788b81165180c876e997a07Keith Busch
1053224042742582c9938788b81165180c876e997a07Keith Busch	for (i = dev->queue_count - 1; i >= 0; i--) {
1054224042742582c9938788b81165180c876e997a07Keith Busch		nvme_free_queue(dev->queues[i]);
1055224042742582c9938788b81165180c876e997a07Keith Busch		dev->queue_count--;
1056224042742582c9938788b81165180c876e997a07Keith Busch		dev->queues[i] = NULL;
1057224042742582c9938788b81165180c876e997a07Keith Busch	}
1058224042742582c9938788b81165180c876e997a07Keith Busch}
1059224042742582c9938788b81165180c876e997a07Keith Busch
1060224042742582c9938788b81165180c876e997a07Keith Buschstatic void nvme_disable_queue(struct nvme_dev *dev, int qid)
1061b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{
1062b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	struct nvme_queue *nvmeq = dev->queues[qid];
1063aba2080f3f1639f9202f1a52993669844abcfb80Matthew Wilcox	int vector = dev->entry[nvmeq->cq_vector].vector;
1064b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
1065a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox	spin_lock_irq(&nvmeq->q_lock);
1066224042742582c9938788b81165180c876e997a07Keith Busch	if (nvmeq->q_suspended) {
1067224042742582c9938788b81165180c876e997a07Keith Busch		spin_unlock_irq(&nvmeq->q_lock);
1068224042742582c9938788b81165180c876e997a07Keith Busch		return;
10693295874b6074d749516d6decd43afad7bf6e38ffKeith Busch	}
1070224042742582c9938788b81165180c876e997a07Keith Busch	nvmeq->q_suspended = 1;
1071a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox	spin_unlock_irq(&nvmeq->q_lock);
1072a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox
1073aba2080f3f1639f9202f1a52993669844abcfb80Matthew Wilcox	irq_set_affinity_hint(vector, NULL);
1074aba2080f3f1639f9202f1a52993669844abcfb80Matthew Wilcox	free_irq(vector, nvmeq);
1075b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
1076b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	/* Don't tell the adapter to delete the admin queue */
1077b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	if (qid) {
1078b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox		adapter_delete_sq(dev, qid);
1079b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox		adapter_delete_cq(dev, qid);
1080b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	}
1081b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
1082224042742582c9938788b81165180c876e997a07Keith Busch	spin_lock_irq(&nvmeq->q_lock);
1083224042742582c9938788b81165180c876e997a07Keith Busch	nvme_process_cq(nvmeq);
1084224042742582c9938788b81165180c876e997a07Keith Busch	nvme_cancel_ios(nvmeq, false);
1085224042742582c9938788b81165180c876e997a07Keith Busch	spin_unlock_irq(&nvmeq->q_lock);
1086b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}
1087b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
1088b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev, int qid,
1089b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox							int depth, int vector)
1090b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{
1091b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	struct device *dmadev = &dev->pci_dev->dev;
1092224042742582c9938788b81165180c876e997a07Keith Busch	unsigned extra = nvme_queue_extra(depth);
1093b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	struct nvme_queue *nvmeq = kzalloc(sizeof(*nvmeq) + extra, GFP_KERNEL);
1094b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	if (!nvmeq)
1095b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox		return NULL;
1096b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
1097b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	nvmeq->cqes = dma_alloc_coherent(dmadev, CQ_SIZE(depth),
1098b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox					&nvmeq->cq_dma_addr, GFP_KERNEL);
1099b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	if (!nvmeq->cqes)
1100b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox		goto free_nvmeq;
1101b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	memset((void *)nvmeq->cqes, 0, CQ_SIZE(depth));
1102b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
1103b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	nvmeq->sq_cmds = dma_alloc_coherent(dmadev, SQ_SIZE(depth),
1104b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox					&nvmeq->sq_dma_addr, GFP_KERNEL);
1105b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	if (!nvmeq->sq_cmds)
1106b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox		goto free_cqdma;
1107b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
1108b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	nvmeq->q_dmadev = dmadev;
1109091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox	nvmeq->dev = dev;
1110b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	spin_lock_init(&nvmeq->q_lock);
1111b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	nvmeq->cq_head = 0;
1112821234603b265f59d7eebce16d9e8beca2a5752dMatthew Wilcox	nvmeq->cq_phase = 1;
1113b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	init_waitqueue_head(&nvmeq->sq_full);
11141fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox	init_waitqueue_entry(&nvmeq->sq_cong_wait, nvme_thread);
1115b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	bio_list_init(&nvmeq->sq_cong);
1116b80d5ccca3a012e91ca64a2a0b13049163a6a698Haiyan Hu	nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
1117b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	nvmeq->q_depth = depth;
1118b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	nvmeq->cq_vector = vector;
1119224042742582c9938788b81165180c876e997a07Keith Busch	nvmeq->q_suspended = 1;
1120224042742582c9938788b81165180c876e997a07Keith Busch	dev->queue_count++;
1121b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
1122b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	return nvmeq;
1123b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
1124b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox free_cqdma:
112568b8eca5f882230e4991df0db0a3094f49420a90Keith Busch	dma_free_coherent(dmadev, CQ_SIZE(depth), (void *)nvmeq->cqes,
1126b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox							nvmeq->cq_dma_addr);
1127b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox free_nvmeq:
1128b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	kfree(nvmeq);
1129b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	return NULL;
1130b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}
1131b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
11323001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcoxstatic int queue_request_irq(struct nvme_dev *dev, struct nvme_queue *nvmeq,
11333001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox							const char *name)
11343001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox{
113558ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox	if (use_threaded_interrupts)
113658ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox		return request_threaded_irq(dev->entry[nvmeq->cq_vector].vector,
1137481e5bad84239b01415b888df2040c1860ae3cfdMichael Opdenacker					nvme_irq_check, nvme_irq, IRQF_SHARED,
113858ffacb545f76fc2c65d1fbfa5acf5184a2a09e6Matthew Wilcox					name, nvmeq);
11393001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox	return request_irq(dev->entry[nvmeq->cq_vector].vector, nvme_irq,
1140481e5bad84239b01415b888df2040c1860ae3cfdMichael Opdenacker				IRQF_SHARED, name, nvmeq);
11413001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox}
11423001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox
1143224042742582c9938788b81165180c876e997a07Keith Buschstatic void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid)
1144b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{
1145224042742582c9938788b81165180c876e997a07Keith Busch	struct nvme_dev *dev = nvmeq->dev;
1146224042742582c9938788b81165180c876e997a07Keith Busch	unsigned extra = nvme_queue_extra(nvmeq->q_depth);
1147b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
1148224042742582c9938788b81165180c876e997a07Keith Busch	nvmeq->sq_tail = 0;
1149224042742582c9938788b81165180c876e997a07Keith Busch	nvmeq->cq_head = 0;
1150224042742582c9938788b81165180c876e997a07Keith Busch	nvmeq->cq_phase = 1;
1151b80d5ccca3a012e91ca64a2a0b13049163a6a698Haiyan Hu	nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
1152224042742582c9938788b81165180c876e997a07Keith Busch	memset(nvmeq->cmdid_data, 0, extra);
1153224042742582c9938788b81165180c876e997a07Keith Busch	memset((void *)nvmeq->cqes, 0, CQ_SIZE(nvmeq->q_depth));
1154224042742582c9938788b81165180c876e997a07Keith Busch	nvme_cancel_ios(nvmeq, false);
1155224042742582c9938788b81165180c876e997a07Keith Busch	nvmeq->q_suspended = 0;
1156224042742582c9938788b81165180c876e997a07Keith Busch}
1157224042742582c9938788b81165180c876e997a07Keith Busch
1158224042742582c9938788b81165180c876e997a07Keith Buschstatic int nvme_create_queue(struct nvme_queue *nvmeq, int qid)
1159224042742582c9938788b81165180c876e997a07Keith Busch{
1160224042742582c9938788b81165180c876e997a07Keith Busch	struct nvme_dev *dev = nvmeq->dev;
1161224042742582c9938788b81165180c876e997a07Keith Busch	int result;
11623f85d50b609e8a5ef151656210203a6e94c19538Matthew Wilcox
1163b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	result = adapter_alloc_cq(dev, qid, nvmeq);
1164b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	if (result < 0)
1165224042742582c9938788b81165180c876e997a07Keith Busch		return result;
1166b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
1167b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	result = adapter_alloc_sq(dev, qid, nvmeq);
1168b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	if (result < 0)
1169b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox		goto release_cq;
1170b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
11713001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox	result = queue_request_irq(dev, nvmeq, "nvme");
1172b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	if (result < 0)
1173b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox		goto release_sq;
1174b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
11750a8d44cb33377969337fa6c1961b631605d5f453Matthew Wilcox	spin_lock_irq(&nvmeq->q_lock);
1176224042742582c9938788b81165180c876e997a07Keith Busch	nvme_init_queue(nvmeq, qid);
11770a8d44cb33377969337fa6c1961b631605d5f453Matthew Wilcox	spin_unlock_irq(&nvmeq->q_lock);
1178224042742582c9938788b81165180c876e997a07Keith Busch
1179224042742582c9938788b81165180c876e997a07Keith Busch	return result;
1180b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
1181b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox release_sq:
1182b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	adapter_delete_sq(dev, qid);
1183b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox release_cq:
1184b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	adapter_delete_cq(dev, qid);
1185224042742582c9938788b81165180c876e997a07Keith Busch	return result;
1186b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}
1187b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
1188ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcoxstatic int nvme_wait_ready(struct nvme_dev *dev, u64 cap, bool enabled)
1189ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox{
1190ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox	unsigned long timeout;
1191ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox	u32 bit = enabled ? NVME_CSTS_RDY : 0;
1192ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox
1193ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox	timeout = ((NVME_CAP_TIMEOUT(cap) + 1) * HZ / 2) + jiffies;
1194ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox
1195ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox	while ((readl(&dev->bar->csts) & NVME_CSTS_RDY) != bit) {
1196ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox		msleep(100);
1197ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox		if (fatal_signal_pending(current))
1198ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox			return -EINTR;
1199ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox		if (time_after(jiffies, timeout)) {
1200ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox			dev_err(&dev->pci_dev->dev,
1201ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox				"Device not ready; aborting initialisation\n");
1202ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox			return -ENODEV;
1203ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox		}
1204ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox	}
1205ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox
1206ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox	return 0;
1207ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox}
1208ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox
1209ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox/*
1210ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox * If the device has been passed off to us in an enabled state, just clear
1211ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox * the enabled bit.  The spec says we should set the 'shutdown notification
1212ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox * bits', but doing so may cause the device to complete commands to the
1213ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox * admin queue ... and we don't know what memory that might be pointing at!
1214ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox */
1215ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcoxstatic int nvme_disable_ctrl(struct nvme_dev *dev, u64 cap)
1216ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox{
121744af146a84fa4a8e136d824207dcd356958a112bMatthew Wilcox	u32 cc = readl(&dev->bar->cc);
121844af146a84fa4a8e136d824207dcd356958a112bMatthew Wilcox
121944af146a84fa4a8e136d824207dcd356958a112bMatthew Wilcox	if (cc & NVME_CC_ENABLE)
122044af146a84fa4a8e136d824207dcd356958a112bMatthew Wilcox		writel(cc & ~NVME_CC_ENABLE, &dev->bar->cc);
1221ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox	return nvme_wait_ready(dev, cap, false);
1222ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox}
1223ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox
1224ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcoxstatic int nvme_enable_ctrl(struct nvme_dev *dev, u64 cap)
1225ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox{
1226ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox	return nvme_wait_ready(dev, cap, true);
1227ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox}
1228ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox
12291894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Buschstatic int nvme_shutdown_ctrl(struct nvme_dev *dev)
12301894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch{
12311894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch	unsigned long timeout;
12321894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch	u32 cc;
12331894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch
12341894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch	cc = (readl(&dev->bar->cc) & ~NVME_CC_SHN_MASK) | NVME_CC_SHN_NORMAL;
12351894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch	writel(cc, &dev->bar->cc);
12361894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch
12371894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch	timeout = 2 * HZ + jiffies;
12381894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch	while ((readl(&dev->bar->csts) & NVME_CSTS_SHST_MASK) !=
12391894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch							NVME_CSTS_SHST_CMPLT) {
12401894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch		msleep(100);
12411894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch		if (fatal_signal_pending(current))
12421894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch			return -EINTR;
12431894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch		if (time_after(jiffies, timeout)) {
12441894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch			dev_err(&dev->pci_dev->dev,
12451894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch				"Device shutdown incomplete; abort shutdown\n");
12461894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch			return -ENODEV;
12471894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch		}
12481894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch	}
12491894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch
12501894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch	return 0;
12511894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch}
12521894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch
12538d85fce77edfc22f1d6dbf78e3af723b4b556f3dGreg Kroah-Hartmanstatic int nvme_configure_admin_queue(struct nvme_dev *dev)
1254b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{
1255ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox	int result;
1256b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	u32 aqa;
1257ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox	u64 cap = readq(&dev->bar->cap);
1258b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	struct nvme_queue *nvmeq;
1259b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
1260ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox	result = nvme_disable_ctrl(dev, cap);
1261ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox	if (result < 0)
1262ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox		return result;
1263b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
1264cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch	nvmeq = dev->queues[0];
1265cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch	if (!nvmeq) {
1266cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch		nvmeq = nvme_alloc_queue(dev, 0, 64, 0);
1267cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch		if (!nvmeq)
1268cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch			return -ENOMEM;
1269cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch		dev->queues[0] = nvmeq;
1270cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch	}
1271b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
1272b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	aqa = nvmeq->q_depth - 1;
1273b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	aqa |= aqa << 16;
1274b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
1275b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	dev->ctrl_config = NVME_CC_ENABLE | NVME_CC_CSS_NVM;
1276b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	dev->ctrl_config |= (PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT;
1277b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	dev->ctrl_config |= NVME_CC_ARB_RR | NVME_CC_SHN_NONE;
12787f53f9d2424533256ae86f7df5661a17de743de8Matthew Wilcox	dev->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
1279b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
1280b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	writel(aqa, &dev->bar->aqa);
1281b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	writeq(nvmeq->sq_dma_addr, &dev->bar->asq);
1282b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	writeq(nvmeq->cq_dma_addr, &dev->bar->acq);
1283b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	writel(dev->ctrl_config, &dev->bar->cc);
1284b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
1285ba47e3865e8023e79b670793a41508222b5f0322Matthew Wilcox	result = nvme_enable_ctrl(dev, cap);
1286025c557a71bd06a9f6d32259f00e2218b15bf0a4Keith Busch	if (result)
1287cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch		return result;
12889e866774aab5d2654b0fa8f97890f68913f05700Matthew Wilcox
12893001082cac4bf6ffd09f72b39e6292ad6394ef17Matthew Wilcox	result = queue_request_irq(dev, nvmeq, "nvme admin");
1290025c557a71bd06a9f6d32259f00e2218b15bf0a4Keith Busch	if (result)
1291cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch		return result;
1292025c557a71bd06a9f6d32259f00e2218b15bf0a4Keith Busch
12930a8d44cb33377969337fa6c1961b631605d5f453Matthew Wilcox	spin_lock_irq(&nvmeq->q_lock);
1294224042742582c9938788b81165180c876e997a07Keith Busch	nvme_init_queue(nvmeq, 0);
12950a8d44cb33377969337fa6c1961b631605d5f453Matthew Wilcox	spin_unlock_irq(&nvmeq->q_lock);
1296b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	return result;
1297b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}
1298b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
12995d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermastruct nvme_iod *nvme_map_user_pages(struct nvme_dev *dev, int write,
1300eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox				unsigned long addr, unsigned length)
1301b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{
130236c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox	int i, err, count, nents, offset;
13037fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox	struct scatterlist *sg;
13047fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox	struct page **pages;
1305eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox	struct nvme_iod *iod;
130636c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox
130736c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox	if (addr & 3)
1308eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox		return ERR_PTR(-EINVAL);
13095460fc03105fbed01fe27aa572d9f65bb410a61dDan Carpenter	if (!length || length > INT_MAX - PAGE_SIZE)
1310eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox		return ERR_PTR(-EINVAL);
13117fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox
131236c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox	offset = offset_in_page(addr);
13137fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox	count = DIV_ROUND_UP(offset + length, PAGE_SIZE);
13147fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox	pages = kcalloc(count, sizeof(*pages), GFP_KERNEL);
131522fff826e715e9727d3c7a69f15e602a9801b673Dan Carpenter	if (!pages)
131622fff826e715e9727d3c7a69f15e602a9801b673Dan Carpenter		return ERR_PTR(-ENOMEM);
131736c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox
131836c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox	err = get_user_pages_fast(addr, count, 1, pages);
131936c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox	if (err < count) {
132036c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox		count = err;
132136c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox		err = -EFAULT;
132236c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox		goto put_pages;
132336c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox	}
13247fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox
1325eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox	iod = nvme_alloc_iod(count, length, GFP_KERNEL);
1326eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox	sg = iod->sg;
132736c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox	sg_init_table(sg, count);
1328d0ba1e497bca83a3d353eb47c9658afc54d83228Matthew Wilcox	for (i = 0; i < count; i++) {
1329d0ba1e497bca83a3d353eb47c9658afc54d83228Matthew Wilcox		sg_set_page(&sg[i], pages[i],
13305460fc03105fbed01fe27aa572d9f65bb410a61dDan Carpenter			    min_t(unsigned, length, PAGE_SIZE - offset),
13315460fc03105fbed01fe27aa572d9f65bb410a61dDan Carpenter			    offset);
1332d0ba1e497bca83a3d353eb47c9658afc54d83228Matthew Wilcox		length -= (PAGE_SIZE - offset);
1333d0ba1e497bca83a3d353eb47c9658afc54d83228Matthew Wilcox		offset = 0;
13347fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox	}
1335fe304c43c6d63e29ed4fc46a874d7a74313788c5Matthew Wilcox	sg_mark_end(&sg[i - 1]);
13361c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox	iod->nents = count;
13377fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox
13387fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox	err = -ENOMEM;
13397fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox	nents = dma_map_sg(&dev->pci_dev->dev, sg, count,
13407fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox				write ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
134136c14ed9caa957c686d4a48fd598a5ec2aa0331bMatthew Wilcox	if (!nents)
1342eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox		goto free_iod;
1343b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
13447fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox	kfree(pages);
1345eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox	return iod;
1346b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
1347eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox free_iod:
1348eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox	kfree(iod);
13497fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox put_pages:
13507fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox	for (i = 0; i < count; i++)
13517fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox		put_page(pages[i]);
13527fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox	kfree(pages);
1353eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox	return ERR_PTR(err);
13547fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox}
1355b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
13565d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Vermavoid nvme_unmap_user_pages(struct nvme_dev *dev, int write,
13571c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox			struct nvme_iod *iod)
13587fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox{
13591c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox	int i;
1360b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
13611c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox	dma_unmap_sg(&dev->pci_dev->dev, iod->sg, iod->nents,
13621c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox				write ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
13637fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox
13641c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox	for (i = 0; i < iod->nents; i++)
13651c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox		put_page(sg_page(&iod->sg[i]));
13667fc3cdabba75c2516b8b645eb0ca7907aea70415Matthew Wilcox}
1367b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
1368a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcoxstatic int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio)
1369a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox{
1370a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox	struct nvme_dev *dev = ns->dev;
1371a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox	struct nvme_queue *nvmeq;
1372a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox	struct nvme_user_io io;
1373a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox	struct nvme_command c;
1374f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch	unsigned length, meta_len;
1375f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch	int status, i;
1376f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch	struct nvme_iod *iod, *meta_iod = NULL;
1377f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch	dma_addr_t meta_dma_addr;
1378f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch	void *meta, *uninitialized_var(meta_mem);
1379a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox
1380a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox	if (copy_from_user(&io, uio, sizeof(io)))
1381a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox		return -EFAULT;
13826c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox	length = (io.nblocks + 1) << ns->lba_shift;
1383f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch	meta_len = (io.nblocks + 1) * ns->ms;
1384f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch
1385f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch	if (meta_len && ((io.metadata & 3) || !io.metadata))
1386f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch		return -EINVAL;
13876c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox
13886c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox	switch (io.opcode) {
13896c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox	case nvme_cmd_write:
13906c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox	case nvme_cmd_read:
13916bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox	case nvme_cmd_compare:
1392eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox		iod = nvme_map_user_pages(dev, io.opcode & 1, io.addr, length);
13936413214c5d424fd5aae6567848340f962ad2ce0fMatthew Wilcox		break;
13946c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox	default:
13956bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox		return -EINVAL;
13966c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox	}
13976c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox
1398eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox	if (IS_ERR(iod))
1399eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox		return PTR_ERR(iod);
1400a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox
1401a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox	memset(&c, 0, sizeof(c));
1402a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox	c.rw.opcode = io.opcode;
1403a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox	c.rw.flags = io.flags;
14046c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox	c.rw.nsid = cpu_to_le32(ns->ns_id);
1405a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox	c.rw.slba = cpu_to_le64(io.slba);
14066c7d49455ceb63064f992347d9185ff5bf43497aMatthew Wilcox	c.rw.length = cpu_to_le16(io.nblocks);
1407a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox	c.rw.control = cpu_to_le16(io.control);
14081c9b52651dad0ff1fa71fc6205c86d972f25bcc0Matthew Wilcox	c.rw.dsmgmt = cpu_to_le32(io.dsmgmt);
14091c9b52651dad0ff1fa71fc6205c86d972f25bcc0Matthew Wilcox	c.rw.reftag = cpu_to_le32(io.reftag);
14101c9b52651dad0ff1fa71fc6205c86d972f25bcc0Matthew Wilcox	c.rw.apptag = cpu_to_le16(io.apptag);
14111c9b52651dad0ff1fa71fc6205c86d972f25bcc0Matthew Wilcox	c.rw.appmask = cpu_to_le16(io.appmask);
1412f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch
1413f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch	if (meta_len) {
14141b56749e541ad59068582f2a28297843e243b856Keith Busch		meta_iod = nvme_map_user_pages(dev, io.opcode & 1, io.metadata,
14151b56749e541ad59068582f2a28297843e243b856Keith Busch								meta_len);
1416f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch		if (IS_ERR(meta_iod)) {
1417f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch			status = PTR_ERR(meta_iod);
1418f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch			meta_iod = NULL;
1419f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch			goto unmap;
1420f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch		}
1421f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch
1422f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch		meta_mem = dma_alloc_coherent(&dev->pci_dev->dev, meta_len,
1423f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch						&meta_dma_addr, GFP_KERNEL);
1424f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch		if (!meta_mem) {
1425f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch			status = -ENOMEM;
1426f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch			goto unmap;
1427f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch		}
1428f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch
1429f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch		if (io.opcode & 1) {
1430f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch			int meta_offset = 0;
1431f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch
1432f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch			for (i = 0; i < meta_iod->nents; i++) {
1433f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch				meta = kmap_atomic(sg_page(&meta_iod->sg[i])) +
1434f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch						meta_iod->sg[i].offset;
1435f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch				memcpy(meta_mem + meta_offset, meta,
1436f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch						meta_iod->sg[i].length);
1437f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch				kunmap_atomic(meta);
1438f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch				meta_offset += meta_iod->sg[i].length;
1439f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch			}
1440f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch		}
1441f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch
1442f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch		c.rw.metadata = cpu_to_le64(meta_dma_addr);
1443f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch	}
1444f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch
1445eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox	length = nvme_setup_prps(dev, &c.common, iod, length, GFP_KERNEL);
1446a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox
1447040a93b52a9eee8177ebaf2ba0ee0f9f518d1bf8Matthew Wilcox	nvmeq = get_nvmeq(dev);
1448fa92282149842645931580225647238428374758Matthew Wilcox	/*
1449fa92282149842645931580225647238428374758Matthew Wilcox	 * Since nvme_submit_sync_cmd sleeps, we can't keep preemption
1450b1ad37efcafe396ac3944853589688dd0ec3c64eMatthew Wilcox	 * disabled.  We may be preempted at any point, and be rescheduled
1451b1ad37efcafe396ac3944853589688dd0ec3c64eMatthew Wilcox	 * to a different CPU.  That will cause cacheline bouncing, but no
1452b1ad37efcafe396ac3944853589688dd0ec3c64eMatthew Wilcox	 * additional races since q_lock already protects against other CPUs.
1453b1ad37efcafe396ac3944853589688dd0ec3c64eMatthew Wilcox	 */
1454a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox	put_nvmeq(nvmeq);
1455b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox	if (length != (io.nblocks + 1) << ns->lba_shift)
1456b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox		status = -ENOMEM;
1457224042742582c9938788b81165180c876e997a07Keith Busch	else if (!nvmeq || nvmeq->q_suspended)
1458224042742582c9938788b81165180c876e997a07Keith Busch		status = -EBUSY;
1459b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox	else
1460ff976d724a74e4522e9ca2de1fb37ac4520f454fMatthew Wilcox		status = nvme_submit_sync_cmd(nvmeq, &c, NULL, NVME_IO_TIMEOUT);
1461a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox
1462f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch	if (meta_len) {
1463f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch		if (status == NVME_SC_SUCCESS && !(io.opcode & 1)) {
1464f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch			int meta_offset = 0;
1465f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch
1466f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch			for (i = 0; i < meta_iod->nents; i++) {
1467f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch				meta = kmap_atomic(sg_page(&meta_iod->sg[i])) +
1468f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch						meta_iod->sg[i].offset;
1469f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch				memcpy(meta, meta_mem + meta_offset,
1470f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch						meta_iod->sg[i].length);
1471f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch				kunmap_atomic(meta);
1472f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch				meta_offset += meta_iod->sg[i].length;
1473f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch			}
1474f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch		}
1475f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch
1476f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch		dma_free_coherent(&dev->pci_dev->dev, meta_len, meta_mem,
1477f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch								meta_dma_addr);
1478f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch	}
1479f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch
1480f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch unmap:
14811c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox	nvme_unmap_user_pages(dev, io.opcode & 1, iod);
1482eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox	nvme_free_iod(dev, iod);
1483f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch
1484f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch	if (meta_iod) {
1485f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch		nvme_unmap_user_pages(dev, io.opcode & 1, meta_iod);
1486f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch		nvme_free_iod(dev, meta_iod);
1487f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch	}
1488f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch
1489a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox	return status;
1490a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox}
1491a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox
149250af8baec46a99a9b81a4600c0374f83a5a590a9Keith Buschstatic int nvme_user_admin_cmd(struct nvme_dev *dev,
14936bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox					struct nvme_admin_cmd __user *ucmd)
14946ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox{
14956bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox	struct nvme_admin_cmd cmd;
14966ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox	struct nvme_command c;
1497eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox	int status, length;
1498c7d36ab8fa04c213328119a9c0d66985fe204ee5Keith Busch	struct nvme_iod *uninitialized_var(iod);
149994f370cab6e5ac514b658c6b2b3aa308cefc5c7aKeith Busch	unsigned timeout;
15006ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox
15016bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox	if (!capable(CAP_SYS_ADMIN))
15026bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox		return -EACCES;
15036bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox	if (copy_from_user(&cmd, ucmd, sizeof(cmd)))
15046ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox		return -EFAULT;
15056ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox
15066ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox	memset(&c, 0, sizeof(c));
15076bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox	c.common.opcode = cmd.opcode;
15086bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox	c.common.flags = cmd.flags;
15096bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox	c.common.nsid = cpu_to_le32(cmd.nsid);
15106bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox	c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
15116bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox	c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
15126bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox	c.common.cdw10[0] = cpu_to_le32(cmd.cdw10);
15136bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox	c.common.cdw10[1] = cpu_to_le32(cmd.cdw11);
15146bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox	c.common.cdw10[2] = cpu_to_le32(cmd.cdw12);
15156bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox	c.common.cdw10[3] = cpu_to_le32(cmd.cdw13);
15166bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox	c.common.cdw10[4] = cpu_to_le32(cmd.cdw14);
15176bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox	c.common.cdw10[5] = cpu_to_le32(cmd.cdw15);
15186bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox
15196bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox	length = cmd.data_len;
15206bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox	if (cmd.data_len) {
1521497421880acecd0281d3182d534f3d28c927caecMatthew Wilcox		iod = nvme_map_user_pages(dev, cmd.opcode & 1, cmd.addr,
1522497421880acecd0281d3182d534f3d28c927caecMatthew Wilcox								length);
1523eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox		if (IS_ERR(iod))
1524eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox			return PTR_ERR(iod);
1525eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox		length = nvme_setup_prps(dev, &c.common, iod, length,
1526eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox								GFP_KERNEL);
15276bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox	}
15286bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox
152994f370cab6e5ac514b658c6b2b3aa308cefc5c7aKeith Busch	timeout = cmd.timeout_ms ? msecs_to_jiffies(cmd.timeout_ms) :
153094f370cab6e5ac514b658c6b2b3aa308cefc5c7aKeith Busch								ADMIN_TIMEOUT;
15316bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox	if (length != cmd.data_len)
1532b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox		status = -ENOMEM;
1533b77954cbddff28d55a36fad3c16f4daebb0f01dfMatthew Wilcox	else
153494f370cab6e5ac514b658c6b2b3aa308cefc5c7aKeith Busch		status = nvme_submit_sync_cmd(dev->queues[0], &c, &cmd.result,
153594f370cab6e5ac514b658c6b2b3aa308cefc5c7aKeith Busch								timeout);
1536eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox
15376bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox	if (cmd.data_len) {
15381c2ad9faaf662b4a525348775deca3ac8e6c35a0Matthew Wilcox		nvme_unmap_user_pages(dev, cmd.opcode & 1, iod);
1539eca18b2394a9387feeaf14cd884ddddd7a809d19Matthew Wilcox		nvme_free_iod(dev, iod);
15406bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox	}
1541f4f117f64baf8840d22266d518227b2a186d294bKeith Busch
1542cf90bc4830b858487fe4b9b9ecd0031e23ca3e83Chayan Biswas	if ((status >= 0) && copy_to_user(&ucmd->result, &cmd.result,
1543f4f117f64baf8840d22266d518227b2a186d294bKeith Busch							sizeof(cmd.result)))
1544f4f117f64baf8840d22266d518227b2a186d294bKeith Busch		status = -EFAULT;
1545f4f117f64baf8840d22266d518227b2a186d294bKeith Busch
15466ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox	return status;
15476ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox}
15486ee44cdced04a53dc4f27eb97067e6cd33784726Matthew Wilcox
1549b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int nvme_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd,
1550b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox							unsigned long arg)
1551b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{
1552b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	struct nvme_ns *ns = bdev->bd_disk->private_data;
1553b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
1554b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	switch (cmd) {
15556bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox	case NVME_IOCTL_ID:
1556c3bfe7176c035a0a2c70bc79180fb13a6c57142aMatthew Wilcox		force_successful_syscall_return();
15576bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox		return ns->ns_id;
15586bbf1acddeed0bfb345a5578f9fcada16f1e514fMatthew Wilcox	case NVME_IOCTL_ADMIN_CMD:
155950af8baec46a99a9b81a4600c0374f83a5a590a9Keith Busch		return nvme_user_admin_cmd(ns->dev, (void __user *)arg);
1560a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox	case NVME_IOCTL_SUBMIT_IO:
1561a53295b6998f62d961c29e54051c1cf1d738c2b3Matthew Wilcox		return nvme_submit_io(ns, (void __user *)arg);
15625d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma	case SG_GET_VERSION_NUM:
15635d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma		return nvme_sg_get_version_num((void __user *)arg);
15645d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma	case SG_IO:
15655d0f6131a79adfa1fb51309c5f81a2a4ef879dd4Vishal Verma		return nvme_sg_io(ns, (void __user *)arg);
1566b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	default:
1567b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox		return -ENOTTY;
1568b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	}
1569b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}
1570b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
1571320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch#ifdef CONFIG_COMPAT
1572320a382746e0ab1304476ea7e986a8d416ab99dbKeith Buschstatic int nvme_compat_ioctl(struct block_device *bdev, fmode_t mode,
1573320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch					unsigned int cmd, unsigned long arg)
1574320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch{
1575320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch	struct nvme_ns *ns = bdev->bd_disk->private_data;
1576320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch
1577320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch	switch (cmd) {
1578320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch	case SG_IO:
1579320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch		return nvme_sg_io32(ns, arg);
1580320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch	}
1581320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch	return nvme_ioctl(bdev, mode, cmd, arg);
1582320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch}
1583320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch#else
1584320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch#define nvme_compat_ioctl	NULL
1585320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch#endif
1586320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch
1587b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic const struct block_device_operations nvme_fops = {
1588b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	.owner		= THIS_MODULE,
1589b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	.ioctl		= nvme_ioctl,
1590320a382746e0ab1304476ea7e986a8d416ab99dbKeith Busch	.compat_ioctl	= nvme_compat_ioctl,
1591b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox};
1592b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
15931fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcoxstatic void nvme_resubmit_bios(struct nvme_queue *nvmeq)
15941fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox{
15951fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox	while (bio_list_peek(&nvmeq->sq_cong)) {
15961fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox		struct bio *bio = bio_list_pop(&nvmeq->sq_cong);
15971fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox		struct nvme_ns *ns = bio->bi_bdev->bd_disk->private_data;
1598427e97080196548557b288517537ab7eb48c309fKeith Busch
1599427e97080196548557b288517537ab7eb48c309fKeith Busch		if (bio_list_empty(&nvmeq->sq_cong))
1600427e97080196548557b288517537ab7eb48c309fKeith Busch			remove_wait_queue(&nvmeq->sq_full,
1601427e97080196548557b288517537ab7eb48c309fKeith Busch							&nvmeq->sq_cong_wait);
16021fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox		if (nvme_submit_bio_queue(nvmeq, ns, bio)) {
1603427e97080196548557b288517537ab7eb48c309fKeith Busch			if (bio_list_empty(&nvmeq->sq_cong))
1604427e97080196548557b288517537ab7eb48c309fKeith Busch				add_wait_queue(&nvmeq->sq_full,
1605427e97080196548557b288517537ab7eb48c309fKeith Busch							&nvmeq->sq_cong_wait);
16061fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox			bio_list_add_head(&nvmeq->sq_cong, bio);
16071fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox			break;
16081fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox		}
16091fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox	}
16101fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox}
16111fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox
16121fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcoxstatic int nvme_kthread(void *data)
16131fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox{
16141fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox	struct nvme_dev *dev;
16151fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox
16161fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox	while (!kthread_should_stop()) {
1617564a232c059913d91b491e04c2b2d670b8f94615Arjan van de Ven		set_current_state(TASK_INTERRUPTIBLE);
16181fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox		spin_lock(&dev_list_lock);
16191fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox		list_for_each_entry(dev, &dev_list, node) {
16201fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox			int i;
16211fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox			for (i = 0; i < dev->queue_count; i++) {
16221fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox				struct nvme_queue *nvmeq = dev->queues[i];
1623740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox				if (!nvmeq)
1624740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox					continue;
16251fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox				spin_lock_irq(&nvmeq->q_lock);
1626224042742582c9938788b81165180c876e997a07Keith Busch				if (nvmeq->q_suspended)
1627224042742582c9938788b81165180c876e997a07Keith Busch					goto unlock;
1628bc57a0f7a44cfcf3e9873f6c6b8dcecdca486b1fMatthew Wilcox				nvme_process_cq(nvmeq);
1629a09115b23e2002bb35b7bfd337683f00875671ecMatthew Wilcox				nvme_cancel_ios(nvmeq, true);
16301fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox				nvme_resubmit_bios(nvmeq);
1631224042742582c9938788b81165180c876e997a07Keith Busch unlock:
16321fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox				spin_unlock_irq(&nvmeq->q_lock);
16331fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox			}
16341fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox		}
16351fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox		spin_unlock(&dev_list_lock);
1636acb7aa0db09b8abd38abeb84334a8a27a52fbb1bArjan van de Ven		schedule_timeout(round_jiffies_relative(HZ));
16371fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox	}
16381fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox	return 0;
16391fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox}
16401fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox
16415aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcoxstatic DEFINE_IDA(nvme_index_ida);
16425aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox
16435aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcoxstatic int nvme_get_ns_idx(void)
16445aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox{
16455aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox	int index, error;
16465aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox
16475aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox	do {
16485aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox		if (!ida_pre_get(&nvme_index_ida, GFP_KERNEL))
16495aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox			return -1;
16505aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox
16515aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox		spin_lock(&dev_list_lock);
16525aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox		error = ida_get_new(&nvme_index_ida, &index);
16535aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox		spin_unlock(&dev_list_lock);
16545aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox	} while (error == -EAGAIN);
16555aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox
16565aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox	if (error)
16575aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox		index = -1;
16585aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox	return index;
16595aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox}
16605aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox
16615aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcoxstatic void nvme_put_ns_idx(int index)
16625aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox{
16635aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox	spin_lock(&dev_list_lock);
16645aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox	ida_remove(&nvme_index_ida, index);
16655aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox	spin_unlock(&dev_list_lock);
16665aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox}
16675aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox
16680e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Buschstatic void nvme_config_discard(struct nvme_ns *ns)
16690e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch{
16700e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch	u32 logical_block_size = queue_logical_block_size(ns->queue);
16710e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch	ns->queue->limits.discard_zeroes_data = 0;
16720e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch	ns->queue->limits.discard_alignment = logical_block_size;
16730e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch	ns->queue->limits.discard_granularity = logical_block_size;
16740e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch	ns->queue->limits.max_discard_sectors = 0xffffffff;
16750e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch	queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, ns->queue);
16760e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch}
16770e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch
1678c3bfe7176c035a0a2c70bc79180fb13a6c57142aMatthew Wilcoxstatic struct nvme_ns *nvme_alloc_ns(struct nvme_dev *dev, unsigned nsid,
1679b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox			struct nvme_id_ns *id, struct nvme_lba_range_type *rt)
1680b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{
1681b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	struct nvme_ns *ns;
1682b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	struct gendisk *disk;
1683b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	int lbaf;
1684b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
1685b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	if (rt->attributes & NVME_LBART_ATTRIB_HIDE)
1686b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox		return NULL;
1687b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
1688b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	ns = kzalloc(sizeof(*ns), GFP_KERNEL);
1689b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	if (!ns)
1690b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox		return NULL;
1691b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	ns->queue = blk_alloc_queue(GFP_KERNEL);
1692b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	if (!ns->queue)
1693b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox		goto out_free_ns;
16944eeb9215a0d5c9494ca8b20158cc8ee82618840cMatthew Wilcox	ns->queue->queue_flags = QUEUE_FLAG_DEFAULT;
16954eeb9215a0d5c9494ca8b20158cc8ee82618840cMatthew Wilcox	queue_flag_set_unlocked(QUEUE_FLAG_NOMERGES, ns->queue);
16964eeb9215a0d5c9494ca8b20158cc8ee82618840cMatthew Wilcox	queue_flag_set_unlocked(QUEUE_FLAG_NONROT, ns->queue);
1697b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	blk_queue_make_request(ns->queue, nvme_make_request);
1698b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	ns->dev = dev;
1699b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	ns->queue->queuedata = ns;
1700b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
1701b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	disk = alloc_disk(NVME_MINORS);
1702b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	if (!disk)
1703b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox		goto out_free_queue;
17045aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox	ns->ns_id = nsid;
1705b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	ns->disk = disk;
1706b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	lbaf = id->flbas & 0xf;
1707b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	ns->lba_shift = id->lbaf[lbaf].ds;
1708f410c680b5344f1cb63ff011c6ae3d4963f45c30Keith Busch	ns->ms = le16_to_cpu(id->lbaf[lbaf].ms);
1709e9ef46369f5107e634a93b7fc4e62a1f53343197Keith Busch	blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift);
17108fc23e032debd682f5ba9fc524a5846c10d2c522Keith Busch	if (dev->max_hw_sectors)
17118fc23e032debd682f5ba9fc524a5846c10d2c522Keith Busch		blk_queue_max_hw_sectors(ns->queue, dev->max_hw_sectors);
1712b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
1713b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	disk->major = nvme_major;
1714b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	disk->minors = NVME_MINORS;
17155aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox	disk->first_minor = NVME_MINORS * nvme_get_ns_idx();
1716b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	disk->fops = &nvme_fops;
1717b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	disk->private_data = ns;
1718b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	disk->queue = ns->queue;
1719388f037f4e7f0a24bac6b1a24f144f5d939f58cfMatthew Wilcox	disk->driverfs_dev = &dev->pci_dev->dev;
17205aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox	sprintf(disk->disk_name, "nvme%dn%d", dev->instance, nsid);
1721b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	set_capacity(disk, le64_to_cpup(&id->nsze) << (ns->lba_shift - 9));
1722b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
17230e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch	if (dev->oncs & NVME_CTRL_ONCS_DSM)
17240e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch		nvme_config_discard(ns);
17250e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch
1726b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	return ns;
1727b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
1728b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox out_free_queue:
1729b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	blk_cleanup_queue(ns->queue);
1730b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox out_free_ns:
1731b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	kfree(ns);
1732b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	return NULL;
1733b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}
1734b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
1735b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void nvme_ns_free(struct nvme_ns *ns)
1736b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{
17375aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox	int index = ns->disk->first_minor / NVME_MINORS;
1738b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	put_disk(ns->disk);
17395aff9382ddc8aac6eb0c70ffbb351652d71da69aMatthew Wilcox	nvme_put_ns_idx(index);
1740b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	blk_cleanup_queue(ns->queue);
1741b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	kfree(ns);
1742b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}
1743b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
1744b3b06812e199f248561ce7824a4a8a9cd573c05aMatthew Wilcoxstatic int set_queue_count(struct nvme_dev *dev, int count)
1745b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{
1746b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	int status;
1747b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	u32 result;
1748b3b06812e199f248561ce7824a4a8a9cd573c05aMatthew Wilcox	u32 q_count = (count - 1) | ((count - 1) << 16);
1749b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
1750df3481399042200792822b6243e36a95a557b57eMatthew Wilcox	status = nvme_set_features(dev, NVME_FEAT_NUM_QUEUES, q_count, 0,
1751bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox								&result);
1752b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	if (status)
17537e03b124065507e72008ef294c30001eca74a031Keith Busch		return status < 0 ? -EIO : -EBUSY;
1754b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	return min(result & 0xffff, result >> 16) + 1;
1755b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}
1756b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
17579d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Buschstatic size_t db_bar_size(struct nvme_dev *dev, unsigned nr_io_queues)
17589d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch{
1759b80d5ccca3a012e91ca64a2a0b13049163a6a698Haiyan Hu	return 4096 + ((nr_io_queues + 1) * 8 * dev->db_stride);
17609d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch}
17619d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch
17628d85fce77edfc22f1d6dbf78e3af723b4b556f3dGreg Kroah-Hartmanstatic int nvme_setup_io_queues(struct nvme_dev *dev)
1763b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{
1764fa08a396647767abd24a9e7015cb177121d0cf15Ramachandra Rao Gajula	struct pci_dev *pdev = dev->pci_dev;
17659d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch	int result, cpu, i, vecs, nr_io_queues, size, q_depth;
1766b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
1767b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox	nr_io_queues = num_online_cpus();
1768b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox	result = set_queue_count(dev, nr_io_queues);
17691b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox	if (result < 0)
17701b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox		return result;
1771b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox	if (result < nr_io_queues)
1772b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox		nr_io_queues = result;
1773b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
17749d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch	size = db_bar_size(dev, nr_io_queues);
17759d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch	if (size > 8192) {
1776f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox		iounmap(dev->bar);
17779d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch		do {
17789d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch			dev->bar = ioremap(pci_resource_start(pdev, 0), size);
17799d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch			if (dev->bar)
17809d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch				break;
17819d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch			if (!--nr_io_queues)
17829d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch				return -ENOMEM;
17839d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch			size = db_bar_size(dev, nr_io_queues);
17849d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch		} while (1);
1785f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox		dev->dbs = ((void __iomem *)dev->bar) + 4096;
1786f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox		dev->queues[0]->q_db = dev->dbs;
1787f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox	}
1788f1938f6e1ee1583c87ec74dc406fdd8694e99ac8Matthew Wilcox
17899d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch	/* Deregister the admin queue's interrupt */
17909d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch	free_irq(dev->entry[0].vector, dev->queues[0]);
17919d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch
1792063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox	vecs = nr_io_queues;
1793063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox	for (i = 0; i < vecs; i++)
17941b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox		dev->entry[i].entry = i;
17951b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox	for (;;) {
1796063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox		result = pci_enable_msix(pdev, dev->entry, vecs);
1797063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox		if (result <= 0)
17981b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox			break;
1799063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox		vecs = result;
18001b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox	}
18011b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox
1802063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox	if (result < 0) {
1803063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox		vecs = nr_io_queues;
1804063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox		if (vecs > 32)
1805063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox			vecs = 32;
1806fa08a396647767abd24a9e7015cb177121d0cf15Ramachandra Rao Gajula		for (;;) {
1807063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox			result = pci_enable_msi_block(pdev, vecs);
1808fa08a396647767abd24a9e7015cb177121d0cf15Ramachandra Rao Gajula			if (result == 0) {
1809063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox				for (i = 0; i < vecs; i++)
1810fa08a396647767abd24a9e7015cb177121d0cf15Ramachandra Rao Gajula					dev->entry[i].vector = i + pdev->irq;
1811fa08a396647767abd24a9e7015cb177121d0cf15Ramachandra Rao Gajula				break;
1812063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox			} else if (result < 0) {
1813063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox				vecs = 1;
1814fa08a396647767abd24a9e7015cb177121d0cf15Ramachandra Rao Gajula				break;
1815fa08a396647767abd24a9e7015cb177121d0cf15Ramachandra Rao Gajula			}
1816063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox			vecs = result;
1817fa08a396647767abd24a9e7015cb177121d0cf15Ramachandra Rao Gajula		}
1818fa08a396647767abd24a9e7015cb177121d0cf15Ramachandra Rao Gajula	}
1819fa08a396647767abd24a9e7015cb177121d0cf15Ramachandra Rao Gajula
1820063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox	/*
1821063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox	 * Should investigate if there's a performance win from allocating
1822063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox	 * more queues than interrupt vectors; it might allow the submission
1823063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox	 * path to scale better, even if the receive path is limited by the
1824063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox	 * number of interrupts.
1825063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox	 */
1826063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox	nr_io_queues = vecs;
1827063a8096f3dbca7521d5918b3aea7ab46c5d2fe9Matthew Wilcox
18281b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox	result = queue_request_irq(dev, dev->queues[0], "nvme admin");
18299d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch	if (result) {
18309d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch		dev->queues[0]->q_suspended = 1;
1831224042742582c9938788b81165180c876e997a07Keith Busch		goto free_queues;
18329d713c2bfb5e1d6abb18a8b12293631f9fcdc708Keith Busch	}
18331b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox
1834cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch	/* Free previously allocated queues that are no longer usable */
1835cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch	spin_lock(&dev_list_lock);
1836cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch	for (i = dev->queue_count - 1; i > nr_io_queues; i--) {
1837cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch		struct nvme_queue *nvmeq = dev->queues[i];
1838cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch
18390a8d44cb33377969337fa6c1961b631605d5f453Matthew Wilcox		spin_lock_irq(&nvmeq->q_lock);
1840cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch		nvme_cancel_ios(nvmeq, false);
18410a8d44cb33377969337fa6c1961b631605d5f453Matthew Wilcox		spin_unlock_irq(&nvmeq->q_lock);
1842cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch
1843cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch		nvme_free_queue(nvmeq);
1844cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch		dev->queue_count--;
1845cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch		dev->queues[i] = NULL;
1846cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch	}
1847cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch	spin_unlock(&dev_list_lock);
1848cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch
18491b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox	cpu = cpumask_first(cpu_online_mask);
1850b348b7d54368c87811907a8e88f0d96713c43009Matthew Wilcox	for (i = 0; i < nr_io_queues; i++) {
18511b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox		irq_set_affinity_hint(dev->entry[i].vector, get_cpu_mask(cpu));
18521b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox		cpu = cpumask_next(cpu, cpu_online_mask);
18531b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox	}
18541b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox
1855a0cadb85b8b758608ae0759151e29de7581c6731Keith Busch	q_depth = min_t(int, NVME_CAP_MQES(readq(&dev->bar->cap)) + 1,
1856a0cadb85b8b758608ae0759151e29de7581c6731Keith Busch								NVME_Q_DEPTH);
1857cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch	for (i = dev->queue_count - 1; i < nr_io_queues; i++) {
1858224042742582c9938788b81165180c876e997a07Keith Busch		dev->queues[i + 1] = nvme_alloc_queue(dev, i + 1, q_depth, i);
1859224042742582c9938788b81165180c876e997a07Keith Busch		if (!dev->queues[i + 1]) {
1860224042742582c9938788b81165180c876e997a07Keith Busch			result = -ENOMEM;
1861224042742582c9938788b81165180c876e997a07Keith Busch			goto free_queues;
1862224042742582c9938788b81165180c876e997a07Keith Busch		}
18631b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox	}
1864b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
18659ecdc946212f7cd592986b2c519b470404caa6b8Matthew Wilcox	for (; i < num_possible_cpus(); i++) {
18669ecdc946212f7cd592986b2c519b470404caa6b8Matthew Wilcox		int target = i % rounddown_pow_of_two(dev->queue_count - 1);
18679ecdc946212f7cd592986b2c519b470404caa6b8Matthew Wilcox		dev->queues[i + 1] = dev->queues[target + 1];
18689ecdc946212f7cd592986b2c519b470404caa6b8Matthew Wilcox	}
18699ecdc946212f7cd592986b2c519b470404caa6b8Matthew Wilcox
1870224042742582c9938788b81165180c876e997a07Keith Busch	for (i = 1; i < dev->queue_count; i++) {
1871224042742582c9938788b81165180c876e997a07Keith Busch		result = nvme_create_queue(dev->queues[i], i);
1872224042742582c9938788b81165180c876e997a07Keith Busch		if (result) {
1873224042742582c9938788b81165180c876e997a07Keith Busch			for (--i; i > 0; i--)
1874224042742582c9938788b81165180c876e997a07Keith Busch				nvme_disable_queue(dev, i);
1875224042742582c9938788b81165180c876e997a07Keith Busch			goto free_queues;
1876224042742582c9938788b81165180c876e997a07Keith Busch		}
1877224042742582c9938788b81165180c876e997a07Keith Busch	}
1878b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
1879224042742582c9938788b81165180c876e997a07Keith Busch	return 0;
1880b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
1881224042742582c9938788b81165180c876e997a07Keith Busch free_queues:
1882224042742582c9938788b81165180c876e997a07Keith Busch	nvme_free_queues(dev);
1883224042742582c9938788b81165180c876e997a07Keith Busch	return result;
1884b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}
1885b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
1886422ef0c7c81934c3344e1623d2dcfda5fa2fab8dMatthew Wilcox/*
1887422ef0c7c81934c3344e1623d2dcfda5fa2fab8dMatthew Wilcox * Return: error value if an error occurred setting up the queues or calling
1888422ef0c7c81934c3344e1623d2dcfda5fa2fab8dMatthew Wilcox * Identify Device.  0 if these succeeded, even if adding some of the
1889422ef0c7c81934c3344e1623d2dcfda5fa2fab8dMatthew Wilcox * namespaces failed.  At the moment, these failures are silent.  TBD which
1890422ef0c7c81934c3344e1623d2dcfda5fa2fab8dMatthew Wilcox * failures should be reported.
1891422ef0c7c81934c3344e1623d2dcfda5fa2fab8dMatthew Wilcox */
18928d85fce77edfc22f1d6dbf78e3af723b4b556f3dGreg Kroah-Hartmanstatic int nvme_dev_add(struct nvme_dev *dev)
1893b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{
1894c3bfe7176c035a0a2c70bc79180fb13a6c57142aMatthew Wilcox	int res;
1895c3bfe7176c035a0a2c70bc79180fb13a6c57142aMatthew Wilcox	unsigned nn, i;
1896cbb6218fd4ae8f98ddf0d66f0826a7a3a9c88298Keith Busch	struct nvme_ns *ns;
189751814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox	struct nvme_id_ctrl *ctrl;
1898bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox	struct nvme_id_ns *id_ns;
1899bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox	void *mem;
1900b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	dma_addr_t dma_addr;
1901159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch	int shift = NVME_CAP_MPSMIN(readq(&dev->bar->cap)) + 12;
1902b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
1903bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox	mem = dma_alloc_coherent(&dev->pci_dev->dev, 8192, &dma_addr,
1904b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox								GFP_KERNEL);
1905a9ef4343afbe67a6abf83c0f0294e80db48e513aKeith Busch	if (!mem)
1906a9ef4343afbe67a6abf83c0f0294e80db48e513aKeith Busch		return -ENOMEM;
1907b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
1908bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox	res = nvme_identify(dev, 0, 1, dma_addr);
1909b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	if (res) {
1910b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox		res = -EIO;
1911cbb6218fd4ae8f98ddf0d66f0826a7a3a9c88298Keith Busch		goto out;
1912b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	}
1913b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
1914bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox	ctrl = mem;
191551814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox	nn = le32_to_cpup(&ctrl->nn);
19160e5e4f0e56aca0df1d5648db0be9028bd573b25cKeith Busch	dev->oncs = le16_to_cpup(&ctrl->oncs);
191751814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox	memcpy(dev->serial, ctrl->sn, sizeof(ctrl->sn));
191851814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox	memcpy(dev->model, ctrl->mn, sizeof(ctrl->mn));
191951814232ecae90f888c902e252306df8d017f0ddMatthew Wilcox	memcpy(dev->firmware_rev, ctrl->fr, sizeof(ctrl->fr));
1920159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch	if (ctrl->mdts)
19218fc23e032debd682f5ba9fc524a5846c10d2c522Keith Busch		dev->max_hw_sectors = 1 << (ctrl->mdts + shift - 9);
1922159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch	if ((dev->pci_dev->vendor == PCI_VENDOR_ID_INTEL) &&
1923159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch			(dev->pci_dev->device == 0x0953) && ctrl->vs[3])
1924159b67d7aefe3902df91075be5d80943c1570aa8Keith Busch		dev->stripe_size = 1 << (ctrl->vs[3] + shift);
1925b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
1926bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox	id_ns = mem;
19272b2c1896871838cdf549442e8ad0264be5fa74e3Matthew Wilcox	for (i = 1; i <= nn; i++) {
1928bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox		res = nvme_identify(dev, i, 0, dma_addr);
1929b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox		if (res)
1930b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox			continue;
1931b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
1932bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox		if (id_ns->ncap == 0)
1933b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox			continue;
1934b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
1935bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox		res = nvme_get_features(dev, NVME_FEAT_LBA_RANGE, i,
193608df1e05657fc6712e520e7c09cc6c86160ceb35Keith Busch							dma_addr + 4096, NULL);
1937b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox		if (res)
1938122090366d1d5c6ec1bfb6dfdb3a6d121ff074aaKeith Busch			memset(mem + 4096, 0, 4096);
1939b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
1940bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox		ns = nvme_alloc_ns(dev, i, mem, mem + 4096);
1941b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox		if (ns)
1942b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox			list_add_tail(&ns->list, &dev->namespaces);
1943b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	}
1944b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	list_for_each_entry(ns, &dev->namespaces, list)
1945b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox		add_disk(ns->disk);
1946422ef0c7c81934c3344e1623d2dcfda5fa2fab8dMatthew Wilcox	res = 0;
1947b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
1948bc5fc7e4b22ca855902aba02b28c96f09b446407Matthew Wilcox out:
1949684f5c2025b067a23722e620d0b3b858d8dc5d01Matthew Wilcox	dma_free_coherent(&dev->pci_dev->dev, 8192, mem, dma_addr);
1950b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	return res;
1951b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}
1952b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
19530877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Buschstatic int nvme_dev_map(struct nvme_dev *dev)
19540877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch{
19550877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch	int bars, result = -ENOMEM;
19560877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch	struct pci_dev *pdev = dev->pci_dev;
19570877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch
19580877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch	if (pci_enable_device_mem(pdev))
19590877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch		return result;
19600877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch
19610877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch	dev->entry[0].vector = pdev->irq;
19620877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch	pci_set_master(pdev);
19630877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch	bars = pci_select_bars(pdev, IORESOURCE_MEM);
19640877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch	if (pci_request_selected_regions(pdev, bars, "nvme"))
19650877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch		goto disable_pci;
19660877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch
1967052d0efada8ccfccd924059b75f5fb119cd10f9aRussell King	if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) &&
1968052d0efada8ccfccd924059b75f5fb119cd10f9aRussell King	    dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)))
1969052d0efada8ccfccd924059b75f5fb119cd10f9aRussell King		goto disable;
19700877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch
19710877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch	pci_set_drvdata(pdev, dev);
19720877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch	dev->bar = ioremap(pci_resource_start(pdev, 0), 8192);
19730877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch	if (!dev->bar)
19740877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch		goto disable;
19750877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch
1976b80d5ccca3a012e91ca64a2a0b13049163a6a698Haiyan Hu	dev->db_stride = 1 << NVME_CAP_STRIDE(readq(&dev->bar->cap));
19770877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch	dev->dbs = ((void __iomem *)dev->bar) + 4096;
19780877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch
19790877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch	return 0;
19800877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch
19810877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch disable:
19820877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch	pci_release_regions(pdev);
19830877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch disable_pci:
19840877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch	pci_disable_device(pdev);
19850877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch	return result;
19860877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch}
19870877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch
19880877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Buschstatic void nvme_dev_unmap(struct nvme_dev *dev)
19890877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch{
19900877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch	if (dev->pci_dev->msi_enabled)
19910877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch		pci_disable_msi(dev->pci_dev);
19920877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch	else if (dev->pci_dev->msix_enabled)
19930877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch		pci_disable_msix(dev->pci_dev);
19940877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch
19950877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch	if (dev->bar) {
19960877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch		iounmap(dev->bar);
19970877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch		dev->bar = NULL;
19980877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch	}
19990877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch
20000877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch	pci_release_regions(dev->pci_dev);
20010877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch	if (pci_is_enabled(dev->pci_dev))
20020877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch		pci_disable_device(dev->pci_dev);
20030877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch}
20040877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch
2005f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Buschstatic void nvme_dev_shutdown(struct nvme_dev *dev)
2006b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{
2007224042742582c9938788b81165180c876e997a07Keith Busch	int i;
2008224042742582c9938788b81165180c876e997a07Keith Busch
2009224042742582c9938788b81165180c876e997a07Keith Busch	for (i = dev->queue_count - 1; i >= 0; i--)
2010224042742582c9938788b81165180c876e997a07Keith Busch		nvme_disable_queue(dev, i);
2011b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
20121fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox	spin_lock(&dev_list_lock);
2013f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch	list_del_init(&dev->node);
20141fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox	spin_unlock(&dev_list_lock);
20151fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox
20161894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch	if (dev->bar)
20171894d8f16afe5ad54b732f0fa6c4e80bd4d40b91Keith Busch		nvme_shutdown_ctrl(dev);
2018f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch	nvme_dev_unmap(dev);
2019f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch}
2020f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch
2021f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Buschstatic void nvme_dev_remove(struct nvme_dev *dev)
2022f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch{
2023f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch	struct nvme_ns *ns, *next;
2024f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch
2025b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	list_for_each_entry_safe(ns, next, &dev->namespaces, list) {
2026b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox		list_del(&ns->list);
2027b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox		del_gendisk(ns->disk);
2028b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox		nvme_ns_free(ns);
2029b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	}
2030b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}
2031b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
2032091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcoxstatic int nvme_setup_prp_pools(struct nvme_dev *dev)
2033091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox{
2034091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox	struct device *dmadev = &dev->pci_dev->dev;
2035091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox	dev->prp_page_pool = dma_pool_create("prp list page", dmadev,
2036091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox						PAGE_SIZE, PAGE_SIZE, 0);
2037091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox	if (!dev->prp_page_pool)
2038091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox		return -ENOMEM;
2039091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox
204099802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox	/* Optimisation for I/Os between 4k and 128k */
204199802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox	dev->prp_small_pool = dma_pool_create("prp list 256", dmadev,
204299802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox						256, 256, 0);
204399802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox	if (!dev->prp_small_pool) {
204499802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox		dma_pool_destroy(dev->prp_page_pool);
204599802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox		return -ENOMEM;
204699802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox	}
2047091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox	return 0;
2048091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox}
2049091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox
2050091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcoxstatic void nvme_release_prp_pools(struct nvme_dev *dev)
2051091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox{
2052091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox	dma_pool_destroy(dev->prp_page_pool);
205399802a7aee2b3dd720e382c52b892cc6a8122b11Matthew Wilcox	dma_pool_destroy(dev->prp_small_pool);
2054091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox}
2055091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox
2056cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anhstatic DEFINE_IDA(nvme_instance_ida);
2057cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh
2058cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anhstatic int nvme_set_instance(struct nvme_dev *dev)
2059b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{
2060cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh	int instance, error;
2061cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh
2062cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh	do {
2063cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh		if (!ida_pre_get(&nvme_instance_ida, GFP_KERNEL))
2064cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh			return -ENODEV;
2065cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh
2066cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh		spin_lock(&dev_list_lock);
2067cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh		error = ida_get_new(&nvme_instance_ida, &instance);
2068cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh		spin_unlock(&dev_list_lock);
2069cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh	} while (error == -EAGAIN);
2070cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh
2071cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh	if (error)
2072cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh		return -ENODEV;
2073cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh
2074cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh	dev->instance = instance;
2075cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh	return 0;
2076b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}
2077b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
2078b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void nvme_release_instance(struct nvme_dev *dev)
2079b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{
2080cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh	spin_lock(&dev_list_lock);
2081cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh	ida_remove(&nvme_instance_ida, dev->instance);
2082cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh	spin_unlock(&dev_list_lock);
2083b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}
2084b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
20855e82e952f04681c10f35e02ee0a4a43ec027137aKeith Buschstatic void nvme_free_dev(struct kref *kref)
20865e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch{
20875e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch	struct nvme_dev *dev = container_of(kref, struct nvme_dev, kref);
20885e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch	nvme_dev_remove(dev);
2089f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch	nvme_dev_shutdown(dev);
2090f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch	nvme_free_queues(dev);
20915e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch	nvme_release_instance(dev);
20925e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch	nvme_release_prp_pools(dev);
20935e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch	kfree(dev->queues);
20945e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch	kfree(dev->entry);
20955e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch	kfree(dev);
20965e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch}
20975e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch
20985e82e952f04681c10f35e02ee0a4a43ec027137aKeith Buschstatic int nvme_dev_open(struct inode *inode, struct file *f)
20995e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch{
21005e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch	struct nvme_dev *dev = container_of(f->private_data, struct nvme_dev,
21015e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch								miscdev);
21025e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch	kref_get(&dev->kref);
21035e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch	f->private_data = dev;
21045e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch	return 0;
21055e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch}
21065e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch
21075e82e952f04681c10f35e02ee0a4a43ec027137aKeith Buschstatic int nvme_dev_release(struct inode *inode, struct file *f)
21085e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch{
21095e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch	struct nvme_dev *dev = f->private_data;
21105e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch	kref_put(&dev->kref, nvme_free_dev);
21115e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch	return 0;
21125e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch}
21135e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch
21145e82e952f04681c10f35e02ee0a4a43ec027137aKeith Buschstatic long nvme_dev_ioctl(struct file *f, unsigned int cmd, unsigned long arg)
21155e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch{
21165e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch	struct nvme_dev *dev = f->private_data;
21175e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch	switch (cmd) {
21185e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch	case NVME_IOCTL_ADMIN_CMD:
21195e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch		return nvme_user_admin_cmd(dev, (void __user *)arg);
21205e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch	default:
21215e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch		return -ENOTTY;
21225e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch	}
21235e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch}
21245e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch
21255e82e952f04681c10f35e02ee0a4a43ec027137aKeith Buschstatic const struct file_operations nvme_dev_fops = {
21265e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch	.owner		= THIS_MODULE,
21275e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch	.open		= nvme_dev_open,
21285e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch	.release	= nvme_dev_release,
21295e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch	.unlocked_ioctl	= nvme_dev_ioctl,
21305e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch	.compat_ioctl	= nvme_dev_ioctl,
21315e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch};
21325e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch
2133f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Buschstatic int nvme_dev_start(struct nvme_dev *dev)
2134f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch{
2135f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch	int result;
2136f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch
2137f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch	result = nvme_dev_map(dev);
2138f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch	if (result)
2139f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch		return result;
2140f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch
2141f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch	result = nvme_configure_admin_queue(dev);
2142f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch	if (result)
2143f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch		goto unmap;
2144f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch
2145f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch	spin_lock(&dev_list_lock);
2146f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch	list_add(&dev->node, &dev_list);
2147f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch	spin_unlock(&dev_list_lock);
2148f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch
2149f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch	result = nvme_setup_io_queues(dev);
2150d82e8bfdef9afae83b894be49af4644d9ac3c359Keith Busch	if (result && result != -EBUSY)
2151f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch		goto disable;
2152f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch
2153d82e8bfdef9afae83b894be49af4644d9ac3c359Keith Busch	return result;
2154f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch
2155f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch disable:
2156f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch	spin_lock(&dev_list_lock);
2157f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch	list_del_init(&dev->node);
2158f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch	spin_unlock(&dev_list_lock);
2159f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch unmap:
2160f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch	nvme_dev_unmap(dev);
2161f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch	return result;
2162f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch}
2163f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch
21648d85fce77edfc22f1d6dbf78e3af723b4b556f3dGreg Kroah-Hartmanstatic int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)
2165b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{
21660877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch	int result = -ENOMEM;
2167b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	struct nvme_dev *dev;
2168b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
2169b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
2170b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	if (!dev)
2171b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox		return -ENOMEM;
2172b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	dev->entry = kcalloc(num_possible_cpus(), sizeof(*dev->entry),
2173b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox								GFP_KERNEL);
2174b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	if (!dev->entry)
2175b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox		goto free;
21761b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox	dev->queues = kcalloc(num_possible_cpus() + 1, sizeof(void *),
21771b23484bd012c078de2ea939249e2fb2e85a0a6eMatthew Wilcox								GFP_KERNEL);
2178b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	if (!dev->queues)
2179b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox		goto free;
2180b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
2181b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	INIT_LIST_HEAD(&dev->namespaces);
2182b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	dev->pci_dev = pdev;
2183052d0efada8ccfccd924059b75f5fb119cd10f9aRussell King
2184cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh	result = nvme_set_instance(dev);
2185cd58ad7d188c643cf572b038909c2f7dd96fdafeQuoc-Son Anh	if (result)
21860877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch		goto free;
2187b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
2188091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox	result = nvme_setup_prp_pools(dev);
2189091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox	if (result)
21900877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch		goto release;
2191091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox
2192f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch	result = nvme_dev_start(dev);
2193d82e8bfdef9afae83b894be49af4644d9ac3c359Keith Busch	if (result) {
2194d82e8bfdef9afae83b894be49af4644d9ac3c359Keith Busch		if (result == -EBUSY)
2195d82e8bfdef9afae83b894be49af4644d9ac3c359Keith Busch			goto create_cdev;
21960877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch		goto release_pools;
2197d82e8bfdef9afae83b894be49af4644d9ac3c359Keith Busch	}
2198b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
2199740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox	result = nvme_dev_add(dev);
2200d82e8bfdef9afae83b894be49af4644d9ac3c359Keith Busch	if (result)
2201f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch		goto shutdown;
2202740216fc59cba54f65187c9ed92f29bce3cf8778Matthew Wilcox
2203d82e8bfdef9afae83b894be49af4644d9ac3c359Keith Busch create_cdev:
22045e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch	scnprintf(dev->name, sizeof(dev->name), "nvme%d", dev->instance);
22055e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch	dev->miscdev.minor = MISC_DYNAMIC_MINOR;
22065e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch	dev->miscdev.parent = &pdev->dev;
22075e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch	dev->miscdev.name = dev->name;
22085e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch	dev->miscdev.fops = &nvme_dev_fops;
22095e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch	result = misc_register(&dev->miscdev);
22105e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch	if (result)
22115e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch		goto remove;
22125e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch
22135e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch	kref_init(&dev->kref);
2214b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	return 0;
2215b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
22165e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch remove:
22175e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch	nvme_dev_remove(dev);
2218f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch shutdown:
2219f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch	nvme_dev_shutdown(dev);
22200877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch release_pools:
2221f0b50732a979c55c2d15fe8ec4503fa5b3260c53Keith Busch	nvme_free_queues(dev);
2222091b609258b8e01cc45b01a41ca5e496f674d989Matthew Wilcox	nvme_release_prp_pools(dev);
22230877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch release:
22240877cb0d285c7f1d53d0b84b360bdea4be4f3f59Keith Busch	nvme_release_instance(dev);
2225b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox free:
2226b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	kfree(dev->queues);
2227b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	kfree(dev->entry);
2228b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	kfree(dev);
2229b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	return result;
2230b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}
2231b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
22328d85fce77edfc22f1d6dbf78e3af723b4b556f3dGreg Kroah-Hartmanstatic void nvme_remove(struct pci_dev *pdev)
2233b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{
2234b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	struct nvme_dev *dev = pci_get_drvdata(pdev);
22355e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch	misc_deregister(&dev->miscdev);
22365e82e952f04681c10f35e02ee0a4a43ec027137aKeith Busch	kref_put(&dev->kref, nvme_free_dev);
2237b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}
2238b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
2239b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* These functions are yet to be implemented */
2240b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_error_detected NULL
2241b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_dump_registers NULL
2242b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_link_reset NULL
2243b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_slot_reset NULL
2244b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define nvme_error_resume NULL
2245cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch
2246cd63894630ab17a192bf97427d16dbec10710a6aKeith Buschstatic int nvme_suspend(struct device *dev)
2247cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch{
2248cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch	struct pci_dev *pdev = to_pci_dev(dev);
2249cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch	struct nvme_dev *ndev = pci_get_drvdata(pdev);
2250cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch
2251cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch	nvme_dev_shutdown(ndev);
2252cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch	return 0;
2253cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch}
2254cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch
2255cd63894630ab17a192bf97427d16dbec10710a6aKeith Buschstatic int nvme_resume(struct device *dev)
2256cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch{
2257cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch	struct pci_dev *pdev = to_pci_dev(dev);
2258cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch	struct nvme_dev *ndev = pci_get_drvdata(pdev);
2259cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch	int ret;
2260cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch
2261cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch	ret = nvme_dev_start(ndev);
2262cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch	/* XXX: should remove gendisks if resume fails */
2263cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch	if (ret)
2264cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch		nvme_free_queues(ndev);
2265cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch	return ret;
2266cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch}
2267cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch
2268cd63894630ab17a192bf97427d16dbec10710a6aKeith Buschstatic SIMPLE_DEV_PM_OPS(nvme_dev_pm_ops, nvme_suspend, nvme_resume);
2269b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
22701d3520357df99baf4ad89f86268ac96cd38092d9Stephen Hemmingerstatic const struct pci_error_handlers nvme_err_handler = {
2271b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	.error_detected	= nvme_error_detected,
2272b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	.mmio_enabled	= nvme_dump_registers,
2273b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	.link_reset	= nvme_link_reset,
2274b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	.slot_reset	= nvme_slot_reset,
2275b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	.resume		= nvme_error_resume,
2276b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox};
2277b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
2278b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox/* Move to pci_ids.h later */
2279b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox#define PCI_CLASS_STORAGE_EXPRESS	0x010802
2280b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
2281b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic DEFINE_PCI_DEVICE_TABLE(nvme_id_table) = {
2282b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	{ PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) },
2283b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	{ 0, }
2284b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox};
2285b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew WilcoxMODULE_DEVICE_TABLE(pci, nvme_id_table);
2286b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
2287b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic struct pci_driver nvme_driver = {
2288b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	.name		= "nvme",
2289b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	.id_table	= nvme_id_table,
2290b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	.probe		= nvme_probe,
22918d85fce77edfc22f1d6dbf78e3af723b4b556f3dGreg Kroah-Hartman	.remove		= nvme_remove,
2292cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch	.driver		= {
2293cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch		.pm	= &nvme_dev_pm_ops,
2294cd63894630ab17a192bf97427d16dbec10710a6aKeith Busch	},
2295b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	.err_handler	= &nvme_err_handler,
2296b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox};
2297b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
2298b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic int __init nvme_init(void)
2299b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{
23000ac13140d796eb1e2f8956aea97a6e5e4ebcf981Matthew Wilcox	int result;
23011fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox
23021fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox	nvme_thread = kthread_run(nvme_kthread, NULL, "nvme");
23031fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox	if (IS_ERR(nvme_thread))
23041fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox		return PTR_ERR(nvme_thread);
2305b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
23065c42ea1643a630060f9e71e06d3933d244970967Keith Busch	result = register_blkdev(nvme_major, "nvme");
23075c42ea1643a630060f9e71e06d3933d244970967Keith Busch	if (result < 0)
23081fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox		goto kill_kthread;
23095c42ea1643a630060f9e71e06d3933d244970967Keith Busch	else if (result > 0)
23100ac13140d796eb1e2f8956aea97a6e5e4ebcf981Matthew Wilcox		nvme_major = result;
2311b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
2312b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	result = pci_register_driver(&nvme_driver);
23131fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox	if (result)
23141fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox		goto unregister_blkdev;
23151fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox	return 0;
2316b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
23171fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox unregister_blkdev:
2318b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	unregister_blkdev(nvme_major, "nvme");
23191fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox kill_kthread:
23201fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox	kthread_stop(nvme_thread);
2321b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	return result;
2322b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}
2323b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
2324b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxstatic void __exit nvme_exit(void)
2325b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox{
2326b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	pci_unregister_driver(&nvme_driver);
2327b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox	unregister_blkdev(nvme_major, "nvme");
23281fa6aeadf18aeebd7a217d7a3a933856448375b6Matthew Wilcox	kthread_stop(nvme_thread);
2329b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox}
2330b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcox
2331b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew WilcoxMODULE_AUTHOR("Matthew Wilcox <willy@linux.intel.com>");
2332b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew WilcoxMODULE_LICENSE("GPL");
2333366e8217e5ec6ce9f73aec19c46d983110fb4a98Matthew WilcoxMODULE_VERSION("0.8");
2334b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxmodule_init(nvme_init);
2335b60503ba432b16fc84442a84e29a7aad2c0c363dMatthew Wilcoxmodule_exit(nvme_exit);
2336