iw_cxgb4.h revision 30c95c2d495c1c8d4d6a97bb9f4e4eacb91ba1d2 
1/*
2 * Copyright (c) 2009-2010 Chelsio, Inc. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses.  You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 *     Redistribution and use in source and binary forms, with or
11 *     without modification, are permitted provided that the following
12 *     conditions are met:
13 *
14 *      - Redistributions of source code must retain the above
15 *	  copyright notice, this list of conditions and the following
16 *	  disclaimer.
17 *      - Redistributions in binary form must reproduce the above
18 *	  copyright notice, this list of conditions and the following
19 *	  disclaimer in the documentation and/or other materials
20 *	  provided with the distribution.
21 *
22 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
23 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
24 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
25 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
26 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
27 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
28 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
29 * SOFTWARE.
30 */
31#ifndef __IW_CXGB4_H__
32#define __IW_CXGB4_H__
33
34#include <linux/mutex.h>
35#include <linux/list.h>
36#include <linux/spinlock.h>
37#include <linux/idr.h>
38#include <linux/workqueue.h>
39#include <linux/netdevice.h>
40#include <linux/sched.h>
41#include <linux/pci.h>
42#include <linux/dma-mapping.h>
43#include <linux/inet.h>
44#include <linux/wait.h>
45#include <linux/kref.h>
46#include <linux/timer.h>
47#include <linux/io.h>
48#include <linux/kfifo.h>
49
50#include <asm/byteorder.h>
51
52#include <net/net_namespace.h>
53
54#include <rdma/ib_verbs.h>
55#include <rdma/iw_cm.h>
56
57#include "cxgb4.h"
58#include "cxgb4_uld.h"
59#include "l2t.h"
60#include "user.h"
61
62#define DRV_NAME "iw_cxgb4"
63#define MOD DRV_NAME ":"
64
65extern int c4iw_debug;
66#define PDBG(fmt, args...) \
67do { \
68	if (c4iw_debug) \
69		printk(MOD fmt, ## args); \
70} while (0)
71
72#include "t4.h"
73
74#define PBL_OFF(rdev_p, a) ((a) - (rdev_p)->lldi.vr->pbl.start)
75#define RQT_OFF(rdev_p, a) ((a) - (rdev_p)->lldi.vr->rq.start)
76
77static inline void *cplhdr(struct sk_buff *skb)
78{
79	return skb->data;
80}
81
82struct c4iw_resource {
83	struct kfifo tpt_fifo;
84	spinlock_t tpt_fifo_lock;
85	struct kfifo qid_fifo;
86	spinlock_t qid_fifo_lock;
87	struct kfifo pdid_fifo;
88	spinlock_t pdid_fifo_lock;
89};
90
91struct c4iw_qid_list {
92	struct list_head entry;
93	u32 qid;
94};
95
96struct c4iw_dev_ucontext {
97	struct list_head qpids;
98	struct list_head cqids;
99	struct mutex lock;
100};
101
102enum c4iw_rdev_flags {
103	T4_FATAL_ERROR = (1<<0),
104};
105
106struct c4iw_rdev {
107	struct c4iw_resource resource;
108	unsigned long qpshift;
109	u32 qpmask;
110	unsigned long cqshift;
111	u32 cqmask;
112	struct c4iw_dev_ucontext uctx;
113	struct gen_pool *pbl_pool;
114	struct gen_pool *rqt_pool;
115	struct gen_pool *ocqp_pool;
116	u32 flags;
117	struct cxgb4_lld_info lldi;
118	unsigned long oc_mw_pa;
119	void __iomem *oc_mw_kva;
120};
121
122static inline int c4iw_fatal_error(struct c4iw_rdev *rdev)
123{
124	return rdev->flags & T4_FATAL_ERROR;
125}
126
127static inline int c4iw_num_stags(struct c4iw_rdev *rdev)
128{
129	return min((int)T4_MAX_NUM_STAG, (int)(rdev->lldi.vr->stag.size >> 5));
130}
131
132#define C4IW_WR_TO (10*HZ)
133
134struct c4iw_wr_wait {
135	wait_queue_head_t wait;
136	int done;
137	int ret;
138};
139
140static inline void c4iw_init_wr_wait(struct c4iw_wr_wait *wr_waitp)
141{
142	wr_waitp->ret = 0;
143	wr_waitp->done = 0;
144	init_waitqueue_head(&wr_waitp->wait);
145}
146
147static inline int c4iw_wait_for_reply(struct c4iw_rdev *rdev,
148				 struct c4iw_wr_wait *wr_waitp,
149				 u32 hwtid, u32 qpid,
150				 const char *func)
151{
152	unsigned to = C4IW_WR_TO;
153	do {
154
155		wait_event_timeout(wr_waitp->wait, wr_waitp->done, to);
156		if (!wr_waitp->done) {
157			printk(KERN_ERR MOD "%s - Device %s not responding - "
158			       "tid %u qpid %u\n", func,
159			       pci_name(rdev->lldi.pdev), hwtid, qpid);
160			to = to << 2;
161		}
162	} while (!wr_waitp->done);
163	if (wr_waitp->ret)
164		PDBG("%s: FW reply %d tid %u qpid %u\n",
165		     pci_name(rdev->lldi.pdev), wr_waitp->ret, hwtid, qpid);
166	return wr_waitp->ret;
167}
168
169
170struct c4iw_dev {
171	struct ib_device ibdev;
172	struct c4iw_rdev rdev;
173	u32 device_cap_flags;
174	struct idr cqidr;
175	struct idr qpidr;
176	struct idr mmidr;
177	spinlock_t lock;
178	struct list_head entry;
179	struct dentry *debugfs_root;
180	u8 registered;
181};
182
183static inline struct c4iw_dev *to_c4iw_dev(struct ib_device *ibdev)
184{
185	return container_of(ibdev, struct c4iw_dev, ibdev);
186}
187
188static inline struct c4iw_dev *rdev_to_c4iw_dev(struct c4iw_rdev *rdev)
189{
190	return container_of(rdev, struct c4iw_dev, rdev);
191}
192
193static inline struct c4iw_cq *get_chp(struct c4iw_dev *rhp, u32 cqid)
194{
195	return idr_find(&rhp->cqidr, cqid);
196}
197
198static inline struct c4iw_qp *get_qhp(struct c4iw_dev *rhp, u32 qpid)
199{
200	return idr_find(&rhp->qpidr, qpid);
201}
202
203static inline struct c4iw_mr *get_mhp(struct c4iw_dev *rhp, u32 mmid)
204{
205	return idr_find(&rhp->mmidr, mmid);
206}
207
208static inline int insert_handle(struct c4iw_dev *rhp, struct idr *idr,
209				void *handle, u32 id)
210{
211	int ret;
212	int newid;
213
214	do {
215		if (!idr_pre_get(idr, GFP_KERNEL))
216			return -ENOMEM;
217		spin_lock_irq(&rhp->lock);
218		ret = idr_get_new_above(idr, handle, id, &newid);
219		BUG_ON(newid != id);
220		spin_unlock_irq(&rhp->lock);
221	} while (ret == -EAGAIN);
222
223	return ret;
224}
225
226static inline void remove_handle(struct c4iw_dev *rhp, struct idr *idr, u32 id)
227{
228	spin_lock_irq(&rhp->lock);
229	idr_remove(idr, id);
230	spin_unlock_irq(&rhp->lock);
231}
232
233struct c4iw_pd {
234	struct ib_pd ibpd;
235	u32 pdid;
236	struct c4iw_dev *rhp;
237};
238
239static inline struct c4iw_pd *to_c4iw_pd(struct ib_pd *ibpd)
240{
241	return container_of(ibpd, struct c4iw_pd, ibpd);
242}
243
244struct tpt_attributes {
245	u64 len;
246	u64 va_fbo;
247	enum fw_ri_mem_perms perms;
248	u32 stag;
249	u32 pdid;
250	u32 qpid;
251	u32 pbl_addr;
252	u32 pbl_size;
253	u32 state:1;
254	u32 type:2;
255	u32 rsvd:1;
256	u32 remote_invaliate_disable:1;
257	u32 zbva:1;
258	u32 mw_bind_enable:1;
259	u32 page_size:5;
260};
261
262struct c4iw_mr {
263	struct ib_mr ibmr;
264	struct ib_umem *umem;
265	struct c4iw_dev *rhp;
266	u64 kva;
267	struct tpt_attributes attr;
268};
269
270static inline struct c4iw_mr *to_c4iw_mr(struct ib_mr *ibmr)
271{
272	return container_of(ibmr, struct c4iw_mr, ibmr);
273}
274
275struct c4iw_mw {
276	struct ib_mw ibmw;
277	struct c4iw_dev *rhp;
278	u64 kva;
279	struct tpt_attributes attr;
280};
281
282static inline struct c4iw_mw *to_c4iw_mw(struct ib_mw *ibmw)
283{
284	return container_of(ibmw, struct c4iw_mw, ibmw);
285}
286
287struct c4iw_fr_page_list {
288	struct ib_fast_reg_page_list ibpl;
289	DEFINE_DMA_UNMAP_ADDR(mapping);
290	dma_addr_t dma_addr;
291	struct c4iw_dev *dev;
292	int size;
293};
294
295static inline struct c4iw_fr_page_list *to_c4iw_fr_page_list(
296					struct ib_fast_reg_page_list *ibpl)
297{
298	return container_of(ibpl, struct c4iw_fr_page_list, ibpl);
299}
300
301struct c4iw_cq {
302	struct ib_cq ibcq;
303	struct c4iw_dev *rhp;
304	struct t4_cq cq;
305	spinlock_t lock;
306	atomic_t refcnt;
307	wait_queue_head_t wait;
308};
309
310static inline struct c4iw_cq *to_c4iw_cq(struct ib_cq *ibcq)
311{
312	return container_of(ibcq, struct c4iw_cq, ibcq);
313}
314
315struct c4iw_mpa_attributes {
316	u8 initiator;
317	u8 recv_marker_enabled;
318	u8 xmit_marker_enabled;
319	u8 crc_enabled;
320	u8 version;
321	u8 p2p_type;
322};
323
324struct c4iw_qp_attributes {
325	u32 scq;
326	u32 rcq;
327	u32 sq_num_entries;
328	u32 rq_num_entries;
329	u32 sq_max_sges;
330	u32 sq_max_sges_rdma_write;
331	u32 rq_max_sges;
332	u32 state;
333	u8 enable_rdma_read;
334	u8 enable_rdma_write;
335	u8 enable_bind;
336	u8 enable_mmid0_fastreg;
337	u32 max_ord;
338	u32 max_ird;
339	u32 pd;
340	u32 next_state;
341	char terminate_buffer[52];
342	u32 terminate_msg_len;
343	u8 is_terminate_local;
344	struct c4iw_mpa_attributes mpa_attr;
345	struct c4iw_ep *llp_stream_handle;
346};
347
348struct c4iw_qp {
349	struct ib_qp ibqp;
350	struct c4iw_dev *rhp;
351	struct c4iw_ep *ep;
352	struct c4iw_qp_attributes attr;
353	struct t4_wq wq;
354	spinlock_t lock;
355	struct mutex mutex;
356	atomic_t refcnt;
357	wait_queue_head_t wait;
358	struct timer_list timer;
359};
360
361static inline struct c4iw_qp *to_c4iw_qp(struct ib_qp *ibqp)
362{
363	return container_of(ibqp, struct c4iw_qp, ibqp);
364}
365
366struct c4iw_ucontext {
367	struct ib_ucontext ibucontext;
368	struct c4iw_dev_ucontext uctx;
369	u32 key;
370	spinlock_t mmap_lock;
371	struct list_head mmaps;
372};
373
374static inline struct c4iw_ucontext *to_c4iw_ucontext(struct ib_ucontext *c)
375{
376	return container_of(c, struct c4iw_ucontext, ibucontext);
377}
378
379struct c4iw_mm_entry {
380	struct list_head entry;
381	u64 addr;
382	u32 key;
383	unsigned len;
384};
385
386static inline struct c4iw_mm_entry *remove_mmap(struct c4iw_ucontext *ucontext,
387						u32 key, unsigned len)
388{
389	struct list_head *pos, *nxt;
390	struct c4iw_mm_entry *mm;
391
392	spin_lock(&ucontext->mmap_lock);
393	list_for_each_safe(pos, nxt, &ucontext->mmaps) {
394
395		mm = list_entry(pos, struct c4iw_mm_entry, entry);
396		if (mm->key == key && mm->len == len) {
397			list_del_init(&mm->entry);
398			spin_unlock(&ucontext->mmap_lock);
399			PDBG("%s key 0x%x addr 0x%llx len %d\n", __func__,
400			     key, (unsigned long long) mm->addr, mm->len);
401			return mm;
402		}
403	}
404	spin_unlock(&ucontext->mmap_lock);
405	return NULL;
406}
407
408static inline void insert_mmap(struct c4iw_ucontext *ucontext,
409			       struct c4iw_mm_entry *mm)
410{
411	spin_lock(&ucontext->mmap_lock);
412	PDBG("%s key 0x%x addr 0x%llx len %d\n", __func__,
413	     mm->key, (unsigned long long) mm->addr, mm->len);
414	list_add_tail(&mm->entry, &ucontext->mmaps);
415	spin_unlock(&ucontext->mmap_lock);
416}
417
418enum c4iw_qp_attr_mask {
419	C4IW_QP_ATTR_NEXT_STATE = 1 << 0,
420	C4IW_QP_ATTR_ENABLE_RDMA_READ = 1 << 7,
421	C4IW_QP_ATTR_ENABLE_RDMA_WRITE = 1 << 8,
422	C4IW_QP_ATTR_ENABLE_RDMA_BIND = 1 << 9,
423	C4IW_QP_ATTR_MAX_ORD = 1 << 11,
424	C4IW_QP_ATTR_MAX_IRD = 1 << 12,
425	C4IW_QP_ATTR_LLP_STREAM_HANDLE = 1 << 22,
426	C4IW_QP_ATTR_STREAM_MSG_BUFFER = 1 << 23,
427	C4IW_QP_ATTR_MPA_ATTR = 1 << 24,
428	C4IW_QP_ATTR_QP_CONTEXT_ACTIVATE = 1 << 25,
429	C4IW_QP_ATTR_VALID_MODIFY = (C4IW_QP_ATTR_ENABLE_RDMA_READ |
430				     C4IW_QP_ATTR_ENABLE_RDMA_WRITE |
431				     C4IW_QP_ATTR_MAX_ORD |
432				     C4IW_QP_ATTR_MAX_IRD |
433				     C4IW_QP_ATTR_LLP_STREAM_HANDLE |
434				     C4IW_QP_ATTR_STREAM_MSG_BUFFER |
435				     C4IW_QP_ATTR_MPA_ATTR |
436				     C4IW_QP_ATTR_QP_CONTEXT_ACTIVATE)
437};
438
439int c4iw_modify_qp(struct c4iw_dev *rhp,
440				struct c4iw_qp *qhp,
441				enum c4iw_qp_attr_mask mask,
442				struct c4iw_qp_attributes *attrs,
443				int internal);
444
445enum c4iw_qp_state {
446	C4IW_QP_STATE_IDLE,
447	C4IW_QP_STATE_RTS,
448	C4IW_QP_STATE_ERROR,
449	C4IW_QP_STATE_TERMINATE,
450	C4IW_QP_STATE_CLOSING,
451	C4IW_QP_STATE_TOT
452};
453
454static inline int c4iw_convert_state(enum ib_qp_state ib_state)
455{
456	switch (ib_state) {
457	case IB_QPS_RESET:
458	case IB_QPS_INIT:
459		return C4IW_QP_STATE_IDLE;
460	case IB_QPS_RTS:
461		return C4IW_QP_STATE_RTS;
462	case IB_QPS_SQD:
463		return C4IW_QP_STATE_CLOSING;
464	case IB_QPS_SQE:
465		return C4IW_QP_STATE_TERMINATE;
466	case IB_QPS_ERR:
467		return C4IW_QP_STATE_ERROR;
468	default:
469		return -1;
470	}
471}
472
473static inline u32 c4iw_ib_to_tpt_access(int a)
474{
475	return (a & IB_ACCESS_REMOTE_WRITE ? FW_RI_MEM_ACCESS_REM_WRITE : 0) |
476	       (a & IB_ACCESS_REMOTE_READ ? FW_RI_MEM_ACCESS_REM_READ : 0) |
477	       (a & IB_ACCESS_LOCAL_WRITE ? FW_RI_MEM_ACCESS_LOCAL_WRITE : 0) |
478	       FW_RI_MEM_ACCESS_LOCAL_READ;
479}
480
481static inline u32 c4iw_ib_to_tpt_bind_access(int acc)
482{
483	return (acc & IB_ACCESS_REMOTE_WRITE ? FW_RI_MEM_ACCESS_REM_WRITE : 0) |
484	       (acc & IB_ACCESS_REMOTE_READ ? FW_RI_MEM_ACCESS_REM_READ : 0);
485}
486
487enum c4iw_mmid_state {
488	C4IW_STAG_STATE_VALID,
489	C4IW_STAG_STATE_INVALID
490};
491
492#define C4IW_NODE_DESC "cxgb4 Chelsio Communications"
493
494#define MPA_KEY_REQ "MPA ID Req Frame"
495#define MPA_KEY_REP "MPA ID Rep Frame"
496
497#define MPA_MAX_PRIVATE_DATA	256
498#define MPA_REJECT		0x20
499#define MPA_CRC			0x40
500#define MPA_MARKERS		0x80
501#define MPA_FLAGS_MASK		0xE0
502
503#define c4iw_put_ep(ep) { \
504	PDBG("put_ep (via %s:%u) ep %p refcnt %d\n", __func__, __LINE__,  \
505	     ep, atomic_read(&((ep)->kref.refcount))); \
506	WARN_ON(atomic_read(&((ep)->kref.refcount)) < 1); \
507	kref_put(&((ep)->kref), _c4iw_free_ep); \
508}
509
510#define c4iw_get_ep(ep) { \
511	PDBG("get_ep (via %s:%u) ep %p, refcnt %d\n", __func__, __LINE__, \
512	     ep, atomic_read(&((ep)->kref.refcount))); \
513	kref_get(&((ep)->kref));  \
514}
515void _c4iw_free_ep(struct kref *kref);
516
517struct mpa_message {
518	u8 key[16];
519	u8 flags;
520	u8 revision;
521	__be16 private_data_size;
522	u8 private_data[0];
523};
524
525struct terminate_message {
526	u8 layer_etype;
527	u8 ecode;
528	__be16 hdrct_rsvd;
529	u8 len_hdrs[0];
530};
531
532#define TERM_MAX_LENGTH (sizeof(struct terminate_message) + 2 + 18 + 28)
533
534enum c4iw_layers_types {
535	LAYER_RDMAP		= 0x00,
536	LAYER_DDP		= 0x10,
537	LAYER_MPA		= 0x20,
538	RDMAP_LOCAL_CATA	= 0x00,
539	RDMAP_REMOTE_PROT	= 0x01,
540	RDMAP_REMOTE_OP		= 0x02,
541	DDP_LOCAL_CATA		= 0x00,
542	DDP_TAGGED_ERR		= 0x01,
543	DDP_UNTAGGED_ERR	= 0x02,
544	DDP_LLP			= 0x03
545};
546
547enum c4iw_rdma_ecodes {
548	RDMAP_INV_STAG		= 0x00,
549	RDMAP_BASE_BOUNDS	= 0x01,
550	RDMAP_ACC_VIOL		= 0x02,
551	RDMAP_STAG_NOT_ASSOC	= 0x03,
552	RDMAP_TO_WRAP		= 0x04,
553	RDMAP_INV_VERS		= 0x05,
554	RDMAP_INV_OPCODE	= 0x06,
555	RDMAP_STREAM_CATA	= 0x07,
556	RDMAP_GLOBAL_CATA	= 0x08,
557	RDMAP_CANT_INV_STAG	= 0x09,
558	RDMAP_UNSPECIFIED	= 0xff
559};
560
561enum c4iw_ddp_ecodes {
562	DDPT_INV_STAG		= 0x00,
563	DDPT_BASE_BOUNDS	= 0x01,
564	DDPT_STAG_NOT_ASSOC	= 0x02,
565	DDPT_TO_WRAP		= 0x03,
566	DDPT_INV_VERS		= 0x04,
567	DDPU_INV_QN		= 0x01,
568	DDPU_INV_MSN_NOBUF	= 0x02,
569	DDPU_INV_MSN_RANGE	= 0x03,
570	DDPU_INV_MO		= 0x04,
571	DDPU_MSG_TOOBIG		= 0x05,
572	DDPU_INV_VERS		= 0x06
573};
574
575enum c4iw_mpa_ecodes {
576	MPA_CRC_ERR		= 0x02,
577	MPA_MARKER_ERR		= 0x03
578};
579
580enum c4iw_ep_state {
581	IDLE = 0,
582	LISTEN,
583	CONNECTING,
584	MPA_REQ_WAIT,
585	MPA_REQ_SENT,
586	MPA_REQ_RCVD,
587	MPA_REP_SENT,
588	FPDU_MODE,
589	ABORTING,
590	CLOSING,
591	MORIBUND,
592	DEAD,
593};
594
595enum c4iw_ep_flags {
596	PEER_ABORT_IN_PROGRESS	= 0,
597	ABORT_REQ_IN_PROGRESS	= 1,
598	RELEASE_RESOURCES	= 2,
599	CLOSE_SENT		= 3,
600};
601
602struct c4iw_ep_common {
603	struct iw_cm_id *cm_id;
604	struct c4iw_qp *qp;
605	struct c4iw_dev *dev;
606	enum c4iw_ep_state state;
607	struct kref kref;
608	struct mutex mutex;
609	struct sockaddr_in local_addr;
610	struct sockaddr_in remote_addr;
611	struct c4iw_wr_wait wr_wait;
612	unsigned long flags;
613};
614
615struct c4iw_listen_ep {
616	struct c4iw_ep_common com;
617	unsigned int stid;
618	int backlog;
619};
620
621struct c4iw_ep {
622	struct c4iw_ep_common com;
623	struct c4iw_ep *parent_ep;
624	struct timer_list timer;
625	struct list_head entry;
626	unsigned int atid;
627	u32 hwtid;
628	u32 snd_seq;
629	u32 rcv_seq;
630	struct l2t_entry *l2t;
631	struct dst_entry *dst;
632	struct sk_buff *mpa_skb;
633	struct c4iw_mpa_attributes mpa_attr;
634	u8 mpa_pkt[sizeof(struct mpa_message) + MPA_MAX_PRIVATE_DATA];
635	unsigned int mpa_pkt_len;
636	u32 ird;
637	u32 ord;
638	u32 smac_idx;
639	u32 tx_chan;
640	u32 mtu;
641	u16 mss;
642	u16 emss;
643	u16 plen;
644	u16 rss_qid;
645	u16 txq_idx;
646	u16 ctrlq_idx;
647	u8 tos;
648};
649
650static inline struct c4iw_ep *to_ep(struct iw_cm_id *cm_id)
651{
652	return cm_id->provider_data;
653}
654
655static inline struct c4iw_listen_ep *to_listen_ep(struct iw_cm_id *cm_id)
656{
657	return cm_id->provider_data;
658}
659
660static inline int compute_wscale(int win)
661{
662	int wscale = 0;
663
664	while (wscale < 14 && (65535<<wscale) < win)
665		wscale++;
666	return wscale;
667}
668
669typedef int (*c4iw_handler_func)(struct c4iw_dev *dev, struct sk_buff *skb);
670
671int c4iw_ep_redirect(void *ctx, struct dst_entry *old, struct dst_entry *new,
672		     struct l2t_entry *l2t);
673void c4iw_put_qpid(struct c4iw_rdev *rdev, u32 qpid,
674		   struct c4iw_dev_ucontext *uctx);
675u32 c4iw_get_resource(struct kfifo *fifo, spinlock_t *lock);
676void c4iw_put_resource(struct kfifo *fifo, u32 entry, spinlock_t *lock);
677int c4iw_init_resource(struct c4iw_rdev *rdev, u32 nr_tpt, u32 nr_pdid);
678int c4iw_init_ctrl_qp(struct c4iw_rdev *rdev);
679int c4iw_pblpool_create(struct c4iw_rdev *rdev);
680int c4iw_rqtpool_create(struct c4iw_rdev *rdev);
681int c4iw_ocqp_pool_create(struct c4iw_rdev *rdev);
682void c4iw_pblpool_destroy(struct c4iw_rdev *rdev);
683void c4iw_rqtpool_destroy(struct c4iw_rdev *rdev);
684void c4iw_ocqp_pool_destroy(struct c4iw_rdev *rdev);
685void c4iw_destroy_resource(struct c4iw_resource *rscp);
686int c4iw_destroy_ctrl_qp(struct c4iw_rdev *rdev);
687int c4iw_register_device(struct c4iw_dev *dev);
688void c4iw_unregister_device(struct c4iw_dev *dev);
689int __init c4iw_cm_init(void);
690void __exit c4iw_cm_term(void);
691void c4iw_release_dev_ucontext(struct c4iw_rdev *rdev,
692			       struct c4iw_dev_ucontext *uctx);
693void c4iw_init_dev_ucontext(struct c4iw_rdev *rdev,
694			    struct c4iw_dev_ucontext *uctx);
695int c4iw_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
696int c4iw_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
697		      struct ib_send_wr **bad_wr);
698int c4iw_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr,
699		      struct ib_recv_wr **bad_wr);
700int c4iw_bind_mw(struct ib_qp *qp, struct ib_mw *mw,
701		 struct ib_mw_bind *mw_bind);
702int c4iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param);
703int c4iw_create_listen(struct iw_cm_id *cm_id, int backlog);
704int c4iw_destroy_listen(struct iw_cm_id *cm_id);
705int c4iw_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param);
706int c4iw_reject_cr(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len);
707void c4iw_qp_add_ref(struct ib_qp *qp);
708void c4iw_qp_rem_ref(struct ib_qp *qp);
709void c4iw_free_fastreg_pbl(struct ib_fast_reg_page_list *page_list);
710struct ib_fast_reg_page_list *c4iw_alloc_fastreg_pbl(
711					struct ib_device *device,
712					int page_list_len);
713struct ib_mr *c4iw_alloc_fast_reg_mr(struct ib_pd *pd, int pbl_depth);
714int c4iw_dealloc_mw(struct ib_mw *mw);
715struct ib_mw *c4iw_alloc_mw(struct ib_pd *pd);
716struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start,
717					   u64 length, u64 virt, int acc,
718					   struct ib_udata *udata);
719struct ib_mr *c4iw_get_dma_mr(struct ib_pd *pd, int acc);
720struct ib_mr *c4iw_register_phys_mem(struct ib_pd *pd,
721					struct ib_phys_buf *buffer_list,
722					int num_phys_buf,
723					int acc,
724					u64 *iova_start);
725int c4iw_reregister_phys_mem(struct ib_mr *mr,
726				     int mr_rereg_mask,
727				     struct ib_pd *pd,
728				     struct ib_phys_buf *buffer_list,
729				     int num_phys_buf,
730				     int acc, u64 *iova_start);
731int c4iw_dereg_mr(struct ib_mr *ib_mr);
732int c4iw_destroy_cq(struct ib_cq *ib_cq);
733struct ib_cq *c4iw_create_cq(struct ib_device *ibdev, int entries,
734					int vector,
735					struct ib_ucontext *ib_context,
736					struct ib_udata *udata);
737int c4iw_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata);
738int c4iw_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
739int c4iw_destroy_qp(struct ib_qp *ib_qp);
740struct ib_qp *c4iw_create_qp(struct ib_pd *pd,
741			     struct ib_qp_init_attr *attrs,
742			     struct ib_udata *udata);
743int c4iw_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
744				 int attr_mask, struct ib_udata *udata);
745struct ib_qp *c4iw_get_qp(struct ib_device *dev, int qpn);
746u32 c4iw_rqtpool_alloc(struct c4iw_rdev *rdev, int size);
747void c4iw_rqtpool_free(struct c4iw_rdev *rdev, u32 addr, int size);
748u32 c4iw_pblpool_alloc(struct c4iw_rdev *rdev, int size);
749void c4iw_pblpool_free(struct c4iw_rdev *rdev, u32 addr, int size);
750u32 c4iw_ocqp_pool_alloc(struct c4iw_rdev *rdev, int size);
751void c4iw_ocqp_pool_free(struct c4iw_rdev *rdev, u32 addr, int size);
752int c4iw_ofld_send(struct c4iw_rdev *rdev, struct sk_buff *skb);
753void c4iw_flush_hw_cq(struct t4_cq *cq);
754void c4iw_count_rcqes(struct t4_cq *cq, struct t4_wq *wq, int *count);
755void c4iw_count_scqes(struct t4_cq *cq, struct t4_wq *wq, int *count);
756int c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp);
757int c4iw_flush_rq(struct t4_wq *wq, struct t4_cq *cq, int count);
758int c4iw_flush_sq(struct t4_wq *wq, struct t4_cq *cq, int count);
759int c4iw_ev_handler(struct c4iw_dev *rnicp, u32 qid);
760u16 c4iw_rqes_posted(struct c4iw_qp *qhp);
761int c4iw_post_terminate(struct c4iw_qp *qhp, struct t4_cqe *err_cqe);
762u32 c4iw_get_cqid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx);
763void c4iw_put_cqid(struct c4iw_rdev *rdev, u32 qid,
764		struct c4iw_dev_ucontext *uctx);
765u32 c4iw_get_qpid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx);
766void c4iw_put_qpid(struct c4iw_rdev *rdev, u32 qid,
767		struct c4iw_dev_ucontext *uctx);
768void c4iw_ev_dispatch(struct c4iw_dev *dev, struct t4_cqe *err_cqe);
769
770extern struct cxgb4_client t4c_client;
771extern c4iw_handler_func c4iw_handlers[NUM_CPL_CMDS];
772extern int c4iw_max_read_depth;
773
774#endif
775