device.c revision 1c01c5388306a4c2245b71da0cba22d521e897ae
1/* 2 * Copyright (c) 2009-2010 Chelsio, Inc. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32#include <linux/module.h> 33#include <linux/moduleparam.h> 34#include <linux/debugfs.h> 35 36#include <rdma/ib_verbs.h> 37 38#include "iw_cxgb4.h" 39 40#define DRV_VERSION "0.1" 41 42MODULE_AUTHOR("Steve Wise"); 43MODULE_DESCRIPTION("Chelsio T4 RDMA Driver"); 44MODULE_LICENSE("Dual BSD/GPL"); 45MODULE_VERSION(DRV_VERSION); 46 47static LIST_HEAD(dev_list); 48static DEFINE_MUTEX(dev_mutex); 49 50static struct dentry *c4iw_debugfs_root; 51 52struct debugfs_qp_data { 53 struct c4iw_dev *devp; 54 char *buf; 55 int bufsize; 56 int pos; 57}; 58 59static int count_qps(int id, void *p, void *data) 60{ 61 struct c4iw_qp *qp = p; 62 int *countp = data; 63 64 if (id != qp->wq.sq.qid) 65 return 0; 66 67 *countp = *countp + 1; 68 return 0; 69} 70 71static int dump_qps(int id, void *p, void *data) 72{ 73 struct c4iw_qp *qp = p; 74 struct debugfs_qp_data *qpd = data; 75 int space; 76 int cc; 77 78 if (id != qp->wq.sq.qid) 79 return 0; 80 81 space = qpd->bufsize - qpd->pos - 1; 82 if (space == 0) 83 return 1; 84 85 if (qp->ep) 86 cc = snprintf(qpd->buf + qpd->pos, space, "qp id %u state %u " 87 "ep tid %u state %u %pI4:%u->%pI4:%u\n", 88 qp->wq.sq.qid, (int)qp->attr.state, 89 qp->ep->hwtid, (int)qp->ep->com.state, 90 &qp->ep->com.local_addr.sin_addr.s_addr, 91 ntohs(qp->ep->com.local_addr.sin_port), 92 &qp->ep->com.remote_addr.sin_addr.s_addr, 93 ntohs(qp->ep->com.remote_addr.sin_port)); 94 else 95 cc = snprintf(qpd->buf + qpd->pos, space, "qp id %u state %u\n", 96 qp->wq.sq.qid, (int)qp->attr.state); 97 if (cc < space) 98 qpd->pos += cc; 99 return 0; 100} 101 102static int qp_release(struct inode *inode, struct file *file) 103{ 104 struct debugfs_qp_data *qpd = file->private_data; 105 if (!qpd) { 106 printk(KERN_INFO "%s null qpd?\n", __func__); 107 return 0; 108 } 109 kfree(qpd->buf); 110 kfree(qpd); 111 return 0; 112} 113 114static int qp_open(struct inode *inode, struct file *file) 115{ 116 struct debugfs_qp_data *qpd; 117 int ret = 0; 118 int count = 1; 119 120 qpd = kmalloc(sizeof *qpd, GFP_KERNEL); 121 if (!qpd) { 122 ret = -ENOMEM; 123 goto out; 124 } 125 qpd->devp = inode->i_private; 126 qpd->pos = 0; 127 128 spin_lock_irq(&qpd->devp->lock); 129 idr_for_each(&qpd->devp->qpidr, count_qps, &count); 130 spin_unlock_irq(&qpd->devp->lock); 131 132 qpd->bufsize = count * 128; 133 qpd->buf = kmalloc(qpd->bufsize, GFP_KERNEL); 134 if (!qpd->buf) { 135 ret = -ENOMEM; 136 goto err1; 137 } 138 139 spin_lock_irq(&qpd->devp->lock); 140 idr_for_each(&qpd->devp->qpidr, dump_qps, qpd); 141 spin_unlock_irq(&qpd->devp->lock); 142 143 qpd->buf[qpd->pos++] = 0; 144 file->private_data = qpd; 145 goto out; 146err1: 147 kfree(qpd); 148out: 149 return ret; 150} 151 152static ssize_t qp_read(struct file *file, char __user *buf, size_t count, 153 loff_t *ppos) 154{ 155 struct debugfs_qp_data *qpd = file->private_data; 156 loff_t pos = *ppos; 157 loff_t avail = qpd->pos; 158 159 if (pos < 0) 160 return -EINVAL; 161 if (pos >= avail) 162 return 0; 163 if (count > avail - pos) 164 count = avail - pos; 165 166 while (count) { 167 size_t len = 0; 168 169 len = min((int)count, (int)qpd->pos - (int)pos); 170 if (copy_to_user(buf, qpd->buf + pos, len)) 171 return -EFAULT; 172 if (len == 0) 173 return -EINVAL; 174 175 buf += len; 176 pos += len; 177 count -= len; 178 } 179 count = pos - *ppos; 180 *ppos = pos; 181 return count; 182} 183 184static const struct file_operations qp_debugfs_fops = { 185 .owner = THIS_MODULE, 186 .open = qp_open, 187 .release = qp_release, 188 .read = qp_read, 189}; 190 191static int setup_debugfs(struct c4iw_dev *devp) 192{ 193 struct dentry *de; 194 195 if (!devp->debugfs_root) 196 return -1; 197 198 de = debugfs_create_file("qps", S_IWUSR, devp->debugfs_root, 199 (void *)devp, &qp_debugfs_fops); 200 if (de && de->d_inode) 201 de->d_inode->i_size = 4096; 202 return 0; 203} 204 205void c4iw_release_dev_ucontext(struct c4iw_rdev *rdev, 206 struct c4iw_dev_ucontext *uctx) 207{ 208 struct list_head *pos, *nxt; 209 struct c4iw_qid_list *entry; 210 211 mutex_lock(&uctx->lock); 212 list_for_each_safe(pos, nxt, &uctx->qpids) { 213 entry = list_entry(pos, struct c4iw_qid_list, entry); 214 list_del_init(&entry->entry); 215 if (!(entry->qid & rdev->qpmask)) 216 c4iw_put_resource(&rdev->resource.qid_fifo, entry->qid, 217 &rdev->resource.qid_fifo_lock); 218 kfree(entry); 219 } 220 221 list_for_each_safe(pos, nxt, &uctx->qpids) { 222 entry = list_entry(pos, struct c4iw_qid_list, entry); 223 list_del_init(&entry->entry); 224 kfree(entry); 225 } 226 mutex_unlock(&uctx->lock); 227} 228 229void c4iw_init_dev_ucontext(struct c4iw_rdev *rdev, 230 struct c4iw_dev_ucontext *uctx) 231{ 232 INIT_LIST_HEAD(&uctx->qpids); 233 INIT_LIST_HEAD(&uctx->cqids); 234 mutex_init(&uctx->lock); 235} 236 237/* Caller takes care of locking if needed */ 238static int c4iw_rdev_open(struct c4iw_rdev *rdev) 239{ 240 int err; 241 242 c4iw_init_dev_ucontext(rdev, &rdev->uctx); 243 244 /* 245 * qpshift is the number of bits to shift the qpid left in order 246 * to get the correct address of the doorbell for that qp. 247 */ 248 rdev->qpshift = PAGE_SHIFT - ilog2(rdev->lldi.udb_density); 249 rdev->qpmask = rdev->lldi.udb_density - 1; 250 rdev->cqshift = PAGE_SHIFT - ilog2(rdev->lldi.ucq_density); 251 rdev->cqmask = rdev->lldi.ucq_density - 1; 252 PDBG("%s dev %s stag start 0x%0x size 0x%0x num stags %d " 253 "pbl start 0x%0x size 0x%0x rq start 0x%0x size 0x%0x\n", 254 __func__, pci_name(rdev->lldi.pdev), rdev->lldi.vr->stag.start, 255 rdev->lldi.vr->stag.size, c4iw_num_stags(rdev), 256 rdev->lldi.vr->pbl.start, 257 rdev->lldi.vr->pbl.size, rdev->lldi.vr->rq.start, 258 rdev->lldi.vr->rq.size); 259 PDBG("udb len 0x%x udb base %p db_reg %p gts_reg %p qpshift %lu " 260 "qpmask 0x%x cqshift %lu cqmask 0x%x\n", 261 (unsigned)pci_resource_len(rdev->lldi.pdev, 2), 262 (void *)pci_resource_start(rdev->lldi.pdev, 2), 263 rdev->lldi.db_reg, 264 rdev->lldi.gts_reg, 265 rdev->qpshift, rdev->qpmask, 266 rdev->cqshift, rdev->cqmask); 267 268 if (c4iw_num_stags(rdev) == 0) { 269 err = -EINVAL; 270 goto err1; 271 } 272 273 err = c4iw_init_resource(rdev, c4iw_num_stags(rdev), T4_MAX_NUM_PD); 274 if (err) { 275 printk(KERN_ERR MOD "error %d initializing resources\n", err); 276 goto err1; 277 } 278 err = c4iw_pblpool_create(rdev); 279 if (err) { 280 printk(KERN_ERR MOD "error %d initializing pbl pool\n", err); 281 goto err2; 282 } 283 err = c4iw_rqtpool_create(rdev); 284 if (err) { 285 printk(KERN_ERR MOD "error %d initializing rqt pool\n", err); 286 goto err3; 287 } 288 return 0; 289err3: 290 c4iw_pblpool_destroy(rdev); 291err2: 292 c4iw_destroy_resource(&rdev->resource); 293err1: 294 return err; 295} 296 297static void c4iw_rdev_close(struct c4iw_rdev *rdev) 298{ 299 c4iw_pblpool_destroy(rdev); 300 c4iw_rqtpool_destroy(rdev); 301 c4iw_destroy_resource(&rdev->resource); 302} 303 304static void c4iw_remove(struct c4iw_dev *dev) 305{ 306 PDBG("%s c4iw_dev %p\n", __func__, dev); 307 cancel_delayed_work_sync(&dev->db_drop_task); 308 list_del(&dev->entry); 309 if (dev->registered) 310 c4iw_unregister_device(dev); 311 c4iw_rdev_close(&dev->rdev); 312 idr_destroy(&dev->cqidr); 313 idr_destroy(&dev->qpidr); 314 idr_destroy(&dev->mmidr); 315 ib_dealloc_device(&dev->ibdev); 316} 317 318static struct c4iw_dev *c4iw_alloc(const struct cxgb4_lld_info *infop) 319{ 320 struct c4iw_dev *devp; 321 int ret; 322 323 devp = (struct c4iw_dev *)ib_alloc_device(sizeof(*devp)); 324 if (!devp) { 325 printk(KERN_ERR MOD "Cannot allocate ib device\n"); 326 return NULL; 327 } 328 devp->rdev.lldi = *infop; 329 330 mutex_lock(&dev_mutex); 331 332 ret = c4iw_rdev_open(&devp->rdev); 333 if (ret) { 334 mutex_unlock(&dev_mutex); 335 printk(KERN_ERR MOD "Unable to open CXIO rdev err %d\n", ret); 336 ib_dealloc_device(&devp->ibdev); 337 return NULL; 338 } 339 340 idr_init(&devp->cqidr); 341 idr_init(&devp->qpidr); 342 idr_init(&devp->mmidr); 343 spin_lock_init(&devp->lock); 344 list_add_tail(&devp->entry, &dev_list); 345 mutex_unlock(&dev_mutex); 346 347 if (c4iw_debugfs_root) { 348 devp->debugfs_root = debugfs_create_dir( 349 pci_name(devp->rdev.lldi.pdev), 350 c4iw_debugfs_root); 351 setup_debugfs(devp); 352 } 353 return devp; 354} 355 356static void *c4iw_uld_add(const struct cxgb4_lld_info *infop) 357{ 358 struct c4iw_dev *dev; 359 static int vers_printed; 360 int i; 361 362 if (!vers_printed++) 363 printk(KERN_INFO MOD "Chelsio T4 RDMA Driver - version %s\n", 364 DRV_VERSION); 365 366 dev = c4iw_alloc(infop); 367 if (!dev) 368 goto out; 369 370 PDBG("%s found device %s nchan %u nrxq %u ntxq %u nports %u\n", 371 __func__, pci_name(dev->rdev.lldi.pdev), 372 dev->rdev.lldi.nchan, dev->rdev.lldi.nrxq, 373 dev->rdev.lldi.ntxq, dev->rdev.lldi.nports); 374 375 for (i = 0; i < dev->rdev.lldi.nrxq; i++) 376 PDBG("rxqid[%u] %u\n", i, dev->rdev.lldi.rxq_ids[i]); 377out: 378 return dev; 379} 380 381static struct sk_buff *t4_pktgl_to_skb(const struct pkt_gl *gl, 382 unsigned int skb_len, 383 unsigned int pull_len) 384{ 385 struct sk_buff *skb; 386 struct skb_shared_info *ssi; 387 388 if (gl->tot_len <= 512) { 389 skb = alloc_skb(gl->tot_len, GFP_ATOMIC); 390 if (unlikely(!skb)) 391 goto out; 392 __skb_put(skb, gl->tot_len); 393 skb_copy_to_linear_data(skb, gl->va, gl->tot_len); 394 } else { 395 skb = alloc_skb(skb_len, GFP_ATOMIC); 396 if (unlikely(!skb)) 397 goto out; 398 __skb_put(skb, pull_len); 399 skb_copy_to_linear_data(skb, gl->va, pull_len); 400 401 ssi = skb_shinfo(skb); 402 ssi->frags[0].page = gl->frags[0].page; 403 ssi->frags[0].page_offset = gl->frags[0].page_offset + pull_len; 404 ssi->frags[0].size = gl->frags[0].size - pull_len; 405 if (gl->nfrags > 1) 406 memcpy(&ssi->frags[1], &gl->frags[1], 407 (gl->nfrags - 1) * sizeof(skb_frag_t)); 408 ssi->nr_frags = gl->nfrags; 409 410 skb->len = gl->tot_len; 411 skb->data_len = skb->len - pull_len; 412 skb->truesize += skb->data_len; 413 414 /* Get a reference for the last page, we don't own it */ 415 get_page(gl->frags[gl->nfrags - 1].page); 416 } 417out: 418 return skb; 419} 420 421static int c4iw_uld_rx_handler(void *handle, const __be64 *rsp, 422 const struct pkt_gl *gl) 423{ 424 struct c4iw_dev *dev = handle; 425 struct sk_buff *skb; 426 const struct cpl_act_establish *rpl; 427 unsigned int opcode; 428 429 if (gl == NULL) { 430 /* omit RSS and rsp_ctrl at end of descriptor */ 431 unsigned int len = 64 - sizeof(struct rsp_ctrl) - 8; 432 433 skb = alloc_skb(256, GFP_ATOMIC); 434 if (!skb) 435 goto nomem; 436 __skb_put(skb, len); 437 skb_copy_to_linear_data(skb, &rsp[1], len); 438 } else if (gl == CXGB4_MSG_AN) { 439 const struct rsp_ctrl *rc = (void *)rsp; 440 441 u32 qid = be32_to_cpu(rc->pldbuflen_qid); 442 c4iw_ev_handler(dev, qid); 443 return 0; 444 } else { 445 skb = t4_pktgl_to_skb(gl, 128, 128); 446 if (unlikely(!skb)) 447 goto nomem; 448 } 449 450 rpl = cplhdr(skb); 451 opcode = rpl->ot.opcode; 452 453 if (c4iw_handlers[opcode]) 454 c4iw_handlers[opcode](dev, skb); 455 else 456 printk(KERN_INFO "%s no handler opcode 0x%x...\n", __func__, 457 opcode); 458 459 return 0; 460nomem: 461 return -1; 462} 463 464static int c4iw_uld_state_change(void *handle, enum cxgb4_state new_state) 465{ 466 struct c4iw_dev *dev = handle; 467 468 PDBG("%s new_state %u\n", __func__, new_state); 469 switch (new_state) { 470 case CXGB4_STATE_UP: 471 printk(KERN_INFO MOD "%s: Up\n", pci_name(dev->rdev.lldi.pdev)); 472 if (!dev->registered) { 473 int ret; 474 ret = c4iw_register_device(dev); 475 if (ret) 476 printk(KERN_ERR MOD 477 "%s: RDMA registration failed: %d\n", 478 pci_name(dev->rdev.lldi.pdev), ret); 479 } 480 break; 481 case CXGB4_STATE_DOWN: 482 printk(KERN_INFO MOD "%s: Down\n", 483 pci_name(dev->rdev.lldi.pdev)); 484 if (dev->registered) 485 c4iw_unregister_device(dev); 486 break; 487 case CXGB4_STATE_START_RECOVERY: 488 printk(KERN_INFO MOD "%s: Fatal Error\n", 489 pci_name(dev->rdev.lldi.pdev)); 490 if (dev->registered) 491 c4iw_unregister_device(dev); 492 break; 493 case CXGB4_STATE_DETACH: 494 printk(KERN_INFO MOD "%s: Detach\n", 495 pci_name(dev->rdev.lldi.pdev)); 496 mutex_lock(&dev_mutex); 497 c4iw_remove(dev); 498 mutex_unlock(&dev_mutex); 499 break; 500 } 501 return 0; 502} 503 504static struct cxgb4_uld_info c4iw_uld_info = { 505 .name = DRV_NAME, 506 .add = c4iw_uld_add, 507 .rx_handler = c4iw_uld_rx_handler, 508 .state_change = c4iw_uld_state_change, 509}; 510 511static int __init c4iw_init_module(void) 512{ 513 int err; 514 515 err = c4iw_cm_init(); 516 if (err) 517 return err; 518 519 c4iw_debugfs_root = debugfs_create_dir(DRV_NAME, NULL); 520 if (!c4iw_debugfs_root) 521 printk(KERN_WARNING MOD 522 "could not create debugfs entry, continuing\n"); 523 524 cxgb4_register_uld(CXGB4_ULD_RDMA, &c4iw_uld_info); 525 526 return 0; 527} 528 529static void __exit c4iw_exit_module(void) 530{ 531 struct c4iw_dev *dev, *tmp; 532 533 mutex_lock(&dev_mutex); 534 list_for_each_entry_safe(dev, tmp, &dev_list, entry) { 535 c4iw_remove(dev); 536 } 537 mutex_unlock(&dev_mutex); 538 cxgb4_unregister_uld(CXGB4_ULD_RDMA); 539 c4iw_cm_term(); 540 debugfs_remove_recursive(c4iw_debugfs_root); 541} 542 543module_init(c4iw_init_module); 544module_exit(c4iw_exit_module); 545