device.c revision 8bbac892fb75d20fa274ca026e24faf00afbf9dd
1/* 2 * Copyright (c) 2009-2010 Chelsio, Inc. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32#include <linux/module.h> 33#include <linux/moduleparam.h> 34#include <linux/debugfs.h> 35 36#include <rdma/ib_verbs.h> 37 38#include "iw_cxgb4.h" 39 40#define DRV_VERSION "0.1" 41 42MODULE_AUTHOR("Steve Wise"); 43MODULE_DESCRIPTION("Chelsio T4 RDMA Driver"); 44MODULE_LICENSE("Dual BSD/GPL"); 45MODULE_VERSION(DRV_VERSION); 46 47static LIST_HEAD(dev_list); 48static DEFINE_MUTEX(dev_mutex); 49 50static struct dentry *c4iw_debugfs_root; 51 52struct c4iw_debugfs_data { 53 struct c4iw_dev *devp; 54 char *buf; 55 int bufsize; 56 int pos; 57}; 58 59static int count_idrs(int id, void *p, void *data) 60{ 61 int *countp = data; 62 63 *countp = *countp + 1; 64 return 0; 65} 66 67static ssize_t debugfs_read(struct file *file, char __user *buf, size_t count, 68 loff_t *ppos) 69{ 70 struct c4iw_debugfs_data *d = file->private_data; 71 loff_t pos = *ppos; 72 loff_t avail = d->pos; 73 74 if (pos < 0) 75 return -EINVAL; 76 if (pos >= avail) 77 return 0; 78 if (count > avail - pos) 79 count = avail - pos; 80 81 while (count) { 82 size_t len = 0; 83 84 len = min((int)count, (int)d->pos - (int)pos); 85 if (copy_to_user(buf, d->buf + pos, len)) 86 return -EFAULT; 87 if (len == 0) 88 return -EINVAL; 89 90 buf += len; 91 pos += len; 92 count -= len; 93 } 94 count = pos - *ppos; 95 *ppos = pos; 96 return count; 97} 98 99static int dump_qp(int id, void *p, void *data) 100{ 101 struct c4iw_qp *qp = p; 102 struct c4iw_debugfs_data *qpd = data; 103 int space; 104 int cc; 105 106 if (id != qp->wq.sq.qid) 107 return 0; 108 109 space = qpd->bufsize - qpd->pos - 1; 110 if (space == 0) 111 return 1; 112 113 if (qp->ep) 114 cc = snprintf(qpd->buf + qpd->pos, space, "qp id %u state %u " 115 "ep tid %u state %u %pI4:%u->%pI4:%u\n", 116 qp->wq.sq.qid, (int)qp->attr.state, 117 qp->ep->hwtid, (int)qp->ep->com.state, 118 &qp->ep->com.local_addr.sin_addr.s_addr, 119 ntohs(qp->ep->com.local_addr.sin_port), 120 &qp->ep->com.remote_addr.sin_addr.s_addr, 121 ntohs(qp->ep->com.remote_addr.sin_port)); 122 else 123 cc = snprintf(qpd->buf + qpd->pos, space, "qp id %u state %u\n", 124 qp->wq.sq.qid, (int)qp->attr.state); 125 if (cc < space) 126 qpd->pos += cc; 127 return 0; 128} 129 130static int qp_release(struct inode *inode, struct file *file) 131{ 132 struct c4iw_debugfs_data *qpd = file->private_data; 133 if (!qpd) { 134 printk(KERN_INFO "%s null qpd?\n", __func__); 135 return 0; 136 } 137 kfree(qpd->buf); 138 kfree(qpd); 139 return 0; 140} 141 142static int qp_open(struct inode *inode, struct file *file) 143{ 144 struct c4iw_debugfs_data *qpd; 145 int ret = 0; 146 int count = 1; 147 148 qpd = kmalloc(sizeof *qpd, GFP_KERNEL); 149 if (!qpd) { 150 ret = -ENOMEM; 151 goto out; 152 } 153 qpd->devp = inode->i_private; 154 qpd->pos = 0; 155 156 spin_lock_irq(&qpd->devp->lock); 157 idr_for_each(&qpd->devp->qpidr, count_idrs, &count); 158 spin_unlock_irq(&qpd->devp->lock); 159 160 qpd->bufsize = count * 128; 161 qpd->buf = kmalloc(qpd->bufsize, GFP_KERNEL); 162 if (!qpd->buf) { 163 ret = -ENOMEM; 164 goto err1; 165 } 166 167 spin_lock_irq(&qpd->devp->lock); 168 idr_for_each(&qpd->devp->qpidr, dump_qp, qpd); 169 spin_unlock_irq(&qpd->devp->lock); 170 171 qpd->buf[qpd->pos++] = 0; 172 file->private_data = qpd; 173 goto out; 174err1: 175 kfree(qpd); 176out: 177 return ret; 178} 179 180static const struct file_operations qp_debugfs_fops = { 181 .owner = THIS_MODULE, 182 .open = qp_open, 183 .release = qp_release, 184 .read = debugfs_read, 185 .llseek = default_llseek, 186}; 187 188static int dump_stag(int id, void *p, void *data) 189{ 190 struct c4iw_debugfs_data *stagd = data; 191 int space; 192 int cc; 193 194 space = stagd->bufsize - stagd->pos - 1; 195 if (space == 0) 196 return 1; 197 198 cc = snprintf(stagd->buf + stagd->pos, space, "0x%x\n", id<<8); 199 if (cc < space) 200 stagd->pos += cc; 201 return 0; 202} 203 204static int stag_release(struct inode *inode, struct file *file) 205{ 206 struct c4iw_debugfs_data *stagd = file->private_data; 207 if (!stagd) { 208 printk(KERN_INFO "%s null stagd?\n", __func__); 209 return 0; 210 } 211 kfree(stagd->buf); 212 kfree(stagd); 213 return 0; 214} 215 216static int stag_open(struct inode *inode, struct file *file) 217{ 218 struct c4iw_debugfs_data *stagd; 219 int ret = 0; 220 int count = 1; 221 222 stagd = kmalloc(sizeof *stagd, GFP_KERNEL); 223 if (!stagd) { 224 ret = -ENOMEM; 225 goto out; 226 } 227 stagd->devp = inode->i_private; 228 stagd->pos = 0; 229 230 spin_lock_irq(&stagd->devp->lock); 231 idr_for_each(&stagd->devp->mmidr, count_idrs, &count); 232 spin_unlock_irq(&stagd->devp->lock); 233 234 stagd->bufsize = count * sizeof("0x12345678\n"); 235 stagd->buf = kmalloc(stagd->bufsize, GFP_KERNEL); 236 if (!stagd->buf) { 237 ret = -ENOMEM; 238 goto err1; 239 } 240 241 spin_lock_irq(&stagd->devp->lock); 242 idr_for_each(&stagd->devp->mmidr, dump_stag, stagd); 243 spin_unlock_irq(&stagd->devp->lock); 244 245 stagd->buf[stagd->pos++] = 0; 246 file->private_data = stagd; 247 goto out; 248err1: 249 kfree(stagd); 250out: 251 return ret; 252} 253 254static const struct file_operations stag_debugfs_fops = { 255 .owner = THIS_MODULE, 256 .open = stag_open, 257 .release = stag_release, 258 .read = debugfs_read, 259 .llseek = default_llseek, 260}; 261 262static int setup_debugfs(struct c4iw_dev *devp) 263{ 264 struct dentry *de; 265 266 if (!devp->debugfs_root) 267 return -1; 268 269 de = debugfs_create_file("qps", S_IWUSR, devp->debugfs_root, 270 (void *)devp, &qp_debugfs_fops); 271 if (de && de->d_inode) 272 de->d_inode->i_size = 4096; 273 274 de = debugfs_create_file("stags", S_IWUSR, devp->debugfs_root, 275 (void *)devp, &stag_debugfs_fops); 276 if (de && de->d_inode) 277 de->d_inode->i_size = 4096; 278 return 0; 279} 280 281void c4iw_release_dev_ucontext(struct c4iw_rdev *rdev, 282 struct c4iw_dev_ucontext *uctx) 283{ 284 struct list_head *pos, *nxt; 285 struct c4iw_qid_list *entry; 286 287 mutex_lock(&uctx->lock); 288 list_for_each_safe(pos, nxt, &uctx->qpids) { 289 entry = list_entry(pos, struct c4iw_qid_list, entry); 290 list_del_init(&entry->entry); 291 if (!(entry->qid & rdev->qpmask)) 292 c4iw_put_resource(&rdev->resource.qid_fifo, entry->qid, 293 &rdev->resource.qid_fifo_lock); 294 kfree(entry); 295 } 296 297 list_for_each_safe(pos, nxt, &uctx->qpids) { 298 entry = list_entry(pos, struct c4iw_qid_list, entry); 299 list_del_init(&entry->entry); 300 kfree(entry); 301 } 302 mutex_unlock(&uctx->lock); 303} 304 305void c4iw_init_dev_ucontext(struct c4iw_rdev *rdev, 306 struct c4iw_dev_ucontext *uctx) 307{ 308 INIT_LIST_HEAD(&uctx->qpids); 309 INIT_LIST_HEAD(&uctx->cqids); 310 mutex_init(&uctx->lock); 311} 312 313/* Caller takes care of locking if needed */ 314static int c4iw_rdev_open(struct c4iw_rdev *rdev) 315{ 316 int err; 317 318 c4iw_init_dev_ucontext(rdev, &rdev->uctx); 319 320 /* 321 * qpshift is the number of bits to shift the qpid left in order 322 * to get the correct address of the doorbell for that qp. 323 */ 324 rdev->qpshift = PAGE_SHIFT - ilog2(rdev->lldi.udb_density); 325 rdev->qpmask = rdev->lldi.udb_density - 1; 326 rdev->cqshift = PAGE_SHIFT - ilog2(rdev->lldi.ucq_density); 327 rdev->cqmask = rdev->lldi.ucq_density - 1; 328 PDBG("%s dev %s stag start 0x%0x size 0x%0x num stags %d " 329 "pbl start 0x%0x size 0x%0x rq start 0x%0x size 0x%0x " 330 "qp qid start %u size %u cq qid start %u size %u\n", 331 __func__, pci_name(rdev->lldi.pdev), rdev->lldi.vr->stag.start, 332 rdev->lldi.vr->stag.size, c4iw_num_stags(rdev), 333 rdev->lldi.vr->pbl.start, 334 rdev->lldi.vr->pbl.size, rdev->lldi.vr->rq.start, 335 rdev->lldi.vr->rq.size, 336 rdev->lldi.vr->qp.start, 337 rdev->lldi.vr->qp.size, 338 rdev->lldi.vr->cq.start, 339 rdev->lldi.vr->cq.size); 340 PDBG("udb len 0x%x udb base %p db_reg %p gts_reg %p qpshift %lu " 341 "qpmask 0x%x cqshift %lu cqmask 0x%x\n", 342 (unsigned)pci_resource_len(rdev->lldi.pdev, 2), 343 (void *)pci_resource_start(rdev->lldi.pdev, 2), 344 rdev->lldi.db_reg, 345 rdev->lldi.gts_reg, 346 rdev->qpshift, rdev->qpmask, 347 rdev->cqshift, rdev->cqmask); 348 349 if (c4iw_num_stags(rdev) == 0) { 350 err = -EINVAL; 351 goto err1; 352 } 353 354 err = c4iw_init_resource(rdev, c4iw_num_stags(rdev), T4_MAX_NUM_PD); 355 if (err) { 356 printk(KERN_ERR MOD "error %d initializing resources\n", err); 357 goto err1; 358 } 359 err = c4iw_pblpool_create(rdev); 360 if (err) { 361 printk(KERN_ERR MOD "error %d initializing pbl pool\n", err); 362 goto err2; 363 } 364 err = c4iw_rqtpool_create(rdev); 365 if (err) { 366 printk(KERN_ERR MOD "error %d initializing rqt pool\n", err); 367 goto err3; 368 } 369 err = c4iw_ocqp_pool_create(rdev); 370 if (err) { 371 printk(KERN_ERR MOD "error %d initializing ocqp pool\n", err); 372 goto err4; 373 } 374 return 0; 375err4: 376 c4iw_rqtpool_destroy(rdev); 377err3: 378 c4iw_pblpool_destroy(rdev); 379err2: 380 c4iw_destroy_resource(&rdev->resource); 381err1: 382 return err; 383} 384 385static void c4iw_rdev_close(struct c4iw_rdev *rdev) 386{ 387 c4iw_pblpool_destroy(rdev); 388 c4iw_rqtpool_destroy(rdev); 389 c4iw_destroy_resource(&rdev->resource); 390} 391 392static void c4iw_remove(struct c4iw_dev *dev) 393{ 394 PDBG("%s c4iw_dev %p\n", __func__, dev); 395 cancel_delayed_work_sync(&dev->db_drop_task); 396 list_del(&dev->entry); 397 if (dev->registered) 398 c4iw_unregister_device(dev); 399 c4iw_rdev_close(&dev->rdev); 400 idr_destroy(&dev->cqidr); 401 idr_destroy(&dev->qpidr); 402 idr_destroy(&dev->mmidr); 403 iounmap(dev->rdev.oc_mw_kva); 404 ib_dealloc_device(&dev->ibdev); 405} 406 407static struct c4iw_dev *c4iw_alloc(const struct cxgb4_lld_info *infop) 408{ 409 struct c4iw_dev *devp; 410 int ret; 411 412 devp = (struct c4iw_dev *)ib_alloc_device(sizeof(*devp)); 413 if (!devp) { 414 printk(KERN_ERR MOD "Cannot allocate ib device\n"); 415 return NULL; 416 } 417 devp->rdev.lldi = *infop; 418 419 devp->rdev.oc_mw_pa = pci_resource_start(devp->rdev.lldi.pdev, 2) + 420 (pci_resource_len(devp->rdev.lldi.pdev, 2) - 421 roundup_pow_of_two(devp->rdev.lldi.vr->ocq.size)); 422 devp->rdev.oc_mw_kva = ioremap_wc(devp->rdev.oc_mw_pa, 423 devp->rdev.lldi.vr->ocq.size); 424 425 printk(KERN_INFO MOD "ocq memory: " 426 "hw_start 0x%x size %u mw_pa 0x%lx mw_kva %p\n", 427 devp->rdev.lldi.vr->ocq.start, devp->rdev.lldi.vr->ocq.size, 428 devp->rdev.oc_mw_pa, devp->rdev.oc_mw_kva); 429 430 mutex_lock(&dev_mutex); 431 432 ret = c4iw_rdev_open(&devp->rdev); 433 if (ret) { 434 mutex_unlock(&dev_mutex); 435 printk(KERN_ERR MOD "Unable to open CXIO rdev err %d\n", ret); 436 ib_dealloc_device(&devp->ibdev); 437 return NULL; 438 } 439 440 idr_init(&devp->cqidr); 441 idr_init(&devp->qpidr); 442 idr_init(&devp->mmidr); 443 spin_lock_init(&devp->lock); 444 list_add_tail(&devp->entry, &dev_list); 445 mutex_unlock(&dev_mutex); 446 447 if (c4iw_debugfs_root) { 448 devp->debugfs_root = debugfs_create_dir( 449 pci_name(devp->rdev.lldi.pdev), 450 c4iw_debugfs_root); 451 setup_debugfs(devp); 452 } 453 return devp; 454} 455 456static void *c4iw_uld_add(const struct cxgb4_lld_info *infop) 457{ 458 struct c4iw_dev *dev; 459 static int vers_printed; 460 int i; 461 462 if (!vers_printed++) 463 printk(KERN_INFO MOD "Chelsio T4 RDMA Driver - version %s\n", 464 DRV_VERSION); 465 466 dev = c4iw_alloc(infop); 467 if (!dev) 468 goto out; 469 470 PDBG("%s found device %s nchan %u nrxq %u ntxq %u nports %u\n", 471 __func__, pci_name(dev->rdev.lldi.pdev), 472 dev->rdev.lldi.nchan, dev->rdev.lldi.nrxq, 473 dev->rdev.lldi.ntxq, dev->rdev.lldi.nports); 474 475 for (i = 0; i < dev->rdev.lldi.nrxq; i++) 476 PDBG("rxqid[%u] %u\n", i, dev->rdev.lldi.rxq_ids[i]); 477out: 478 return dev; 479} 480 481static struct sk_buff *t4_pktgl_to_skb(const struct pkt_gl *gl, 482 unsigned int skb_len, 483 unsigned int pull_len) 484{ 485 struct sk_buff *skb; 486 struct skb_shared_info *ssi; 487 488 if (gl->tot_len <= 512) { 489 skb = alloc_skb(gl->tot_len, GFP_ATOMIC); 490 if (unlikely(!skb)) 491 goto out; 492 __skb_put(skb, gl->tot_len); 493 skb_copy_to_linear_data(skb, gl->va, gl->tot_len); 494 } else { 495 skb = alloc_skb(skb_len, GFP_ATOMIC); 496 if (unlikely(!skb)) 497 goto out; 498 __skb_put(skb, pull_len); 499 skb_copy_to_linear_data(skb, gl->va, pull_len); 500 501 ssi = skb_shinfo(skb); 502 ssi->frags[0].page = gl->frags[0].page; 503 ssi->frags[0].page_offset = gl->frags[0].page_offset + pull_len; 504 ssi->frags[0].size = gl->frags[0].size - pull_len; 505 if (gl->nfrags > 1) 506 memcpy(&ssi->frags[1], &gl->frags[1], 507 (gl->nfrags - 1) * sizeof(skb_frag_t)); 508 ssi->nr_frags = gl->nfrags; 509 510 skb->len = gl->tot_len; 511 skb->data_len = skb->len - pull_len; 512 skb->truesize += skb->data_len; 513 514 /* Get a reference for the last page, we don't own it */ 515 get_page(gl->frags[gl->nfrags - 1].page); 516 } 517out: 518 return skb; 519} 520 521static int c4iw_uld_rx_handler(void *handle, const __be64 *rsp, 522 const struct pkt_gl *gl) 523{ 524 struct c4iw_dev *dev = handle; 525 struct sk_buff *skb; 526 const struct cpl_act_establish *rpl; 527 unsigned int opcode; 528 529 if (gl == NULL) { 530 /* omit RSS and rsp_ctrl at end of descriptor */ 531 unsigned int len = 64 - sizeof(struct rsp_ctrl) - 8; 532 533 skb = alloc_skb(256, GFP_ATOMIC); 534 if (!skb) 535 goto nomem; 536 __skb_put(skb, len); 537 skb_copy_to_linear_data(skb, &rsp[1], len); 538 } else if (gl == CXGB4_MSG_AN) { 539 const struct rsp_ctrl *rc = (void *)rsp; 540 541 u32 qid = be32_to_cpu(rc->pldbuflen_qid); 542 c4iw_ev_handler(dev, qid); 543 return 0; 544 } else { 545 skb = t4_pktgl_to_skb(gl, 128, 128); 546 if (unlikely(!skb)) 547 goto nomem; 548 } 549 550 rpl = cplhdr(skb); 551 opcode = rpl->ot.opcode; 552 553 if (c4iw_handlers[opcode]) 554 c4iw_handlers[opcode](dev, skb); 555 else 556 printk(KERN_INFO "%s no handler opcode 0x%x...\n", __func__, 557 opcode); 558 559 return 0; 560nomem: 561 return -1; 562} 563 564static int c4iw_uld_state_change(void *handle, enum cxgb4_state new_state) 565{ 566 struct c4iw_dev *dev = handle; 567 568 PDBG("%s new_state %u\n", __func__, new_state); 569 switch (new_state) { 570 case CXGB4_STATE_UP: 571 printk(KERN_INFO MOD "%s: Up\n", pci_name(dev->rdev.lldi.pdev)); 572 if (!dev->registered) { 573 int ret; 574 ret = c4iw_register_device(dev); 575 if (ret) 576 printk(KERN_ERR MOD 577 "%s: RDMA registration failed: %d\n", 578 pci_name(dev->rdev.lldi.pdev), ret); 579 } 580 break; 581 case CXGB4_STATE_DOWN: 582 printk(KERN_INFO MOD "%s: Down\n", 583 pci_name(dev->rdev.lldi.pdev)); 584 if (dev->registered) 585 c4iw_unregister_device(dev); 586 break; 587 case CXGB4_STATE_START_RECOVERY: 588 printk(KERN_INFO MOD "%s: Fatal Error\n", 589 pci_name(dev->rdev.lldi.pdev)); 590 if (dev->registered) 591 c4iw_unregister_device(dev); 592 break; 593 case CXGB4_STATE_DETACH: 594 printk(KERN_INFO MOD "%s: Detach\n", 595 pci_name(dev->rdev.lldi.pdev)); 596 mutex_lock(&dev_mutex); 597 c4iw_remove(dev); 598 mutex_unlock(&dev_mutex); 599 break; 600 } 601 return 0; 602} 603 604static struct cxgb4_uld_info c4iw_uld_info = { 605 .name = DRV_NAME, 606 .add = c4iw_uld_add, 607 .rx_handler = c4iw_uld_rx_handler, 608 .state_change = c4iw_uld_state_change, 609}; 610 611static int __init c4iw_init_module(void) 612{ 613 int err; 614 615 err = c4iw_cm_init(); 616 if (err) 617 return err; 618 619 c4iw_debugfs_root = debugfs_create_dir(DRV_NAME, NULL); 620 if (!c4iw_debugfs_root) 621 printk(KERN_WARNING MOD 622 "could not create debugfs entry, continuing\n"); 623 624 cxgb4_register_uld(CXGB4_ULD_RDMA, &c4iw_uld_info); 625 626 return 0; 627} 628 629static void __exit c4iw_exit_module(void) 630{ 631 struct c4iw_dev *dev, *tmp; 632 633 mutex_lock(&dev_mutex); 634 list_for_each_entry_safe(dev, tmp, &dev_list, entry) { 635 c4iw_remove(dev); 636 } 637 mutex_unlock(&dev_mutex); 638 cxgb4_unregister_uld(CXGB4_ULD_RDMA); 639 c4iw_cm_term(); 640 debugfs_remove_recursive(c4iw_debugfs_root); 641} 642 643module_init(c4iw_init_module); 644module_exit(c4iw_exit_module); 645