1/* 2 * Copyright (c) 2004 Mellanox Technologies Ltd. All rights reserved. 3 * Copyright (c) 2004 Infinicon Corporation. All rights reserved. 4 * Copyright (c) 2004 Intel Corporation. All rights reserved. 5 * Copyright (c) 2004 Topspin Corporation. All rights reserved. 6 * Copyright (c) 2004 Voltaire Corporation. All rights reserved. 7 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. 8 * Copyright (c) 2005, 2006 Cisco Systems. All rights reserved. 9 * 10 * This software is available to you under a choice of one of two 11 * licenses. You may choose to be licensed under the terms of the GNU 12 * General Public License (GPL) Version 2, available from the file 13 * COPYING in the main directory of this source tree, or the 14 * OpenIB.org BSD license below: 15 * 16 * Redistribution and use in source and binary forms, with or 17 * without modification, are permitted provided that the following 18 * conditions are met: 19 * 20 * - Redistributions of source code must retain the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer. 23 * 24 * - Redistributions in binary form must reproduce the above 25 * copyright notice, this list of conditions and the following 26 * disclaimer in the documentation and/or other materials 27 * provided with the distribution. 28 * 29 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 30 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 31 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 32 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 33 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 34 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 35 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 36 * SOFTWARE. 37 */ 38 39#include <linux/errno.h> 40#include <linux/err.h> 41#include <linux/export.h> 42#include <linux/string.h> 43#include <linux/slab.h> 44 45#include <rdma/ib_verbs.h> 46#include <rdma/ib_cache.h> 47 48int ib_rate_to_mult(enum ib_rate rate) 49{ 50 switch (rate) { 51 case IB_RATE_2_5_GBPS: return 1; 52 case IB_RATE_5_GBPS: return 2; 53 case IB_RATE_10_GBPS: return 4; 54 case IB_RATE_20_GBPS: return 8; 55 case IB_RATE_30_GBPS: return 12; 56 case IB_RATE_40_GBPS: return 16; 57 case IB_RATE_60_GBPS: return 24; 58 case IB_RATE_80_GBPS: return 32; 59 case IB_RATE_120_GBPS: return 48; 60 default: return -1; 61 } 62} 63EXPORT_SYMBOL(ib_rate_to_mult); 64 65enum ib_rate mult_to_ib_rate(int mult) 66{ 67 switch (mult) { 68 case 1: return IB_RATE_2_5_GBPS; 69 case 2: return IB_RATE_5_GBPS; 70 case 4: return IB_RATE_10_GBPS; 71 case 8: return IB_RATE_20_GBPS; 72 case 12: return IB_RATE_30_GBPS; 73 case 16: return IB_RATE_40_GBPS; 74 case 24: return IB_RATE_60_GBPS; 75 case 32: return IB_RATE_80_GBPS; 76 case 48: return IB_RATE_120_GBPS; 77 default: return IB_RATE_PORT_CURRENT; 78 } 79} 80EXPORT_SYMBOL(mult_to_ib_rate); 81 82int ib_rate_to_mbps(enum ib_rate rate) 83{ 84 switch (rate) { 85 case IB_RATE_2_5_GBPS: return 2500; 86 case IB_RATE_5_GBPS: return 5000; 87 case IB_RATE_10_GBPS: return 10000; 88 case IB_RATE_20_GBPS: return 20000; 89 case IB_RATE_30_GBPS: return 30000; 90 case IB_RATE_40_GBPS: return 40000; 91 case IB_RATE_60_GBPS: return 60000; 92 case IB_RATE_80_GBPS: return 80000; 93 case IB_RATE_120_GBPS: return 120000; 94 case IB_RATE_14_GBPS: return 14062; 95 case IB_RATE_56_GBPS: return 56250; 96 case IB_RATE_112_GBPS: return 112500; 97 case IB_RATE_168_GBPS: return 168750; 98 case IB_RATE_25_GBPS: return 25781; 99 case IB_RATE_100_GBPS: return 103125; 100 case IB_RATE_200_GBPS: return 206250; 101 case IB_RATE_300_GBPS: return 309375; 102 default: return -1; 103 } 104} 105EXPORT_SYMBOL(ib_rate_to_mbps); 106 107enum rdma_transport_type 108rdma_node_get_transport(enum rdma_node_type node_type) 109{ 110 switch (node_type) { 111 case RDMA_NODE_IB_CA: 112 case RDMA_NODE_IB_SWITCH: 113 case RDMA_NODE_IB_ROUTER: 114 return RDMA_TRANSPORT_IB; 115 case RDMA_NODE_RNIC: 116 return RDMA_TRANSPORT_IWARP; 117 default: 118 BUG(); 119 return 0; 120 } 121} 122EXPORT_SYMBOL(rdma_node_get_transport); 123 124enum rdma_link_layer rdma_port_get_link_layer(struct ib_device *device, u8 port_num) 125{ 126 if (device->get_link_layer) 127 return device->get_link_layer(device, port_num); 128 129 switch (rdma_node_get_transport(device->node_type)) { 130 case RDMA_TRANSPORT_IB: 131 return IB_LINK_LAYER_INFINIBAND; 132 case RDMA_TRANSPORT_IWARP: 133 return IB_LINK_LAYER_ETHERNET; 134 default: 135 return IB_LINK_LAYER_UNSPECIFIED; 136 } 137} 138EXPORT_SYMBOL(rdma_port_get_link_layer); 139 140/* Protection domains */ 141 142struct ib_pd *ib_alloc_pd(struct ib_device *device) 143{ 144 struct ib_pd *pd; 145 146 pd = device->alloc_pd(device, NULL, NULL); 147 148 if (!IS_ERR(pd)) { 149 pd->device = device; 150 pd->uobject = NULL; 151 atomic_set(&pd->usecnt, 0); 152 } 153 154 return pd; 155} 156EXPORT_SYMBOL(ib_alloc_pd); 157 158int ib_dealloc_pd(struct ib_pd *pd) 159{ 160 if (atomic_read(&pd->usecnt)) 161 return -EBUSY; 162 163 return pd->device->dealloc_pd(pd); 164} 165EXPORT_SYMBOL(ib_dealloc_pd); 166 167/* Address handles */ 168 169struct ib_ah *ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr) 170{ 171 struct ib_ah *ah; 172 173 ah = pd->device->create_ah(pd, ah_attr); 174 175 if (!IS_ERR(ah)) { 176 ah->device = pd->device; 177 ah->pd = pd; 178 ah->uobject = NULL; 179 atomic_inc(&pd->usecnt); 180 } 181 182 return ah; 183} 184EXPORT_SYMBOL(ib_create_ah); 185 186int ib_init_ah_from_wc(struct ib_device *device, u8 port_num, struct ib_wc *wc, 187 struct ib_grh *grh, struct ib_ah_attr *ah_attr) 188{ 189 u32 flow_class; 190 u16 gid_index; 191 int ret; 192 193 memset(ah_attr, 0, sizeof *ah_attr); 194 ah_attr->dlid = wc->slid; 195 ah_attr->sl = wc->sl; 196 ah_attr->src_path_bits = wc->dlid_path_bits; 197 ah_attr->port_num = port_num; 198 199 if (wc->wc_flags & IB_WC_GRH) { 200 ah_attr->ah_flags = IB_AH_GRH; 201 ah_attr->grh.dgid = grh->sgid; 202 203 ret = ib_find_cached_gid(device, &grh->dgid, &port_num, 204 &gid_index); 205 if (ret) 206 return ret; 207 208 ah_attr->grh.sgid_index = (u8) gid_index; 209 flow_class = be32_to_cpu(grh->version_tclass_flow); 210 ah_attr->grh.flow_label = flow_class & 0xFFFFF; 211 ah_attr->grh.hop_limit = 0xFF; 212 ah_attr->grh.traffic_class = (flow_class >> 20) & 0xFF; 213 } 214 return 0; 215} 216EXPORT_SYMBOL(ib_init_ah_from_wc); 217 218struct ib_ah *ib_create_ah_from_wc(struct ib_pd *pd, struct ib_wc *wc, 219 struct ib_grh *grh, u8 port_num) 220{ 221 struct ib_ah_attr ah_attr; 222 int ret; 223 224 ret = ib_init_ah_from_wc(pd->device, port_num, wc, grh, &ah_attr); 225 if (ret) 226 return ERR_PTR(ret); 227 228 return ib_create_ah(pd, &ah_attr); 229} 230EXPORT_SYMBOL(ib_create_ah_from_wc); 231 232int ib_modify_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr) 233{ 234 return ah->device->modify_ah ? 235 ah->device->modify_ah(ah, ah_attr) : 236 -ENOSYS; 237} 238EXPORT_SYMBOL(ib_modify_ah); 239 240int ib_query_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr) 241{ 242 return ah->device->query_ah ? 243 ah->device->query_ah(ah, ah_attr) : 244 -ENOSYS; 245} 246EXPORT_SYMBOL(ib_query_ah); 247 248int ib_destroy_ah(struct ib_ah *ah) 249{ 250 struct ib_pd *pd; 251 int ret; 252 253 pd = ah->pd; 254 ret = ah->device->destroy_ah(ah); 255 if (!ret) 256 atomic_dec(&pd->usecnt); 257 258 return ret; 259} 260EXPORT_SYMBOL(ib_destroy_ah); 261 262/* Shared receive queues */ 263 264struct ib_srq *ib_create_srq(struct ib_pd *pd, 265 struct ib_srq_init_attr *srq_init_attr) 266{ 267 struct ib_srq *srq; 268 269 if (!pd->device->create_srq) 270 return ERR_PTR(-ENOSYS); 271 272 srq = pd->device->create_srq(pd, srq_init_attr, NULL); 273 274 if (!IS_ERR(srq)) { 275 srq->device = pd->device; 276 srq->pd = pd; 277 srq->uobject = NULL; 278 srq->event_handler = srq_init_attr->event_handler; 279 srq->srq_context = srq_init_attr->srq_context; 280 srq->srq_type = srq_init_attr->srq_type; 281 if (srq->srq_type == IB_SRQT_XRC) { 282 srq->ext.xrc.xrcd = srq_init_attr->ext.xrc.xrcd; 283 srq->ext.xrc.cq = srq_init_attr->ext.xrc.cq; 284 atomic_inc(&srq->ext.xrc.xrcd->usecnt); 285 atomic_inc(&srq->ext.xrc.cq->usecnt); 286 } 287 atomic_inc(&pd->usecnt); 288 atomic_set(&srq->usecnt, 0); 289 } 290 291 return srq; 292} 293EXPORT_SYMBOL(ib_create_srq); 294 295int ib_modify_srq(struct ib_srq *srq, 296 struct ib_srq_attr *srq_attr, 297 enum ib_srq_attr_mask srq_attr_mask) 298{ 299 return srq->device->modify_srq ? 300 srq->device->modify_srq(srq, srq_attr, srq_attr_mask, NULL) : 301 -ENOSYS; 302} 303EXPORT_SYMBOL(ib_modify_srq); 304 305int ib_query_srq(struct ib_srq *srq, 306 struct ib_srq_attr *srq_attr) 307{ 308 return srq->device->query_srq ? 309 srq->device->query_srq(srq, srq_attr) : -ENOSYS; 310} 311EXPORT_SYMBOL(ib_query_srq); 312 313int ib_destroy_srq(struct ib_srq *srq) 314{ 315 struct ib_pd *pd; 316 enum ib_srq_type srq_type; 317 struct ib_xrcd *uninitialized_var(xrcd); 318 struct ib_cq *uninitialized_var(cq); 319 int ret; 320 321 if (atomic_read(&srq->usecnt)) 322 return -EBUSY; 323 324 pd = srq->pd; 325 srq_type = srq->srq_type; 326 if (srq_type == IB_SRQT_XRC) { 327 xrcd = srq->ext.xrc.xrcd; 328 cq = srq->ext.xrc.cq; 329 } 330 331 ret = srq->device->destroy_srq(srq); 332 if (!ret) { 333 atomic_dec(&pd->usecnt); 334 if (srq_type == IB_SRQT_XRC) { 335 atomic_dec(&xrcd->usecnt); 336 atomic_dec(&cq->usecnt); 337 } 338 } 339 340 return ret; 341} 342EXPORT_SYMBOL(ib_destroy_srq); 343 344/* Queue pairs */ 345 346static void __ib_shared_qp_event_handler(struct ib_event *event, void *context) 347{ 348 struct ib_qp *qp = context; 349 350 list_for_each_entry(event->element.qp, &qp->open_list, open_list) 351 event->element.qp->event_handler(event, event->element.qp->qp_context); 352} 353 354static void __ib_insert_xrcd_qp(struct ib_xrcd *xrcd, struct ib_qp *qp) 355{ 356 mutex_lock(&xrcd->tgt_qp_mutex); 357 list_add(&qp->xrcd_list, &xrcd->tgt_qp_list); 358 mutex_unlock(&xrcd->tgt_qp_mutex); 359} 360 361static struct ib_qp *__ib_open_qp(struct ib_qp *real_qp, 362 void (*event_handler)(struct ib_event *, void *), 363 void *qp_context) 364{ 365 struct ib_qp *qp; 366 unsigned long flags; 367 368 qp = kzalloc(sizeof *qp, GFP_KERNEL); 369 if (!qp) 370 return ERR_PTR(-ENOMEM); 371 372 qp->real_qp = real_qp; 373 atomic_inc(&real_qp->usecnt); 374 qp->device = real_qp->device; 375 qp->event_handler = event_handler; 376 qp->qp_context = qp_context; 377 qp->qp_num = real_qp->qp_num; 378 qp->qp_type = real_qp->qp_type; 379 380 spin_lock_irqsave(&real_qp->device->event_handler_lock, flags); 381 list_add(&qp->open_list, &real_qp->open_list); 382 spin_unlock_irqrestore(&real_qp->device->event_handler_lock, flags); 383 384 return qp; 385} 386 387struct ib_qp *ib_open_qp(struct ib_xrcd *xrcd, 388 struct ib_qp_open_attr *qp_open_attr) 389{ 390 struct ib_qp *qp, *real_qp; 391 392 if (qp_open_attr->qp_type != IB_QPT_XRC_TGT) 393 return ERR_PTR(-EINVAL); 394 395 qp = ERR_PTR(-EINVAL); 396 mutex_lock(&xrcd->tgt_qp_mutex); 397 list_for_each_entry(real_qp, &xrcd->tgt_qp_list, xrcd_list) { 398 if (real_qp->qp_num == qp_open_attr->qp_num) { 399 qp = __ib_open_qp(real_qp, qp_open_attr->event_handler, 400 qp_open_attr->qp_context); 401 break; 402 } 403 } 404 mutex_unlock(&xrcd->tgt_qp_mutex); 405 return qp; 406} 407EXPORT_SYMBOL(ib_open_qp); 408 409struct ib_qp *ib_create_qp(struct ib_pd *pd, 410 struct ib_qp_init_attr *qp_init_attr) 411{ 412 struct ib_qp *qp, *real_qp; 413 struct ib_device *device; 414 415 device = pd ? pd->device : qp_init_attr->xrcd->device; 416 qp = device->create_qp(pd, qp_init_attr, NULL); 417 418 if (!IS_ERR(qp)) { 419 qp->device = device; 420 qp->real_qp = qp; 421 qp->uobject = NULL; 422 qp->qp_type = qp_init_attr->qp_type; 423 424 atomic_set(&qp->usecnt, 0); 425 if (qp_init_attr->qp_type == IB_QPT_XRC_TGT) { 426 qp->event_handler = __ib_shared_qp_event_handler; 427 qp->qp_context = qp; 428 qp->pd = NULL; 429 qp->send_cq = qp->recv_cq = NULL; 430 qp->srq = NULL; 431 qp->xrcd = qp_init_attr->xrcd; 432 atomic_inc(&qp_init_attr->xrcd->usecnt); 433 INIT_LIST_HEAD(&qp->open_list); 434 435 real_qp = qp; 436 qp = __ib_open_qp(real_qp, qp_init_attr->event_handler, 437 qp_init_attr->qp_context); 438 if (!IS_ERR(qp)) 439 __ib_insert_xrcd_qp(qp_init_attr->xrcd, real_qp); 440 else 441 real_qp->device->destroy_qp(real_qp); 442 } else { 443 qp->event_handler = qp_init_attr->event_handler; 444 qp->qp_context = qp_init_attr->qp_context; 445 if (qp_init_attr->qp_type == IB_QPT_XRC_INI) { 446 qp->recv_cq = NULL; 447 qp->srq = NULL; 448 } else { 449 qp->recv_cq = qp_init_attr->recv_cq; 450 atomic_inc(&qp_init_attr->recv_cq->usecnt); 451 qp->srq = qp_init_attr->srq; 452 if (qp->srq) 453 atomic_inc(&qp_init_attr->srq->usecnt); 454 } 455 456 qp->pd = pd; 457 qp->send_cq = qp_init_attr->send_cq; 458 qp->xrcd = NULL; 459 460 atomic_inc(&pd->usecnt); 461 atomic_inc(&qp_init_attr->send_cq->usecnt); 462 } 463 } 464 465 return qp; 466} 467EXPORT_SYMBOL(ib_create_qp); 468 469static const struct { 470 int valid; 471 enum ib_qp_attr_mask req_param[IB_QPT_MAX]; 472 enum ib_qp_attr_mask opt_param[IB_QPT_MAX]; 473} qp_state_table[IB_QPS_ERR + 1][IB_QPS_ERR + 1] = { 474 [IB_QPS_RESET] = { 475 [IB_QPS_RESET] = { .valid = 1 }, 476 [IB_QPS_INIT] = { 477 .valid = 1, 478 .req_param = { 479 [IB_QPT_UD] = (IB_QP_PKEY_INDEX | 480 IB_QP_PORT | 481 IB_QP_QKEY), 482 [IB_QPT_UC] = (IB_QP_PKEY_INDEX | 483 IB_QP_PORT | 484 IB_QP_ACCESS_FLAGS), 485 [IB_QPT_RC] = (IB_QP_PKEY_INDEX | 486 IB_QP_PORT | 487 IB_QP_ACCESS_FLAGS), 488 [IB_QPT_XRC_INI] = (IB_QP_PKEY_INDEX | 489 IB_QP_PORT | 490 IB_QP_ACCESS_FLAGS), 491 [IB_QPT_XRC_TGT] = (IB_QP_PKEY_INDEX | 492 IB_QP_PORT | 493 IB_QP_ACCESS_FLAGS), 494 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX | 495 IB_QP_QKEY), 496 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX | 497 IB_QP_QKEY), 498 } 499 }, 500 }, 501 [IB_QPS_INIT] = { 502 [IB_QPS_RESET] = { .valid = 1 }, 503 [IB_QPS_ERR] = { .valid = 1 }, 504 [IB_QPS_INIT] = { 505 .valid = 1, 506 .opt_param = { 507 [IB_QPT_UD] = (IB_QP_PKEY_INDEX | 508 IB_QP_PORT | 509 IB_QP_QKEY), 510 [IB_QPT_UC] = (IB_QP_PKEY_INDEX | 511 IB_QP_PORT | 512 IB_QP_ACCESS_FLAGS), 513 [IB_QPT_RC] = (IB_QP_PKEY_INDEX | 514 IB_QP_PORT | 515 IB_QP_ACCESS_FLAGS), 516 [IB_QPT_XRC_INI] = (IB_QP_PKEY_INDEX | 517 IB_QP_PORT | 518 IB_QP_ACCESS_FLAGS), 519 [IB_QPT_XRC_TGT] = (IB_QP_PKEY_INDEX | 520 IB_QP_PORT | 521 IB_QP_ACCESS_FLAGS), 522 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX | 523 IB_QP_QKEY), 524 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX | 525 IB_QP_QKEY), 526 } 527 }, 528 [IB_QPS_RTR] = { 529 .valid = 1, 530 .req_param = { 531 [IB_QPT_UC] = (IB_QP_AV | 532 IB_QP_PATH_MTU | 533 IB_QP_DEST_QPN | 534 IB_QP_RQ_PSN), 535 [IB_QPT_RC] = (IB_QP_AV | 536 IB_QP_PATH_MTU | 537 IB_QP_DEST_QPN | 538 IB_QP_RQ_PSN | 539 IB_QP_MAX_DEST_RD_ATOMIC | 540 IB_QP_MIN_RNR_TIMER), 541 [IB_QPT_XRC_INI] = (IB_QP_AV | 542 IB_QP_PATH_MTU | 543 IB_QP_DEST_QPN | 544 IB_QP_RQ_PSN), 545 [IB_QPT_XRC_TGT] = (IB_QP_AV | 546 IB_QP_PATH_MTU | 547 IB_QP_DEST_QPN | 548 IB_QP_RQ_PSN | 549 IB_QP_MAX_DEST_RD_ATOMIC | 550 IB_QP_MIN_RNR_TIMER), 551 }, 552 .opt_param = { 553 [IB_QPT_UD] = (IB_QP_PKEY_INDEX | 554 IB_QP_QKEY), 555 [IB_QPT_UC] = (IB_QP_ALT_PATH | 556 IB_QP_ACCESS_FLAGS | 557 IB_QP_PKEY_INDEX), 558 [IB_QPT_RC] = (IB_QP_ALT_PATH | 559 IB_QP_ACCESS_FLAGS | 560 IB_QP_PKEY_INDEX), 561 [IB_QPT_XRC_INI] = (IB_QP_ALT_PATH | 562 IB_QP_ACCESS_FLAGS | 563 IB_QP_PKEY_INDEX), 564 [IB_QPT_XRC_TGT] = (IB_QP_ALT_PATH | 565 IB_QP_ACCESS_FLAGS | 566 IB_QP_PKEY_INDEX), 567 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX | 568 IB_QP_QKEY), 569 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX | 570 IB_QP_QKEY), 571 } 572 } 573 }, 574 [IB_QPS_RTR] = { 575 [IB_QPS_RESET] = { .valid = 1 }, 576 [IB_QPS_ERR] = { .valid = 1 }, 577 [IB_QPS_RTS] = { 578 .valid = 1, 579 .req_param = { 580 [IB_QPT_UD] = IB_QP_SQ_PSN, 581 [IB_QPT_UC] = IB_QP_SQ_PSN, 582 [IB_QPT_RC] = (IB_QP_TIMEOUT | 583 IB_QP_RETRY_CNT | 584 IB_QP_RNR_RETRY | 585 IB_QP_SQ_PSN | 586 IB_QP_MAX_QP_RD_ATOMIC), 587 [IB_QPT_XRC_INI] = (IB_QP_TIMEOUT | 588 IB_QP_RETRY_CNT | 589 IB_QP_RNR_RETRY | 590 IB_QP_SQ_PSN | 591 IB_QP_MAX_QP_RD_ATOMIC), 592 [IB_QPT_XRC_TGT] = (IB_QP_TIMEOUT | 593 IB_QP_SQ_PSN), 594 [IB_QPT_SMI] = IB_QP_SQ_PSN, 595 [IB_QPT_GSI] = IB_QP_SQ_PSN, 596 }, 597 .opt_param = { 598 [IB_QPT_UD] = (IB_QP_CUR_STATE | 599 IB_QP_QKEY), 600 [IB_QPT_UC] = (IB_QP_CUR_STATE | 601 IB_QP_ALT_PATH | 602 IB_QP_ACCESS_FLAGS | 603 IB_QP_PATH_MIG_STATE), 604 [IB_QPT_RC] = (IB_QP_CUR_STATE | 605 IB_QP_ALT_PATH | 606 IB_QP_ACCESS_FLAGS | 607 IB_QP_MIN_RNR_TIMER | 608 IB_QP_PATH_MIG_STATE), 609 [IB_QPT_XRC_INI] = (IB_QP_CUR_STATE | 610 IB_QP_ALT_PATH | 611 IB_QP_ACCESS_FLAGS | 612 IB_QP_PATH_MIG_STATE), 613 [IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE | 614 IB_QP_ALT_PATH | 615 IB_QP_ACCESS_FLAGS | 616 IB_QP_MIN_RNR_TIMER | 617 IB_QP_PATH_MIG_STATE), 618 [IB_QPT_SMI] = (IB_QP_CUR_STATE | 619 IB_QP_QKEY), 620 [IB_QPT_GSI] = (IB_QP_CUR_STATE | 621 IB_QP_QKEY), 622 } 623 } 624 }, 625 [IB_QPS_RTS] = { 626 [IB_QPS_RESET] = { .valid = 1 }, 627 [IB_QPS_ERR] = { .valid = 1 }, 628 [IB_QPS_RTS] = { 629 .valid = 1, 630 .opt_param = { 631 [IB_QPT_UD] = (IB_QP_CUR_STATE | 632 IB_QP_QKEY), 633 [IB_QPT_UC] = (IB_QP_CUR_STATE | 634 IB_QP_ACCESS_FLAGS | 635 IB_QP_ALT_PATH | 636 IB_QP_PATH_MIG_STATE), 637 [IB_QPT_RC] = (IB_QP_CUR_STATE | 638 IB_QP_ACCESS_FLAGS | 639 IB_QP_ALT_PATH | 640 IB_QP_PATH_MIG_STATE | 641 IB_QP_MIN_RNR_TIMER), 642 [IB_QPT_XRC_INI] = (IB_QP_CUR_STATE | 643 IB_QP_ACCESS_FLAGS | 644 IB_QP_ALT_PATH | 645 IB_QP_PATH_MIG_STATE), 646 [IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE | 647 IB_QP_ACCESS_FLAGS | 648 IB_QP_ALT_PATH | 649 IB_QP_PATH_MIG_STATE | 650 IB_QP_MIN_RNR_TIMER), 651 [IB_QPT_SMI] = (IB_QP_CUR_STATE | 652 IB_QP_QKEY), 653 [IB_QPT_GSI] = (IB_QP_CUR_STATE | 654 IB_QP_QKEY), 655 } 656 }, 657 [IB_QPS_SQD] = { 658 .valid = 1, 659 .opt_param = { 660 [IB_QPT_UD] = IB_QP_EN_SQD_ASYNC_NOTIFY, 661 [IB_QPT_UC] = IB_QP_EN_SQD_ASYNC_NOTIFY, 662 [IB_QPT_RC] = IB_QP_EN_SQD_ASYNC_NOTIFY, 663 [IB_QPT_XRC_INI] = IB_QP_EN_SQD_ASYNC_NOTIFY, 664 [IB_QPT_XRC_TGT] = IB_QP_EN_SQD_ASYNC_NOTIFY, /* ??? */ 665 [IB_QPT_SMI] = IB_QP_EN_SQD_ASYNC_NOTIFY, 666 [IB_QPT_GSI] = IB_QP_EN_SQD_ASYNC_NOTIFY 667 } 668 }, 669 }, 670 [IB_QPS_SQD] = { 671 [IB_QPS_RESET] = { .valid = 1 }, 672 [IB_QPS_ERR] = { .valid = 1 }, 673 [IB_QPS_RTS] = { 674 .valid = 1, 675 .opt_param = { 676 [IB_QPT_UD] = (IB_QP_CUR_STATE | 677 IB_QP_QKEY), 678 [IB_QPT_UC] = (IB_QP_CUR_STATE | 679 IB_QP_ALT_PATH | 680 IB_QP_ACCESS_FLAGS | 681 IB_QP_PATH_MIG_STATE), 682 [IB_QPT_RC] = (IB_QP_CUR_STATE | 683 IB_QP_ALT_PATH | 684 IB_QP_ACCESS_FLAGS | 685 IB_QP_MIN_RNR_TIMER | 686 IB_QP_PATH_MIG_STATE), 687 [IB_QPT_XRC_INI] = (IB_QP_CUR_STATE | 688 IB_QP_ALT_PATH | 689 IB_QP_ACCESS_FLAGS | 690 IB_QP_PATH_MIG_STATE), 691 [IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE | 692 IB_QP_ALT_PATH | 693 IB_QP_ACCESS_FLAGS | 694 IB_QP_MIN_RNR_TIMER | 695 IB_QP_PATH_MIG_STATE), 696 [IB_QPT_SMI] = (IB_QP_CUR_STATE | 697 IB_QP_QKEY), 698 [IB_QPT_GSI] = (IB_QP_CUR_STATE | 699 IB_QP_QKEY), 700 } 701 }, 702 [IB_QPS_SQD] = { 703 .valid = 1, 704 .opt_param = { 705 [IB_QPT_UD] = (IB_QP_PKEY_INDEX | 706 IB_QP_QKEY), 707 [IB_QPT_UC] = (IB_QP_AV | 708 IB_QP_ALT_PATH | 709 IB_QP_ACCESS_FLAGS | 710 IB_QP_PKEY_INDEX | 711 IB_QP_PATH_MIG_STATE), 712 [IB_QPT_RC] = (IB_QP_PORT | 713 IB_QP_AV | 714 IB_QP_TIMEOUT | 715 IB_QP_RETRY_CNT | 716 IB_QP_RNR_RETRY | 717 IB_QP_MAX_QP_RD_ATOMIC | 718 IB_QP_MAX_DEST_RD_ATOMIC | 719 IB_QP_ALT_PATH | 720 IB_QP_ACCESS_FLAGS | 721 IB_QP_PKEY_INDEX | 722 IB_QP_MIN_RNR_TIMER | 723 IB_QP_PATH_MIG_STATE), 724 [IB_QPT_XRC_INI] = (IB_QP_PORT | 725 IB_QP_AV | 726 IB_QP_TIMEOUT | 727 IB_QP_RETRY_CNT | 728 IB_QP_RNR_RETRY | 729 IB_QP_MAX_QP_RD_ATOMIC | 730 IB_QP_ALT_PATH | 731 IB_QP_ACCESS_FLAGS | 732 IB_QP_PKEY_INDEX | 733 IB_QP_PATH_MIG_STATE), 734 [IB_QPT_XRC_TGT] = (IB_QP_PORT | 735 IB_QP_AV | 736 IB_QP_TIMEOUT | 737 IB_QP_MAX_DEST_RD_ATOMIC | 738 IB_QP_ALT_PATH | 739 IB_QP_ACCESS_FLAGS | 740 IB_QP_PKEY_INDEX | 741 IB_QP_MIN_RNR_TIMER | 742 IB_QP_PATH_MIG_STATE), 743 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX | 744 IB_QP_QKEY), 745 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX | 746 IB_QP_QKEY), 747 } 748 } 749 }, 750 [IB_QPS_SQE] = { 751 [IB_QPS_RESET] = { .valid = 1 }, 752 [IB_QPS_ERR] = { .valid = 1 }, 753 [IB_QPS_RTS] = { 754 .valid = 1, 755 .opt_param = { 756 [IB_QPT_UD] = (IB_QP_CUR_STATE | 757 IB_QP_QKEY), 758 [IB_QPT_UC] = (IB_QP_CUR_STATE | 759 IB_QP_ACCESS_FLAGS), 760 [IB_QPT_SMI] = (IB_QP_CUR_STATE | 761 IB_QP_QKEY), 762 [IB_QPT_GSI] = (IB_QP_CUR_STATE | 763 IB_QP_QKEY), 764 } 765 } 766 }, 767 [IB_QPS_ERR] = { 768 [IB_QPS_RESET] = { .valid = 1 }, 769 [IB_QPS_ERR] = { .valid = 1 } 770 } 771}; 772 773int ib_modify_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state next_state, 774 enum ib_qp_type type, enum ib_qp_attr_mask mask) 775{ 776 enum ib_qp_attr_mask req_param, opt_param; 777 778 if (cur_state < 0 || cur_state > IB_QPS_ERR || 779 next_state < 0 || next_state > IB_QPS_ERR) 780 return 0; 781 782 if (mask & IB_QP_CUR_STATE && 783 cur_state != IB_QPS_RTR && cur_state != IB_QPS_RTS && 784 cur_state != IB_QPS_SQD && cur_state != IB_QPS_SQE) 785 return 0; 786 787 if (!qp_state_table[cur_state][next_state].valid) 788 return 0; 789 790 req_param = qp_state_table[cur_state][next_state].req_param[type]; 791 opt_param = qp_state_table[cur_state][next_state].opt_param[type]; 792 793 if ((mask & req_param) != req_param) 794 return 0; 795 796 if (mask & ~(req_param | opt_param | IB_QP_STATE)) 797 return 0; 798 799 return 1; 800} 801EXPORT_SYMBOL(ib_modify_qp_is_ok); 802 803int ib_modify_qp(struct ib_qp *qp, 804 struct ib_qp_attr *qp_attr, 805 int qp_attr_mask) 806{ 807 return qp->device->modify_qp(qp->real_qp, qp_attr, qp_attr_mask, NULL); 808} 809EXPORT_SYMBOL(ib_modify_qp); 810 811int ib_query_qp(struct ib_qp *qp, 812 struct ib_qp_attr *qp_attr, 813 int qp_attr_mask, 814 struct ib_qp_init_attr *qp_init_attr) 815{ 816 return qp->device->query_qp ? 817 qp->device->query_qp(qp->real_qp, qp_attr, qp_attr_mask, qp_init_attr) : 818 -ENOSYS; 819} 820EXPORT_SYMBOL(ib_query_qp); 821 822int ib_close_qp(struct ib_qp *qp) 823{ 824 struct ib_qp *real_qp; 825 unsigned long flags; 826 827 real_qp = qp->real_qp; 828 if (real_qp == qp) 829 return -EINVAL; 830 831 spin_lock_irqsave(&real_qp->device->event_handler_lock, flags); 832 list_del(&qp->open_list); 833 spin_unlock_irqrestore(&real_qp->device->event_handler_lock, flags); 834 835 atomic_dec(&real_qp->usecnt); 836 kfree(qp); 837 838 return 0; 839} 840EXPORT_SYMBOL(ib_close_qp); 841 842static int __ib_destroy_shared_qp(struct ib_qp *qp) 843{ 844 struct ib_xrcd *xrcd; 845 struct ib_qp *real_qp; 846 int ret; 847 848 real_qp = qp->real_qp; 849 xrcd = real_qp->xrcd; 850 851 mutex_lock(&xrcd->tgt_qp_mutex); 852 ib_close_qp(qp); 853 if (atomic_read(&real_qp->usecnt) == 0) 854 list_del(&real_qp->xrcd_list); 855 else 856 real_qp = NULL; 857 mutex_unlock(&xrcd->tgt_qp_mutex); 858 859 if (real_qp) { 860 ret = ib_destroy_qp(real_qp); 861 if (!ret) 862 atomic_dec(&xrcd->usecnt); 863 else 864 __ib_insert_xrcd_qp(xrcd, real_qp); 865 } 866 867 return 0; 868} 869 870int ib_destroy_qp(struct ib_qp *qp) 871{ 872 struct ib_pd *pd; 873 struct ib_cq *scq, *rcq; 874 struct ib_srq *srq; 875 int ret; 876 877 if (atomic_read(&qp->usecnt)) 878 return -EBUSY; 879 880 if (qp->real_qp != qp) 881 return __ib_destroy_shared_qp(qp); 882 883 pd = qp->pd; 884 scq = qp->send_cq; 885 rcq = qp->recv_cq; 886 srq = qp->srq; 887 888 ret = qp->device->destroy_qp(qp); 889 if (!ret) { 890 if (pd) 891 atomic_dec(&pd->usecnt); 892 if (scq) 893 atomic_dec(&scq->usecnt); 894 if (rcq) 895 atomic_dec(&rcq->usecnt); 896 if (srq) 897 atomic_dec(&srq->usecnt); 898 } 899 900 return ret; 901} 902EXPORT_SYMBOL(ib_destroy_qp); 903 904/* Completion queues */ 905 906struct ib_cq *ib_create_cq(struct ib_device *device, 907 ib_comp_handler comp_handler, 908 void (*event_handler)(struct ib_event *, void *), 909 void *cq_context, int cqe, int comp_vector) 910{ 911 struct ib_cq *cq; 912 913 cq = device->create_cq(device, cqe, comp_vector, NULL, NULL); 914 915 if (!IS_ERR(cq)) { 916 cq->device = device; 917 cq->uobject = NULL; 918 cq->comp_handler = comp_handler; 919 cq->event_handler = event_handler; 920 cq->cq_context = cq_context; 921 atomic_set(&cq->usecnt, 0); 922 } 923 924 return cq; 925} 926EXPORT_SYMBOL(ib_create_cq); 927 928int ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period) 929{ 930 return cq->device->modify_cq ? 931 cq->device->modify_cq(cq, cq_count, cq_period) : -ENOSYS; 932} 933EXPORT_SYMBOL(ib_modify_cq); 934 935int ib_destroy_cq(struct ib_cq *cq) 936{ 937 if (atomic_read(&cq->usecnt)) 938 return -EBUSY; 939 940 return cq->device->destroy_cq(cq); 941} 942EXPORT_SYMBOL(ib_destroy_cq); 943 944int ib_resize_cq(struct ib_cq *cq, int cqe) 945{ 946 return cq->device->resize_cq ? 947 cq->device->resize_cq(cq, cqe, NULL) : -ENOSYS; 948} 949EXPORT_SYMBOL(ib_resize_cq); 950 951/* Memory regions */ 952 953struct ib_mr *ib_get_dma_mr(struct ib_pd *pd, int mr_access_flags) 954{ 955 struct ib_mr *mr; 956 957 mr = pd->device->get_dma_mr(pd, mr_access_flags); 958 959 if (!IS_ERR(mr)) { 960 mr->device = pd->device; 961 mr->pd = pd; 962 mr->uobject = NULL; 963 atomic_inc(&pd->usecnt); 964 atomic_set(&mr->usecnt, 0); 965 } 966 967 return mr; 968} 969EXPORT_SYMBOL(ib_get_dma_mr); 970 971struct ib_mr *ib_reg_phys_mr(struct ib_pd *pd, 972 struct ib_phys_buf *phys_buf_array, 973 int num_phys_buf, 974 int mr_access_flags, 975 u64 *iova_start) 976{ 977 struct ib_mr *mr; 978 979 if (!pd->device->reg_phys_mr) 980 return ERR_PTR(-ENOSYS); 981 982 mr = pd->device->reg_phys_mr(pd, phys_buf_array, num_phys_buf, 983 mr_access_flags, iova_start); 984 985 if (!IS_ERR(mr)) { 986 mr->device = pd->device; 987 mr->pd = pd; 988 mr->uobject = NULL; 989 atomic_inc(&pd->usecnt); 990 atomic_set(&mr->usecnt, 0); 991 } 992 993 return mr; 994} 995EXPORT_SYMBOL(ib_reg_phys_mr); 996 997int ib_rereg_phys_mr(struct ib_mr *mr, 998 int mr_rereg_mask, 999 struct ib_pd *pd, 1000 struct ib_phys_buf *phys_buf_array, 1001 int num_phys_buf, 1002 int mr_access_flags, 1003 u64 *iova_start) 1004{ 1005 struct ib_pd *old_pd; 1006 int ret; 1007 1008 if (!mr->device->rereg_phys_mr) 1009 return -ENOSYS; 1010 1011 if (atomic_read(&mr->usecnt)) 1012 return -EBUSY; 1013 1014 old_pd = mr->pd; 1015 1016 ret = mr->device->rereg_phys_mr(mr, mr_rereg_mask, pd, 1017 phys_buf_array, num_phys_buf, 1018 mr_access_flags, iova_start); 1019 1020 if (!ret && (mr_rereg_mask & IB_MR_REREG_PD)) { 1021 atomic_dec(&old_pd->usecnt); 1022 atomic_inc(&pd->usecnt); 1023 } 1024 1025 return ret; 1026} 1027EXPORT_SYMBOL(ib_rereg_phys_mr); 1028 1029int ib_query_mr(struct ib_mr *mr, struct ib_mr_attr *mr_attr) 1030{ 1031 return mr->device->query_mr ? 1032 mr->device->query_mr(mr, mr_attr) : -ENOSYS; 1033} 1034EXPORT_SYMBOL(ib_query_mr); 1035 1036int ib_dereg_mr(struct ib_mr *mr) 1037{ 1038 struct ib_pd *pd; 1039 int ret; 1040 1041 if (atomic_read(&mr->usecnt)) 1042 return -EBUSY; 1043 1044 pd = mr->pd; 1045 ret = mr->device->dereg_mr(mr); 1046 if (!ret) 1047 atomic_dec(&pd->usecnt); 1048 1049 return ret; 1050} 1051EXPORT_SYMBOL(ib_dereg_mr); 1052 1053struct ib_mr *ib_alloc_fast_reg_mr(struct ib_pd *pd, int max_page_list_len) 1054{ 1055 struct ib_mr *mr; 1056 1057 if (!pd->device->alloc_fast_reg_mr) 1058 return ERR_PTR(-ENOSYS); 1059 1060 mr = pd->device->alloc_fast_reg_mr(pd, max_page_list_len); 1061 1062 if (!IS_ERR(mr)) { 1063 mr->device = pd->device; 1064 mr->pd = pd; 1065 mr->uobject = NULL; 1066 atomic_inc(&pd->usecnt); 1067 atomic_set(&mr->usecnt, 0); 1068 } 1069 1070 return mr; 1071} 1072EXPORT_SYMBOL(ib_alloc_fast_reg_mr); 1073 1074struct ib_fast_reg_page_list *ib_alloc_fast_reg_page_list(struct ib_device *device, 1075 int max_page_list_len) 1076{ 1077 struct ib_fast_reg_page_list *page_list; 1078 1079 if (!device->alloc_fast_reg_page_list) 1080 return ERR_PTR(-ENOSYS); 1081 1082 page_list = device->alloc_fast_reg_page_list(device, max_page_list_len); 1083 1084 if (!IS_ERR(page_list)) { 1085 page_list->device = device; 1086 page_list->max_page_list_len = max_page_list_len; 1087 } 1088 1089 return page_list; 1090} 1091EXPORT_SYMBOL(ib_alloc_fast_reg_page_list); 1092 1093void ib_free_fast_reg_page_list(struct ib_fast_reg_page_list *page_list) 1094{ 1095 page_list->device->free_fast_reg_page_list(page_list); 1096} 1097EXPORT_SYMBOL(ib_free_fast_reg_page_list); 1098 1099/* Memory windows */ 1100 1101struct ib_mw *ib_alloc_mw(struct ib_pd *pd) 1102{ 1103 struct ib_mw *mw; 1104 1105 if (!pd->device->alloc_mw) 1106 return ERR_PTR(-ENOSYS); 1107 1108 mw = pd->device->alloc_mw(pd); 1109 if (!IS_ERR(mw)) { 1110 mw->device = pd->device; 1111 mw->pd = pd; 1112 mw->uobject = NULL; 1113 atomic_inc(&pd->usecnt); 1114 } 1115 1116 return mw; 1117} 1118EXPORT_SYMBOL(ib_alloc_mw); 1119 1120int ib_dealloc_mw(struct ib_mw *mw) 1121{ 1122 struct ib_pd *pd; 1123 int ret; 1124 1125 pd = mw->pd; 1126 ret = mw->device->dealloc_mw(mw); 1127 if (!ret) 1128 atomic_dec(&pd->usecnt); 1129 1130 return ret; 1131} 1132EXPORT_SYMBOL(ib_dealloc_mw); 1133 1134/* "Fast" memory regions */ 1135 1136struct ib_fmr *ib_alloc_fmr(struct ib_pd *pd, 1137 int mr_access_flags, 1138 struct ib_fmr_attr *fmr_attr) 1139{ 1140 struct ib_fmr *fmr; 1141 1142 if (!pd->device->alloc_fmr) 1143 return ERR_PTR(-ENOSYS); 1144 1145 fmr = pd->device->alloc_fmr(pd, mr_access_flags, fmr_attr); 1146 if (!IS_ERR(fmr)) { 1147 fmr->device = pd->device; 1148 fmr->pd = pd; 1149 atomic_inc(&pd->usecnt); 1150 } 1151 1152 return fmr; 1153} 1154EXPORT_SYMBOL(ib_alloc_fmr); 1155 1156int ib_unmap_fmr(struct list_head *fmr_list) 1157{ 1158 struct ib_fmr *fmr; 1159 1160 if (list_empty(fmr_list)) 1161 return 0; 1162 1163 fmr = list_entry(fmr_list->next, struct ib_fmr, list); 1164 return fmr->device->unmap_fmr(fmr_list); 1165} 1166EXPORT_SYMBOL(ib_unmap_fmr); 1167 1168int ib_dealloc_fmr(struct ib_fmr *fmr) 1169{ 1170 struct ib_pd *pd; 1171 int ret; 1172 1173 pd = fmr->pd; 1174 ret = fmr->device->dealloc_fmr(fmr); 1175 if (!ret) 1176 atomic_dec(&pd->usecnt); 1177 1178 return ret; 1179} 1180EXPORT_SYMBOL(ib_dealloc_fmr); 1181 1182/* Multicast groups */ 1183 1184int ib_attach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid) 1185{ 1186 if (!qp->device->attach_mcast) 1187 return -ENOSYS; 1188 if (gid->raw[0] != 0xff || qp->qp_type != IB_QPT_UD) 1189 return -EINVAL; 1190 1191 return qp->device->attach_mcast(qp, gid, lid); 1192} 1193EXPORT_SYMBOL(ib_attach_mcast); 1194 1195int ib_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid) 1196{ 1197 if (!qp->device->detach_mcast) 1198 return -ENOSYS; 1199 if (gid->raw[0] != 0xff || qp->qp_type != IB_QPT_UD) 1200 return -EINVAL; 1201 1202 return qp->device->detach_mcast(qp, gid, lid); 1203} 1204EXPORT_SYMBOL(ib_detach_mcast); 1205 1206struct ib_xrcd *ib_alloc_xrcd(struct ib_device *device) 1207{ 1208 struct ib_xrcd *xrcd; 1209 1210 if (!device->alloc_xrcd) 1211 return ERR_PTR(-ENOSYS); 1212 1213 xrcd = device->alloc_xrcd(device, NULL, NULL); 1214 if (!IS_ERR(xrcd)) { 1215 xrcd->device = device; 1216 xrcd->inode = NULL; 1217 atomic_set(&xrcd->usecnt, 0); 1218 mutex_init(&xrcd->tgt_qp_mutex); 1219 INIT_LIST_HEAD(&xrcd->tgt_qp_list); 1220 } 1221 1222 return xrcd; 1223} 1224EXPORT_SYMBOL(ib_alloc_xrcd); 1225 1226int ib_dealloc_xrcd(struct ib_xrcd *xrcd) 1227{ 1228 struct ib_qp *qp; 1229 int ret; 1230 1231 if (atomic_read(&xrcd->usecnt)) 1232 return -EBUSY; 1233 1234 while (!list_empty(&xrcd->tgt_qp_list)) { 1235 qp = list_entry(xrcd->tgt_qp_list.next, struct ib_qp, xrcd_list); 1236 ret = ib_destroy_qp(qp); 1237 if (ret) 1238 return ret; 1239 } 1240 1241 return xrcd->device->dealloc_xrcd(xrcd); 1242} 1243EXPORT_SYMBOL(ib_dealloc_xrcd); 1244