nes_verbs.c revision 24797a344293601f14f49e2d259c3ca447c4f802
1/* 2 * Copyright (c) 2006 - 2008 NetEffect, Inc. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 * 32 */ 33 34#include <linux/module.h> 35#include <linux/moduleparam.h> 36#include <linux/random.h> 37#include <linux/highmem.h> 38#include <asm/byteorder.h> 39 40#include <rdma/ib_verbs.h> 41#include <rdma/iw_cm.h> 42#include <rdma/ib_user_verbs.h> 43 44#include "nes.h" 45 46#include <rdma/ib_umem.h> 47 48atomic_t mod_qp_timouts; 49atomic_t qps_created; 50atomic_t sw_qps_destroyed; 51 52static void nes_unregister_ofa_device(struct nes_ib_device *nesibdev); 53 54/** 55 * nes_alloc_mw 56 */ 57static struct ib_mw *nes_alloc_mw(struct ib_pd *ibpd) { 58 unsigned long flags; 59 struct nes_pd *nespd = to_nespd(ibpd); 60 struct nes_vnic *nesvnic = to_nesvnic(ibpd->device); 61 struct nes_device *nesdev = nesvnic->nesdev; 62 struct nes_adapter *nesadapter = nesdev->nesadapter; 63 struct nes_cqp_request *cqp_request; 64 struct nes_mr *nesmr; 65 struct ib_mw *ibmw; 66 struct nes_hw_cqp_wqe *cqp_wqe; 67 int ret; 68 u32 stag; 69 u32 stag_index = 0; 70 u32 next_stag_index = 0; 71 u32 driver_key = 0; 72 u8 stag_key = 0; 73 74 get_random_bytes(&next_stag_index, sizeof(next_stag_index)); 75 stag_key = (u8)next_stag_index; 76 77 driver_key = 0; 78 79 next_stag_index >>= 8; 80 next_stag_index %= nesadapter->max_mr; 81 82 ret = nes_alloc_resource(nesadapter, nesadapter->allocated_mrs, 83 nesadapter->max_mr, &stag_index, &next_stag_index); 84 if (ret) { 85 return ERR_PTR(ret); 86 } 87 88 nesmr = kzalloc(sizeof(*nesmr), GFP_KERNEL); 89 if (!nesmr) { 90 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); 91 return ERR_PTR(-ENOMEM); 92 } 93 94 stag = stag_index << 8; 95 stag |= driver_key; 96 stag += (u32)stag_key; 97 98 nes_debug(NES_DBG_MR, "Registering STag 0x%08X, index = 0x%08X\n", 99 stag, stag_index); 100 101 /* Register the region with the adapter */ 102 cqp_request = nes_get_cqp_request(nesdev); 103 if (cqp_request == NULL) { 104 kfree(nesmr); 105 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); 106 return ERR_PTR(-ENOMEM); 107 } 108 109 cqp_request->waiting = 1; 110 cqp_wqe = &cqp_request->cqp_wqe; 111 112 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = 113 cpu_to_le32( NES_CQP_ALLOCATE_STAG | NES_CQP_STAG_RIGHTS_REMOTE_READ | 114 NES_CQP_STAG_RIGHTS_REMOTE_WRITE | NES_CQP_STAG_VA_TO | 115 NES_CQP_STAG_REM_ACC_EN); 116 117 nes_fill_init_cqp_wqe(cqp_wqe, nesdev); 118 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX, (nespd->pd_id & 0x00007fff)); 119 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_STAG_IDX, stag); 120 121 atomic_set(&cqp_request->refcount, 2); 122 nes_post_cqp_request(nesdev, cqp_request, NES_CQP_REQUEST_RING_DOORBELL); 123 124 /* Wait for CQP */ 125 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0), 126 NES_EVENT_TIMEOUT); 127 nes_debug(NES_DBG_MR, "Register STag 0x%08X completed, wait_event_timeout ret = %u," 128 " CQP Major:Minor codes = 0x%04X:0x%04X.\n", 129 stag, ret, cqp_request->major_code, cqp_request->minor_code); 130 if ((!ret) || (cqp_request->major_code)) { 131 if (atomic_dec_and_test(&cqp_request->refcount)) { 132 if (cqp_request->dynamic) { 133 kfree(cqp_request); 134 } else { 135 spin_lock_irqsave(&nesdev->cqp.lock, flags); 136 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs); 137 spin_unlock_irqrestore(&nesdev->cqp.lock, flags); 138 } 139 } 140 kfree(nesmr); 141 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); 142 if (!ret) { 143 return ERR_PTR(-ETIME); 144 } else { 145 return ERR_PTR(-ENOMEM); 146 } 147 } else { 148 if (atomic_dec_and_test(&cqp_request->refcount)) { 149 if (cqp_request->dynamic) { 150 kfree(cqp_request); 151 } else { 152 spin_lock_irqsave(&nesdev->cqp.lock, flags); 153 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs); 154 spin_unlock_irqrestore(&nesdev->cqp.lock, flags); 155 } 156 } 157 } 158 159 nesmr->ibmw.rkey = stag; 160 nesmr->mode = IWNES_MEMREG_TYPE_MW; 161 ibmw = &nesmr->ibmw; 162 nesmr->pbl_4k = 0; 163 nesmr->pbls_used = 0; 164 165 return ibmw; 166} 167 168 169/** 170 * nes_dealloc_mw 171 */ 172static int nes_dealloc_mw(struct ib_mw *ibmw) 173{ 174 struct nes_mr *nesmr = to_nesmw(ibmw); 175 struct nes_vnic *nesvnic = to_nesvnic(ibmw->device); 176 struct nes_device *nesdev = nesvnic->nesdev; 177 struct nes_adapter *nesadapter = nesdev->nesadapter; 178 struct nes_hw_cqp_wqe *cqp_wqe; 179 struct nes_cqp_request *cqp_request; 180 int err = 0; 181 unsigned long flags; 182 int ret; 183 184 /* Deallocate the window with the adapter */ 185 cqp_request = nes_get_cqp_request(nesdev); 186 if (cqp_request == NULL) { 187 nes_debug(NES_DBG_MR, "Failed to get a cqp_request.\n"); 188 return -ENOMEM; 189 } 190 cqp_request->waiting = 1; 191 cqp_wqe = &cqp_request->cqp_wqe; 192 nes_fill_init_cqp_wqe(cqp_wqe, nesdev); 193 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, NES_CQP_DEALLOCATE_STAG); 194 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_STAG_IDX, ibmw->rkey); 195 196 atomic_set(&cqp_request->refcount, 2); 197 nes_post_cqp_request(nesdev, cqp_request, NES_CQP_REQUEST_RING_DOORBELL); 198 199 /* Wait for CQP */ 200 nes_debug(NES_DBG_MR, "Waiting for deallocate STag 0x%08X to complete.\n", 201 ibmw->rkey); 202 ret = wait_event_timeout(cqp_request->waitq, (0 != cqp_request->request_done), 203 NES_EVENT_TIMEOUT); 204 nes_debug(NES_DBG_MR, "Deallocate STag completed, wait_event_timeout ret = %u," 205 " CQP Major:Minor codes = 0x%04X:0x%04X.\n", 206 ret, cqp_request->major_code, cqp_request->minor_code); 207 if ((!ret) || (cqp_request->major_code)) { 208 if (atomic_dec_and_test(&cqp_request->refcount)) { 209 if (cqp_request->dynamic) { 210 kfree(cqp_request); 211 } else { 212 spin_lock_irqsave(&nesdev->cqp.lock, flags); 213 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs); 214 spin_unlock_irqrestore(&nesdev->cqp.lock, flags); 215 } 216 } 217 if (!ret) { 218 err = -ETIME; 219 } else { 220 err = -EIO; 221 } 222 } else { 223 if (atomic_dec_and_test(&cqp_request->refcount)) { 224 if (cqp_request->dynamic) { 225 kfree(cqp_request); 226 } else { 227 spin_lock_irqsave(&nesdev->cqp.lock, flags); 228 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs); 229 spin_unlock_irqrestore(&nesdev->cqp.lock, flags); 230 } 231 } 232 } 233 234 nes_free_resource(nesadapter, nesadapter->allocated_mrs, 235 (ibmw->rkey & 0x0fffff00) >> 8); 236 kfree(nesmr); 237 238 return err; 239} 240 241 242/** 243 * nes_bind_mw 244 */ 245static int nes_bind_mw(struct ib_qp *ibqp, struct ib_mw *ibmw, 246 struct ib_mw_bind *ibmw_bind) 247{ 248 u64 u64temp; 249 struct nes_vnic *nesvnic = to_nesvnic(ibqp->device); 250 struct nes_device *nesdev = nesvnic->nesdev; 251 /* struct nes_mr *nesmr = to_nesmw(ibmw); */ 252 struct nes_qp *nesqp = to_nesqp(ibqp); 253 struct nes_hw_qp_wqe *wqe; 254 unsigned long flags = 0; 255 u32 head; 256 u32 wqe_misc = 0; 257 u32 qsize; 258 259 if (nesqp->ibqp_state > IB_QPS_RTS) 260 return -EINVAL; 261 262 spin_lock_irqsave(&nesqp->lock, flags); 263 264 head = nesqp->hwqp.sq_head; 265 qsize = nesqp->hwqp.sq_tail; 266 267 /* Check for SQ overflow */ 268 if (((head + (2 * qsize) - nesqp->hwqp.sq_tail) % qsize) == (qsize - 1)) { 269 spin_unlock_irqrestore(&nesqp->lock, flags); 270 return -EINVAL; 271 } 272 273 wqe = &nesqp->hwqp.sq_vbase[head]; 274 /* nes_debug(NES_DBG_MR, "processing sq wqe at %p, head = %u.\n", wqe, head); */ 275 nes_fill_init_qp_wqe(wqe, nesqp, head); 276 u64temp = ibmw_bind->wr_id; 277 set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX, u64temp); 278 wqe_misc = NES_IWARP_SQ_OP_BIND; 279 280 wqe_misc |= NES_IWARP_SQ_WQE_LOCAL_FENCE; 281 282 if (ibmw_bind->send_flags & IB_SEND_SIGNALED) 283 wqe_misc |= NES_IWARP_SQ_WQE_SIGNALED_COMPL; 284 285 if (ibmw_bind->mw_access_flags & IB_ACCESS_REMOTE_WRITE) { 286 wqe_misc |= NES_CQP_STAG_RIGHTS_REMOTE_WRITE; 287 } 288 if (ibmw_bind->mw_access_flags & IB_ACCESS_REMOTE_READ) { 289 wqe_misc |= NES_CQP_STAG_RIGHTS_REMOTE_READ; 290 } 291 292 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_MISC_IDX, wqe_misc); 293 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_BIND_WQE_MR_IDX, ibmw_bind->mr->lkey); 294 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_BIND_WQE_MW_IDX, ibmw->rkey); 295 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_BIND_WQE_LENGTH_LOW_IDX, 296 ibmw_bind->length); 297 wqe->wqe_words[NES_IWARP_SQ_BIND_WQE_LENGTH_HIGH_IDX] = 0; 298 u64temp = (u64)ibmw_bind->addr; 299 set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_BIND_WQE_VA_FBO_LOW_IDX, u64temp); 300 301 head++; 302 if (head >= qsize) 303 head = 0; 304 305 nesqp->hwqp.sq_head = head; 306 barrier(); 307 308 nes_write32(nesdev->regs+NES_WQE_ALLOC, 309 (1 << 24) | 0x00800000 | nesqp->hwqp.qp_id); 310 311 spin_unlock_irqrestore(&nesqp->lock, flags); 312 313 return 0; 314} 315 316 317/** 318 * nes_alloc_fmr 319 */ 320static struct ib_fmr *nes_alloc_fmr(struct ib_pd *ibpd, 321 int ibmr_access_flags, 322 struct ib_fmr_attr *ibfmr_attr) 323{ 324 unsigned long flags; 325 struct nes_pd *nespd = to_nespd(ibpd); 326 struct nes_vnic *nesvnic = to_nesvnic(ibpd->device); 327 struct nes_device *nesdev = nesvnic->nesdev; 328 struct nes_adapter *nesadapter = nesdev->nesadapter; 329 struct nes_fmr *nesfmr; 330 struct nes_cqp_request *cqp_request; 331 struct nes_hw_cqp_wqe *cqp_wqe; 332 int ret; 333 u32 stag; 334 u32 stag_index = 0; 335 u32 next_stag_index = 0; 336 u32 driver_key = 0; 337 u32 opcode = 0; 338 u8 stag_key = 0; 339 int i=0; 340 struct nes_vpbl vpbl; 341 342 get_random_bytes(&next_stag_index, sizeof(next_stag_index)); 343 stag_key = (u8)next_stag_index; 344 345 driver_key = 0; 346 347 next_stag_index >>= 8; 348 next_stag_index %= nesadapter->max_mr; 349 350 ret = nes_alloc_resource(nesadapter, nesadapter->allocated_mrs, 351 nesadapter->max_mr, &stag_index, &next_stag_index); 352 if (ret) { 353 goto failed_resource_alloc; 354 } 355 356 nesfmr = kzalloc(sizeof(*nesfmr), GFP_KERNEL); 357 if (!nesfmr) { 358 ret = -ENOMEM; 359 goto failed_fmr_alloc; 360 } 361 362 nesfmr->nesmr.mode = IWNES_MEMREG_TYPE_FMR; 363 if (ibfmr_attr->max_pages == 1) { 364 /* use zero length PBL */ 365 nesfmr->nesmr.pbl_4k = 0; 366 nesfmr->nesmr.pbls_used = 0; 367 } else if (ibfmr_attr->max_pages <= 32) { 368 /* use PBL 256 */ 369 nesfmr->nesmr.pbl_4k = 0; 370 nesfmr->nesmr.pbls_used = 1; 371 } else if (ibfmr_attr->max_pages <= 512) { 372 /* use 4K PBLs */ 373 nesfmr->nesmr.pbl_4k = 1; 374 nesfmr->nesmr.pbls_used = 1; 375 } else { 376 /* use two level 4K PBLs */ 377 /* add support for two level 256B PBLs */ 378 nesfmr->nesmr.pbl_4k = 1; 379 nesfmr->nesmr.pbls_used = 1 + (ibfmr_attr->max_pages >> 9) + 380 ((ibfmr_attr->max_pages & 511) ? 1 : 0); 381 } 382 /* Register the region with the adapter */ 383 spin_lock_irqsave(&nesadapter->pbl_lock, flags); 384 385 /* track PBL resources */ 386 if (nesfmr->nesmr.pbls_used != 0) { 387 if (nesfmr->nesmr.pbl_4k) { 388 if (nesfmr->nesmr.pbls_used > nesadapter->free_4kpbl) { 389 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 390 ret = -ENOMEM; 391 goto failed_vpbl_alloc; 392 } else { 393 nesadapter->free_4kpbl -= nesfmr->nesmr.pbls_used; 394 } 395 } else { 396 if (nesfmr->nesmr.pbls_used > nesadapter->free_256pbl) { 397 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 398 ret = -ENOMEM; 399 goto failed_vpbl_alloc; 400 } else { 401 nesadapter->free_256pbl -= nesfmr->nesmr.pbls_used; 402 } 403 } 404 } 405 406 /* one level pbl */ 407 if (nesfmr->nesmr.pbls_used == 0) { 408 nesfmr->root_vpbl.pbl_vbase = NULL; 409 nes_debug(NES_DBG_MR, "zero level pbl \n"); 410 } else if (nesfmr->nesmr.pbls_used == 1) { 411 /* can change it to kmalloc & dma_map_single */ 412 nesfmr->root_vpbl.pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 4096, 413 &nesfmr->root_vpbl.pbl_pbase); 414 if (!nesfmr->root_vpbl.pbl_vbase) { 415 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 416 ret = -ENOMEM; 417 goto failed_vpbl_alloc; 418 } 419 nesfmr->leaf_pbl_cnt = 0; 420 nes_debug(NES_DBG_MR, "one level pbl, root_vpbl.pbl_vbase=%p \n", 421 nesfmr->root_vpbl.pbl_vbase); 422 } 423 /* two level pbl */ 424 else { 425 nesfmr->root_vpbl.pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 8192, 426 &nesfmr->root_vpbl.pbl_pbase); 427 if (!nesfmr->root_vpbl.pbl_vbase) { 428 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 429 ret = -ENOMEM; 430 goto failed_vpbl_alloc; 431 } 432 433 nesfmr->root_vpbl.leaf_vpbl = kzalloc(sizeof(*nesfmr->root_vpbl.leaf_vpbl)*1024, GFP_KERNEL); 434 if (!nesfmr->root_vpbl.leaf_vpbl) { 435 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 436 ret = -ENOMEM; 437 goto failed_leaf_vpbl_alloc; 438 } 439 440 nesfmr->leaf_pbl_cnt = nesfmr->nesmr.pbls_used-1; 441 nes_debug(NES_DBG_MR, "two level pbl, root_vpbl.pbl_vbase=%p" 442 " leaf_pbl_cnt=%d root_vpbl.leaf_vpbl=%p\n", 443 nesfmr->root_vpbl.pbl_vbase, nesfmr->leaf_pbl_cnt, nesfmr->root_vpbl.leaf_vpbl); 444 445 for (i=0; i<nesfmr->leaf_pbl_cnt; i++) 446 nesfmr->root_vpbl.leaf_vpbl[i].pbl_vbase = NULL; 447 448 for (i=0; i<nesfmr->leaf_pbl_cnt; i++) { 449 vpbl.pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 4096, 450 &vpbl.pbl_pbase); 451 452 if (!vpbl.pbl_vbase) { 453 ret = -ENOMEM; 454 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 455 goto failed_leaf_vpbl_pages_alloc; 456 } 457 458 nesfmr->root_vpbl.pbl_vbase[i].pa_low = cpu_to_le32((u32)vpbl.pbl_pbase); 459 nesfmr->root_vpbl.pbl_vbase[i].pa_high = cpu_to_le32((u32)((((u64)vpbl.pbl_pbase)>>32))); 460 nesfmr->root_vpbl.leaf_vpbl[i] = vpbl; 461 462 nes_debug(NES_DBG_MR, "pbase_low=0x%x, pbase_high=0x%x, vpbl=%p\n", 463 nesfmr->root_vpbl.pbl_vbase[i].pa_low, 464 nesfmr->root_vpbl.pbl_vbase[i].pa_high, 465 &nesfmr->root_vpbl.leaf_vpbl[i]); 466 } 467 } 468 nesfmr->ib_qp = NULL; 469 nesfmr->access_rights =0; 470 471 stag = stag_index << 8; 472 stag |= driver_key; 473 stag += (u32)stag_key; 474 475 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 476 cqp_request = nes_get_cqp_request(nesdev); 477 if (cqp_request == NULL) { 478 nes_debug(NES_DBG_MR, "Failed to get a cqp_request.\n"); 479 ret = -ENOMEM; 480 goto failed_leaf_vpbl_pages_alloc; 481 } 482 cqp_request->waiting = 1; 483 cqp_wqe = &cqp_request->cqp_wqe; 484 485 nes_debug(NES_DBG_MR, "Registering STag 0x%08X, index = 0x%08X\n", 486 stag, stag_index); 487 488 opcode = NES_CQP_ALLOCATE_STAG | NES_CQP_STAG_VA_TO | NES_CQP_STAG_MR; 489 490 if (nesfmr->nesmr.pbl_4k == 1) 491 opcode |= NES_CQP_STAG_PBL_BLK_SIZE; 492 493 if (ibmr_access_flags & IB_ACCESS_REMOTE_WRITE) { 494 opcode |= NES_CQP_STAG_RIGHTS_REMOTE_WRITE | 495 NES_CQP_STAG_RIGHTS_LOCAL_WRITE | NES_CQP_STAG_REM_ACC_EN; 496 nesfmr->access_rights |= 497 NES_CQP_STAG_RIGHTS_REMOTE_WRITE | NES_CQP_STAG_RIGHTS_LOCAL_WRITE | 498 NES_CQP_STAG_REM_ACC_EN; 499 } 500 501 if (ibmr_access_flags & IB_ACCESS_REMOTE_READ) { 502 opcode |= NES_CQP_STAG_RIGHTS_REMOTE_READ | 503 NES_CQP_STAG_RIGHTS_LOCAL_READ | NES_CQP_STAG_REM_ACC_EN; 504 nesfmr->access_rights |= 505 NES_CQP_STAG_RIGHTS_REMOTE_READ | NES_CQP_STAG_RIGHTS_LOCAL_READ | 506 NES_CQP_STAG_REM_ACC_EN; 507 } 508 509 nes_fill_init_cqp_wqe(cqp_wqe, nesdev); 510 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, opcode); 511 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX, (nespd->pd_id & 0x00007fff)); 512 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_STAG_IDX, stag); 513 514 cqp_wqe->wqe_words[NES_CQP_STAG_WQE_PBL_BLK_COUNT_IDX] = 515 cpu_to_le32((nesfmr->nesmr.pbls_used>1) ? 516 (nesfmr->nesmr.pbls_used-1) : nesfmr->nesmr.pbls_used); 517 518 atomic_set(&cqp_request->refcount, 2); 519 nes_post_cqp_request(nesdev, cqp_request, NES_CQP_REQUEST_RING_DOORBELL); 520 521 /* Wait for CQP */ 522 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0), 523 NES_EVENT_TIMEOUT); 524 nes_debug(NES_DBG_MR, "Register STag 0x%08X completed, wait_event_timeout ret = %u," 525 " CQP Major:Minor codes = 0x%04X:0x%04X.\n", 526 stag, ret, cqp_request->major_code, cqp_request->minor_code); 527 528 if ((!ret) || (cqp_request->major_code)) { 529 if (atomic_dec_and_test(&cqp_request->refcount)) { 530 if (cqp_request->dynamic) { 531 kfree(cqp_request); 532 } else { 533 spin_lock_irqsave(&nesdev->cqp.lock, flags); 534 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs); 535 spin_unlock_irqrestore(&nesdev->cqp.lock, flags); 536 } 537 } 538 ret = (!ret) ? -ETIME : -EIO; 539 goto failed_leaf_vpbl_pages_alloc; 540 } else { 541 if (atomic_dec_and_test(&cqp_request->refcount)) { 542 if (cqp_request->dynamic) { 543 kfree(cqp_request); 544 } else { 545 spin_lock_irqsave(&nesdev->cqp.lock, flags); 546 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs); 547 spin_unlock_irqrestore(&nesdev->cqp.lock, flags); 548 } 549 } 550 } 551 552 nesfmr->nesmr.ibfmr.lkey = stag; 553 nesfmr->nesmr.ibfmr.rkey = stag; 554 nesfmr->attr = *ibfmr_attr; 555 556 return &nesfmr->nesmr.ibfmr; 557 558 failed_leaf_vpbl_pages_alloc: 559 /* unroll all allocated pages */ 560 for (i=0; i<nesfmr->leaf_pbl_cnt; i++) { 561 if (nesfmr->root_vpbl.leaf_vpbl[i].pbl_vbase) { 562 pci_free_consistent(nesdev->pcidev, 4096, nesfmr->root_vpbl.leaf_vpbl[i].pbl_vbase, 563 nesfmr->root_vpbl.leaf_vpbl[i].pbl_pbase); 564 } 565 } 566 if (nesfmr->root_vpbl.leaf_vpbl) 567 kfree(nesfmr->root_vpbl.leaf_vpbl); 568 569 failed_leaf_vpbl_alloc: 570 if (nesfmr->leaf_pbl_cnt == 0) { 571 if (nesfmr->root_vpbl.pbl_vbase) 572 pci_free_consistent(nesdev->pcidev, 4096, nesfmr->root_vpbl.pbl_vbase, 573 nesfmr->root_vpbl.pbl_pbase); 574 } else 575 pci_free_consistent(nesdev->pcidev, 8192, nesfmr->root_vpbl.pbl_vbase, 576 nesfmr->root_vpbl.pbl_pbase); 577 578 failed_vpbl_alloc: 579 kfree(nesfmr); 580 581 failed_fmr_alloc: 582 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); 583 584 failed_resource_alloc: 585 return ERR_PTR(ret); 586} 587 588 589/** 590 * nes_dealloc_fmr 591 */ 592static int nes_dealloc_fmr(struct ib_fmr *ibfmr) 593{ 594 struct nes_mr *nesmr = to_nesmr_from_ibfmr(ibfmr); 595 struct nes_fmr *nesfmr = to_nesfmr(nesmr); 596 struct nes_vnic *nesvnic = to_nesvnic(ibfmr->device); 597 struct nes_device *nesdev = nesvnic->nesdev; 598 struct nes_mr temp_nesmr = *nesmr; 599 int i = 0; 600 601 temp_nesmr.ibmw.device = ibfmr->device; 602 temp_nesmr.ibmw.pd = ibfmr->pd; 603 temp_nesmr.ibmw.rkey = ibfmr->rkey; 604 temp_nesmr.ibmw.uobject = NULL; 605 606 /* free the resources */ 607 if (nesfmr->leaf_pbl_cnt == 0) { 608 /* single PBL case */ 609 if (nesfmr->root_vpbl.pbl_vbase) 610 pci_free_consistent(nesdev->pcidev, 4096, nesfmr->root_vpbl.pbl_vbase, 611 nesfmr->root_vpbl.pbl_pbase); 612 } else { 613 for (i = 0; i < nesfmr->leaf_pbl_cnt; i++) { 614 pci_free_consistent(nesdev->pcidev, 4096, nesfmr->root_vpbl.leaf_vpbl[i].pbl_vbase, 615 nesfmr->root_vpbl.leaf_vpbl[i].pbl_pbase); 616 } 617 kfree(nesfmr->root_vpbl.leaf_vpbl); 618 pci_free_consistent(nesdev->pcidev, 8192, nesfmr->root_vpbl.pbl_vbase, 619 nesfmr->root_vpbl.pbl_pbase); 620 } 621 622 return nes_dealloc_mw(&temp_nesmr.ibmw); 623} 624 625 626/** 627 * nes_map_phys_fmr 628 */ 629static int nes_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list, 630 int list_len, u64 iova) 631{ 632 return 0; 633} 634 635 636/** 637 * nes_unmap_frm 638 */ 639static int nes_unmap_fmr(struct list_head *ibfmr_list) 640{ 641 return 0; 642} 643 644 645 646/** 647 * nes_query_device 648 */ 649static int nes_query_device(struct ib_device *ibdev, struct ib_device_attr *props) 650{ 651 struct nes_vnic *nesvnic = to_nesvnic(ibdev); 652 struct nes_device *nesdev = nesvnic->nesdev; 653 struct nes_ib_device *nesibdev = nesvnic->nesibdev; 654 655 memset(props, 0, sizeof(*props)); 656 memcpy(&props->sys_image_guid, nesvnic->netdev->dev_addr, 6); 657 658 props->fw_ver = nesdev->nesadapter->fw_ver; 659 props->device_cap_flags = nesdev->nesadapter->device_cap_flags; 660 props->vendor_id = nesdev->nesadapter->vendor_id; 661 props->vendor_part_id = nesdev->nesadapter->vendor_part_id; 662 props->hw_ver = nesdev->nesadapter->hw_rev; 663 props->max_mr_size = 0x80000000; 664 props->max_qp = nesibdev->max_qp; 665 props->max_qp_wr = nesdev->nesadapter->max_qp_wr - 2; 666 props->max_sge = nesdev->nesadapter->max_sge; 667 props->max_cq = nesibdev->max_cq; 668 props->max_cqe = nesdev->nesadapter->max_cqe - 1; 669 props->max_mr = nesibdev->max_mr; 670 props->max_mw = nesibdev->max_mr; 671 props->max_pd = nesibdev->max_pd; 672 props->max_sge_rd = 1; 673 switch (nesdev->nesadapter->max_irrq_wr) { 674 case 0: 675 props->max_qp_rd_atom = 1; 676 break; 677 case 1: 678 props->max_qp_rd_atom = 4; 679 break; 680 case 2: 681 props->max_qp_rd_atom = 16; 682 break; 683 case 3: 684 props->max_qp_rd_atom = 32; 685 break; 686 default: 687 props->max_qp_rd_atom = 0; 688 } 689 props->max_qp_init_rd_atom = props->max_qp_wr; 690 props->atomic_cap = IB_ATOMIC_NONE; 691 props->max_map_per_fmr = 1; 692 693 return 0; 694} 695 696 697/** 698 * nes_query_port 699 */ 700static int nes_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *props) 701{ 702 memset(props, 0, sizeof(*props)); 703 704 props->max_mtu = IB_MTU_2048; 705 props->active_mtu = IB_MTU_2048; 706 props->lid = 1; 707 props->lmc = 0; 708 props->sm_lid = 0; 709 props->sm_sl = 0; 710 props->state = IB_PORT_ACTIVE; 711 props->phys_state = 0; 712 props->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP | 713 IB_PORT_VENDOR_CLASS_SUP | IB_PORT_BOOT_MGMT_SUP; 714 props->gid_tbl_len = 1; 715 props->pkey_tbl_len = 1; 716 props->qkey_viol_cntr = 0; 717 props->active_width = IB_WIDTH_4X; 718 props->active_speed = 1; 719 props->max_msg_sz = 0x80000000; 720 721 return 0; 722} 723 724 725/** 726 * nes_modify_port 727 */ 728static int nes_modify_port(struct ib_device *ibdev, u8 port, 729 int port_modify_mask, struct ib_port_modify *props) 730{ 731 return 0; 732} 733 734 735/** 736 * nes_query_pkey 737 */ 738static int nes_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey) 739{ 740 *pkey = 0; 741 return 0; 742} 743 744 745/** 746 * nes_query_gid 747 */ 748static int nes_query_gid(struct ib_device *ibdev, u8 port, 749 int index, union ib_gid *gid) 750{ 751 struct nes_vnic *nesvnic = to_nesvnic(ibdev); 752 753 memset(&(gid->raw[0]), 0, sizeof(gid->raw)); 754 memcpy(&(gid->raw[0]), nesvnic->netdev->dev_addr, 6); 755 756 return 0; 757} 758 759 760/** 761 * nes_alloc_ucontext - Allocate the user context data structure. This keeps track 762 * of all objects associated with a particular user-mode client. 763 */ 764static struct ib_ucontext *nes_alloc_ucontext(struct ib_device *ibdev, 765 struct ib_udata *udata) 766{ 767 struct nes_vnic *nesvnic = to_nesvnic(ibdev); 768 struct nes_device *nesdev = nesvnic->nesdev; 769 struct nes_adapter *nesadapter = nesdev->nesadapter; 770 struct nes_alloc_ucontext_req req; 771 struct nes_alloc_ucontext_resp uresp; 772 struct nes_ucontext *nes_ucontext; 773 struct nes_ib_device *nesibdev = nesvnic->nesibdev; 774 775 776 if (ib_copy_from_udata(&req, udata, sizeof(struct nes_alloc_ucontext_req))) { 777 printk(KERN_ERR PFX "Invalid structure size on allocate user context.\n"); 778 return ERR_PTR(-EINVAL); 779 } 780 781 if (req.userspace_ver != NES_ABI_USERSPACE_VER) { 782 printk(KERN_ERR PFX "Invalid userspace driver version detected. Detected version %d, should be %d\n", 783 req.userspace_ver, NES_ABI_USERSPACE_VER); 784 return ERR_PTR(-EINVAL); 785 } 786 787 788 memset(&uresp, 0, sizeof uresp); 789 790 uresp.max_qps = nesibdev->max_qp; 791 uresp.max_pds = nesibdev->max_pd; 792 uresp.wq_size = nesdev->nesadapter->max_qp_wr * 2; 793 uresp.virtwq = nesadapter->virtwq; 794 uresp.kernel_ver = NES_ABI_KERNEL_VER; 795 796 nes_ucontext = kzalloc(sizeof *nes_ucontext, GFP_KERNEL); 797 if (!nes_ucontext) 798 return ERR_PTR(-ENOMEM); 799 800 nes_ucontext->nesdev = nesdev; 801 nes_ucontext->mmap_wq_offset = uresp.max_pds; 802 nes_ucontext->mmap_cq_offset = nes_ucontext->mmap_wq_offset + 803 ((sizeof(struct nes_hw_qp_wqe) * uresp.max_qps * 2) + PAGE_SIZE-1) / 804 PAGE_SIZE; 805 806 807 if (ib_copy_to_udata(udata, &uresp, sizeof uresp)) { 808 kfree(nes_ucontext); 809 return ERR_PTR(-EFAULT); 810 } 811 812 INIT_LIST_HEAD(&nes_ucontext->cq_reg_mem_list); 813 INIT_LIST_HEAD(&nes_ucontext->qp_reg_mem_list); 814 atomic_set(&nes_ucontext->usecnt, 1); 815 return &nes_ucontext->ibucontext; 816} 817 818 819/** 820 * nes_dealloc_ucontext 821 */ 822static int nes_dealloc_ucontext(struct ib_ucontext *context) 823{ 824 /* struct nes_vnic *nesvnic = to_nesvnic(context->device); */ 825 /* struct nes_device *nesdev = nesvnic->nesdev; */ 826 struct nes_ucontext *nes_ucontext = to_nesucontext(context); 827 828 if (!atomic_dec_and_test(&nes_ucontext->usecnt)) 829 return 0; 830 kfree(nes_ucontext); 831 return 0; 832} 833 834 835/** 836 * nes_mmap 837 */ 838static int nes_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) 839{ 840 unsigned long index; 841 struct nes_vnic *nesvnic = to_nesvnic(context->device); 842 struct nes_device *nesdev = nesvnic->nesdev; 843 /* struct nes_adapter *nesadapter = nesdev->nesadapter; */ 844 struct nes_ucontext *nes_ucontext; 845 struct nes_qp *nesqp; 846 847 nes_ucontext = to_nesucontext(context); 848 849 850 if (vma->vm_pgoff >= nes_ucontext->mmap_wq_offset) { 851 index = (vma->vm_pgoff - nes_ucontext->mmap_wq_offset) * PAGE_SIZE; 852 index /= ((sizeof(struct nes_hw_qp_wqe) * nesdev->nesadapter->max_qp_wr * 2) + 853 PAGE_SIZE-1) & (~(PAGE_SIZE-1)); 854 if (!test_bit(index, nes_ucontext->allocated_wqs)) { 855 nes_debug(NES_DBG_MMAP, "wq %lu not allocated\n", index); 856 return -EFAULT; 857 } 858 nesqp = nes_ucontext->mmap_nesqp[index]; 859 if (nesqp == NULL) { 860 nes_debug(NES_DBG_MMAP, "wq %lu has a NULL QP base.\n", index); 861 return -EFAULT; 862 } 863 if (remap_pfn_range(vma, vma->vm_start, 864 virt_to_phys(nesqp->hwqp.sq_vbase) >> PAGE_SHIFT, 865 vma->vm_end - vma->vm_start, 866 vma->vm_page_prot)) { 867 nes_debug(NES_DBG_MMAP, "remap_pfn_range failed.\n"); 868 return -EAGAIN; 869 } 870 vma->vm_private_data = nesqp; 871 return 0; 872 } else { 873 index = vma->vm_pgoff; 874 if (!test_bit(index, nes_ucontext->allocated_doorbells)) 875 return -EFAULT; 876 877 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 878 if (io_remap_pfn_range(vma, vma->vm_start, 879 (nesdev->doorbell_start + 880 ((nes_ucontext->mmap_db_index[index] - nesdev->base_doorbell_index) * 4096)) 881 >> PAGE_SHIFT, PAGE_SIZE, vma->vm_page_prot)) 882 return -EAGAIN; 883 vma->vm_private_data = nes_ucontext; 884 return 0; 885 } 886 887 return -ENOSYS; 888} 889 890 891/** 892 * nes_alloc_pd 893 */ 894static struct ib_pd *nes_alloc_pd(struct ib_device *ibdev, 895 struct ib_ucontext *context, struct ib_udata *udata) 896{ 897 struct nes_pd *nespd; 898 struct nes_vnic *nesvnic = to_nesvnic(ibdev); 899 struct nes_device *nesdev = nesvnic->nesdev; 900 struct nes_adapter *nesadapter = nesdev->nesadapter; 901 struct nes_ucontext *nesucontext; 902 struct nes_alloc_pd_resp uresp; 903 u32 pd_num = 0; 904 int err; 905 906 nes_debug(NES_DBG_PD, "nesvnic=%p, netdev=%p %s, ibdev=%p, context=%p, netdev refcnt=%u\n", 907 nesvnic, nesdev->netdev[0], nesdev->netdev[0]->name, ibdev, context, 908 atomic_read(&nesvnic->netdev->refcnt)); 909 910 err = nes_alloc_resource(nesadapter, nesadapter->allocated_pds, 911 nesadapter->max_pd, &pd_num, &nesadapter->next_pd); 912 if (err) { 913 return ERR_PTR(err); 914 } 915 916 nespd = kzalloc(sizeof (struct nes_pd), GFP_KERNEL); 917 if (!nespd) { 918 nes_free_resource(nesadapter, nesadapter->allocated_pds, pd_num); 919 return ERR_PTR(-ENOMEM); 920 } 921 922 nes_debug(NES_DBG_PD, "Allocating PD (%p) for ib device %s\n", 923 nespd, nesvnic->nesibdev->ibdev.name); 924 925 nespd->pd_id = (pd_num << (PAGE_SHIFT-12)) + nesadapter->base_pd; 926 927 if (context) { 928 nesucontext = to_nesucontext(context); 929 nespd->mmap_db_index = find_next_zero_bit(nesucontext->allocated_doorbells, 930 NES_MAX_USER_DB_REGIONS, nesucontext->first_free_db); 931 nes_debug(NES_DBG_PD, "find_first_zero_biton doorbells returned %u, mapping pd_id %u.\n", 932 nespd->mmap_db_index, nespd->pd_id); 933 if (nespd->mmap_db_index >= NES_MAX_USER_DB_REGIONS) { 934 nes_debug(NES_DBG_PD, "mmap_db_index > MAX\n"); 935 nes_free_resource(nesadapter, nesadapter->allocated_pds, pd_num); 936 kfree(nespd); 937 return ERR_PTR(-ENOMEM); 938 } 939 940 uresp.pd_id = nespd->pd_id; 941 uresp.mmap_db_index = nespd->mmap_db_index; 942 if (ib_copy_to_udata(udata, &uresp, sizeof (struct nes_alloc_pd_resp))) { 943 nes_free_resource(nesadapter, nesadapter->allocated_pds, pd_num); 944 kfree(nespd); 945 return ERR_PTR(-EFAULT); 946 } 947 948 set_bit(nespd->mmap_db_index, nesucontext->allocated_doorbells); 949 nesucontext->mmap_db_index[nespd->mmap_db_index] = nespd->pd_id; 950 nesucontext->first_free_db = nespd->mmap_db_index + 1; 951 } 952 953 nes_debug(NES_DBG_PD, "PD%u structure located @%p.\n", nespd->pd_id, nespd); 954 return &nespd->ibpd; 955} 956 957 958/** 959 * nes_dealloc_pd 960 */ 961static int nes_dealloc_pd(struct ib_pd *ibpd) 962{ 963 struct nes_ucontext *nesucontext; 964 struct nes_pd *nespd = to_nespd(ibpd); 965 struct nes_vnic *nesvnic = to_nesvnic(ibpd->device); 966 struct nes_device *nesdev = nesvnic->nesdev; 967 struct nes_adapter *nesadapter = nesdev->nesadapter; 968 969 if ((ibpd->uobject) && (ibpd->uobject->context)) { 970 nesucontext = to_nesucontext(ibpd->uobject->context); 971 nes_debug(NES_DBG_PD, "Clearing bit %u from allocated doorbells\n", 972 nespd->mmap_db_index); 973 clear_bit(nespd->mmap_db_index, nesucontext->allocated_doorbells); 974 nesucontext->mmap_db_index[nespd->mmap_db_index] = 0; 975 if (nesucontext->first_free_db > nespd->mmap_db_index) { 976 nesucontext->first_free_db = nespd->mmap_db_index; 977 } 978 } 979 980 nes_debug(NES_DBG_PD, "Deallocating PD%u structure located @%p.\n", 981 nespd->pd_id, nespd); 982 nes_free_resource(nesadapter, nesadapter->allocated_pds, 983 (nespd->pd_id-nesadapter->base_pd)>>(PAGE_SHIFT-12)); 984 kfree(nespd); 985 986 return 0; 987} 988 989 990/** 991 * nes_create_ah 992 */ 993static struct ib_ah *nes_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr) 994{ 995 return ERR_PTR(-ENOSYS); 996} 997 998 999/** 1000 * nes_destroy_ah 1001 */ 1002static int nes_destroy_ah(struct ib_ah *ah) 1003{ 1004 return -ENOSYS; 1005} 1006 1007 1008/** 1009 * nes_get_encoded_size 1010 */ 1011static inline u8 nes_get_encoded_size(int *size) 1012{ 1013 u8 encoded_size = 0; 1014 if (*size <= 32) { 1015 *size = 32; 1016 encoded_size = 1; 1017 } else if (*size <= 128) { 1018 *size = 128; 1019 encoded_size = 2; 1020 } else if (*size <= 512) { 1021 *size = 512; 1022 encoded_size = 3; 1023 } 1024 return (encoded_size); 1025} 1026 1027 1028 1029/** 1030 * nes_setup_virt_qp 1031 */ 1032static int nes_setup_virt_qp(struct nes_qp *nesqp, struct nes_pbl *nespbl, 1033 struct nes_vnic *nesvnic, int sq_size, int rq_size) 1034{ 1035 unsigned long flags; 1036 void *mem; 1037 __le64 *pbl = NULL; 1038 __le64 *tpbl; 1039 __le64 *pblbuffer; 1040 struct nes_device *nesdev = nesvnic->nesdev; 1041 struct nes_adapter *nesadapter = nesdev->nesadapter; 1042 u32 pbl_entries; 1043 u8 rq_pbl_entries; 1044 u8 sq_pbl_entries; 1045 1046 pbl_entries = nespbl->pbl_size >> 3; 1047 nes_debug(NES_DBG_QP, "Userspace PBL, pbl_size=%u, pbl_entries = %d pbl_vbase=%p, pbl_pbase=%lx\n", 1048 nespbl->pbl_size, pbl_entries, 1049 (void *)nespbl->pbl_vbase, 1050 (unsigned long) nespbl->pbl_pbase); 1051 pbl = (__le64 *) nespbl->pbl_vbase; /* points to first pbl entry */ 1052 /* now lets set the sq_vbase as well as rq_vbase addrs we will assign */ 1053 /* the first pbl to be fro the rq_vbase... */ 1054 rq_pbl_entries = (rq_size * sizeof(struct nes_hw_qp_wqe)) >> 12; 1055 sq_pbl_entries = (sq_size * sizeof(struct nes_hw_qp_wqe)) >> 12; 1056 nesqp->hwqp.sq_pbase = (le32_to_cpu(((__le32 *)pbl)[0])) | ((u64)((le32_to_cpu(((__le32 *)pbl)[1]))) << 32); 1057 if (!nespbl->page) { 1058 nes_debug(NES_DBG_QP, "QP nespbl->page is NULL \n"); 1059 kfree(nespbl); 1060 return -ENOMEM; 1061 } 1062 1063 nesqp->hwqp.sq_vbase = kmap(nespbl->page); 1064 nesqp->page = nespbl->page; 1065 if (!nesqp->hwqp.sq_vbase) { 1066 nes_debug(NES_DBG_QP, "QP sq_vbase kmap failed\n"); 1067 kfree(nespbl); 1068 return -ENOMEM; 1069 } 1070 1071 /* Now to get to sq.. we need to calculate how many */ 1072 /* PBL entries were used by the rq.. */ 1073 pbl += sq_pbl_entries; 1074 nesqp->hwqp.rq_pbase = (le32_to_cpu(((__le32 *)pbl)[0])) | ((u64)((le32_to_cpu(((__le32 *)pbl)[1]))) << 32); 1075 /* nesqp->hwqp.rq_vbase = bus_to_virt(*pbl); */ 1076 /*nesqp->hwqp.rq_vbase = phys_to_virt(*pbl); */ 1077 1078 nes_debug(NES_DBG_QP, "QP sq_vbase= %p sq_pbase=%lx rq_vbase=%p rq_pbase=%lx\n", 1079 nesqp->hwqp.sq_vbase, (unsigned long) nesqp->hwqp.sq_pbase, 1080 nesqp->hwqp.rq_vbase, (unsigned long) nesqp->hwqp.rq_pbase); 1081 spin_lock_irqsave(&nesadapter->pbl_lock, flags); 1082 if (!nesadapter->free_256pbl) { 1083 pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, nespbl->pbl_vbase, 1084 nespbl->pbl_pbase); 1085 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 1086 kunmap(nesqp->page); 1087 kfree(nespbl); 1088 return -ENOMEM; 1089 } 1090 nesadapter->free_256pbl--; 1091 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 1092 1093 nesqp->pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 256, &nesqp->pbl_pbase); 1094 pblbuffer = nesqp->pbl_vbase; 1095 if (!nesqp->pbl_vbase) { 1096 /* memory allocated during nes_reg_user_mr() */ 1097 pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, nespbl->pbl_vbase, 1098 nespbl->pbl_pbase); 1099 kfree(nespbl); 1100 spin_lock_irqsave(&nesadapter->pbl_lock, flags); 1101 nesadapter->free_256pbl++; 1102 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 1103 kunmap(nesqp->page); 1104 return -ENOMEM; 1105 } 1106 memset(nesqp->pbl_vbase, 0, 256); 1107 /* fill in the page address in the pbl buffer.. */ 1108 tpbl = pblbuffer + 16; 1109 pbl = (__le64 *)nespbl->pbl_vbase; 1110 while (sq_pbl_entries--) 1111 *tpbl++ = *pbl++; 1112 tpbl = pblbuffer; 1113 while (rq_pbl_entries--) 1114 *tpbl++ = *pbl++; 1115 1116 /* done with memory allocated during nes_reg_user_mr() */ 1117 pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, nespbl->pbl_vbase, 1118 nespbl->pbl_pbase); 1119 kfree(nespbl); 1120 1121 nesqp->qp_mem_size = 1122 max((u32)sizeof(struct nes_qp_context), ((u32)256)) + 256; /* this is Q2 */ 1123 /* Round up to a multiple of a page */ 1124 nesqp->qp_mem_size += PAGE_SIZE - 1; 1125 nesqp->qp_mem_size &= ~(PAGE_SIZE - 1); 1126 1127 mem = pci_alloc_consistent(nesdev->pcidev, nesqp->qp_mem_size, 1128 &nesqp->hwqp.q2_pbase); 1129 1130 if (!mem) { 1131 pci_free_consistent(nesdev->pcidev, 256, nesqp->pbl_vbase, nesqp->pbl_pbase); 1132 nesqp->pbl_vbase = NULL; 1133 spin_lock_irqsave(&nesadapter->pbl_lock, flags); 1134 nesadapter->free_256pbl++; 1135 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 1136 kunmap(nesqp->page); 1137 return -ENOMEM; 1138 } 1139 nesqp->hwqp.q2_vbase = mem; 1140 mem += 256; 1141 memset(nesqp->hwqp.q2_vbase, 0, 256); 1142 nesqp->nesqp_context = mem; 1143 memset(nesqp->nesqp_context, 0, sizeof(*nesqp->nesqp_context)); 1144 nesqp->nesqp_context_pbase = nesqp->hwqp.q2_pbase + 256; 1145 1146 return 0; 1147} 1148 1149 1150/** 1151 * nes_setup_mmap_qp 1152 */ 1153static int nes_setup_mmap_qp(struct nes_qp *nesqp, struct nes_vnic *nesvnic, 1154 int sq_size, int rq_size) 1155{ 1156 void *mem; 1157 struct nes_device *nesdev = nesvnic->nesdev; 1158 1159 nesqp->qp_mem_size = (sizeof(struct nes_hw_qp_wqe) * sq_size) + 1160 (sizeof(struct nes_hw_qp_wqe) * rq_size) + 1161 max((u32)sizeof(struct nes_qp_context), ((u32)256)) + 1162 256; /* this is Q2 */ 1163 /* Round up to a multiple of a page */ 1164 nesqp->qp_mem_size += PAGE_SIZE - 1; 1165 nesqp->qp_mem_size &= ~(PAGE_SIZE - 1); 1166 1167 mem = pci_alloc_consistent(nesdev->pcidev, nesqp->qp_mem_size, 1168 &nesqp->hwqp.sq_pbase); 1169 if (!mem) 1170 return -ENOMEM; 1171 nes_debug(NES_DBG_QP, "PCI consistent memory for " 1172 "host descriptor rings located @ %p (pa = 0x%08lX.) size = %u.\n", 1173 mem, (unsigned long)nesqp->hwqp.sq_pbase, nesqp->qp_mem_size); 1174 1175 memset(mem, 0, nesqp->qp_mem_size); 1176 1177 nesqp->hwqp.sq_vbase = mem; 1178 mem += sizeof(struct nes_hw_qp_wqe) * sq_size; 1179 1180 nesqp->hwqp.rq_vbase = mem; 1181 nesqp->hwqp.rq_pbase = nesqp->hwqp.sq_pbase + 1182 sizeof(struct nes_hw_qp_wqe) * sq_size; 1183 mem += sizeof(struct nes_hw_qp_wqe) * rq_size; 1184 1185 nesqp->hwqp.q2_vbase = mem; 1186 nesqp->hwqp.q2_pbase = nesqp->hwqp.rq_pbase + 1187 sizeof(struct nes_hw_qp_wqe) * rq_size; 1188 mem += 256; 1189 memset(nesqp->hwqp.q2_vbase, 0, 256); 1190 1191 nesqp->nesqp_context = mem; 1192 nesqp->nesqp_context_pbase = nesqp->hwqp.q2_pbase + 256; 1193 memset(nesqp->nesqp_context, 0, sizeof(*nesqp->nesqp_context)); 1194 return 0; 1195} 1196 1197 1198/** 1199 * nes_free_qp_mem() is to free up the qp's pci_alloc_consistent() memory. 1200 */ 1201static inline void nes_free_qp_mem(struct nes_device *nesdev, 1202 struct nes_qp *nesqp, int virt_wqs) 1203{ 1204 unsigned long flags; 1205 struct nes_adapter *nesadapter = nesdev->nesadapter; 1206 if (!virt_wqs) { 1207 pci_free_consistent(nesdev->pcidev, nesqp->qp_mem_size, 1208 nesqp->hwqp.sq_vbase, nesqp->hwqp.sq_pbase); 1209 }else { 1210 spin_lock_irqsave(&nesadapter->pbl_lock, flags); 1211 nesadapter->free_256pbl++; 1212 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 1213 pci_free_consistent(nesdev->pcidev, nesqp->qp_mem_size, nesqp->hwqp.q2_vbase, nesqp->hwqp.q2_pbase); 1214 pci_free_consistent(nesdev->pcidev, 256, nesqp->pbl_vbase, nesqp->pbl_pbase ); 1215 nesqp->pbl_vbase = NULL; 1216 kunmap(nesqp->page); 1217 } 1218} 1219 1220 1221/** 1222 * nes_create_qp 1223 */ 1224static struct ib_qp *nes_create_qp(struct ib_pd *ibpd, 1225 struct ib_qp_init_attr *init_attr, struct ib_udata *udata) 1226{ 1227 u64 u64temp= 0; 1228 u64 u64nesqp = 0; 1229 struct nes_pd *nespd = to_nespd(ibpd); 1230 struct nes_vnic *nesvnic = to_nesvnic(ibpd->device); 1231 struct nes_device *nesdev = nesvnic->nesdev; 1232 struct nes_adapter *nesadapter = nesdev->nesadapter; 1233 struct nes_qp *nesqp; 1234 struct nes_cq *nescq; 1235 struct nes_ucontext *nes_ucontext; 1236 struct nes_hw_cqp_wqe *cqp_wqe; 1237 struct nes_cqp_request *cqp_request; 1238 struct nes_create_qp_req req; 1239 struct nes_create_qp_resp uresp; 1240 struct nes_pbl *nespbl = NULL; 1241 u32 qp_num = 0; 1242 u32 opcode = 0; 1243 /* u32 counter = 0; */ 1244 void *mem; 1245 unsigned long flags; 1246 int ret; 1247 int err; 1248 int virt_wqs = 0; 1249 int sq_size; 1250 int rq_size; 1251 u8 sq_encoded_size; 1252 u8 rq_encoded_size; 1253 /* int counter; */ 1254 1255 if (init_attr->create_flags) 1256 return ERR_PTR(-EINVAL); 1257 1258 atomic_inc(&qps_created); 1259 switch (init_attr->qp_type) { 1260 case IB_QPT_RC: 1261 if (nes_drv_opt & NES_DRV_OPT_NO_INLINE_DATA) { 1262 init_attr->cap.max_inline_data = 0; 1263 } else { 1264 init_attr->cap.max_inline_data = 64; 1265 } 1266 sq_size = init_attr->cap.max_send_wr; 1267 rq_size = init_attr->cap.max_recv_wr; 1268 1269 /* check if the encoded sizes are OK or not... */ 1270 sq_encoded_size = nes_get_encoded_size(&sq_size); 1271 rq_encoded_size = nes_get_encoded_size(&rq_size); 1272 1273 if ((!sq_encoded_size) || (!rq_encoded_size)) { 1274 nes_debug(NES_DBG_QP, "ERROR bad rq (%u) or sq (%u) size\n", 1275 rq_size, sq_size); 1276 return ERR_PTR(-EINVAL); 1277 } 1278 1279 init_attr->cap.max_send_wr = sq_size -2; 1280 init_attr->cap.max_recv_wr = rq_size -1; 1281 nes_debug(NES_DBG_QP, "RQ size=%u, SQ Size=%u\n", rq_size, sq_size); 1282 1283 ret = nes_alloc_resource(nesadapter, nesadapter->allocated_qps, 1284 nesadapter->max_qp, &qp_num, &nesadapter->next_qp); 1285 if (ret) { 1286 return ERR_PTR(ret); 1287 } 1288 1289 /* Need 512 (actually now 1024) byte alignment on this structure */ 1290 mem = kzalloc(sizeof(*nesqp)+NES_SW_CONTEXT_ALIGN-1, GFP_KERNEL); 1291 if (!mem) { 1292 nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num); 1293 nes_debug(NES_DBG_QP, "Unable to allocate QP\n"); 1294 return ERR_PTR(-ENOMEM); 1295 } 1296 u64nesqp = (unsigned long)mem; 1297 u64nesqp += ((u64)NES_SW_CONTEXT_ALIGN) - 1; 1298 u64temp = ((u64)NES_SW_CONTEXT_ALIGN) - 1; 1299 u64nesqp &= ~u64temp; 1300 nesqp = (struct nes_qp *)(unsigned long)u64nesqp; 1301 /* nes_debug(NES_DBG_QP, "nesqp=%p, allocated buffer=%p. Rounded to closest %u\n", 1302 nesqp, mem, NES_SW_CONTEXT_ALIGN); */ 1303 nesqp->allocated_buffer = mem; 1304 1305 if (udata) { 1306 if (ib_copy_from_udata(&req, udata, sizeof(struct nes_create_qp_req))) { 1307 nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num); 1308 kfree(nesqp->allocated_buffer); 1309 nes_debug(NES_DBG_QP, "ib_copy_from_udata() Failed \n"); 1310 return NULL; 1311 } 1312 if (req.user_wqe_buffers) { 1313 virt_wqs = 1; 1314 } 1315 if ((ibpd->uobject) && (ibpd->uobject->context)) { 1316 nesqp->user_mode = 1; 1317 nes_ucontext = to_nesucontext(ibpd->uobject->context); 1318 if (virt_wqs) { 1319 err = 1; 1320 list_for_each_entry(nespbl, &nes_ucontext->qp_reg_mem_list, list) { 1321 if (nespbl->user_base == (unsigned long )req.user_wqe_buffers) { 1322 list_del(&nespbl->list); 1323 err = 0; 1324 nes_debug(NES_DBG_QP, "Found PBL for virtual QP. nespbl=%p. user_base=0x%lx\n", 1325 nespbl, nespbl->user_base); 1326 break; 1327 } 1328 } 1329 if (err) { 1330 nes_debug(NES_DBG_QP, "Didn't Find PBL for virtual QP. address = %llx.\n", 1331 (long long unsigned int)req.user_wqe_buffers); 1332 nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num); 1333 kfree(nesqp->allocated_buffer); 1334 return ERR_PTR(-EFAULT); 1335 } 1336 } 1337 1338 nes_ucontext = to_nesucontext(ibpd->uobject->context); 1339 nesqp->mmap_sq_db_index = 1340 find_next_zero_bit(nes_ucontext->allocated_wqs, 1341 NES_MAX_USER_WQ_REGIONS, nes_ucontext->first_free_wq); 1342 /* nes_debug(NES_DBG_QP, "find_first_zero_biton wqs returned %u\n", 1343 nespd->mmap_db_index); */ 1344 if (nesqp->mmap_sq_db_index >= NES_MAX_USER_WQ_REGIONS) { 1345 nes_debug(NES_DBG_QP, 1346 "db index > max user regions, failing create QP\n"); 1347 nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num); 1348 if (virt_wqs) { 1349 pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, nespbl->pbl_vbase, 1350 nespbl->pbl_pbase); 1351 kfree(nespbl); 1352 } 1353 kfree(nesqp->allocated_buffer); 1354 return ERR_PTR(-ENOMEM); 1355 } 1356 set_bit(nesqp->mmap_sq_db_index, nes_ucontext->allocated_wqs); 1357 nes_ucontext->mmap_nesqp[nesqp->mmap_sq_db_index] = nesqp; 1358 nes_ucontext->first_free_wq = nesqp->mmap_sq_db_index + 1; 1359 } else { 1360 nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num); 1361 kfree(nesqp->allocated_buffer); 1362 return ERR_PTR(-EFAULT); 1363 } 1364 } 1365 err = (!virt_wqs) ? nes_setup_mmap_qp(nesqp, nesvnic, sq_size, rq_size) : 1366 nes_setup_virt_qp(nesqp, nespbl, nesvnic, sq_size, rq_size); 1367 if (err) { 1368 nes_debug(NES_DBG_QP, 1369 "error geting qp mem code = %d\n", err); 1370 nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num); 1371 kfree(nesqp->allocated_buffer); 1372 return ERR_PTR(-ENOMEM); 1373 } 1374 1375 nesqp->hwqp.sq_size = sq_size; 1376 nesqp->hwqp.sq_encoded_size = sq_encoded_size; 1377 nesqp->hwqp.sq_head = 1; 1378 nesqp->hwqp.rq_size = rq_size; 1379 nesqp->hwqp.rq_encoded_size = rq_encoded_size; 1380 /* nes_debug(NES_DBG_QP, "nesqp->nesqp_context_pbase = %p\n", 1381 (void *)nesqp->nesqp_context_pbase); 1382 */ 1383 nesqp->hwqp.qp_id = qp_num; 1384 nesqp->ibqp.qp_num = nesqp->hwqp.qp_id; 1385 nesqp->nespd = nespd; 1386 1387 nescq = to_nescq(init_attr->send_cq); 1388 nesqp->nesscq = nescq; 1389 nescq = to_nescq(init_attr->recv_cq); 1390 nesqp->nesrcq = nescq; 1391 1392 nesqp->nesqp_context->misc |= cpu_to_le32((u32)PCI_FUNC(nesdev->pcidev->devfn) << 1393 NES_QPCONTEXT_MISC_PCI_FCN_SHIFT); 1394 nesqp->nesqp_context->misc |= cpu_to_le32((u32)nesqp->hwqp.rq_encoded_size << 1395 NES_QPCONTEXT_MISC_RQ_SIZE_SHIFT); 1396 nesqp->nesqp_context->misc |= cpu_to_le32((u32)nesqp->hwqp.sq_encoded_size << 1397 NES_QPCONTEXT_MISC_SQ_SIZE_SHIFT); 1398 nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_PRIV_EN); 1399 nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_FAST_REGISTER_EN); 1400 nesqp->nesqp_context->cqs = cpu_to_le32(nesqp->nesscq->hw_cq.cq_number + 1401 ((u32)nesqp->nesrcq->hw_cq.cq_number << 16)); 1402 u64temp = (u64)nesqp->hwqp.sq_pbase; 1403 nesqp->nesqp_context->sq_addr_low = cpu_to_le32((u32)u64temp); 1404 nesqp->nesqp_context->sq_addr_high = cpu_to_le32((u32)(u64temp >> 32)); 1405 1406 1407 if (!virt_wqs) { 1408 u64temp = (u64)nesqp->hwqp.sq_pbase; 1409 nesqp->nesqp_context->sq_addr_low = cpu_to_le32((u32)u64temp); 1410 nesqp->nesqp_context->sq_addr_high = cpu_to_le32((u32)(u64temp >> 32)); 1411 u64temp = (u64)nesqp->hwqp.rq_pbase; 1412 nesqp->nesqp_context->rq_addr_low = cpu_to_le32((u32)u64temp); 1413 nesqp->nesqp_context->rq_addr_high = cpu_to_le32((u32)(u64temp >> 32)); 1414 } else { 1415 u64temp = (u64)nesqp->pbl_pbase; 1416 nesqp->nesqp_context->rq_addr_low = cpu_to_le32((u32)u64temp); 1417 nesqp->nesqp_context->rq_addr_high = cpu_to_le32((u32)(u64temp >> 32)); 1418 } 1419 1420 /* nes_debug(NES_DBG_QP, "next_qp_nic_index=%u, using nic_index=%d\n", 1421 nesvnic->next_qp_nic_index, 1422 nesvnic->qp_nic_index[nesvnic->next_qp_nic_index]); */ 1423 spin_lock_irqsave(&nesdev->cqp.lock, flags); 1424 nesqp->nesqp_context->misc2 |= cpu_to_le32( 1425 (u32)nesvnic->qp_nic_index[nesvnic->next_qp_nic_index] << 1426 NES_QPCONTEXT_MISC2_NIC_INDEX_SHIFT); 1427 nesvnic->next_qp_nic_index++; 1428 if ((nesvnic->next_qp_nic_index > 3) || 1429 (nesvnic->qp_nic_index[nesvnic->next_qp_nic_index] == 0xf)) { 1430 nesvnic->next_qp_nic_index = 0; 1431 } 1432 spin_unlock_irqrestore(&nesdev->cqp.lock, flags); 1433 1434 nesqp->nesqp_context->pd_index_wscale |= cpu_to_le32((u32)nesqp->nespd->pd_id << 16); 1435 u64temp = (u64)nesqp->hwqp.q2_pbase; 1436 nesqp->nesqp_context->q2_addr_low = cpu_to_le32((u32)u64temp); 1437 nesqp->nesqp_context->q2_addr_high = cpu_to_le32((u32)(u64temp >> 32)); 1438 nesqp->nesqp_context->aeq_token_low = cpu_to_le32((u32)((unsigned long)(nesqp))); 1439 nesqp->nesqp_context->aeq_token_high = cpu_to_le32((u32)(upper_32_bits((unsigned long)(nesqp)))); 1440 nesqp->nesqp_context->ird_ord_sizes = cpu_to_le32(NES_QPCONTEXT_ORDIRD_ALSMM | 1441 ((((u32)nesadapter->max_irrq_wr) << 1442 NES_QPCONTEXT_ORDIRD_IRDSIZE_SHIFT) & NES_QPCONTEXT_ORDIRD_IRDSIZE_MASK)); 1443 if (disable_mpa_crc) { 1444 nes_debug(NES_DBG_QP, "Disabling MPA crc checking due to module option.\n"); 1445 nesqp->nesqp_context->ird_ord_sizes |= cpu_to_le32(NES_QPCONTEXT_ORDIRD_RNMC); 1446 } 1447 1448 1449 /* Create the QP */ 1450 cqp_request = nes_get_cqp_request(nesdev); 1451 if (cqp_request == NULL) { 1452 nes_debug(NES_DBG_QP, "Failed to get a cqp_request\n"); 1453 nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num); 1454 nes_free_qp_mem(nesdev, nesqp,virt_wqs); 1455 kfree(nesqp->allocated_buffer); 1456 return ERR_PTR(-ENOMEM); 1457 } 1458 cqp_request->waiting = 1; 1459 cqp_wqe = &cqp_request->cqp_wqe; 1460 1461 if (!virt_wqs) { 1462 opcode = NES_CQP_CREATE_QP | NES_CQP_QP_TYPE_IWARP | 1463 NES_CQP_QP_IWARP_STATE_IDLE; 1464 } else { 1465 opcode = NES_CQP_CREATE_QP | NES_CQP_QP_TYPE_IWARP | NES_CQP_QP_VIRT_WQS | 1466 NES_CQP_QP_IWARP_STATE_IDLE; 1467 } 1468 opcode |= NES_CQP_QP_CQS_VALID; 1469 nes_fill_init_cqp_wqe(cqp_wqe, nesdev); 1470 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, opcode); 1471 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, nesqp->hwqp.qp_id); 1472 1473 u64temp = (u64)nesqp->nesqp_context_pbase; 1474 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp); 1475 1476 atomic_set(&cqp_request->refcount, 2); 1477 nes_post_cqp_request(nesdev, cqp_request, NES_CQP_REQUEST_RING_DOORBELL); 1478 1479 /* Wait for CQP */ 1480 nes_debug(NES_DBG_QP, "Waiting for create iWARP QP%u to complete.\n", 1481 nesqp->hwqp.qp_id); 1482 ret = wait_event_timeout(cqp_request->waitq, 1483 (cqp_request->request_done != 0), NES_EVENT_TIMEOUT); 1484 nes_debug(NES_DBG_QP, "Create iwarp QP%u completed, wait_event_timeout ret=%u," 1485 " nesdev->cqp_head = %u, nesdev->cqp.sq_tail = %u," 1486 " CQP Major:Minor codes = 0x%04X:0x%04X.\n", 1487 nesqp->hwqp.qp_id, ret, nesdev->cqp.sq_head, nesdev->cqp.sq_tail, 1488 cqp_request->major_code, cqp_request->minor_code); 1489 if ((!ret) || (cqp_request->major_code)) { 1490 if (atomic_dec_and_test(&cqp_request->refcount)) { 1491 if (cqp_request->dynamic) { 1492 kfree(cqp_request); 1493 } else { 1494 spin_lock_irqsave(&nesdev->cqp.lock, flags); 1495 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs); 1496 spin_unlock_irqrestore(&nesdev->cqp.lock, flags); 1497 } 1498 } 1499 nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num); 1500 nes_free_qp_mem(nesdev, nesqp,virt_wqs); 1501 kfree(nesqp->allocated_buffer); 1502 if (!ret) { 1503 return ERR_PTR(-ETIME); 1504 } else { 1505 return ERR_PTR(-EIO); 1506 } 1507 } else { 1508 if (atomic_dec_and_test(&cqp_request->refcount)) { 1509 if (cqp_request->dynamic) { 1510 kfree(cqp_request); 1511 } else { 1512 spin_lock_irqsave(&nesdev->cqp.lock, flags); 1513 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs); 1514 spin_unlock_irqrestore(&nesdev->cqp.lock, flags); 1515 } 1516 } 1517 } 1518 1519 if (ibpd->uobject) { 1520 uresp.mmap_sq_db_index = nesqp->mmap_sq_db_index; 1521 uresp.actual_sq_size = sq_size; 1522 uresp.actual_rq_size = rq_size; 1523 uresp.qp_id = nesqp->hwqp.qp_id; 1524 uresp.nes_drv_opt = nes_drv_opt; 1525 if (ib_copy_to_udata(udata, &uresp, sizeof uresp)) { 1526 nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num); 1527 nes_free_qp_mem(nesdev, nesqp,virt_wqs); 1528 kfree(nesqp->allocated_buffer); 1529 return ERR_PTR(-EFAULT); 1530 } 1531 } 1532 1533 nes_debug(NES_DBG_QP, "QP%u structure located @%p.Size = %u.\n", 1534 nesqp->hwqp.qp_id, nesqp, (u32)sizeof(*nesqp)); 1535 spin_lock_init(&nesqp->lock); 1536 init_waitqueue_head(&nesqp->state_waitq); 1537 init_waitqueue_head(&nesqp->kick_waitq); 1538 nes_add_ref(&nesqp->ibqp); 1539 break; 1540 default: 1541 nes_debug(NES_DBG_QP, "Invalid QP type: %d\n", init_attr->qp_type); 1542 return ERR_PTR(-EINVAL); 1543 break; 1544 } 1545 1546 /* update the QP table */ 1547 nesdev->nesadapter->qp_table[nesqp->hwqp.qp_id-NES_FIRST_QPN] = nesqp; 1548 nes_debug(NES_DBG_QP, "netdev refcnt=%u\n", 1549 atomic_read(&nesvnic->netdev->refcnt)); 1550 1551 return &nesqp->ibqp; 1552} 1553 1554 1555/** 1556 * nes_destroy_qp 1557 */ 1558static int nes_destroy_qp(struct ib_qp *ibqp) 1559{ 1560 struct nes_qp *nesqp = to_nesqp(ibqp); 1561 /* struct nes_vnic *nesvnic = to_nesvnic(ibqp->device); */ 1562 struct nes_ucontext *nes_ucontext; 1563 struct ib_qp_attr attr; 1564 struct iw_cm_id *cm_id; 1565 struct iw_cm_event cm_event; 1566 int ret; 1567 1568 atomic_inc(&sw_qps_destroyed); 1569 nesqp->destroyed = 1; 1570 1571 /* Blow away the connection if it exists. */ 1572 if (nesqp->ibqp_state >= IB_QPS_INIT && nesqp->ibqp_state <= IB_QPS_RTS) { 1573 /* if (nesqp->ibqp_state == IB_QPS_RTS) { */ 1574 attr.qp_state = IB_QPS_ERR; 1575 nes_modify_qp(&nesqp->ibqp, &attr, IB_QP_STATE, NULL); 1576 } 1577 1578 if (((nesqp->ibqp_state == IB_QPS_INIT) || 1579 (nesqp->ibqp_state == IB_QPS_RTR)) && (nesqp->cm_id)) { 1580 cm_id = nesqp->cm_id; 1581 cm_event.event = IW_CM_EVENT_CONNECT_REPLY; 1582 cm_event.status = IW_CM_EVENT_STATUS_TIMEOUT; 1583 cm_event.local_addr = cm_id->local_addr; 1584 cm_event.remote_addr = cm_id->remote_addr; 1585 cm_event.private_data = NULL; 1586 cm_event.private_data_len = 0; 1587 1588 nes_debug(NES_DBG_QP, "Generating a CM Timeout Event for " 1589 "QP%u. cm_id = %p, refcount = %u. \n", 1590 nesqp->hwqp.qp_id, cm_id, atomic_read(&nesqp->refcount)); 1591 1592 cm_id->rem_ref(cm_id); 1593 ret = cm_id->event_handler(cm_id, &cm_event); 1594 if (ret) 1595 nes_debug(NES_DBG_QP, "OFA CM event_handler returned, ret=%d\n", ret); 1596 } 1597 1598 1599 if (nesqp->user_mode) { 1600 if ((ibqp->uobject)&&(ibqp->uobject->context)) { 1601 nes_ucontext = to_nesucontext(ibqp->uobject->context); 1602 clear_bit(nesqp->mmap_sq_db_index, nes_ucontext->allocated_wqs); 1603 nes_ucontext->mmap_nesqp[nesqp->mmap_sq_db_index] = NULL; 1604 if (nes_ucontext->first_free_wq > nesqp->mmap_sq_db_index) { 1605 nes_ucontext->first_free_wq = nesqp->mmap_sq_db_index; 1606 } 1607 } 1608 if (nesqp->pbl_pbase) 1609 kunmap(nesqp->page); 1610 } 1611 1612 nes_rem_ref(&nesqp->ibqp); 1613 return 0; 1614} 1615 1616 1617/** 1618 * nes_create_cq 1619 */ 1620static struct ib_cq *nes_create_cq(struct ib_device *ibdev, int entries, 1621 int comp_vector, 1622 struct ib_ucontext *context, struct ib_udata *udata) 1623{ 1624 u64 u64temp; 1625 struct nes_vnic *nesvnic = to_nesvnic(ibdev); 1626 struct nes_device *nesdev = nesvnic->nesdev; 1627 struct nes_adapter *nesadapter = nesdev->nesadapter; 1628 struct nes_cq *nescq; 1629 struct nes_ucontext *nes_ucontext = NULL; 1630 struct nes_cqp_request *cqp_request; 1631 void *mem = NULL; 1632 struct nes_hw_cqp_wqe *cqp_wqe; 1633 struct nes_pbl *nespbl = NULL; 1634 struct nes_create_cq_req req; 1635 struct nes_create_cq_resp resp; 1636 u32 cq_num = 0; 1637 u32 opcode = 0; 1638 u32 pbl_entries = 1; 1639 int err; 1640 unsigned long flags; 1641 int ret; 1642 1643 err = nes_alloc_resource(nesadapter, nesadapter->allocated_cqs, 1644 nesadapter->max_cq, &cq_num, &nesadapter->next_cq); 1645 if (err) { 1646 return ERR_PTR(err); 1647 } 1648 1649 nescq = kzalloc(sizeof(struct nes_cq), GFP_KERNEL); 1650 if (!nescq) { 1651 nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); 1652 nes_debug(NES_DBG_CQ, "Unable to allocate nes_cq struct\n"); 1653 return ERR_PTR(-ENOMEM); 1654 } 1655 1656 nescq->hw_cq.cq_size = max(entries + 1, 5); 1657 nescq->hw_cq.cq_number = cq_num; 1658 nescq->ibcq.cqe = nescq->hw_cq.cq_size - 1; 1659 1660 1661 if (context) { 1662 nes_ucontext = to_nesucontext(context); 1663 if (ib_copy_from_udata(&req, udata, sizeof (struct nes_create_cq_req))) { 1664 nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); 1665 kfree(nescq); 1666 return ERR_PTR(-EFAULT); 1667 } 1668 nesvnic->mcrq_ucontext = nes_ucontext; 1669 nes_ucontext->mcrqf = req.mcrqf; 1670 if (nes_ucontext->mcrqf) { 1671 if (nes_ucontext->mcrqf & 0x80000000) 1672 nescq->hw_cq.cq_number = nesvnic->nic.qp_id + 12 + (nes_ucontext->mcrqf & 0xf) - 1; 1673 else if (nes_ucontext->mcrqf & 0x40000000) 1674 nescq->hw_cq.cq_number = nes_ucontext->mcrqf & 0xffff; 1675 else 1676 nescq->hw_cq.cq_number = nesvnic->mcrq_qp_id + nes_ucontext->mcrqf-1; 1677 nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); 1678 } 1679 nes_debug(NES_DBG_CQ, "CQ Virtual Address = %08lX, size = %u.\n", 1680 (unsigned long)req.user_cq_buffer, entries); 1681 err = 1; 1682 list_for_each_entry(nespbl, &nes_ucontext->cq_reg_mem_list, list) { 1683 if (nespbl->user_base == (unsigned long )req.user_cq_buffer) { 1684 list_del(&nespbl->list); 1685 err = 0; 1686 nes_debug(NES_DBG_CQ, "Found PBL for virtual CQ. nespbl=%p.\n", 1687 nespbl); 1688 break; 1689 } 1690 } 1691 if (err) { 1692 nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); 1693 kfree(nescq); 1694 return ERR_PTR(-EFAULT); 1695 } 1696 1697 pbl_entries = nespbl->pbl_size >> 3; 1698 nescq->cq_mem_size = 0; 1699 } else { 1700 nescq->cq_mem_size = nescq->hw_cq.cq_size * sizeof(struct nes_hw_cqe); 1701 nes_debug(NES_DBG_CQ, "Attempting to allocate pci memory (%u entries, %u bytes) for CQ%u.\n", 1702 entries, nescq->cq_mem_size, nescq->hw_cq.cq_number); 1703 1704 /* allocate the physical buffer space */ 1705 mem = pci_alloc_consistent(nesdev->pcidev, nescq->cq_mem_size, 1706 &nescq->hw_cq.cq_pbase); 1707 if (!mem) { 1708 printk(KERN_ERR PFX "Unable to allocate pci memory for cq\n"); 1709 nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); 1710 kfree(nescq); 1711 return ERR_PTR(-ENOMEM); 1712 } 1713 1714 memset(mem, 0, nescq->cq_mem_size); 1715 nescq->hw_cq.cq_vbase = mem; 1716 nescq->hw_cq.cq_head = 0; 1717 nes_debug(NES_DBG_CQ, "CQ%u virtual address @ %p, phys = 0x%08X\n", 1718 nescq->hw_cq.cq_number, nescq->hw_cq.cq_vbase, 1719 (u32)nescq->hw_cq.cq_pbase); 1720 } 1721 1722 nescq->hw_cq.ce_handler = nes_iwarp_ce_handler; 1723 spin_lock_init(&nescq->lock); 1724 1725 /* send CreateCQ request to CQP */ 1726 cqp_request = nes_get_cqp_request(nesdev); 1727 if (cqp_request == NULL) { 1728 nes_debug(NES_DBG_CQ, "Failed to get a cqp_request.\n"); 1729 if (!context) 1730 pci_free_consistent(nesdev->pcidev, nescq->cq_mem_size, mem, 1731 nescq->hw_cq.cq_pbase); 1732 nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); 1733 kfree(nescq); 1734 return ERR_PTR(-ENOMEM); 1735 } 1736 cqp_request->waiting = 1; 1737 cqp_wqe = &cqp_request->cqp_wqe; 1738 1739 opcode = NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID | 1740 NES_CQP_CQ_CHK_OVERFLOW | 1741 NES_CQP_CQ_CEQE_MASK | ((u32)nescq->hw_cq.cq_size << 16); 1742 1743 spin_lock_irqsave(&nesadapter->pbl_lock, flags); 1744 1745 if (pbl_entries != 1) { 1746 if (pbl_entries > 32) { 1747 /* use 4k pbl */ 1748 nes_debug(NES_DBG_CQ, "pbl_entries=%u, use a 4k PBL\n", pbl_entries); 1749 if (nesadapter->free_4kpbl == 0) { 1750 if (cqp_request->dynamic) { 1751 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 1752 kfree(cqp_request); 1753 } else { 1754 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs); 1755 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 1756 } 1757 if (!context) 1758 pci_free_consistent(nesdev->pcidev, nescq->cq_mem_size, mem, 1759 nescq->hw_cq.cq_pbase); 1760 nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); 1761 kfree(nescq); 1762 return ERR_PTR(-ENOMEM); 1763 } else { 1764 opcode |= (NES_CQP_CQ_VIRT | NES_CQP_CQ_4KB_CHUNK); 1765 nescq->virtual_cq = 2; 1766 nesadapter->free_4kpbl--; 1767 } 1768 } else { 1769 /* use 256 byte pbl */ 1770 nes_debug(NES_DBG_CQ, "pbl_entries=%u, use a 256 byte PBL\n", pbl_entries); 1771 if (nesadapter->free_256pbl == 0) { 1772 if (cqp_request->dynamic) { 1773 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 1774 kfree(cqp_request); 1775 } else { 1776 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs); 1777 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 1778 } 1779 if (!context) 1780 pci_free_consistent(nesdev->pcidev, nescq->cq_mem_size, mem, 1781 nescq->hw_cq.cq_pbase); 1782 nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); 1783 kfree(nescq); 1784 return ERR_PTR(-ENOMEM); 1785 } else { 1786 opcode |= NES_CQP_CQ_VIRT; 1787 nescq->virtual_cq = 1; 1788 nesadapter->free_256pbl--; 1789 } 1790 } 1791 } 1792 1793 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 1794 1795 nes_fill_init_cqp_wqe(cqp_wqe, nesdev); 1796 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, opcode); 1797 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, 1798 (nescq->hw_cq.cq_number | ((u32)nesdev->ceq_index << 16))); 1799 1800 if (context) { 1801 if (pbl_entries != 1) 1802 u64temp = (u64)nespbl->pbl_pbase; 1803 else 1804 u64temp = le64_to_cpu(nespbl->pbl_vbase[0]); 1805 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX, 1806 nes_ucontext->mmap_db_index[0]); 1807 } else { 1808 u64temp = (u64)nescq->hw_cq.cq_pbase; 1809 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0; 1810 } 1811 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp); 1812 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0; 1813 u64temp = (u64)(unsigned long)&nescq->hw_cq; 1814 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] = 1815 cpu_to_le32((u32)(u64temp >> 1)); 1816 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 1817 cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF); 1818 1819 atomic_set(&cqp_request->refcount, 2); 1820 nes_post_cqp_request(nesdev, cqp_request, NES_CQP_REQUEST_RING_DOORBELL); 1821 1822 /* Wait for CQP */ 1823 nes_debug(NES_DBG_CQ, "Waiting for create iWARP CQ%u to complete.\n", 1824 nescq->hw_cq.cq_number); 1825 ret = wait_event_timeout(cqp_request->waitq, (0 != cqp_request->request_done), 1826 NES_EVENT_TIMEOUT * 2); 1827 nes_debug(NES_DBG_CQ, "Create iWARP CQ%u completed, wait_event_timeout ret = %d.\n", 1828 nescq->hw_cq.cq_number, ret); 1829 if ((!ret) || (cqp_request->major_code)) { 1830 if (atomic_dec_and_test(&cqp_request->refcount)) { 1831 if (cqp_request->dynamic) { 1832 kfree(cqp_request); 1833 } else { 1834 spin_lock_irqsave(&nesdev->cqp.lock, flags); 1835 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs); 1836 spin_unlock_irqrestore(&nesdev->cqp.lock, flags); 1837 } 1838 } 1839 if (!context) 1840 pci_free_consistent(nesdev->pcidev, nescq->cq_mem_size, mem, 1841 nescq->hw_cq.cq_pbase); 1842 nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); 1843 kfree(nescq); 1844 return ERR_PTR(-EIO); 1845 } else { 1846 if (atomic_dec_and_test(&cqp_request->refcount)) { 1847 if (cqp_request->dynamic) { 1848 kfree(cqp_request); 1849 } else { 1850 spin_lock_irqsave(&nesdev->cqp.lock, flags); 1851 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs); 1852 spin_unlock_irqrestore(&nesdev->cqp.lock, flags); 1853 } 1854 } 1855 } 1856 1857 if (context) { 1858 /* free the nespbl */ 1859 pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, nespbl->pbl_vbase, 1860 nespbl->pbl_pbase); 1861 kfree(nespbl); 1862 resp.cq_id = nescq->hw_cq.cq_number; 1863 resp.cq_size = nescq->hw_cq.cq_size; 1864 resp.mmap_db_index = 0; 1865 if (ib_copy_to_udata(udata, &resp, sizeof resp)) { 1866 nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); 1867 kfree(nescq); 1868 return ERR_PTR(-EFAULT); 1869 } 1870 } 1871 1872 return &nescq->ibcq; 1873} 1874 1875 1876/** 1877 * nes_destroy_cq 1878 */ 1879static int nes_destroy_cq(struct ib_cq *ib_cq) 1880{ 1881 struct nes_cq *nescq; 1882 struct nes_device *nesdev; 1883 struct nes_vnic *nesvnic; 1884 struct nes_adapter *nesadapter; 1885 struct nes_hw_cqp_wqe *cqp_wqe; 1886 struct nes_cqp_request *cqp_request; 1887 unsigned long flags; 1888 u32 opcode = 0; 1889 int ret; 1890 1891 if (ib_cq == NULL) 1892 return 0; 1893 1894 nescq = to_nescq(ib_cq); 1895 nesvnic = to_nesvnic(ib_cq->device); 1896 nesdev = nesvnic->nesdev; 1897 nesadapter = nesdev->nesadapter; 1898 1899 nes_debug(NES_DBG_CQ, "Destroy CQ%u\n", nescq->hw_cq.cq_number); 1900 1901 /* Send DestroyCQ request to CQP */ 1902 cqp_request = nes_get_cqp_request(nesdev); 1903 if (cqp_request == NULL) { 1904 nes_debug(NES_DBG_CQ, "Failed to get a cqp_request.\n"); 1905 return -ENOMEM; 1906 } 1907 cqp_request->waiting = 1; 1908 cqp_wqe = &cqp_request->cqp_wqe; 1909 opcode = NES_CQP_DESTROY_CQ | (nescq->hw_cq.cq_size << 16); 1910 spin_lock_irqsave(&nesadapter->pbl_lock, flags); 1911 if (nescq->virtual_cq == 1) { 1912 nesadapter->free_256pbl++; 1913 if (nesadapter->free_256pbl > nesadapter->max_256pbl) { 1914 printk(KERN_ERR PFX "%s: free 256B PBLs(%u) has exceeded the max(%u)\n", 1915 __func__, nesadapter->free_256pbl, nesadapter->max_256pbl); 1916 } 1917 } else if (nescq->virtual_cq == 2) { 1918 nesadapter->free_4kpbl++; 1919 if (nesadapter->free_4kpbl > nesadapter->max_4kpbl) { 1920 printk(KERN_ERR PFX "%s: free 4K PBLs(%u) has exceeded the max(%u)\n", 1921 __func__, nesadapter->free_4kpbl, nesadapter->max_4kpbl); 1922 } 1923 opcode |= NES_CQP_CQ_4KB_CHUNK; 1924 } 1925 1926 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 1927 1928 nes_fill_init_cqp_wqe(cqp_wqe, nesdev); 1929 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, opcode); 1930 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, 1931 (nescq->hw_cq.cq_number | ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 16))); 1932 nes_free_resource(nesadapter, nesadapter->allocated_cqs, nescq->hw_cq.cq_number); 1933 atomic_set(&cqp_request->refcount, 2); 1934 nes_post_cqp_request(nesdev, cqp_request, NES_CQP_REQUEST_RING_DOORBELL); 1935 1936 /* Wait for CQP */ 1937 nes_debug(NES_DBG_CQ, "Waiting for destroy iWARP CQ%u to complete.\n", 1938 nescq->hw_cq.cq_number); 1939 ret = wait_event_timeout(cqp_request->waitq, (0 != cqp_request->request_done), 1940 NES_EVENT_TIMEOUT); 1941 nes_debug(NES_DBG_CQ, "Destroy iWARP CQ%u completed, wait_event_timeout ret = %u," 1942 " CQP Major:Minor codes = 0x%04X:0x%04X.\n", 1943 nescq->hw_cq.cq_number, ret, cqp_request->major_code, 1944 cqp_request->minor_code); 1945 if ((!ret) || (cqp_request->major_code)) { 1946 if (atomic_dec_and_test(&cqp_request->refcount)) { 1947 if (cqp_request->dynamic) { 1948 kfree(cqp_request); 1949 } else { 1950 spin_lock_irqsave(&nesdev->cqp.lock, flags); 1951 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs); 1952 spin_unlock_irqrestore(&nesdev->cqp.lock, flags); 1953 } 1954 } 1955 if (!ret) { 1956 nes_debug(NES_DBG_CQ, "iWARP CQ%u destroy timeout expired\n", 1957 nescq->hw_cq.cq_number); 1958 ret = -ETIME; 1959 } else { 1960 nes_debug(NES_DBG_CQ, "iWARP CQ%u destroy failed\n", 1961 nescq->hw_cq.cq_number); 1962 ret = -EIO; 1963 } 1964 } else { 1965 ret = 0; 1966 if (atomic_dec_and_test(&cqp_request->refcount)) { 1967 if (cqp_request->dynamic) { 1968 kfree(cqp_request); 1969 } else { 1970 spin_lock_irqsave(&nesdev->cqp.lock, flags); 1971 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs); 1972 spin_unlock_irqrestore(&nesdev->cqp.lock, flags); 1973 } 1974 } 1975 } 1976 1977 if (nescq->cq_mem_size) 1978 pci_free_consistent(nesdev->pcidev, nescq->cq_mem_size, 1979 nescq->hw_cq.cq_vbase, nescq->hw_cq.cq_pbase); 1980 kfree(nescq); 1981 1982 return ret; 1983} 1984 1985 1986/** 1987 * nes_reg_mr 1988 */ 1989static int nes_reg_mr(struct nes_device *nesdev, struct nes_pd *nespd, 1990 u32 stag, u64 region_length, struct nes_root_vpbl *root_vpbl, 1991 dma_addr_t single_buffer, u16 pbl_count, u16 residual_page_count, 1992 int acc, u64 *iova_start) 1993{ 1994 struct nes_hw_cqp_wqe *cqp_wqe; 1995 struct nes_cqp_request *cqp_request; 1996 unsigned long flags; 1997 int ret; 1998 struct nes_adapter *nesadapter = nesdev->nesadapter; 1999 /* int count; */ 2000 u32 opcode = 0; 2001 u16 major_code; 2002 2003 /* Register the region with the adapter */ 2004 cqp_request = nes_get_cqp_request(nesdev); 2005 if (cqp_request == NULL) { 2006 nes_debug(NES_DBG_MR, "Failed to get a cqp_request.\n"); 2007 return -ENOMEM; 2008 } 2009 cqp_request->waiting = 1; 2010 cqp_wqe = &cqp_request->cqp_wqe; 2011 2012 spin_lock_irqsave(&nesadapter->pbl_lock, flags); 2013 /* track PBL resources */ 2014 if (pbl_count != 0) { 2015 if (pbl_count > 1) { 2016 /* Two level PBL */ 2017 if ((pbl_count+1) > nesadapter->free_4kpbl) { 2018 nes_debug(NES_DBG_MR, "Out of 4KB Pbls for two level request.\n"); 2019 if (cqp_request->dynamic) { 2020 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 2021 kfree(cqp_request); 2022 } else { 2023 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs); 2024 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 2025 } 2026 return -ENOMEM; 2027 } else { 2028 nesadapter->free_4kpbl -= pbl_count+1; 2029 } 2030 } else if (residual_page_count > 32) { 2031 if (pbl_count > nesadapter->free_4kpbl) { 2032 nes_debug(NES_DBG_MR, "Out of 4KB Pbls.\n"); 2033 if (cqp_request->dynamic) { 2034 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 2035 kfree(cqp_request); 2036 } else { 2037 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs); 2038 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 2039 } 2040 return -ENOMEM; 2041 } else { 2042 nesadapter->free_4kpbl -= pbl_count; 2043 } 2044 } else { 2045 if (pbl_count > nesadapter->free_256pbl) { 2046 nes_debug(NES_DBG_MR, "Out of 256B Pbls.\n"); 2047 if (cqp_request->dynamic) { 2048 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 2049 kfree(cqp_request); 2050 } else { 2051 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs); 2052 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 2053 } 2054 return -ENOMEM; 2055 } else { 2056 nesadapter->free_256pbl -= pbl_count; 2057 } 2058 } 2059 } 2060 2061 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 2062 2063 opcode = NES_CQP_REGISTER_STAG | NES_CQP_STAG_RIGHTS_LOCAL_READ | 2064 NES_CQP_STAG_VA_TO | NES_CQP_STAG_MR; 2065 if (acc & IB_ACCESS_LOCAL_WRITE) 2066 opcode |= NES_CQP_STAG_RIGHTS_LOCAL_WRITE; 2067 if (acc & IB_ACCESS_REMOTE_WRITE) 2068 opcode |= NES_CQP_STAG_RIGHTS_REMOTE_WRITE | NES_CQP_STAG_REM_ACC_EN; 2069 if (acc & IB_ACCESS_REMOTE_READ) 2070 opcode |= NES_CQP_STAG_RIGHTS_REMOTE_READ | NES_CQP_STAG_REM_ACC_EN; 2071 if (acc & IB_ACCESS_MW_BIND) 2072 opcode |= NES_CQP_STAG_RIGHTS_WINDOW_BIND | NES_CQP_STAG_REM_ACC_EN; 2073 2074 nes_fill_init_cqp_wqe(cqp_wqe, nesdev); 2075 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, opcode); 2076 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_VA_LOW_IDX, *iova_start); 2077 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_LEN_LOW_IDX, region_length); 2078 2079 cqp_wqe->wqe_words[NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX] = 2080 cpu_to_le32((u32)(region_length >> 8) & 0xff000000); 2081 cqp_wqe->wqe_words[NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX] |= 2082 cpu_to_le32(nespd->pd_id & 0x00007fff); 2083 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_STAG_IDX, stag); 2084 2085 if (pbl_count == 0) { 2086 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PA_LOW_IDX, single_buffer); 2087 } else { 2088 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PA_LOW_IDX, root_vpbl->pbl_pbase); 2089 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PBL_BLK_COUNT_IDX, pbl_count); 2090 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PBL_LEN_IDX, 2091 (((pbl_count - 1) * 4096) + (residual_page_count*8))); 2092 2093 if ((pbl_count > 1) || (residual_page_count > 32)) 2094 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_STAG_PBL_BLK_SIZE); 2095 } 2096 barrier(); 2097 2098 atomic_set(&cqp_request->refcount, 2); 2099 nes_post_cqp_request(nesdev, cqp_request, NES_CQP_REQUEST_RING_DOORBELL); 2100 2101 /* Wait for CQP */ 2102 ret = wait_event_timeout(cqp_request->waitq, (0 != cqp_request->request_done), 2103 NES_EVENT_TIMEOUT); 2104 nes_debug(NES_DBG_MR, "Register STag 0x%08X completed, wait_event_timeout ret = %u," 2105 " CQP Major:Minor codes = 0x%04X:0x%04X.\n", 2106 stag, ret, cqp_request->major_code, cqp_request->minor_code); 2107 major_code = cqp_request->major_code; 2108 if (atomic_dec_and_test(&cqp_request->refcount)) { 2109 if (cqp_request->dynamic) { 2110 kfree(cqp_request); 2111 } else { 2112 spin_lock_irqsave(&nesdev->cqp.lock, flags); 2113 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs); 2114 spin_unlock_irqrestore(&nesdev->cqp.lock, flags); 2115 } 2116 } 2117 if (!ret) 2118 return -ETIME; 2119 else if (major_code) 2120 return -EIO; 2121 else 2122 return 0; 2123 2124 return 0; 2125} 2126 2127 2128/** 2129 * nes_reg_phys_mr 2130 */ 2131static struct ib_mr *nes_reg_phys_mr(struct ib_pd *ib_pd, 2132 struct ib_phys_buf *buffer_list, int num_phys_buf, int acc, 2133 u64 * iova_start) 2134{ 2135 u64 region_length; 2136 struct nes_pd *nespd = to_nespd(ib_pd); 2137 struct nes_vnic *nesvnic = to_nesvnic(ib_pd->device); 2138 struct nes_device *nesdev = nesvnic->nesdev; 2139 struct nes_adapter *nesadapter = nesdev->nesadapter; 2140 struct nes_mr *nesmr; 2141 struct ib_mr *ibmr; 2142 struct nes_vpbl vpbl; 2143 struct nes_root_vpbl root_vpbl; 2144 u32 stag; 2145 u32 i; 2146 u32 stag_index = 0; 2147 u32 next_stag_index = 0; 2148 u32 driver_key = 0; 2149 u32 root_pbl_index = 0; 2150 u32 cur_pbl_index = 0; 2151 int err = 0, pbl_depth = 0; 2152 int ret = 0; 2153 u16 pbl_count = 0; 2154 u8 single_page = 1; 2155 u8 stag_key = 0; 2156 2157 pbl_depth = 0; 2158 region_length = 0; 2159 vpbl.pbl_vbase = NULL; 2160 root_vpbl.pbl_vbase = NULL; 2161 root_vpbl.pbl_pbase = 0; 2162 2163 get_random_bytes(&next_stag_index, sizeof(next_stag_index)); 2164 stag_key = (u8)next_stag_index; 2165 2166 driver_key = 0; 2167 2168 next_stag_index >>= 8; 2169 next_stag_index %= nesadapter->max_mr; 2170 if (num_phys_buf > (1024*512)) { 2171 return ERR_PTR(-E2BIG); 2172 } 2173 2174 err = nes_alloc_resource(nesadapter, nesadapter->allocated_mrs, nesadapter->max_mr, 2175 &stag_index, &next_stag_index); 2176 if (err) { 2177 return ERR_PTR(err); 2178 } 2179 2180 nesmr = kzalloc(sizeof(*nesmr), GFP_KERNEL); 2181 if (!nesmr) { 2182 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); 2183 return ERR_PTR(-ENOMEM); 2184 } 2185 2186 for (i = 0; i < num_phys_buf; i++) { 2187 2188 if ((i & 0x01FF) == 0) { 2189 if (root_pbl_index == 1) { 2190 /* Allocate the root PBL */ 2191 root_vpbl.pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 8192, 2192 &root_vpbl.pbl_pbase); 2193 nes_debug(NES_DBG_MR, "Allocating root PBL, va = %p, pa = 0x%08X\n", 2194 root_vpbl.pbl_vbase, (unsigned int)root_vpbl.pbl_pbase); 2195 if (!root_vpbl.pbl_vbase) { 2196 pci_free_consistent(nesdev->pcidev, 4096, vpbl.pbl_vbase, 2197 vpbl.pbl_pbase); 2198 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); 2199 kfree(nesmr); 2200 return ERR_PTR(-ENOMEM); 2201 } 2202 root_vpbl.leaf_vpbl = kzalloc(sizeof(*root_vpbl.leaf_vpbl)*1024, GFP_KERNEL); 2203 if (!root_vpbl.leaf_vpbl) { 2204 pci_free_consistent(nesdev->pcidev, 8192, root_vpbl.pbl_vbase, 2205 root_vpbl.pbl_pbase); 2206 pci_free_consistent(nesdev->pcidev, 4096, vpbl.pbl_vbase, 2207 vpbl.pbl_pbase); 2208 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); 2209 kfree(nesmr); 2210 return ERR_PTR(-ENOMEM); 2211 } 2212 root_vpbl.pbl_vbase[0].pa_low = cpu_to_le32((u32)vpbl.pbl_pbase); 2213 root_vpbl.pbl_vbase[0].pa_high = 2214 cpu_to_le32((u32)((((u64)vpbl.pbl_pbase) >> 32))); 2215 root_vpbl.leaf_vpbl[0] = vpbl; 2216 } 2217 /* Allocate a 4K buffer for the PBL */ 2218 vpbl.pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 4096, 2219 &vpbl.pbl_pbase); 2220 nes_debug(NES_DBG_MR, "Allocating leaf PBL, va = %p, pa = 0x%016lX\n", 2221 vpbl.pbl_vbase, (unsigned long)vpbl.pbl_pbase); 2222 if (!vpbl.pbl_vbase) { 2223 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); 2224 ibmr = ERR_PTR(-ENOMEM); 2225 kfree(nesmr); 2226 goto reg_phys_err; 2227 } 2228 /* Fill in the root table */ 2229 if (1 <= root_pbl_index) { 2230 root_vpbl.pbl_vbase[root_pbl_index].pa_low = 2231 cpu_to_le32((u32)vpbl.pbl_pbase); 2232 root_vpbl.pbl_vbase[root_pbl_index].pa_high = 2233 cpu_to_le32((u32)((((u64)vpbl.pbl_pbase) >> 32))); 2234 root_vpbl.leaf_vpbl[root_pbl_index] = vpbl; 2235 } 2236 root_pbl_index++; 2237 cur_pbl_index = 0; 2238 } 2239 if (buffer_list[i].addr & ~PAGE_MASK) { 2240 /* TODO: Unwind allocated buffers */ 2241 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); 2242 nes_debug(NES_DBG_MR, "Unaligned Memory Buffer: 0x%x\n", 2243 (unsigned int) buffer_list[i].addr); 2244 ibmr = ERR_PTR(-EINVAL); 2245 kfree(nesmr); 2246 goto reg_phys_err; 2247 } 2248 2249 if (!buffer_list[i].size) { 2250 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); 2251 nes_debug(NES_DBG_MR, "Invalid Buffer Size\n"); 2252 ibmr = ERR_PTR(-EINVAL); 2253 kfree(nesmr); 2254 goto reg_phys_err; 2255 } 2256 2257 region_length += buffer_list[i].size; 2258 if ((i != 0) && (single_page)) { 2259 if ((buffer_list[i-1].addr+PAGE_SIZE) != buffer_list[i].addr) 2260 single_page = 0; 2261 } 2262 vpbl.pbl_vbase[cur_pbl_index].pa_low = cpu_to_le32((u32)buffer_list[i].addr); 2263 vpbl.pbl_vbase[cur_pbl_index++].pa_high = 2264 cpu_to_le32((u32)((((u64)buffer_list[i].addr) >> 32))); 2265 } 2266 2267 stag = stag_index << 8; 2268 stag |= driver_key; 2269 stag += (u32)stag_key; 2270 2271 nes_debug(NES_DBG_MR, "Registering STag 0x%08X, VA = 0x%016lX," 2272 " length = 0x%016lX, index = 0x%08X\n", 2273 stag, (unsigned long)*iova_start, (unsigned long)region_length, stag_index); 2274 2275 region_length -= (*iova_start)&PAGE_MASK; 2276 2277 /* Make the leaf PBL the root if only one PBL */ 2278 if (root_pbl_index == 1) { 2279 root_vpbl.pbl_pbase = vpbl.pbl_pbase; 2280 } 2281 2282 if (single_page) { 2283 pbl_count = 0; 2284 } else { 2285 pbl_count = root_pbl_index; 2286 } 2287 ret = nes_reg_mr(nesdev, nespd, stag, region_length, &root_vpbl, 2288 buffer_list[0].addr, pbl_count, (u16)cur_pbl_index, acc, iova_start); 2289 2290 if (ret == 0) { 2291 nesmr->ibmr.rkey = stag; 2292 nesmr->ibmr.lkey = stag; 2293 nesmr->mode = IWNES_MEMREG_TYPE_MEM; 2294 ibmr = &nesmr->ibmr; 2295 nesmr->pbl_4k = ((pbl_count > 1) || (cur_pbl_index > 32)) ? 1 : 0; 2296 nesmr->pbls_used = pbl_count; 2297 if (pbl_count > 1) { 2298 nesmr->pbls_used++; 2299 } 2300 } else { 2301 kfree(nesmr); 2302 ibmr = ERR_PTR(-ENOMEM); 2303 } 2304 2305 reg_phys_err: 2306 /* free the resources */ 2307 if (root_pbl_index == 1) { 2308 /* single PBL case */ 2309 pci_free_consistent(nesdev->pcidev, 4096, vpbl.pbl_vbase, vpbl.pbl_pbase); 2310 } else { 2311 for (i=0; i<root_pbl_index; i++) { 2312 pci_free_consistent(nesdev->pcidev, 4096, root_vpbl.leaf_vpbl[i].pbl_vbase, 2313 root_vpbl.leaf_vpbl[i].pbl_pbase); 2314 } 2315 kfree(root_vpbl.leaf_vpbl); 2316 pci_free_consistent(nesdev->pcidev, 8192, root_vpbl.pbl_vbase, 2317 root_vpbl.pbl_pbase); 2318 } 2319 2320 return ibmr; 2321} 2322 2323 2324/** 2325 * nes_get_dma_mr 2326 */ 2327static struct ib_mr *nes_get_dma_mr(struct ib_pd *pd, int acc) 2328{ 2329 struct ib_phys_buf bl; 2330 u64 kva = 0; 2331 2332 nes_debug(NES_DBG_MR, "\n"); 2333 2334 bl.size = (u64)0xffffffffffULL; 2335 bl.addr = 0; 2336 return nes_reg_phys_mr(pd, &bl, 1, acc, &kva); 2337} 2338 2339 2340/** 2341 * nes_reg_user_mr 2342 */ 2343static struct ib_mr *nes_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, 2344 u64 virt, int acc, struct ib_udata *udata) 2345{ 2346 u64 iova_start; 2347 __le64 *pbl; 2348 u64 region_length; 2349 dma_addr_t last_dma_addr = 0; 2350 dma_addr_t first_dma_addr = 0; 2351 struct nes_pd *nespd = to_nespd(pd); 2352 struct nes_vnic *nesvnic = to_nesvnic(pd->device); 2353 struct nes_device *nesdev = nesvnic->nesdev; 2354 struct nes_adapter *nesadapter = nesdev->nesadapter; 2355 struct ib_mr *ibmr = ERR_PTR(-EINVAL); 2356 struct ib_umem_chunk *chunk; 2357 struct nes_ucontext *nes_ucontext; 2358 struct nes_pbl *nespbl; 2359 struct nes_mr *nesmr; 2360 struct ib_umem *region; 2361 struct nes_mem_reg_req req; 2362 struct nes_vpbl vpbl; 2363 struct nes_root_vpbl root_vpbl; 2364 int nmap_index, page_index; 2365 int page_count = 0; 2366 int err, pbl_depth = 0; 2367 int chunk_pages; 2368 int ret; 2369 u32 stag; 2370 u32 stag_index = 0; 2371 u32 next_stag_index; 2372 u32 driver_key; 2373 u32 root_pbl_index = 0; 2374 u32 cur_pbl_index = 0; 2375 u32 skip_pages; 2376 u16 pbl_count; 2377 u8 single_page = 1; 2378 u8 stag_key; 2379 2380 region = ib_umem_get(pd->uobject->context, start, length, acc, 0); 2381 if (IS_ERR(region)) { 2382 return (struct ib_mr *)region; 2383 } 2384 2385 nes_debug(NES_DBG_MR, "User base = 0x%lX, Virt base = 0x%lX, length = %u," 2386 " offset = %u, page size = %u.\n", 2387 (unsigned long int)start, (unsigned long int)virt, (u32)length, 2388 region->offset, region->page_size); 2389 2390 skip_pages = ((u32)region->offset) >> 12; 2391 2392 if (ib_copy_from_udata(&req, udata, sizeof(req))) 2393 return ERR_PTR(-EFAULT); 2394 nes_debug(NES_DBG_MR, "Memory Registration type = %08X.\n", req.reg_type); 2395 2396 switch (req.reg_type) { 2397 case IWNES_MEMREG_TYPE_MEM: 2398 pbl_depth = 0; 2399 region_length = 0; 2400 vpbl.pbl_vbase = NULL; 2401 root_vpbl.pbl_vbase = NULL; 2402 root_vpbl.pbl_pbase = 0; 2403 2404 get_random_bytes(&next_stag_index, sizeof(next_stag_index)); 2405 stag_key = (u8)next_stag_index; 2406 2407 driver_key = next_stag_index & 0x70000000; 2408 2409 next_stag_index >>= 8; 2410 next_stag_index %= nesadapter->max_mr; 2411 2412 err = nes_alloc_resource(nesadapter, nesadapter->allocated_mrs, 2413 nesadapter->max_mr, &stag_index, &next_stag_index); 2414 if (err) { 2415 ib_umem_release(region); 2416 return ERR_PTR(err); 2417 } 2418 2419 nesmr = kzalloc(sizeof(*nesmr), GFP_KERNEL); 2420 if (!nesmr) { 2421 ib_umem_release(region); 2422 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); 2423 return ERR_PTR(-ENOMEM); 2424 } 2425 nesmr->region = region; 2426 2427 list_for_each_entry(chunk, ®ion->chunk_list, list) { 2428 nes_debug(NES_DBG_MR, "Chunk: nents = %u, nmap = %u .\n", 2429 chunk->nents, chunk->nmap); 2430 for (nmap_index = 0; nmap_index < chunk->nmap; ++nmap_index) { 2431 if (sg_dma_address(&chunk->page_list[nmap_index]) & ~PAGE_MASK) { 2432 ib_umem_release(region); 2433 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); 2434 nes_debug(NES_DBG_MR, "Unaligned Memory Buffer: 0x%x\n", 2435 (unsigned int) sg_dma_address(&chunk->page_list[nmap_index])); 2436 ibmr = ERR_PTR(-EINVAL); 2437 kfree(nesmr); 2438 goto reg_user_mr_err; 2439 } 2440 2441 if (!sg_dma_len(&chunk->page_list[nmap_index])) { 2442 ib_umem_release(region); 2443 nes_free_resource(nesadapter, nesadapter->allocated_mrs, 2444 stag_index); 2445 nes_debug(NES_DBG_MR, "Invalid Buffer Size\n"); 2446 ibmr = ERR_PTR(-EINVAL); 2447 kfree(nesmr); 2448 goto reg_user_mr_err; 2449 } 2450 2451 region_length += sg_dma_len(&chunk->page_list[nmap_index]); 2452 chunk_pages = sg_dma_len(&chunk->page_list[nmap_index]) >> 12; 2453 region_length -= skip_pages << 12; 2454 for (page_index=skip_pages; page_index < chunk_pages; page_index++) { 2455 skip_pages = 0; 2456 if ((page_count!=0)&&(page_count<<12)-(region->offset&(4096-1))>=region->length) 2457 goto enough_pages; 2458 if ((page_count&0x01FF) == 0) { 2459 if (page_count >= 1024 * 512) { 2460 ib_umem_release(region); 2461 nes_free_resource(nesadapter, 2462 nesadapter->allocated_mrs, stag_index); 2463 kfree(nesmr); 2464 ibmr = ERR_PTR(-E2BIG); 2465 goto reg_user_mr_err; 2466 } 2467 if (root_pbl_index == 1) { 2468 root_vpbl.pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 2469 8192, &root_vpbl.pbl_pbase); 2470 nes_debug(NES_DBG_MR, "Allocating root PBL, va = %p, pa = 0x%08X\n", 2471 root_vpbl.pbl_vbase, (unsigned int)root_vpbl.pbl_pbase); 2472 if (!root_vpbl.pbl_vbase) { 2473 ib_umem_release(region); 2474 pci_free_consistent(nesdev->pcidev, 4096, vpbl.pbl_vbase, 2475 vpbl.pbl_pbase); 2476 nes_free_resource(nesadapter, nesadapter->allocated_mrs, 2477 stag_index); 2478 kfree(nesmr); 2479 ibmr = ERR_PTR(-ENOMEM); 2480 goto reg_user_mr_err; 2481 } 2482 root_vpbl.leaf_vpbl = kzalloc(sizeof(*root_vpbl.leaf_vpbl)*1024, 2483 GFP_KERNEL); 2484 if (!root_vpbl.leaf_vpbl) { 2485 ib_umem_release(region); 2486 pci_free_consistent(nesdev->pcidev, 8192, root_vpbl.pbl_vbase, 2487 root_vpbl.pbl_pbase); 2488 pci_free_consistent(nesdev->pcidev, 4096, vpbl.pbl_vbase, 2489 vpbl.pbl_pbase); 2490 nes_free_resource(nesadapter, nesadapter->allocated_mrs, 2491 stag_index); 2492 kfree(nesmr); 2493 ibmr = ERR_PTR(-ENOMEM); 2494 goto reg_user_mr_err; 2495 } 2496 root_vpbl.pbl_vbase[0].pa_low = 2497 cpu_to_le32((u32)vpbl.pbl_pbase); 2498 root_vpbl.pbl_vbase[0].pa_high = 2499 cpu_to_le32((u32)((((u64)vpbl.pbl_pbase) >> 32))); 2500 root_vpbl.leaf_vpbl[0] = vpbl; 2501 } 2502 vpbl.pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 4096, 2503 &vpbl.pbl_pbase); 2504 nes_debug(NES_DBG_MR, "Allocating leaf PBL, va = %p, pa = 0x%08X\n", 2505 vpbl.pbl_vbase, (unsigned int)vpbl.pbl_pbase); 2506 if (!vpbl.pbl_vbase) { 2507 ib_umem_release(region); 2508 nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); 2509 ibmr = ERR_PTR(-ENOMEM); 2510 kfree(nesmr); 2511 goto reg_user_mr_err; 2512 } 2513 if (1 <= root_pbl_index) { 2514 root_vpbl.pbl_vbase[root_pbl_index].pa_low = 2515 cpu_to_le32((u32)vpbl.pbl_pbase); 2516 root_vpbl.pbl_vbase[root_pbl_index].pa_high = 2517 cpu_to_le32((u32)((((u64)vpbl.pbl_pbase)>>32))); 2518 root_vpbl.leaf_vpbl[root_pbl_index] = vpbl; 2519 } 2520 root_pbl_index++; 2521 cur_pbl_index = 0; 2522 } 2523 if (single_page) { 2524 if (page_count != 0) { 2525 if ((last_dma_addr+4096) != 2526 (sg_dma_address(&chunk->page_list[nmap_index])+ 2527 (page_index*4096))) 2528 single_page = 0; 2529 last_dma_addr = sg_dma_address(&chunk->page_list[nmap_index])+ 2530 (page_index*4096); 2531 } else { 2532 first_dma_addr = sg_dma_address(&chunk->page_list[nmap_index])+ 2533 (page_index*4096); 2534 last_dma_addr = first_dma_addr; 2535 } 2536 } 2537 2538 vpbl.pbl_vbase[cur_pbl_index].pa_low = 2539 cpu_to_le32((u32)(sg_dma_address(&chunk->page_list[nmap_index])+ 2540 (page_index*4096))); 2541 vpbl.pbl_vbase[cur_pbl_index].pa_high = 2542 cpu_to_le32((u32)((((u64)(sg_dma_address(&chunk->page_list[nmap_index])+ 2543 (page_index*4096))) >> 32))); 2544 cur_pbl_index++; 2545 page_count++; 2546 } 2547 } 2548 } 2549 enough_pages: 2550 nes_debug(NES_DBG_MR, "calculating stag, stag_index=0x%08x, driver_key=0x%08x," 2551 " stag_key=0x%08x\n", 2552 stag_index, driver_key, stag_key); 2553 stag = stag_index << 8; 2554 stag |= driver_key; 2555 stag += (u32)stag_key; 2556 if (stag == 0) { 2557 stag = 1; 2558 } 2559 2560 iova_start = virt; 2561 /* Make the leaf PBL the root if only one PBL */ 2562 if (root_pbl_index == 1) { 2563 root_vpbl.pbl_pbase = vpbl.pbl_pbase; 2564 } 2565 2566 if (single_page) { 2567 pbl_count = 0; 2568 } else { 2569 pbl_count = root_pbl_index; 2570 first_dma_addr = 0; 2571 } 2572 nes_debug(NES_DBG_MR, "Registering STag 0x%08X, VA = 0x%08X, length = 0x%08X," 2573 " index = 0x%08X, region->length=0x%08llx, pbl_count = %u\n", 2574 stag, (unsigned int)iova_start, 2575 (unsigned int)region_length, stag_index, 2576 (unsigned long long)region->length, pbl_count); 2577 ret = nes_reg_mr( nesdev, nespd, stag, region->length, &root_vpbl, 2578 first_dma_addr, pbl_count, (u16)cur_pbl_index, acc, &iova_start); 2579 2580 nes_debug(NES_DBG_MR, "ret=%d\n", ret); 2581 2582 if (ret == 0) { 2583 nesmr->ibmr.rkey = stag; 2584 nesmr->ibmr.lkey = stag; 2585 nesmr->mode = IWNES_MEMREG_TYPE_MEM; 2586 ibmr = &nesmr->ibmr; 2587 nesmr->pbl_4k = ((pbl_count > 1) || (cur_pbl_index > 32)) ? 1 : 0; 2588 nesmr->pbls_used = pbl_count; 2589 if (pbl_count > 1) { 2590 nesmr->pbls_used++; 2591 } 2592 } else { 2593 ib_umem_release(region); 2594 kfree(nesmr); 2595 ibmr = ERR_PTR(-ENOMEM); 2596 } 2597 2598 reg_user_mr_err: 2599 /* free the resources */ 2600 if (root_pbl_index == 1) { 2601 pci_free_consistent(nesdev->pcidev, 4096, vpbl.pbl_vbase, 2602 vpbl.pbl_pbase); 2603 } else { 2604 for (page_index=0; page_index<root_pbl_index; page_index++) { 2605 pci_free_consistent(nesdev->pcidev, 4096, 2606 root_vpbl.leaf_vpbl[page_index].pbl_vbase, 2607 root_vpbl.leaf_vpbl[page_index].pbl_pbase); 2608 } 2609 kfree(root_vpbl.leaf_vpbl); 2610 pci_free_consistent(nesdev->pcidev, 8192, root_vpbl.pbl_vbase, 2611 root_vpbl.pbl_pbase); 2612 } 2613 2614 nes_debug(NES_DBG_MR, "Leaving, ibmr=%p", ibmr); 2615 2616 return ibmr; 2617 break; 2618 case IWNES_MEMREG_TYPE_QP: 2619 case IWNES_MEMREG_TYPE_CQ: 2620 nespbl = kzalloc(sizeof(*nespbl), GFP_KERNEL); 2621 if (!nespbl) { 2622 nes_debug(NES_DBG_MR, "Unable to allocate PBL\n"); 2623 ib_umem_release(region); 2624 return ERR_PTR(-ENOMEM); 2625 } 2626 nesmr = kzalloc(sizeof(*nesmr), GFP_KERNEL); 2627 if (!nesmr) { 2628 ib_umem_release(region); 2629 kfree(nespbl); 2630 nes_debug(NES_DBG_MR, "Unable to allocate nesmr\n"); 2631 return ERR_PTR(-ENOMEM); 2632 } 2633 nesmr->region = region; 2634 nes_ucontext = to_nesucontext(pd->uobject->context); 2635 pbl_depth = region->length >> 12; 2636 pbl_depth += (region->length & (4096-1)) ? 1 : 0; 2637 nespbl->pbl_size = pbl_depth*sizeof(u64); 2638 if (req.reg_type == IWNES_MEMREG_TYPE_QP) { 2639 nes_debug(NES_DBG_MR, "Attempting to allocate QP PBL memory"); 2640 } else { 2641 nes_debug(NES_DBG_MR, "Attempting to allocate CP PBL memory"); 2642 } 2643 2644 nes_debug(NES_DBG_MR, " %u bytes, %u entries.\n", 2645 nespbl->pbl_size, pbl_depth); 2646 pbl = pci_alloc_consistent(nesdev->pcidev, nespbl->pbl_size, 2647 &nespbl->pbl_pbase); 2648 if (!pbl) { 2649 ib_umem_release(region); 2650 kfree(nesmr); 2651 kfree(nespbl); 2652 nes_debug(NES_DBG_MR, "Unable to allocate PBL memory\n"); 2653 return ERR_PTR(-ENOMEM); 2654 } 2655 2656 nespbl->pbl_vbase = (u64 *)pbl; 2657 nespbl->user_base = start; 2658 nes_debug(NES_DBG_MR, "Allocated PBL memory, %u bytes, pbl_pbase=%lx," 2659 " pbl_vbase=%p user_base=0x%lx\n", 2660 nespbl->pbl_size, (unsigned long) nespbl->pbl_pbase, 2661 (void *) nespbl->pbl_vbase, nespbl->user_base); 2662 2663 list_for_each_entry(chunk, ®ion->chunk_list, list) { 2664 for (nmap_index = 0; nmap_index < chunk->nmap; ++nmap_index) { 2665 chunk_pages = sg_dma_len(&chunk->page_list[nmap_index]) >> 12; 2666 chunk_pages += (sg_dma_len(&chunk->page_list[nmap_index]) & (4096-1)) ? 1 : 0; 2667 nespbl->page = sg_page(&chunk->page_list[0]); 2668 for (page_index=0; page_index<chunk_pages; page_index++) { 2669 ((__le32 *)pbl)[0] = cpu_to_le32((u32) 2670 (sg_dma_address(&chunk->page_list[nmap_index])+ 2671 (page_index*4096))); 2672 ((__le32 *)pbl)[1] = cpu_to_le32(((u64) 2673 (sg_dma_address(&chunk->page_list[nmap_index])+ 2674 (page_index*4096)))>>32); 2675 nes_debug(NES_DBG_MR, "pbl=%p, *pbl=0x%016llx, 0x%08x%08x\n", pbl, 2676 (unsigned long long)*pbl, 2677 le32_to_cpu(((__le32 *)pbl)[1]), le32_to_cpu(((__le32 *)pbl)[0])); 2678 pbl++; 2679 } 2680 } 2681 } 2682 if (req.reg_type == IWNES_MEMREG_TYPE_QP) { 2683 list_add_tail(&nespbl->list, &nes_ucontext->qp_reg_mem_list); 2684 } else { 2685 list_add_tail(&nespbl->list, &nes_ucontext->cq_reg_mem_list); 2686 } 2687 nesmr->ibmr.rkey = -1; 2688 nesmr->ibmr.lkey = -1; 2689 nesmr->mode = req.reg_type; 2690 return &nesmr->ibmr; 2691 break; 2692 } 2693 2694 return ERR_PTR(-ENOSYS); 2695} 2696 2697 2698/** 2699 * nes_dereg_mr 2700 */ 2701static int nes_dereg_mr(struct ib_mr *ib_mr) 2702{ 2703 struct nes_mr *nesmr = to_nesmr(ib_mr); 2704 struct nes_vnic *nesvnic = to_nesvnic(ib_mr->device); 2705 struct nes_device *nesdev = nesvnic->nesdev; 2706 struct nes_adapter *nesadapter = nesdev->nesadapter; 2707 struct nes_hw_cqp_wqe *cqp_wqe; 2708 struct nes_cqp_request *cqp_request; 2709 unsigned long flags; 2710 int ret; 2711 u16 major_code; 2712 u16 minor_code; 2713 2714 if (nesmr->region) { 2715 ib_umem_release(nesmr->region); 2716 } 2717 if (nesmr->mode != IWNES_MEMREG_TYPE_MEM) { 2718 kfree(nesmr); 2719 return 0; 2720 } 2721 2722 /* Deallocate the region with the adapter */ 2723 2724 cqp_request = nes_get_cqp_request(nesdev); 2725 if (cqp_request == NULL) { 2726 nes_debug(NES_DBG_MR, "Failed to get a cqp_request.\n"); 2727 return -ENOMEM; 2728 } 2729 cqp_request->waiting = 1; 2730 cqp_wqe = &cqp_request->cqp_wqe; 2731 2732 spin_lock_irqsave(&nesadapter->pbl_lock, flags); 2733 if (nesmr->pbls_used != 0) { 2734 if (nesmr->pbl_4k) { 2735 nesadapter->free_4kpbl += nesmr->pbls_used; 2736 if (nesadapter->free_4kpbl > nesadapter->max_4kpbl) { 2737 printk(KERN_ERR PFX "free 4KB PBLs(%u) has exceeded the max(%u)\n", 2738 nesadapter->free_4kpbl, nesadapter->max_4kpbl); 2739 } 2740 } else { 2741 nesadapter->free_256pbl += nesmr->pbls_used; 2742 if (nesadapter->free_256pbl > nesadapter->max_256pbl) { 2743 printk(KERN_ERR PFX "free 256B PBLs(%u) has exceeded the max(%u)\n", 2744 nesadapter->free_256pbl, nesadapter->max_256pbl); 2745 } 2746 } 2747 } 2748 2749 spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); 2750 nes_fill_init_cqp_wqe(cqp_wqe, nesdev); 2751 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, 2752 NES_CQP_DEALLOCATE_STAG | NES_CQP_STAG_VA_TO | 2753 NES_CQP_STAG_DEALLOC_PBLS | NES_CQP_STAG_MR); 2754 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_STAG_IDX, ib_mr->rkey); 2755 2756 atomic_set(&cqp_request->refcount, 2); 2757 nes_post_cqp_request(nesdev, cqp_request, NES_CQP_REQUEST_RING_DOORBELL); 2758 2759 /* Wait for CQP */ 2760 nes_debug(NES_DBG_MR, "Waiting for deallocate STag 0x%08X completed\n", ib_mr->rkey); 2761 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0), 2762 NES_EVENT_TIMEOUT); 2763 nes_debug(NES_DBG_MR, "Deallocate STag 0x%08X completed, wait_event_timeout ret = %u," 2764 " CQP Major:Minor codes = 0x%04X:0x%04X\n", 2765 ib_mr->rkey, ret, cqp_request->major_code, cqp_request->minor_code); 2766 2767 nes_free_resource(nesadapter, nesadapter->allocated_mrs, 2768 (ib_mr->rkey & 0x0fffff00) >> 8); 2769 2770 kfree(nesmr); 2771 2772 major_code = cqp_request->major_code; 2773 minor_code = cqp_request->minor_code; 2774 if (atomic_dec_and_test(&cqp_request->refcount)) { 2775 if (cqp_request->dynamic) { 2776 kfree(cqp_request); 2777 } else { 2778 spin_lock_irqsave(&nesdev->cqp.lock, flags); 2779 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs); 2780 spin_unlock_irqrestore(&nesdev->cqp.lock, flags); 2781 } 2782 } 2783 if (!ret) { 2784 nes_debug(NES_DBG_MR, "Timeout waiting to destroy STag," 2785 " ib_mr=%p, rkey = 0x%08X\n", 2786 ib_mr, ib_mr->rkey); 2787 return -ETIME; 2788 } else if (major_code) { 2789 nes_debug(NES_DBG_MR, "Error (0x%04X:0x%04X) while attempting" 2790 " to destroy STag, ib_mr=%p, rkey = 0x%08X\n", 2791 major_code, minor_code, ib_mr, ib_mr->rkey); 2792 return -EIO; 2793 } else 2794 return 0; 2795} 2796 2797 2798/** 2799 * show_rev 2800 */ 2801static ssize_t show_rev(struct device *dev, struct device_attribute *attr, 2802 char *buf) 2803{ 2804 struct nes_ib_device *nesibdev = 2805 container_of(dev, struct nes_ib_device, ibdev.dev); 2806 struct nes_vnic *nesvnic = nesibdev->nesvnic; 2807 2808 nes_debug(NES_DBG_INIT, "\n"); 2809 return sprintf(buf, "%x\n", nesvnic->nesdev->nesadapter->hw_rev); 2810} 2811 2812 2813/** 2814 * show_fw_ver 2815 */ 2816static ssize_t show_fw_ver(struct device *dev, struct device_attribute *attr, 2817 char *buf) 2818{ 2819 struct nes_ib_device *nesibdev = 2820 container_of(dev, struct nes_ib_device, ibdev.dev); 2821 struct nes_vnic *nesvnic = nesibdev->nesvnic; 2822 2823 nes_debug(NES_DBG_INIT, "\n"); 2824 return sprintf(buf, "%x.%x.%x\n", 2825 (int)(nesvnic->nesdev->nesadapter->fw_ver >> 32), 2826 (int)(nesvnic->nesdev->nesadapter->fw_ver >> 16) & 0xffff, 2827 (int)(nesvnic->nesdev->nesadapter->fw_ver & 0xffff)); 2828} 2829 2830 2831/** 2832 * show_hca 2833 */ 2834static ssize_t show_hca(struct device *dev, struct device_attribute *attr, 2835 char *buf) 2836{ 2837 nes_debug(NES_DBG_INIT, "\n"); 2838 return sprintf(buf, "NES020\n"); 2839} 2840 2841 2842/** 2843 * show_board 2844 */ 2845static ssize_t show_board(struct device *dev, struct device_attribute *attr, 2846 char *buf) 2847{ 2848 nes_debug(NES_DBG_INIT, "\n"); 2849 return sprintf(buf, "%.*s\n", 32, "NES020 Board ID"); 2850} 2851 2852 2853static DEVICE_ATTR(hw_rev, S_IRUGO, show_rev, NULL); 2854static DEVICE_ATTR(fw_ver, S_IRUGO, show_fw_ver, NULL); 2855static DEVICE_ATTR(hca_type, S_IRUGO, show_hca, NULL); 2856static DEVICE_ATTR(board_id, S_IRUGO, show_board, NULL); 2857 2858static struct device_attribute *nes_dev_attributes[] = { 2859 &dev_attr_hw_rev, 2860 &dev_attr_fw_ver, 2861 &dev_attr_hca_type, 2862 &dev_attr_board_id 2863}; 2864 2865 2866/** 2867 * nes_query_qp 2868 */ 2869static int nes_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 2870 int attr_mask, struct ib_qp_init_attr *init_attr) 2871{ 2872 struct nes_qp *nesqp = to_nesqp(ibqp); 2873 2874 nes_debug(NES_DBG_QP, "\n"); 2875 2876 attr->qp_access_flags = 0; 2877 attr->cap.max_send_wr = nesqp->hwqp.sq_size; 2878 attr->cap.max_recv_wr = nesqp->hwqp.rq_size; 2879 attr->cap.max_recv_sge = 1; 2880 if (nes_drv_opt & NES_DRV_OPT_NO_INLINE_DATA) { 2881 init_attr->cap.max_inline_data = 0; 2882 } else { 2883 init_attr->cap.max_inline_data = 64; 2884 } 2885 2886 init_attr->event_handler = nesqp->ibqp.event_handler; 2887 init_attr->qp_context = nesqp->ibqp.qp_context; 2888 init_attr->send_cq = nesqp->ibqp.send_cq; 2889 init_attr->recv_cq = nesqp->ibqp.recv_cq; 2890 init_attr->srq = nesqp->ibqp.srq = nesqp->ibqp.srq; 2891 init_attr->cap = attr->cap; 2892 2893 return 0; 2894} 2895 2896 2897/** 2898 * nes_hw_modify_qp 2899 */ 2900int nes_hw_modify_qp(struct nes_device *nesdev, struct nes_qp *nesqp, 2901 u32 next_iwarp_state, u32 wait_completion) 2902{ 2903 struct nes_hw_cqp_wqe *cqp_wqe; 2904 /* struct iw_cm_id *cm_id = nesqp->cm_id; */ 2905 /* struct iw_cm_event cm_event; */ 2906 struct nes_cqp_request *cqp_request; 2907 unsigned long flags; 2908 int ret; 2909 u16 major_code; 2910 2911 nes_debug(NES_DBG_MOD_QP, "QP%u, refcount=%d\n", 2912 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount)); 2913 2914 cqp_request = nes_get_cqp_request(nesdev); 2915 if (cqp_request == NULL) { 2916 nes_debug(NES_DBG_MOD_QP, "Failed to get a cqp_request.\n"); 2917 return -ENOMEM; 2918 } 2919 if (wait_completion) { 2920 cqp_request->waiting = 1; 2921 } else { 2922 cqp_request->waiting = 0; 2923 } 2924 cqp_wqe = &cqp_request->cqp_wqe; 2925 2926 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, 2927 NES_CQP_MODIFY_QP | NES_CQP_QP_TYPE_IWARP | next_iwarp_state); 2928 nes_debug(NES_DBG_MOD_QP, "using next_iwarp_state=%08x, wqe_words=%08x\n", 2929 next_iwarp_state, le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX])); 2930 nes_fill_init_cqp_wqe(cqp_wqe, nesdev); 2931 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, nesqp->hwqp.qp_id); 2932 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, (u64)nesqp->nesqp_context_pbase); 2933 2934 atomic_set(&cqp_request->refcount, 2); 2935 nes_post_cqp_request(nesdev, cqp_request, NES_CQP_REQUEST_RING_DOORBELL); 2936 2937 /* Wait for CQP */ 2938 if (wait_completion) { 2939 /* nes_debug(NES_DBG_MOD_QP, "Waiting for modify iWARP QP%u to complete.\n", 2940 nesqp->hwqp.qp_id); */ 2941 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0), 2942 NES_EVENT_TIMEOUT); 2943 nes_debug(NES_DBG_MOD_QP, "Modify iwarp QP%u completed, wait_event_timeout ret=%u, " 2944 "CQP Major:Minor codes = 0x%04X:0x%04X.\n", 2945 nesqp->hwqp.qp_id, ret, cqp_request->major_code, cqp_request->minor_code); 2946 major_code = cqp_request->major_code; 2947 if (major_code) { 2948 nes_debug(NES_DBG_MOD_QP, "Modify iwarp QP%u failed" 2949 "CQP Major:Minor codes = 0x%04X:0x%04X, intended next state = 0x%08X.\n", 2950 nesqp->hwqp.qp_id, cqp_request->major_code, 2951 cqp_request->minor_code, next_iwarp_state); 2952 } 2953 if (atomic_dec_and_test(&cqp_request->refcount)) { 2954 if (cqp_request->dynamic) { 2955 kfree(cqp_request); 2956 } else { 2957 spin_lock_irqsave(&nesdev->cqp.lock, flags); 2958 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs); 2959 spin_unlock_irqrestore(&nesdev->cqp.lock, flags); 2960 } 2961 } 2962 if (!ret) 2963 return -ETIME; 2964 else if (major_code) 2965 return -EIO; 2966 else 2967 return 0; 2968 } else { 2969 return 0; 2970 } 2971} 2972 2973 2974/** 2975 * nes_modify_qp 2976 */ 2977int nes_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 2978 int attr_mask, struct ib_udata *udata) 2979{ 2980 struct nes_qp *nesqp = to_nesqp(ibqp); 2981 struct nes_vnic *nesvnic = to_nesvnic(ibqp->device); 2982 struct nes_device *nesdev = nesvnic->nesdev; 2983 /* u32 cqp_head; */ 2984 /* u32 counter; */ 2985 u32 next_iwarp_state = 0; 2986 int err; 2987 unsigned long qplockflags; 2988 int ret; 2989 u16 original_last_aeq; 2990 u8 issue_modify_qp = 0; 2991 u8 issue_disconnect = 0; 2992 u8 dont_wait = 0; 2993 2994 nes_debug(NES_DBG_MOD_QP, "QP%u: QP State=%u, cur QP State=%u," 2995 " iwarp_state=0x%X, refcount=%d\n", 2996 nesqp->hwqp.qp_id, attr->qp_state, nesqp->ibqp_state, 2997 nesqp->iwarp_state, atomic_read(&nesqp->refcount)); 2998 2999 nes_add_ref(&nesqp->ibqp); 3000 spin_lock_irqsave(&nesqp->lock, qplockflags); 3001 3002 nes_debug(NES_DBG_MOD_QP, "QP%u: hw_iwarp_state=0x%X, hw_tcp_state=0x%X," 3003 " QP Access Flags=0x%X, attr_mask = 0x%0x\n", 3004 nesqp->hwqp.qp_id, nesqp->hw_iwarp_state, 3005 nesqp->hw_tcp_state, attr->qp_access_flags, attr_mask); 3006 3007 if (attr_mask & IB_QP_STATE) { 3008 switch (attr->qp_state) { 3009 case IB_QPS_INIT: 3010 nes_debug(NES_DBG_MOD_QP, "QP%u: new state = init\n", 3011 nesqp->hwqp.qp_id); 3012 if (nesqp->iwarp_state > (u32)NES_CQP_QP_IWARP_STATE_IDLE) { 3013 spin_unlock_irqrestore(&nesqp->lock, qplockflags); 3014 nes_rem_ref(&nesqp->ibqp); 3015 return -EINVAL; 3016 } 3017 next_iwarp_state = NES_CQP_QP_IWARP_STATE_IDLE; 3018 issue_modify_qp = 1; 3019 break; 3020 case IB_QPS_RTR: 3021 nes_debug(NES_DBG_MOD_QP, "QP%u: new state = rtr\n", 3022 nesqp->hwqp.qp_id); 3023 if (nesqp->iwarp_state>(u32)NES_CQP_QP_IWARP_STATE_IDLE) { 3024 spin_unlock_irqrestore(&nesqp->lock, qplockflags); 3025 nes_rem_ref(&nesqp->ibqp); 3026 return -EINVAL; 3027 } 3028 next_iwarp_state = NES_CQP_QP_IWARP_STATE_IDLE; 3029 issue_modify_qp = 1; 3030 break; 3031 case IB_QPS_RTS: 3032 nes_debug(NES_DBG_MOD_QP, "QP%u: new state = rts\n", 3033 nesqp->hwqp.qp_id); 3034 if (nesqp->iwarp_state>(u32)NES_CQP_QP_IWARP_STATE_RTS) { 3035 spin_unlock_irqrestore(&nesqp->lock, qplockflags); 3036 nes_rem_ref(&nesqp->ibqp); 3037 return -EINVAL; 3038 } 3039 if (nesqp->cm_id == NULL) { 3040 nes_debug(NES_DBG_MOD_QP, "QP%u: Failing attempt to move QP to RTS without a CM_ID. \n", 3041 nesqp->hwqp.qp_id ); 3042 spin_unlock_irqrestore(&nesqp->lock, qplockflags); 3043 nes_rem_ref(&nesqp->ibqp); 3044 return -EINVAL; 3045 } 3046 next_iwarp_state = NES_CQP_QP_IWARP_STATE_RTS; 3047 if (nesqp->iwarp_state != NES_CQP_QP_IWARP_STATE_RTS) 3048 next_iwarp_state |= NES_CQP_QP_CONTEXT_VALID | 3049 NES_CQP_QP_ARP_VALID | NES_CQP_QP_ORD_VALID; 3050 issue_modify_qp = 1; 3051 nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_ESTABLISHED; 3052 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_RTS; 3053 nesqp->hte_added = 1; 3054 break; 3055 case IB_QPS_SQD: 3056 issue_modify_qp = 1; 3057 nes_debug(NES_DBG_MOD_QP, "QP%u: new state=closing. SQ head=%u, SQ tail=%u\n", 3058 nesqp->hwqp.qp_id, nesqp->hwqp.sq_head, nesqp->hwqp.sq_tail); 3059 if (nesqp->iwarp_state == (u32)NES_CQP_QP_IWARP_STATE_CLOSING) { 3060 spin_unlock_irqrestore(&nesqp->lock, qplockflags); 3061 nes_rem_ref(&nesqp->ibqp); 3062 return 0; 3063 } else { 3064 if (nesqp->iwarp_state > (u32)NES_CQP_QP_IWARP_STATE_CLOSING) { 3065 nes_debug(NES_DBG_MOD_QP, "QP%u: State change to closing" 3066 " ignored due to current iWARP state\n", 3067 nesqp->hwqp.qp_id); 3068 spin_unlock_irqrestore(&nesqp->lock, qplockflags); 3069 nes_rem_ref(&nesqp->ibqp); 3070 return -EINVAL; 3071 } 3072 if (nesqp->hw_iwarp_state != NES_AEQE_IWARP_STATE_RTS) { 3073 nes_debug(NES_DBG_MOD_QP, "QP%u: State change to closing" 3074 " already done based on hw state.\n", 3075 nesqp->hwqp.qp_id); 3076 issue_modify_qp = 0; 3077 nesqp->in_disconnect = 0; 3078 } 3079 switch (nesqp->hw_iwarp_state) { 3080 case NES_AEQE_IWARP_STATE_CLOSING: 3081 next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING; 3082 case NES_AEQE_IWARP_STATE_TERMINATE: 3083 next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE; 3084 break; 3085 case NES_AEQE_IWARP_STATE_ERROR: 3086 next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR; 3087 break; 3088 default: 3089 next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING; 3090 nesqp->in_disconnect = 1; 3091 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING; 3092 break; 3093 } 3094 } 3095 break; 3096 case IB_QPS_SQE: 3097 nes_debug(NES_DBG_MOD_QP, "QP%u: new state = terminate\n", 3098 nesqp->hwqp.qp_id); 3099 if (nesqp->iwarp_state>=(u32)NES_CQP_QP_IWARP_STATE_TERMINATE) { 3100 spin_unlock_irqrestore(&nesqp->lock, qplockflags); 3101 nes_rem_ref(&nesqp->ibqp); 3102 return -EINVAL; 3103 } 3104 /* next_iwarp_state = (NES_CQP_QP_IWARP_STATE_TERMINATE | 0x02000000); */ 3105 next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE; 3106 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_TERMINATE; 3107 issue_modify_qp = 1; 3108 nesqp->in_disconnect = 1; 3109 break; 3110 case IB_QPS_ERR: 3111 case IB_QPS_RESET: 3112 if (nesqp->iwarp_state == (u32)NES_CQP_QP_IWARP_STATE_ERROR) { 3113 spin_unlock_irqrestore(&nesqp->lock, qplockflags); 3114 nes_rem_ref(&nesqp->ibqp); 3115 return -EINVAL; 3116 } 3117 nes_debug(NES_DBG_MOD_QP, "QP%u: new state = error\n", 3118 nesqp->hwqp.qp_id); 3119 next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR; 3120 /* next_iwarp_state = (NES_CQP_QP_IWARP_STATE_TERMINATE | 0x02000000); */ 3121 if (nesqp->hte_added) { 3122 nes_debug(NES_DBG_MOD_QP, "set CQP_QP_DEL_HTE\n"); 3123 next_iwarp_state |= NES_CQP_QP_DEL_HTE; 3124 nesqp->hte_added = 0; 3125 } 3126 if ((nesqp->hw_tcp_state > NES_AEQE_TCP_STATE_CLOSED) && 3127 (nesqp->hw_tcp_state != NES_AEQE_TCP_STATE_TIME_WAIT)) { 3128 next_iwarp_state |= NES_CQP_QP_RESET; 3129 nesqp->in_disconnect = 1; 3130 } else { 3131 nes_debug(NES_DBG_MOD_QP, "QP%u NOT setting NES_CQP_QP_RESET since TCP state = %u\n", 3132 nesqp->hwqp.qp_id, nesqp->hw_tcp_state); 3133 dont_wait = 1; 3134 } 3135 issue_modify_qp = 1; 3136 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_ERROR; 3137 break; 3138 default: 3139 spin_unlock_irqrestore(&nesqp->lock, qplockflags); 3140 nes_rem_ref(&nesqp->ibqp); 3141 return -EINVAL; 3142 break; 3143 } 3144 3145 nesqp->ibqp_state = attr->qp_state; 3146 if (((nesqp->iwarp_state & NES_CQP_QP_IWARP_STATE_MASK) == 3147 (u32)NES_CQP_QP_IWARP_STATE_RTS) && 3148 ((next_iwarp_state & NES_CQP_QP_IWARP_STATE_MASK) > 3149 (u32)NES_CQP_QP_IWARP_STATE_RTS)) { 3150 nesqp->iwarp_state = next_iwarp_state & NES_CQP_QP_IWARP_STATE_MASK; 3151 nes_debug(NES_DBG_MOD_QP, "Change nesqp->iwarp_state=%08x\n", 3152 nesqp->iwarp_state); 3153 issue_disconnect = 1; 3154 } else { 3155 nesqp->iwarp_state = next_iwarp_state & NES_CQP_QP_IWARP_STATE_MASK; 3156 nes_debug(NES_DBG_MOD_QP, "Change nesqp->iwarp_state=%08x\n", 3157 nesqp->iwarp_state); 3158 } 3159 } 3160 3161 if (attr_mask & IB_QP_ACCESS_FLAGS) { 3162 if (attr->qp_access_flags & IB_ACCESS_LOCAL_WRITE) { 3163 nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_RDMA_WRITE_EN | 3164 NES_QPCONTEXT_MISC_RDMA_READ_EN); 3165 issue_modify_qp = 1; 3166 } 3167 if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE) { 3168 nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_RDMA_WRITE_EN); 3169 issue_modify_qp = 1; 3170 } 3171 if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ) { 3172 nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_RDMA_READ_EN); 3173 issue_modify_qp = 1; 3174 } 3175 if (attr->qp_access_flags & IB_ACCESS_MW_BIND) { 3176 nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_WBIND_EN); 3177 issue_modify_qp = 1; 3178 } 3179 3180 if (nesqp->user_mode) { 3181 nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_RDMA_WRITE_EN | 3182 NES_QPCONTEXT_MISC_RDMA_READ_EN); 3183 issue_modify_qp = 1; 3184 } 3185 } 3186 3187 original_last_aeq = nesqp->last_aeq; 3188 spin_unlock_irqrestore(&nesqp->lock, qplockflags); 3189 3190 nes_debug(NES_DBG_MOD_QP, "issue_modify_qp=%u\n", issue_modify_qp); 3191 3192 ret = 0; 3193 3194 3195 if (issue_modify_qp) { 3196 nes_debug(NES_DBG_MOD_QP, "call nes_hw_modify_qp\n"); 3197 ret = nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 1); 3198 if (ret) 3199 nes_debug(NES_DBG_MOD_QP, "nes_hw_modify_qp (next_iwarp_state = 0x%08X)" 3200 " failed for QP%u.\n", 3201 next_iwarp_state, nesqp->hwqp.qp_id); 3202 3203 } 3204 3205 if ((issue_modify_qp) && (nesqp->ibqp_state > IB_QPS_RTS)) { 3206 nes_debug(NES_DBG_MOD_QP, "QP%u Issued ModifyQP refcount (%d)," 3207 " original_last_aeq = 0x%04X. last_aeq = 0x%04X.\n", 3208 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount), 3209 original_last_aeq, nesqp->last_aeq); 3210 if ((!ret) || 3211 ((original_last_aeq != NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) && 3212 (ret))) { 3213 if (dont_wait) { 3214 if (nesqp->cm_id && nesqp->hw_tcp_state != 0) { 3215 nes_debug(NES_DBG_MOD_QP, "QP%u Queuing fake disconnect for QP refcount (%d)," 3216 " original_last_aeq = 0x%04X. last_aeq = 0x%04X.\n", 3217 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount), 3218 original_last_aeq, nesqp->last_aeq); 3219 /* this one is for the cm_disconnect thread */ 3220 nes_add_ref(&nesqp->ibqp); 3221 spin_lock_irqsave(&nesqp->lock, qplockflags); 3222 nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED; 3223 nesqp->last_aeq = NES_AEQE_AEID_RESET_SENT; 3224 spin_unlock_irqrestore(&nesqp->lock, qplockflags); 3225 nes_cm_disconn(nesqp); 3226 } else { 3227 nes_debug(NES_DBG_MOD_QP, "QP%u No fake disconnect, QP refcount=%d\n", 3228 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount)); 3229 nes_rem_ref(&nesqp->ibqp); 3230 } 3231 } else { 3232 spin_lock_irqsave(&nesqp->lock, qplockflags); 3233 if (nesqp->cm_id) { 3234 /* These two are for the timer thread */ 3235 if (atomic_inc_return(&nesqp->close_timer_started) == 1) { 3236 nes_add_ref(&nesqp->ibqp); 3237 nesqp->cm_id->add_ref(nesqp->cm_id); 3238 nes_debug(NES_DBG_MOD_QP, "QP%u Not decrementing QP refcount (%d)," 3239 " need ae to finish up, original_last_aeq = 0x%04X." 3240 " last_aeq = 0x%04X, scheduling timer.\n", 3241 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount), 3242 original_last_aeq, nesqp->last_aeq); 3243 schedule_nes_timer(nesqp->cm_node, (struct sk_buff *) nesqp, NES_TIMER_TYPE_CLOSE, 1, 0); 3244 } 3245 spin_unlock_irqrestore(&nesqp->lock, qplockflags); 3246 } else { 3247 spin_unlock_irqrestore(&nesqp->lock, qplockflags); 3248 nes_debug(NES_DBG_MOD_QP, "QP%u Not decrementing QP refcount (%d)," 3249 " need ae to finish up, original_last_aeq = 0x%04X." 3250 " last_aeq = 0x%04X.\n", 3251 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount), 3252 original_last_aeq, nesqp->last_aeq); 3253 } 3254 } 3255 } else { 3256 nes_debug(NES_DBG_MOD_QP, "QP%u Decrementing QP refcount (%d), No ae to finish up," 3257 " original_last_aeq = 0x%04X. last_aeq = 0x%04X.\n", 3258 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount), 3259 original_last_aeq, nesqp->last_aeq); 3260 nes_rem_ref(&nesqp->ibqp); 3261 } 3262 } else { 3263 nes_debug(NES_DBG_MOD_QP, "QP%u Decrementing QP refcount (%d), No ae to finish up," 3264 " original_last_aeq = 0x%04X. last_aeq = 0x%04X.\n", 3265 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount), 3266 original_last_aeq, nesqp->last_aeq); 3267 nes_rem_ref(&nesqp->ibqp); 3268 } 3269 3270 err = 0; 3271 3272 nes_debug(NES_DBG_MOD_QP, "QP%u Leaving, refcount=%d\n", 3273 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount)); 3274 3275 return err; 3276} 3277 3278 3279/** 3280 * nes_muticast_attach 3281 */ 3282static int nes_multicast_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid) 3283{ 3284 nes_debug(NES_DBG_INIT, "\n"); 3285 return -ENOSYS; 3286} 3287 3288 3289/** 3290 * nes_multicast_detach 3291 */ 3292static int nes_multicast_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid) 3293{ 3294 nes_debug(NES_DBG_INIT, "\n"); 3295 return -ENOSYS; 3296} 3297 3298 3299/** 3300 * nes_process_mad 3301 */ 3302static int nes_process_mad(struct ib_device *ibdev, int mad_flags, 3303 u8 port_num, struct ib_wc *in_wc, struct ib_grh *in_grh, 3304 struct ib_mad *in_mad, struct ib_mad *out_mad) 3305{ 3306 nes_debug(NES_DBG_INIT, "\n"); 3307 return -ENOSYS; 3308} 3309 3310static inline void 3311fill_wqe_sg_send(struct nes_hw_qp_wqe *wqe, struct ib_send_wr *ib_wr, u32 uselkey) 3312{ 3313 int sge_index; 3314 int total_payload_length = 0; 3315 for (sge_index = 0; sge_index < ib_wr->num_sge; sge_index++) { 3316 set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_FRAG0_LOW_IDX+(sge_index*4), 3317 ib_wr->sg_list[sge_index].addr); 3318 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_LENGTH0_IDX + (sge_index*4), 3319 ib_wr->sg_list[sge_index].length); 3320 if (uselkey) 3321 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_STAG0_IDX + (sge_index*4), 3322 (ib_wr->sg_list[sge_index].lkey)); 3323 else 3324 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_STAG0_IDX + (sge_index*4), 0); 3325 3326 total_payload_length += ib_wr->sg_list[sge_index].length; 3327 } 3328 nes_debug(NES_DBG_IW_TX, "UC UC UC, sending total_payload_length=%u \n", 3329 total_payload_length); 3330 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX, 3331 total_payload_length); 3332} 3333 3334/** 3335 * nes_post_send 3336 */ 3337static int nes_post_send(struct ib_qp *ibqp, struct ib_send_wr *ib_wr, 3338 struct ib_send_wr **bad_wr) 3339{ 3340 u64 u64temp; 3341 unsigned long flags = 0; 3342 struct nes_vnic *nesvnic = to_nesvnic(ibqp->device); 3343 struct nes_device *nesdev = nesvnic->nesdev; 3344 struct nes_qp *nesqp = to_nesqp(ibqp); 3345 struct nes_hw_qp_wqe *wqe; 3346 int err; 3347 u32 qsize = nesqp->hwqp.sq_size; 3348 u32 head; 3349 u32 wqe_misc; 3350 u32 wqe_count; 3351 u32 counter; 3352 u32 total_payload_length; 3353 3354 err = 0; 3355 wqe_misc = 0; 3356 wqe_count = 0; 3357 total_payload_length = 0; 3358 3359 if (nesqp->ibqp_state > IB_QPS_RTS) 3360 return -EINVAL; 3361 3362 spin_lock_irqsave(&nesqp->lock, flags); 3363 3364 head = nesqp->hwqp.sq_head; 3365 3366 while (ib_wr) { 3367 /* Check for SQ overflow */ 3368 if (((head + (2 * qsize) - nesqp->hwqp.sq_tail) % qsize) == (qsize - 1)) { 3369 err = -EINVAL; 3370 break; 3371 } 3372 3373 wqe = &nesqp->hwqp.sq_vbase[head]; 3374 /* nes_debug(NES_DBG_IW_TX, "processing sq wqe for QP%u at %p, head = %u.\n", 3375 nesqp->hwqp.qp_id, wqe, head); */ 3376 nes_fill_init_qp_wqe(wqe, nesqp, head); 3377 u64temp = (u64)(ib_wr->wr_id); 3378 set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX, 3379 u64temp); 3380 switch (ib_wr->opcode) { 3381 case IB_WR_SEND: 3382 if (ib_wr->send_flags & IB_SEND_SOLICITED) { 3383 wqe_misc = NES_IWARP_SQ_OP_SENDSE; 3384 } else { 3385 wqe_misc = NES_IWARP_SQ_OP_SEND; 3386 } 3387 if (ib_wr->num_sge > nesdev->nesadapter->max_sge) { 3388 err = -EINVAL; 3389 break; 3390 } 3391 if (ib_wr->send_flags & IB_SEND_FENCE) { 3392 wqe_misc |= NES_IWARP_SQ_WQE_LOCAL_FENCE; 3393 } 3394 if ((ib_wr->send_flags & IB_SEND_INLINE) && 3395 ((nes_drv_opt & NES_DRV_OPT_NO_INLINE_DATA) == 0) && 3396 (ib_wr->sg_list[0].length <= 64)) { 3397 memcpy(&wqe->wqe_words[NES_IWARP_SQ_WQE_IMM_DATA_START_IDX], 3398 (void *)(unsigned long)ib_wr->sg_list[0].addr, ib_wr->sg_list[0].length); 3399 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX, 3400 ib_wr->sg_list[0].length); 3401 wqe_misc |= NES_IWARP_SQ_WQE_IMM_DATA; 3402 } else { 3403 fill_wqe_sg_send(wqe, ib_wr, 1); 3404 } 3405 3406 break; 3407 case IB_WR_RDMA_WRITE: 3408 wqe_misc = NES_IWARP_SQ_OP_RDMAW; 3409 if (ib_wr->num_sge > nesdev->nesadapter->max_sge) { 3410 nes_debug(NES_DBG_IW_TX, "Exceeded max sge, ib_wr=%u, max=%u\n", 3411 ib_wr->num_sge, 3412 nesdev->nesadapter->max_sge); 3413 err = -EINVAL; 3414 break; 3415 } 3416 if (ib_wr->send_flags & IB_SEND_FENCE) { 3417 wqe_misc |= NES_IWARP_SQ_WQE_LOCAL_FENCE; 3418 } 3419 3420 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_STAG_IDX, 3421 ib_wr->wr.rdma.rkey); 3422 set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_TO_LOW_IDX, 3423 ib_wr->wr.rdma.remote_addr); 3424 3425 if ((ib_wr->send_flags & IB_SEND_INLINE) && 3426 ((nes_drv_opt & NES_DRV_OPT_NO_INLINE_DATA) == 0) && 3427 (ib_wr->sg_list[0].length <= 64)) { 3428 memcpy(&wqe->wqe_words[NES_IWARP_SQ_WQE_IMM_DATA_START_IDX], 3429 (void *)(unsigned long)ib_wr->sg_list[0].addr, ib_wr->sg_list[0].length); 3430 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX, 3431 ib_wr->sg_list[0].length); 3432 wqe_misc |= NES_IWARP_SQ_WQE_IMM_DATA; 3433 } else { 3434 fill_wqe_sg_send(wqe, ib_wr, 1); 3435 } 3436 wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX] = 3437 wqe->wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX]; 3438 break; 3439 case IB_WR_RDMA_READ: 3440 /* iWARP only supports 1 sge for RDMA reads */ 3441 if (ib_wr->num_sge > 1) { 3442 nes_debug(NES_DBG_IW_TX, "Exceeded max sge, ib_wr=%u, max=1\n", 3443 ib_wr->num_sge); 3444 err = -EINVAL; 3445 break; 3446 } 3447 wqe_misc = NES_IWARP_SQ_OP_RDMAR; 3448 set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_TO_LOW_IDX, 3449 ib_wr->wr.rdma.remote_addr); 3450 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_STAG_IDX, 3451 ib_wr->wr.rdma.rkey); 3452 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX, 3453 ib_wr->sg_list->length); 3454 set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_FRAG0_LOW_IDX, 3455 ib_wr->sg_list->addr); 3456 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_STAG0_IDX, 3457 ib_wr->sg_list->lkey); 3458 break; 3459 default: 3460 /* error */ 3461 err = -EINVAL; 3462 break; 3463 } 3464 3465 if (ib_wr->send_flags & IB_SEND_SIGNALED) { 3466 wqe_misc |= NES_IWARP_SQ_WQE_SIGNALED_COMPL; 3467 } 3468 wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] = cpu_to_le32(wqe_misc); 3469 3470 ib_wr = ib_wr->next; 3471 head++; 3472 wqe_count++; 3473 if (head >= qsize) 3474 head = 0; 3475 3476 } 3477 3478 nesqp->hwqp.sq_head = head; 3479 barrier(); 3480 while (wqe_count) { 3481 counter = min(wqe_count, ((u32)255)); 3482 wqe_count -= counter; 3483 nes_write32(nesdev->regs + NES_WQE_ALLOC, 3484 (counter << 24) | 0x00800000 | nesqp->hwqp.qp_id); 3485 } 3486 3487 spin_unlock_irqrestore(&nesqp->lock, flags); 3488 3489 if (err) 3490 *bad_wr = ib_wr; 3491 return err; 3492} 3493 3494 3495/** 3496 * nes_post_recv 3497 */ 3498static int nes_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *ib_wr, 3499 struct ib_recv_wr **bad_wr) 3500{ 3501 u64 u64temp; 3502 unsigned long flags = 0; 3503 struct nes_vnic *nesvnic = to_nesvnic(ibqp->device); 3504 struct nes_device *nesdev = nesvnic->nesdev; 3505 struct nes_qp *nesqp = to_nesqp(ibqp); 3506 struct nes_hw_qp_wqe *wqe; 3507 int err = 0; 3508 int sge_index; 3509 u32 qsize = nesqp->hwqp.rq_size; 3510 u32 head; 3511 u32 wqe_count = 0; 3512 u32 counter; 3513 u32 total_payload_length; 3514 3515 if (nesqp->ibqp_state > IB_QPS_RTS) 3516 return -EINVAL; 3517 3518 spin_lock_irqsave(&nesqp->lock, flags); 3519 3520 head = nesqp->hwqp.rq_head; 3521 3522 while (ib_wr) { 3523 if (ib_wr->num_sge > nesdev->nesadapter->max_sge) { 3524 err = -EINVAL; 3525 break; 3526 } 3527 /* Check for RQ overflow */ 3528 if (((head + (2 * qsize) - nesqp->hwqp.rq_tail) % qsize) == (qsize - 1)) { 3529 err = -EINVAL; 3530 break; 3531 } 3532 3533 nes_debug(NES_DBG_IW_RX, "ibwr sge count = %u.\n", ib_wr->num_sge); 3534 wqe = &nesqp->hwqp.rq_vbase[head]; 3535 3536 /* nes_debug(NES_DBG_IW_RX, "QP%u:processing rq wqe at %p, head = %u.\n", 3537 nesqp->hwqp.qp_id, wqe, head); */ 3538 nes_fill_init_qp_wqe(wqe, nesqp, head); 3539 u64temp = (u64)(ib_wr->wr_id); 3540 set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX, 3541 u64temp); 3542 total_payload_length = 0; 3543 for (sge_index=0; sge_index < ib_wr->num_sge; sge_index++) { 3544 set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_RQ_WQE_FRAG0_LOW_IDX+(sge_index*4), 3545 ib_wr->sg_list[sge_index].addr); 3546 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_RQ_WQE_LENGTH0_IDX+(sge_index*4), 3547 ib_wr->sg_list[sge_index].length); 3548 set_wqe_32bit_value(wqe->wqe_words,NES_IWARP_RQ_WQE_STAG0_IDX+(sge_index*4), 3549 ib_wr->sg_list[sge_index].lkey); 3550 3551 total_payload_length += ib_wr->sg_list[sge_index].length; 3552 } 3553 set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_RQ_WQE_TOTAL_PAYLOAD_IDX, 3554 total_payload_length); 3555 3556 ib_wr = ib_wr->next; 3557 head++; 3558 wqe_count++; 3559 if (head >= qsize) 3560 head = 0; 3561 } 3562 3563 nesqp->hwqp.rq_head = head; 3564 barrier(); 3565 while (wqe_count) { 3566 counter = min(wqe_count, ((u32)255)); 3567 wqe_count -= counter; 3568 nes_write32(nesdev->regs+NES_WQE_ALLOC, (counter<<24) | nesqp->hwqp.qp_id); 3569 } 3570 3571 spin_unlock_irqrestore(&nesqp->lock, flags); 3572 3573 if (err) 3574 *bad_wr = ib_wr; 3575 return err; 3576} 3577 3578 3579/** 3580 * nes_poll_cq 3581 */ 3582static int nes_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *entry) 3583{ 3584 u64 u64temp; 3585 u64 wrid; 3586 /* u64 u64temp; */ 3587 unsigned long flags = 0; 3588 struct nes_vnic *nesvnic = to_nesvnic(ibcq->device); 3589 struct nes_device *nesdev = nesvnic->nesdev; 3590 struct nes_cq *nescq = to_nescq(ibcq); 3591 struct nes_qp *nesqp; 3592 struct nes_hw_cqe cqe; 3593 u32 head; 3594 u32 wq_tail; 3595 u32 cq_size; 3596 u32 cqe_count = 0; 3597 u32 wqe_index; 3598 u32 u32temp; 3599 /* u32 counter; */ 3600 3601 nes_debug(NES_DBG_CQ, "\n"); 3602 3603 spin_lock_irqsave(&nescq->lock, flags); 3604 3605 head = nescq->hw_cq.cq_head; 3606 cq_size = nescq->hw_cq.cq_size; 3607 3608 while (cqe_count < num_entries) { 3609 if (le32_to_cpu(nescq->hw_cq.cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]) & 3610 NES_CQE_VALID) { 3611 /* 3612 * Make sure we read CQ entry contents *after* 3613 * we've checked the valid bit. 3614 */ 3615 rmb(); 3616 3617 cqe = nescq->hw_cq.cq_vbase[head]; 3618 nescq->hw_cq.cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0; 3619 u32temp = le32_to_cpu(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]); 3620 wqe_index = u32temp & 3621 (nesdev->nesadapter->max_qp_wr - 1); 3622 u32temp &= ~(NES_SW_CONTEXT_ALIGN-1); 3623 /* parse CQE, get completion context from WQE (either rq or sq */ 3624 u64temp = (((u64)(le32_to_cpu(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX])))<<32) | 3625 ((u64)u32temp); 3626 nesqp = *((struct nes_qp **)&u64temp); 3627 memset(entry, 0, sizeof *entry); 3628 if (cqe.cqe_words[NES_CQE_ERROR_CODE_IDX] == 0) { 3629 entry->status = IB_WC_SUCCESS; 3630 } else { 3631 entry->status = IB_WC_WR_FLUSH_ERR; 3632 } 3633 3634 entry->qp = &nesqp->ibqp; 3635 entry->src_qp = nesqp->hwqp.qp_id; 3636 3637 if (le32_to_cpu(cqe.cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_SQ) { 3638 if (nesqp->skip_lsmm) { 3639 nesqp->skip_lsmm = 0; 3640 wq_tail = nesqp->hwqp.sq_tail++; 3641 } 3642 3643 /* Working on a SQ Completion*/ 3644 wq_tail = wqe_index; 3645 nesqp->hwqp.sq_tail = (wqe_index+1)&(nesqp->hwqp.sq_size - 1); 3646 wrid = (((u64)(cpu_to_le32((u32)nesqp->hwqp.sq_vbase[wq_tail]. 3647 wqe_words[NES_IWARP_SQ_WQE_COMP_SCRATCH_HIGH_IDX]))) << 32) | 3648 ((u64)(cpu_to_le32((u32)nesqp->hwqp.sq_vbase[wq_tail]. 3649 wqe_words[NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX]))); 3650 entry->byte_len = le32_to_cpu(nesqp->hwqp.sq_vbase[wq_tail]. 3651 wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX]); 3652 3653 switch (le32_to_cpu(nesqp->hwqp.sq_vbase[wq_tail]. 3654 wqe_words[NES_IWARP_SQ_WQE_MISC_IDX]) & 0x3f) { 3655 case NES_IWARP_SQ_OP_RDMAW: 3656 nes_debug(NES_DBG_CQ, "Operation = RDMA WRITE.\n"); 3657 entry->opcode = IB_WC_RDMA_WRITE; 3658 break; 3659 case NES_IWARP_SQ_OP_RDMAR: 3660 nes_debug(NES_DBG_CQ, "Operation = RDMA READ.\n"); 3661 entry->opcode = IB_WC_RDMA_READ; 3662 entry->byte_len = le32_to_cpu(nesqp->hwqp.sq_vbase[wq_tail]. 3663 wqe_words[NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX]); 3664 break; 3665 case NES_IWARP_SQ_OP_SENDINV: 3666 case NES_IWARP_SQ_OP_SENDSEINV: 3667 case NES_IWARP_SQ_OP_SEND: 3668 case NES_IWARP_SQ_OP_SENDSE: 3669 nes_debug(NES_DBG_CQ, "Operation = Send.\n"); 3670 entry->opcode = IB_WC_SEND; 3671 break; 3672 } 3673 } else { 3674 /* Working on a RQ Completion*/ 3675 wq_tail = wqe_index; 3676 nesqp->hwqp.rq_tail = (wqe_index+1)&(nesqp->hwqp.rq_size - 1); 3677 entry->byte_len = le32_to_cpu(cqe.cqe_words[NES_CQE_PAYLOAD_LENGTH_IDX]); 3678 wrid = ((u64)(le32_to_cpu(nesqp->hwqp.rq_vbase[wq_tail].wqe_words[NES_IWARP_RQ_WQE_COMP_SCRATCH_LOW_IDX]))) | 3679 ((u64)(le32_to_cpu(nesqp->hwqp.rq_vbase[wq_tail].wqe_words[NES_IWARP_RQ_WQE_COMP_SCRATCH_HIGH_IDX]))<<32); 3680 entry->opcode = IB_WC_RECV; 3681 } 3682 entry->wr_id = wrid; 3683 3684 if (++head >= cq_size) 3685 head = 0; 3686 cqe_count++; 3687 nescq->polled_completions++; 3688 if ((nescq->polled_completions > (cq_size / 2)) || 3689 (nescq->polled_completions == 255)) { 3690 nes_debug(NES_DBG_CQ, "CQ%u Issuing CQE Allocate since more than half of cqes" 3691 " are pending %u of %u.\n", 3692 nescq->hw_cq.cq_number, nescq->polled_completions, cq_size); 3693 nes_write32(nesdev->regs+NES_CQE_ALLOC, 3694 nescq->hw_cq.cq_number | (nescq->polled_completions << 16)); 3695 nescq->polled_completions = 0; 3696 } 3697 entry++; 3698 } else 3699 break; 3700 } 3701 3702 if (nescq->polled_completions) { 3703 nes_write32(nesdev->regs+NES_CQE_ALLOC, 3704 nescq->hw_cq.cq_number | (nescq->polled_completions << 16)); 3705 nescq->polled_completions = 0; 3706 } 3707 3708 nescq->hw_cq.cq_head = head; 3709 nes_debug(NES_DBG_CQ, "Reporting %u completions for CQ%u.\n", 3710 cqe_count, nescq->hw_cq.cq_number); 3711 3712 spin_unlock_irqrestore(&nescq->lock, flags); 3713 3714 return cqe_count; 3715} 3716 3717 3718/** 3719 * nes_req_notify_cq 3720 */ 3721static int nes_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags notify_flags) 3722 { 3723 struct nes_vnic *nesvnic = to_nesvnic(ibcq->device); 3724 struct nes_device *nesdev = nesvnic->nesdev; 3725 struct nes_cq *nescq = to_nescq(ibcq); 3726 u32 cq_arm; 3727 3728 nes_debug(NES_DBG_CQ, "Requesting notification for CQ%u.\n", 3729 nescq->hw_cq.cq_number); 3730 3731 cq_arm = nescq->hw_cq.cq_number; 3732 if ((notify_flags & IB_CQ_SOLICITED_MASK) == IB_CQ_NEXT_COMP) 3733 cq_arm |= NES_CQE_ALLOC_NOTIFY_NEXT; 3734 else if ((notify_flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED) 3735 cq_arm |= NES_CQE_ALLOC_NOTIFY_SE; 3736 else 3737 return -EINVAL; 3738 3739 nes_write32(nesdev->regs+NES_CQE_ALLOC, cq_arm); 3740 nes_read32(nesdev->regs+NES_CQE_ALLOC); 3741 3742 return 0; 3743} 3744 3745 3746/** 3747 * nes_init_ofa_device 3748 */ 3749struct nes_ib_device *nes_init_ofa_device(struct net_device *netdev) 3750{ 3751 struct nes_ib_device *nesibdev; 3752 struct nes_vnic *nesvnic = netdev_priv(netdev); 3753 struct nes_device *nesdev = nesvnic->nesdev; 3754 3755 nesibdev = (struct nes_ib_device *)ib_alloc_device(sizeof(struct nes_ib_device)); 3756 if (nesibdev == NULL) { 3757 return NULL; 3758 } 3759 strlcpy(nesibdev->ibdev.name, "nes%d", IB_DEVICE_NAME_MAX); 3760 nesibdev->ibdev.owner = THIS_MODULE; 3761 3762 nesibdev->ibdev.node_type = RDMA_NODE_RNIC; 3763 memset(&nesibdev->ibdev.node_guid, 0, sizeof(nesibdev->ibdev.node_guid)); 3764 memcpy(&nesibdev->ibdev.node_guid, netdev->dev_addr, 6); 3765 3766 nesibdev->ibdev.uverbs_cmd_mask = 3767 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) | 3768 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) | 3769 (1ull << IB_USER_VERBS_CMD_QUERY_PORT) | 3770 (1ull << IB_USER_VERBS_CMD_ALLOC_PD) | 3771 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) | 3772 (1ull << IB_USER_VERBS_CMD_REG_MR) | 3773 (1ull << IB_USER_VERBS_CMD_DEREG_MR) | 3774 (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) | 3775 (1ull << IB_USER_VERBS_CMD_CREATE_CQ) | 3776 (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) | 3777 (1ull << IB_USER_VERBS_CMD_CREATE_AH) | 3778 (1ull << IB_USER_VERBS_CMD_DESTROY_AH) | 3779 (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ) | 3780 (1ull << IB_USER_VERBS_CMD_CREATE_QP) | 3781 (1ull << IB_USER_VERBS_CMD_MODIFY_QP) | 3782 (1ull << IB_USER_VERBS_CMD_POLL_CQ) | 3783 (1ull << IB_USER_VERBS_CMD_DESTROY_QP) | 3784 (1ull << IB_USER_VERBS_CMD_ALLOC_MW) | 3785 (1ull << IB_USER_VERBS_CMD_BIND_MW) | 3786 (1ull << IB_USER_VERBS_CMD_DEALLOC_MW) | 3787 (1ull << IB_USER_VERBS_CMD_POST_RECV) | 3788 (1ull << IB_USER_VERBS_CMD_POST_SEND); 3789 3790 nesibdev->ibdev.phys_port_cnt = 1; 3791 nesibdev->ibdev.num_comp_vectors = 1; 3792 nesibdev->ibdev.dma_device = &nesdev->pcidev->dev; 3793 nesibdev->ibdev.dev.parent = &nesdev->pcidev->dev; 3794 nesibdev->ibdev.query_device = nes_query_device; 3795 nesibdev->ibdev.query_port = nes_query_port; 3796 nesibdev->ibdev.modify_port = nes_modify_port; 3797 nesibdev->ibdev.query_pkey = nes_query_pkey; 3798 nesibdev->ibdev.query_gid = nes_query_gid; 3799 nesibdev->ibdev.alloc_ucontext = nes_alloc_ucontext; 3800 nesibdev->ibdev.dealloc_ucontext = nes_dealloc_ucontext; 3801 nesibdev->ibdev.mmap = nes_mmap; 3802 nesibdev->ibdev.alloc_pd = nes_alloc_pd; 3803 nesibdev->ibdev.dealloc_pd = nes_dealloc_pd; 3804 nesibdev->ibdev.create_ah = nes_create_ah; 3805 nesibdev->ibdev.destroy_ah = nes_destroy_ah; 3806 nesibdev->ibdev.create_qp = nes_create_qp; 3807 nesibdev->ibdev.modify_qp = nes_modify_qp; 3808 nesibdev->ibdev.query_qp = nes_query_qp; 3809 nesibdev->ibdev.destroy_qp = nes_destroy_qp; 3810 nesibdev->ibdev.create_cq = nes_create_cq; 3811 nesibdev->ibdev.destroy_cq = nes_destroy_cq; 3812 nesibdev->ibdev.poll_cq = nes_poll_cq; 3813 nesibdev->ibdev.get_dma_mr = nes_get_dma_mr; 3814 nesibdev->ibdev.reg_phys_mr = nes_reg_phys_mr; 3815 nesibdev->ibdev.reg_user_mr = nes_reg_user_mr; 3816 nesibdev->ibdev.dereg_mr = nes_dereg_mr; 3817 nesibdev->ibdev.alloc_mw = nes_alloc_mw; 3818 nesibdev->ibdev.dealloc_mw = nes_dealloc_mw; 3819 nesibdev->ibdev.bind_mw = nes_bind_mw; 3820 3821 nesibdev->ibdev.alloc_fmr = nes_alloc_fmr; 3822 nesibdev->ibdev.unmap_fmr = nes_unmap_fmr; 3823 nesibdev->ibdev.dealloc_fmr = nes_dealloc_fmr; 3824 nesibdev->ibdev.map_phys_fmr = nes_map_phys_fmr; 3825 3826 nesibdev->ibdev.attach_mcast = nes_multicast_attach; 3827 nesibdev->ibdev.detach_mcast = nes_multicast_detach; 3828 nesibdev->ibdev.process_mad = nes_process_mad; 3829 3830 nesibdev->ibdev.req_notify_cq = nes_req_notify_cq; 3831 nesibdev->ibdev.post_send = nes_post_send; 3832 nesibdev->ibdev.post_recv = nes_post_recv; 3833 3834 nesibdev->ibdev.iwcm = kzalloc(sizeof(*nesibdev->ibdev.iwcm), GFP_KERNEL); 3835 if (nesibdev->ibdev.iwcm == NULL) { 3836 ib_dealloc_device(&nesibdev->ibdev); 3837 return NULL; 3838 } 3839 nesibdev->ibdev.iwcm->add_ref = nes_add_ref; 3840 nesibdev->ibdev.iwcm->rem_ref = nes_rem_ref; 3841 nesibdev->ibdev.iwcm->get_qp = nes_get_qp; 3842 nesibdev->ibdev.iwcm->connect = nes_connect; 3843 nesibdev->ibdev.iwcm->accept = nes_accept; 3844 nesibdev->ibdev.iwcm->reject = nes_reject; 3845 nesibdev->ibdev.iwcm->create_listen = nes_create_listen; 3846 nesibdev->ibdev.iwcm->destroy_listen = nes_destroy_listen; 3847 3848 return nesibdev; 3849} 3850 3851 3852/** 3853 * nes_destroy_ofa_device 3854 */ 3855void nes_destroy_ofa_device(struct nes_ib_device *nesibdev) 3856{ 3857 if (nesibdev == NULL) 3858 return; 3859 3860 nes_unregister_ofa_device(nesibdev); 3861 3862 kfree(nesibdev->ibdev.iwcm); 3863 ib_dealloc_device(&nesibdev->ibdev); 3864} 3865 3866 3867/** 3868 * nes_register_ofa_device 3869 */ 3870int nes_register_ofa_device(struct nes_ib_device *nesibdev) 3871{ 3872 struct nes_vnic *nesvnic = nesibdev->nesvnic; 3873 struct nes_device *nesdev = nesvnic->nesdev; 3874 struct nes_adapter *nesadapter = nesdev->nesadapter; 3875 int i, ret; 3876 3877 ret = ib_register_device(&nesvnic->nesibdev->ibdev); 3878 if (ret) { 3879 return ret; 3880 } 3881 3882 /* Get the resources allocated to this device */ 3883 nesibdev->max_cq = (nesadapter->max_cq-NES_FIRST_QPN) / nesadapter->port_count; 3884 nesibdev->max_mr = nesadapter->max_mr / nesadapter->port_count; 3885 nesibdev->max_qp = (nesadapter->max_qp-NES_FIRST_QPN) / nesadapter->port_count; 3886 nesibdev->max_pd = nesadapter->max_pd / nesadapter->port_count; 3887 3888 for (i = 0; i < ARRAY_SIZE(nes_dev_attributes); ++i) { 3889 ret = device_create_file(&nesibdev->ibdev.dev, nes_dev_attributes[i]); 3890 if (ret) { 3891 while (i > 0) { 3892 i--; 3893 device_remove_file(&nesibdev->ibdev.dev, 3894 nes_dev_attributes[i]); 3895 } 3896 ib_unregister_device(&nesibdev->ibdev); 3897 return ret; 3898 } 3899 } 3900 3901 nesvnic->of_device_registered = 1; 3902 3903 return 0; 3904} 3905 3906 3907/** 3908 * nes_unregister_ofa_device 3909 */ 3910static void nes_unregister_ofa_device(struct nes_ib_device *nesibdev) 3911{ 3912 struct nes_vnic *nesvnic = nesibdev->nesvnic; 3913 int i; 3914 3915 for (i = 0; i < ARRAY_SIZE(nes_dev_attributes); ++i) { 3916 device_remove_file(&nesibdev->ibdev.dev, nes_dev_attributes[i]); 3917 } 3918 3919 if (nesvnic->of_device_registered) { 3920 ib_unregister_device(&nesibdev->ibdev); 3921 } 3922 3923 nesvnic->of_device_registered = 0; 3924} 3925