iwch_qp.c revision de3d353072f9342f04112ba0504c3e294220cb8f
1/* 2 * Copyright (c) 2006 Chelsio, Inc. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32#include "iwch_provider.h" 33#include "iwch.h" 34#include "iwch_cm.h" 35#include "cxio_hal.h" 36 37#define NO_SUPPORT -1 38 39static int iwch_build_rdma_send(union t3_wr *wqe, struct ib_send_wr *wr, 40 u8 * flit_cnt) 41{ 42 int i; 43 u32 plen; 44 45 switch (wr->opcode) { 46 case IB_WR_SEND: 47 case IB_WR_SEND_WITH_IMM: 48 if (wr->send_flags & IB_SEND_SOLICITED) 49 wqe->send.rdmaop = T3_SEND_WITH_SE; 50 else 51 wqe->send.rdmaop = T3_SEND; 52 wqe->send.rem_stag = 0; 53 break; 54#if 0 /* Not currently supported */ 55 case TYPE_SEND_INVALIDATE: 56 case TYPE_SEND_INVALIDATE_IMMEDIATE: 57 wqe->send.rdmaop = T3_SEND_WITH_INV; 58 wqe->send.rem_stag = cpu_to_be32(wr->wr.rdma.rkey); 59 break; 60 case TYPE_SEND_SE_INVALIDATE: 61 wqe->send.rdmaop = T3_SEND_WITH_SE_INV; 62 wqe->send.rem_stag = cpu_to_be32(wr->wr.rdma.rkey); 63 break; 64#endif 65 default: 66 break; 67 } 68 if (wr->num_sge > T3_MAX_SGE) 69 return -EINVAL; 70 wqe->send.reserved[0] = 0; 71 wqe->send.reserved[1] = 0; 72 wqe->send.reserved[2] = 0; 73 if (wr->opcode == IB_WR_SEND_WITH_IMM) { 74 plen = 4; 75 wqe->send.sgl[0].stag = wr->imm_data; 76 wqe->send.sgl[0].len = __constant_cpu_to_be32(0); 77 wqe->send.num_sgle = __constant_cpu_to_be32(0); 78 *flit_cnt = 5; 79 } else { 80 plen = 0; 81 for (i = 0; i < wr->num_sge; i++) { 82 if ((plen + wr->sg_list[i].length) < plen) { 83 return -EMSGSIZE; 84 } 85 plen += wr->sg_list[i].length; 86 wqe->send.sgl[i].stag = 87 cpu_to_be32(wr->sg_list[i].lkey); 88 wqe->send.sgl[i].len = 89 cpu_to_be32(wr->sg_list[i].length); 90 wqe->send.sgl[i].to = cpu_to_be64(wr->sg_list[i].addr); 91 } 92 wqe->send.num_sgle = cpu_to_be32(wr->num_sge); 93 *flit_cnt = 4 + ((wr->num_sge) << 1); 94 } 95 wqe->send.plen = cpu_to_be32(plen); 96 return 0; 97} 98 99static int iwch_build_rdma_write(union t3_wr *wqe, struct ib_send_wr *wr, 100 u8 *flit_cnt) 101{ 102 int i; 103 u32 plen; 104 if (wr->num_sge > T3_MAX_SGE) 105 return -EINVAL; 106 wqe->write.rdmaop = T3_RDMA_WRITE; 107 wqe->write.reserved[0] = 0; 108 wqe->write.reserved[1] = 0; 109 wqe->write.reserved[2] = 0; 110 wqe->write.stag_sink = cpu_to_be32(wr->wr.rdma.rkey); 111 wqe->write.to_sink = cpu_to_be64(wr->wr.rdma.remote_addr); 112 113 if (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) { 114 plen = 4; 115 wqe->write.sgl[0].stag = wr->imm_data; 116 wqe->write.sgl[0].len = __constant_cpu_to_be32(0); 117 wqe->write.num_sgle = __constant_cpu_to_be32(0); 118 *flit_cnt = 6; 119 } else { 120 plen = 0; 121 for (i = 0; i < wr->num_sge; i++) { 122 if ((plen + wr->sg_list[i].length) < plen) { 123 return -EMSGSIZE; 124 } 125 plen += wr->sg_list[i].length; 126 wqe->write.sgl[i].stag = 127 cpu_to_be32(wr->sg_list[i].lkey); 128 wqe->write.sgl[i].len = 129 cpu_to_be32(wr->sg_list[i].length); 130 wqe->write.sgl[i].to = 131 cpu_to_be64(wr->sg_list[i].addr); 132 } 133 wqe->write.num_sgle = cpu_to_be32(wr->num_sge); 134 *flit_cnt = 5 + ((wr->num_sge) << 1); 135 } 136 wqe->write.plen = cpu_to_be32(plen); 137 return 0; 138} 139 140static int iwch_build_rdma_read(union t3_wr *wqe, struct ib_send_wr *wr, 141 u8 *flit_cnt) 142{ 143 if (wr->num_sge > 1) 144 return -EINVAL; 145 wqe->read.rdmaop = T3_READ_REQ; 146 wqe->read.reserved[0] = 0; 147 wqe->read.reserved[1] = 0; 148 wqe->read.reserved[2] = 0; 149 wqe->read.rem_stag = cpu_to_be32(wr->wr.rdma.rkey); 150 wqe->read.rem_to = cpu_to_be64(wr->wr.rdma.remote_addr); 151 wqe->read.local_stag = cpu_to_be32(wr->sg_list[0].lkey); 152 wqe->read.local_len = cpu_to_be32(wr->sg_list[0].length); 153 wqe->read.local_to = cpu_to_be64(wr->sg_list[0].addr); 154 *flit_cnt = sizeof(struct t3_rdma_read_wr) >> 3; 155 return 0; 156} 157 158/* 159 * TBD: this is going to be moved to firmware. Missing pdid/qpid check for now. 160 */ 161static int iwch_sgl2pbl_map(struct iwch_dev *rhp, struct ib_sge *sg_list, 162 u32 num_sgle, u32 * pbl_addr, u8 * page_size) 163{ 164 int i; 165 struct iwch_mr *mhp; 166 u32 offset; 167 for (i = 0; i < num_sgle; i++) { 168 169 mhp = get_mhp(rhp, (sg_list[i].lkey) >> 8); 170 if (!mhp) { 171 PDBG("%s %d\n", __FUNCTION__, __LINE__); 172 return -EIO; 173 } 174 if (!mhp->attr.state) { 175 PDBG("%s %d\n", __FUNCTION__, __LINE__); 176 return -EIO; 177 } 178 if (mhp->attr.zbva) { 179 PDBG("%s %d\n", __FUNCTION__, __LINE__); 180 return -EIO; 181 } 182 183 if (sg_list[i].addr < mhp->attr.va_fbo) { 184 PDBG("%s %d\n", __FUNCTION__, __LINE__); 185 return -EINVAL; 186 } 187 if (sg_list[i].addr + ((u64) sg_list[i].length) < 188 sg_list[i].addr) { 189 PDBG("%s %d\n", __FUNCTION__, __LINE__); 190 return -EINVAL; 191 } 192 if (sg_list[i].addr + ((u64) sg_list[i].length) > 193 mhp->attr.va_fbo + ((u64) mhp->attr.len)) { 194 PDBG("%s %d\n", __FUNCTION__, __LINE__); 195 return -EINVAL; 196 } 197 offset = sg_list[i].addr - mhp->attr.va_fbo; 198 offset += ((u32) mhp->attr.va_fbo) % 199 (1UL << (12 + mhp->attr.page_size)); 200 pbl_addr[i] = ((mhp->attr.pbl_addr - 201 rhp->rdev.rnic_info.pbl_base) >> 3) + 202 (offset >> (12 + mhp->attr.page_size)); 203 page_size[i] = mhp->attr.page_size; 204 } 205 return 0; 206} 207 208static int iwch_build_rdma_recv(struct iwch_dev *rhp, union t3_wr *wqe, 209 struct ib_recv_wr *wr) 210{ 211 int i, err = 0; 212 u32 pbl_addr[4]; 213 u8 page_size[4]; 214 if (wr->num_sge > T3_MAX_SGE) 215 return -EINVAL; 216 err = iwch_sgl2pbl_map(rhp, wr->sg_list, wr->num_sge, pbl_addr, 217 page_size); 218 if (err) 219 return err; 220 wqe->recv.pagesz[0] = page_size[0]; 221 wqe->recv.pagesz[1] = page_size[1]; 222 wqe->recv.pagesz[2] = page_size[2]; 223 wqe->recv.pagesz[3] = page_size[3]; 224 wqe->recv.num_sgle = cpu_to_be32(wr->num_sge); 225 for (i = 0; i < wr->num_sge; i++) { 226 wqe->recv.sgl[i].stag = cpu_to_be32(wr->sg_list[i].lkey); 227 wqe->recv.sgl[i].len = cpu_to_be32(wr->sg_list[i].length); 228 229 /* to in the WQE == the offset into the page */ 230 wqe->recv.sgl[i].to = cpu_to_be64(((u32) wr->sg_list[i].addr) % 231 (1UL << (12 + page_size[i]))); 232 233 /* pbl_addr is the adapters address in the PBL */ 234 wqe->recv.pbl_addr[i] = cpu_to_be32(pbl_addr[i]); 235 } 236 for (; i < T3_MAX_SGE; i++) { 237 wqe->recv.sgl[i].stag = 0; 238 wqe->recv.sgl[i].len = 0; 239 wqe->recv.sgl[i].to = 0; 240 wqe->recv.pbl_addr[i] = 0; 241 } 242 return 0; 243} 244 245int iwch_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, 246 struct ib_send_wr **bad_wr) 247{ 248 int err = 0; 249 u8 t3_wr_flit_cnt; 250 enum t3_wr_opcode t3_wr_opcode = 0; 251 enum t3_wr_flags t3_wr_flags; 252 struct iwch_qp *qhp; 253 u32 idx; 254 union t3_wr *wqe; 255 u32 num_wrs; 256 unsigned long flag; 257 struct t3_swsq *sqp; 258 259 qhp = to_iwch_qp(ibqp); 260 spin_lock_irqsave(&qhp->lock, flag); 261 if (qhp->attr.state > IWCH_QP_STATE_RTS) { 262 spin_unlock_irqrestore(&qhp->lock, flag); 263 return -EINVAL; 264 } 265 num_wrs = Q_FREECNT(qhp->wq.sq_rptr, qhp->wq.sq_wptr, 266 qhp->wq.sq_size_log2); 267 if (num_wrs <= 0) { 268 spin_unlock_irqrestore(&qhp->lock, flag); 269 return -ENOMEM; 270 } 271 while (wr) { 272 if (num_wrs == 0) { 273 err = -ENOMEM; 274 *bad_wr = wr; 275 break; 276 } 277 idx = Q_PTR2IDX(qhp->wq.wptr, qhp->wq.size_log2); 278 wqe = (union t3_wr *) (qhp->wq.queue + idx); 279 t3_wr_flags = 0; 280 if (wr->send_flags & IB_SEND_SOLICITED) 281 t3_wr_flags |= T3_SOLICITED_EVENT_FLAG; 282 if (wr->send_flags & IB_SEND_FENCE) 283 t3_wr_flags |= T3_READ_FENCE_FLAG; 284 if (wr->send_flags & IB_SEND_SIGNALED) 285 t3_wr_flags |= T3_COMPLETION_FLAG; 286 sqp = qhp->wq.sq + 287 Q_PTR2IDX(qhp->wq.sq_wptr, qhp->wq.sq_size_log2); 288 switch (wr->opcode) { 289 case IB_WR_SEND: 290 case IB_WR_SEND_WITH_IMM: 291 t3_wr_opcode = T3_WR_SEND; 292 err = iwch_build_rdma_send(wqe, wr, &t3_wr_flit_cnt); 293 break; 294 case IB_WR_RDMA_WRITE: 295 case IB_WR_RDMA_WRITE_WITH_IMM: 296 t3_wr_opcode = T3_WR_WRITE; 297 err = iwch_build_rdma_write(wqe, wr, &t3_wr_flit_cnt); 298 break; 299 case IB_WR_RDMA_READ: 300 t3_wr_opcode = T3_WR_READ; 301 t3_wr_flags = 0; /* T3 reads are always signaled */ 302 err = iwch_build_rdma_read(wqe, wr, &t3_wr_flit_cnt); 303 if (err) 304 break; 305 sqp->read_len = wqe->read.local_len; 306 if (!qhp->wq.oldest_read) 307 qhp->wq.oldest_read = sqp; 308 break; 309 default: 310 PDBG("%s post of type=%d TBD!\n", __FUNCTION__, 311 wr->opcode); 312 err = -EINVAL; 313 } 314 if (err) { 315 *bad_wr = wr; 316 break; 317 } 318 wqe->send.wrid.id0.hi = qhp->wq.sq_wptr; 319 sqp->wr_id = wr->wr_id; 320 sqp->opcode = wr2opcode(t3_wr_opcode); 321 sqp->sq_wptr = qhp->wq.sq_wptr; 322 sqp->complete = 0; 323 sqp->signaled = (wr->send_flags & IB_SEND_SIGNALED); 324 325 build_fw_riwrh((void *) wqe, t3_wr_opcode, t3_wr_flags, 326 Q_GENBIT(qhp->wq.wptr, qhp->wq.size_log2), 327 0, t3_wr_flit_cnt); 328 PDBG("%s cookie 0x%llx wq idx 0x%x swsq idx %ld opcode %d\n", 329 __FUNCTION__, (unsigned long long) wr->wr_id, idx, 330 Q_PTR2IDX(qhp->wq.sq_wptr, qhp->wq.sq_size_log2), 331 sqp->opcode); 332 wr = wr->next; 333 num_wrs--; 334 ++(qhp->wq.wptr); 335 ++(qhp->wq.sq_wptr); 336 } 337 spin_unlock_irqrestore(&qhp->lock, flag); 338 ring_doorbell(qhp->wq.doorbell, qhp->wq.qpid); 339 return err; 340} 341 342int iwch_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr, 343 struct ib_recv_wr **bad_wr) 344{ 345 int err = 0; 346 struct iwch_qp *qhp; 347 u32 idx; 348 union t3_wr *wqe; 349 u32 num_wrs; 350 unsigned long flag; 351 352 qhp = to_iwch_qp(ibqp); 353 spin_lock_irqsave(&qhp->lock, flag); 354 if (qhp->attr.state > IWCH_QP_STATE_RTS) { 355 spin_unlock_irqrestore(&qhp->lock, flag); 356 return -EINVAL; 357 } 358 num_wrs = Q_FREECNT(qhp->wq.rq_rptr, qhp->wq.rq_wptr, 359 qhp->wq.rq_size_log2) - 1; 360 if (!wr) { 361 spin_unlock_irqrestore(&qhp->lock, flag); 362 return -EINVAL; 363 } 364 while (wr) { 365 idx = Q_PTR2IDX(qhp->wq.wptr, qhp->wq.size_log2); 366 wqe = (union t3_wr *) (qhp->wq.queue + idx); 367 if (num_wrs) 368 err = iwch_build_rdma_recv(qhp->rhp, wqe, wr); 369 else 370 err = -ENOMEM; 371 if (err) { 372 *bad_wr = wr; 373 break; 374 } 375 qhp->wq.rq[Q_PTR2IDX(qhp->wq.rq_wptr, qhp->wq.rq_size_log2)] = 376 wr->wr_id; 377 build_fw_riwrh((void *) wqe, T3_WR_RCV, T3_COMPLETION_FLAG, 378 Q_GENBIT(qhp->wq.wptr, qhp->wq.size_log2), 379 0, sizeof(struct t3_receive_wr) >> 3); 380 PDBG("%s cookie 0x%llx idx 0x%x rq_wptr 0x%x rw_rptr 0x%x " 381 "wqe %p \n", __FUNCTION__, (unsigned long long) wr->wr_id, 382 idx, qhp->wq.rq_wptr, qhp->wq.rq_rptr, wqe); 383 ++(qhp->wq.rq_wptr); 384 ++(qhp->wq.wptr); 385 wr = wr->next; 386 num_wrs--; 387 } 388 spin_unlock_irqrestore(&qhp->lock, flag); 389 ring_doorbell(qhp->wq.doorbell, qhp->wq.qpid); 390 return err; 391} 392 393int iwch_bind_mw(struct ib_qp *qp, 394 struct ib_mw *mw, 395 struct ib_mw_bind *mw_bind) 396{ 397 struct iwch_dev *rhp; 398 struct iwch_mw *mhp; 399 struct iwch_qp *qhp; 400 union t3_wr *wqe; 401 u32 pbl_addr; 402 u8 page_size; 403 u32 num_wrs; 404 unsigned long flag; 405 struct ib_sge sgl; 406 int err=0; 407 enum t3_wr_flags t3_wr_flags; 408 u32 idx; 409 struct t3_swsq *sqp; 410 411 qhp = to_iwch_qp(qp); 412 mhp = to_iwch_mw(mw); 413 rhp = qhp->rhp; 414 415 spin_lock_irqsave(&qhp->lock, flag); 416 if (qhp->attr.state > IWCH_QP_STATE_RTS) { 417 spin_unlock_irqrestore(&qhp->lock, flag); 418 return -EINVAL; 419 } 420 num_wrs = Q_FREECNT(qhp->wq.sq_rptr, qhp->wq.sq_wptr, 421 qhp->wq.sq_size_log2); 422 if ((num_wrs) <= 0) { 423 spin_unlock_irqrestore(&qhp->lock, flag); 424 return -ENOMEM; 425 } 426 idx = Q_PTR2IDX(qhp->wq.wptr, qhp->wq.size_log2); 427 PDBG("%s: idx 0x%0x, mw 0x%p, mw_bind 0x%p\n", __FUNCTION__, idx, 428 mw, mw_bind); 429 wqe = (union t3_wr *) (qhp->wq.queue + idx); 430 431 t3_wr_flags = 0; 432 if (mw_bind->send_flags & IB_SEND_SIGNALED) 433 t3_wr_flags = T3_COMPLETION_FLAG; 434 435 sgl.addr = mw_bind->addr; 436 sgl.lkey = mw_bind->mr->lkey; 437 sgl.length = mw_bind->length; 438 wqe->bind.reserved = 0; 439 wqe->bind.type = T3_VA_BASED_TO; 440 441 /* TBD: check perms */ 442 wqe->bind.perms = iwch_ib_to_mwbind_access(mw_bind->mw_access_flags); 443 wqe->bind.mr_stag = cpu_to_be32(mw_bind->mr->lkey); 444 wqe->bind.mw_stag = cpu_to_be32(mw->rkey); 445 wqe->bind.mw_len = cpu_to_be32(mw_bind->length); 446 wqe->bind.mw_va = cpu_to_be64(mw_bind->addr); 447 err = iwch_sgl2pbl_map(rhp, &sgl, 1, &pbl_addr, &page_size); 448 if (err) { 449 spin_unlock_irqrestore(&qhp->lock, flag); 450 return err; 451 } 452 wqe->send.wrid.id0.hi = qhp->wq.sq_wptr; 453 sqp = qhp->wq.sq + Q_PTR2IDX(qhp->wq.sq_wptr, qhp->wq.sq_size_log2); 454 sqp->wr_id = mw_bind->wr_id; 455 sqp->opcode = T3_BIND_MW; 456 sqp->sq_wptr = qhp->wq.sq_wptr; 457 sqp->complete = 0; 458 sqp->signaled = (mw_bind->send_flags & IB_SEND_SIGNALED); 459 wqe->bind.mr_pbl_addr = cpu_to_be32(pbl_addr); 460 wqe->bind.mr_pagesz = page_size; 461 wqe->flit[T3_SQ_COOKIE_FLIT] = mw_bind->wr_id; 462 build_fw_riwrh((void *)wqe, T3_WR_BIND, t3_wr_flags, 463 Q_GENBIT(qhp->wq.wptr, qhp->wq.size_log2), 0, 464 sizeof(struct t3_bind_mw_wr) >> 3); 465 ++(qhp->wq.wptr); 466 ++(qhp->wq.sq_wptr); 467 spin_unlock_irqrestore(&qhp->lock, flag); 468 469 ring_doorbell(qhp->wq.doorbell, qhp->wq.qpid); 470 471 return err; 472} 473 474static inline void build_term_codes(struct respQ_msg_t *rsp_msg, 475 u8 *layer_type, u8 *ecode) 476{ 477 int status = TPT_ERR_INTERNAL_ERR; 478 int tagged = 0; 479 int opcode = -1; 480 int rqtype = 0; 481 int send_inv = 0; 482 483 if (rsp_msg) { 484 status = CQE_STATUS(rsp_msg->cqe); 485 opcode = CQE_OPCODE(rsp_msg->cqe); 486 rqtype = RQ_TYPE(rsp_msg->cqe); 487 send_inv = (opcode == T3_SEND_WITH_INV) || 488 (opcode == T3_SEND_WITH_SE_INV); 489 tagged = (opcode == T3_RDMA_WRITE) || 490 (rqtype && (opcode == T3_READ_RESP)); 491 } 492 493 switch (status) { 494 case TPT_ERR_STAG: 495 if (send_inv) { 496 *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; 497 *ecode = RDMAP_CANT_INV_STAG; 498 } else { 499 *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; 500 *ecode = RDMAP_INV_STAG; 501 } 502 break; 503 case TPT_ERR_PDID: 504 *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; 505 if ((opcode == T3_SEND_WITH_INV) || 506 (opcode == T3_SEND_WITH_SE_INV)) 507 *ecode = RDMAP_CANT_INV_STAG; 508 else 509 *ecode = RDMAP_STAG_NOT_ASSOC; 510 break; 511 case TPT_ERR_QPID: 512 *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; 513 *ecode = RDMAP_STAG_NOT_ASSOC; 514 break; 515 case TPT_ERR_ACCESS: 516 *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; 517 *ecode = RDMAP_ACC_VIOL; 518 break; 519 case TPT_ERR_WRAP: 520 *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; 521 *ecode = RDMAP_TO_WRAP; 522 break; 523 case TPT_ERR_BOUND: 524 if (tagged) { 525 *layer_type = LAYER_DDP|DDP_TAGGED_ERR; 526 *ecode = DDPT_BASE_BOUNDS; 527 } else { 528 *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; 529 *ecode = RDMAP_BASE_BOUNDS; 530 } 531 break; 532 case TPT_ERR_INVALIDATE_SHARED_MR: 533 case TPT_ERR_INVALIDATE_MR_WITH_MW_BOUND: 534 *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; 535 *ecode = RDMAP_CANT_INV_STAG; 536 break; 537 case TPT_ERR_ECC: 538 case TPT_ERR_ECC_PSTAG: 539 case TPT_ERR_INTERNAL_ERR: 540 *layer_type = LAYER_RDMAP|RDMAP_LOCAL_CATA; 541 *ecode = 0; 542 break; 543 case TPT_ERR_OUT_OF_RQE: 544 *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; 545 *ecode = DDPU_INV_MSN_NOBUF; 546 break; 547 case TPT_ERR_PBL_ADDR_BOUND: 548 *layer_type = LAYER_DDP|DDP_TAGGED_ERR; 549 *ecode = DDPT_BASE_BOUNDS; 550 break; 551 case TPT_ERR_CRC: 552 *layer_type = LAYER_MPA|DDP_LLP; 553 *ecode = MPA_CRC_ERR; 554 break; 555 case TPT_ERR_MARKER: 556 *layer_type = LAYER_MPA|DDP_LLP; 557 *ecode = MPA_MARKER_ERR; 558 break; 559 case TPT_ERR_PDU_LEN_ERR: 560 *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; 561 *ecode = DDPU_MSG_TOOBIG; 562 break; 563 case TPT_ERR_DDP_VERSION: 564 if (tagged) { 565 *layer_type = LAYER_DDP|DDP_TAGGED_ERR; 566 *ecode = DDPT_INV_VERS; 567 } else { 568 *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; 569 *ecode = DDPU_INV_VERS; 570 } 571 break; 572 case TPT_ERR_RDMA_VERSION: 573 *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; 574 *ecode = RDMAP_INV_VERS; 575 break; 576 case TPT_ERR_OPCODE: 577 *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; 578 *ecode = RDMAP_INV_OPCODE; 579 break; 580 case TPT_ERR_DDP_QUEUE_NUM: 581 *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; 582 *ecode = DDPU_INV_QN; 583 break; 584 case TPT_ERR_MSN: 585 case TPT_ERR_MSN_GAP: 586 case TPT_ERR_MSN_RANGE: 587 case TPT_ERR_IRD_OVERFLOW: 588 *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; 589 *ecode = DDPU_INV_MSN_RANGE; 590 break; 591 case TPT_ERR_TBIT: 592 *layer_type = LAYER_DDP|DDP_LOCAL_CATA; 593 *ecode = 0; 594 break; 595 case TPT_ERR_MO: 596 *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; 597 *ecode = DDPU_INV_MO; 598 break; 599 default: 600 *layer_type = LAYER_RDMAP|DDP_LOCAL_CATA; 601 *ecode = 0; 602 break; 603 } 604} 605 606/* 607 * This posts a TERMINATE with layer=RDMA, type=catastrophic. 608 */ 609int iwch_post_terminate(struct iwch_qp *qhp, struct respQ_msg_t *rsp_msg) 610{ 611 union t3_wr *wqe; 612 struct terminate_message *term; 613 struct sk_buff *skb; 614 615 PDBG("%s %d\n", __FUNCTION__, __LINE__); 616 skb = alloc_skb(40, GFP_ATOMIC); 617 if (!skb) { 618 printk(KERN_ERR "%s cannot send TERMINATE!\n", __FUNCTION__); 619 return -ENOMEM; 620 } 621 wqe = (union t3_wr *)skb_put(skb, 40); 622 memset(wqe, 0, 40); 623 wqe->send.rdmaop = T3_TERMINATE; 624 625 /* immediate data length */ 626 wqe->send.plen = htonl(4); 627 628 /* immediate data starts here. */ 629 term = (struct terminate_message *)wqe->send.sgl; 630 build_term_codes(rsp_msg, &term->layer_etype, &term->ecode); 631 build_fw_riwrh((void *)wqe, T3_WR_SEND, 632 T3_COMPLETION_FLAG | T3_NOTIFY_FLAG, 1, 633 qhp->ep->hwtid, 5); 634 skb->priority = CPL_PRIORITY_DATA; 635 return cxgb3_ofld_send(qhp->rhp->rdev.t3cdev_p, skb); 636} 637 638/* 639 * Assumes qhp lock is held. 640 */ 641static void __flush_qp(struct iwch_qp *qhp, unsigned long *flag) 642{ 643 struct iwch_cq *rchp, *schp; 644 int count; 645 646 rchp = get_chp(qhp->rhp, qhp->attr.rcq); 647 schp = get_chp(qhp->rhp, qhp->attr.scq); 648 649 PDBG("%s qhp %p rchp %p schp %p\n", __FUNCTION__, qhp, rchp, schp); 650 /* take a ref on the qhp since we must release the lock */ 651 atomic_inc(&qhp->refcnt); 652 spin_unlock_irqrestore(&qhp->lock, *flag); 653 654 /* locking heirarchy: cq lock first, then qp lock. */ 655 spin_lock_irqsave(&rchp->lock, *flag); 656 spin_lock(&qhp->lock); 657 cxio_flush_hw_cq(&rchp->cq); 658 cxio_count_rcqes(&rchp->cq, &qhp->wq, &count); 659 cxio_flush_rq(&qhp->wq, &rchp->cq, count); 660 spin_unlock(&qhp->lock); 661 spin_unlock_irqrestore(&rchp->lock, *flag); 662 663 /* locking heirarchy: cq lock first, then qp lock. */ 664 spin_lock_irqsave(&schp->lock, *flag); 665 spin_lock(&qhp->lock); 666 cxio_flush_hw_cq(&schp->cq); 667 cxio_count_scqes(&schp->cq, &qhp->wq, &count); 668 cxio_flush_sq(&qhp->wq, &schp->cq, count); 669 spin_unlock(&qhp->lock); 670 spin_unlock_irqrestore(&schp->lock, *flag); 671 672 /* deref */ 673 if (atomic_dec_and_test(&qhp->refcnt)) 674 wake_up(&qhp->wait); 675 676 spin_lock_irqsave(&qhp->lock, *flag); 677} 678 679static void flush_qp(struct iwch_qp *qhp, unsigned long *flag) 680{ 681 if (t3b_device(qhp->rhp)) 682 cxio_set_wq_in_error(&qhp->wq); 683 else 684 __flush_qp(qhp, flag); 685} 686 687 688/* 689 * Return non zero if at least one RECV was pre-posted. 690 */ 691static int rqes_posted(struct iwch_qp *qhp) 692{ 693 return fw_riwrh_opcode((struct fw_riwrh *)qhp->wq.queue) == T3_WR_RCV; 694} 695 696static int rdma_init(struct iwch_dev *rhp, struct iwch_qp *qhp, 697 enum iwch_qp_attr_mask mask, 698 struct iwch_qp_attributes *attrs) 699{ 700 struct t3_rdma_init_attr init_attr; 701 int ret; 702 703 init_attr.tid = qhp->ep->hwtid; 704 init_attr.qpid = qhp->wq.qpid; 705 init_attr.pdid = qhp->attr.pd; 706 init_attr.scqid = qhp->attr.scq; 707 init_attr.rcqid = qhp->attr.rcq; 708 init_attr.rq_addr = qhp->wq.rq_addr; 709 init_attr.rq_size = 1 << qhp->wq.rq_size_log2; 710 init_attr.mpaattrs = uP_RI_MPA_IETF_ENABLE | 711 qhp->attr.mpa_attr.recv_marker_enabled | 712 (qhp->attr.mpa_attr.xmit_marker_enabled << 1) | 713 (qhp->attr.mpa_attr.crc_enabled << 2); 714 715 /* 716 * XXX - The IWCM doesn't quite handle getting these 717 * attrs set before going into RTS. For now, just turn 718 * them on always... 719 */ 720#if 0 721 init_attr.qpcaps = qhp->attr.enableRdmaRead | 722 (qhp->attr.enableRdmaWrite << 1) | 723 (qhp->attr.enableBind << 2) | 724 (qhp->attr.enable_stag0_fastreg << 3) | 725 (qhp->attr.enable_stag0_fastreg << 4); 726#else 727 init_attr.qpcaps = 0x1f; 728#endif 729 init_attr.tcp_emss = qhp->ep->emss; 730 init_attr.ord = qhp->attr.max_ord; 731 init_attr.ird = qhp->attr.max_ird; 732 init_attr.qp_dma_addr = qhp->wq.dma_addr; 733 init_attr.qp_dma_size = (1UL << qhp->wq.size_log2); 734 init_attr.flags = rqes_posted(qhp) ? RECVS_POSTED : 0; 735 init_attr.irs = qhp->ep->rcv_seq; 736 PDBG("%s init_attr.rq_addr 0x%x init_attr.rq_size = %d " 737 "flags 0x%x qpcaps 0x%x\n", __FUNCTION__, 738 init_attr.rq_addr, init_attr.rq_size, 739 init_attr.flags, init_attr.qpcaps); 740 ret = cxio_rdma_init(&rhp->rdev, &init_attr); 741 PDBG("%s ret %d\n", __FUNCTION__, ret); 742 return ret; 743} 744 745int iwch_modify_qp(struct iwch_dev *rhp, struct iwch_qp *qhp, 746 enum iwch_qp_attr_mask mask, 747 struct iwch_qp_attributes *attrs, 748 int internal) 749{ 750 int ret = 0; 751 struct iwch_qp_attributes newattr = qhp->attr; 752 unsigned long flag; 753 int disconnect = 0; 754 int terminate = 0; 755 int abort = 0; 756 int free = 0; 757 struct iwch_ep *ep = NULL; 758 759 PDBG("%s qhp %p qpid 0x%x ep %p state %d -> %d\n", __FUNCTION__, 760 qhp, qhp->wq.qpid, qhp->ep, qhp->attr.state, 761 (mask & IWCH_QP_ATTR_NEXT_STATE) ? attrs->next_state : -1); 762 763 spin_lock_irqsave(&qhp->lock, flag); 764 765 /* Process attr changes if in IDLE */ 766 if (mask & IWCH_QP_ATTR_VALID_MODIFY) { 767 if (qhp->attr.state != IWCH_QP_STATE_IDLE) { 768 ret = -EIO; 769 goto out; 770 } 771 if (mask & IWCH_QP_ATTR_ENABLE_RDMA_READ) 772 newattr.enable_rdma_read = attrs->enable_rdma_read; 773 if (mask & IWCH_QP_ATTR_ENABLE_RDMA_WRITE) 774 newattr.enable_rdma_write = attrs->enable_rdma_write; 775 if (mask & IWCH_QP_ATTR_ENABLE_RDMA_BIND) 776 newattr.enable_bind = attrs->enable_bind; 777 if (mask & IWCH_QP_ATTR_MAX_ORD) { 778 if (attrs->max_ord > 779 rhp->attr.max_rdma_read_qp_depth) { 780 ret = -EINVAL; 781 goto out; 782 } 783 newattr.max_ord = attrs->max_ord; 784 } 785 if (mask & IWCH_QP_ATTR_MAX_IRD) { 786 if (attrs->max_ird > 787 rhp->attr.max_rdma_reads_per_qp) { 788 ret = -EINVAL; 789 goto out; 790 } 791 newattr.max_ird = attrs->max_ird; 792 } 793 qhp->attr = newattr; 794 } 795 796 if (!(mask & IWCH_QP_ATTR_NEXT_STATE)) 797 goto out; 798 if (qhp->attr.state == attrs->next_state) 799 goto out; 800 801 switch (qhp->attr.state) { 802 case IWCH_QP_STATE_IDLE: 803 switch (attrs->next_state) { 804 case IWCH_QP_STATE_RTS: 805 if (!(mask & IWCH_QP_ATTR_LLP_STREAM_HANDLE)) { 806 ret = -EINVAL; 807 goto out; 808 } 809 if (!(mask & IWCH_QP_ATTR_MPA_ATTR)) { 810 ret = -EINVAL; 811 goto out; 812 } 813 qhp->attr.mpa_attr = attrs->mpa_attr; 814 qhp->attr.llp_stream_handle = attrs->llp_stream_handle; 815 qhp->ep = qhp->attr.llp_stream_handle; 816 qhp->attr.state = IWCH_QP_STATE_RTS; 817 818 /* 819 * Ref the endpoint here and deref when we 820 * disassociate the endpoint from the QP. This 821 * happens in CLOSING->IDLE transition or *->ERROR 822 * transition. 823 */ 824 get_ep(&qhp->ep->com); 825 spin_unlock_irqrestore(&qhp->lock, flag); 826 ret = rdma_init(rhp, qhp, mask, attrs); 827 spin_lock_irqsave(&qhp->lock, flag); 828 if (ret) 829 goto err; 830 break; 831 case IWCH_QP_STATE_ERROR: 832 qhp->attr.state = IWCH_QP_STATE_ERROR; 833 flush_qp(qhp, &flag); 834 break; 835 default: 836 ret = -EINVAL; 837 goto out; 838 } 839 break; 840 case IWCH_QP_STATE_RTS: 841 switch (attrs->next_state) { 842 case IWCH_QP_STATE_CLOSING: 843 BUG_ON(atomic_read(&qhp->ep->com.kref.refcount) < 2); 844 qhp->attr.state = IWCH_QP_STATE_CLOSING; 845 if (!internal) { 846 abort=0; 847 disconnect = 1; 848 ep = qhp->ep; 849 } 850 break; 851 case IWCH_QP_STATE_TERMINATE: 852 qhp->attr.state = IWCH_QP_STATE_TERMINATE; 853 if (t3b_device(qhp->rhp)) 854 cxio_set_wq_in_error(&qhp->wq); 855 if (!internal) 856 terminate = 1; 857 break; 858 case IWCH_QP_STATE_ERROR: 859 qhp->attr.state = IWCH_QP_STATE_ERROR; 860 if (!internal) { 861 abort=1; 862 disconnect = 1; 863 ep = qhp->ep; 864 } 865 goto err; 866 break; 867 default: 868 ret = -EINVAL; 869 goto out; 870 } 871 break; 872 case IWCH_QP_STATE_CLOSING: 873 if (!internal) { 874 ret = -EINVAL; 875 goto out; 876 } 877 switch (attrs->next_state) { 878 case IWCH_QP_STATE_IDLE: 879 qhp->attr.state = IWCH_QP_STATE_IDLE; 880 qhp->attr.llp_stream_handle = NULL; 881 put_ep(&qhp->ep->com); 882 qhp->ep = NULL; 883 wake_up(&qhp->wait); 884 break; 885 case IWCH_QP_STATE_ERROR: 886 goto err; 887 default: 888 ret = -EINVAL; 889 goto err; 890 } 891 break; 892 case IWCH_QP_STATE_ERROR: 893 if (attrs->next_state != IWCH_QP_STATE_IDLE) { 894 ret = -EINVAL; 895 goto out; 896 } 897 898 if (!Q_EMPTY(qhp->wq.sq_rptr, qhp->wq.sq_wptr) || 899 !Q_EMPTY(qhp->wq.rq_rptr, qhp->wq.rq_wptr)) { 900 ret = -EINVAL; 901 goto out; 902 } 903 qhp->attr.state = IWCH_QP_STATE_IDLE; 904 memset(&qhp->attr, 0, sizeof(qhp->attr)); 905 break; 906 case IWCH_QP_STATE_TERMINATE: 907 if (!internal) { 908 ret = -EINVAL; 909 goto out; 910 } 911 goto err; 912 break; 913 default: 914 printk(KERN_ERR "%s in a bad state %d\n", 915 __FUNCTION__, qhp->attr.state); 916 ret = -EINVAL; 917 goto err; 918 break; 919 } 920 goto out; 921err: 922 PDBG("%s disassociating ep %p qpid 0x%x\n", __FUNCTION__, qhp->ep, 923 qhp->wq.qpid); 924 925 /* disassociate the LLP connection */ 926 qhp->attr.llp_stream_handle = NULL; 927 ep = qhp->ep; 928 qhp->ep = NULL; 929 qhp->attr.state = IWCH_QP_STATE_ERROR; 930 free=1; 931 wake_up(&qhp->wait); 932 BUG_ON(!ep); 933 flush_qp(qhp, &flag); 934out: 935 spin_unlock_irqrestore(&qhp->lock, flag); 936 937 if (terminate) 938 iwch_post_terminate(qhp, NULL); 939 940 /* 941 * If disconnect is 1, then we need to initiate a disconnect 942 * on the EP. This can be a normal close (RTS->CLOSING) or 943 * an abnormal close (RTS/CLOSING->ERROR). 944 */ 945 if (disconnect) 946 iwch_ep_disconnect(ep, abort, GFP_KERNEL); 947 948 /* 949 * If free is 1, then we've disassociated the EP from the QP 950 * and we need to dereference the EP. 951 */ 952 if (free) 953 put_ep(&ep->com); 954 955 PDBG("%s exit state %d\n", __FUNCTION__, qhp->attr.state); 956 return ret; 957} 958 959static int quiesce_qp(struct iwch_qp *qhp) 960{ 961 spin_lock_irq(&qhp->lock); 962 iwch_quiesce_tid(qhp->ep); 963 qhp->flags |= QP_QUIESCED; 964 spin_unlock_irq(&qhp->lock); 965 return 0; 966} 967 968static int resume_qp(struct iwch_qp *qhp) 969{ 970 spin_lock_irq(&qhp->lock); 971 iwch_resume_tid(qhp->ep); 972 qhp->flags &= ~QP_QUIESCED; 973 spin_unlock_irq(&qhp->lock); 974 return 0; 975} 976 977int iwch_quiesce_qps(struct iwch_cq *chp) 978{ 979 int i; 980 struct iwch_qp *qhp; 981 982 for (i=0; i < T3_MAX_NUM_QP; i++) { 983 qhp = get_qhp(chp->rhp, i); 984 if (!qhp) 985 continue; 986 if ((qhp->attr.rcq == chp->cq.cqid) && !qp_quiesced(qhp)) { 987 quiesce_qp(qhp); 988 continue; 989 } 990 if ((qhp->attr.scq == chp->cq.cqid) && !qp_quiesced(qhp)) 991 quiesce_qp(qhp); 992 } 993 return 0; 994} 995 996int iwch_resume_qps(struct iwch_cq *chp) 997{ 998 int i; 999 struct iwch_qp *qhp; 1000 1001 for (i=0; i < T3_MAX_NUM_QP; i++) { 1002 qhp = get_qhp(chp->rhp, i); 1003 if (!qhp) 1004 continue; 1005 if ((qhp->attr.rcq == chp->cq.cqid) && qp_quiesced(qhp)) { 1006 resume_qp(qhp); 1007 continue; 1008 } 1009 if ((qhp->attr.scq == chp->cq.cqid) && qp_quiesced(qhp)) 1010 resume_qp(qhp); 1011 } 1012 return 0; 1013} 1014