ehca_mrmw.c revision 2492398e616451788bc7c7905cadb8734b082bc7
1/* 2 * IBM eServer eHCA Infiniband device driver for Linux on POWER 3 * 4 * MR/MW functions 5 * 6 * Authors: Dietmar Decker <ddecker@de.ibm.com> 7 * Christoph Raisch <raisch@de.ibm.com> 8 * 9 * Copyright (c) 2005 IBM Corporation 10 * 11 * All rights reserved. 12 * 13 * This source code is distributed under a dual license of GPL v2.0 and OpenIB 14 * BSD. 15 * 16 * OpenIB BSD License 17 * 18 * Redistribution and use in source and binary forms, with or without 19 * modification, are permitted provided that the following conditions are met: 20 * 21 * Redistributions of source code must retain the above copyright notice, this 22 * list of conditions and the following disclaimer. 23 * 24 * Redistributions in binary form must reproduce the above copyright notice, 25 * this list of conditions and the following disclaimer in the documentation 26 * and/or other materials 27 * provided with the distribution. 28 * 29 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 30 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 31 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 32 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 33 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 34 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 35 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 36 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER 37 * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 38 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 39 * POSSIBILITY OF SUCH DAMAGE. 40 */ 41 42#include <rdma/ib_umem.h> 43 44#include <asm/current.h> 45 46#include "ehca_iverbs.h" 47#include "ehca_mrmw.h" 48#include "hcp_if.h" 49#include "hipz_hw.h" 50 51#define NUM_CHUNKS(length, chunk_size) \ 52 (((length) + (chunk_size - 1)) / (chunk_size)) 53/* max number of rpages (per hcall register_rpages) */ 54#define MAX_RPAGES 512 55 56static struct kmem_cache *mr_cache; 57static struct kmem_cache *mw_cache; 58 59static struct ehca_mr *ehca_mr_new(void) 60{ 61 struct ehca_mr *me; 62 63 me = kmem_cache_zalloc(mr_cache, GFP_KERNEL); 64 if (me) { 65 spin_lock_init(&me->mrlock); 66 } else 67 ehca_gen_err("alloc failed"); 68 69 return me; 70} 71 72static void ehca_mr_delete(struct ehca_mr *me) 73{ 74 kmem_cache_free(mr_cache, me); 75} 76 77static struct ehca_mw *ehca_mw_new(void) 78{ 79 struct ehca_mw *me; 80 81 me = kmem_cache_zalloc(mw_cache, GFP_KERNEL); 82 if (me) { 83 spin_lock_init(&me->mwlock); 84 } else 85 ehca_gen_err("alloc failed"); 86 87 return me; 88} 89 90static void ehca_mw_delete(struct ehca_mw *me) 91{ 92 kmem_cache_free(mw_cache, me); 93} 94 95/*----------------------------------------------------------------------*/ 96 97struct ib_mr *ehca_get_dma_mr(struct ib_pd *pd, int mr_access_flags) 98{ 99 struct ib_mr *ib_mr; 100 int ret; 101 struct ehca_mr *e_maxmr; 102 struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd); 103 struct ehca_shca *shca = 104 container_of(pd->device, struct ehca_shca, ib_device); 105 106 if (shca->maxmr) { 107 e_maxmr = ehca_mr_new(); 108 if (!e_maxmr) { 109 ehca_err(&shca->ib_device, "out of memory"); 110 ib_mr = ERR_PTR(-ENOMEM); 111 goto get_dma_mr_exit0; 112 } 113 114 ret = ehca_reg_maxmr(shca, e_maxmr, (u64*)KERNELBASE, 115 mr_access_flags, e_pd, 116 &e_maxmr->ib.ib_mr.lkey, 117 &e_maxmr->ib.ib_mr.rkey); 118 if (ret) { 119 ehca_mr_delete(e_maxmr); 120 ib_mr = ERR_PTR(ret); 121 goto get_dma_mr_exit0; 122 } 123 ib_mr = &e_maxmr->ib.ib_mr; 124 } else { 125 ehca_err(&shca->ib_device, "no internal max-MR exist!"); 126 ib_mr = ERR_PTR(-EINVAL); 127 goto get_dma_mr_exit0; 128 } 129 130get_dma_mr_exit0: 131 if (IS_ERR(ib_mr)) 132 ehca_err(&shca->ib_device, "rc=%lx pd=%p mr_access_flags=%x ", 133 PTR_ERR(ib_mr), pd, mr_access_flags); 134 return ib_mr; 135} /* end ehca_get_dma_mr() */ 136 137/*----------------------------------------------------------------------*/ 138 139struct ib_mr *ehca_reg_phys_mr(struct ib_pd *pd, 140 struct ib_phys_buf *phys_buf_array, 141 int num_phys_buf, 142 int mr_access_flags, 143 u64 *iova_start) 144{ 145 struct ib_mr *ib_mr; 146 int ret; 147 struct ehca_mr *e_mr; 148 struct ehca_shca *shca = 149 container_of(pd->device, struct ehca_shca, ib_device); 150 struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd); 151 152 u64 size; 153 struct ehca_mr_pginfo pginfo={0,0,0,0,0,0,0,NULL,0,NULL,NULL,0,NULL,0}; 154 u32 num_pages_mr; 155 u32 num_pages_4k; /* 4k portion "pages" */ 156 157 if ((num_phys_buf <= 0) || !phys_buf_array) { 158 ehca_err(pd->device, "bad input values: num_phys_buf=%x " 159 "phys_buf_array=%p", num_phys_buf, phys_buf_array); 160 ib_mr = ERR_PTR(-EINVAL); 161 goto reg_phys_mr_exit0; 162 } 163 if (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) && 164 !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)) || 165 ((mr_access_flags & IB_ACCESS_REMOTE_ATOMIC) && 166 !(mr_access_flags & IB_ACCESS_LOCAL_WRITE))) { 167 /* 168 * Remote Write Access requires Local Write Access 169 * Remote Atomic Access requires Local Write Access 170 */ 171 ehca_err(pd->device, "bad input values: mr_access_flags=%x", 172 mr_access_flags); 173 ib_mr = ERR_PTR(-EINVAL); 174 goto reg_phys_mr_exit0; 175 } 176 177 /* check physical buffer list and calculate size */ 178 ret = ehca_mr_chk_buf_and_calc_size(phys_buf_array, num_phys_buf, 179 iova_start, &size); 180 if (ret) { 181 ib_mr = ERR_PTR(ret); 182 goto reg_phys_mr_exit0; 183 } 184 if ((size == 0) || 185 (((u64)iova_start + size) < (u64)iova_start)) { 186 ehca_err(pd->device, "bad input values: size=%lx iova_start=%p", 187 size, iova_start); 188 ib_mr = ERR_PTR(-EINVAL); 189 goto reg_phys_mr_exit0; 190 } 191 192 e_mr = ehca_mr_new(); 193 if (!e_mr) { 194 ehca_err(pd->device, "out of memory"); 195 ib_mr = ERR_PTR(-ENOMEM); 196 goto reg_phys_mr_exit0; 197 } 198 199 /* determine number of MR pages */ 200 num_pages_mr = NUM_CHUNKS(((u64)iova_start % PAGE_SIZE) + size, 201 PAGE_SIZE); 202 num_pages_4k = NUM_CHUNKS(((u64)iova_start % EHCA_PAGESIZE) + size, 203 EHCA_PAGESIZE); 204 205 /* register MR on HCA */ 206 if (ehca_mr_is_maxmr(size, iova_start)) { 207 e_mr->flags |= EHCA_MR_FLAG_MAXMR; 208 ret = ehca_reg_maxmr(shca, e_mr, iova_start, mr_access_flags, 209 e_pd, &e_mr->ib.ib_mr.lkey, 210 &e_mr->ib.ib_mr.rkey); 211 if (ret) { 212 ib_mr = ERR_PTR(ret); 213 goto reg_phys_mr_exit1; 214 } 215 } else { 216 pginfo.type = EHCA_MR_PGI_PHYS; 217 pginfo.num_pages = num_pages_mr; 218 pginfo.num_4k = num_pages_4k; 219 pginfo.num_phys_buf = num_phys_buf; 220 pginfo.phys_buf_array = phys_buf_array; 221 pginfo.next_4k = (((u64)iova_start & ~PAGE_MASK) / 222 EHCA_PAGESIZE); 223 224 ret = ehca_reg_mr(shca, e_mr, iova_start, size, mr_access_flags, 225 e_pd, &pginfo, &e_mr->ib.ib_mr.lkey, 226 &e_mr->ib.ib_mr.rkey); 227 if (ret) { 228 ib_mr = ERR_PTR(ret); 229 goto reg_phys_mr_exit1; 230 } 231 } 232 233 /* successful registration of all pages */ 234 return &e_mr->ib.ib_mr; 235 236reg_phys_mr_exit1: 237 ehca_mr_delete(e_mr); 238reg_phys_mr_exit0: 239 if (IS_ERR(ib_mr)) 240 ehca_err(pd->device, "rc=%lx pd=%p phys_buf_array=%p " 241 "num_phys_buf=%x mr_access_flags=%x iova_start=%p", 242 PTR_ERR(ib_mr), pd, phys_buf_array, 243 num_phys_buf, mr_access_flags, iova_start); 244 return ib_mr; 245} /* end ehca_reg_phys_mr() */ 246 247/*----------------------------------------------------------------------*/ 248 249struct ib_mr *ehca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, u64 virt, 250 int mr_access_flags, struct ib_udata *udata) 251{ 252 struct ib_mr *ib_mr; 253 struct ehca_mr *e_mr; 254 struct ehca_shca *shca = 255 container_of(pd->device, struct ehca_shca, ib_device); 256 struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd); 257 struct ehca_mr_pginfo pginfo={0,0,0,0,0,0,0,NULL,0,NULL,NULL,0,NULL,0}; 258 int ret; 259 u32 num_pages_mr; 260 u32 num_pages_4k; /* 4k portion "pages" */ 261 262 if (!pd) { 263 ehca_gen_err("bad pd=%p", pd); 264 return ERR_PTR(-EFAULT); 265 } 266 267 if (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) && 268 !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)) || 269 ((mr_access_flags & IB_ACCESS_REMOTE_ATOMIC) && 270 !(mr_access_flags & IB_ACCESS_LOCAL_WRITE))) { 271 /* 272 * Remote Write Access requires Local Write Access 273 * Remote Atomic Access requires Local Write Access 274 */ 275 ehca_err(pd->device, "bad input values: mr_access_flags=%x", 276 mr_access_flags); 277 ib_mr = ERR_PTR(-EINVAL); 278 goto reg_user_mr_exit0; 279 } 280 281 if (length == 0 || virt + length < virt) { 282 ehca_err(pd->device, "bad input values: length=%lx " 283 "virt_base=%lx", length, virt); 284 ib_mr = ERR_PTR(-EINVAL); 285 goto reg_user_mr_exit0; 286 } 287 288 e_mr = ehca_mr_new(); 289 if (!e_mr) { 290 ehca_err(pd->device, "out of memory"); 291 ib_mr = ERR_PTR(-ENOMEM); 292 goto reg_user_mr_exit0; 293 } 294 295 e_mr->umem = ib_umem_get(pd->uobject->context, start, length, 296 mr_access_flags); 297 if (IS_ERR(e_mr->umem)) { 298 ib_mr = (void *) e_mr->umem; 299 goto reg_user_mr_exit1; 300 } 301 302 if (e_mr->umem->page_size != PAGE_SIZE) { 303 ehca_err(pd->device, "page size not supported, " 304 "e_mr->umem->page_size=%x", e_mr->umem->page_size); 305 ib_mr = ERR_PTR(-EINVAL); 306 goto reg_user_mr_exit2; 307 } 308 309 /* determine number of MR pages */ 310 num_pages_mr = NUM_CHUNKS((virt % PAGE_SIZE) + length, PAGE_SIZE); 311 num_pages_4k = NUM_CHUNKS((virt % EHCA_PAGESIZE) + length, 312 EHCA_PAGESIZE); 313 314 /* register MR on HCA */ 315 pginfo.type = EHCA_MR_PGI_USER; 316 pginfo.num_pages = num_pages_mr; 317 pginfo.num_4k = num_pages_4k; 318 pginfo.region = e_mr->umem; 319 pginfo.next_4k = e_mr->umem->offset / EHCA_PAGESIZE; 320 pginfo.next_chunk = list_prepare_entry(pginfo.next_chunk, 321 (&e_mr->umem->chunk_list), 322 list); 323 324 ret = ehca_reg_mr(shca, e_mr, (u64*) virt, length, mr_access_flags, e_pd, 325 &pginfo, &e_mr->ib.ib_mr.lkey, &e_mr->ib.ib_mr.rkey); 326 if (ret) { 327 ib_mr = ERR_PTR(ret); 328 goto reg_user_mr_exit2; 329 } 330 331 /* successful registration of all pages */ 332 return &e_mr->ib.ib_mr; 333 334reg_user_mr_exit2: 335 ib_umem_release(e_mr->umem); 336reg_user_mr_exit1: 337 ehca_mr_delete(e_mr); 338reg_user_mr_exit0: 339 if (IS_ERR(ib_mr)) 340 ehca_err(pd->device, "rc=%lx pd=%p mr_access_flags=%x" 341 " udata=%p", 342 PTR_ERR(ib_mr), pd, mr_access_flags, udata); 343 return ib_mr; 344} /* end ehca_reg_user_mr() */ 345 346/*----------------------------------------------------------------------*/ 347 348int ehca_rereg_phys_mr(struct ib_mr *mr, 349 int mr_rereg_mask, 350 struct ib_pd *pd, 351 struct ib_phys_buf *phys_buf_array, 352 int num_phys_buf, 353 int mr_access_flags, 354 u64 *iova_start) 355{ 356 int ret; 357 358 struct ehca_shca *shca = 359 container_of(mr->device, struct ehca_shca, ib_device); 360 struct ehca_mr *e_mr = container_of(mr, struct ehca_mr, ib.ib_mr); 361 struct ehca_pd *my_pd = container_of(mr->pd, struct ehca_pd, ib_pd); 362 u64 new_size; 363 u64 *new_start; 364 u32 new_acl; 365 struct ehca_pd *new_pd; 366 u32 tmp_lkey, tmp_rkey; 367 unsigned long sl_flags; 368 u32 num_pages_mr = 0; 369 u32 num_pages_4k = 0; /* 4k portion "pages" */ 370 struct ehca_mr_pginfo pginfo={0,0,0,0,0,0,0,NULL,0,NULL,NULL,0,NULL,0}; 371 u32 cur_pid = current->tgid; 372 373 if (my_pd->ib_pd.uobject && my_pd->ib_pd.uobject->context && 374 (my_pd->ownpid != cur_pid)) { 375 ehca_err(mr->device, "Invalid caller pid=%x ownpid=%x", 376 cur_pid, my_pd->ownpid); 377 ret = -EINVAL; 378 goto rereg_phys_mr_exit0; 379 } 380 381 if (!(mr_rereg_mask & IB_MR_REREG_TRANS)) { 382 /* TODO not supported, because PHYP rereg hCall needs pages */ 383 ehca_err(mr->device, "rereg without IB_MR_REREG_TRANS not " 384 "supported yet, mr_rereg_mask=%x", mr_rereg_mask); 385 ret = -EINVAL; 386 goto rereg_phys_mr_exit0; 387 } 388 389 if (mr_rereg_mask & IB_MR_REREG_PD) { 390 if (!pd) { 391 ehca_err(mr->device, "rereg with bad pd, pd=%p " 392 "mr_rereg_mask=%x", pd, mr_rereg_mask); 393 ret = -EINVAL; 394 goto rereg_phys_mr_exit0; 395 } 396 } 397 398 if ((mr_rereg_mask & 399 ~(IB_MR_REREG_TRANS | IB_MR_REREG_PD | IB_MR_REREG_ACCESS)) || 400 (mr_rereg_mask == 0)) { 401 ret = -EINVAL; 402 goto rereg_phys_mr_exit0; 403 } 404 405 /* check other parameters */ 406 if (e_mr == shca->maxmr) { 407 /* should be impossible, however reject to be sure */ 408 ehca_err(mr->device, "rereg internal max-MR impossible, mr=%p " 409 "shca->maxmr=%p mr->lkey=%x", 410 mr, shca->maxmr, mr->lkey); 411 ret = -EINVAL; 412 goto rereg_phys_mr_exit0; 413 } 414 if (mr_rereg_mask & IB_MR_REREG_TRANS) { /* transl., i.e. addr/size */ 415 if (e_mr->flags & EHCA_MR_FLAG_FMR) { 416 ehca_err(mr->device, "not supported for FMR, mr=%p " 417 "flags=%x", mr, e_mr->flags); 418 ret = -EINVAL; 419 goto rereg_phys_mr_exit0; 420 } 421 if (!phys_buf_array || num_phys_buf <= 0) { 422 ehca_err(mr->device, "bad input values: mr_rereg_mask=%x" 423 " phys_buf_array=%p num_phys_buf=%x", 424 mr_rereg_mask, phys_buf_array, num_phys_buf); 425 ret = -EINVAL; 426 goto rereg_phys_mr_exit0; 427 } 428 } 429 if ((mr_rereg_mask & IB_MR_REREG_ACCESS) && /* change ACL */ 430 (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) && 431 !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)) || 432 ((mr_access_flags & IB_ACCESS_REMOTE_ATOMIC) && 433 !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)))) { 434 /* 435 * Remote Write Access requires Local Write Access 436 * Remote Atomic Access requires Local Write Access 437 */ 438 ehca_err(mr->device, "bad input values: mr_rereg_mask=%x " 439 "mr_access_flags=%x", mr_rereg_mask, mr_access_flags); 440 ret = -EINVAL; 441 goto rereg_phys_mr_exit0; 442 } 443 444 /* set requested values dependent on rereg request */ 445 spin_lock_irqsave(&e_mr->mrlock, sl_flags); 446 new_start = e_mr->start; /* new == old address */ 447 new_size = e_mr->size; /* new == old length */ 448 new_acl = e_mr->acl; /* new == old access control */ 449 new_pd = container_of(mr->pd,struct ehca_pd,ib_pd); /*new == old PD*/ 450 451 if (mr_rereg_mask & IB_MR_REREG_TRANS) { 452 new_start = iova_start; /* change address */ 453 /* check physical buffer list and calculate size */ 454 ret = ehca_mr_chk_buf_and_calc_size(phys_buf_array, 455 num_phys_buf, iova_start, 456 &new_size); 457 if (ret) 458 goto rereg_phys_mr_exit1; 459 if ((new_size == 0) || 460 (((u64)iova_start + new_size) < (u64)iova_start)) { 461 ehca_err(mr->device, "bad input values: new_size=%lx " 462 "iova_start=%p", new_size, iova_start); 463 ret = -EINVAL; 464 goto rereg_phys_mr_exit1; 465 } 466 num_pages_mr = NUM_CHUNKS(((u64)new_start % PAGE_SIZE) + 467 new_size, PAGE_SIZE); 468 num_pages_4k = NUM_CHUNKS(((u64)new_start % EHCA_PAGESIZE) + 469 new_size, EHCA_PAGESIZE); 470 pginfo.type = EHCA_MR_PGI_PHYS; 471 pginfo.num_pages = num_pages_mr; 472 pginfo.num_4k = num_pages_4k; 473 pginfo.num_phys_buf = num_phys_buf; 474 pginfo.phys_buf_array = phys_buf_array; 475 pginfo.next_4k = (((u64)iova_start & ~PAGE_MASK) / 476 EHCA_PAGESIZE); 477 } 478 if (mr_rereg_mask & IB_MR_REREG_ACCESS) 479 new_acl = mr_access_flags; 480 if (mr_rereg_mask & IB_MR_REREG_PD) 481 new_pd = container_of(pd, struct ehca_pd, ib_pd); 482 483 ret = ehca_rereg_mr(shca, e_mr, new_start, new_size, new_acl, 484 new_pd, &pginfo, &tmp_lkey, &tmp_rkey); 485 if (ret) 486 goto rereg_phys_mr_exit1; 487 488 /* successful reregistration */ 489 if (mr_rereg_mask & IB_MR_REREG_PD) 490 mr->pd = pd; 491 mr->lkey = tmp_lkey; 492 mr->rkey = tmp_rkey; 493 494rereg_phys_mr_exit1: 495 spin_unlock_irqrestore(&e_mr->mrlock, sl_flags); 496rereg_phys_mr_exit0: 497 if (ret) 498 ehca_err(mr->device, "ret=%x mr=%p mr_rereg_mask=%x pd=%p " 499 "phys_buf_array=%p num_phys_buf=%x mr_access_flags=%x " 500 "iova_start=%p", 501 ret, mr, mr_rereg_mask, pd, phys_buf_array, 502 num_phys_buf, mr_access_flags, iova_start); 503 return ret; 504} /* end ehca_rereg_phys_mr() */ 505 506/*----------------------------------------------------------------------*/ 507 508int ehca_query_mr(struct ib_mr *mr, struct ib_mr_attr *mr_attr) 509{ 510 int ret = 0; 511 u64 h_ret; 512 struct ehca_shca *shca = 513 container_of(mr->device, struct ehca_shca, ib_device); 514 struct ehca_mr *e_mr = container_of(mr, struct ehca_mr, ib.ib_mr); 515 struct ehca_pd *my_pd = container_of(mr->pd, struct ehca_pd, ib_pd); 516 u32 cur_pid = current->tgid; 517 unsigned long sl_flags; 518 struct ehca_mr_hipzout_parms hipzout = {{0},0,0,0,0,0}; 519 520 if (my_pd->ib_pd.uobject && my_pd->ib_pd.uobject->context && 521 (my_pd->ownpid != cur_pid)) { 522 ehca_err(mr->device, "Invalid caller pid=%x ownpid=%x", 523 cur_pid, my_pd->ownpid); 524 ret = -EINVAL; 525 goto query_mr_exit0; 526 } 527 528 if ((e_mr->flags & EHCA_MR_FLAG_FMR)) { 529 ehca_err(mr->device, "not supported for FMR, mr=%p e_mr=%p " 530 "e_mr->flags=%x", mr, e_mr, e_mr->flags); 531 ret = -EINVAL; 532 goto query_mr_exit0; 533 } 534 535 memset(mr_attr, 0, sizeof(struct ib_mr_attr)); 536 spin_lock_irqsave(&e_mr->mrlock, sl_flags); 537 538 h_ret = hipz_h_query_mr(shca->ipz_hca_handle, e_mr, &hipzout); 539 if (h_ret != H_SUCCESS) { 540 ehca_err(mr->device, "hipz_mr_query failed, h_ret=%lx mr=%p " 541 "hca_hndl=%lx mr_hndl=%lx lkey=%x", 542 h_ret, mr, shca->ipz_hca_handle.handle, 543 e_mr->ipz_mr_handle.handle, mr->lkey); 544 ret = ehca2ib_return_code(h_ret); 545 goto query_mr_exit1; 546 } 547 mr_attr->pd = mr->pd; 548 mr_attr->device_virt_addr = hipzout.vaddr; 549 mr_attr->size = hipzout.len; 550 mr_attr->lkey = hipzout.lkey; 551 mr_attr->rkey = hipzout.rkey; 552 ehca_mrmw_reverse_map_acl(&hipzout.acl, &mr_attr->mr_access_flags); 553 554query_mr_exit1: 555 spin_unlock_irqrestore(&e_mr->mrlock, sl_flags); 556query_mr_exit0: 557 if (ret) 558 ehca_err(mr->device, "ret=%x mr=%p mr_attr=%p", 559 ret, mr, mr_attr); 560 return ret; 561} /* end ehca_query_mr() */ 562 563/*----------------------------------------------------------------------*/ 564 565int ehca_dereg_mr(struct ib_mr *mr) 566{ 567 int ret = 0; 568 u64 h_ret; 569 struct ehca_shca *shca = 570 container_of(mr->device, struct ehca_shca, ib_device); 571 struct ehca_mr *e_mr = container_of(mr, struct ehca_mr, ib.ib_mr); 572 struct ehca_pd *my_pd = container_of(mr->pd, struct ehca_pd, ib_pd); 573 u32 cur_pid = current->tgid; 574 575 if (my_pd->ib_pd.uobject && my_pd->ib_pd.uobject->context && 576 (my_pd->ownpid != cur_pid)) { 577 ehca_err(mr->device, "Invalid caller pid=%x ownpid=%x", 578 cur_pid, my_pd->ownpid); 579 ret = -EINVAL; 580 goto dereg_mr_exit0; 581 } 582 583 if ((e_mr->flags & EHCA_MR_FLAG_FMR)) { 584 ehca_err(mr->device, "not supported for FMR, mr=%p e_mr=%p " 585 "e_mr->flags=%x", mr, e_mr, e_mr->flags); 586 ret = -EINVAL; 587 goto dereg_mr_exit0; 588 } else if (e_mr == shca->maxmr) { 589 /* should be impossible, however reject to be sure */ 590 ehca_err(mr->device, "dereg internal max-MR impossible, mr=%p " 591 "shca->maxmr=%p mr->lkey=%x", 592 mr, shca->maxmr, mr->lkey); 593 ret = -EINVAL; 594 goto dereg_mr_exit0; 595 } 596 597 /* TODO: BUSY: MR still has bound window(s) */ 598 h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_mr); 599 if (h_ret != H_SUCCESS) { 600 ehca_err(mr->device, "hipz_free_mr failed, h_ret=%lx shca=%p " 601 "e_mr=%p hca_hndl=%lx mr_hndl=%lx mr->lkey=%x", 602 h_ret, shca, e_mr, shca->ipz_hca_handle.handle, 603 e_mr->ipz_mr_handle.handle, mr->lkey); 604 ret = ehca2ib_return_code(h_ret); 605 goto dereg_mr_exit0; 606 } 607 608 if (e_mr->umem) 609 ib_umem_release(e_mr->umem); 610 611 /* successful deregistration */ 612 ehca_mr_delete(e_mr); 613 614dereg_mr_exit0: 615 if (ret) 616 ehca_err(mr->device, "ret=%x mr=%p", ret, mr); 617 return ret; 618} /* end ehca_dereg_mr() */ 619 620/*----------------------------------------------------------------------*/ 621 622struct ib_mw *ehca_alloc_mw(struct ib_pd *pd) 623{ 624 struct ib_mw *ib_mw; 625 u64 h_ret; 626 struct ehca_mw *e_mw; 627 struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd); 628 struct ehca_shca *shca = 629 container_of(pd->device, struct ehca_shca, ib_device); 630 struct ehca_mw_hipzout_parms hipzout = {{0},0}; 631 632 e_mw = ehca_mw_new(); 633 if (!e_mw) { 634 ib_mw = ERR_PTR(-ENOMEM); 635 goto alloc_mw_exit0; 636 } 637 638 h_ret = hipz_h_alloc_resource_mw(shca->ipz_hca_handle, e_mw, 639 e_pd->fw_pd, &hipzout); 640 if (h_ret != H_SUCCESS) { 641 ehca_err(pd->device, "hipz_mw_allocate failed, h_ret=%lx " 642 "shca=%p hca_hndl=%lx mw=%p", 643 h_ret, shca, shca->ipz_hca_handle.handle, e_mw); 644 ib_mw = ERR_PTR(ehca2ib_return_code(h_ret)); 645 goto alloc_mw_exit1; 646 } 647 /* successful MW allocation */ 648 e_mw->ipz_mw_handle = hipzout.handle; 649 e_mw->ib_mw.rkey = hipzout.rkey; 650 return &e_mw->ib_mw; 651 652alloc_mw_exit1: 653 ehca_mw_delete(e_mw); 654alloc_mw_exit0: 655 if (IS_ERR(ib_mw)) 656 ehca_err(pd->device, "rc=%lx pd=%p", PTR_ERR(ib_mw), pd); 657 return ib_mw; 658} /* end ehca_alloc_mw() */ 659 660/*----------------------------------------------------------------------*/ 661 662int ehca_bind_mw(struct ib_qp *qp, 663 struct ib_mw *mw, 664 struct ib_mw_bind *mw_bind) 665{ 666 /* TODO: not supported up to now */ 667 ehca_gen_err("bind MW currently not supported by HCAD"); 668 669 return -EPERM; 670} /* end ehca_bind_mw() */ 671 672/*----------------------------------------------------------------------*/ 673 674int ehca_dealloc_mw(struct ib_mw *mw) 675{ 676 u64 h_ret; 677 struct ehca_shca *shca = 678 container_of(mw->device, struct ehca_shca, ib_device); 679 struct ehca_mw *e_mw = container_of(mw, struct ehca_mw, ib_mw); 680 681 h_ret = hipz_h_free_resource_mw(shca->ipz_hca_handle, e_mw); 682 if (h_ret != H_SUCCESS) { 683 ehca_err(mw->device, "hipz_free_mw failed, h_ret=%lx shca=%p " 684 "mw=%p rkey=%x hca_hndl=%lx mw_hndl=%lx", 685 h_ret, shca, mw, mw->rkey, shca->ipz_hca_handle.handle, 686 e_mw->ipz_mw_handle.handle); 687 return ehca2ib_return_code(h_ret); 688 } 689 /* successful deallocation */ 690 ehca_mw_delete(e_mw); 691 return 0; 692} /* end ehca_dealloc_mw() */ 693 694/*----------------------------------------------------------------------*/ 695 696struct ib_fmr *ehca_alloc_fmr(struct ib_pd *pd, 697 int mr_access_flags, 698 struct ib_fmr_attr *fmr_attr) 699{ 700 struct ib_fmr *ib_fmr; 701 struct ehca_shca *shca = 702 container_of(pd->device, struct ehca_shca, ib_device); 703 struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd); 704 struct ehca_mr *e_fmr; 705 int ret; 706 u32 tmp_lkey, tmp_rkey; 707 struct ehca_mr_pginfo pginfo={0,0,0,0,0,0,0,NULL,0,NULL,NULL,0,NULL,0}; 708 709 /* check other parameters */ 710 if (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) && 711 !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)) || 712 ((mr_access_flags & IB_ACCESS_REMOTE_ATOMIC) && 713 !(mr_access_flags & IB_ACCESS_LOCAL_WRITE))) { 714 /* 715 * Remote Write Access requires Local Write Access 716 * Remote Atomic Access requires Local Write Access 717 */ 718 ehca_err(pd->device, "bad input values: mr_access_flags=%x", 719 mr_access_flags); 720 ib_fmr = ERR_PTR(-EINVAL); 721 goto alloc_fmr_exit0; 722 } 723 if (mr_access_flags & IB_ACCESS_MW_BIND) { 724 ehca_err(pd->device, "bad input values: mr_access_flags=%x", 725 mr_access_flags); 726 ib_fmr = ERR_PTR(-EINVAL); 727 goto alloc_fmr_exit0; 728 } 729 if ((fmr_attr->max_pages == 0) || (fmr_attr->max_maps == 0)) { 730 ehca_err(pd->device, "bad input values: fmr_attr->max_pages=%x " 731 "fmr_attr->max_maps=%x fmr_attr->page_shift=%x", 732 fmr_attr->max_pages, fmr_attr->max_maps, 733 fmr_attr->page_shift); 734 ib_fmr = ERR_PTR(-EINVAL); 735 goto alloc_fmr_exit0; 736 } 737 if (((1 << fmr_attr->page_shift) != EHCA_PAGESIZE) && 738 ((1 << fmr_attr->page_shift) != PAGE_SIZE)) { 739 ehca_err(pd->device, "unsupported fmr_attr->page_shift=%x", 740 fmr_attr->page_shift); 741 ib_fmr = ERR_PTR(-EINVAL); 742 goto alloc_fmr_exit0; 743 } 744 745 e_fmr = ehca_mr_new(); 746 if (!e_fmr) { 747 ib_fmr = ERR_PTR(-ENOMEM); 748 goto alloc_fmr_exit0; 749 } 750 e_fmr->flags |= EHCA_MR_FLAG_FMR; 751 752 /* register MR on HCA */ 753 ret = ehca_reg_mr(shca, e_fmr, NULL, 754 fmr_attr->max_pages * (1 << fmr_attr->page_shift), 755 mr_access_flags, e_pd, &pginfo, 756 &tmp_lkey, &tmp_rkey); 757 if (ret) { 758 ib_fmr = ERR_PTR(ret); 759 goto alloc_fmr_exit1; 760 } 761 762 /* successful */ 763 e_fmr->fmr_page_size = 1 << fmr_attr->page_shift; 764 e_fmr->fmr_max_pages = fmr_attr->max_pages; 765 e_fmr->fmr_max_maps = fmr_attr->max_maps; 766 e_fmr->fmr_map_cnt = 0; 767 return &e_fmr->ib.ib_fmr; 768 769alloc_fmr_exit1: 770 ehca_mr_delete(e_fmr); 771alloc_fmr_exit0: 772 if (IS_ERR(ib_fmr)) 773 ehca_err(pd->device, "rc=%lx pd=%p mr_access_flags=%x " 774 "fmr_attr=%p", PTR_ERR(ib_fmr), pd, 775 mr_access_flags, fmr_attr); 776 return ib_fmr; 777} /* end ehca_alloc_fmr() */ 778 779/*----------------------------------------------------------------------*/ 780 781int ehca_map_phys_fmr(struct ib_fmr *fmr, 782 u64 *page_list, 783 int list_len, 784 u64 iova) 785{ 786 int ret; 787 struct ehca_shca *shca = 788 container_of(fmr->device, struct ehca_shca, ib_device); 789 struct ehca_mr *e_fmr = container_of(fmr, struct ehca_mr, ib.ib_fmr); 790 struct ehca_pd *e_pd = container_of(fmr->pd, struct ehca_pd, ib_pd); 791 struct ehca_mr_pginfo pginfo={0,0,0,0,0,0,0,NULL,0,NULL,NULL,0,NULL,0}; 792 u32 tmp_lkey, tmp_rkey; 793 794 if (!(e_fmr->flags & EHCA_MR_FLAG_FMR)) { 795 ehca_err(fmr->device, "not a FMR, e_fmr=%p e_fmr->flags=%x", 796 e_fmr, e_fmr->flags); 797 ret = -EINVAL; 798 goto map_phys_fmr_exit0; 799 } 800 ret = ehca_fmr_check_page_list(e_fmr, page_list, list_len); 801 if (ret) 802 goto map_phys_fmr_exit0; 803 if (iova % e_fmr->fmr_page_size) { 804 /* only whole-numbered pages */ 805 ehca_err(fmr->device, "bad iova, iova=%lx fmr_page_size=%x", 806 iova, e_fmr->fmr_page_size); 807 ret = -EINVAL; 808 goto map_phys_fmr_exit0; 809 } 810 if (e_fmr->fmr_map_cnt >= e_fmr->fmr_max_maps) { 811 /* HCAD does not limit the maps, however trace this anyway */ 812 ehca_info(fmr->device, "map limit exceeded, fmr=%p " 813 "e_fmr->fmr_map_cnt=%x e_fmr->fmr_max_maps=%x", 814 fmr, e_fmr->fmr_map_cnt, e_fmr->fmr_max_maps); 815 } 816 817 pginfo.type = EHCA_MR_PGI_FMR; 818 pginfo.num_pages = list_len; 819 pginfo.num_4k = list_len * (e_fmr->fmr_page_size / EHCA_PAGESIZE); 820 pginfo.page_list = page_list; 821 pginfo.next_4k = ((iova & (e_fmr->fmr_page_size-1)) / 822 EHCA_PAGESIZE); 823 824 ret = ehca_rereg_mr(shca, e_fmr, (u64*)iova, 825 list_len * e_fmr->fmr_page_size, 826 e_fmr->acl, e_pd, &pginfo, &tmp_lkey, &tmp_rkey); 827 if (ret) 828 goto map_phys_fmr_exit0; 829 830 /* successful reregistration */ 831 e_fmr->fmr_map_cnt++; 832 e_fmr->ib.ib_fmr.lkey = tmp_lkey; 833 e_fmr->ib.ib_fmr.rkey = tmp_rkey; 834 return 0; 835 836map_phys_fmr_exit0: 837 if (ret) 838 ehca_err(fmr->device, "ret=%x fmr=%p page_list=%p list_len=%x " 839 "iova=%lx", 840 ret, fmr, page_list, list_len, iova); 841 return ret; 842} /* end ehca_map_phys_fmr() */ 843 844/*----------------------------------------------------------------------*/ 845 846int ehca_unmap_fmr(struct list_head *fmr_list) 847{ 848 int ret = 0; 849 struct ib_fmr *ib_fmr; 850 struct ehca_shca *shca = NULL; 851 struct ehca_shca *prev_shca; 852 struct ehca_mr *e_fmr; 853 u32 num_fmr = 0; 854 u32 unmap_fmr_cnt = 0; 855 856 /* check all FMR belong to same SHCA, and check internal flag */ 857 list_for_each_entry(ib_fmr, fmr_list, list) { 858 prev_shca = shca; 859 if (!ib_fmr) { 860 ehca_gen_err("bad fmr=%p in list", ib_fmr); 861 ret = -EINVAL; 862 goto unmap_fmr_exit0; 863 } 864 shca = container_of(ib_fmr->device, struct ehca_shca, 865 ib_device); 866 e_fmr = container_of(ib_fmr, struct ehca_mr, ib.ib_fmr); 867 if ((shca != prev_shca) && prev_shca) { 868 ehca_err(&shca->ib_device, "SHCA mismatch, shca=%p " 869 "prev_shca=%p e_fmr=%p", 870 shca, prev_shca, e_fmr); 871 ret = -EINVAL; 872 goto unmap_fmr_exit0; 873 } 874 if (!(e_fmr->flags & EHCA_MR_FLAG_FMR)) { 875 ehca_err(&shca->ib_device, "not a FMR, e_fmr=%p " 876 "e_fmr->flags=%x", e_fmr, e_fmr->flags); 877 ret = -EINVAL; 878 goto unmap_fmr_exit0; 879 } 880 num_fmr++; 881 } 882 883 /* loop over all FMRs to unmap */ 884 list_for_each_entry(ib_fmr, fmr_list, list) { 885 unmap_fmr_cnt++; 886 e_fmr = container_of(ib_fmr, struct ehca_mr, ib.ib_fmr); 887 shca = container_of(ib_fmr->device, struct ehca_shca, 888 ib_device); 889 ret = ehca_unmap_one_fmr(shca, e_fmr); 890 if (ret) { 891 /* unmap failed, stop unmapping of rest of FMRs */ 892 ehca_err(&shca->ib_device, "unmap of one FMR failed, " 893 "stop rest, e_fmr=%p num_fmr=%x " 894 "unmap_fmr_cnt=%x lkey=%x", e_fmr, num_fmr, 895 unmap_fmr_cnt, e_fmr->ib.ib_fmr.lkey); 896 goto unmap_fmr_exit0; 897 } 898 } 899 900unmap_fmr_exit0: 901 if (ret) 902 ehca_gen_err("ret=%x fmr_list=%p num_fmr=%x unmap_fmr_cnt=%x", 903 ret, fmr_list, num_fmr, unmap_fmr_cnt); 904 return ret; 905} /* end ehca_unmap_fmr() */ 906 907/*----------------------------------------------------------------------*/ 908 909int ehca_dealloc_fmr(struct ib_fmr *fmr) 910{ 911 int ret; 912 u64 h_ret; 913 struct ehca_shca *shca = 914 container_of(fmr->device, struct ehca_shca, ib_device); 915 struct ehca_mr *e_fmr = container_of(fmr, struct ehca_mr, ib.ib_fmr); 916 917 if (!(e_fmr->flags & EHCA_MR_FLAG_FMR)) { 918 ehca_err(fmr->device, "not a FMR, e_fmr=%p e_fmr->flags=%x", 919 e_fmr, e_fmr->flags); 920 ret = -EINVAL; 921 goto free_fmr_exit0; 922 } 923 924 h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_fmr); 925 if (h_ret != H_SUCCESS) { 926 ehca_err(fmr->device, "hipz_free_mr failed, h_ret=%lx e_fmr=%p " 927 "hca_hndl=%lx fmr_hndl=%lx fmr->lkey=%x", 928 h_ret, e_fmr, shca->ipz_hca_handle.handle, 929 e_fmr->ipz_mr_handle.handle, fmr->lkey); 930 ret = ehca2ib_return_code(h_ret); 931 goto free_fmr_exit0; 932 } 933 /* successful deregistration */ 934 ehca_mr_delete(e_fmr); 935 return 0; 936 937free_fmr_exit0: 938 if (ret) 939 ehca_err(&shca->ib_device, "ret=%x fmr=%p", ret, fmr); 940 return ret; 941} /* end ehca_dealloc_fmr() */ 942 943/*----------------------------------------------------------------------*/ 944 945int ehca_reg_mr(struct ehca_shca *shca, 946 struct ehca_mr *e_mr, 947 u64 *iova_start, 948 u64 size, 949 int acl, 950 struct ehca_pd *e_pd, 951 struct ehca_mr_pginfo *pginfo, 952 u32 *lkey, /*OUT*/ 953 u32 *rkey) /*OUT*/ 954{ 955 int ret; 956 u64 h_ret; 957 u32 hipz_acl; 958 struct ehca_mr_hipzout_parms hipzout = {{0},0,0,0,0,0}; 959 960 ehca_mrmw_map_acl(acl, &hipz_acl); 961 ehca_mrmw_set_pgsize_hipz_acl(&hipz_acl); 962 if (ehca_use_hp_mr == 1) 963 hipz_acl |= 0x00000001; 964 965 h_ret = hipz_h_alloc_resource_mr(shca->ipz_hca_handle, e_mr, 966 (u64)iova_start, size, hipz_acl, 967 e_pd->fw_pd, &hipzout); 968 if (h_ret != H_SUCCESS) { 969 ehca_err(&shca->ib_device, "hipz_alloc_mr failed, h_ret=%lx " 970 "hca_hndl=%lx", h_ret, shca->ipz_hca_handle.handle); 971 ret = ehca2ib_return_code(h_ret); 972 goto ehca_reg_mr_exit0; 973 } 974 975 e_mr->ipz_mr_handle = hipzout.handle; 976 977 ret = ehca_reg_mr_rpages(shca, e_mr, pginfo); 978 if (ret) 979 goto ehca_reg_mr_exit1; 980 981 /* successful registration */ 982 e_mr->num_pages = pginfo->num_pages; 983 e_mr->num_4k = pginfo->num_4k; 984 e_mr->start = iova_start; 985 e_mr->size = size; 986 e_mr->acl = acl; 987 *lkey = hipzout.lkey; 988 *rkey = hipzout.rkey; 989 return 0; 990 991ehca_reg_mr_exit1: 992 h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_mr); 993 if (h_ret != H_SUCCESS) { 994 ehca_err(&shca->ib_device, "h_ret=%lx shca=%p e_mr=%p " 995 "iova_start=%p size=%lx acl=%x e_pd=%p lkey=%x " 996 "pginfo=%p num_pages=%lx num_4k=%lx ret=%x", 997 h_ret, shca, e_mr, iova_start, size, acl, e_pd, 998 hipzout.lkey, pginfo, pginfo->num_pages, 999 pginfo->num_4k, ret); 1000 ehca_err(&shca->ib_device, "internal error in ehca_reg_mr, " 1001 "not recoverable"); 1002 } 1003ehca_reg_mr_exit0: 1004 if (ret) 1005 ehca_err(&shca->ib_device, "ret=%x shca=%p e_mr=%p " 1006 "iova_start=%p size=%lx acl=%x e_pd=%p pginfo=%p " 1007 "num_pages=%lx num_4k=%lx", 1008 ret, shca, e_mr, iova_start, size, acl, e_pd, pginfo, 1009 pginfo->num_pages, pginfo->num_4k); 1010 return ret; 1011} /* end ehca_reg_mr() */ 1012 1013/*----------------------------------------------------------------------*/ 1014 1015int ehca_reg_mr_rpages(struct ehca_shca *shca, 1016 struct ehca_mr *e_mr, 1017 struct ehca_mr_pginfo *pginfo) 1018{ 1019 int ret = 0; 1020 u64 h_ret; 1021 u32 rnum; 1022 u64 rpage; 1023 u32 i; 1024 u64 *kpage; 1025 1026 kpage = ehca_alloc_fw_ctrlblock(GFP_KERNEL); 1027 if (!kpage) { 1028 ehca_err(&shca->ib_device, "kpage alloc failed"); 1029 ret = -ENOMEM; 1030 goto ehca_reg_mr_rpages_exit0; 1031 } 1032 1033 /* max 512 pages per shot */ 1034 for (i = 0; i < NUM_CHUNKS(pginfo->num_4k, MAX_RPAGES); i++) { 1035 1036 if (i == NUM_CHUNKS(pginfo->num_4k, MAX_RPAGES) - 1) { 1037 rnum = pginfo->num_4k % MAX_RPAGES; /* last shot */ 1038 if (rnum == 0) 1039 rnum = MAX_RPAGES; /* last shot is full */ 1040 } else 1041 rnum = MAX_RPAGES; 1042 1043 if (rnum > 1) { 1044 ret = ehca_set_pagebuf(e_mr, pginfo, rnum, kpage); 1045 if (ret) { 1046 ehca_err(&shca->ib_device, "ehca_set_pagebuf " 1047 "bad rc, ret=%x rnum=%x kpage=%p", 1048 ret, rnum, kpage); 1049 ret = -EFAULT; 1050 goto ehca_reg_mr_rpages_exit1; 1051 } 1052 rpage = virt_to_abs(kpage); 1053 if (!rpage) { 1054 ehca_err(&shca->ib_device, "kpage=%p i=%x", 1055 kpage, i); 1056 ret = -EFAULT; 1057 goto ehca_reg_mr_rpages_exit1; 1058 } 1059 } else { /* rnum==1 */ 1060 ret = ehca_set_pagebuf_1(e_mr, pginfo, &rpage); 1061 if (ret) { 1062 ehca_err(&shca->ib_device, "ehca_set_pagebuf_1 " 1063 "bad rc, ret=%x i=%x", ret, i); 1064 ret = -EFAULT; 1065 goto ehca_reg_mr_rpages_exit1; 1066 } 1067 } 1068 1069 h_ret = hipz_h_register_rpage_mr(shca->ipz_hca_handle, e_mr, 1070 0, /* pagesize 4k */ 1071 0, rpage, rnum); 1072 1073 if (i == NUM_CHUNKS(pginfo->num_4k, MAX_RPAGES) - 1) { 1074 /* 1075 * check for 'registration complete'==H_SUCCESS 1076 * and for 'page registered'==H_PAGE_REGISTERED 1077 */ 1078 if (h_ret != H_SUCCESS) { 1079 ehca_err(&shca->ib_device, "last " 1080 "hipz_reg_rpage_mr failed, h_ret=%lx " 1081 "e_mr=%p i=%x hca_hndl=%lx mr_hndl=%lx" 1082 " lkey=%x", h_ret, e_mr, i, 1083 shca->ipz_hca_handle.handle, 1084 e_mr->ipz_mr_handle.handle, 1085 e_mr->ib.ib_mr.lkey); 1086 ret = ehca2ib_return_code(h_ret); 1087 break; 1088 } else 1089 ret = 0; 1090 } else if (h_ret != H_PAGE_REGISTERED) { 1091 ehca_err(&shca->ib_device, "hipz_reg_rpage_mr failed, " 1092 "h_ret=%lx e_mr=%p i=%x lkey=%x hca_hndl=%lx " 1093 "mr_hndl=%lx", h_ret, e_mr, i, 1094 e_mr->ib.ib_mr.lkey, 1095 shca->ipz_hca_handle.handle, 1096 e_mr->ipz_mr_handle.handle); 1097 ret = ehca2ib_return_code(h_ret); 1098 break; 1099 } else 1100 ret = 0; 1101 } /* end for(i) */ 1102 1103 1104ehca_reg_mr_rpages_exit1: 1105 ehca_free_fw_ctrlblock(kpage); 1106ehca_reg_mr_rpages_exit0: 1107 if (ret) 1108 ehca_err(&shca->ib_device, "ret=%x shca=%p e_mr=%p pginfo=%p " 1109 "num_pages=%lx num_4k=%lx", ret, shca, e_mr, pginfo, 1110 pginfo->num_pages, pginfo->num_4k); 1111 return ret; 1112} /* end ehca_reg_mr_rpages() */ 1113 1114/*----------------------------------------------------------------------*/ 1115 1116inline int ehca_rereg_mr_rereg1(struct ehca_shca *shca, 1117 struct ehca_mr *e_mr, 1118 u64 *iova_start, 1119 u64 size, 1120 u32 acl, 1121 struct ehca_pd *e_pd, 1122 struct ehca_mr_pginfo *pginfo, 1123 u32 *lkey, /*OUT*/ 1124 u32 *rkey) /*OUT*/ 1125{ 1126 int ret; 1127 u64 h_ret; 1128 u32 hipz_acl; 1129 u64 *kpage; 1130 u64 rpage; 1131 struct ehca_mr_pginfo pginfo_save; 1132 struct ehca_mr_hipzout_parms hipzout = {{0},0,0,0,0,0}; 1133 1134 ehca_mrmw_map_acl(acl, &hipz_acl); 1135 ehca_mrmw_set_pgsize_hipz_acl(&hipz_acl); 1136 1137 kpage = ehca_alloc_fw_ctrlblock(GFP_KERNEL); 1138 if (!kpage) { 1139 ehca_err(&shca->ib_device, "kpage alloc failed"); 1140 ret = -ENOMEM; 1141 goto ehca_rereg_mr_rereg1_exit0; 1142 } 1143 1144 pginfo_save = *pginfo; 1145 ret = ehca_set_pagebuf(e_mr, pginfo, pginfo->num_4k, kpage); 1146 if (ret) { 1147 ehca_err(&shca->ib_device, "set pagebuf failed, e_mr=%p " 1148 "pginfo=%p type=%x num_pages=%lx num_4k=%lx kpage=%p", 1149 e_mr, pginfo, pginfo->type, pginfo->num_pages, 1150 pginfo->num_4k,kpage); 1151 goto ehca_rereg_mr_rereg1_exit1; 1152 } 1153 rpage = virt_to_abs(kpage); 1154 if (!rpage) { 1155 ehca_err(&shca->ib_device, "kpage=%p", kpage); 1156 ret = -EFAULT; 1157 goto ehca_rereg_mr_rereg1_exit1; 1158 } 1159 h_ret = hipz_h_reregister_pmr(shca->ipz_hca_handle, e_mr, 1160 (u64)iova_start, size, hipz_acl, 1161 e_pd->fw_pd, rpage, &hipzout); 1162 if (h_ret != H_SUCCESS) { 1163 /* 1164 * reregistration unsuccessful, try it again with the 3 hCalls, 1165 * e.g. this is required in case H_MR_CONDITION 1166 * (MW bound or MR is shared) 1167 */ 1168 ehca_warn(&shca->ib_device, "hipz_h_reregister_pmr failed " 1169 "(Rereg1), h_ret=%lx e_mr=%p", h_ret, e_mr); 1170 *pginfo = pginfo_save; 1171 ret = -EAGAIN; 1172 } else if ((u64*)hipzout.vaddr != iova_start) { 1173 ehca_err(&shca->ib_device, "PHYP changed iova_start in " 1174 "rereg_pmr, iova_start=%p iova_start_out=%lx e_mr=%p " 1175 "mr_handle=%lx lkey=%x lkey_out=%x", iova_start, 1176 hipzout.vaddr, e_mr, e_mr->ipz_mr_handle.handle, 1177 e_mr->ib.ib_mr.lkey, hipzout.lkey); 1178 ret = -EFAULT; 1179 } else { 1180 /* 1181 * successful reregistration 1182 * note: start and start_out are identical for eServer HCAs 1183 */ 1184 e_mr->num_pages = pginfo->num_pages; 1185 e_mr->num_4k = pginfo->num_4k; 1186 e_mr->start = iova_start; 1187 e_mr->size = size; 1188 e_mr->acl = acl; 1189 *lkey = hipzout.lkey; 1190 *rkey = hipzout.rkey; 1191 } 1192 1193ehca_rereg_mr_rereg1_exit1: 1194 ehca_free_fw_ctrlblock(kpage); 1195ehca_rereg_mr_rereg1_exit0: 1196 if ( ret && (ret != -EAGAIN) ) 1197 ehca_err(&shca->ib_device, "ret=%x lkey=%x rkey=%x " 1198 "pginfo=%p num_pages=%lx num_4k=%lx", 1199 ret, *lkey, *rkey, pginfo, pginfo->num_pages, 1200 pginfo->num_4k); 1201 return ret; 1202} /* end ehca_rereg_mr_rereg1() */ 1203 1204/*----------------------------------------------------------------------*/ 1205 1206int ehca_rereg_mr(struct ehca_shca *shca, 1207 struct ehca_mr *e_mr, 1208 u64 *iova_start, 1209 u64 size, 1210 int acl, 1211 struct ehca_pd *e_pd, 1212 struct ehca_mr_pginfo *pginfo, 1213 u32 *lkey, 1214 u32 *rkey) 1215{ 1216 int ret = 0; 1217 u64 h_ret; 1218 int rereg_1_hcall = 1; /* 1: use hipz_h_reregister_pmr directly */ 1219 int rereg_3_hcall = 0; /* 1: use 3 hipz calls for reregistration */ 1220 1221 /* first determine reregistration hCall(s) */ 1222 if ((pginfo->num_4k > MAX_RPAGES) || (e_mr->num_4k > MAX_RPAGES) || 1223 (pginfo->num_4k > e_mr->num_4k)) { 1224 ehca_dbg(&shca->ib_device, "Rereg3 case, pginfo->num_4k=%lx " 1225 "e_mr->num_4k=%x", pginfo->num_4k, e_mr->num_4k); 1226 rereg_1_hcall = 0; 1227 rereg_3_hcall = 1; 1228 } 1229 1230 if (e_mr->flags & EHCA_MR_FLAG_MAXMR) { /* check for max-MR */ 1231 rereg_1_hcall = 0; 1232 rereg_3_hcall = 1; 1233 e_mr->flags &= ~EHCA_MR_FLAG_MAXMR; 1234 ehca_err(&shca->ib_device, "Rereg MR for max-MR! e_mr=%p", 1235 e_mr); 1236 } 1237 1238 if (rereg_1_hcall) { 1239 ret = ehca_rereg_mr_rereg1(shca, e_mr, iova_start, size, 1240 acl, e_pd, pginfo, lkey, rkey); 1241 if (ret) { 1242 if (ret == -EAGAIN) 1243 rereg_3_hcall = 1; 1244 else 1245 goto ehca_rereg_mr_exit0; 1246 } 1247 } 1248 1249 if (rereg_3_hcall) { 1250 struct ehca_mr save_mr; 1251 1252 /* first deregister old MR */ 1253 h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_mr); 1254 if (h_ret != H_SUCCESS) { 1255 ehca_err(&shca->ib_device, "hipz_free_mr failed, " 1256 "h_ret=%lx e_mr=%p hca_hndl=%lx mr_hndl=%lx " 1257 "mr->lkey=%x", 1258 h_ret, e_mr, shca->ipz_hca_handle.handle, 1259 e_mr->ipz_mr_handle.handle, 1260 e_mr->ib.ib_mr.lkey); 1261 ret = ehca2ib_return_code(h_ret); 1262 goto ehca_rereg_mr_exit0; 1263 } 1264 /* clean ehca_mr_t, without changing struct ib_mr and lock */ 1265 save_mr = *e_mr; 1266 ehca_mr_deletenew(e_mr); 1267 1268 /* set some MR values */ 1269 e_mr->flags = save_mr.flags; 1270 e_mr->fmr_page_size = save_mr.fmr_page_size; 1271 e_mr->fmr_max_pages = save_mr.fmr_max_pages; 1272 e_mr->fmr_max_maps = save_mr.fmr_max_maps; 1273 e_mr->fmr_map_cnt = save_mr.fmr_map_cnt; 1274 1275 ret = ehca_reg_mr(shca, e_mr, iova_start, size, acl, 1276 e_pd, pginfo, lkey, rkey); 1277 if (ret) { 1278 u32 offset = (u64)(&e_mr->flags) - (u64)e_mr; 1279 memcpy(&e_mr->flags, &(save_mr.flags), 1280 sizeof(struct ehca_mr) - offset); 1281 goto ehca_rereg_mr_exit0; 1282 } 1283 } 1284 1285ehca_rereg_mr_exit0: 1286 if (ret) 1287 ehca_err(&shca->ib_device, "ret=%x shca=%p e_mr=%p " 1288 "iova_start=%p size=%lx acl=%x e_pd=%p pginfo=%p " 1289 "num_pages=%lx lkey=%x rkey=%x rereg_1_hcall=%x " 1290 "rereg_3_hcall=%x", ret, shca, e_mr, iova_start, size, 1291 acl, e_pd, pginfo, pginfo->num_pages, *lkey, *rkey, 1292 rereg_1_hcall, rereg_3_hcall); 1293 return ret; 1294} /* end ehca_rereg_mr() */ 1295 1296/*----------------------------------------------------------------------*/ 1297 1298int ehca_unmap_one_fmr(struct ehca_shca *shca, 1299 struct ehca_mr *e_fmr) 1300{ 1301 int ret = 0; 1302 u64 h_ret; 1303 int rereg_1_hcall = 1; /* 1: use hipz_mr_reregister directly */ 1304 int rereg_3_hcall = 0; /* 1: use 3 hipz calls for unmapping */ 1305 struct ehca_pd *e_pd = 1306 container_of(e_fmr->ib.ib_fmr.pd, struct ehca_pd, ib_pd); 1307 struct ehca_mr save_fmr; 1308 u32 tmp_lkey, tmp_rkey; 1309 struct ehca_mr_pginfo pginfo={0,0,0,0,0,0,0,NULL,0,NULL,NULL,0,NULL,0}; 1310 struct ehca_mr_hipzout_parms hipzout = {{0},0,0,0,0,0}; 1311 1312 /* first check if reregistration hCall can be used for unmap */ 1313 if (e_fmr->fmr_max_pages > MAX_RPAGES) { 1314 rereg_1_hcall = 0; 1315 rereg_3_hcall = 1; 1316 } 1317 1318 if (rereg_1_hcall) { 1319 /* 1320 * note: after using rereg hcall with len=0, 1321 * rereg hcall must be used again for registering pages 1322 */ 1323 h_ret = hipz_h_reregister_pmr(shca->ipz_hca_handle, e_fmr, 0, 1324 0, 0, e_pd->fw_pd, 0, &hipzout); 1325 if (h_ret != H_SUCCESS) { 1326 /* 1327 * should not happen, because length checked above, 1328 * FMRs are not shared and no MW bound to FMRs 1329 */ 1330 ehca_err(&shca->ib_device, "hipz_reregister_pmr failed " 1331 "(Rereg1), h_ret=%lx e_fmr=%p hca_hndl=%lx " 1332 "mr_hndl=%lx lkey=%x lkey_out=%x", 1333 h_ret, e_fmr, shca->ipz_hca_handle.handle, 1334 e_fmr->ipz_mr_handle.handle, 1335 e_fmr->ib.ib_fmr.lkey, hipzout.lkey); 1336 rereg_3_hcall = 1; 1337 } else { 1338 /* successful reregistration */ 1339 e_fmr->start = NULL; 1340 e_fmr->size = 0; 1341 tmp_lkey = hipzout.lkey; 1342 tmp_rkey = hipzout.rkey; 1343 } 1344 } 1345 1346 if (rereg_3_hcall) { 1347 struct ehca_mr save_mr; 1348 1349 /* first free old FMR */ 1350 h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_fmr); 1351 if (h_ret != H_SUCCESS) { 1352 ehca_err(&shca->ib_device, "hipz_free_mr failed, " 1353 "h_ret=%lx e_fmr=%p hca_hndl=%lx mr_hndl=%lx " 1354 "lkey=%x", 1355 h_ret, e_fmr, shca->ipz_hca_handle.handle, 1356 e_fmr->ipz_mr_handle.handle, 1357 e_fmr->ib.ib_fmr.lkey); 1358 ret = ehca2ib_return_code(h_ret); 1359 goto ehca_unmap_one_fmr_exit0; 1360 } 1361 /* clean ehca_mr_t, without changing lock */ 1362 save_fmr = *e_fmr; 1363 ehca_mr_deletenew(e_fmr); 1364 1365 /* set some MR values */ 1366 e_fmr->flags = save_fmr.flags; 1367 e_fmr->fmr_page_size = save_fmr.fmr_page_size; 1368 e_fmr->fmr_max_pages = save_fmr.fmr_max_pages; 1369 e_fmr->fmr_max_maps = save_fmr.fmr_max_maps; 1370 e_fmr->fmr_map_cnt = save_fmr.fmr_map_cnt; 1371 e_fmr->acl = save_fmr.acl; 1372 1373 pginfo.type = EHCA_MR_PGI_FMR; 1374 pginfo.num_pages = 0; 1375 pginfo.num_4k = 0; 1376 ret = ehca_reg_mr(shca, e_fmr, NULL, 1377 (e_fmr->fmr_max_pages * e_fmr->fmr_page_size), 1378 e_fmr->acl, e_pd, &pginfo, &tmp_lkey, 1379 &tmp_rkey); 1380 if (ret) { 1381 u32 offset = (u64)(&e_fmr->flags) - (u64)e_fmr; 1382 memcpy(&e_fmr->flags, &(save_mr.flags), 1383 sizeof(struct ehca_mr) - offset); 1384 goto ehca_unmap_one_fmr_exit0; 1385 } 1386 } 1387 1388ehca_unmap_one_fmr_exit0: 1389 if (ret) 1390 ehca_err(&shca->ib_device, "ret=%x tmp_lkey=%x tmp_rkey=%x " 1391 "fmr_max_pages=%x rereg_1_hcall=%x rereg_3_hcall=%x", 1392 ret, tmp_lkey, tmp_rkey, e_fmr->fmr_max_pages, 1393 rereg_1_hcall, rereg_3_hcall); 1394 return ret; 1395} /* end ehca_unmap_one_fmr() */ 1396 1397/*----------------------------------------------------------------------*/ 1398 1399int ehca_reg_smr(struct ehca_shca *shca, 1400 struct ehca_mr *e_origmr, 1401 struct ehca_mr *e_newmr, 1402 u64 *iova_start, 1403 int acl, 1404 struct ehca_pd *e_pd, 1405 u32 *lkey, /*OUT*/ 1406 u32 *rkey) /*OUT*/ 1407{ 1408 int ret = 0; 1409 u64 h_ret; 1410 u32 hipz_acl; 1411 struct ehca_mr_hipzout_parms hipzout = {{0},0,0,0,0,0}; 1412 1413 ehca_mrmw_map_acl(acl, &hipz_acl); 1414 ehca_mrmw_set_pgsize_hipz_acl(&hipz_acl); 1415 1416 h_ret = hipz_h_register_smr(shca->ipz_hca_handle, e_newmr, e_origmr, 1417 (u64)iova_start, hipz_acl, e_pd->fw_pd, 1418 &hipzout); 1419 if (h_ret != H_SUCCESS) { 1420 ehca_err(&shca->ib_device, "hipz_reg_smr failed, h_ret=%lx " 1421 "shca=%p e_origmr=%p e_newmr=%p iova_start=%p acl=%x " 1422 "e_pd=%p hca_hndl=%lx mr_hndl=%lx lkey=%x", 1423 h_ret, shca, e_origmr, e_newmr, iova_start, acl, e_pd, 1424 shca->ipz_hca_handle.handle, 1425 e_origmr->ipz_mr_handle.handle, 1426 e_origmr->ib.ib_mr.lkey); 1427 ret = ehca2ib_return_code(h_ret); 1428 goto ehca_reg_smr_exit0; 1429 } 1430 /* successful registration */ 1431 e_newmr->num_pages = e_origmr->num_pages; 1432 e_newmr->num_4k = e_origmr->num_4k; 1433 e_newmr->start = iova_start; 1434 e_newmr->size = e_origmr->size; 1435 e_newmr->acl = acl; 1436 e_newmr->ipz_mr_handle = hipzout.handle; 1437 *lkey = hipzout.lkey; 1438 *rkey = hipzout.rkey; 1439 return 0; 1440 1441ehca_reg_smr_exit0: 1442 if (ret) 1443 ehca_err(&shca->ib_device, "ret=%x shca=%p e_origmr=%p " 1444 "e_newmr=%p iova_start=%p acl=%x e_pd=%p", 1445 ret, shca, e_origmr, e_newmr, iova_start, acl, e_pd); 1446 return ret; 1447} /* end ehca_reg_smr() */ 1448 1449/*----------------------------------------------------------------------*/ 1450 1451/* register internal max-MR to internal SHCA */ 1452int ehca_reg_internal_maxmr( 1453 struct ehca_shca *shca, 1454 struct ehca_pd *e_pd, 1455 struct ehca_mr **e_maxmr) /*OUT*/ 1456{ 1457 int ret; 1458 struct ehca_mr *e_mr; 1459 u64 *iova_start; 1460 u64 size_maxmr; 1461 struct ehca_mr_pginfo pginfo={0,0,0,0,0,0,0,NULL,0,NULL,NULL,0,NULL,0}; 1462 struct ib_phys_buf ib_pbuf; 1463 u32 num_pages_mr; 1464 u32 num_pages_4k; /* 4k portion "pages" */ 1465 1466 e_mr = ehca_mr_new(); 1467 if (!e_mr) { 1468 ehca_err(&shca->ib_device, "out of memory"); 1469 ret = -ENOMEM; 1470 goto ehca_reg_internal_maxmr_exit0; 1471 } 1472 e_mr->flags |= EHCA_MR_FLAG_MAXMR; 1473 1474 /* register internal max-MR on HCA */ 1475 size_maxmr = (u64)high_memory - PAGE_OFFSET; 1476 iova_start = (u64*)KERNELBASE; 1477 ib_pbuf.addr = 0; 1478 ib_pbuf.size = size_maxmr; 1479 num_pages_mr = NUM_CHUNKS(((u64)iova_start % PAGE_SIZE) + size_maxmr, 1480 PAGE_SIZE); 1481 num_pages_4k = NUM_CHUNKS(((u64)iova_start % EHCA_PAGESIZE) 1482 + size_maxmr, EHCA_PAGESIZE); 1483 1484 pginfo.type = EHCA_MR_PGI_PHYS; 1485 pginfo.num_pages = num_pages_mr; 1486 pginfo.num_4k = num_pages_4k; 1487 pginfo.num_phys_buf = 1; 1488 pginfo.phys_buf_array = &ib_pbuf; 1489 1490 ret = ehca_reg_mr(shca, e_mr, iova_start, size_maxmr, 0, e_pd, 1491 &pginfo, &e_mr->ib.ib_mr.lkey, 1492 &e_mr->ib.ib_mr.rkey); 1493 if (ret) { 1494 ehca_err(&shca->ib_device, "reg of internal max MR failed, " 1495 "e_mr=%p iova_start=%p size_maxmr=%lx num_pages_mr=%x " 1496 "num_pages_4k=%x", e_mr, iova_start, size_maxmr, 1497 num_pages_mr, num_pages_4k); 1498 goto ehca_reg_internal_maxmr_exit1; 1499 } 1500 1501 /* successful registration of all pages */ 1502 e_mr->ib.ib_mr.device = e_pd->ib_pd.device; 1503 e_mr->ib.ib_mr.pd = &e_pd->ib_pd; 1504 e_mr->ib.ib_mr.uobject = NULL; 1505 atomic_inc(&(e_pd->ib_pd.usecnt)); 1506 atomic_set(&(e_mr->ib.ib_mr.usecnt), 0); 1507 *e_maxmr = e_mr; 1508 return 0; 1509 1510ehca_reg_internal_maxmr_exit1: 1511 ehca_mr_delete(e_mr); 1512ehca_reg_internal_maxmr_exit0: 1513 if (ret) 1514 ehca_err(&shca->ib_device, "ret=%x shca=%p e_pd=%p e_maxmr=%p", 1515 ret, shca, e_pd, e_maxmr); 1516 return ret; 1517} /* end ehca_reg_internal_maxmr() */ 1518 1519/*----------------------------------------------------------------------*/ 1520 1521int ehca_reg_maxmr(struct ehca_shca *shca, 1522 struct ehca_mr *e_newmr, 1523 u64 *iova_start, 1524 int acl, 1525 struct ehca_pd *e_pd, 1526 u32 *lkey, 1527 u32 *rkey) 1528{ 1529 u64 h_ret; 1530 struct ehca_mr *e_origmr = shca->maxmr; 1531 u32 hipz_acl; 1532 struct ehca_mr_hipzout_parms hipzout = {{0},0,0,0,0,0}; 1533 1534 ehca_mrmw_map_acl(acl, &hipz_acl); 1535 ehca_mrmw_set_pgsize_hipz_acl(&hipz_acl); 1536 1537 h_ret = hipz_h_register_smr(shca->ipz_hca_handle, e_newmr, e_origmr, 1538 (u64)iova_start, hipz_acl, e_pd->fw_pd, 1539 &hipzout); 1540 if (h_ret != H_SUCCESS) { 1541 ehca_err(&shca->ib_device, "hipz_reg_smr failed, h_ret=%lx " 1542 "e_origmr=%p hca_hndl=%lx mr_hndl=%lx lkey=%x", 1543 h_ret, e_origmr, shca->ipz_hca_handle.handle, 1544 e_origmr->ipz_mr_handle.handle, 1545 e_origmr->ib.ib_mr.lkey); 1546 return ehca2ib_return_code(h_ret); 1547 } 1548 /* successful registration */ 1549 e_newmr->num_pages = e_origmr->num_pages; 1550 e_newmr->num_4k = e_origmr->num_4k; 1551 e_newmr->start = iova_start; 1552 e_newmr->size = e_origmr->size; 1553 e_newmr->acl = acl; 1554 e_newmr->ipz_mr_handle = hipzout.handle; 1555 *lkey = hipzout.lkey; 1556 *rkey = hipzout.rkey; 1557 return 0; 1558} /* end ehca_reg_maxmr() */ 1559 1560/*----------------------------------------------------------------------*/ 1561 1562int ehca_dereg_internal_maxmr(struct ehca_shca *shca) 1563{ 1564 int ret; 1565 struct ehca_mr *e_maxmr; 1566 struct ib_pd *ib_pd; 1567 1568 if (!shca->maxmr) { 1569 ehca_err(&shca->ib_device, "bad call, shca=%p", shca); 1570 ret = -EINVAL; 1571 goto ehca_dereg_internal_maxmr_exit0; 1572 } 1573 1574 e_maxmr = shca->maxmr; 1575 ib_pd = e_maxmr->ib.ib_mr.pd; 1576 shca->maxmr = NULL; /* remove internal max-MR indication from SHCA */ 1577 1578 ret = ehca_dereg_mr(&e_maxmr->ib.ib_mr); 1579 if (ret) { 1580 ehca_err(&shca->ib_device, "dereg internal max-MR failed, " 1581 "ret=%x e_maxmr=%p shca=%p lkey=%x", 1582 ret, e_maxmr, shca, e_maxmr->ib.ib_mr.lkey); 1583 shca->maxmr = e_maxmr; 1584 goto ehca_dereg_internal_maxmr_exit0; 1585 } 1586 1587 atomic_dec(&ib_pd->usecnt); 1588 1589ehca_dereg_internal_maxmr_exit0: 1590 if (ret) 1591 ehca_err(&shca->ib_device, "ret=%x shca=%p shca->maxmr=%p", 1592 ret, shca, shca->maxmr); 1593 return ret; 1594} /* end ehca_dereg_internal_maxmr() */ 1595 1596/*----------------------------------------------------------------------*/ 1597 1598/* 1599 * check physical buffer array of MR verbs for validness and 1600 * calculates MR size 1601 */ 1602int ehca_mr_chk_buf_and_calc_size(struct ib_phys_buf *phys_buf_array, 1603 int num_phys_buf, 1604 u64 *iova_start, 1605 u64 *size) 1606{ 1607 struct ib_phys_buf *pbuf = phys_buf_array; 1608 u64 size_count = 0; 1609 u32 i; 1610 1611 if (num_phys_buf == 0) { 1612 ehca_gen_err("bad phys buf array len, num_phys_buf=0"); 1613 return -EINVAL; 1614 } 1615 /* check first buffer */ 1616 if (((u64)iova_start & ~PAGE_MASK) != (pbuf->addr & ~PAGE_MASK)) { 1617 ehca_gen_err("iova_start/addr mismatch, iova_start=%p " 1618 "pbuf->addr=%lx pbuf->size=%lx", 1619 iova_start, pbuf->addr, pbuf->size); 1620 return -EINVAL; 1621 } 1622 if (((pbuf->addr + pbuf->size) % PAGE_SIZE) && 1623 (num_phys_buf > 1)) { 1624 ehca_gen_err("addr/size mismatch in 1st buf, pbuf->addr=%lx " 1625 "pbuf->size=%lx", pbuf->addr, pbuf->size); 1626 return -EINVAL; 1627 } 1628 1629 for (i = 0; i < num_phys_buf; i++) { 1630 if ((i > 0) && (pbuf->addr % PAGE_SIZE)) { 1631 ehca_gen_err("bad address, i=%x pbuf->addr=%lx " 1632 "pbuf->size=%lx", 1633 i, pbuf->addr, pbuf->size); 1634 return -EINVAL; 1635 } 1636 if (((i > 0) && /* not 1st */ 1637 (i < (num_phys_buf - 1)) && /* not last */ 1638 (pbuf->size % PAGE_SIZE)) || (pbuf->size == 0)) { 1639 ehca_gen_err("bad size, i=%x pbuf->size=%lx", 1640 i, pbuf->size); 1641 return -EINVAL; 1642 } 1643 size_count += pbuf->size; 1644 pbuf++; 1645 } 1646 1647 *size = size_count; 1648 return 0; 1649} /* end ehca_mr_chk_buf_and_calc_size() */ 1650 1651/*----------------------------------------------------------------------*/ 1652 1653/* check page list of map FMR verb for validness */ 1654int ehca_fmr_check_page_list(struct ehca_mr *e_fmr, 1655 u64 *page_list, 1656 int list_len) 1657{ 1658 u32 i; 1659 u64 *page; 1660 1661 if ((list_len == 0) || (list_len > e_fmr->fmr_max_pages)) { 1662 ehca_gen_err("bad list_len, list_len=%x " 1663 "e_fmr->fmr_max_pages=%x fmr=%p", 1664 list_len, e_fmr->fmr_max_pages, e_fmr); 1665 return -EINVAL; 1666 } 1667 1668 /* each page must be aligned */ 1669 page = page_list; 1670 for (i = 0; i < list_len; i++) { 1671 if (*page % e_fmr->fmr_page_size) { 1672 ehca_gen_err("bad page, i=%x *page=%lx page=%p fmr=%p " 1673 "fmr_page_size=%x", i, *page, page, e_fmr, 1674 e_fmr->fmr_page_size); 1675 return -EINVAL; 1676 } 1677 page++; 1678 } 1679 1680 return 0; 1681} /* end ehca_fmr_check_page_list() */ 1682 1683/*----------------------------------------------------------------------*/ 1684 1685/* setup page buffer from page info */ 1686int ehca_set_pagebuf(struct ehca_mr *e_mr, 1687 struct ehca_mr_pginfo *pginfo, 1688 u32 number, 1689 u64 *kpage) 1690{ 1691 int ret = 0; 1692 struct ib_umem_chunk *prev_chunk; 1693 struct ib_umem_chunk *chunk; 1694 struct ib_phys_buf *pbuf; 1695 u64 *fmrlist; 1696 u64 num4k, pgaddr, offs4k; 1697 u32 i = 0; 1698 u32 j = 0; 1699 1700 if (pginfo->type == EHCA_MR_PGI_PHYS) { 1701 /* loop over desired phys_buf_array entries */ 1702 while (i < number) { 1703 pbuf = pginfo->phys_buf_array + pginfo->next_buf; 1704 num4k = NUM_CHUNKS((pbuf->addr % EHCA_PAGESIZE) 1705 + pbuf->size, EHCA_PAGESIZE); 1706 offs4k = (pbuf->addr & ~PAGE_MASK) / EHCA_PAGESIZE; 1707 while (pginfo->next_4k < offs4k + num4k) { 1708 /* sanity check */ 1709 if ((pginfo->page_cnt >= pginfo->num_pages) || 1710 (pginfo->page_4k_cnt >= pginfo->num_4k)) { 1711 ehca_gen_err("page_cnt >= num_pages, " 1712 "page_cnt=%lx " 1713 "num_pages=%lx " 1714 "page_4k_cnt=%lx " 1715 "num_4k=%lx i=%x", 1716 pginfo->page_cnt, 1717 pginfo->num_pages, 1718 pginfo->page_4k_cnt, 1719 pginfo->num_4k, i); 1720 ret = -EFAULT; 1721 goto ehca_set_pagebuf_exit0; 1722 } 1723 *kpage = phys_to_abs( 1724 (pbuf->addr & EHCA_PAGEMASK) 1725 + (pginfo->next_4k * EHCA_PAGESIZE)); 1726 if ( !(*kpage) && pbuf->addr ) { 1727 ehca_gen_err("pbuf->addr=%lx " 1728 "pbuf->size=%lx " 1729 "next_4k=%lx", pbuf->addr, 1730 pbuf->size, 1731 pginfo->next_4k); 1732 ret = -EFAULT; 1733 goto ehca_set_pagebuf_exit0; 1734 } 1735 (pginfo->page_4k_cnt)++; 1736 (pginfo->next_4k)++; 1737 if (pginfo->next_4k % 1738 (PAGE_SIZE / EHCA_PAGESIZE) == 0) 1739 (pginfo->page_cnt)++; 1740 kpage++; 1741 i++; 1742 if (i >= number) break; 1743 } 1744 if (pginfo->next_4k >= offs4k + num4k) { 1745 (pginfo->next_buf)++; 1746 pginfo->next_4k = 0; 1747 } 1748 } 1749 } else if (pginfo->type == EHCA_MR_PGI_USER) { 1750 /* loop over desired chunk entries */ 1751 chunk = pginfo->next_chunk; 1752 prev_chunk = pginfo->next_chunk; 1753 list_for_each_entry_continue(chunk, 1754 (&(pginfo->region->chunk_list)), 1755 list) { 1756 for (i = pginfo->next_nmap; i < chunk->nmap; ) { 1757 pgaddr = ( page_to_pfn(chunk->page_list[i].page) 1758 << PAGE_SHIFT ); 1759 *kpage = phys_to_abs(pgaddr + 1760 (pginfo->next_4k * 1761 EHCA_PAGESIZE)); 1762 if ( !(*kpage) ) { 1763 ehca_gen_err("pgaddr=%lx " 1764 "chunk->page_list[i]=%lx " 1765 "i=%x next_4k=%lx mr=%p", 1766 pgaddr, 1767 (u64)sg_dma_address( 1768 &chunk-> 1769 page_list[i]), 1770 i, pginfo->next_4k, e_mr); 1771 ret = -EFAULT; 1772 goto ehca_set_pagebuf_exit0; 1773 } 1774 (pginfo->page_4k_cnt)++; 1775 (pginfo->next_4k)++; 1776 kpage++; 1777 if (pginfo->next_4k % 1778 (PAGE_SIZE / EHCA_PAGESIZE) == 0) { 1779 (pginfo->page_cnt)++; 1780 (pginfo->next_nmap)++; 1781 pginfo->next_4k = 0; 1782 i++; 1783 } 1784 j++; 1785 if (j >= number) break; 1786 } 1787 if ((pginfo->next_nmap >= chunk->nmap) && 1788 (j >= number)) { 1789 pginfo->next_nmap = 0; 1790 prev_chunk = chunk; 1791 break; 1792 } else if (pginfo->next_nmap >= chunk->nmap) { 1793 pginfo->next_nmap = 0; 1794 prev_chunk = chunk; 1795 } else if (j >= number) 1796 break; 1797 else 1798 prev_chunk = chunk; 1799 } 1800 pginfo->next_chunk = 1801 list_prepare_entry(prev_chunk, 1802 (&(pginfo->region->chunk_list)), 1803 list); 1804 } else if (pginfo->type == EHCA_MR_PGI_FMR) { 1805 /* loop over desired page_list entries */ 1806 fmrlist = pginfo->page_list + pginfo->next_listelem; 1807 for (i = 0; i < number; i++) { 1808 *kpage = phys_to_abs((*fmrlist & EHCA_PAGEMASK) + 1809 pginfo->next_4k * EHCA_PAGESIZE); 1810 if ( !(*kpage) ) { 1811 ehca_gen_err("*fmrlist=%lx fmrlist=%p " 1812 "next_listelem=%lx next_4k=%lx", 1813 *fmrlist, fmrlist, 1814 pginfo->next_listelem, 1815 pginfo->next_4k); 1816 ret = -EFAULT; 1817 goto ehca_set_pagebuf_exit0; 1818 } 1819 (pginfo->page_4k_cnt)++; 1820 (pginfo->next_4k)++; 1821 kpage++; 1822 if (pginfo->next_4k % 1823 (e_mr->fmr_page_size / EHCA_PAGESIZE) == 0) { 1824 (pginfo->page_cnt)++; 1825 (pginfo->next_listelem)++; 1826 fmrlist++; 1827 pginfo->next_4k = 0; 1828 } 1829 } 1830 } else { 1831 ehca_gen_err("bad pginfo->type=%x", pginfo->type); 1832 ret = -EFAULT; 1833 goto ehca_set_pagebuf_exit0; 1834 } 1835 1836ehca_set_pagebuf_exit0: 1837 if (ret) 1838 ehca_gen_err("ret=%x e_mr=%p pginfo=%p type=%x num_pages=%lx " 1839 "num_4k=%lx next_buf=%lx next_4k=%lx number=%x " 1840 "kpage=%p page_cnt=%lx page_4k_cnt=%lx i=%x " 1841 "next_listelem=%lx region=%p next_chunk=%p " 1842 "next_nmap=%lx", ret, e_mr, pginfo, pginfo->type, 1843 pginfo->num_pages, pginfo->num_4k, 1844 pginfo->next_buf, pginfo->next_4k, number, kpage, 1845 pginfo->page_cnt, pginfo->page_4k_cnt, i, 1846 pginfo->next_listelem, pginfo->region, 1847 pginfo->next_chunk, pginfo->next_nmap); 1848 return ret; 1849} /* end ehca_set_pagebuf() */ 1850 1851/*----------------------------------------------------------------------*/ 1852 1853/* setup 1 page from page info page buffer */ 1854int ehca_set_pagebuf_1(struct ehca_mr *e_mr, 1855 struct ehca_mr_pginfo *pginfo, 1856 u64 *rpage) 1857{ 1858 int ret = 0; 1859 struct ib_phys_buf *tmp_pbuf; 1860 u64 *fmrlist; 1861 struct ib_umem_chunk *chunk; 1862 struct ib_umem_chunk *prev_chunk; 1863 u64 pgaddr, num4k, offs4k; 1864 1865 if (pginfo->type == EHCA_MR_PGI_PHYS) { 1866 /* sanity check */ 1867 if ((pginfo->page_cnt >= pginfo->num_pages) || 1868 (pginfo->page_4k_cnt >= pginfo->num_4k)) { 1869 ehca_gen_err("page_cnt >= num_pages, page_cnt=%lx " 1870 "num_pages=%lx page_4k_cnt=%lx num_4k=%lx", 1871 pginfo->page_cnt, pginfo->num_pages, 1872 pginfo->page_4k_cnt, pginfo->num_4k); 1873 ret = -EFAULT; 1874 goto ehca_set_pagebuf_1_exit0; 1875 } 1876 tmp_pbuf = pginfo->phys_buf_array + pginfo->next_buf; 1877 num4k = NUM_CHUNKS((tmp_pbuf->addr % EHCA_PAGESIZE) + 1878 tmp_pbuf->size, EHCA_PAGESIZE); 1879 offs4k = (tmp_pbuf->addr & ~PAGE_MASK) / EHCA_PAGESIZE; 1880 *rpage = phys_to_abs((tmp_pbuf->addr & EHCA_PAGEMASK) + 1881 (pginfo->next_4k * EHCA_PAGESIZE)); 1882 if ( !(*rpage) && tmp_pbuf->addr ) { 1883 ehca_gen_err("tmp_pbuf->addr=%lx" 1884 " tmp_pbuf->size=%lx next_4k=%lx", 1885 tmp_pbuf->addr, tmp_pbuf->size, 1886 pginfo->next_4k); 1887 ret = -EFAULT; 1888 goto ehca_set_pagebuf_1_exit0; 1889 } 1890 (pginfo->page_4k_cnt)++; 1891 (pginfo->next_4k)++; 1892 if (pginfo->next_4k % (PAGE_SIZE / EHCA_PAGESIZE) == 0) 1893 (pginfo->page_cnt)++; 1894 if (pginfo->next_4k >= offs4k + num4k) { 1895 (pginfo->next_buf)++; 1896 pginfo->next_4k = 0; 1897 } 1898 } else if (pginfo->type == EHCA_MR_PGI_USER) { 1899 chunk = pginfo->next_chunk; 1900 prev_chunk = pginfo->next_chunk; 1901 list_for_each_entry_continue(chunk, 1902 (&(pginfo->region->chunk_list)), 1903 list) { 1904 pgaddr = ( page_to_pfn(chunk->page_list[ 1905 pginfo->next_nmap].page) 1906 << PAGE_SHIFT); 1907 *rpage = phys_to_abs(pgaddr + 1908 (pginfo->next_4k * EHCA_PAGESIZE)); 1909 if ( !(*rpage) ) { 1910 ehca_gen_err("pgaddr=%lx chunk->page_list[]=%lx" 1911 " next_nmap=%lx next_4k=%lx mr=%p", 1912 pgaddr, (u64)sg_dma_address( 1913 &chunk->page_list[ 1914 pginfo-> 1915 next_nmap]), 1916 pginfo->next_nmap, pginfo->next_4k, 1917 e_mr); 1918 ret = -EFAULT; 1919 goto ehca_set_pagebuf_1_exit0; 1920 } 1921 (pginfo->page_4k_cnt)++; 1922 (pginfo->next_4k)++; 1923 if (pginfo->next_4k % 1924 (PAGE_SIZE / EHCA_PAGESIZE) == 0) { 1925 (pginfo->page_cnt)++; 1926 (pginfo->next_nmap)++; 1927 pginfo->next_4k = 0; 1928 } 1929 if (pginfo->next_nmap >= chunk->nmap) { 1930 pginfo->next_nmap = 0; 1931 prev_chunk = chunk; 1932 } 1933 break; 1934 } 1935 pginfo->next_chunk = 1936 list_prepare_entry(prev_chunk, 1937 (&(pginfo->region->chunk_list)), 1938 list); 1939 } else if (pginfo->type == EHCA_MR_PGI_FMR) { 1940 fmrlist = pginfo->page_list + pginfo->next_listelem; 1941 *rpage = phys_to_abs((*fmrlist & EHCA_PAGEMASK) + 1942 pginfo->next_4k * EHCA_PAGESIZE); 1943 if ( !(*rpage) ) { 1944 ehca_gen_err("*fmrlist=%lx fmrlist=%p " 1945 "next_listelem=%lx next_4k=%lx", 1946 *fmrlist, fmrlist, pginfo->next_listelem, 1947 pginfo->next_4k); 1948 ret = -EFAULT; 1949 goto ehca_set_pagebuf_1_exit0; 1950 } 1951 (pginfo->page_4k_cnt)++; 1952 (pginfo->next_4k)++; 1953 if (pginfo->next_4k % 1954 (e_mr->fmr_page_size / EHCA_PAGESIZE) == 0) { 1955 (pginfo->page_cnt)++; 1956 (pginfo->next_listelem)++; 1957 pginfo->next_4k = 0; 1958 } 1959 } else { 1960 ehca_gen_err("bad pginfo->type=%x", pginfo->type); 1961 ret = -EFAULT; 1962 goto ehca_set_pagebuf_1_exit0; 1963 } 1964 1965ehca_set_pagebuf_1_exit0: 1966 if (ret) 1967 ehca_gen_err("ret=%x e_mr=%p pginfo=%p type=%x num_pages=%lx " 1968 "num_4k=%lx next_buf=%lx next_4k=%lx rpage=%p " 1969 "page_cnt=%lx page_4k_cnt=%lx next_listelem=%lx " 1970 "region=%p next_chunk=%p next_nmap=%lx", ret, e_mr, 1971 pginfo, pginfo->type, pginfo->num_pages, 1972 pginfo->num_4k, pginfo->next_buf, pginfo->next_4k, 1973 rpage, pginfo->page_cnt, pginfo->page_4k_cnt, 1974 pginfo->next_listelem, pginfo->region, 1975 pginfo->next_chunk, pginfo->next_nmap); 1976 return ret; 1977} /* end ehca_set_pagebuf_1() */ 1978 1979/*----------------------------------------------------------------------*/ 1980 1981/* 1982 * check MR if it is a max-MR, i.e. uses whole memory 1983 * in case it's a max-MR 1 is returned, else 0 1984 */ 1985int ehca_mr_is_maxmr(u64 size, 1986 u64 *iova_start) 1987{ 1988 /* a MR is treated as max-MR only if it fits following: */ 1989 if ((size == ((u64)high_memory - PAGE_OFFSET)) && 1990 (iova_start == (void*)KERNELBASE)) { 1991 ehca_gen_dbg("this is a max-MR"); 1992 return 1; 1993 } else 1994 return 0; 1995} /* end ehca_mr_is_maxmr() */ 1996 1997/*----------------------------------------------------------------------*/ 1998 1999/* map access control for MR/MW. This routine is used for MR and MW. */ 2000void ehca_mrmw_map_acl(int ib_acl, 2001 u32 *hipz_acl) 2002{ 2003 *hipz_acl = 0; 2004 if (ib_acl & IB_ACCESS_REMOTE_READ) 2005 *hipz_acl |= HIPZ_ACCESSCTRL_R_READ; 2006 if (ib_acl & IB_ACCESS_REMOTE_WRITE) 2007 *hipz_acl |= HIPZ_ACCESSCTRL_R_WRITE; 2008 if (ib_acl & IB_ACCESS_REMOTE_ATOMIC) 2009 *hipz_acl |= HIPZ_ACCESSCTRL_R_ATOMIC; 2010 if (ib_acl & IB_ACCESS_LOCAL_WRITE) 2011 *hipz_acl |= HIPZ_ACCESSCTRL_L_WRITE; 2012 if (ib_acl & IB_ACCESS_MW_BIND) 2013 *hipz_acl |= HIPZ_ACCESSCTRL_MW_BIND; 2014} /* end ehca_mrmw_map_acl() */ 2015 2016/*----------------------------------------------------------------------*/ 2017 2018/* sets page size in hipz access control for MR/MW. */ 2019void ehca_mrmw_set_pgsize_hipz_acl(u32 *hipz_acl) /*INOUT*/ 2020{ 2021 return; /* HCA supports only 4k */ 2022} /* end ehca_mrmw_set_pgsize_hipz_acl() */ 2023 2024/*----------------------------------------------------------------------*/ 2025 2026/* 2027 * reverse map access control for MR/MW. 2028 * This routine is used for MR and MW. 2029 */ 2030void ehca_mrmw_reverse_map_acl(const u32 *hipz_acl, 2031 int *ib_acl) /*OUT*/ 2032{ 2033 *ib_acl = 0; 2034 if (*hipz_acl & HIPZ_ACCESSCTRL_R_READ) 2035 *ib_acl |= IB_ACCESS_REMOTE_READ; 2036 if (*hipz_acl & HIPZ_ACCESSCTRL_R_WRITE) 2037 *ib_acl |= IB_ACCESS_REMOTE_WRITE; 2038 if (*hipz_acl & HIPZ_ACCESSCTRL_R_ATOMIC) 2039 *ib_acl |= IB_ACCESS_REMOTE_ATOMIC; 2040 if (*hipz_acl & HIPZ_ACCESSCTRL_L_WRITE) 2041 *ib_acl |= IB_ACCESS_LOCAL_WRITE; 2042 if (*hipz_acl & HIPZ_ACCESSCTRL_MW_BIND) 2043 *ib_acl |= IB_ACCESS_MW_BIND; 2044} /* end ehca_mrmw_reverse_map_acl() */ 2045 2046 2047/*----------------------------------------------------------------------*/ 2048 2049/* 2050 * MR destructor and constructor 2051 * used in Reregister MR verb, sets all fields in ehca_mr_t to 0, 2052 * except struct ib_mr and spinlock 2053 */ 2054void ehca_mr_deletenew(struct ehca_mr *mr) 2055{ 2056 mr->flags = 0; 2057 mr->num_pages = 0; 2058 mr->num_4k = 0; 2059 mr->acl = 0; 2060 mr->start = NULL; 2061 mr->fmr_page_size = 0; 2062 mr->fmr_max_pages = 0; 2063 mr->fmr_max_maps = 0; 2064 mr->fmr_map_cnt = 0; 2065 memset(&mr->ipz_mr_handle, 0, sizeof(mr->ipz_mr_handle)); 2066 memset(&mr->galpas, 0, sizeof(mr->galpas)); 2067 mr->nr_of_pages = 0; 2068 mr->pagearray = NULL; 2069} /* end ehca_mr_deletenew() */ 2070 2071int ehca_init_mrmw_cache(void) 2072{ 2073 mr_cache = kmem_cache_create("ehca_cache_mr", 2074 sizeof(struct ehca_mr), 0, 2075 SLAB_HWCACHE_ALIGN, 2076 NULL, NULL); 2077 if (!mr_cache) 2078 return -ENOMEM; 2079 mw_cache = kmem_cache_create("ehca_cache_mw", 2080 sizeof(struct ehca_mw), 0, 2081 SLAB_HWCACHE_ALIGN, 2082 NULL, NULL); 2083 if (!mw_cache) { 2084 kmem_cache_destroy(mr_cache); 2085 mr_cache = NULL; 2086 return -ENOMEM; 2087 } 2088 return 0; 2089} 2090 2091void ehca_cleanup_mrmw_cache(void) 2092{ 2093 if (mr_cache) 2094 kmem_cache_destroy(mr_cache); 2095 if (mw_cache) 2096 kmem_cache_destroy(mw_cache); 2097} 2098