mthca_mr.c revision 399d7921299fc4f146bd62bfa6312382a5429bcc
1/* 2 * Copyright (c) 2004 Topspin Communications. All rights reserved. 3 * Copyright (c) 2005 Mellanox Technologies. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 * 33 * $Id: mthca_mr.c 1349 2004-12-16 21:09:43Z roland $ 34 */ 35 36#include <linux/slab.h> 37#include <linux/init.h> 38#include <linux/errno.h> 39 40#include "mthca_dev.h" 41#include "mthca_cmd.h" 42#include "mthca_memfree.h" 43 44struct mthca_mtt { 45 struct mthca_buddy *buddy; 46 int order; 47 u32 first_seg; 48}; 49 50/* 51 * Must be packed because mtt_seg is 64 bits but only aligned to 32 bits. 52 */ 53struct mthca_mpt_entry { 54 __be32 flags; 55 __be32 page_size; 56 __be32 key; 57 __be32 pd; 58 __be64 start; 59 __be64 length; 60 __be32 lkey; 61 __be32 window_count; 62 __be32 window_count_limit; 63 __be64 mtt_seg; 64 __be32 mtt_sz; /* Arbel only */ 65 u32 reserved[2]; 66} __attribute__((packed)); 67 68#define MTHCA_MPT_FLAG_SW_OWNS (0xfUL << 28) 69#define MTHCA_MPT_FLAG_MIO (1 << 17) 70#define MTHCA_MPT_FLAG_BIND_ENABLE (1 << 15) 71#define MTHCA_MPT_FLAG_PHYSICAL (1 << 9) 72#define MTHCA_MPT_FLAG_REGION (1 << 8) 73 74#define MTHCA_MTT_FLAG_PRESENT 1 75 76#define MTHCA_MPT_STATUS_SW 0xF0 77#define MTHCA_MPT_STATUS_HW 0x00 78 79/* 80 * Buddy allocator for MTT segments (currently not very efficient 81 * since it doesn't keep a free list and just searches linearly 82 * through the bitmaps) 83 */ 84 85static u32 mthca_buddy_alloc(struct mthca_buddy *buddy, int order) 86{ 87 int o; 88 int m; 89 u32 seg; 90 91 spin_lock(&buddy->lock); 92 93 for (o = order; o <= buddy->max_order; ++o) { 94 m = 1 << (buddy->max_order - o); 95 seg = find_first_bit(buddy->bits[o], m); 96 if (seg < m) 97 goto found; 98 } 99 100 spin_unlock(&buddy->lock); 101 return -1; 102 103 found: 104 clear_bit(seg, buddy->bits[o]); 105 106 while (o > order) { 107 --o; 108 seg <<= 1; 109 set_bit(seg ^ 1, buddy->bits[o]); 110 } 111 112 spin_unlock(&buddy->lock); 113 114 seg <<= order; 115 116 return seg; 117} 118 119static void mthca_buddy_free(struct mthca_buddy *buddy, u32 seg, int order) 120{ 121 seg >>= order; 122 123 spin_lock(&buddy->lock); 124 125 while (test_bit(seg ^ 1, buddy->bits[order])) { 126 clear_bit(seg ^ 1, buddy->bits[order]); 127 seg >>= 1; 128 ++order; 129 } 130 131 set_bit(seg, buddy->bits[order]); 132 133 spin_unlock(&buddy->lock); 134} 135 136static int __devinit mthca_buddy_init(struct mthca_buddy *buddy, int max_order) 137{ 138 int i, s; 139 140 buddy->max_order = max_order; 141 spin_lock_init(&buddy->lock); 142 143 buddy->bits = kzalloc((buddy->max_order + 1) * sizeof (long *), 144 GFP_KERNEL); 145 if (!buddy->bits) 146 goto err_out; 147 148 for (i = 0; i <= buddy->max_order; ++i) { 149 s = BITS_TO_LONGS(1 << (buddy->max_order - i)); 150 buddy->bits[i] = kmalloc(s * sizeof (long), GFP_KERNEL); 151 if (!buddy->bits[i]) 152 goto err_out_free; 153 bitmap_zero(buddy->bits[i], 154 1 << (buddy->max_order - i)); 155 } 156 157 set_bit(0, buddy->bits[buddy->max_order]); 158 159 return 0; 160 161err_out_free: 162 for (i = 0; i <= buddy->max_order; ++i) 163 kfree(buddy->bits[i]); 164 165 kfree(buddy->bits); 166 167err_out: 168 return -ENOMEM; 169} 170 171static void __devexit mthca_buddy_cleanup(struct mthca_buddy *buddy) 172{ 173 int i; 174 175 for (i = 0; i <= buddy->max_order; ++i) 176 kfree(buddy->bits[i]); 177 178 kfree(buddy->bits); 179} 180 181static u32 mthca_alloc_mtt_range(struct mthca_dev *dev, int order, 182 struct mthca_buddy *buddy) 183{ 184 u32 seg = mthca_buddy_alloc(buddy, order); 185 186 if (seg == -1) 187 return -1; 188 189 if (mthca_is_memfree(dev)) 190 if (mthca_table_get_range(dev, dev->mr_table.mtt_table, seg, 191 seg + (1 << order) - 1)) { 192 mthca_buddy_free(buddy, seg, order); 193 seg = -1; 194 } 195 196 return seg; 197} 198 199static struct mthca_mtt *__mthca_alloc_mtt(struct mthca_dev *dev, int size, 200 struct mthca_buddy *buddy) 201{ 202 struct mthca_mtt *mtt; 203 int i; 204 205 if (size <= 0) 206 return ERR_PTR(-EINVAL); 207 208 mtt = kmalloc(sizeof *mtt, GFP_KERNEL); 209 if (!mtt) 210 return ERR_PTR(-ENOMEM); 211 212 mtt->buddy = buddy; 213 mtt->order = 0; 214 for (i = MTHCA_MTT_SEG_SIZE / 8; i < size; i <<= 1) 215 ++mtt->order; 216 217 mtt->first_seg = mthca_alloc_mtt_range(dev, mtt->order, buddy); 218 if (mtt->first_seg == -1) { 219 kfree(mtt); 220 return ERR_PTR(-ENOMEM); 221 } 222 223 return mtt; 224} 225 226struct mthca_mtt *mthca_alloc_mtt(struct mthca_dev *dev, int size) 227{ 228 return __mthca_alloc_mtt(dev, size, &dev->mr_table.mtt_buddy); 229} 230 231void mthca_free_mtt(struct mthca_dev *dev, struct mthca_mtt *mtt) 232{ 233 if (!mtt) 234 return; 235 236 mthca_buddy_free(mtt->buddy, mtt->first_seg, mtt->order); 237 238 mthca_table_put_range(dev, dev->mr_table.mtt_table, 239 mtt->first_seg, 240 mtt->first_seg + (1 << mtt->order) - 1); 241 242 kfree(mtt); 243} 244 245int mthca_write_mtt(struct mthca_dev *dev, struct mthca_mtt *mtt, 246 int start_index, u64 *buffer_list, int list_len) 247{ 248 struct mthca_mailbox *mailbox; 249 __be64 *mtt_entry; 250 int err = 0; 251 u8 status; 252 int i; 253 254 mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); 255 if (IS_ERR(mailbox)) 256 return PTR_ERR(mailbox); 257 mtt_entry = mailbox->buf; 258 259 while (list_len > 0) { 260 mtt_entry[0] = cpu_to_be64(dev->mr_table.mtt_base + 261 mtt->first_seg * MTHCA_MTT_SEG_SIZE + 262 start_index * 8); 263 mtt_entry[1] = 0; 264 for (i = 0; i < list_len && i < MTHCA_MAILBOX_SIZE / 8 - 2; ++i) 265 mtt_entry[i + 2] = cpu_to_be64(buffer_list[i] | 266 MTHCA_MTT_FLAG_PRESENT); 267 268 /* 269 * If we have an odd number of entries to write, add 270 * one more dummy entry for firmware efficiency. 271 */ 272 if (i & 1) 273 mtt_entry[i + 2] = 0; 274 275 err = mthca_WRITE_MTT(dev, mailbox, (i + 1) & ~1, &status); 276 if (err) { 277 mthca_warn(dev, "WRITE_MTT failed (%d)\n", err); 278 goto out; 279 } 280 if (status) { 281 mthca_warn(dev, "WRITE_MTT returned status 0x%02x\n", 282 status); 283 err = -EINVAL; 284 goto out; 285 } 286 287 list_len -= i; 288 start_index += i; 289 buffer_list += i; 290 } 291 292out: 293 mthca_free_mailbox(dev, mailbox); 294 return err; 295} 296 297static inline u32 tavor_hw_index_to_key(u32 ind) 298{ 299 return ind; 300} 301 302static inline u32 tavor_key_to_hw_index(u32 key) 303{ 304 return key; 305} 306 307static inline u32 arbel_hw_index_to_key(u32 ind) 308{ 309 return (ind >> 24) | (ind << 8); 310} 311 312static inline u32 arbel_key_to_hw_index(u32 key) 313{ 314 return (key << 24) | (key >> 8); 315} 316 317static inline u32 hw_index_to_key(struct mthca_dev *dev, u32 ind) 318{ 319 if (mthca_is_memfree(dev)) 320 return arbel_hw_index_to_key(ind); 321 else 322 return tavor_hw_index_to_key(ind); 323} 324 325static inline u32 key_to_hw_index(struct mthca_dev *dev, u32 key) 326{ 327 if (mthca_is_memfree(dev)) 328 return arbel_key_to_hw_index(key); 329 else 330 return tavor_key_to_hw_index(key); 331} 332 333int mthca_mr_alloc(struct mthca_dev *dev, u32 pd, int buffer_size_shift, 334 u64 iova, u64 total_size, u32 access, struct mthca_mr *mr) 335{ 336 struct mthca_mailbox *mailbox; 337 struct mthca_mpt_entry *mpt_entry; 338 u32 key; 339 int i; 340 int err; 341 u8 status; 342 343 WARN_ON(buffer_size_shift >= 32); 344 345 key = mthca_alloc(&dev->mr_table.mpt_alloc); 346 if (key == -1) 347 return -ENOMEM; 348 mr->ibmr.rkey = mr->ibmr.lkey = hw_index_to_key(dev, key); 349 350 if (mthca_is_memfree(dev)) { 351 err = mthca_table_get(dev, dev->mr_table.mpt_table, key); 352 if (err) 353 goto err_out_mpt_free; 354 } 355 356 mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); 357 if (IS_ERR(mailbox)) { 358 err = PTR_ERR(mailbox); 359 goto err_out_table; 360 } 361 mpt_entry = mailbox->buf; 362 363 mpt_entry->flags = cpu_to_be32(MTHCA_MPT_FLAG_SW_OWNS | 364 MTHCA_MPT_FLAG_MIO | 365 MTHCA_MPT_FLAG_REGION | 366 access); 367 if (!mr->mtt) 368 mpt_entry->flags |= cpu_to_be32(MTHCA_MPT_FLAG_PHYSICAL); 369 370 mpt_entry->page_size = cpu_to_be32(buffer_size_shift - 12); 371 mpt_entry->key = cpu_to_be32(key); 372 mpt_entry->pd = cpu_to_be32(pd); 373 mpt_entry->start = cpu_to_be64(iova); 374 mpt_entry->length = cpu_to_be64(total_size); 375 376 memset(&mpt_entry->lkey, 0, 377 sizeof *mpt_entry - offsetof(struct mthca_mpt_entry, lkey)); 378 379 if (mr->mtt) 380 mpt_entry->mtt_seg = 381 cpu_to_be64(dev->mr_table.mtt_base + 382 mr->mtt->first_seg * MTHCA_MTT_SEG_SIZE); 383 384 if (0) { 385 mthca_dbg(dev, "Dumping MPT entry %08x:\n", mr->ibmr.lkey); 386 for (i = 0; i < sizeof (struct mthca_mpt_entry) / 4; ++i) { 387 if (i % 4 == 0) 388 printk("[%02x] ", i * 4); 389 printk(" %08x", be32_to_cpu(((__be32 *) mpt_entry)[i])); 390 if ((i + 1) % 4 == 0) 391 printk("\n"); 392 } 393 } 394 395 err = mthca_SW2HW_MPT(dev, mailbox, 396 key & (dev->limits.num_mpts - 1), 397 &status); 398 if (err) { 399 mthca_warn(dev, "SW2HW_MPT failed (%d)\n", err); 400 goto err_out_mailbox; 401 } else if (status) { 402 mthca_warn(dev, "SW2HW_MPT returned status 0x%02x\n", 403 status); 404 err = -EINVAL; 405 goto err_out_mailbox; 406 } 407 408 mthca_free_mailbox(dev, mailbox); 409 return err; 410 411err_out_mailbox: 412 mthca_free_mailbox(dev, mailbox); 413 414err_out_table: 415 mthca_table_put(dev, dev->mr_table.mpt_table, key); 416 417err_out_mpt_free: 418 mthca_free(&dev->mr_table.mpt_alloc, key); 419 return err; 420} 421 422int mthca_mr_alloc_notrans(struct mthca_dev *dev, u32 pd, 423 u32 access, struct mthca_mr *mr) 424{ 425 mr->mtt = NULL; 426 return mthca_mr_alloc(dev, pd, 12, 0, ~0ULL, access, mr); 427} 428 429int mthca_mr_alloc_phys(struct mthca_dev *dev, u32 pd, 430 u64 *buffer_list, int buffer_size_shift, 431 int list_len, u64 iova, u64 total_size, 432 u32 access, struct mthca_mr *mr) 433{ 434 int err; 435 436 mr->mtt = mthca_alloc_mtt(dev, list_len); 437 if (IS_ERR(mr->mtt)) 438 return PTR_ERR(mr->mtt); 439 440 err = mthca_write_mtt(dev, mr->mtt, 0, buffer_list, list_len); 441 if (err) { 442 mthca_free_mtt(dev, mr->mtt); 443 return err; 444 } 445 446 err = mthca_mr_alloc(dev, pd, buffer_size_shift, iova, 447 total_size, access, mr); 448 if (err) 449 mthca_free_mtt(dev, mr->mtt); 450 451 return err; 452} 453 454/* Free mr or fmr */ 455static void mthca_free_region(struct mthca_dev *dev, u32 lkey) 456{ 457 mthca_table_put(dev, dev->mr_table.mpt_table, 458 key_to_hw_index(dev, lkey)); 459 460 mthca_free(&dev->mr_table.mpt_alloc, key_to_hw_index(dev, lkey)); 461} 462 463void mthca_free_mr(struct mthca_dev *dev, struct mthca_mr *mr) 464{ 465 int err; 466 u8 status; 467 468 err = mthca_HW2SW_MPT(dev, NULL, 469 key_to_hw_index(dev, mr->ibmr.lkey) & 470 (dev->limits.num_mpts - 1), 471 &status); 472 if (err) 473 mthca_warn(dev, "HW2SW_MPT failed (%d)\n", err); 474 else if (status) 475 mthca_warn(dev, "HW2SW_MPT returned status 0x%02x\n", 476 status); 477 478 mthca_free_region(dev, mr->ibmr.lkey); 479 mthca_free_mtt(dev, mr->mtt); 480} 481 482int mthca_fmr_alloc(struct mthca_dev *dev, u32 pd, 483 u32 access, struct mthca_fmr *mr) 484{ 485 struct mthca_mpt_entry *mpt_entry; 486 struct mthca_mailbox *mailbox; 487 u64 mtt_seg; 488 u32 key, idx; 489 u8 status; 490 int list_len = mr->attr.max_pages; 491 int err = -ENOMEM; 492 int i; 493 494 if (mr->attr.page_size < 12 || mr->attr.page_size >= 32) 495 return -EINVAL; 496 497 /* For Arbel, all MTTs must fit in the same page. */ 498 if (mthca_is_memfree(dev) && 499 mr->attr.max_pages * sizeof *mr->mem.arbel.mtts > PAGE_SIZE) 500 return -EINVAL; 501 502 mr->maps = 0; 503 504 key = mthca_alloc(&dev->mr_table.mpt_alloc); 505 if (key == -1) 506 return -ENOMEM; 507 508 idx = key & (dev->limits.num_mpts - 1); 509 mr->ibmr.rkey = mr->ibmr.lkey = hw_index_to_key(dev, key); 510 511 if (mthca_is_memfree(dev)) { 512 err = mthca_table_get(dev, dev->mr_table.mpt_table, key); 513 if (err) 514 goto err_out_mpt_free; 515 516 mr->mem.arbel.mpt = mthca_table_find(dev->mr_table.mpt_table, key); 517 BUG_ON(!mr->mem.arbel.mpt); 518 } else 519 mr->mem.tavor.mpt = dev->mr_table.tavor_fmr.mpt_base + 520 sizeof *(mr->mem.tavor.mpt) * idx; 521 522 mr->mtt = __mthca_alloc_mtt(dev, list_len, dev->mr_table.fmr_mtt_buddy); 523 if (IS_ERR(mr->mtt)) 524 goto err_out_table; 525 526 mtt_seg = mr->mtt->first_seg * MTHCA_MTT_SEG_SIZE; 527 528 if (mthca_is_memfree(dev)) { 529 mr->mem.arbel.mtts = mthca_table_find(dev->mr_table.mtt_table, 530 mr->mtt->first_seg); 531 BUG_ON(!mr->mem.arbel.mtts); 532 } else 533 mr->mem.tavor.mtts = dev->mr_table.tavor_fmr.mtt_base + mtt_seg; 534 535 mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); 536 if (IS_ERR(mailbox)) 537 goto err_out_free_mtt; 538 539 mpt_entry = mailbox->buf; 540 541 mpt_entry->flags = cpu_to_be32(MTHCA_MPT_FLAG_SW_OWNS | 542 MTHCA_MPT_FLAG_MIO | 543 MTHCA_MPT_FLAG_REGION | 544 access); 545 546 mpt_entry->page_size = cpu_to_be32(mr->attr.page_size - 12); 547 mpt_entry->key = cpu_to_be32(key); 548 mpt_entry->pd = cpu_to_be32(pd); 549 memset(&mpt_entry->start, 0, 550 sizeof *mpt_entry - offsetof(struct mthca_mpt_entry, start)); 551 mpt_entry->mtt_seg = cpu_to_be64(dev->mr_table.mtt_base + mtt_seg); 552 553 if (0) { 554 mthca_dbg(dev, "Dumping MPT entry %08x:\n", mr->ibmr.lkey); 555 for (i = 0; i < sizeof (struct mthca_mpt_entry) / 4; ++i) { 556 if (i % 4 == 0) 557 printk("[%02x] ", i * 4); 558 printk(" %08x", be32_to_cpu(((__be32 *) mpt_entry)[i])); 559 if ((i + 1) % 4 == 0) 560 printk("\n"); 561 } 562 } 563 564 err = mthca_SW2HW_MPT(dev, mailbox, 565 key & (dev->limits.num_mpts - 1), 566 &status); 567 if (err) { 568 mthca_warn(dev, "SW2HW_MPT failed (%d)\n", err); 569 goto err_out_mailbox_free; 570 } 571 if (status) { 572 mthca_warn(dev, "SW2HW_MPT returned status 0x%02x\n", 573 status); 574 err = -EINVAL; 575 goto err_out_mailbox_free; 576 } 577 578 mthca_free_mailbox(dev, mailbox); 579 return 0; 580 581err_out_mailbox_free: 582 mthca_free_mailbox(dev, mailbox); 583 584err_out_free_mtt: 585 mthca_free_mtt(dev, mr->mtt); 586 587err_out_table: 588 mthca_table_put(dev, dev->mr_table.mpt_table, key); 589 590err_out_mpt_free: 591 mthca_free(&dev->mr_table.mpt_alloc, mr->ibmr.lkey); 592 return err; 593} 594 595int mthca_free_fmr(struct mthca_dev *dev, struct mthca_fmr *fmr) 596{ 597 if (fmr->maps) 598 return -EBUSY; 599 600 mthca_free_region(dev, fmr->ibmr.lkey); 601 mthca_free_mtt(dev, fmr->mtt); 602 603 return 0; 604} 605 606static inline int mthca_check_fmr(struct mthca_fmr *fmr, u64 *page_list, 607 int list_len, u64 iova) 608{ 609 int i, page_mask; 610 611 if (list_len > fmr->attr.max_pages) 612 return -EINVAL; 613 614 page_mask = (1 << fmr->attr.page_size) - 1; 615 616 /* We are getting page lists, so va must be page aligned. */ 617 if (iova & page_mask) 618 return -EINVAL; 619 620 /* Trust the user not to pass misaligned data in page_list */ 621 if (0) 622 for (i = 0; i < list_len; ++i) { 623 if (page_list[i] & ~page_mask) 624 return -EINVAL; 625 } 626 627 if (fmr->maps >= fmr->attr.max_maps) 628 return -EINVAL; 629 630 return 0; 631} 632 633 634int mthca_tavor_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list, 635 int list_len, u64 iova) 636{ 637 struct mthca_fmr *fmr = to_mfmr(ibfmr); 638 struct mthca_dev *dev = to_mdev(ibfmr->device); 639 struct mthca_mpt_entry mpt_entry; 640 u32 key; 641 int i, err; 642 643 err = mthca_check_fmr(fmr, page_list, list_len, iova); 644 if (err) 645 return err; 646 647 ++fmr->maps; 648 649 key = tavor_key_to_hw_index(fmr->ibmr.lkey); 650 key += dev->limits.num_mpts; 651 fmr->ibmr.lkey = fmr->ibmr.rkey = tavor_hw_index_to_key(key); 652 653 writeb(MTHCA_MPT_STATUS_SW, fmr->mem.tavor.mpt); 654 655 for (i = 0; i < list_len; ++i) { 656 __be64 mtt_entry = cpu_to_be64(page_list[i] | 657 MTHCA_MTT_FLAG_PRESENT); 658 mthca_write64_raw(mtt_entry, fmr->mem.tavor.mtts + i); 659 } 660 661 mpt_entry.lkey = cpu_to_be32(key); 662 mpt_entry.length = cpu_to_be64(list_len * (1ull << fmr->attr.page_size)); 663 mpt_entry.start = cpu_to_be64(iova); 664 665 __raw_writel((__force u32) mpt_entry.lkey, &fmr->mem.tavor.mpt->key); 666 memcpy_toio(&fmr->mem.tavor.mpt->start, &mpt_entry.start, 667 offsetof(struct mthca_mpt_entry, window_count) - 668 offsetof(struct mthca_mpt_entry, start)); 669 670 writeb(MTHCA_MPT_STATUS_HW, fmr->mem.tavor.mpt); 671 672 return 0; 673} 674 675int mthca_arbel_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list, 676 int list_len, u64 iova) 677{ 678 struct mthca_fmr *fmr = to_mfmr(ibfmr); 679 struct mthca_dev *dev = to_mdev(ibfmr->device); 680 u32 key; 681 int i, err; 682 683 err = mthca_check_fmr(fmr, page_list, list_len, iova); 684 if (err) 685 return err; 686 687 ++fmr->maps; 688 689 key = arbel_key_to_hw_index(fmr->ibmr.lkey); 690 key += dev->limits.num_mpts; 691 fmr->ibmr.lkey = fmr->ibmr.rkey = arbel_hw_index_to_key(key); 692 693 *(u8 *) fmr->mem.arbel.mpt = MTHCA_MPT_STATUS_SW; 694 695 wmb(); 696 697 for (i = 0; i < list_len; ++i) 698 fmr->mem.arbel.mtts[i] = cpu_to_be64(page_list[i] | 699 MTHCA_MTT_FLAG_PRESENT); 700 701 fmr->mem.arbel.mpt->key = cpu_to_be32(key); 702 fmr->mem.arbel.mpt->lkey = cpu_to_be32(key); 703 fmr->mem.arbel.mpt->length = cpu_to_be64(list_len * (1ull << fmr->attr.page_size)); 704 fmr->mem.arbel.mpt->start = cpu_to_be64(iova); 705 706 wmb(); 707 708 *(u8 *) fmr->mem.arbel.mpt = MTHCA_MPT_STATUS_HW; 709 710 wmb(); 711 712 return 0; 713} 714 715void mthca_tavor_fmr_unmap(struct mthca_dev *dev, struct mthca_fmr *fmr) 716{ 717 u32 key; 718 719 if (!fmr->maps) 720 return; 721 722 key = tavor_key_to_hw_index(fmr->ibmr.lkey); 723 key &= dev->limits.num_mpts - 1; 724 fmr->ibmr.lkey = fmr->ibmr.rkey = tavor_hw_index_to_key(key); 725 726 fmr->maps = 0; 727 728 writeb(MTHCA_MPT_STATUS_SW, fmr->mem.tavor.mpt); 729} 730 731void mthca_arbel_fmr_unmap(struct mthca_dev *dev, struct mthca_fmr *fmr) 732{ 733 u32 key; 734 735 if (!fmr->maps) 736 return; 737 738 key = arbel_key_to_hw_index(fmr->ibmr.lkey); 739 key &= dev->limits.num_mpts - 1; 740 fmr->ibmr.lkey = fmr->ibmr.rkey = arbel_hw_index_to_key(key); 741 742 fmr->maps = 0; 743 744 *(u8 *) fmr->mem.arbel.mpt = MTHCA_MPT_STATUS_SW; 745} 746 747int __devinit mthca_init_mr_table(struct mthca_dev *dev) 748{ 749 int err, i; 750 751 err = mthca_alloc_init(&dev->mr_table.mpt_alloc, 752 dev->limits.num_mpts, 753 ~0, dev->limits.reserved_mrws); 754 if (err) 755 return err; 756 757 if (!mthca_is_memfree(dev) && 758 (dev->mthca_flags & MTHCA_FLAG_DDR_HIDDEN)) 759 dev->limits.fmr_reserved_mtts = 0; 760 else 761 dev->mthca_flags |= MTHCA_FLAG_FMR; 762 763 err = mthca_buddy_init(&dev->mr_table.mtt_buddy, 764 fls(dev->limits.num_mtt_segs - 1)); 765 766 if (err) 767 goto err_mtt_buddy; 768 769 dev->mr_table.tavor_fmr.mpt_base = NULL; 770 dev->mr_table.tavor_fmr.mtt_base = NULL; 771 772 if (dev->limits.fmr_reserved_mtts) { 773 i = fls(dev->limits.fmr_reserved_mtts - 1); 774 775 if (i >= 31) { 776 mthca_warn(dev, "Unable to reserve 2^31 FMR MTTs.\n"); 777 err = -EINVAL; 778 goto err_fmr_mpt; 779 } 780 781 dev->mr_table.tavor_fmr.mpt_base = 782 ioremap(dev->mr_table.mpt_base, 783 (1 << i) * sizeof (struct mthca_mpt_entry)); 784 785 if (!dev->mr_table.tavor_fmr.mpt_base) { 786 mthca_warn(dev, "MPT ioremap for FMR failed.\n"); 787 err = -ENOMEM; 788 goto err_fmr_mpt; 789 } 790 791 dev->mr_table.tavor_fmr.mtt_base = 792 ioremap(dev->mr_table.mtt_base, 793 (1 << i) * MTHCA_MTT_SEG_SIZE); 794 if (!dev->mr_table.tavor_fmr.mtt_base) { 795 mthca_warn(dev, "MTT ioremap for FMR failed.\n"); 796 err = -ENOMEM; 797 goto err_fmr_mtt; 798 } 799 800 err = mthca_buddy_init(&dev->mr_table.tavor_fmr.mtt_buddy, i); 801 if (err) 802 goto err_fmr_mtt_buddy; 803 804 /* Prevent regular MRs from using FMR keys */ 805 err = mthca_buddy_alloc(&dev->mr_table.mtt_buddy, i); 806 if (err) 807 goto err_reserve_fmr; 808 809 dev->mr_table.fmr_mtt_buddy = 810 &dev->mr_table.tavor_fmr.mtt_buddy; 811 } else 812 dev->mr_table.fmr_mtt_buddy = &dev->mr_table.mtt_buddy; 813 814 /* FMR table is always the first, take reserved MTTs out of there */ 815 if (dev->limits.reserved_mtts) { 816 i = fls(dev->limits.reserved_mtts - 1); 817 818 if (mthca_alloc_mtt_range(dev, i, 819 dev->mr_table.fmr_mtt_buddy) == -1) { 820 mthca_warn(dev, "MTT table of order %d is too small.\n", 821 dev->mr_table.fmr_mtt_buddy->max_order); 822 err = -ENOMEM; 823 goto err_reserve_mtts; 824 } 825 } 826 827 return 0; 828 829err_reserve_mtts: 830err_reserve_fmr: 831 if (dev->limits.fmr_reserved_mtts) 832 mthca_buddy_cleanup(&dev->mr_table.tavor_fmr.mtt_buddy); 833 834err_fmr_mtt_buddy: 835 if (dev->mr_table.tavor_fmr.mtt_base) 836 iounmap(dev->mr_table.tavor_fmr.mtt_base); 837 838err_fmr_mtt: 839 if (dev->mr_table.tavor_fmr.mpt_base) 840 iounmap(dev->mr_table.tavor_fmr.mpt_base); 841 842err_fmr_mpt: 843 mthca_buddy_cleanup(&dev->mr_table.mtt_buddy); 844 845err_mtt_buddy: 846 mthca_alloc_cleanup(&dev->mr_table.mpt_alloc); 847 848 return err; 849} 850 851void __devexit mthca_cleanup_mr_table(struct mthca_dev *dev) 852{ 853 /* XXX check if any MRs are still allocated? */ 854 if (dev->limits.fmr_reserved_mtts) 855 mthca_buddy_cleanup(&dev->mr_table.tavor_fmr.mtt_buddy); 856 857 mthca_buddy_cleanup(&dev->mr_table.mtt_buddy); 858 859 if (dev->mr_table.tavor_fmr.mtt_base) 860 iounmap(dev->mr_table.tavor_fmr.mtt_base); 861 if (dev->mr_table.tavor_fmr.mpt_base) 862 iounmap(dev->mr_table.tavor_fmr.mpt_base); 863 864 mthca_alloc_cleanup(&dev->mr_table.mpt_alloc); 865} 866