mthca_memfree.c revision 85665c9816a2b38866e03c019bf5c2e47eb9fb04
1/* 2 * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 * 32 * $Id$ 33 */ 34 35#include "mthca_memfree.h" 36#include "mthca_dev.h" 37#include "mthca_cmd.h" 38 39/* 40 * We allocate in as big chunks as we can, up to a maximum of 256 KB 41 * per chunk. 42 */ 43enum { 44 MTHCA_ICM_ALLOC_SIZE = 1 << 18, 45 MTHCA_TABLE_CHUNK_SIZE = 1 << 18 46}; 47 48void mthca_free_icm(struct mthca_dev *dev, struct mthca_icm *icm) 49{ 50 struct mthca_icm_chunk *chunk, *tmp; 51 int i; 52 53 if (!icm) 54 return; 55 56 list_for_each_entry_safe(chunk, tmp, &icm->chunk_list, list) { 57 if (chunk->nsg > 0) 58 pci_unmap_sg(dev->pdev, chunk->mem, chunk->npages, 59 PCI_DMA_BIDIRECTIONAL); 60 61 for (i = 0; i < chunk->npages; ++i) 62 __free_pages(chunk->mem[i].page, 63 get_order(chunk->mem[i].length)); 64 65 kfree(chunk); 66 } 67 68 kfree(icm); 69} 70 71struct mthca_icm *mthca_alloc_icm(struct mthca_dev *dev, int npages, 72 unsigned int gfp_mask) 73{ 74 struct mthca_icm *icm; 75 struct mthca_icm_chunk *chunk = NULL; 76 int cur_order; 77 78 icm = kmalloc(sizeof *icm, gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN)); 79 if (!icm) 80 return icm; 81 82 icm->refcount = 0; 83 INIT_LIST_HEAD(&icm->chunk_list); 84 85 cur_order = get_order(MTHCA_ICM_ALLOC_SIZE); 86 87 while (npages > 0) { 88 if (!chunk) { 89 chunk = kmalloc(sizeof *chunk, 90 gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN)); 91 if (!chunk) 92 goto fail; 93 94 chunk->npages = 0; 95 chunk->nsg = 0; 96 list_add_tail(&chunk->list, &icm->chunk_list); 97 } 98 99 while (1 << cur_order > npages) 100 --cur_order; 101 102 chunk->mem[chunk->npages].page = alloc_pages(gfp_mask, cur_order); 103 if (chunk->mem[chunk->npages].page) { 104 chunk->mem[chunk->npages].length = PAGE_SIZE << cur_order; 105 chunk->mem[chunk->npages].offset = 0; 106 107 if (++chunk->npages == MTHCA_ICM_CHUNK_LEN) { 108 chunk->nsg = pci_map_sg(dev->pdev, chunk->mem, 109 chunk->npages, 110 PCI_DMA_BIDIRECTIONAL); 111 112 if (chunk->nsg <= 0) 113 goto fail; 114 115 chunk = NULL; 116 } 117 118 npages -= 1 << cur_order; 119 } else { 120 --cur_order; 121 if (cur_order < 0) 122 goto fail; 123 } 124 } 125 126 if (chunk) { 127 chunk->nsg = pci_map_sg(dev->pdev, chunk->mem, 128 chunk->npages, 129 PCI_DMA_BIDIRECTIONAL); 130 131 if (chunk->nsg <= 0) 132 goto fail; 133 } 134 135 return icm; 136 137fail: 138 mthca_free_icm(dev, icm); 139 return NULL; 140} 141 142int mthca_table_get(struct mthca_dev *dev, struct mthca_icm_table *table, int obj) 143{ 144 int i = (obj & (table->num_obj - 1)) * table->obj_size / MTHCA_TABLE_CHUNK_SIZE; 145 int ret = 0; 146 u8 status; 147 148 down(&table->mutex); 149 150 if (table->icm[i]) { 151 ++table->icm[i]->refcount; 152 goto out; 153 } 154 155 table->icm[i] = mthca_alloc_icm(dev, MTHCA_TABLE_CHUNK_SIZE >> PAGE_SHIFT, 156 (table->lowmem ? GFP_KERNEL : GFP_HIGHUSER) | 157 __GFP_NOWARN); 158 if (!table->icm[i]) { 159 ret = -ENOMEM; 160 goto out; 161 } 162 163 if (mthca_MAP_ICM(dev, table->icm[i], table->virt + i * MTHCA_TABLE_CHUNK_SIZE, 164 &status) || status) { 165 mthca_free_icm(dev, table->icm[i]); 166 table->icm[i] = NULL; 167 ret = -ENOMEM; 168 goto out; 169 } 170 171 ++table->icm[i]->refcount; 172 173out: 174 up(&table->mutex); 175 return ret; 176} 177 178void mthca_table_put(struct mthca_dev *dev, struct mthca_icm_table *table, int obj) 179{ 180 int i = (obj & (table->num_obj - 1)) * table->obj_size / MTHCA_TABLE_CHUNK_SIZE; 181 u8 status; 182 183 down(&table->mutex); 184 185 if (--table->icm[i]->refcount == 0) { 186 mthca_UNMAP_ICM(dev, table->virt + i * MTHCA_TABLE_CHUNK_SIZE, 187 MTHCA_TABLE_CHUNK_SIZE >> 12, &status); 188 mthca_free_icm(dev, table->icm[i]); 189 table->icm[i] = NULL; 190 } 191 192 up(&table->mutex); 193} 194 195int mthca_table_get_range(struct mthca_dev *dev, struct mthca_icm_table *table, 196 int start, int end) 197{ 198 int inc = MTHCA_TABLE_CHUNK_SIZE / table->obj_size; 199 int i, err; 200 201 for (i = start; i <= end; i += inc) { 202 err = mthca_table_get(dev, table, i); 203 if (err) 204 goto fail; 205 } 206 207 return 0; 208 209fail: 210 while (i > start) { 211 i -= inc; 212 mthca_table_put(dev, table, i); 213 } 214 215 return err; 216} 217 218void mthca_table_put_range(struct mthca_dev *dev, struct mthca_icm_table *table, 219 int start, int end) 220{ 221 int i; 222 223 for (i = start; i <= end; i += MTHCA_TABLE_CHUNK_SIZE / table->obj_size) 224 mthca_table_put(dev, table, i); 225} 226 227struct mthca_icm_table *mthca_alloc_icm_table(struct mthca_dev *dev, 228 u64 virt, int obj_size, 229 int nobj, int reserved, 230 int use_lowmem) 231{ 232 struct mthca_icm_table *table; 233 int num_icm; 234 int i; 235 u8 status; 236 237 num_icm = obj_size * nobj / MTHCA_TABLE_CHUNK_SIZE; 238 239 table = kmalloc(sizeof *table + num_icm * sizeof *table->icm, GFP_KERNEL); 240 if (!table) 241 return NULL; 242 243 table->virt = virt; 244 table->num_icm = num_icm; 245 table->num_obj = nobj; 246 table->obj_size = obj_size; 247 table->lowmem = use_lowmem; 248 init_MUTEX(&table->mutex); 249 250 for (i = 0; i < num_icm; ++i) 251 table->icm[i] = NULL; 252 253 for (i = 0; i * MTHCA_TABLE_CHUNK_SIZE < reserved * obj_size; ++i) { 254 table->icm[i] = mthca_alloc_icm(dev, MTHCA_TABLE_CHUNK_SIZE >> PAGE_SHIFT, 255 (use_lowmem ? GFP_KERNEL : GFP_HIGHUSER) | 256 __GFP_NOWARN); 257 if (!table->icm[i]) 258 goto err; 259 if (mthca_MAP_ICM(dev, table->icm[i], virt + i * MTHCA_TABLE_CHUNK_SIZE, 260 &status) || status) { 261 mthca_free_icm(dev, table->icm[i]); 262 table->icm[i] = NULL; 263 goto err; 264 } 265 266 /* 267 * Add a reference to this ICM chunk so that it never 268 * gets freed (since it contains reserved firmware objects). 269 */ 270 ++table->icm[i]->refcount; 271 } 272 273 return table; 274 275err: 276 for (i = 0; i < num_icm; ++i) 277 if (table->icm[i]) { 278 mthca_UNMAP_ICM(dev, virt + i * MTHCA_TABLE_CHUNK_SIZE, 279 MTHCA_TABLE_CHUNK_SIZE >> 12, &status); 280 mthca_free_icm(dev, table->icm[i]); 281 } 282 283 kfree(table); 284 285 return NULL; 286} 287 288void mthca_free_icm_table(struct mthca_dev *dev, struct mthca_icm_table *table) 289{ 290 int i; 291 u8 status; 292 293 for (i = 0; i < table->num_icm; ++i) 294 if (table->icm[i]) { 295 mthca_UNMAP_ICM(dev, table->virt + i * MTHCA_TABLE_CHUNK_SIZE, 296 MTHCA_TABLE_CHUNK_SIZE >> 12, &status); 297 mthca_free_icm(dev, table->icm[i]); 298 } 299 300 kfree(table); 301} 302 303static u64 mthca_uarc_virt(struct mthca_dev *dev, int page) 304{ 305 return dev->uar_table.uarc_base + 306 dev->driver_uar.index * dev->uar_table.uarc_size + 307 page * 4096; 308} 309 310int mthca_alloc_db(struct mthca_dev *dev, int type, u32 qn, u32 **db) 311{ 312 int group; 313 int start, end, dir; 314 int i, j; 315 struct mthca_db_page *page; 316 int ret = 0; 317 u8 status; 318 319 down(&dev->db_tab->mutex); 320 321 switch (type) { 322 case MTHCA_DB_TYPE_CQ_ARM: 323 case MTHCA_DB_TYPE_SQ: 324 group = 0; 325 start = 0; 326 end = dev->db_tab->max_group1; 327 dir = 1; 328 break; 329 330 case MTHCA_DB_TYPE_CQ_SET_CI: 331 case MTHCA_DB_TYPE_RQ: 332 case MTHCA_DB_TYPE_SRQ: 333 group = 1; 334 start = dev->db_tab->npages - 1; 335 end = dev->db_tab->min_group2; 336 dir = -1; 337 break; 338 339 default: 340 return -1; 341 } 342 343 for (i = start; i != end; i += dir) 344 if (dev->db_tab->page[i].db_rec && 345 !bitmap_full(dev->db_tab->page[i].used, 346 MTHCA_DB_REC_PER_PAGE)) { 347 page = dev->db_tab->page + i; 348 goto found; 349 } 350 351 if (dev->db_tab->max_group1 >= dev->db_tab->min_group2 - 1) { 352 ret = -ENOMEM; 353 goto out; 354 } 355 356 page = dev->db_tab->page + end; 357 page->db_rec = dma_alloc_coherent(&dev->pdev->dev, 4096, 358 &page->mapping, GFP_KERNEL); 359 if (!page->db_rec) { 360 ret = -ENOMEM; 361 goto out; 362 } 363 memset(page->db_rec, 0, 4096); 364 365 ret = mthca_MAP_ICM_page(dev, page->mapping, mthca_uarc_virt(dev, i), &status); 366 if (!ret && status) 367 ret = -EINVAL; 368 if (ret) { 369 dma_free_coherent(&dev->pdev->dev, 4096, 370 page->db_rec, page->mapping); 371 goto out; 372 } 373 374 bitmap_zero(page->used, MTHCA_DB_REC_PER_PAGE); 375 if (group == 0) 376 ++dev->db_tab->max_group1; 377 else 378 --dev->db_tab->min_group2; 379 380found: 381 j = find_first_zero_bit(page->used, MTHCA_DB_REC_PER_PAGE); 382 set_bit(j, page->used); 383 384 if (group == 1) 385 j = MTHCA_DB_REC_PER_PAGE - 1 - j; 386 387 ret = i * MTHCA_DB_REC_PER_PAGE + j; 388 389 page->db_rec[j] = cpu_to_be64((qn << 8) | (type << 5)); 390 391 *db = (u32 *) &page->db_rec[j]; 392 393out: 394 up(&dev->db_tab->mutex); 395 396 return ret; 397} 398 399void mthca_free_db(struct mthca_dev *dev, int type, int db_index) 400{ 401 int i, j; 402 struct mthca_db_page *page; 403 u8 status; 404 405 i = db_index / MTHCA_DB_REC_PER_PAGE; 406 j = db_index % MTHCA_DB_REC_PER_PAGE; 407 408 page = dev->db_tab->page + i; 409 410 down(&dev->db_tab->mutex); 411 412 page->db_rec[j] = 0; 413 if (i >= dev->db_tab->min_group2) 414 j = MTHCA_DB_REC_PER_PAGE - 1 - j; 415 clear_bit(j, page->used); 416 417 if (bitmap_empty(page->used, MTHCA_DB_REC_PER_PAGE) && 418 i >= dev->db_tab->max_group1 - 1) { 419 mthca_UNMAP_ICM(dev, mthca_uarc_virt(dev, i), 1, &status); 420 421 dma_free_coherent(&dev->pdev->dev, 4096, 422 page->db_rec, page->mapping); 423 page->db_rec = NULL; 424 425 if (i == dev->db_tab->max_group1) { 426 --dev->db_tab->max_group1; 427 /* XXX may be able to unmap more pages now */ 428 } 429 if (i == dev->db_tab->min_group2) 430 ++dev->db_tab->min_group2; 431 } 432 433 up(&dev->db_tab->mutex); 434} 435 436int mthca_init_db_tab(struct mthca_dev *dev) 437{ 438 int i; 439 440 if (dev->hca_type != ARBEL_NATIVE) 441 return 0; 442 443 dev->db_tab = kmalloc(sizeof *dev->db_tab, GFP_KERNEL); 444 if (!dev->db_tab) 445 return -ENOMEM; 446 447 init_MUTEX(&dev->db_tab->mutex); 448 449 dev->db_tab->npages = dev->uar_table.uarc_size / 4096; 450 dev->db_tab->max_group1 = 0; 451 dev->db_tab->min_group2 = dev->db_tab->npages - 1; 452 453 dev->db_tab->page = kmalloc(dev->db_tab->npages * 454 sizeof *dev->db_tab->page, 455 GFP_KERNEL); 456 if (!dev->db_tab->page) { 457 kfree(dev->db_tab); 458 return -ENOMEM; 459 } 460 461 for (i = 0; i < dev->db_tab->npages; ++i) 462 dev->db_tab->page[i].db_rec = NULL; 463 464 return 0; 465} 466 467void mthca_cleanup_db_tab(struct mthca_dev *dev) 468{ 469 int i; 470 u8 status; 471 472 if (dev->hca_type != ARBEL_NATIVE) 473 return; 474 475 /* 476 * Because we don't always free our UARC pages when they 477 * become empty to make mthca_free_db() simpler we need to 478 * make a sweep through the doorbell pages and free any 479 * leftover pages now. 480 */ 481 for (i = 0; i < dev->db_tab->npages; ++i) { 482 if (!dev->db_tab->page[i].db_rec) 483 continue; 484 485 if (!bitmap_empty(dev->db_tab->page[i].used, MTHCA_DB_REC_PER_PAGE)) 486 mthca_warn(dev, "Kernel UARC page %d not empty\n", i); 487 488 mthca_UNMAP_ICM(dev, mthca_uarc_virt(dev, i), 1, &status); 489 490 dma_free_coherent(&dev->pdev->dev, 4096, 491 dev->db_tab->page[i].db_rec, 492 dev->db_tab->page[i].mapping); 493 } 494 495 kfree(dev->db_tab->page); 496 kfree(dev->db_tab); 497} 498