1/* 2 * Copyright (c) 2013, Mellanox Technologies inc. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33#include <asm-generic/kmap_types.h> 34#include <linux/kernel.h> 35#include <linux/module.h> 36#include <linux/mlx5/driver.h> 37#include <linux/mlx5/cmd.h> 38#include "mlx5_core.h" 39 40enum { 41 MLX5_PAGES_CANT_GIVE = 0, 42 MLX5_PAGES_GIVE = 1, 43 MLX5_PAGES_TAKE = 2 44}; 45 46enum { 47 MLX5_BOOT_PAGES = 1, 48 MLX5_INIT_PAGES = 2, 49 MLX5_POST_INIT_PAGES = 3 50}; 51 52struct mlx5_pages_req { 53 struct mlx5_core_dev *dev; 54 u16 func_id; 55 s32 npages; 56 struct work_struct work; 57}; 58 59struct fw_page { 60 struct rb_node rb_node; 61 u64 addr; 62 struct page *page; 63 u16 func_id; 64 unsigned long bitmask; 65 struct list_head list; 66 unsigned free_count; 67}; 68 69struct mlx5_query_pages_inbox { 70 struct mlx5_inbox_hdr hdr; 71 u8 rsvd[8]; 72}; 73 74struct mlx5_query_pages_outbox { 75 struct mlx5_outbox_hdr hdr; 76 __be16 rsvd; 77 __be16 func_id; 78 __be32 num_pages; 79}; 80 81struct mlx5_manage_pages_inbox { 82 struct mlx5_inbox_hdr hdr; 83 __be16 rsvd; 84 __be16 func_id; 85 __be32 num_entries; 86 __be64 pas[0]; 87}; 88 89struct mlx5_manage_pages_outbox { 90 struct mlx5_outbox_hdr hdr; 91 __be32 num_entries; 92 u8 rsvd[4]; 93 __be64 pas[0]; 94}; 95 96enum { 97 MAX_RECLAIM_TIME_MSECS = 5000, 98}; 99 100enum { 101 MLX5_MAX_RECLAIM_TIME_MILI = 5000, 102 MLX5_NUM_4K_IN_PAGE = PAGE_SIZE / MLX5_ADAPTER_PAGE_SIZE, 103}; 104 105static int insert_page(struct mlx5_core_dev *dev, u64 addr, struct page *page, u16 func_id) 106{ 107 struct rb_root *root = &dev->priv.page_root; 108 struct rb_node **new = &root->rb_node; 109 struct rb_node *parent = NULL; 110 struct fw_page *nfp; 111 struct fw_page *tfp; 112 int i; 113 114 while (*new) { 115 parent = *new; 116 tfp = rb_entry(parent, struct fw_page, rb_node); 117 if (tfp->addr < addr) 118 new = &parent->rb_left; 119 else if (tfp->addr > addr) 120 new = &parent->rb_right; 121 else 122 return -EEXIST; 123 } 124 125 nfp = kzalloc(sizeof(*nfp), GFP_KERNEL); 126 if (!nfp) 127 return -ENOMEM; 128 129 nfp->addr = addr; 130 nfp->page = page; 131 nfp->func_id = func_id; 132 nfp->free_count = MLX5_NUM_4K_IN_PAGE; 133 for (i = 0; i < MLX5_NUM_4K_IN_PAGE; i++) 134 set_bit(i, &nfp->bitmask); 135 136 rb_link_node(&nfp->rb_node, parent, new); 137 rb_insert_color(&nfp->rb_node, root); 138 list_add(&nfp->list, &dev->priv.free_list); 139 140 return 0; 141} 142 143static struct fw_page *find_fw_page(struct mlx5_core_dev *dev, u64 addr) 144{ 145 struct rb_root *root = &dev->priv.page_root; 146 struct rb_node *tmp = root->rb_node; 147 struct fw_page *result = NULL; 148 struct fw_page *tfp; 149 150 while (tmp) { 151 tfp = rb_entry(tmp, struct fw_page, rb_node); 152 if (tfp->addr < addr) { 153 tmp = tmp->rb_left; 154 } else if (tfp->addr > addr) { 155 tmp = tmp->rb_right; 156 } else { 157 result = tfp; 158 break; 159 } 160 } 161 162 return result; 163} 164 165static int mlx5_cmd_query_pages(struct mlx5_core_dev *dev, u16 *func_id, 166 s32 *npages, int boot) 167{ 168 struct mlx5_query_pages_inbox in; 169 struct mlx5_query_pages_outbox out; 170 int err; 171 172 memset(&in, 0, sizeof(in)); 173 memset(&out, 0, sizeof(out)); 174 in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_QUERY_PAGES); 175 in.hdr.opmod = boot ? cpu_to_be16(MLX5_BOOT_PAGES) : cpu_to_be16(MLX5_INIT_PAGES); 176 177 err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out)); 178 if (err) 179 return err; 180 181 if (out.hdr.status) 182 return mlx5_cmd_status_to_err(&out.hdr); 183 184 *npages = be32_to_cpu(out.num_pages); 185 *func_id = be16_to_cpu(out.func_id); 186 187 return err; 188} 189 190static int alloc_4k(struct mlx5_core_dev *dev, u64 *addr) 191{ 192 struct fw_page *fp; 193 unsigned n; 194 195 if (list_empty(&dev->priv.free_list)) 196 return -ENOMEM; 197 198 fp = list_entry(dev->priv.free_list.next, struct fw_page, list); 199 n = find_first_bit(&fp->bitmask, 8 * sizeof(fp->bitmask)); 200 if (n >= MLX5_NUM_4K_IN_PAGE) { 201 mlx5_core_warn(dev, "alloc 4k bug\n"); 202 return -ENOENT; 203 } 204 clear_bit(n, &fp->bitmask); 205 fp->free_count--; 206 if (!fp->free_count) 207 list_del(&fp->list); 208 209 *addr = fp->addr + n * MLX5_ADAPTER_PAGE_SIZE; 210 211 return 0; 212} 213 214static void free_4k(struct mlx5_core_dev *dev, u64 addr) 215{ 216 struct fw_page *fwp; 217 int n; 218 219 fwp = find_fw_page(dev, addr & PAGE_MASK); 220 if (!fwp) { 221 mlx5_core_warn(dev, "page not found\n"); 222 return; 223 } 224 225 n = (addr & ~PAGE_MASK) >> MLX5_ADAPTER_PAGE_SHIFT; 226 fwp->free_count++; 227 set_bit(n, &fwp->bitmask); 228 if (fwp->free_count == MLX5_NUM_4K_IN_PAGE) { 229 rb_erase(&fwp->rb_node, &dev->priv.page_root); 230 if (fwp->free_count != 1) 231 list_del(&fwp->list); 232 dma_unmap_page(&dev->pdev->dev, addr & PAGE_MASK, PAGE_SIZE, 233 DMA_BIDIRECTIONAL); 234 __free_page(fwp->page); 235 kfree(fwp); 236 } else if (fwp->free_count == 1) { 237 list_add(&fwp->list, &dev->priv.free_list); 238 } 239} 240 241static int alloc_system_page(struct mlx5_core_dev *dev, u16 func_id) 242{ 243 struct page *page; 244 u64 addr; 245 int err; 246 247 page = alloc_page(GFP_HIGHUSER); 248 if (!page) { 249 mlx5_core_warn(dev, "failed to allocate page\n"); 250 return -ENOMEM; 251 } 252 addr = dma_map_page(&dev->pdev->dev, page, 0, 253 PAGE_SIZE, DMA_BIDIRECTIONAL); 254 if (dma_mapping_error(&dev->pdev->dev, addr)) { 255 mlx5_core_warn(dev, "failed dma mapping page\n"); 256 err = -ENOMEM; 257 goto out_alloc; 258 } 259 err = insert_page(dev, addr, page, func_id); 260 if (err) { 261 mlx5_core_err(dev, "failed to track allocated page\n"); 262 goto out_mapping; 263 } 264 265 return 0; 266 267out_mapping: 268 dma_unmap_page(&dev->pdev->dev, addr, PAGE_SIZE, DMA_BIDIRECTIONAL); 269 270out_alloc: 271 __free_page(page); 272 273 return err; 274} 275static int give_pages(struct mlx5_core_dev *dev, u16 func_id, int npages, 276 int notify_fail) 277{ 278 struct mlx5_manage_pages_inbox *in; 279 struct mlx5_manage_pages_outbox out; 280 struct mlx5_manage_pages_inbox *nin; 281 int inlen; 282 u64 addr; 283 int err; 284 int i; 285 286 inlen = sizeof(*in) + npages * sizeof(in->pas[0]); 287 in = mlx5_vzalloc(inlen); 288 if (!in) { 289 mlx5_core_warn(dev, "vzalloc failed %d\n", inlen); 290 return -ENOMEM; 291 } 292 memset(&out, 0, sizeof(out)); 293 294 for (i = 0; i < npages; i++) { 295retry: 296 err = alloc_4k(dev, &addr); 297 if (err) { 298 if (err == -ENOMEM) 299 err = alloc_system_page(dev, func_id); 300 if (err) 301 goto out_4k; 302 303 goto retry; 304 } 305 in->pas[i] = cpu_to_be64(addr); 306 } 307 308 in->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_MANAGE_PAGES); 309 in->hdr.opmod = cpu_to_be16(MLX5_PAGES_GIVE); 310 in->func_id = cpu_to_be16(func_id); 311 in->num_entries = cpu_to_be32(npages); 312 err = mlx5_cmd_exec(dev, in, inlen, &out, sizeof(out)); 313 if (err) { 314 mlx5_core_warn(dev, "func_id 0x%x, npages %d, err %d\n", 315 func_id, npages, err); 316 goto out_alloc; 317 } 318 dev->priv.fw_pages += npages; 319 320 if (out.hdr.status) { 321 err = mlx5_cmd_status_to_err(&out.hdr); 322 if (err) { 323 mlx5_core_warn(dev, "func_id 0x%x, npages %d, status %d\n", 324 func_id, npages, out.hdr.status); 325 goto out_alloc; 326 } 327 } 328 329 mlx5_core_dbg(dev, "err %d\n", err); 330 331 goto out_free; 332 333out_alloc: 334 if (notify_fail) { 335 nin = kzalloc(sizeof(*nin), GFP_KERNEL); 336 if (!nin) { 337 mlx5_core_warn(dev, "allocation failed\n"); 338 goto out_4k; 339 } 340 memset(&out, 0, sizeof(out)); 341 nin->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_MANAGE_PAGES); 342 nin->hdr.opmod = cpu_to_be16(MLX5_PAGES_CANT_GIVE); 343 if (mlx5_cmd_exec(dev, nin, sizeof(*nin), &out, sizeof(out))) 344 mlx5_core_warn(dev, "page notify failed\n"); 345 kfree(nin); 346 } 347 348out_4k: 349 for (i--; i >= 0; i--) 350 free_4k(dev, be64_to_cpu(in->pas[i])); 351out_free: 352 mlx5_vfree(in); 353 return err; 354} 355 356static int reclaim_pages(struct mlx5_core_dev *dev, u32 func_id, int npages, 357 int *nclaimed) 358{ 359 struct mlx5_manage_pages_inbox in; 360 struct mlx5_manage_pages_outbox *out; 361 int num_claimed; 362 int outlen; 363 u64 addr; 364 int err; 365 int i; 366 367 if (nclaimed) 368 *nclaimed = 0; 369 370 memset(&in, 0, sizeof(in)); 371 outlen = sizeof(*out) + npages * sizeof(out->pas[0]); 372 out = mlx5_vzalloc(outlen); 373 if (!out) 374 return -ENOMEM; 375 376 in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_MANAGE_PAGES); 377 in.hdr.opmod = cpu_to_be16(MLX5_PAGES_TAKE); 378 in.func_id = cpu_to_be16(func_id); 379 in.num_entries = cpu_to_be32(npages); 380 mlx5_core_dbg(dev, "npages %d, outlen %d\n", npages, outlen); 381 err = mlx5_cmd_exec(dev, &in, sizeof(in), out, outlen); 382 if (err) { 383 mlx5_core_err(dev, "failed reclaiming pages\n"); 384 goto out_free; 385 } 386 dev->priv.fw_pages -= npages; 387 388 if (out->hdr.status) { 389 err = mlx5_cmd_status_to_err(&out->hdr); 390 goto out_free; 391 } 392 393 num_claimed = be32_to_cpu(out->num_entries); 394 if (nclaimed) 395 *nclaimed = num_claimed; 396 397 for (i = 0; i < num_claimed; i++) { 398 addr = be64_to_cpu(out->pas[i]); 399 free_4k(dev, addr); 400 } 401 402out_free: 403 mlx5_vfree(out); 404 return err; 405} 406 407static void pages_work_handler(struct work_struct *work) 408{ 409 struct mlx5_pages_req *req = container_of(work, struct mlx5_pages_req, work); 410 struct mlx5_core_dev *dev = req->dev; 411 int err = 0; 412 413 if (req->npages < 0) 414 err = reclaim_pages(dev, req->func_id, -1 * req->npages, NULL); 415 else if (req->npages > 0) 416 err = give_pages(dev, req->func_id, req->npages, 1); 417 418 if (err) 419 mlx5_core_warn(dev, "%s fail %d\n", 420 req->npages < 0 ? "reclaim" : "give", err); 421 422 kfree(req); 423} 424 425void mlx5_core_req_pages_handler(struct mlx5_core_dev *dev, u16 func_id, 426 s32 npages) 427{ 428 struct mlx5_pages_req *req; 429 430 req = kzalloc(sizeof(*req), GFP_ATOMIC); 431 if (!req) { 432 mlx5_core_warn(dev, "failed to allocate pages request\n"); 433 return; 434 } 435 436 req->dev = dev; 437 req->func_id = func_id; 438 req->npages = npages; 439 INIT_WORK(&req->work, pages_work_handler); 440 queue_work(dev->priv.pg_wq, &req->work); 441} 442 443int mlx5_satisfy_startup_pages(struct mlx5_core_dev *dev, int boot) 444{ 445 u16 uninitialized_var(func_id); 446 s32 uninitialized_var(npages); 447 int err; 448 449 err = mlx5_cmd_query_pages(dev, &func_id, &npages, boot); 450 if (err) 451 return err; 452 453 mlx5_core_dbg(dev, "requested %d %s pages for func_id 0x%x\n", 454 npages, boot ? "boot" : "init", func_id); 455 456 return give_pages(dev, func_id, npages, 0); 457} 458 459enum { 460 MLX5_BLKS_FOR_RECLAIM_PAGES = 12 461}; 462 463static int optimal_reclaimed_pages(void) 464{ 465 struct mlx5_cmd_prot_block *block; 466 struct mlx5_cmd_layout *lay; 467 int ret; 468 469 ret = (sizeof(lay->out) + MLX5_BLKS_FOR_RECLAIM_PAGES * sizeof(block->data) - 470 sizeof(struct mlx5_manage_pages_outbox)) / 471 FIELD_SIZEOF(struct mlx5_manage_pages_outbox, pas[0]); 472 473 return ret; 474} 475 476int mlx5_reclaim_startup_pages(struct mlx5_core_dev *dev) 477{ 478 unsigned long end = jiffies + msecs_to_jiffies(MAX_RECLAIM_TIME_MSECS); 479 struct fw_page *fwp; 480 struct rb_node *p; 481 int nclaimed = 0; 482 int err; 483 484 do { 485 p = rb_first(&dev->priv.page_root); 486 if (p) { 487 fwp = rb_entry(p, struct fw_page, rb_node); 488 err = reclaim_pages(dev, fwp->func_id, 489 optimal_reclaimed_pages(), 490 &nclaimed); 491 if (err) { 492 mlx5_core_warn(dev, "failed reclaiming pages (%d)\n", 493 err); 494 return err; 495 } 496 if (nclaimed) 497 end = jiffies + msecs_to_jiffies(MAX_RECLAIM_TIME_MSECS); 498 } 499 if (time_after(jiffies, end)) { 500 mlx5_core_warn(dev, "FW did not return all pages. giving up...\n"); 501 break; 502 } 503 } while (p); 504 505 return 0; 506} 507 508void mlx5_pagealloc_init(struct mlx5_core_dev *dev) 509{ 510 dev->priv.page_root = RB_ROOT; 511 INIT_LIST_HEAD(&dev->priv.free_list); 512} 513 514void mlx5_pagealloc_cleanup(struct mlx5_core_dev *dev) 515{ 516 /* nothing */ 517} 518 519int mlx5_pagealloc_start(struct mlx5_core_dev *dev) 520{ 521 dev->priv.pg_wq = create_singlethread_workqueue("mlx5_page_allocator"); 522 if (!dev->priv.pg_wq) 523 return -ENOMEM; 524 525 return 0; 526} 527 528void mlx5_pagealloc_stop(struct mlx5_core_dev *dev) 529{ 530 destroy_workqueue(dev->priv.pg_wq); 531} 532