1/* 2 * Copyright 2012 Red Hat Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Ben Skeggs 23 */ 24 25#ifdef HAVE_CONFIG_H 26#include <config.h> 27#endif 28 29#include <stdio.h> 30#include <stdlib.h> 31#include <stdint.h> 32#include <stdbool.h> 33#include <string.h> 34#include <assert.h> 35#include <errno.h> 36 37#include <xf86drm.h> 38#include <xf86atomic.h> 39#include "libdrm_lists.h" 40#include "nouveau_drm.h" 41 42#include "nouveau.h" 43#include "private.h" 44 45struct nouveau_pushbuf_krec { 46 struct nouveau_pushbuf_krec *next; 47 struct drm_nouveau_gem_pushbuf_bo buffer[NOUVEAU_GEM_MAX_BUFFERS]; 48 struct drm_nouveau_gem_pushbuf_reloc reloc[NOUVEAU_GEM_MAX_RELOCS]; 49 struct drm_nouveau_gem_pushbuf_push push[NOUVEAU_GEM_MAX_PUSH]; 50 int nr_buffer; 51 int nr_reloc; 52 int nr_push; 53 uint64_t vram_used; 54 uint64_t gart_used; 55}; 56 57struct nouveau_pushbuf_priv { 58 struct nouveau_pushbuf base; 59 struct nouveau_pushbuf_krec *list; 60 struct nouveau_pushbuf_krec *krec; 61 struct nouveau_list bctx_list; 62 struct nouveau_bo *bo; 63 uint32_t type; 64 uint32_t suffix0; 65 uint32_t suffix1; 66 uint32_t *ptr; 67 uint32_t *bgn; 68 int bo_next; 69 int bo_nr; 70 struct nouveau_bo *bos[]; 71}; 72 73static inline struct nouveau_pushbuf_priv * 74nouveau_pushbuf(struct nouveau_pushbuf *push) 75{ 76 return (struct nouveau_pushbuf_priv *)push; 77} 78 79static int pushbuf_validate(struct nouveau_pushbuf *, bool); 80static int pushbuf_flush(struct nouveau_pushbuf *); 81 82static bool 83pushbuf_kref_fits(struct nouveau_pushbuf *push, struct nouveau_bo *bo, 84 uint32_t *domains) 85{ 86 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push); 87 struct nouveau_pushbuf_krec *krec = nvpb->krec; 88 struct nouveau_device *dev = push->client->device; 89 struct nouveau_bo *kbo; 90 struct drm_nouveau_gem_pushbuf_bo *kref; 91 int i; 92 93 /* VRAM is the only valid domain. GART and VRAM|GART buffers 94 * are all accounted to GART, so if this doesn't fit in VRAM 95 * straight up, a flush is needed. 96 */ 97 if (*domains == NOUVEAU_GEM_DOMAIN_VRAM) { 98 if (krec->vram_used + bo->size > dev->vram_limit) 99 return false; 100 krec->vram_used += bo->size; 101 return true; 102 } 103 104 /* GART or VRAM|GART buffer. Account both of these buffer types 105 * to GART only for the moment, which simplifies things. If the 106 * buffer can fit already, we're done here. 107 */ 108 if (krec->gart_used + bo->size <= dev->gart_limit) { 109 krec->gart_used += bo->size; 110 return true; 111 } 112 113 /* Ran out of GART space, if it's a VRAM|GART buffer and it'll 114 * fit into available VRAM, turn it into a VRAM buffer 115 */ 116 if ((*domains & NOUVEAU_GEM_DOMAIN_VRAM) && 117 krec->vram_used + bo->size <= dev->vram_limit) { 118 *domains &= NOUVEAU_GEM_DOMAIN_VRAM; 119 krec->vram_used += bo->size; 120 return true; 121 } 122 123 /* Still couldn't fit the buffer in anywhere, so as a last resort; 124 * scan the buffer list for VRAM|GART buffers and turn them into 125 * VRAM buffers until we have enough space in GART for this one 126 */ 127 kref = krec->buffer; 128 for (i = 0; i < krec->nr_buffer; i++, kref++) { 129 if (!(kref->valid_domains & NOUVEAU_GEM_DOMAIN_GART)) 130 continue; 131 132 kbo = (void *)(unsigned long)kref->user_priv; 133 if (!(kref->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) || 134 krec->vram_used + kbo->size > dev->vram_limit) 135 continue; 136 137 kref->valid_domains &= NOUVEAU_GEM_DOMAIN_VRAM; 138 krec->gart_used -= kbo->size; 139 krec->vram_used += kbo->size; 140 if (krec->gart_used + bo->size <= dev->gart_limit) { 141 krec->gart_used += bo->size; 142 return true; 143 } 144 } 145 146 /* Couldn't resolve a placement, need to force a flush */ 147 return false; 148} 149 150static struct drm_nouveau_gem_pushbuf_bo * 151pushbuf_kref(struct nouveau_pushbuf *push, struct nouveau_bo *bo, 152 uint32_t flags) 153{ 154 struct nouveau_device *dev = push->client->device; 155 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push); 156 struct nouveau_pushbuf_krec *krec = nvpb->krec; 157 struct nouveau_pushbuf *fpush; 158 struct drm_nouveau_gem_pushbuf_bo *kref; 159 uint32_t domains, domains_wr, domains_rd; 160 161 domains = 0; 162 if (flags & NOUVEAU_BO_VRAM) 163 domains |= NOUVEAU_GEM_DOMAIN_VRAM; 164 if (flags & NOUVEAU_BO_GART) 165 domains |= NOUVEAU_GEM_DOMAIN_GART; 166 domains_wr = domains * !!(flags & NOUVEAU_BO_WR); 167 domains_rd = domains * !!(flags & NOUVEAU_BO_RD); 168 169 /* if buffer is referenced on another pushbuf that is owned by the 170 * same client, we need to flush the other pushbuf first to ensure 171 * the correct ordering of commands 172 */ 173 fpush = cli_push_get(push->client, bo); 174 if (fpush && fpush != push) 175 pushbuf_flush(fpush); 176 177 kref = cli_kref_get(push->client, bo); 178 if (kref) { 179 /* possible conflict in memory types - flush and retry */ 180 if (!(kref->valid_domains & domains)) 181 return NULL; 182 183 /* VRAM|GART buffer turning into a VRAM buffer. Make sure 184 * it'll fit in VRAM and force a flush if not. 185 */ 186 if ((kref->valid_domains & NOUVEAU_GEM_DOMAIN_GART) && 187 ( domains == NOUVEAU_GEM_DOMAIN_VRAM)) { 188 if (krec->vram_used + bo->size > dev->vram_limit) 189 return NULL; 190 krec->vram_used += bo->size; 191 krec->gart_used -= bo->size; 192 } 193 194 kref->valid_domains &= domains; 195 kref->write_domains |= domains_wr; 196 kref->read_domains |= domains_rd; 197 } else { 198 if (krec->nr_buffer == NOUVEAU_GEM_MAX_BUFFERS || 199 !pushbuf_kref_fits(push, bo, &domains)) 200 return NULL; 201 202 kref = &krec->buffer[krec->nr_buffer++]; 203 kref->user_priv = (unsigned long)bo; 204 kref->handle = bo->handle; 205 kref->valid_domains = domains; 206 kref->write_domains = domains_wr; 207 kref->read_domains = domains_rd; 208 kref->presumed.valid = 1; 209 kref->presumed.offset = bo->offset; 210 if (bo->flags & NOUVEAU_BO_VRAM) 211 kref->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM; 212 else 213 kref->presumed.domain = NOUVEAU_GEM_DOMAIN_GART; 214 215 cli_kref_set(push->client, bo, kref, push); 216 atomic_inc(&nouveau_bo(bo)->refcnt); 217 } 218 219 return kref; 220} 221 222static uint32_t 223pushbuf_krel(struct nouveau_pushbuf *push, struct nouveau_bo *bo, 224 uint32_t data, uint32_t flags, uint32_t vor, uint32_t tor) 225{ 226 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push); 227 struct nouveau_pushbuf_krec *krec = nvpb->krec; 228 struct drm_nouveau_gem_pushbuf_reloc *krel; 229 struct drm_nouveau_gem_pushbuf_bo *pkref; 230 struct drm_nouveau_gem_pushbuf_bo *bkref; 231 uint32_t reloc = data; 232 233 pkref = cli_kref_get(push->client, nvpb->bo); 234 bkref = cli_kref_get(push->client, bo); 235 krel = &krec->reloc[krec->nr_reloc++]; 236 237 assert(pkref); 238 assert(bkref); 239 krel->reloc_bo_index = pkref - krec->buffer; 240 krel->reloc_bo_offset = (push->cur - nvpb->ptr) * 4; 241 krel->bo_index = bkref - krec->buffer; 242 krel->flags = 0; 243 krel->data = data; 244 krel->vor = vor; 245 krel->tor = tor; 246 247 if (flags & NOUVEAU_BO_LOW) { 248 reloc = (bkref->presumed.offset + data); 249 krel->flags |= NOUVEAU_GEM_RELOC_LOW; 250 } else 251 if (flags & NOUVEAU_BO_HIGH) { 252 reloc = (bkref->presumed.offset + data) >> 32; 253 krel->flags |= NOUVEAU_GEM_RELOC_HIGH; 254 } 255 if (flags & NOUVEAU_BO_OR) { 256 if (bkref->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) 257 reloc |= vor; 258 else 259 reloc |= tor; 260 krel->flags |= NOUVEAU_GEM_RELOC_OR; 261 } 262 263 return reloc; 264} 265 266static void 267pushbuf_dump(struct nouveau_pushbuf_krec *krec, int krec_id, int chid) 268{ 269 struct drm_nouveau_gem_pushbuf_reloc *krel; 270 struct drm_nouveau_gem_pushbuf_push *kpsh; 271 struct drm_nouveau_gem_pushbuf_bo *kref; 272 struct nouveau_bo *bo; 273 uint32_t *bgn, *end; 274 int i; 275 276 err("ch%d: krec %d pushes %d bufs %d relocs %d\n", chid, 277 krec_id, krec->nr_push, krec->nr_buffer, krec->nr_reloc); 278 279 kref = krec->buffer; 280 for (i = 0; i < krec->nr_buffer; i++, kref++) { 281 err("ch%d: buf %08x %08x %08x %08x %08x\n", chid, i, 282 kref->handle, kref->valid_domains, 283 kref->read_domains, kref->write_domains); 284 } 285 286 krel = krec->reloc; 287 for (i = 0; i < krec->nr_reloc; i++, krel++) { 288 err("ch%d: rel %08x %08x %08x %08x %08x %08x %08x\n", 289 chid, krel->reloc_bo_index, krel->reloc_bo_offset, 290 krel->bo_index, krel->flags, krel->data, 291 krel->vor, krel->tor); 292 } 293 294 kpsh = krec->push; 295 for (i = 0; i < krec->nr_push; i++, kpsh++) { 296 kref = krec->buffer + kpsh->bo_index; 297 bo = (void *)(unsigned long)kref->user_priv; 298 bgn = (uint32_t *)((char *)bo->map + kpsh->offset); 299 end = bgn + (kpsh->length /4); 300 301 err("ch%d: psh %08x %010llx %010llx\n", chid, kpsh->bo_index, 302 (unsigned long long)kpsh->offset, 303 (unsigned long long)(kpsh->offset + kpsh->length)); 304 while (bgn < end) 305 err("\t0x%08x\n", *bgn++); 306 } 307} 308 309static int 310pushbuf_submit(struct nouveau_pushbuf *push, struct nouveau_object *chan) 311{ 312 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push); 313 struct nouveau_pushbuf_krec *krec = nvpb->list; 314 struct nouveau_device *dev = push->client->device; 315 struct nouveau_drm *drm = nouveau_drm(&dev->object); 316 struct drm_nouveau_gem_pushbuf_bo_presumed *info; 317 struct drm_nouveau_gem_pushbuf_bo *kref; 318 struct drm_nouveau_gem_pushbuf req; 319 struct nouveau_fifo *fifo = chan->data; 320 struct nouveau_bo *bo; 321 int krec_id = 0; 322 int ret = 0, i; 323 324 if (chan->oclass != NOUVEAU_FIFO_CHANNEL_CLASS) 325 return -EINVAL; 326 327 if (push->kick_notify) 328 push->kick_notify(push); 329 330 nouveau_pushbuf_data(push, NULL, 0, 0); 331 332 while (krec && krec->nr_push) { 333 req.channel = fifo->channel; 334 req.nr_buffers = krec->nr_buffer; 335 req.buffers = (uint64_t)(unsigned long)krec->buffer; 336 req.nr_relocs = krec->nr_reloc; 337 req.nr_push = krec->nr_push; 338 req.relocs = (uint64_t)(unsigned long)krec->reloc; 339 req.push = (uint64_t)(unsigned long)krec->push; 340 req.suffix0 = nvpb->suffix0; 341 req.suffix1 = nvpb->suffix1; 342 req.vram_available = 0; /* for valgrind */ 343 req.gart_available = 0; 344 345 if (dbg_on(0)) 346 pushbuf_dump(krec, krec_id++, fifo->channel); 347 348#ifndef SIMULATE 349 ret = drmCommandWriteRead(drm->fd, DRM_NOUVEAU_GEM_PUSHBUF, 350 &req, sizeof(req)); 351 nvpb->suffix0 = req.suffix0; 352 nvpb->suffix1 = req.suffix1; 353 dev->vram_limit = (req.vram_available * 354 nouveau_device(dev)->vram_limit_percent) / 100; 355 dev->gart_limit = (req.gart_available * 356 nouveau_device(dev)->gart_limit_percent) / 100; 357#else 358 if (dbg_on(31)) 359 ret = -EINVAL; 360#endif 361 362 if (ret) { 363 err("kernel rejected pushbuf: %s\n", strerror(-ret)); 364 pushbuf_dump(krec, krec_id++, fifo->channel); 365 break; 366 } 367 368 kref = krec->buffer; 369 for (i = 0; i < krec->nr_buffer; i++, kref++) { 370 bo = (void *)(unsigned long)kref->user_priv; 371 372 info = &kref->presumed; 373 if (!info->valid) { 374 bo->flags &= ~NOUVEAU_BO_APER; 375 if (info->domain == NOUVEAU_GEM_DOMAIN_VRAM) 376 bo->flags |= NOUVEAU_BO_VRAM; 377 else 378 bo->flags |= NOUVEAU_BO_GART; 379 bo->offset = info->offset; 380 } 381 382 if (kref->write_domains) 383 nouveau_bo(bo)->access |= NOUVEAU_BO_WR; 384 if (kref->read_domains) 385 nouveau_bo(bo)->access |= NOUVEAU_BO_RD; 386 } 387 388 krec = krec->next; 389 } 390 391 return ret; 392} 393 394static int 395pushbuf_flush(struct nouveau_pushbuf *push) 396{ 397 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push); 398 struct nouveau_pushbuf_krec *krec = nvpb->krec; 399 struct drm_nouveau_gem_pushbuf_bo *kref; 400 struct nouveau_bufctx *bctx, *btmp; 401 struct nouveau_bo *bo; 402 int ret = 0, i; 403 404 if (push->channel) { 405 ret = pushbuf_submit(push, push->channel); 406 } else { 407 nouveau_pushbuf_data(push, NULL, 0, 0); 408 krec->next = malloc(sizeof(*krec)); 409 nvpb->krec = krec->next; 410 } 411 412 kref = krec->buffer; 413 for (i = 0; i < krec->nr_buffer; i++, kref++) { 414 bo = (void *)(unsigned long)kref->user_priv; 415 cli_kref_set(push->client, bo, NULL, NULL); 416 if (push->channel) 417 nouveau_bo_ref(NULL, &bo); 418 } 419 420 krec = nvpb->krec; 421 krec->vram_used = 0; 422 krec->gart_used = 0; 423 krec->nr_buffer = 0; 424 krec->nr_reloc = 0; 425 krec->nr_push = 0; 426 427 DRMLISTFOREACHENTRYSAFE(bctx, btmp, &nvpb->bctx_list, head) { 428 DRMLISTJOIN(&bctx->current, &bctx->pending); 429 DRMINITLISTHEAD(&bctx->current); 430 DRMLISTDELINIT(&bctx->head); 431 } 432 433 return ret; 434} 435 436static void 437pushbuf_refn_fail(struct nouveau_pushbuf *push, int sref, int srel) 438{ 439 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push); 440 struct nouveau_pushbuf_krec *krec = nvpb->krec; 441 struct drm_nouveau_gem_pushbuf_bo *kref; 442 443 kref = krec->buffer + sref; 444 while (krec->nr_buffer-- > sref) { 445 struct nouveau_bo *bo = (void *)(unsigned long)kref->user_priv; 446 cli_kref_set(push->client, bo, NULL, NULL); 447 nouveau_bo_ref(NULL, &bo); 448 kref++; 449 } 450 krec->nr_buffer = sref; 451 krec->nr_reloc = srel; 452} 453 454static int 455pushbuf_refn(struct nouveau_pushbuf *push, bool retry, 456 struct nouveau_pushbuf_refn *refs, int nr) 457{ 458 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push); 459 struct nouveau_pushbuf_krec *krec = nvpb->krec; 460 struct drm_nouveau_gem_pushbuf_bo *kref; 461 int sref = krec->nr_buffer; 462 int ret = 0, i; 463 464 for (i = 0; i < nr; i++) { 465 kref = pushbuf_kref(push, refs[i].bo, refs[i].flags); 466 if (!kref) { 467 ret = -ENOSPC; 468 break; 469 } 470 } 471 472 if (ret) { 473 pushbuf_refn_fail(push, sref, krec->nr_reloc); 474 if (retry) { 475 pushbuf_flush(push); 476 nouveau_pushbuf_space(push, 0, 0, 0); 477 return pushbuf_refn(push, false, refs, nr); 478 } 479 } 480 481 return ret; 482} 483 484static int 485pushbuf_validate(struct nouveau_pushbuf *push, bool retry) 486{ 487 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push); 488 struct nouveau_pushbuf_krec *krec = nvpb->krec; 489 struct drm_nouveau_gem_pushbuf_bo *kref; 490 struct nouveau_bufctx *bctx = push->bufctx; 491 struct nouveau_bufref *bref; 492 int relocs = bctx ? bctx->relocs * 2: 0; 493 int sref, srel, ret; 494 495 ret = nouveau_pushbuf_space(push, relocs, relocs, 0); 496 if (ret || bctx == NULL) 497 return ret; 498 499 sref = krec->nr_buffer; 500 srel = krec->nr_reloc; 501 502 DRMLISTDEL(&bctx->head); 503 DRMLISTADD(&bctx->head, &nvpb->bctx_list); 504 505 DRMLISTFOREACHENTRY(bref, &bctx->pending, thead) { 506 kref = pushbuf_kref(push, bref->bo, bref->flags); 507 if (!kref) { 508 ret = -ENOSPC; 509 break; 510 } 511 512 if (bref->packet) { 513 pushbuf_krel(push, bref->bo, bref->packet, 0, 0, 0); 514 *push->cur++ = 0; 515 pushbuf_krel(push, bref->bo, bref->data, bref->flags, 516 bref->vor, bref->tor); 517 *push->cur++ = 0; 518 } 519 } 520 521 DRMLISTJOIN(&bctx->pending, &bctx->current); 522 DRMINITLISTHEAD(&bctx->pending); 523 524 if (ret) { 525 pushbuf_refn_fail(push, sref, srel); 526 if (retry) { 527 pushbuf_flush(push); 528 return pushbuf_validate(push, false); 529 } 530 } 531 532 return ret; 533} 534 535int 536nouveau_pushbuf_new(struct nouveau_client *client, struct nouveau_object *chan, 537 int nr, uint32_t size, bool immediate, 538 struct nouveau_pushbuf **ppush) 539{ 540 struct nouveau_drm *drm = nouveau_drm(&client->device->object); 541 struct nouveau_fifo *fifo = chan->data; 542 struct nouveau_pushbuf_priv *nvpb; 543 struct nouveau_pushbuf *push; 544 struct drm_nouveau_gem_pushbuf req = {}; 545 int ret; 546 547 if (chan->oclass != NOUVEAU_FIFO_CHANNEL_CLASS) 548 return -EINVAL; 549 550 /* nop pushbuf call, to get the current "return to main" sequence 551 * we need to append to the pushbuf on early chipsets 552 */ 553 req.channel = fifo->channel; 554 req.nr_push = 0; 555 ret = drmCommandWriteRead(drm->fd, DRM_NOUVEAU_GEM_PUSHBUF, 556 &req, sizeof(req)); 557 if (ret) 558 return ret; 559 560 nvpb = calloc(1, sizeof(*nvpb) + nr * sizeof(*nvpb->bos)); 561 if (!nvpb) 562 return -ENOMEM; 563 564#ifndef SIMULATE 565 nvpb->suffix0 = req.suffix0; 566 nvpb->suffix1 = req.suffix1; 567#else 568 nvpb->suffix0 = 0xffffffff; 569 nvpb->suffix1 = 0xffffffff; 570#endif 571 nvpb->krec = calloc(1, sizeof(*nvpb->krec)); 572 nvpb->list = nvpb->krec; 573 if (!nvpb->krec) { 574 free(nvpb); 575 return -ENOMEM; 576 } 577 578 push = &nvpb->base; 579 push->client = client; 580 push->channel = immediate ? chan : NULL; 581 push->flags = NOUVEAU_BO_RD; 582 if (fifo->pushbuf & NOUVEAU_GEM_DOMAIN_GART) { 583 push->flags |= NOUVEAU_BO_GART; 584 nvpb->type = NOUVEAU_BO_GART; 585 } else 586 if (fifo->pushbuf & NOUVEAU_GEM_DOMAIN_VRAM) { 587 push->flags |= NOUVEAU_BO_VRAM; 588 nvpb->type = NOUVEAU_BO_VRAM; 589 } 590 nvpb->type |= NOUVEAU_BO_MAP; 591 592 for (nvpb->bo_nr = 0; nvpb->bo_nr < nr; nvpb->bo_nr++) { 593 ret = nouveau_bo_new(client->device, nvpb->type, 0, size, 594 NULL, &nvpb->bos[nvpb->bo_nr]); 595 if (ret) { 596 nouveau_pushbuf_del(&push); 597 return ret; 598 } 599 } 600 601 DRMINITLISTHEAD(&nvpb->bctx_list); 602 *ppush = push; 603 return 0; 604} 605 606void 607nouveau_pushbuf_del(struct nouveau_pushbuf **ppush) 608{ 609 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(*ppush); 610 if (nvpb) { 611 struct drm_nouveau_gem_pushbuf_bo *kref; 612 struct nouveau_pushbuf_krec *krec; 613 while ((krec = nvpb->list)) { 614 kref = krec->buffer; 615 while (krec->nr_buffer--) { 616 unsigned long priv = kref++->user_priv; 617 struct nouveau_bo *bo = (void *)priv; 618 cli_kref_set(nvpb->base.client, bo, NULL, NULL); 619 nouveau_bo_ref(NULL, &bo); 620 } 621 nvpb->list = krec->next; 622 free(krec); 623 } 624 while (nvpb->bo_nr--) 625 nouveau_bo_ref(NULL, &nvpb->bos[nvpb->bo_nr]); 626 nouveau_bo_ref(NULL, &nvpb->bo); 627 free(nvpb); 628 } 629 *ppush = NULL; 630} 631 632struct nouveau_bufctx * 633nouveau_pushbuf_bufctx(struct nouveau_pushbuf *push, struct nouveau_bufctx *ctx) 634{ 635 struct nouveau_bufctx *prev = push->bufctx; 636 push->bufctx = ctx; 637 return prev; 638} 639 640int 641nouveau_pushbuf_space(struct nouveau_pushbuf *push, 642 uint32_t dwords, uint32_t relocs, uint32_t pushes) 643{ 644 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push); 645 struct nouveau_pushbuf_krec *krec = nvpb->krec; 646 struct nouveau_client *client = push->client; 647 struct nouveau_bo *bo = NULL; 648 bool flushed = false; 649 int ret = 0; 650 651 /* switch to next buffer if insufficient space in the current one */ 652 if (push->cur + dwords >= push->end) { 653 if (nvpb->bo_next < nvpb->bo_nr) { 654 nouveau_bo_ref(nvpb->bos[nvpb->bo_next++], &bo); 655 if (nvpb->bo_next == nvpb->bo_nr && push->channel) 656 nvpb->bo_next = 0; 657 } else { 658 ret = nouveau_bo_new(client->device, nvpb->type, 0, 659 nvpb->bos[0]->size, NULL, &bo); 660 if (ret) 661 return ret; 662 } 663 } 664 665 /* make sure there's always enough space to queue up the pending 666 * data in the pushbuf proper 667 */ 668 pushes++; 669 670 /* need to flush if we've run out of space on an immediate pushbuf, 671 * if the new buffer won't fit, or if the kernel push/reloc limits 672 * have been hit 673 */ 674 if ((bo && ( push->channel || 675 !pushbuf_kref(push, bo, push->flags))) || 676 krec->nr_reloc + relocs >= NOUVEAU_GEM_MAX_RELOCS || 677 krec->nr_push + pushes >= NOUVEAU_GEM_MAX_PUSH) { 678 if (nvpb->bo && krec->nr_buffer) 679 pushbuf_flush(push); 680 flushed = true; 681 } 682 683 /* if necessary, switch to new buffer */ 684 if (bo) { 685 ret = nouveau_bo_map(bo, NOUVEAU_BO_WR, push->client); 686 if (ret) 687 return ret; 688 689 nouveau_pushbuf_data(push, NULL, 0, 0); 690 nouveau_bo_ref(bo, &nvpb->bo); 691 nouveau_bo_ref(NULL, &bo); 692 693 nvpb->bgn = nvpb->bo->map; 694 nvpb->ptr = nvpb->bgn; 695 push->cur = nvpb->bgn; 696 push->end = push->cur + (nvpb->bo->size / 4); 697 push->end -= 2 + push->rsvd_kick; /* space for suffix */ 698 } 699 700 pushbuf_kref(push, nvpb->bo, push->flags); 701 return flushed ? pushbuf_validate(push, false) : 0; 702} 703 704void 705nouveau_pushbuf_data(struct nouveau_pushbuf *push, struct nouveau_bo *bo, 706 uint64_t offset, uint64_t length) 707{ 708 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push); 709 struct nouveau_pushbuf_krec *krec = nvpb->krec; 710 struct drm_nouveau_gem_pushbuf_push *kpsh; 711 struct drm_nouveau_gem_pushbuf_bo *kref; 712 713 if (bo != nvpb->bo && nvpb->bgn != push->cur) { 714 if (nvpb->suffix0 || nvpb->suffix1) { 715 *push->cur++ = nvpb->suffix0; 716 *push->cur++ = nvpb->suffix1; 717 } 718 719 nouveau_pushbuf_data(push, nvpb->bo, 720 (nvpb->bgn - nvpb->ptr) * 4, 721 (push->cur - nvpb->bgn) * 4); 722 nvpb->bgn = push->cur; 723 } 724 725 if (bo) { 726 kref = cli_kref_get(push->client, bo); 727 assert(kref); 728 kpsh = &krec->push[krec->nr_push++]; 729 kpsh->bo_index = kref - krec->buffer; 730 kpsh->offset = offset; 731 kpsh->length = length; 732 } 733} 734 735int 736nouveau_pushbuf_refn(struct nouveau_pushbuf *push, 737 struct nouveau_pushbuf_refn *refs, int nr) 738{ 739 return pushbuf_refn(push, true, refs, nr); 740} 741 742void 743nouveau_pushbuf_reloc(struct nouveau_pushbuf *push, struct nouveau_bo *bo, 744 uint32_t data, uint32_t flags, uint32_t vor, uint32_t tor) 745{ 746 *push->cur = pushbuf_krel(push, bo, data, flags, vor, tor); 747 push->cur++; 748} 749 750int 751nouveau_pushbuf_validate(struct nouveau_pushbuf *push) 752{ 753 return pushbuf_validate(push, true); 754} 755 756uint32_t 757nouveau_pushbuf_refd(struct nouveau_pushbuf *push, struct nouveau_bo *bo) 758{ 759 struct drm_nouveau_gem_pushbuf_bo *kref; 760 uint32_t flags = 0; 761 762 if (cli_push_get(push->client, bo) == push) { 763 kref = cli_kref_get(push->client, bo); 764 assert(kref); 765 if (kref->read_domains) 766 flags |= NOUVEAU_BO_RD; 767 if (kref->write_domains) 768 flags |= NOUVEAU_BO_WR; 769 } 770 771 return flags; 772} 773 774int 775nouveau_pushbuf_kick(struct nouveau_pushbuf *push, struct nouveau_object *chan) 776{ 777 if (!push->channel) 778 return pushbuf_submit(push, chan); 779 pushbuf_flush(push); 780 return pushbuf_validate(push, false); 781} 782