job.c revision 5060d8ec7cfc29dd399b4fe952ba96e7a88aa778
1/* 2 * Tegra host1x Job 3 * 4 * Copyright (c) 2010-2013, NVIDIA Corporation. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms and conditions of the GNU General Public License, 8 * version 2, as published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 * more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program. If not, see <http://www.gnu.org/licenses/>. 17 */ 18 19#include <linux/dma-mapping.h> 20#include <linux/err.h> 21#include <linux/kref.h> 22#include <linux/module.h> 23#include <linux/scatterlist.h> 24#include <linux/slab.h> 25#include <linux/vmalloc.h> 26#include <trace/events/host1x.h> 27 28#include "channel.h" 29#include "dev.h" 30#include "host1x_bo.h" 31#include "job.h" 32#include "syncpt.h" 33 34struct host1x_job *host1x_job_alloc(struct host1x_channel *ch, 35 u32 num_cmdbufs, u32 num_relocs, 36 u32 num_waitchks) 37{ 38 struct host1x_job *job = NULL; 39 unsigned int num_unpins = num_cmdbufs + num_relocs; 40 u64 total; 41 void *mem; 42 43 /* Check that we're not going to overflow */ 44 total = sizeof(struct host1x_job) + 45 num_relocs * sizeof(struct host1x_reloc) + 46 num_unpins * sizeof(struct host1x_job_unpin_data) + 47 num_waitchks * sizeof(struct host1x_waitchk) + 48 num_cmdbufs * sizeof(struct host1x_job_gather) + 49 num_unpins * sizeof(dma_addr_t) + 50 num_unpins * sizeof(u32 *); 51 if (total > ULONG_MAX) 52 return NULL; 53 54 mem = job = kzalloc(total, GFP_KERNEL); 55 if (!job) 56 return NULL; 57 58 kref_init(&job->ref); 59 job->channel = ch; 60 61 /* Redistribute memory to the structs */ 62 mem += sizeof(struct host1x_job); 63 job->relocarray = num_relocs ? mem : NULL; 64 mem += num_relocs * sizeof(struct host1x_reloc); 65 job->unpins = num_unpins ? mem : NULL; 66 mem += num_unpins * sizeof(struct host1x_job_unpin_data); 67 job->waitchk = num_waitchks ? mem : NULL; 68 mem += num_waitchks * sizeof(struct host1x_waitchk); 69 job->gathers = num_cmdbufs ? mem : NULL; 70 mem += num_cmdbufs * sizeof(struct host1x_job_gather); 71 job->addr_phys = num_unpins ? mem : NULL; 72 73 job->reloc_addr_phys = job->addr_phys; 74 job->gather_addr_phys = &job->addr_phys[num_relocs]; 75 76 return job; 77} 78 79struct host1x_job *host1x_job_get(struct host1x_job *job) 80{ 81 kref_get(&job->ref); 82 return job; 83} 84 85static void job_free(struct kref *ref) 86{ 87 struct host1x_job *job = container_of(ref, struct host1x_job, ref); 88 89 kfree(job); 90} 91 92void host1x_job_put(struct host1x_job *job) 93{ 94 kref_put(&job->ref, job_free); 95} 96 97void host1x_job_add_gather(struct host1x_job *job, struct host1x_bo *bo, 98 u32 words, u32 offset) 99{ 100 struct host1x_job_gather *cur_gather = &job->gathers[job->num_gathers]; 101 102 cur_gather->words = words; 103 cur_gather->bo = bo; 104 cur_gather->offset = offset; 105 job->num_gathers++; 106} 107 108/* 109 * NULL an already satisfied WAIT_SYNCPT host method, by patching its 110 * args in the command stream. The method data is changed to reference 111 * a reserved (never given out or incr) HOST1X_SYNCPT_RESERVED syncpt 112 * with a matching threshold value of 0, so is guaranteed to be popped 113 * by the host HW. 114 */ 115static void host1x_syncpt_patch_offset(struct host1x_syncpt *sp, 116 struct host1x_bo *h, u32 offset) 117{ 118 void *patch_addr = NULL; 119 120 /* patch the wait */ 121 patch_addr = host1x_bo_kmap(h, offset >> PAGE_SHIFT); 122 if (patch_addr) { 123 host1x_syncpt_patch_wait(sp, 124 patch_addr + (offset & ~PAGE_MASK)); 125 host1x_bo_kunmap(h, offset >> PAGE_SHIFT, patch_addr); 126 } else 127 pr_err("Could not map cmdbuf for wait check\n"); 128} 129 130/* 131 * Check driver supplied waitchk structs for syncpt thresholds 132 * that have already been satisfied and NULL the comparison (to 133 * avoid a wrap condition in the HW). 134 */ 135static int do_waitchks(struct host1x_job *job, struct host1x *host, 136 struct host1x_bo *patch) 137{ 138 int i; 139 140 /* compare syncpt vs wait threshold */ 141 for (i = 0; i < job->num_waitchk; i++) { 142 struct host1x_waitchk *wait = &job->waitchk[i]; 143 struct host1x_syncpt *sp = 144 host1x_syncpt_get(host, wait->syncpt_id); 145 146 /* validate syncpt id */ 147 if (wait->syncpt_id > host1x_syncpt_nb_pts(host)) 148 continue; 149 150 /* skip all other gathers */ 151 if (patch != wait->bo) 152 continue; 153 154 trace_host1x_syncpt_wait_check(wait->bo, wait->offset, 155 wait->syncpt_id, wait->thresh, 156 host1x_syncpt_read_min(sp)); 157 158 if (host1x_syncpt_is_expired(sp, wait->thresh)) { 159 dev_dbg(host->dev, 160 "drop WAIT id %d (%s) thresh 0x%x, min 0x%x\n", 161 wait->syncpt_id, sp->name, wait->thresh, 162 host1x_syncpt_read_min(sp)); 163 164 host1x_syncpt_patch_offset(sp, patch, wait->offset); 165 } 166 167 wait->bo = NULL; 168 } 169 170 return 0; 171} 172 173static unsigned int pin_job(struct host1x_job *job) 174{ 175 unsigned int i; 176 177 job->num_unpins = 0; 178 179 for (i = 0; i < job->num_relocs; i++) { 180 struct host1x_reloc *reloc = &job->relocarray[i]; 181 struct sg_table *sgt; 182 dma_addr_t phys_addr; 183 184 reloc->target = host1x_bo_get(reloc->target); 185 if (!reloc->target) 186 goto unpin; 187 188 phys_addr = host1x_bo_pin(reloc->target, &sgt); 189 if (!phys_addr) 190 goto unpin; 191 192 job->addr_phys[job->num_unpins] = phys_addr; 193 job->unpins[job->num_unpins].bo = reloc->target; 194 job->unpins[job->num_unpins].sgt = sgt; 195 job->num_unpins++; 196 } 197 198 for (i = 0; i < job->num_gathers; i++) { 199 struct host1x_job_gather *g = &job->gathers[i]; 200 struct sg_table *sgt; 201 dma_addr_t phys_addr; 202 203 g->bo = host1x_bo_get(g->bo); 204 if (!g->bo) 205 goto unpin; 206 207 phys_addr = host1x_bo_pin(g->bo, &sgt); 208 if (!phys_addr) 209 goto unpin; 210 211 job->addr_phys[job->num_unpins] = phys_addr; 212 job->unpins[job->num_unpins].bo = g->bo; 213 job->unpins[job->num_unpins].sgt = sgt; 214 job->num_unpins++; 215 } 216 217 return job->num_unpins; 218 219unpin: 220 host1x_job_unpin(job); 221 return 0; 222} 223 224static unsigned int do_relocs(struct host1x_job *job, struct host1x_bo *cmdbuf) 225{ 226 int i = 0; 227 u32 last_page = ~0; 228 void *cmdbuf_page_addr = NULL; 229 230 /* pin & patch the relocs for one gather */ 231 while (i < job->num_relocs) { 232 struct host1x_reloc *reloc = &job->relocarray[i]; 233 u32 reloc_addr = (job->reloc_addr_phys[i] + 234 reloc->target_offset) >> reloc->shift; 235 u32 *target; 236 237 /* skip all other gathers */ 238 if (!(reloc->cmdbuf && cmdbuf == reloc->cmdbuf)) { 239 i++; 240 continue; 241 } 242 243 if (last_page != reloc->cmdbuf_offset >> PAGE_SHIFT) { 244 if (cmdbuf_page_addr) 245 host1x_bo_kunmap(cmdbuf, last_page, 246 cmdbuf_page_addr); 247 248 cmdbuf_page_addr = host1x_bo_kmap(cmdbuf, 249 reloc->cmdbuf_offset >> PAGE_SHIFT); 250 last_page = reloc->cmdbuf_offset >> PAGE_SHIFT; 251 252 if (unlikely(!cmdbuf_page_addr)) { 253 pr_err("Could not map cmdbuf for relocation\n"); 254 return -ENOMEM; 255 } 256 } 257 258 target = cmdbuf_page_addr + (reloc->cmdbuf_offset & ~PAGE_MASK); 259 *target = reloc_addr; 260 261 /* mark this gather as handled */ 262 reloc->cmdbuf = 0; 263 } 264 265 if (cmdbuf_page_addr) 266 host1x_bo_kunmap(cmdbuf, last_page, cmdbuf_page_addr); 267 268 return 0; 269} 270 271static bool check_reloc(struct host1x_reloc *reloc, struct host1x_bo *cmdbuf, 272 unsigned int offset) 273{ 274 offset *= sizeof(u32); 275 276 if (reloc->cmdbuf != cmdbuf || reloc->cmdbuf_offset != offset) 277 return false; 278 279 return true; 280} 281 282struct host1x_firewall { 283 struct host1x_job *job; 284 struct device *dev; 285 286 unsigned int num_relocs; 287 struct host1x_reloc *reloc; 288 289 struct host1x_bo *cmdbuf_id; 290 unsigned int offset; 291 292 u32 words; 293 u32 class; 294 u32 reg; 295 u32 mask; 296 u32 count; 297}; 298 299static int check_mask(struct host1x_firewall *fw) 300{ 301 u32 mask = fw->mask; 302 u32 reg = fw->reg; 303 304 while (mask) { 305 if (fw->words == 0) 306 return -EINVAL; 307 308 if (mask & 1) { 309 if (fw->job->is_addr_reg(fw->dev, fw->class, reg)) { 310 if (!fw->num_relocs) 311 return -EINVAL; 312 if (!check_reloc(fw->reloc, fw->cmdbuf_id, 313 fw->offset)) 314 return -EINVAL; 315 fw->reloc++; 316 fw->num_relocs--; 317 } 318 fw->words--; 319 fw->offset++; 320 } 321 mask >>= 1; 322 reg++; 323 } 324 325 return 0; 326} 327 328static int check_incr(struct host1x_firewall *fw) 329{ 330 u32 count = fw->count; 331 u32 reg = fw->reg; 332 333 while (count) { 334 if (fw->words == 0) 335 return -EINVAL; 336 337 if (fw->job->is_addr_reg(fw->dev, fw->class, reg)) { 338 if (!fw->num_relocs) 339 return -EINVAL; 340 if (!check_reloc(fw->reloc, fw->cmdbuf_id, fw->offset)) 341 return -EINVAL; 342 fw->reloc++; 343 fw->num_relocs--; 344 } 345 reg++; 346 fw->words--; 347 fw->offset++; 348 count--; 349 } 350 351 return 0; 352} 353 354static int check_nonincr(struct host1x_firewall *fw) 355{ 356 int is_addr_reg = fw->job->is_addr_reg(fw->dev, fw->class, fw->reg); 357 u32 count = fw->count; 358 359 while (count) { 360 if (fw->words == 0) 361 return -EINVAL; 362 363 if (is_addr_reg) { 364 if (!fw->num_relocs) 365 return -EINVAL; 366 if (!check_reloc(fw->reloc, fw->cmdbuf_id, fw->offset)) 367 return -EINVAL; 368 fw->reloc++; 369 fw->num_relocs--; 370 } 371 fw->words--; 372 fw->offset++; 373 count--; 374 } 375 376 return 0; 377} 378 379static int validate(struct host1x_job *job, struct device *dev, 380 struct host1x_job_gather *g) 381{ 382 u32 *cmdbuf_base; 383 int err = 0; 384 struct host1x_firewall fw; 385 386 fw.job = job; 387 fw.dev = dev; 388 fw.reloc = job->relocarray; 389 fw.num_relocs = job->num_relocs; 390 fw.cmdbuf_id = g->bo; 391 392 fw.offset = 0; 393 fw.class = 0; 394 395 if (!job->is_addr_reg) 396 return 0; 397 398 cmdbuf_base = host1x_bo_mmap(g->bo); 399 if (!cmdbuf_base) 400 return -ENOMEM; 401 402 fw.words = g->words; 403 while (fw.words && !err) { 404 u32 word = cmdbuf_base[fw.offset]; 405 u32 opcode = (word & 0xf0000000) >> 28; 406 407 fw.mask = 0; 408 fw.reg = 0; 409 fw.count = 0; 410 fw.words--; 411 fw.offset++; 412 413 switch (opcode) { 414 case 0: 415 fw.class = word >> 6 & 0x3ff; 416 fw.mask = word & 0x3f; 417 fw.reg = word >> 16 & 0xfff; 418 err = check_mask(&fw); 419 if (err) 420 goto out; 421 break; 422 case 1: 423 fw.reg = word >> 16 & 0xfff; 424 fw.count = word & 0xffff; 425 err = check_incr(&fw); 426 if (err) 427 goto out; 428 break; 429 430 case 2: 431 fw.reg = word >> 16 & 0xfff; 432 fw.count = word & 0xffff; 433 err = check_nonincr(&fw); 434 if (err) 435 goto out; 436 break; 437 438 case 3: 439 fw.mask = word & 0xffff; 440 fw.reg = word >> 16 & 0xfff; 441 err = check_mask(&fw); 442 if (err) 443 goto out; 444 break; 445 case 4: 446 case 5: 447 case 14: 448 break; 449 default: 450 err = -EINVAL; 451 break; 452 } 453 } 454 455 /* No relocs should remain at this point */ 456 if (fw.num_relocs) 457 err = -EINVAL; 458 459out: 460 host1x_bo_munmap(g->bo, cmdbuf_base); 461 462 return err; 463} 464 465static inline int copy_gathers(struct host1x_job *job, struct device *dev) 466{ 467 size_t size = 0; 468 size_t offset = 0; 469 int i; 470 471 for (i = 0; i < job->num_gathers; i++) { 472 struct host1x_job_gather *g = &job->gathers[i]; 473 size += g->words * sizeof(u32); 474 } 475 476 job->gather_copy_mapped = dma_alloc_writecombine(dev, size, 477 &job->gather_copy, 478 GFP_KERNEL); 479 if (!job->gather_copy_mapped) { 480 int err = PTR_ERR(job->gather_copy_mapped); 481 job->gather_copy_mapped = NULL; 482 return err; 483 } 484 485 job->gather_copy_size = size; 486 487 for (i = 0; i < job->num_gathers; i++) { 488 struct host1x_job_gather *g = &job->gathers[i]; 489 void *gather; 490 491 gather = host1x_bo_mmap(g->bo); 492 memcpy(job->gather_copy_mapped + offset, gather + g->offset, 493 g->words * sizeof(u32)); 494 host1x_bo_munmap(g->bo, gather); 495 496 g->base = job->gather_copy; 497 g->offset = offset; 498 g->bo = NULL; 499 500 offset += g->words * sizeof(u32); 501 } 502 503 return 0; 504} 505 506int host1x_job_pin(struct host1x_job *job, struct device *dev) 507{ 508 int err; 509 unsigned int i, j; 510 struct host1x *host = dev_get_drvdata(dev->parent); 511 DECLARE_BITMAP(waitchk_mask, host1x_syncpt_nb_pts(host)); 512 513 bitmap_zero(waitchk_mask, host1x_syncpt_nb_pts(host)); 514 for (i = 0; i < job->num_waitchk; i++) { 515 u32 syncpt_id = job->waitchk[i].syncpt_id; 516 if (syncpt_id < host1x_syncpt_nb_pts(host)) 517 set_bit(syncpt_id, waitchk_mask); 518 } 519 520 /* get current syncpt values for waitchk */ 521 for_each_set_bit(i, waitchk_mask, host1x_syncpt_nb_pts(host)) 522 host1x_syncpt_load(host->syncpt + i); 523 524 /* pin memory */ 525 err = pin_job(job); 526 if (!err) 527 goto out; 528 529 /* patch gathers */ 530 for (i = 0; i < job->num_gathers; i++) { 531 struct host1x_job_gather *g = &job->gathers[i]; 532 533 /* process each gather mem only once */ 534 if (g->handled) 535 continue; 536 537 g->base = job->gather_addr_phys[i]; 538 539 for (j = 0; j < job->num_gathers; j++) 540 if (job->gathers[j].bo == g->bo) 541 job->gathers[j].handled = true; 542 543 err = 0; 544 545 if (IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL)) 546 err = validate(job, dev, g); 547 548 if (err) 549 dev_err(dev, "Job invalid (err=%d)\n", err); 550 551 if (!err) 552 err = do_relocs(job, g->bo); 553 554 if (!err) 555 err = do_waitchks(job, host, g->bo); 556 557 if (err) 558 break; 559 } 560 561 if (IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL) && !err) { 562 err = copy_gathers(job, dev); 563 if (err) { 564 host1x_job_unpin(job); 565 return err; 566 } 567 } 568 569out: 570 wmb(); 571 572 return err; 573} 574 575void host1x_job_unpin(struct host1x_job *job) 576{ 577 unsigned int i; 578 579 for (i = 0; i < job->num_unpins; i++) { 580 struct host1x_job_unpin_data *unpin = &job->unpins[i]; 581 host1x_bo_unpin(unpin->bo, unpin->sgt); 582 host1x_bo_put(unpin->bo); 583 } 584 job->num_unpins = 0; 585 586 if (job->gather_copy_size) 587 dma_free_writecombine(job->channel->dev, job->gather_copy_size, 588 job->gather_copy_mapped, 589 job->gather_copy); 590} 591 592/* 593 * Debug routine used to dump job entries 594 */ 595void host1x_job_dump(struct device *dev, struct host1x_job *job) 596{ 597 dev_dbg(dev, " SYNCPT_ID %d\n", job->syncpt_id); 598 dev_dbg(dev, " SYNCPT_VAL %d\n", job->syncpt_end); 599 dev_dbg(dev, " FIRST_GET 0x%x\n", job->first_get); 600 dev_dbg(dev, " TIMEOUT %d\n", job->timeout); 601 dev_dbg(dev, " NUM_SLOTS %d\n", job->num_slots); 602 dev_dbg(dev, " NUM_HANDLES %d\n", job->num_unpins); 603} 604