mmu.c revision f0c5b592a5fbc7478fde6d0f84022e6bec9a4302
1/************************************************************************** 2 * Copyright (c) 2007, Intel Corporation. 3 * 4 * This program is free software; you can redistribute it and/or modify it 5 * under the terms and conditions of the GNU General Public License, 6 * version 2, as published by the Free Software Foundation. 7 * 8 * This program is distributed in the hope it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 11 * more details. 12 * 13 * You should have received a copy of the GNU General Public License along with 14 * this program; if not, write to the Free Software Foundation, Inc., 15 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. 16 * 17 **************************************************************************/ 18#include <drm/drmP.h> 19#include "psb_drv.h" 20#include "psb_reg.h" 21 22/* 23 * Code for the SGX MMU: 24 */ 25 26/* 27 * clflush on one processor only: 28 * clflush should apparently flush the cache line on all processors in an 29 * SMP system. 30 */ 31 32/* 33 * kmap atomic: 34 * The usage of the slots must be completely encapsulated within a spinlock, and 35 * no other functions that may be using the locks for other purposed may be 36 * called from within the locked region. 37 * Since the slots are per processor, this will guarantee that we are the only 38 * user. 39 */ 40 41/* 42 * TODO: Inserting ptes from an interrupt handler: 43 * This may be desirable for some SGX functionality where the GPU can fault in 44 * needed pages. For that, we need to make an atomic insert_pages function, that 45 * may fail. 46 * If it fails, the caller need to insert the page using a workqueue function, 47 * but on average it should be fast. 48 */ 49 50struct psb_mmu_driver { 51 /* protects driver- and pd structures. Always take in read mode 52 * before taking the page table spinlock. 53 */ 54 struct rw_semaphore sem; 55 56 /* protects page tables, directory tables and pt tables. 57 * and pt structures. 58 */ 59 spinlock_t lock; 60 61 atomic_t needs_tlbflush; 62 63 uint8_t __iomem *register_map; 64 struct psb_mmu_pd *default_pd; 65 /*uint32_t bif_ctrl;*/ 66 int has_clflush; 67 int clflush_add; 68 unsigned long clflush_mask; 69 70 struct drm_psb_private *dev_priv; 71}; 72 73struct psb_mmu_pd; 74 75struct psb_mmu_pt { 76 struct psb_mmu_pd *pd; 77 uint32_t index; 78 uint32_t count; 79 struct page *p; 80 uint32_t *v; 81}; 82 83struct psb_mmu_pd { 84 struct psb_mmu_driver *driver; 85 int hw_context; 86 struct psb_mmu_pt **tables; 87 struct page *p; 88 struct page *dummy_pt; 89 struct page *dummy_page; 90 uint32_t pd_mask; 91 uint32_t invalid_pde; 92 uint32_t invalid_pte; 93}; 94 95static inline uint32_t psb_mmu_pt_index(uint32_t offset) 96{ 97 return (offset >> PSB_PTE_SHIFT) & 0x3FF; 98} 99 100static inline uint32_t psb_mmu_pd_index(uint32_t offset) 101{ 102 return offset >> PSB_PDE_SHIFT; 103} 104 105static inline void psb_clflush(void *addr) 106{ 107 __asm__ __volatile__("clflush (%0)\n" : : "r"(addr) : "memory"); 108} 109 110static inline void psb_mmu_clflush(struct psb_mmu_driver *driver, 111 void *addr) 112{ 113 if (!driver->has_clflush) 114 return; 115 116 mb(); 117 psb_clflush(addr); 118 mb(); 119} 120 121static void psb_page_clflush(struct psb_mmu_driver *driver, struct page* page) 122{ 123 uint32_t clflush_add = driver->clflush_add >> PAGE_SHIFT; 124 uint32_t clflush_count = PAGE_SIZE / clflush_add; 125 int i; 126 uint8_t *clf; 127 128 clf = kmap_atomic(page); 129 mb(); 130 for (i = 0; i < clflush_count; ++i) { 131 psb_clflush(clf); 132 clf += clflush_add; 133 } 134 mb(); 135 kunmap_atomic(clf); 136} 137 138static void psb_pages_clflush(struct psb_mmu_driver *driver, 139 struct page *page[], unsigned long num_pages) 140{ 141 int i; 142 143 if (!driver->has_clflush) 144 return ; 145 146 for (i = 0; i < num_pages; i++) 147 psb_page_clflush(driver, *page++); 148} 149 150static void psb_mmu_flush_pd_locked(struct psb_mmu_driver *driver, 151 int force) 152{ 153 atomic_set(&driver->needs_tlbflush, 0); 154} 155 156static void psb_mmu_flush_pd(struct psb_mmu_driver *driver, int force) 157{ 158 down_write(&driver->sem); 159 psb_mmu_flush_pd_locked(driver, force); 160 up_write(&driver->sem); 161} 162 163void psb_mmu_flush(struct psb_mmu_driver *driver, int rc_prot) 164{ 165 if (rc_prot) 166 down_write(&driver->sem); 167 if (rc_prot) 168 up_write(&driver->sem); 169} 170 171void psb_mmu_set_pd_context(struct psb_mmu_pd *pd, int hw_context) 172{ 173 /*ttm_tt_cache_flush(&pd->p, 1);*/ 174 psb_pages_clflush(pd->driver, &pd->p, 1); 175 down_write(&pd->driver->sem); 176 wmb(); 177 psb_mmu_flush_pd_locked(pd->driver, 1); 178 pd->hw_context = hw_context; 179 up_write(&pd->driver->sem); 180 181} 182 183static inline unsigned long psb_pd_addr_end(unsigned long addr, 184 unsigned long end) 185{ 186 187 addr = (addr + PSB_PDE_MASK + 1) & ~PSB_PDE_MASK; 188 return (addr < end) ? addr : end; 189} 190 191static inline uint32_t psb_mmu_mask_pte(uint32_t pfn, int type) 192{ 193 uint32_t mask = PSB_PTE_VALID; 194 195 if (type & PSB_MMU_CACHED_MEMORY) 196 mask |= PSB_PTE_CACHED; 197 if (type & PSB_MMU_RO_MEMORY) 198 mask |= PSB_PTE_RO; 199 if (type & PSB_MMU_WO_MEMORY) 200 mask |= PSB_PTE_WO; 201 202 return (pfn << PAGE_SHIFT) | mask; 203} 204 205struct psb_mmu_pd *psb_mmu_alloc_pd(struct psb_mmu_driver *driver, 206 int trap_pagefaults, int invalid_type) 207{ 208 struct psb_mmu_pd *pd = kmalloc(sizeof(*pd), GFP_KERNEL); 209 uint32_t *v; 210 int i; 211 212 if (!pd) 213 return NULL; 214 215 pd->p = alloc_page(GFP_DMA32); 216 if (!pd->p) 217 goto out_err1; 218 pd->dummy_pt = alloc_page(GFP_DMA32); 219 if (!pd->dummy_pt) 220 goto out_err2; 221 pd->dummy_page = alloc_page(GFP_DMA32); 222 if (!pd->dummy_page) 223 goto out_err3; 224 225 if (!trap_pagefaults) { 226 pd->invalid_pde = 227 psb_mmu_mask_pte(page_to_pfn(pd->dummy_pt), 228 invalid_type); 229 pd->invalid_pte = 230 psb_mmu_mask_pte(page_to_pfn(pd->dummy_page), 231 invalid_type); 232 } else { 233 pd->invalid_pde = 0; 234 pd->invalid_pte = 0; 235 } 236 237 v = kmap(pd->dummy_pt); 238 for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i) 239 v[i] = pd->invalid_pte; 240 241 kunmap(pd->dummy_pt); 242 243 v = kmap(pd->p); 244 for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i) 245 v[i] = pd->invalid_pde; 246 247 kunmap(pd->p); 248 249 clear_page(kmap(pd->dummy_page)); 250 kunmap(pd->dummy_page); 251 252 pd->tables = vmalloc_user(sizeof(struct psb_mmu_pt *) * 1024); 253 if (!pd->tables) 254 goto out_err4; 255 256 pd->hw_context = -1; 257 pd->pd_mask = PSB_PTE_VALID; 258 pd->driver = driver; 259 260 return pd; 261 262out_err4: 263 __free_page(pd->dummy_page); 264out_err3: 265 __free_page(pd->dummy_pt); 266out_err2: 267 __free_page(pd->p); 268out_err1: 269 kfree(pd); 270 return NULL; 271} 272 273void psb_mmu_free_pt(struct psb_mmu_pt *pt) 274{ 275 __free_page(pt->p); 276 kfree(pt); 277} 278 279void psb_mmu_free_pagedir(struct psb_mmu_pd *pd) 280{ 281 struct psb_mmu_driver *driver = pd->driver; 282 struct psb_mmu_pt *pt; 283 int i; 284 285 down_write(&driver->sem); 286 if (pd->hw_context != -1) 287 psb_mmu_flush_pd_locked(driver, 1); 288 289 /* Should take the spinlock here, but we don't need to do that 290 since we have the semaphore in write mode. */ 291 292 for (i = 0; i < 1024; ++i) { 293 pt = pd->tables[i]; 294 if (pt) 295 psb_mmu_free_pt(pt); 296 } 297 298 vfree(pd->tables); 299 __free_page(pd->dummy_page); 300 __free_page(pd->dummy_pt); 301 __free_page(pd->p); 302 kfree(pd); 303 up_write(&driver->sem); 304} 305 306static struct psb_mmu_pt *psb_mmu_alloc_pt(struct psb_mmu_pd *pd) 307{ 308 struct psb_mmu_pt *pt = kmalloc(sizeof(*pt), GFP_KERNEL); 309 void *v; 310 uint32_t clflush_add = pd->driver->clflush_add >> PAGE_SHIFT; 311 uint32_t clflush_count = PAGE_SIZE / clflush_add; 312 spinlock_t *lock = &pd->driver->lock; 313 uint8_t *clf; 314 uint32_t *ptes; 315 int i; 316 317 if (!pt) 318 return NULL; 319 320 pt->p = alloc_page(GFP_DMA32); 321 if (!pt->p) { 322 kfree(pt); 323 return NULL; 324 } 325 326 spin_lock(lock); 327 328 v = kmap_atomic(pt->p); 329 clf = (uint8_t *) v; 330 ptes = (uint32_t *) v; 331 for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i) 332 *ptes++ = pd->invalid_pte; 333 334 335 if (pd->driver->has_clflush && pd->hw_context != -1) { 336 mb(); 337 for (i = 0; i < clflush_count; ++i) { 338 psb_clflush(clf); 339 clf += clflush_add; 340 } 341 mb(); 342 } 343 344 kunmap_atomic(v); 345 spin_unlock(lock); 346 347 pt->count = 0; 348 pt->pd = pd; 349 pt->index = 0; 350 351 return pt; 352} 353 354struct psb_mmu_pt *psb_mmu_pt_alloc_map_lock(struct psb_mmu_pd *pd, 355 unsigned long addr) 356{ 357 uint32_t index = psb_mmu_pd_index(addr); 358 struct psb_mmu_pt *pt; 359 uint32_t *v; 360 spinlock_t *lock = &pd->driver->lock; 361 362 spin_lock(lock); 363 pt = pd->tables[index]; 364 while (!pt) { 365 spin_unlock(lock); 366 pt = psb_mmu_alloc_pt(pd); 367 if (!pt) 368 return NULL; 369 spin_lock(lock); 370 371 if (pd->tables[index]) { 372 spin_unlock(lock); 373 psb_mmu_free_pt(pt); 374 spin_lock(lock); 375 pt = pd->tables[index]; 376 continue; 377 } 378 379 v = kmap_atomic(pd->p); 380 pd->tables[index] = pt; 381 v[index] = (page_to_pfn(pt->p) << 12) | pd->pd_mask; 382 pt->index = index; 383 kunmap_atomic((void *) v); 384 385 if (pd->hw_context != -1) { 386 psb_mmu_clflush(pd->driver, (void *) &v[index]); 387 atomic_set(&pd->driver->needs_tlbflush, 1); 388 } 389 } 390 pt->v = kmap_atomic(pt->p); 391 return pt; 392} 393 394static struct psb_mmu_pt *psb_mmu_pt_map_lock(struct psb_mmu_pd *pd, 395 unsigned long addr) 396{ 397 uint32_t index = psb_mmu_pd_index(addr); 398 struct psb_mmu_pt *pt; 399 spinlock_t *lock = &pd->driver->lock; 400 401 spin_lock(lock); 402 pt = pd->tables[index]; 403 if (!pt) { 404 spin_unlock(lock); 405 return NULL; 406 } 407 pt->v = kmap_atomic(pt->p); 408 return pt; 409} 410 411static void psb_mmu_pt_unmap_unlock(struct psb_mmu_pt *pt) 412{ 413 struct psb_mmu_pd *pd = pt->pd; 414 uint32_t *v; 415 416 kunmap_atomic(pt->v); 417 if (pt->count == 0) { 418 v = kmap_atomic(pd->p); 419 v[pt->index] = pd->invalid_pde; 420 pd->tables[pt->index] = NULL; 421 422 if (pd->hw_context != -1) { 423 psb_mmu_clflush(pd->driver, 424 (void *) &v[pt->index]); 425 atomic_set(&pd->driver->needs_tlbflush, 1); 426 } 427 kunmap_atomic(pt->v); 428 spin_unlock(&pd->driver->lock); 429 psb_mmu_free_pt(pt); 430 return; 431 } 432 spin_unlock(&pd->driver->lock); 433} 434 435static inline void psb_mmu_set_pte(struct psb_mmu_pt *pt, 436 unsigned long addr, uint32_t pte) 437{ 438 pt->v[psb_mmu_pt_index(addr)] = pte; 439} 440 441static inline void psb_mmu_invalidate_pte(struct psb_mmu_pt *pt, 442 unsigned long addr) 443{ 444 pt->v[psb_mmu_pt_index(addr)] = pt->pd->invalid_pte; 445} 446 447 448void psb_mmu_mirror_gtt(struct psb_mmu_pd *pd, 449 uint32_t mmu_offset, uint32_t gtt_start, 450 uint32_t gtt_pages) 451{ 452 uint32_t *v; 453 uint32_t start = psb_mmu_pd_index(mmu_offset); 454 struct psb_mmu_driver *driver = pd->driver; 455 int num_pages = gtt_pages; 456 457 down_read(&driver->sem); 458 spin_lock(&driver->lock); 459 460 v = kmap_atomic(pd->p); 461 v += start; 462 463 while (gtt_pages--) { 464 *v++ = gtt_start | pd->pd_mask; 465 gtt_start += PAGE_SIZE; 466 } 467 468 /*ttm_tt_cache_flush(&pd->p, num_pages);*/ 469 psb_pages_clflush(pd->driver, &pd->p, num_pages); 470 kunmap_atomic(v); 471 spin_unlock(&driver->lock); 472 473 if (pd->hw_context != -1) 474 atomic_set(&pd->driver->needs_tlbflush, 1); 475 476 up_read(&pd->driver->sem); 477 psb_mmu_flush_pd(pd->driver, 0); 478} 479 480struct psb_mmu_pd *psb_mmu_get_default_pd(struct psb_mmu_driver *driver) 481{ 482 struct psb_mmu_pd *pd; 483 484 /* down_read(&driver->sem); */ 485 pd = driver->default_pd; 486 /* up_read(&driver->sem); */ 487 488 return pd; 489} 490 491/* Returns the physical address of the PD shared by sgx/msvdx */ 492uint32_t psb_get_default_pd_addr(struct psb_mmu_driver *driver) 493{ 494 struct psb_mmu_pd *pd; 495 496 pd = psb_mmu_get_default_pd(driver); 497 return page_to_pfn(pd->p) << PAGE_SHIFT; 498} 499 500void psb_mmu_driver_takedown(struct psb_mmu_driver *driver) 501{ 502 psb_mmu_free_pagedir(driver->default_pd); 503 kfree(driver); 504} 505 506struct psb_mmu_driver *psb_mmu_driver_init(uint8_t __iomem * registers, 507 int trap_pagefaults, 508 int invalid_type, 509 struct drm_psb_private *dev_priv) 510{ 511 struct psb_mmu_driver *driver; 512 513 driver = kmalloc(sizeof(*driver), GFP_KERNEL); 514 515 if (!driver) 516 return NULL; 517 driver->dev_priv = dev_priv; 518 519 driver->default_pd = psb_mmu_alloc_pd(driver, trap_pagefaults, 520 invalid_type); 521 if (!driver->default_pd) 522 goto out_err1; 523 524 spin_lock_init(&driver->lock); 525 init_rwsem(&driver->sem); 526 down_write(&driver->sem); 527 driver->register_map = registers; 528 atomic_set(&driver->needs_tlbflush, 1); 529 530 driver->has_clflush = 0; 531 532 if (boot_cpu_has(X86_FEATURE_CLFLSH)) { 533 uint32_t tfms, misc, cap0, cap4, clflush_size; 534 535 /* 536 * clflush size is determined at kernel setup for x86_64 537 * but not for i386. We have to do it here. 538 */ 539 540 cpuid(0x00000001, &tfms, &misc, &cap0, &cap4); 541 clflush_size = ((misc >> 8) & 0xff) * 8; 542 driver->has_clflush = 1; 543 driver->clflush_add = 544 PAGE_SIZE * clflush_size / sizeof(uint32_t); 545 driver->clflush_mask = driver->clflush_add - 1; 546 driver->clflush_mask = ~driver->clflush_mask; 547 } 548 549 up_write(&driver->sem); 550 return driver; 551 552out_err1: 553 kfree(driver); 554 return NULL; 555} 556 557static void psb_mmu_flush_ptes(struct psb_mmu_pd *pd, 558 unsigned long address, uint32_t num_pages, 559 uint32_t desired_tile_stride, 560 uint32_t hw_tile_stride) 561{ 562 struct psb_mmu_pt *pt; 563 uint32_t rows = 1; 564 uint32_t i; 565 unsigned long addr; 566 unsigned long end; 567 unsigned long next; 568 unsigned long add; 569 unsigned long row_add; 570 unsigned long clflush_add = pd->driver->clflush_add; 571 unsigned long clflush_mask = pd->driver->clflush_mask; 572 573 if (!pd->driver->has_clflush) { 574 /*ttm_tt_cache_flush(&pd->p, num_pages);*/ 575 psb_pages_clflush(pd->driver, &pd->p, num_pages); 576 return; 577 } 578 579 if (hw_tile_stride) 580 rows = num_pages / desired_tile_stride; 581 else 582 desired_tile_stride = num_pages; 583 584 add = desired_tile_stride << PAGE_SHIFT; 585 row_add = hw_tile_stride << PAGE_SHIFT; 586 mb(); 587 for (i = 0; i < rows; ++i) { 588 589 addr = address; 590 end = addr + add; 591 592 do { 593 next = psb_pd_addr_end(addr, end); 594 pt = psb_mmu_pt_map_lock(pd, addr); 595 if (!pt) 596 continue; 597 do { 598 psb_clflush(&pt->v 599 [psb_mmu_pt_index(addr)]); 600 } while (addr += 601 clflush_add, 602 (addr & clflush_mask) < next); 603 604 psb_mmu_pt_unmap_unlock(pt); 605 } while (addr = next, next != end); 606 address += row_add; 607 } 608 mb(); 609} 610 611void psb_mmu_remove_pfn_sequence(struct psb_mmu_pd *pd, 612 unsigned long address, uint32_t num_pages) 613{ 614 struct psb_mmu_pt *pt; 615 unsigned long addr; 616 unsigned long end; 617 unsigned long next; 618 unsigned long f_address = address; 619 620 down_read(&pd->driver->sem); 621 622 addr = address; 623 end = addr + (num_pages << PAGE_SHIFT); 624 625 do { 626 next = psb_pd_addr_end(addr, end); 627 pt = psb_mmu_pt_alloc_map_lock(pd, addr); 628 if (!pt) 629 goto out; 630 do { 631 psb_mmu_invalidate_pte(pt, addr); 632 --pt->count; 633 } while (addr += PAGE_SIZE, addr < next); 634 psb_mmu_pt_unmap_unlock(pt); 635 636 } while (addr = next, next != end); 637 638out: 639 if (pd->hw_context != -1) 640 psb_mmu_flush_ptes(pd, f_address, num_pages, 1, 1); 641 642 up_read(&pd->driver->sem); 643 644 if (pd->hw_context != -1) 645 psb_mmu_flush(pd->driver, 0); 646 647 return; 648} 649 650void psb_mmu_remove_pages(struct psb_mmu_pd *pd, unsigned long address, 651 uint32_t num_pages, uint32_t desired_tile_stride, 652 uint32_t hw_tile_stride) 653{ 654 struct psb_mmu_pt *pt; 655 uint32_t rows = 1; 656 uint32_t i; 657 unsigned long addr; 658 unsigned long end; 659 unsigned long next; 660 unsigned long add; 661 unsigned long row_add; 662 unsigned long f_address = address; 663 664 if (hw_tile_stride) 665 rows = num_pages / desired_tile_stride; 666 else 667 desired_tile_stride = num_pages; 668 669 add = desired_tile_stride << PAGE_SHIFT; 670 row_add = hw_tile_stride << PAGE_SHIFT; 671 672 /* down_read(&pd->driver->sem); */ 673 674 /* Make sure we only need to flush this processor's cache */ 675 676 for (i = 0; i < rows; ++i) { 677 678 addr = address; 679 end = addr + add; 680 681 do { 682 next = psb_pd_addr_end(addr, end); 683 pt = psb_mmu_pt_map_lock(pd, addr); 684 if (!pt) 685 continue; 686 do { 687 psb_mmu_invalidate_pte(pt, addr); 688 --pt->count; 689 690 } while (addr += PAGE_SIZE, addr < next); 691 psb_mmu_pt_unmap_unlock(pt); 692 693 } while (addr = next, next != end); 694 address += row_add; 695 } 696 if (pd->hw_context != -1) 697 psb_mmu_flush_ptes(pd, f_address, num_pages, 698 desired_tile_stride, hw_tile_stride); 699 700 /* up_read(&pd->driver->sem); */ 701 702 if (pd->hw_context != -1) 703 psb_mmu_flush(pd->driver, 0); 704} 705 706int psb_mmu_insert_pfn_sequence(struct psb_mmu_pd *pd, uint32_t start_pfn, 707 unsigned long address, uint32_t num_pages, 708 int type) 709{ 710 struct psb_mmu_pt *pt; 711 uint32_t pte; 712 unsigned long addr; 713 unsigned long end; 714 unsigned long next; 715 unsigned long f_address = address; 716 int ret = 0; 717 718 down_read(&pd->driver->sem); 719 720 addr = address; 721 end = addr + (num_pages << PAGE_SHIFT); 722 723 do { 724 next = psb_pd_addr_end(addr, end); 725 pt = psb_mmu_pt_alloc_map_lock(pd, addr); 726 if (!pt) { 727 ret = -ENOMEM; 728 goto out; 729 } 730 do { 731 pte = psb_mmu_mask_pte(start_pfn++, type); 732 psb_mmu_set_pte(pt, addr, pte); 733 pt->count++; 734 } while (addr += PAGE_SIZE, addr < next); 735 psb_mmu_pt_unmap_unlock(pt); 736 737 } while (addr = next, next != end); 738 739out: 740 if (pd->hw_context != -1) 741 psb_mmu_flush_ptes(pd, f_address, num_pages, 1, 1); 742 743 up_read(&pd->driver->sem); 744 745 if (pd->hw_context != -1) 746 psb_mmu_flush(pd->driver, 1); 747 748 return ret; 749} 750 751int psb_mmu_insert_pages(struct psb_mmu_pd *pd, struct page **pages, 752 unsigned long address, uint32_t num_pages, 753 uint32_t desired_tile_stride, 754 uint32_t hw_tile_stride, int type) 755{ 756 struct psb_mmu_pt *pt; 757 uint32_t rows = 1; 758 uint32_t i; 759 uint32_t pte; 760 unsigned long addr; 761 unsigned long end; 762 unsigned long next; 763 unsigned long add; 764 unsigned long row_add; 765 unsigned long f_address = address; 766 int ret = 0; 767 768 if (hw_tile_stride) { 769 if (num_pages % desired_tile_stride != 0) 770 return -EINVAL; 771 rows = num_pages / desired_tile_stride; 772 } else { 773 desired_tile_stride = num_pages; 774 } 775 776 add = desired_tile_stride << PAGE_SHIFT; 777 row_add = hw_tile_stride << PAGE_SHIFT; 778 779 down_read(&pd->driver->sem); 780 781 for (i = 0; i < rows; ++i) { 782 783 addr = address; 784 end = addr + add; 785 786 do { 787 next = psb_pd_addr_end(addr, end); 788 pt = psb_mmu_pt_alloc_map_lock(pd, addr); 789 if (!pt) { 790 ret = -ENOMEM; 791 goto out; 792 } 793 do { 794 pte = 795 psb_mmu_mask_pte(page_to_pfn(*pages++), 796 type); 797 psb_mmu_set_pte(pt, addr, pte); 798 pt->count++; 799 } while (addr += PAGE_SIZE, addr < next); 800 psb_mmu_pt_unmap_unlock(pt); 801 802 } while (addr = next, next != end); 803 804 address += row_add; 805 } 806out: 807 if (pd->hw_context != -1) 808 psb_mmu_flush_ptes(pd, f_address, num_pages, 809 desired_tile_stride, hw_tile_stride); 810 811 up_read(&pd->driver->sem); 812 813 if (pd->hw_context != -1) 814 psb_mmu_flush(pd->driver, 1); 815 816 return ret; 817} 818 819int psb_mmu_virtual_to_pfn(struct psb_mmu_pd *pd, uint32_t virtual, 820 unsigned long *pfn) 821{ 822 int ret; 823 struct psb_mmu_pt *pt; 824 uint32_t tmp; 825 spinlock_t *lock = &pd->driver->lock; 826 827 down_read(&pd->driver->sem); 828 pt = psb_mmu_pt_map_lock(pd, virtual); 829 if (!pt) { 830 uint32_t *v; 831 832 spin_lock(lock); 833 v = kmap_atomic(pd->p); 834 tmp = v[psb_mmu_pd_index(virtual)]; 835 kunmap_atomic(v); 836 spin_unlock(lock); 837 838 if (tmp != pd->invalid_pde || !(tmp & PSB_PTE_VALID) || 839 !(pd->invalid_pte & PSB_PTE_VALID)) { 840 ret = -EINVAL; 841 goto out; 842 } 843 ret = 0; 844 *pfn = pd->invalid_pte >> PAGE_SHIFT; 845 goto out; 846 } 847 tmp = pt->v[psb_mmu_pt_index(virtual)]; 848 if (!(tmp & PSB_PTE_VALID)) { 849 ret = -EINVAL; 850 } else { 851 ret = 0; 852 *pfn = tmp >> PAGE_SHIFT; 853 } 854 psb_mmu_pt_unmap_unlock(pt); 855out: 856 up_read(&pd->driver->sem); 857 return ret; 858} 859