1771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse/* 2771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * Copyright 2008 Jerome Glisse. 3771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * All Rights Reserved. 4771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * 5771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * Permission is hereby granted, free of charge, to any person obtaining a 6771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * copy of this software and associated documentation files (the "Software"), 7771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * to deal in the Software without restriction, including without limitation 8771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * and/or sell copies of the Software, and to permit persons to whom the 10771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * Software is furnished to do so, subject to the following conditions: 11771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * 12771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * The above copyright notice and this permission notice (including the next 13771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * paragraph) shall be included in all copies or substantial portions of the 14771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * Software. 15771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * 16771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR 20771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 22771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * DEALINGS IN THE SOFTWARE. 23771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * 24771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * Authors: 25771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * Jerome Glisse <glisse@freedesktop.org> 26771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse */ 27771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse#include "drmP.h" 28771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse#include "radeon_drm.h" 29771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse#include "radeon_reg.h" 30771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse#include "radeon.h" 31771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 32771fe6b912fca54f03e8a72eb63058b582775362Jerome Glissevoid r100_cs_dump_packet(struct radeon_cs_parser *p, 33771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse struct radeon_cs_packet *pkt); 34771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 35771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisseint radeon_cs_parser_relocs(struct radeon_cs_parser *p) 36771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse{ 37771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse struct drm_device *ddev = p->rdev->ddev; 38771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse struct radeon_cs_chunk *chunk; 39771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse unsigned i, j; 40771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse bool duplicate; 41771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 42771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse if (p->chunk_relocs_idx == -1) { 43771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return 0; 44771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 45771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse chunk = &p->chunks[p->chunk_relocs_idx]; 46771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse /* FIXME: we assume that each relocs use 4 dwords */ 47771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse p->nrelocs = chunk->length_dw / 4; 48771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse p->relocs_ptr = kcalloc(p->nrelocs, sizeof(void *), GFP_KERNEL); 49771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse if (p->relocs_ptr == NULL) { 50771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return -ENOMEM; 51771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 52771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse p->relocs = kcalloc(p->nrelocs, sizeof(struct radeon_cs_reloc), GFP_KERNEL); 53771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse if (p->relocs == NULL) { 54771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return -ENOMEM; 55771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 56771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse for (i = 0; i < p->nrelocs; i++) { 57771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse struct drm_radeon_cs_reloc *r; 58771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 59771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse duplicate = false; 60771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse r = (struct drm_radeon_cs_reloc *)&chunk->kdata[i*4]; 6116557f1ef0e14fd43b0a46b7dc89c431020ff3e2Christian König for (j = 0; j < i; j++) { 62771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse if (r->handle == p->relocs[j].handle) { 63771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse p->relocs_ptr[i] = &p->relocs[j]; 64771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse duplicate = true; 65771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse break; 66771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 67771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 68771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse if (!duplicate) { 69771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse p->relocs[i].gobj = drm_gem_object_lookup(ddev, 70771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse p->filp, 71771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse r->handle); 72771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse if (p->relocs[i].gobj == NULL) { 73771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse DRM_ERROR("gem object lookup failed 0x%x\n", 74771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse r->handle); 75bf79cb914dbfe848add8bb76cbb8ff89110d29ffChris Wilson return -ENOENT; 76771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 77771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse p->relocs_ptr[i] = &p->relocs[i]; 787e4d15d90afe46d34b510f3c70217d3469a7dd70Daniel Vetter p->relocs[i].robj = gem_to_radeon_bo(p->relocs[i].gobj); 794c7886791264f03428d5424befb1b96f08fc90f4Jerome Glisse p->relocs[i].lobj.bo = p->relocs[i].robj; 80771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse p->relocs[i].lobj.wdomain = r->write_domain; 81147666fb3b93b8c484f562da33a37f886ddff768Thomas Hellstrom p->relocs[i].lobj.rdomain = r->read_domains; 82147666fb3b93b8c484f562da33a37f886ddff768Thomas Hellstrom p->relocs[i].lobj.tv.bo = &p->relocs[i].robj->tbo; 83771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse p->relocs[i].handle = r->handle; 84771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse p->relocs[i].flags = r->flags; 854c7886791264f03428d5424befb1b96f08fc90f4Jerome Glisse radeon_bo_list_add_object(&p->relocs[i].lobj, 86147666fb3b93b8c484f562da33a37f886ddff768Thomas Hellstrom &p->validated); 8793504fce28b1a387ec01f81b26637d237dca2b36Christian König 8816557f1ef0e14fd43b0a46b7dc89c431020ff3e2Christian König } else 8916557f1ef0e14fd43b0a46b7dc89c431020ff3e2Christian König p->relocs[i].handle = 0; 90771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 9194429bb6c8343722544e282d89dc4638672e49b4Jerome Glisse return radeon_bo_list_validate(&p->validated); 92771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse} 93771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 94721604a15b934f0a8d1909acb8017f029128be2fJerome Glissestatic int radeon_cs_get_ring(struct radeon_cs_parser *p, u32 ring, s32 priority) 95721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse{ 96721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse p->priority = priority; 97721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse 98721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse switch (ring) { 99721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse default: 100721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse DRM_ERROR("unknown ring id: %d\n", ring); 101721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse return -EINVAL; 102721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse case RADEON_CS_RING_GFX: 103721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse p->ring = RADEON_RING_TYPE_GFX_INDEX; 104721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse break; 105721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse case RADEON_CS_RING_COMPUTE: 1068d5ef7b1f67a2f8e6594b38a68c1566ed0740ec1Alex Deucher if (p->rdev->family >= CHIP_TAHITI) { 1078d5ef7b1f67a2f8e6594b38a68c1566ed0740ec1Alex Deucher if (p->priority > 0) 1088d5ef7b1f67a2f8e6594b38a68c1566ed0740ec1Alex Deucher p->ring = CAYMAN_RING_TYPE_CP1_INDEX; 1098d5ef7b1f67a2f8e6594b38a68c1566ed0740ec1Alex Deucher else 1108d5ef7b1f67a2f8e6594b38a68c1566ed0740ec1Alex Deucher p->ring = CAYMAN_RING_TYPE_CP2_INDEX; 1118d5ef7b1f67a2f8e6594b38a68c1566ed0740ec1Alex Deucher } else 1128d5ef7b1f67a2f8e6594b38a68c1566ed0740ec1Alex Deucher p->ring = RADEON_RING_TYPE_GFX_INDEX; 113721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse break; 114721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse } 115721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse return 0; 116721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse} 117721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse 11893504fce28b1a387ec01f81b26637d237dca2b36Christian Königstatic int radeon_cs_sync_rings(struct radeon_cs_parser *p) 11993504fce28b1a387ec01f81b26637d237dca2b36Christian König{ 120cdac5504b2981520f993790f5d30c0cdc15c3214Christian König bool sync_to_ring[RADEON_NUM_RINGS] = { }; 12193504fce28b1a387ec01f81b26637d237dca2b36Christian König int i, r; 12293504fce28b1a387ec01f81b26637d237dca2b36Christian König 123cdac5504b2981520f993790f5d30c0cdc15c3214Christian König for (i = 0; i < p->nrelocs; i++) { 124cdac5504b2981520f993790f5d30c0cdc15c3214Christian König if (!p->relocs[i].robj || !p->relocs[i].robj->tbo.sync_obj) 125cdac5504b2981520f993790f5d30c0cdc15c3214Christian König continue; 126cdac5504b2981520f993790f5d30c0cdc15c3214Christian König 127cdac5504b2981520f993790f5d30c0cdc15c3214Christian König if (!(p->relocs[i].flags & RADEON_RELOC_DONT_SYNC)) { 128cdac5504b2981520f993790f5d30c0cdc15c3214Christian König struct radeon_fence *fence = p->relocs[i].robj->tbo.sync_obj; 129cdac5504b2981520f993790f5d30c0cdc15c3214Christian König if (!radeon_fence_signaled(fence)) { 130cdac5504b2981520f993790f5d30c0cdc15c3214Christian König sync_to_ring[fence->ring] = true; 131cdac5504b2981520f993790f5d30c0cdc15c3214Christian König } 132cdac5504b2981520f993790f5d30c0cdc15c3214Christian König } 133cdac5504b2981520f993790f5d30c0cdc15c3214Christian König } 134cdac5504b2981520f993790f5d30c0cdc15c3214Christian König 13593504fce28b1a387ec01f81b26637d237dca2b36Christian König for (i = 0; i < RADEON_NUM_RINGS; ++i) { 13693504fce28b1a387ec01f81b26637d237dca2b36Christian König /* no need to sync to our own or unused rings */ 137cdac5504b2981520f993790f5d30c0cdc15c3214Christian König if (i == p->ring || !sync_to_ring[i] || !p->rdev->ring[i].ready) 13893504fce28b1a387ec01f81b26637d237dca2b36Christian König continue; 13993504fce28b1a387ec01f81b26637d237dca2b36Christian König 14093504fce28b1a387ec01f81b26637d237dca2b36Christian König if (!p->ib->fence->semaphore) { 14193504fce28b1a387ec01f81b26637d237dca2b36Christian König r = radeon_semaphore_create(p->rdev, &p->ib->fence->semaphore); 14293504fce28b1a387ec01f81b26637d237dca2b36Christian König if (r) 14393504fce28b1a387ec01f81b26637d237dca2b36Christian König return r; 14493504fce28b1a387ec01f81b26637d237dca2b36Christian König } 14593504fce28b1a387ec01f81b26637d237dca2b36Christian König 14693504fce28b1a387ec01f81b26637d237dca2b36Christian König r = radeon_ring_lock(p->rdev, &p->rdev->ring[i], 3); 14793504fce28b1a387ec01f81b26637d237dca2b36Christian König if (r) 14893504fce28b1a387ec01f81b26637d237dca2b36Christian König return r; 14993504fce28b1a387ec01f81b26637d237dca2b36Christian König radeon_semaphore_emit_signal(p->rdev, i, p->ib->fence->semaphore); 15093504fce28b1a387ec01f81b26637d237dca2b36Christian König radeon_ring_unlock_commit(p->rdev, &p->rdev->ring[i]); 15193504fce28b1a387ec01f81b26637d237dca2b36Christian König 15293504fce28b1a387ec01f81b26637d237dca2b36Christian König r = radeon_ring_lock(p->rdev, &p->rdev->ring[p->ring], 3); 15393504fce28b1a387ec01f81b26637d237dca2b36Christian König if (r) 15493504fce28b1a387ec01f81b26637d237dca2b36Christian König return r; 15593504fce28b1a387ec01f81b26637d237dca2b36Christian König radeon_semaphore_emit_wait(p->rdev, p->ring, p->ib->fence->semaphore); 15693504fce28b1a387ec01f81b26637d237dca2b36Christian König radeon_ring_unlock_commit(p->rdev, &p->rdev->ring[p->ring]); 15793504fce28b1a387ec01f81b26637d237dca2b36Christian König } 15893504fce28b1a387ec01f81b26637d237dca2b36Christian König return 0; 15993504fce28b1a387ec01f81b26637d237dca2b36Christian König} 16093504fce28b1a387ec01f81b26637d237dca2b36Christian König 16144920b1991d9182bc3ff8f90df71b0e5311448dcAlex Deucher/* XXX: note that this is called from the legacy UMS CS ioctl as well */ 162771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisseint radeon_cs_parser_init(struct radeon_cs_parser *p, void *data) 163771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse{ 164771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse struct drm_radeon_cs *cs = data; 165771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse uint64_t *chunk_array_ptr; 166721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse unsigned size, i; 167721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse u32 ring = RADEON_CS_RING_GFX; 168721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse s32 priority = 0; 169771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 170771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse if (!cs->num_chunks) { 171771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return 0; 172771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 173771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse /* get chunks */ 174771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse INIT_LIST_HEAD(&p->validated); 175771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse p->idx = 0; 176771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse p->chunk_ib_idx = -1; 177771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse p->chunk_relocs_idx = -1; 178721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse p->chunk_flags_idx = -1; 179dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher p->chunk_const_ib_idx = -1; 180771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse p->chunks_array = kcalloc(cs->num_chunks, sizeof(uint64_t), GFP_KERNEL); 181771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse if (p->chunks_array == NULL) { 182771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return -ENOMEM; 183771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 184771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse chunk_array_ptr = (uint64_t *)(unsigned long)(cs->chunks); 185771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse if (DRM_COPY_FROM_USER(p->chunks_array, chunk_array_ptr, 186771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse sizeof(uint64_t)*cs->num_chunks)) { 187771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return -EFAULT; 188771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 189721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse p->cs_flags = 0; 190771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse p->nchunks = cs->num_chunks; 191771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse p->chunks = kcalloc(p->nchunks, sizeof(struct radeon_cs_chunk), GFP_KERNEL); 192771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse if (p->chunks == NULL) { 193771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return -ENOMEM; 194771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 195771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse for (i = 0; i < p->nchunks; i++) { 196771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse struct drm_radeon_cs_chunk __user **chunk_ptr = NULL; 197771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse struct drm_radeon_cs_chunk user_chunk; 198771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse uint32_t __user *cdata; 199771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 200771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse chunk_ptr = (void __user*)(unsigned long)p->chunks_array[i]; 201771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse if (DRM_COPY_FROM_USER(&user_chunk, chunk_ptr, 202771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse sizeof(struct drm_radeon_cs_chunk))) { 203771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return -EFAULT; 204771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 2055176fdc4c5873e52f9cb6e166d80e843847e7eb4Dave Airlie p->chunks[i].length_dw = user_chunk.length_dw; 2065176fdc4c5873e52f9cb6e166d80e843847e7eb4Dave Airlie p->chunks[i].kdata = NULL; 207771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse p->chunks[i].chunk_id = user_chunk.chunk_id; 2085176fdc4c5873e52f9cb6e166d80e843847e7eb4Dave Airlie 209771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_RELOCS) { 210771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse p->chunk_relocs_idx = i; 211771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 212771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_IB) { 213771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse p->chunk_ib_idx = i; 2145176fdc4c5873e52f9cb6e166d80e843847e7eb4Dave Airlie /* zero length IB isn't useful */ 2155176fdc4c5873e52f9cb6e166d80e843847e7eb4Dave Airlie if (p->chunks[i].length_dw == 0) 2165176fdc4c5873e52f9cb6e166d80e843847e7eb4Dave Airlie return -EINVAL; 217771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 218dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_CONST_IB) { 219dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher p->chunk_const_ib_idx = i; 220dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher /* zero length CONST IB isn't useful */ 221dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher if (p->chunks[i].length_dw == 0) 222dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher return -EINVAL; 223dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher } 224721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_FLAGS) { 225721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse p->chunk_flags_idx = i; 226721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse /* zero length flags aren't useful */ 227721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse if (p->chunks[i].length_dw == 0) 228721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse return -EINVAL; 229e70f224c1938af208b64b02c5cec27889fefcaecMarek Olšák } 2305176fdc4c5873e52f9cb6e166d80e843847e7eb4Dave Airlie 231771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse p->chunks[i].length_dw = user_chunk.length_dw; 232513bcb4655e68706594e45dfa1d4b181500110baDave Airlie p->chunks[i].user_ptr = (void __user *)(unsigned long)user_chunk.chunk_data; 233771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 234513bcb4655e68706594e45dfa1d4b181500110baDave Airlie cdata = (uint32_t *)(unsigned long)user_chunk.chunk_data; 235721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse if ((p->chunks[i].chunk_id == RADEON_CHUNK_ID_RELOCS) || 236721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse (p->chunks[i].chunk_id == RADEON_CHUNK_ID_FLAGS)) { 237513bcb4655e68706594e45dfa1d4b181500110baDave Airlie size = p->chunks[i].length_dw * sizeof(uint32_t); 238513bcb4655e68706594e45dfa1d4b181500110baDave Airlie p->chunks[i].kdata = kmalloc(size, GFP_KERNEL); 239513bcb4655e68706594e45dfa1d4b181500110baDave Airlie if (p->chunks[i].kdata == NULL) { 240513bcb4655e68706594e45dfa1d4b181500110baDave Airlie return -ENOMEM; 241513bcb4655e68706594e45dfa1d4b181500110baDave Airlie } 242513bcb4655e68706594e45dfa1d4b181500110baDave Airlie if (DRM_COPY_FROM_USER(p->chunks[i].kdata, 243513bcb4655e68706594e45dfa1d4b181500110baDave Airlie p->chunks[i].user_ptr, size)) { 244513bcb4655e68706594e45dfa1d4b181500110baDave Airlie return -EFAULT; 245513bcb4655e68706594e45dfa1d4b181500110baDave Airlie } 246e70f224c1938af208b64b02c5cec27889fefcaecMarek Olšák if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_FLAGS) { 247721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse p->cs_flags = p->chunks[i].kdata[0]; 248721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse if (p->chunks[i].length_dw > 1) 249721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse ring = p->chunks[i].kdata[1]; 250721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse if (p->chunks[i].length_dw > 2) 251721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse priority = (s32)p->chunks[i].kdata[2]; 252e70f224c1938af208b64b02c5cec27889fefcaecMarek Olšák } 253771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 254771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 255721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse 25644920b1991d9182bc3ff8f90df71b0e5311448dcAlex Deucher /* these are KMS only */ 25744920b1991d9182bc3ff8f90df71b0e5311448dcAlex Deucher if (p->rdev) { 25844920b1991d9182bc3ff8f90df71b0e5311448dcAlex Deucher if ((p->cs_flags & RADEON_CS_USE_VM) && 25944920b1991d9182bc3ff8f90df71b0e5311448dcAlex Deucher !p->rdev->vm_manager.enabled) { 26044920b1991d9182bc3ff8f90df71b0e5311448dcAlex Deucher DRM_ERROR("VM not active on asic!\n"); 26144920b1991d9182bc3ff8f90df71b0e5311448dcAlex Deucher return -EINVAL; 26244920b1991d9182bc3ff8f90df71b0e5311448dcAlex Deucher } 2631b5475dba7b19cace11c3f466ff7b7a58c23aab0Alex Deucher 26444920b1991d9182bc3ff8f90df71b0e5311448dcAlex Deucher /* we only support VM on SI+ */ 26544920b1991d9182bc3ff8f90df71b0e5311448dcAlex Deucher if ((p->rdev->family >= CHIP_TAHITI) && 26644920b1991d9182bc3ff8f90df71b0e5311448dcAlex Deucher ((p->cs_flags & RADEON_CS_USE_VM) == 0)) { 26744920b1991d9182bc3ff8f90df71b0e5311448dcAlex Deucher DRM_ERROR("VM required on SI+!\n"); 26844920b1991d9182bc3ff8f90df71b0e5311448dcAlex Deucher return -EINVAL; 26944920b1991d9182bc3ff8f90df71b0e5311448dcAlex Deucher } 270721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse 27144920b1991d9182bc3ff8f90df71b0e5311448dcAlex Deucher if (radeon_cs_get_ring(p, ring, priority)) 27244920b1991d9182bc3ff8f90df71b0e5311448dcAlex Deucher return -EINVAL; 27344920b1991d9182bc3ff8f90df71b0e5311448dcAlex Deucher } 274721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse 275721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse /* deal with non-vm */ 276721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse if ((p->chunk_ib_idx != -1) && 277721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse ((p->cs_flags & RADEON_CS_USE_VM) == 0) && 278721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse (p->chunks[p->chunk_ib_idx].chunk_id == RADEON_CHUNK_ID_IB)) { 279721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse if (p->chunks[p->chunk_ib_idx].length_dw > (16 * 1024)) { 280721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse DRM_ERROR("cs IB too big: %d\n", 281721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse p->chunks[p->chunk_ib_idx].length_dw); 282721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse return -EINVAL; 283721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse } 284721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse p->chunks[p->chunk_ib_idx].kpage[0] = kmalloc(PAGE_SIZE, GFP_KERNEL); 285721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse p->chunks[p->chunk_ib_idx].kpage[1] = kmalloc(PAGE_SIZE, GFP_KERNEL); 286721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse if (p->chunks[p->chunk_ib_idx].kpage[0] == NULL || 287f48bb04a56fd1f582a45f3882255477417bc0ed7Julia Lawall p->chunks[p->chunk_ib_idx].kpage[1] == NULL) 288721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse return -ENOMEM; 289721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse p->chunks[p->chunk_ib_idx].kpage_idx[0] = -1; 290721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse p->chunks[p->chunk_ib_idx].kpage_idx[1] = -1; 291721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse p->chunks[p->chunk_ib_idx].last_copied_page = -1; 292721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse p->chunks[p->chunk_ib_idx].last_page_index = 293721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse ((p->chunks[p->chunk_ib_idx].length_dw * 4) - 1) / PAGE_SIZE; 294721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse } 295721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse 296771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return 0; 297771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse} 298771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 299771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse/** 300771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * cs_parser_fini() - clean parser states 301771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * @parser: parser structure holding parsing context. 302771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * @error: error number 303771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * 304771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * If error is set than unvalidate buffer, otherwise just free memory 305771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * used by parsing context. 306771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse **/ 307771fe6b912fca54f03e8a72eb63058b582775362Jerome Glissestatic void radeon_cs_parser_fini(struct radeon_cs_parser *parser, int error) 308771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse{ 309771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse unsigned i; 310771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 311147666fb3b93b8c484f562da33a37f886ddff768Thomas Hellstrom 312147666fb3b93b8c484f562da33a37f886ddff768Thomas Hellstrom if (!error && parser->ib) 313147666fb3b93b8c484f562da33a37f886ddff768Thomas Hellstrom ttm_eu_fence_buffer_objects(&parser->validated, 314147666fb3b93b8c484f562da33a37f886ddff768Thomas Hellstrom parser->ib->fence); 315147666fb3b93b8c484f562da33a37f886ddff768Thomas Hellstrom else 316147666fb3b93b8c484f562da33a37f886ddff768Thomas Hellstrom ttm_eu_backoff_reservation(&parser->validated); 317147666fb3b93b8c484f562da33a37f886ddff768Thomas Hellstrom 318fcbc451ba1948fba967198bd150ecbd10bbb7075Pauli Nieminen if (parser->relocs != NULL) { 319fcbc451ba1948fba967198bd150ecbd10bbb7075Pauli Nieminen for (i = 0; i < parser->nrelocs; i++) { 320fcbc451ba1948fba967198bd150ecbd10bbb7075Pauli Nieminen if (parser->relocs[i].gobj) 321fcbc451ba1948fba967198bd150ecbd10bbb7075Pauli Nieminen drm_gem_object_unreference_unlocked(parser->relocs[i].gobj); 322fcbc451ba1948fba967198bd150ecbd10bbb7075Pauli Nieminen } 323771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 32448e113e516a94ac9622a8326c4de670beb3366e6Michel Dänzer kfree(parser->track); 325771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse kfree(parser->relocs); 326771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse kfree(parser->relocs_ptr); 327771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse for (i = 0; i < parser->nchunks; i++) { 328771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse kfree(parser->chunks[i].kdata); 329513bcb4655e68706594e45dfa1d4b181500110baDave Airlie kfree(parser->chunks[i].kpage[0]); 330513bcb4655e68706594e45dfa1d4b181500110baDave Airlie kfree(parser->chunks[i].kpage[1]); 331771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 332771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse kfree(parser->chunks); 333771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse kfree(parser->chunks_array); 334771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse radeon_ib_free(parser->rdev, &parser->ib); 335771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse} 336771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 337721604a15b934f0a8d1909acb8017f029128be2fJerome Glissestatic int radeon_cs_ib_chunk(struct radeon_device *rdev, 338721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse struct radeon_cs_parser *parser) 339721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse{ 340721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse struct radeon_cs_chunk *ib_chunk; 341721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse int r; 342721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse 343721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse if (parser->chunk_ib_idx == -1) 344721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse return 0; 345721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse 346721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse if (parser->cs_flags & RADEON_CS_USE_VM) 347721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse return 0; 348721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse 349721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse ib_chunk = &parser->chunks[parser->chunk_ib_idx]; 350721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse /* Copy the packet into the IB, the parser will read from the 351721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse * input memory (cached) and write to the IB (which can be 352721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse * uncached). 353721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse */ 354721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse r = radeon_ib_get(rdev, parser->ring, &parser->ib, 355721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse ib_chunk->length_dw * 4); 356721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse if (r) { 357721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse DRM_ERROR("Failed to get ib !\n"); 358721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse return r; 359721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse } 360721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse parser->ib->length_dw = ib_chunk->length_dw; 361eb0c19c539f72d94c33cfd0860f4f8c899c64f4dChristian König r = radeon_cs_parse(rdev, parser->ring, parser); 362721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse if (r || parser->parser_error) { 363721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse DRM_ERROR("Invalid command stream !\n"); 364721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse return r; 365721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse } 366721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse r = radeon_cs_finish_pages(parser); 367721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse if (r) { 368721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse DRM_ERROR("Invalid command stream !\n"); 369721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse return r; 370721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse } 37193504fce28b1a387ec01f81b26637d237dca2b36Christian König r = radeon_cs_sync_rings(parser); 37293504fce28b1a387ec01f81b26637d237dca2b36Christian König if (r) { 37393504fce28b1a387ec01f81b26637d237dca2b36Christian König DRM_ERROR("Failed to synchronize rings !\n"); 37493504fce28b1a387ec01f81b26637d237dca2b36Christian König } 375721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse parser->ib->vm_id = 0; 376721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse r = radeon_ib_schedule(rdev, parser->ib); 377721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse if (r) { 378721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse DRM_ERROR("Failed to schedule IB !\n"); 379721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse } 380721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse return 0; 381721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse} 382721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse 383721604a15b934f0a8d1909acb8017f029128be2fJerome Glissestatic int radeon_bo_vm_update_pte(struct radeon_cs_parser *parser, 384721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse struct radeon_vm *vm) 385721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse{ 386721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse struct radeon_bo_list *lobj; 387721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse struct radeon_bo *bo; 388721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse int r; 389721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse 390721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse list_for_each_entry(lobj, &parser->validated, tv.head) { 391721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse bo = lobj->bo; 392721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse r = radeon_vm_bo_update_pte(parser->rdev, vm, bo, &bo->tbo.mem); 393721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse if (r) { 394721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse return r; 395721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse } 396721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse } 397721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse return 0; 398721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse} 399721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse 400721604a15b934f0a8d1909acb8017f029128be2fJerome Glissestatic int radeon_cs_ib_vm_chunk(struct radeon_device *rdev, 401721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse struct radeon_cs_parser *parser) 402721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse{ 403721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse struct radeon_cs_chunk *ib_chunk; 404721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse struct radeon_fpriv *fpriv = parser->filp->driver_priv; 405721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse struct radeon_vm *vm = &fpriv->vm; 406721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse int r; 407721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse 408721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse if (parser->chunk_ib_idx == -1) 409721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse return 0; 410721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse 411721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse if ((parser->cs_flags & RADEON_CS_USE_VM) == 0) 412721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse return 0; 413721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse 414dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher if ((rdev->family >= CHIP_TAHITI) && 415dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher (parser->chunk_const_ib_idx != -1)) { 416dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher ib_chunk = &parser->chunks[parser->chunk_const_ib_idx]; 417dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher if (ib_chunk->length_dw > RADEON_IB_VM_MAX_SIZE) { 418dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher DRM_ERROR("cs IB CONST too big: %d\n", ib_chunk->length_dw); 419dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher return -EINVAL; 420dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher } 421dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher r = radeon_ib_get(rdev, parser->ring, &parser->const_ib, 422dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher ib_chunk->length_dw * 4); 423dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher if (r) { 424dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher DRM_ERROR("Failed to get const ib !\n"); 425dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher return r; 426dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher } 427dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher parser->const_ib->is_const_ib = true; 428dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher parser->const_ib->length_dw = ib_chunk->length_dw; 429dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher /* Copy the packet into the IB */ 430dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher if (DRM_COPY_FROM_USER(parser->const_ib->ptr, ib_chunk->user_ptr, 431dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher ib_chunk->length_dw * 4)) { 432dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher return -EFAULT; 433dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher } 434dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher r = radeon_ring_ib_parse(rdev, parser->ring, parser->const_ib); 435dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher if (r) { 436dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher return r; 437dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher } 438dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher } 439dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher 440721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse ib_chunk = &parser->chunks[parser->chunk_ib_idx]; 441721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse if (ib_chunk->length_dw > RADEON_IB_VM_MAX_SIZE) { 442721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse DRM_ERROR("cs IB too big: %d\n", ib_chunk->length_dw); 443721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse return -EINVAL; 444721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse } 445721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse r = radeon_ib_get(rdev, parser->ring, &parser->ib, 446721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse ib_chunk->length_dw * 4); 447721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse if (r) { 448721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse DRM_ERROR("Failed to get ib !\n"); 449721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse return r; 450721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse } 451721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse parser->ib->length_dw = ib_chunk->length_dw; 452721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse /* Copy the packet into the IB */ 453721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse if (DRM_COPY_FROM_USER(parser->ib->ptr, ib_chunk->user_ptr, 454721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse ib_chunk->length_dw * 4)) { 455721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse return -EFAULT; 456721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse } 457721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse r = radeon_ring_ib_parse(rdev, parser->ring, parser->ib); 458721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse if (r) { 459721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse return r; 460721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse } 461721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse 462721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse mutex_lock(&vm->mutex); 463721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse r = radeon_vm_bind(rdev, vm); 464721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse if (r) { 465721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse goto out; 466721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse } 467721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse r = radeon_bo_vm_update_pte(parser, vm); 468721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse if (r) { 469721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse goto out; 470721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse } 47193504fce28b1a387ec01f81b26637d237dca2b36Christian König r = radeon_cs_sync_rings(parser); 47293504fce28b1a387ec01f81b26637d237dca2b36Christian König if (r) { 47393504fce28b1a387ec01f81b26637d237dca2b36Christian König DRM_ERROR("Failed to synchronize rings !\n"); 47493504fce28b1a387ec01f81b26637d237dca2b36Christian König } 475dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher 476dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher if ((rdev->family >= CHIP_TAHITI) && 477dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher (parser->chunk_const_ib_idx != -1)) { 478dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher parser->const_ib->vm_id = vm->id; 479dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher /* ib pool is bind at 0 in virtual address space to gpu_addr is the 480dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher * offset inside the pool bo 481dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher */ 482dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher parser->const_ib->gpu_addr = parser->const_ib->sa_bo.offset; 483dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher r = radeon_ib_schedule(rdev, parser->const_ib); 484dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher if (r) 485dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher goto out; 486dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher } 487dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher 488721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse parser->ib->vm_id = vm->id; 489721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse /* ib pool is bind at 0 in virtual address space to gpu_addr is the 490721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse * offset inside the pool bo 491721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse */ 492721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse parser->ib->gpu_addr = parser->ib->sa_bo.offset; 493dfcf5f36529d69eb35f4fdedfa6f244c5249698cAlex Deucher parser->ib->is_const_ib = false; 494721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse r = radeon_ib_schedule(rdev, parser->ib); 495721604a15b934f0a8d1909acb8017f029128be2fJerome Glisseout: 496721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse if (!r) { 497721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse if (vm->fence) { 498721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse radeon_fence_unref(&vm->fence); 499721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse } 500721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse vm->fence = radeon_fence_ref(parser->ib->fence); 501721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse } 502721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse mutex_unlock(&fpriv->vm.mutex); 503721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse return r; 504721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse} 505721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse 506771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisseint radeon_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 507771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse{ 508771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse struct radeon_device *rdev = dev->dev_private; 509771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse struct radeon_cs_parser parser; 510771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse int r; 511771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 5127a1619b97e978bb9c05fa4bbe64171068bd5bf85Michel Dänzer radeon_mutex_lock(&rdev->cs_mutex); 5136b7746e8768e1c550b320d5af761f73e5aa37f76Jerome Glisse if (!rdev->accel_working) { 5146b7746e8768e1c550b320d5af761f73e5aa37f76Jerome Glisse radeon_mutex_unlock(&rdev->cs_mutex); 5156b7746e8768e1c550b320d5af761f73e5aa37f76Jerome Glisse return -EBUSY; 5166b7746e8768e1c550b320d5af761f73e5aa37f76Jerome Glisse } 517771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse /* initialize parser */ 518771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse memset(&parser, 0, sizeof(struct radeon_cs_parser)); 519771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse parser.filp = filp; 520771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse parser.rdev = rdev; 521c8c15ff1e90bfc4a2db1ba77a01b3b2783e723fcJerome Glisse parser.dev = rdev->dev; 522428c6e3630b343d5b3034af914015d10c94bb90eDave Airlie parser.family = rdev->family; 523771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse r = radeon_cs_parser_init(&parser, data); 524771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse if (r) { 525771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse DRM_ERROR("Failed to initialize parser !\n"); 526771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse radeon_cs_parser_fini(&parser, r); 5277a1619b97e978bb9c05fa4bbe64171068bd5bf85Michel Dänzer radeon_mutex_unlock(&rdev->cs_mutex); 528771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return r; 529771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 530771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse r = radeon_cs_parser_relocs(&parser); 531771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse if (r) { 53297f23b3d85a4d734a8584dade3a34579931c8f8dDave Airlie if (r != -ERESTARTSYS) 53397f23b3d85a4d734a8584dade3a34579931c8f8dDave Airlie DRM_ERROR("Failed to parse relocation %d!\n", r); 534771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse radeon_cs_parser_fini(&parser, r); 5357a1619b97e978bb9c05fa4bbe64171068bd5bf85Michel Dänzer radeon_mutex_unlock(&rdev->cs_mutex); 536771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return r; 537771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 538721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse r = radeon_cs_ib_chunk(rdev, &parser); 539771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse if (r) { 540721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse goto out; 541771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 542721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse r = radeon_cs_ib_vm_chunk(rdev, &parser); 543771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse if (r) { 544721604a15b934f0a8d1909acb8017f029128be2fJerome Glisse goto out; 545771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 546721604a15b934f0a8d1909acb8017f029128be2fJerome Glisseout: 547771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse radeon_cs_parser_fini(&parser, r); 5487a1619b97e978bb9c05fa4bbe64171068bd5bf85Michel Dänzer radeon_mutex_unlock(&rdev->cs_mutex); 549771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return r; 550771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse} 551513bcb4655e68706594e45dfa1d4b181500110baDave Airlie 552513bcb4655e68706594e45dfa1d4b181500110baDave Airlieint radeon_cs_finish_pages(struct radeon_cs_parser *p) 553513bcb4655e68706594e45dfa1d4b181500110baDave Airlie{ 554513bcb4655e68706594e45dfa1d4b181500110baDave Airlie struct radeon_cs_chunk *ibc = &p->chunks[p->chunk_ib_idx]; 555513bcb4655e68706594e45dfa1d4b181500110baDave Airlie int i; 556513bcb4655e68706594e45dfa1d4b181500110baDave Airlie int size = PAGE_SIZE; 557513bcb4655e68706594e45dfa1d4b181500110baDave Airlie 558513bcb4655e68706594e45dfa1d4b181500110baDave Airlie for (i = ibc->last_copied_page + 1; i <= ibc->last_page_index; i++) { 559513bcb4655e68706594e45dfa1d4b181500110baDave Airlie if (i == ibc->last_page_index) { 560513bcb4655e68706594e45dfa1d4b181500110baDave Airlie size = (ibc->length_dw * 4) % PAGE_SIZE; 561513bcb4655e68706594e45dfa1d4b181500110baDave Airlie if (size == 0) 562513bcb4655e68706594e45dfa1d4b181500110baDave Airlie size = PAGE_SIZE; 563513bcb4655e68706594e45dfa1d4b181500110baDave Airlie } 564513bcb4655e68706594e45dfa1d4b181500110baDave Airlie 565513bcb4655e68706594e45dfa1d4b181500110baDave Airlie if (DRM_COPY_FROM_USER(p->ib->ptr + (i * (PAGE_SIZE/4)), 566513bcb4655e68706594e45dfa1d4b181500110baDave Airlie ibc->user_ptr + (i * PAGE_SIZE), 567513bcb4655e68706594e45dfa1d4b181500110baDave Airlie size)) 568513bcb4655e68706594e45dfa1d4b181500110baDave Airlie return -EFAULT; 569513bcb4655e68706594e45dfa1d4b181500110baDave Airlie } 570513bcb4655e68706594e45dfa1d4b181500110baDave Airlie return 0; 571513bcb4655e68706594e45dfa1d4b181500110baDave Airlie} 572513bcb4655e68706594e45dfa1d4b181500110baDave Airlie 573513bcb4655e68706594e45dfa1d4b181500110baDave Airlieint radeon_cs_update_pages(struct radeon_cs_parser *p, int pg_idx) 574513bcb4655e68706594e45dfa1d4b181500110baDave Airlie{ 575513bcb4655e68706594e45dfa1d4b181500110baDave Airlie int new_page; 576513bcb4655e68706594e45dfa1d4b181500110baDave Airlie struct radeon_cs_chunk *ibc = &p->chunks[p->chunk_ib_idx]; 577513bcb4655e68706594e45dfa1d4b181500110baDave Airlie int i; 578513bcb4655e68706594e45dfa1d4b181500110baDave Airlie int size = PAGE_SIZE; 579513bcb4655e68706594e45dfa1d4b181500110baDave Airlie 580c5e617e2f84225a28823a3e19951273b9f59eb27Dave Airlie for (i = ibc->last_copied_page + 1; i < pg_idx; i++) { 581513bcb4655e68706594e45dfa1d4b181500110baDave Airlie if (DRM_COPY_FROM_USER(p->ib->ptr + (i * (PAGE_SIZE/4)), 582513bcb4655e68706594e45dfa1d4b181500110baDave Airlie ibc->user_ptr + (i * PAGE_SIZE), 583513bcb4655e68706594e45dfa1d4b181500110baDave Airlie PAGE_SIZE)) { 584513bcb4655e68706594e45dfa1d4b181500110baDave Airlie p->parser_error = -EFAULT; 585513bcb4655e68706594e45dfa1d4b181500110baDave Airlie return 0; 586513bcb4655e68706594e45dfa1d4b181500110baDave Airlie } 587513bcb4655e68706594e45dfa1d4b181500110baDave Airlie } 588513bcb4655e68706594e45dfa1d4b181500110baDave Airlie 589513bcb4655e68706594e45dfa1d4b181500110baDave Airlie new_page = ibc->kpage_idx[0] < ibc->kpage_idx[1] ? 0 : 1; 590513bcb4655e68706594e45dfa1d4b181500110baDave Airlie 591513bcb4655e68706594e45dfa1d4b181500110baDave Airlie if (pg_idx == ibc->last_page_index) { 592513bcb4655e68706594e45dfa1d4b181500110baDave Airlie size = (ibc->length_dw * 4) % PAGE_SIZE; 593513bcb4655e68706594e45dfa1d4b181500110baDave Airlie if (size == 0) 594513bcb4655e68706594e45dfa1d4b181500110baDave Airlie size = PAGE_SIZE; 595513bcb4655e68706594e45dfa1d4b181500110baDave Airlie } 596513bcb4655e68706594e45dfa1d4b181500110baDave Airlie 597513bcb4655e68706594e45dfa1d4b181500110baDave Airlie if (DRM_COPY_FROM_USER(ibc->kpage[new_page], 598513bcb4655e68706594e45dfa1d4b181500110baDave Airlie ibc->user_ptr + (pg_idx * PAGE_SIZE), 599513bcb4655e68706594e45dfa1d4b181500110baDave Airlie size)) { 600513bcb4655e68706594e45dfa1d4b181500110baDave Airlie p->parser_error = -EFAULT; 601513bcb4655e68706594e45dfa1d4b181500110baDave Airlie return 0; 602513bcb4655e68706594e45dfa1d4b181500110baDave Airlie } 603513bcb4655e68706594e45dfa1d4b181500110baDave Airlie 604513bcb4655e68706594e45dfa1d4b181500110baDave Airlie /* copy to IB here */ 605513bcb4655e68706594e45dfa1d4b181500110baDave Airlie memcpy((void *)(p->ib->ptr+(pg_idx*(PAGE_SIZE/4))), ibc->kpage[new_page], size); 606513bcb4655e68706594e45dfa1d4b181500110baDave Airlie 607513bcb4655e68706594e45dfa1d4b181500110baDave Airlie ibc->last_copied_page = pg_idx; 608513bcb4655e68706594e45dfa1d4b181500110baDave Airlie ibc->kpage_idx[new_page] = pg_idx; 609513bcb4655e68706594e45dfa1d4b181500110baDave Airlie 610513bcb4655e68706594e45dfa1d4b181500110baDave Airlie return new_page; 611513bcb4655e68706594e45dfa1d4b181500110baDave Airlie} 612