1771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse/* 2771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * Copyright 2009 Jerome Glisse. 3771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * All Rights Reserved. 4771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * 5771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * Permission is hereby granted, free of charge, to any person obtaining a 6771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * copy of this software and associated documentation files (the 7771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * "Software"), to deal in the Software without restriction, including 8771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * without limitation the rights to use, copy, modify, merge, publish, 9771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * distribute, sub license, and/or sell copies of the Software, and to 10771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * permit persons to whom the Software is furnished to do so, subject to 11771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * the following conditions: 12771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * 13771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 16771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 17771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 18771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 19771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * USE OR OTHER DEALINGS IN THE SOFTWARE. 20771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * 21771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * The above copyright notice and this permission notice (including the 22771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * next paragraph) shall be included in all copies or substantial portions 23771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * of the Software. 24771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * 25771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse */ 26771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse/* 27771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * Authors: 28771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * Jerome Glisse <glisse@freedesktop.org> 29771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * Dave Airlie 30771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse */ 31771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse#include <linux/seq_file.h> 3260063497a95e716c9a689af3be2687d261f115b4Arun Sharma#include <linux/atomic.h> 33771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse#include <linux/wait.h> 34771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse#include <linux/kref.h> 355a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/slab.h> 36f2ba57b5eab8817d86d0f108fdf1878e51dc0a37Christian König#include <linux/firmware.h> 37760285e7e7ab282c25b5e90816f7c47000557f4fDavid Howells#include <drm/drmP.h> 38771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse#include "radeon_reg.h" 39771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse#include "radeon.h" 4099ee7fac189893c90145a22b86bbcfdc98f69a9cDave Airlie#include "radeon_trace.h" 41771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 42d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher/* 43d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * Fences 44d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * Fences mark an event in the GPUs pipeline and are used 45d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * for GPU/CPU synchronization. When the fence is written, 46d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * it is expected that all buffers associated with that fence 47d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * are no longer in use by the associated ring on the GPU and 48d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * that the the relevant GPU caches have been flushed. Whether 49d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * we use a scratch register or memory location depends on the asic 50d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * and whether writeback is enabled. 51d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher */ 52d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher 53d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher/** 54d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * radeon_fence_write - write a fence value 55d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * 56d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * @rdev: radeon_device pointer 57d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * @seq: sequence number to write 58d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * @ring: ring index the fence is associated with 59d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * 60d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * Writes a fence value to memory or a scratch register (all asics). 61d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher */ 627465280c076d6440e5908c158c83b542dc063a30Alex Deucherstatic void radeon_fence_write(struct radeon_device *rdev, u32 seq, int ring) 63b81157d016a48b8025ccfcb286827679b35f16aaAlex Deucher{ 64bf66625e02ac8e4535627394bcfb237bbf8a157fChristian König struct radeon_fence_driver *drv = &rdev->fence_drv[ring]; 65bf66625e02ac8e4535627394bcfb237bbf8a157fChristian König if (likely(rdev->wb.enabled || !drv->scratch_reg)) { 66089920f21db0108fb105ecfd81de4c92d88f06d0Jerome Glisse if (drv->cpu_addr) { 67089920f21db0108fb105ecfd81de4c92d88f06d0Jerome Glisse *drv->cpu_addr = cpu_to_le32(seq); 68089920f21db0108fb105ecfd81de4c92d88f06d0Jerome Glisse } 6930eb77f4e6ba20f797af4ff79807fae7cb67429eJerome Glisse } else { 70bf66625e02ac8e4535627394bcfb237bbf8a157fChristian König WREG32(drv->scratch_reg, seq); 7130eb77f4e6ba20f797af4ff79807fae7cb67429eJerome Glisse } 72b81157d016a48b8025ccfcb286827679b35f16aaAlex Deucher} 73b81157d016a48b8025ccfcb286827679b35f16aaAlex Deucher 74d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher/** 75d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * radeon_fence_read - read a fence value 76d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * 77d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * @rdev: radeon_device pointer 78d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * @ring: ring index the fence is associated with 79d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * 80d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * Reads a fence value from memory or a scratch register (all asics). 81d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * Returns the value of the fence read from memory or register. 82d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher */ 837465280c076d6440e5908c158c83b542dc063a30Alex Deucherstatic u32 radeon_fence_read(struct radeon_device *rdev, int ring) 84b81157d016a48b8025ccfcb286827679b35f16aaAlex Deucher{ 85bf66625e02ac8e4535627394bcfb237bbf8a157fChristian König struct radeon_fence_driver *drv = &rdev->fence_drv[ring]; 867465280c076d6440e5908c158c83b542dc063a30Alex Deucher u32 seq = 0; 87b81157d016a48b8025ccfcb286827679b35f16aaAlex Deucher 88bf66625e02ac8e4535627394bcfb237bbf8a157fChristian König if (likely(rdev->wb.enabled || !drv->scratch_reg)) { 89089920f21db0108fb105ecfd81de4c92d88f06d0Jerome Glisse if (drv->cpu_addr) { 90089920f21db0108fb105ecfd81de4c92d88f06d0Jerome Glisse seq = le32_to_cpu(*drv->cpu_addr); 91089920f21db0108fb105ecfd81de4c92d88f06d0Jerome Glisse } else { 92089920f21db0108fb105ecfd81de4c92d88f06d0Jerome Glisse seq = lower_32_bits(atomic64_read(&drv->last_seq)); 93089920f21db0108fb105ecfd81de4c92d88f06d0Jerome Glisse } 9430eb77f4e6ba20f797af4ff79807fae7cb67429eJerome Glisse } else { 95bf66625e02ac8e4535627394bcfb237bbf8a157fChristian König seq = RREG32(drv->scratch_reg); 9630eb77f4e6ba20f797af4ff79807fae7cb67429eJerome Glisse } 97b81157d016a48b8025ccfcb286827679b35f16aaAlex Deucher return seq; 98b81157d016a48b8025ccfcb286827679b35f16aaAlex Deucher} 99b81157d016a48b8025ccfcb286827679b35f16aaAlex Deucher 100d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher/** 1010bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König * radeon_fence_schedule_check - schedule lockup check 1020bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König * 1030bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König * @rdev: radeon_device pointer 1040bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König * @ring: ring index we should work with 1050bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König * 1060bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König * Queues a delayed work item to check for lockups. 1070bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König */ 1080bfa4b41268ad5fd741f16f484e4fee190822ec6Christian Königstatic void radeon_fence_schedule_check(struct radeon_device *rdev, int ring) 1090bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König{ 1100bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König /* 1110bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König * Do not reset the timer here with mod_delayed_work, 1120bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König * this can livelock in an interaction with TTM delayed destroy. 1130bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König */ 1140bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König queue_delayed_work(system_power_efficient_wq, 1150bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König &rdev->fence_drv[ring].lockup_work, 1160bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König RADEON_FENCE_JIFFIES_TIMEOUT); 1170bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König} 1180bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König 1190bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König/** 120d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * radeon_fence_emit - emit a fence on the requested ring 121d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * 122d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * @rdev: radeon_device pointer 123d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * @fence: radeon fence object 124d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * @ring: ring index the fence is associated with 125d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * 126d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * Emits a fence command on the requested ring (all asics). 127d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * Returns 0 on success, -ENOMEM on failure. 128d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher */ 129876dc9f32907e57e0298bcd0f1607cb7a2582f63Christian Königint radeon_fence_emit(struct radeon_device *rdev, 130876dc9f32907e57e0298bcd0f1607cb7a2582f63Christian König struct radeon_fence **fence, 131876dc9f32907e57e0298bcd0f1607cb7a2582f63Christian König int ring) 132771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse{ 133954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst u64 seq = ++rdev->fence_drv[ring].sync_seq[ring]; 134954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst 1353b7a2b24ea2b703b3af595d0d4ee233ab0b36377Jerome Glisse /* we are protected by the ring emission mutex */ 136876dc9f32907e57e0298bcd0f1607cb7a2582f63Christian König *fence = kmalloc(sizeof(struct radeon_fence), GFP_KERNEL); 137876dc9f32907e57e0298bcd0f1607cb7a2582f63Christian König if ((*fence) == NULL) { 138876dc9f32907e57e0298bcd0f1607cb7a2582f63Christian König return -ENOMEM; 139771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 140876dc9f32907e57e0298bcd0f1607cb7a2582f63Christian König (*fence)->rdev = rdev; 141954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst (*fence)->seq = seq; 142876dc9f32907e57e0298bcd0f1607cb7a2582f63Christian König (*fence)->ring = ring; 143954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst fence_init(&(*fence)->base, &radeon_fence_ops, 144954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst &rdev->fence_queue.lock, rdev->fence_context + ring, seq); 145876dc9f32907e57e0298bcd0f1607cb7a2582f63Christian König radeon_fence_ring_emit(rdev, ring, *fence); 1461d7841676676691c205bfcb3d99c5997466a3408Christian König trace_radeon_fence_emit(rdev->ddev, ring, (*fence)->seq); 1470bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König radeon_fence_schedule_check(rdev, ring); 148771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return 0; 149771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse} 150771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 151d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher/** 152954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst * radeon_fence_check_signaled - callback from fence_queue 153954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst * 154954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst * this function is called with fence_queue lock held, which is also used 155954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst * for the fence locking itself, so unlocked variants are used for 156954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst * fence_signal, and remove_wait_queue. 157954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst */ 158954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorststatic int radeon_fence_check_signaled(wait_queue_t *wait, unsigned mode, int flags, void *key) 159954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst{ 160954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst struct radeon_fence *fence; 161954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst u64 seq; 162954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst 163954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst fence = container_of(wait, struct radeon_fence, fence_wake); 164954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst 165954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst /* 166954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst * We cannot use radeon_fence_process here because we're already 167954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst * in the waitqueue, in a call from wake_up_all. 168954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst */ 169954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst seq = atomic64_read(&fence->rdev->fence_drv[fence->ring].last_seq); 170954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst if (seq >= fence->seq) { 171954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst int ret = fence_signal_locked(&fence->base); 172954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst 173954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst if (!ret) 174954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst FENCE_TRACE(&fence->base, "signaled from irq context\n"); 175954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst else 176954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst FENCE_TRACE(&fence->base, "was already signaled\n"); 177954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst 178954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst radeon_irq_kms_sw_irq_put(fence->rdev, fence->ring); 179954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst __remove_wait_queue(&fence->rdev->fence_queue, &fence->fence_wake); 180954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst fence_put(&fence->base); 181954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst } else 182954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst FENCE_TRACE(&fence->base, "pending\n"); 183954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst return 0; 184954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst} 185954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst 186954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst/** 1870bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König * radeon_fence_activity - check for fence activity 188d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * 189d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * @rdev: radeon_device pointer 190d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * @ring: ring index the fence is associated with 191d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * 1920bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König * Checks the current fence value and calculates the last 1930bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König * signalled fence value. Returns true if activity occured 1940bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König * on the ring, and the fence_queue should be waken up. 195d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher */ 1960bfa4b41268ad5fd741f16f484e4fee190822ec6Christian Königstatic bool radeon_fence_activity(struct radeon_device *rdev, int ring) 197771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse{ 198f492c171a38d77fc13a8998a0721f2da50835224Christian König uint64_t seq, last_seq, last_emitted; 199bb635567291482a87e4cc46e6683419c1f365ddfJerome Glisse unsigned count_loop = 0; 200771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse bool wake = false; 201771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 202bb635567291482a87e4cc46e6683419c1f365ddfJerome Glisse /* Note there is a scenario here for an infinite loop but it's 203bb635567291482a87e4cc46e6683419c1f365ddfJerome Glisse * very unlikely to happen. For it to happen, the current polling 204bb635567291482a87e4cc46e6683419c1f365ddfJerome Glisse * process need to be interrupted by another process and another 205bb635567291482a87e4cc46e6683419c1f365ddfJerome Glisse * process needs to update the last_seq btw the atomic read and 206bb635567291482a87e4cc46e6683419c1f365ddfJerome Glisse * xchg of the current process. 207bb635567291482a87e4cc46e6683419c1f365ddfJerome Glisse * 208bb635567291482a87e4cc46e6683419c1f365ddfJerome Glisse * More over for this to go in infinite loop there need to be 209bb635567291482a87e4cc46e6683419c1f365ddfJerome Glisse * continuously new fence signaled ie radeon_fence_read needs 210bb635567291482a87e4cc46e6683419c1f365ddfJerome Glisse * to return a different value each time for both the currently 211bb635567291482a87e4cc46e6683419c1f365ddfJerome Glisse * polling process and the other process that xchg the last_seq 212bb635567291482a87e4cc46e6683419c1f365ddfJerome Glisse * btw atomic read and xchg of the current process. And the 213bb635567291482a87e4cc46e6683419c1f365ddfJerome Glisse * value the other process set as last seq must be higher than 214bb635567291482a87e4cc46e6683419c1f365ddfJerome Glisse * the seq value we just read. Which means that current process 215bb635567291482a87e4cc46e6683419c1f365ddfJerome Glisse * need to be interrupted after radeon_fence_read and before 216bb635567291482a87e4cc46e6683419c1f365ddfJerome Glisse * atomic xchg. 217bb635567291482a87e4cc46e6683419c1f365ddfJerome Glisse * 218bb635567291482a87e4cc46e6683419c1f365ddfJerome Glisse * To be even more safe we count the number of time we loop and 219bb635567291482a87e4cc46e6683419c1f365ddfJerome Glisse * we bail after 10 loop just accepting the fact that we might 220bb635567291482a87e4cc46e6683419c1f365ddfJerome Glisse * have temporarly set the last_seq not to the true real last 221bb635567291482a87e4cc46e6683419c1f365ddfJerome Glisse * seq but to an older one. 222bb635567291482a87e4cc46e6683419c1f365ddfJerome Glisse */ 223bb635567291482a87e4cc46e6683419c1f365ddfJerome Glisse last_seq = atomic64_read(&rdev->fence_drv[ring].last_seq); 224bb635567291482a87e4cc46e6683419c1f365ddfJerome Glisse do { 225f492c171a38d77fc13a8998a0721f2da50835224Christian König last_emitted = rdev->fence_drv[ring].sync_seq[ring]; 226bb635567291482a87e4cc46e6683419c1f365ddfJerome Glisse seq = radeon_fence_read(rdev, ring); 227bb635567291482a87e4cc46e6683419c1f365ddfJerome Glisse seq |= last_seq & 0xffffffff00000000LL; 228bb635567291482a87e4cc46e6683419c1f365ddfJerome Glisse if (seq < last_seq) { 229f492c171a38d77fc13a8998a0721f2da50835224Christian König seq &= 0xffffffff; 230f492c171a38d77fc13a8998a0721f2da50835224Christian König seq |= last_emitted & 0xffffffff00000000LL; 231bb635567291482a87e4cc46e6683419c1f365ddfJerome Glisse } 23236abacaed34bc1f5bcb11ca611dd3a06c5c0ef39Christian König 233f492c171a38d77fc13a8998a0721f2da50835224Christian König if (seq <= last_seq || seq > last_emitted) { 2343b7a2b24ea2b703b3af595d0d4ee233ab0b36377Jerome Glisse break; 235bb635567291482a87e4cc46e6683419c1f365ddfJerome Glisse } 236bb635567291482a87e4cc46e6683419c1f365ddfJerome Glisse /* If we loop over we don't want to return without 237bb635567291482a87e4cc46e6683419c1f365ddfJerome Glisse * checking if a fence is signaled as it means that the 238bb635567291482a87e4cc46e6683419c1f365ddfJerome Glisse * seq we just read is different from the previous on. 239bb635567291482a87e4cc46e6683419c1f365ddfJerome Glisse */ 240bb635567291482a87e4cc46e6683419c1f365ddfJerome Glisse wake = true; 2413b7a2b24ea2b703b3af595d0d4ee233ab0b36377Jerome Glisse last_seq = seq; 242bb635567291482a87e4cc46e6683419c1f365ddfJerome Glisse if ((count_loop++) > 10) { 243bb635567291482a87e4cc46e6683419c1f365ddfJerome Glisse /* We looped over too many time leave with the 244bb635567291482a87e4cc46e6683419c1f365ddfJerome Glisse * fact that we might have set an older fence 245bb635567291482a87e4cc46e6683419c1f365ddfJerome Glisse * seq then the current real last seq as signaled 246bb635567291482a87e4cc46e6683419c1f365ddfJerome Glisse * by the hw. 247bb635567291482a87e4cc46e6683419c1f365ddfJerome Glisse */ 248bb635567291482a87e4cc46e6683419c1f365ddfJerome Glisse break; 249bb635567291482a87e4cc46e6683419c1f365ddfJerome Glisse } 250bb635567291482a87e4cc46e6683419c1f365ddfJerome Glisse } while (atomic64_xchg(&rdev->fence_drv[ring].last_seq, seq) > seq); 251bb635567291482a87e4cc46e6683419c1f365ddfJerome Glisse 2520bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König if (seq < last_emitted) 2530bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König radeon_fence_schedule_check(rdev, ring); 2540bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König 2550bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König return wake; 2560bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König} 2570bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König 2580bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König/** 2590bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König * radeon_fence_check_lockup - check for hardware lockup 2600bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König * 2610bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König * @work: delayed work item 2620bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König * 2630bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König * Checks for fence activity and if there is none probe 2640bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König * the hardware if a lockup occured. 2650bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König */ 2660bfa4b41268ad5fd741f16f484e4fee190822ec6Christian Königstatic void radeon_fence_check_lockup(struct work_struct *work) 2670bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König{ 2680bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König struct radeon_fence_driver *fence_drv; 2690bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König struct radeon_device *rdev; 2700bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König int ring; 2710bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König 2720bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König fence_drv = container_of(work, struct radeon_fence_driver, 2730bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König lockup_work.work); 2740bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König rdev = fence_drv->rdev; 2750bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König ring = fence_drv - &rdev->fence_drv[0]; 2760bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König 2770bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König if (!down_read_trylock(&rdev->exclusive_lock)) { 2780bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König /* just reschedule the check if a reset is going on */ 2790bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König radeon_fence_schedule_check(rdev, ring); 2800bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König return; 2810bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König } 2820bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König 283954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst if (fence_drv->delayed_irq && rdev->ddev->irq_enabled) { 284954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst unsigned long irqflags; 285954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst 286954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst fence_drv->delayed_irq = false; 287954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst spin_lock_irqsave(&rdev->irq.lock, irqflags); 288954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst radeon_irq_set(rdev); 289954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst spin_unlock_irqrestore(&rdev->irq.lock, irqflags); 290954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst } 291954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst 2920bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König if (radeon_fence_activity(rdev, ring)) 2930bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König wake_up_all(&rdev->fence_queue); 2940bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König 2950bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König else if (radeon_ring_is_lockup(rdev, ring, &rdev->ring[ring])) { 2960bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König 2970bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König /* good news we believe it's a lockup */ 2980bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König dev_warn(rdev->dev, "GPU lockup (current fence id " 2990bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König "0x%016llx last fence id 0x%016llx on ring %d)\n", 3000bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König (uint64_t)atomic64_read(&fence_drv->last_seq), 3010bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König fence_drv->sync_seq[ring], ring); 3020bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König 3030bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König /* remember that we need an reset */ 3040bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König rdev->needs_reset = true; 3050bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König wake_up_all(&rdev->fence_queue); 3060bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König } 3070bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König up_read(&rdev->exclusive_lock); 3080bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König} 3090bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König 3100bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König/** 3110bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König * radeon_fence_process - process a fence 3120bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König * 3130bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König * @rdev: radeon_device pointer 3140bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König * @ring: ring index the fence is associated with 3150bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König * 3160bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König * Checks the current fence value and wakes the fence queue 3170bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König * if the sequence number has increased (all asics). 3180bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König */ 3190bfa4b41268ad5fd741f16f484e4fee190822ec6Christian Königvoid radeon_fence_process(struct radeon_device *rdev, int ring) 3200bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König{ 3210bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König if (radeon_fence_activity(rdev, ring)) 3220085c95061e836f3ed489d042b502733c094e7e4Jerome Glisse wake_up_all(&rdev->fence_queue); 323771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse} 324771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 325d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher/** 326f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König * radeon_fence_seq_signaled - check if a fence sequence number has signaled 327d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * 328d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * @rdev: radeon device pointer 329d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * @seq: sequence number 330d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * @ring: ring index the fence is associated with 331d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * 332f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König * Check if the last signaled fence sequnce number is >= the requested 333d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * sequence number (all asics). 334d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * Returns true if the fence has signaled (current fence value 335d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * is >= requested value) or false if it has not (current fence 336d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * value is < the requested value. Helper function for 337d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * radeon_fence_signaled(). 338d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher */ 3393b7a2b24ea2b703b3af595d0d4ee233ab0b36377Jerome Glissestatic bool radeon_fence_seq_signaled(struct radeon_device *rdev, 3403b7a2b24ea2b703b3af595d0d4ee233ab0b36377Jerome Glisse u64 seq, unsigned ring) 341771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse{ 3423b7a2b24ea2b703b3af595d0d4ee233ab0b36377Jerome Glisse if (atomic64_read(&rdev->fence_drv[ring].last_seq) >= seq) { 3433b7a2b24ea2b703b3af595d0d4ee233ab0b36377Jerome Glisse return true; 3443b7a2b24ea2b703b3af595d0d4ee233ab0b36377Jerome Glisse } 3453b7a2b24ea2b703b3af595d0d4ee233ab0b36377Jerome Glisse /* poll new last sequence at least once */ 3463b7a2b24ea2b703b3af595d0d4ee233ab0b36377Jerome Glisse radeon_fence_process(rdev, ring); 3473b7a2b24ea2b703b3af595d0d4ee233ab0b36377Jerome Glisse if (atomic64_read(&rdev->fence_drv[ring].last_seq) >= seq) { 348771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return true; 3493b7a2b24ea2b703b3af595d0d4ee233ab0b36377Jerome Glisse } 3503b7a2b24ea2b703b3af595d0d4ee233ab0b36377Jerome Glisse return false; 3513b7a2b24ea2b703b3af595d0d4ee233ab0b36377Jerome Glisse} 3523655d54af8dd85788c3e5088387469703a0f8f12Darren Jenkins 353954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorststatic bool radeon_fence_is_signaled(struct fence *f) 354954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst{ 355954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst struct radeon_fence *fence = to_radeon_fence(f); 356954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst struct radeon_device *rdev = fence->rdev; 357954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst unsigned ring = fence->ring; 358954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst u64 seq = fence->seq; 359954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst 360954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst if (atomic64_read(&rdev->fence_drv[ring].last_seq) >= seq) { 361954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst return true; 362954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst } 363954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst 364954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst if (down_read_trylock(&rdev->exclusive_lock)) { 365954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst radeon_fence_process(rdev, ring); 366954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst up_read(&rdev->exclusive_lock); 367954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst 368954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst if (atomic64_read(&rdev->fence_drv[ring].last_seq) >= seq) { 369954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst return true; 370954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst } 371954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst } 372954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst return false; 373954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst} 374954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst 375954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst/** 376954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst * radeon_fence_enable_signaling - enable signalling on fence 377954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst * @fence: fence 378954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst * 379954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst * This function is called with fence_queue lock held, and adds a callback 380954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst * to fence_queue that checks if this fence is signaled, and if so it 381954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst * signals the fence and removes itself. 382954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst */ 383954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorststatic bool radeon_fence_enable_signaling(struct fence *f) 384954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst{ 385954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst struct radeon_fence *fence = to_radeon_fence(f); 386954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst struct radeon_device *rdev = fence->rdev; 387954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst 388954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst if (atomic64_read(&rdev->fence_drv[fence->ring].last_seq) >= fence->seq) 389954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst return false; 390954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst 391954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst if (down_read_trylock(&rdev->exclusive_lock)) { 392954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst radeon_irq_kms_sw_irq_get(rdev, fence->ring); 393954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst 394954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst if (radeon_fence_activity(rdev, fence->ring)) 395954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst wake_up_all_locked(&rdev->fence_queue); 396954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst 397954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst /* did fence get signaled after we enabled the sw irq? */ 398954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst if (atomic64_read(&rdev->fence_drv[fence->ring].last_seq) >= fence->seq) { 399954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst radeon_irq_kms_sw_irq_put(rdev, fence->ring); 400954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst up_read(&rdev->exclusive_lock); 401954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst return false; 402954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst } 403954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst 404954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst up_read(&rdev->exclusive_lock); 405954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst } else { 406954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst /* we're probably in a lockup, lets not fiddle too much */ 407954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst if (radeon_irq_kms_sw_irq_get_delayed(rdev, fence->ring)) 408954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst rdev->fence_drv[fence->ring].delayed_irq = true; 409954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst radeon_fence_schedule_check(rdev, fence->ring); 410954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst } 411954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst 412954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst fence->fence_wake.flags = 0; 413954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst fence->fence_wake.private = NULL; 414954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst fence->fence_wake.func = radeon_fence_check_signaled; 415954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst __add_wait_queue(&rdev->fence_queue, &fence->fence_wake); 416954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst fence_get(f); 417954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst 418954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst FENCE_TRACE(&fence->base, "armed on ring %i!\n", fence->ring); 419954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst return true; 420954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst} 421954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst 422d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher/** 423d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * radeon_fence_signaled - check if a fence has signaled 424d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * 425d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * @fence: radeon fence object 426d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * 427d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * Check if the requested fence has signaled (all asics). 428d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * Returns true if the fence has signaled or false if it has not. 429d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher */ 4303b7a2b24ea2b703b3af595d0d4ee233ab0b36377Jerome Glissebool radeon_fence_signaled(struct radeon_fence *fence) 4313b7a2b24ea2b703b3af595d0d4ee233ab0b36377Jerome Glisse{ 432d6d5c5b8364bcc4d52cddc68bcb0a330d2af20f3Christian König if (!fence) 4333b7a2b24ea2b703b3af595d0d4ee233ab0b36377Jerome Glisse return true; 434954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst 435954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst if (radeon_fence_seq_signaled(fence->rdev, fence->seq, fence->ring)) { 436954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst int ret; 437954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst 438954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst ret = fence_signal(&fence->base); 439954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst if (!ret) 440954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst FENCE_TRACE(&fence->base, "signaled from radeon_fence_signaled\n"); 4413b7a2b24ea2b703b3af595d0d4ee233ab0b36377Jerome Glisse return true; 442954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst } 4433b7a2b24ea2b703b3af595d0d4ee233ab0b36377Jerome Glisse return false; 444771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse} 445771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 446d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher/** 447f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König * radeon_fence_any_seq_signaled - check if any sequence number is signaled 448d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * 449d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * @rdev: radeon device pointer 450f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König * @seq: sequence numbers 451f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König * 452f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König * Check if the last signaled fence sequnce number is >= the requested 453f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König * sequence number (all asics). 454f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König * Returns true if any has signaled (current value is >= requested value) 455f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König * or false if it has not. Helper function for radeon_fence_wait_seq. 456f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König */ 457f9eaf9ae782d6480f179850e27e6f4911ac10227Christian Königstatic bool radeon_fence_any_seq_signaled(struct radeon_device *rdev, u64 *seq) 458f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König{ 459f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König unsigned i; 460f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König 461f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König for (i = 0; i < RADEON_NUM_RINGS; ++i) { 462f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König if (seq[i] && radeon_fence_seq_signaled(rdev, seq[i], i)) 463f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König return true; 464f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König } 465f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König return false; 466f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König} 467f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König 468f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König/** 4699867d00dbaef42e346e5d12eaa9591b057fea6d8Maarten Lankhorst * radeon_fence_wait_seq_timeout - wait for a specific sequence numbers 470f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König * 471f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König * @rdev: radeon device pointer 472f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König * @target_seq: sequence number(s) we want to wait for 473d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * @intr: use interruptable sleep 4749867d00dbaef42e346e5d12eaa9591b057fea6d8Maarten Lankhorst * @timeout: maximum time to wait, or MAX_SCHEDULE_TIMEOUT for infinite wait 475d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * 476f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König * Wait for the requested sequence number(s) to be written by any ring 477f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König * (all asics). Sequnce number array is indexed by ring id. 478d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * @intr selects whether to use interruptable (true) or non-interruptable 479d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * (false) sleep when waiting for the sequence number. Helper function 480f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König * for radeon_fence_wait_*(). 4819867d00dbaef42e346e5d12eaa9591b057fea6d8Maarten Lankhorst * Returns remaining time if the sequence number has passed, 0 when 4829867d00dbaef42e346e5d12eaa9591b057fea6d8Maarten Lankhorst * the wait timeout, or an error for all other cases. 483f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König * -EDEADLK is returned when a GPU lockup has been detected. 484d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher */ 4859867d00dbaef42e346e5d12eaa9591b057fea6d8Maarten Lankhorststatic long radeon_fence_wait_seq_timeout(struct radeon_device *rdev, 4869867d00dbaef42e346e5d12eaa9591b057fea6d8Maarten Lankhorst u64 *target_seq, bool intr, 4879867d00dbaef42e346e5d12eaa9591b057fea6d8Maarten Lankhorst long timeout) 488771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse{ 4890bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König long r; 4900bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König int i; 491f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König 4920bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König if (radeon_fence_any_seq_signaled(rdev, target_seq)) 4939867d00dbaef42e346e5d12eaa9591b057fea6d8Maarten Lankhorst return timeout; 494771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 4950bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König /* enable IRQs and tracing */ 4960bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König for (i = 0; i < RADEON_NUM_RINGS; ++i) { 4970bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König if (!target_seq[i]) 4980bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König continue; 49936abacaed34bc1f5bcb11ca611dd3a06c5c0ef39Christian König 5000bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König trace_radeon_fence_wait_begin(rdev->ddev, i, target_seq[i]); 5010bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König radeon_irq_kms_sw_irq_get(rdev, i); 5020bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König } 50336abacaed34bc1f5bcb11ca611dd3a06c5c0ef39Christian König 5040bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König if (intr) { 5050bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König r = wait_event_interruptible_timeout(rdev->fence_queue, ( 5060bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König radeon_fence_any_seq_signaled(rdev, target_seq) 5079867d00dbaef42e346e5d12eaa9591b057fea6d8Maarten Lankhorst || rdev->needs_reset), timeout); 5080bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König } else { 5090bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König r = wait_event_timeout(rdev->fence_queue, ( 5100bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König radeon_fence_any_seq_signaled(rdev, target_seq) 5119867d00dbaef42e346e5d12eaa9591b057fea6d8Maarten Lankhorst || rdev->needs_reset), timeout); 5120bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König } 513f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König 5140bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König if (rdev->needs_reset) 5150bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König r = -EDEADLK; 516f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König 5170bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König for (i = 0; i < RADEON_NUM_RINGS; ++i) { 5180bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König if (!target_seq[i]) 5190bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König continue; 52025a9e35218c66e7a3a5344cfe94a31a3c10ff810Christian König 5210bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König radeon_irq_kms_sw_irq_put(rdev, i); 5220bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König trace_radeon_fence_wait_end(rdev->ddev, i, target_seq[i]); 523771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 5240bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König 5259867d00dbaef42e346e5d12eaa9591b057fea6d8Maarten Lankhorst return r; 526771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse} 527771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 528d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher/** 529d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * radeon_fence_wait - wait for a fence to signal 530d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * 531d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * @fence: radeon fence object 5329867d00dbaef42e346e5d12eaa9591b057fea6d8Maarten Lankhorst * @intr: use interruptible sleep 533d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * 534d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * Wait for the requested fence to signal (all asics). 535d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * @intr selects whether to use interruptable (true) or non-interruptable 536d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * (false) sleep when waiting for the fence. 537d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * Returns 0 if the fence has passed, error for all other cases. 538d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher */ 5393b7a2b24ea2b703b3af595d0d4ee233ab0b36377Jerome Glisseint radeon_fence_wait(struct radeon_fence *fence, bool intr) 540771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse{ 541f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König uint64_t seq[RADEON_NUM_RINGS] = {}; 5429867d00dbaef42e346e5d12eaa9591b057fea6d8Maarten Lankhorst long r; 543771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 544392a250bd080e296f97ccc7e91b051a6b5da0ff1Maarten Lankhorst /* 545392a250bd080e296f97ccc7e91b051a6b5da0ff1Maarten Lankhorst * This function should not be called on !radeon fences. 546392a250bd080e296f97ccc7e91b051a6b5da0ff1Maarten Lankhorst * If this is the case, it would mean this function can 547392a250bd080e296f97ccc7e91b051a6b5da0ff1Maarten Lankhorst * also be called on radeon fences belonging to another card. 548392a250bd080e296f97ccc7e91b051a6b5da0ff1Maarten Lankhorst * exclusive_lock is not held in that case. 549392a250bd080e296f97ccc7e91b051a6b5da0ff1Maarten Lankhorst */ 550392a250bd080e296f97ccc7e91b051a6b5da0ff1Maarten Lankhorst if (WARN_ON_ONCE(!to_radeon_fence(&fence->base))) 551392a250bd080e296f97ccc7e91b051a6b5da0ff1Maarten Lankhorst return fence_wait(&fence->base, intr); 552392a250bd080e296f97ccc7e91b051a6b5da0ff1Maarten Lankhorst 553f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König seq[fence->ring] = fence->seq; 5549867d00dbaef42e346e5d12eaa9591b057fea6d8Maarten Lankhorst r = radeon_fence_wait_seq_timeout(fence->rdev, seq, intr, MAX_SCHEDULE_TIMEOUT); 5559867d00dbaef42e346e5d12eaa9591b057fea6d8Maarten Lankhorst if (r < 0) { 556f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König return r; 5579867d00dbaef42e346e5d12eaa9591b057fea6d8Maarten Lankhorst } 5580085c95061e836f3ed489d042b502733c094e7e4Jerome Glisse 559954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst r = fence_signal(&fence->base); 560954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst if (!r) 561954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst FENCE_TRACE(&fence->base, "signaled from fence_wait\n"); 5620085c95061e836f3ed489d042b502733c094e7e4Jerome Glisse return 0; 5630085c95061e836f3ed489d042b502733c094e7e4Jerome Glisse} 5640085c95061e836f3ed489d042b502733c094e7e4Jerome Glisse 565d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher/** 566d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * radeon_fence_wait_any - wait for a fence to signal on any ring 567d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * 568d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * @rdev: radeon device pointer 569d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * @fences: radeon fence object(s) 570d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * @intr: use interruptable sleep 571d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * 572d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * Wait for any requested fence to signal (all asics). Fence 573d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * array is indexed by ring id. @intr selects whether to use 574d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * interruptable (true) or non-interruptable (false) sleep when 575d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * waiting for the fences. Used by the suballocator. 576d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * Returns 0 if any fence has passed, error for all other cases. 577d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher */ 5780085c95061e836f3ed489d042b502733c094e7e4Jerome Glisseint radeon_fence_wait_any(struct radeon_device *rdev, 5790085c95061e836f3ed489d042b502733c094e7e4Jerome Glisse struct radeon_fence **fences, 5800085c95061e836f3ed489d042b502733c094e7e4Jerome Glisse bool intr) 5810085c95061e836f3ed489d042b502733c094e7e4Jerome Glisse{ 5820085c95061e836f3ed489d042b502733c094e7e4Jerome Glisse uint64_t seq[RADEON_NUM_RINGS]; 583f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König unsigned i, num_rings = 0; 5849867d00dbaef42e346e5d12eaa9591b057fea6d8Maarten Lankhorst long r; 5850085c95061e836f3ed489d042b502733c094e7e4Jerome Glisse 5860085c95061e836f3ed489d042b502733c094e7e4Jerome Glisse for (i = 0; i < RADEON_NUM_RINGS; ++i) { 5870085c95061e836f3ed489d042b502733c094e7e4Jerome Glisse seq[i] = 0; 5880085c95061e836f3ed489d042b502733c094e7e4Jerome Glisse 5890085c95061e836f3ed489d042b502733c094e7e4Jerome Glisse if (!fences[i]) { 5900085c95061e836f3ed489d042b502733c094e7e4Jerome Glisse continue; 5910085c95061e836f3ed489d042b502733c094e7e4Jerome Glisse } 5920085c95061e836f3ed489d042b502733c094e7e4Jerome Glisse 593876dc9f32907e57e0298bcd0f1607cb7a2582f63Christian König seq[i] = fences[i]->seq; 594f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König ++num_rings; 5950085c95061e836f3ed489d042b502733c094e7e4Jerome Glisse } 5960085c95061e836f3ed489d042b502733c094e7e4Jerome Glisse 597f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König /* nothing to wait for ? */ 598f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König if (num_rings == 0) 599f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König return -ENOENT; 600f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König 6019867d00dbaef42e346e5d12eaa9591b057fea6d8Maarten Lankhorst r = radeon_fence_wait_seq_timeout(rdev, seq, intr, MAX_SCHEDULE_TIMEOUT); 6029867d00dbaef42e346e5d12eaa9591b057fea6d8Maarten Lankhorst if (r < 0) { 6030085c95061e836f3ed489d042b502733c094e7e4Jerome Glisse return r; 6040085c95061e836f3ed489d042b502733c094e7e4Jerome Glisse } 6050085c95061e836f3ed489d042b502733c094e7e4Jerome Glisse return 0; 6060085c95061e836f3ed489d042b502733c094e7e4Jerome Glisse} 6070085c95061e836f3ed489d042b502733c094e7e4Jerome Glisse 608d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher/** 60937615527c5669f0c332534a797e5aaa175b6f3cbChristian König * radeon_fence_wait_next - wait for the next fence to signal 610d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * 611d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * @rdev: radeon device pointer 612d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * @ring: ring index the fence is associated with 613d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * 614d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * Wait for the next fence on the requested ring to signal (all asics). 615d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * Returns 0 if the next fence has passed, error for all other cases. 616d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * Caller must hold ring lock. 617d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher */ 61837615527c5669f0c332534a797e5aaa175b6f3cbChristian Königint radeon_fence_wait_next(struct radeon_device *rdev, int ring) 619771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse{ 620f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König uint64_t seq[RADEON_NUM_RINGS] = {}; 6219867d00dbaef42e346e5d12eaa9591b057fea6d8Maarten Lankhorst long r; 622771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 623f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König seq[ring] = atomic64_read(&rdev->fence_drv[ring].last_seq) + 1ULL; 624f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König if (seq[ring] >= rdev->fence_drv[ring].sync_seq[ring]) { 6258a47cc9ec1249eefd600adb273148c62879a560dChristian König /* nothing to wait for, last_seq is 6268a47cc9ec1249eefd600adb273148c62879a560dChristian König already the last emited fence */ 6278a47cc9ec1249eefd600adb273148c62879a560dChristian König return -ENOENT; 628771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 6299867d00dbaef42e346e5d12eaa9591b057fea6d8Maarten Lankhorst r = radeon_fence_wait_seq_timeout(rdev, seq, false, MAX_SCHEDULE_TIMEOUT); 6309867d00dbaef42e346e5d12eaa9591b057fea6d8Maarten Lankhorst if (r < 0) 6319867d00dbaef42e346e5d12eaa9591b057fea6d8Maarten Lankhorst return r; 6329867d00dbaef42e346e5d12eaa9591b057fea6d8Maarten Lankhorst return 0; 6333b7a2b24ea2b703b3af595d0d4ee233ab0b36377Jerome Glisse} 6343b7a2b24ea2b703b3af595d0d4ee233ab0b36377Jerome Glisse 635d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher/** 63637615527c5669f0c332534a797e5aaa175b6f3cbChristian König * radeon_fence_wait_empty - wait for all fences to signal 637d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * 638d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * @rdev: radeon device pointer 639d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * @ring: ring index the fence is associated with 640d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * 641d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * Wait for all fences on the requested ring to signal (all asics). 642d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * Returns 0 if the fences have passed, error for all other cases. 643d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * Caller must hold ring lock. 644d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher */ 64537615527c5669f0c332534a797e5aaa175b6f3cbChristian Königint radeon_fence_wait_empty(struct radeon_device *rdev, int ring) 6463b7a2b24ea2b703b3af595d0d4ee233ab0b36377Jerome Glisse{ 647f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König uint64_t seq[RADEON_NUM_RINGS] = {}; 6489867d00dbaef42e346e5d12eaa9591b057fea6d8Maarten Lankhorst long r; 6497ecc45e3ef8468abb062be2a8bb427029342f42dChristian König 650f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König seq[ring] = rdev->fence_drv[ring].sync_seq[ring]; 651721529b5780dd40d2d27310c69ae2615a878221cChristian König if (!seq[ring]) 652721529b5780dd40d2d27310c69ae2615a878221cChristian König return 0; 653721529b5780dd40d2d27310c69ae2615a878221cChristian König 6549867d00dbaef42e346e5d12eaa9591b057fea6d8Maarten Lankhorst r = radeon_fence_wait_seq_timeout(rdev, seq, false, MAX_SCHEDULE_TIMEOUT); 6559867d00dbaef42e346e5d12eaa9591b057fea6d8Maarten Lankhorst if (r < 0) { 656f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König if (r == -EDEADLK) 6575f8f635edd8ad5a6416bff4c5ff486500357f473Jerome Glisse return -EDEADLK; 658f9eaf9ae782d6480f179850e27e6f4911ac10227Christian König 6599867d00dbaef42e346e5d12eaa9591b057fea6d8Maarten Lankhorst dev_err(rdev->dev, "error waiting for ring[%d] to become idle (%ld)\n", 6605f8f635edd8ad5a6416bff4c5ff486500357f473Jerome Glisse ring, r); 6617ecc45e3ef8468abb062be2a8bb427029342f42dChristian König } 6625f8f635edd8ad5a6416bff4c5ff486500357f473Jerome Glisse return 0; 663771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse} 664771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 665d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher/** 666d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * radeon_fence_ref - take a ref on a fence 667d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * 668d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * @fence: radeon fence object 669d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * 670d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * Take a reference on a fence (all asics). 671d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * Returns the fence. 672d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher */ 673771fe6b912fca54f03e8a72eb63058b582775362Jerome Glissestruct radeon_fence *radeon_fence_ref(struct radeon_fence *fence) 674771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse{ 675954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst fence_get(&fence->base); 676771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return fence; 677771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse} 678771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 679d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher/** 680d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * radeon_fence_unref - remove a ref on a fence 681d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * 682d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * @fence: radeon fence object 683d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * 684d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * Remove a reference on a fence (all asics). 685d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher */ 686771fe6b912fca54f03e8a72eb63058b582775362Jerome Glissevoid radeon_fence_unref(struct radeon_fence **fence) 687771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse{ 688771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse struct radeon_fence *tmp = *fence; 689771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 690771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse *fence = NULL; 691771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse if (tmp) { 692954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst fence_put(&tmp->base); 693771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 694771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse} 695771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 696d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher/** 697d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * radeon_fence_count_emitted - get the count of emitted fences 698d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * 699d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * @rdev: radeon device pointer 700d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * @ring: ring index the fence is associated with 701d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * 702d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * Get the number of fences emitted on the requested ring (all asics). 703d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * Returns the number of emitted fences on the ring. Used by the 704d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * dynpm code to ring track activity. 705d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher */ 7063b7a2b24ea2b703b3af595d0d4ee233ab0b36377Jerome Glisseunsigned radeon_fence_count_emitted(struct radeon_device *rdev, int ring) 707771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse{ 7083b7a2b24ea2b703b3af595d0d4ee233ab0b36377Jerome Glisse uint64_t emitted; 709771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 7103b7a2b24ea2b703b3af595d0d4ee233ab0b36377Jerome Glisse /* We are not protected by ring lock when reading the last sequence 7113b7a2b24ea2b703b3af595d0d4ee233ab0b36377Jerome Glisse * but it's ok to report slightly wrong fence count here. 7123b7a2b24ea2b703b3af595d0d4ee233ab0b36377Jerome Glisse */ 7130085c95061e836f3ed489d042b502733c094e7e4Jerome Glisse radeon_fence_process(rdev, ring); 71468e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König emitted = rdev->fence_drv[ring].sync_seq[ring] 71568e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König - atomic64_read(&rdev->fence_drv[ring].last_seq); 7163b7a2b24ea2b703b3af595d0d4ee233ab0b36377Jerome Glisse /* to avoid 32bits warp around */ 7173b7a2b24ea2b703b3af595d0d4ee233ab0b36377Jerome Glisse if (emitted > 0x10000000) { 7183b7a2b24ea2b703b3af595d0d4ee233ab0b36377Jerome Glisse emitted = 0x10000000; 71947492a23a128e953bd5087b1cac909cd8124ca5eChristian König } 7203b7a2b24ea2b703b3af595d0d4ee233ab0b36377Jerome Glisse return (unsigned)emitted; 72147492a23a128e953bd5087b1cac909cd8124ca5eChristian König} 72247492a23a128e953bd5087b1cac909cd8124ca5eChristian König 723d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher/** 724d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * radeon_fence_need_sync - do we need a semaphore 725d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * 726d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * @fence: radeon fence object 727d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * @dst_ring: which ring to check against 728d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * 729d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * Check if the fence needs to be synced against another ring 730d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * (all asics). If so, we need to emit a semaphore. 731d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * Returns true if we need to sync with another ring, false if 732d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * not. 733d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher */ 73468e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian Königbool radeon_fence_need_sync(struct radeon_fence *fence, int dst_ring) 73568e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König{ 73668e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König struct radeon_fence_driver *fdrv; 73768e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König 73868e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König if (!fence) { 73968e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König return false; 74068e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König } 74168e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König 74268e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König if (fence->ring == dst_ring) { 74368e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König return false; 74468e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König } 74568e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König 74668e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König /* we are protected by the ring mutex */ 74768e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König fdrv = &fence->rdev->fence_drv[dst_ring]; 74868e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König if (fence->seq <= fdrv->sync_seq[fence->ring]) { 74968e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König return false; 75068e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König } 75168e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König 75268e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König return true; 75368e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König} 75468e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König 755d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher/** 756d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * radeon_fence_note_sync - record the sync point 757d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * 758d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * @fence: radeon fence object 759d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * @dst_ring: which ring to check against 760d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * 761d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * Note the sequence number at which point the fence will 762d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * be synced with the requested ring (all asics). 763d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher */ 76468e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian Königvoid radeon_fence_note_sync(struct radeon_fence *fence, int dst_ring) 76568e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König{ 76668e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König struct radeon_fence_driver *dst, *src; 76768e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König unsigned i; 76868e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König 76968e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König if (!fence) { 77068e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König return; 77168e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König } 77268e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König 77368e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König if (fence->ring == dst_ring) { 77468e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König return; 77568e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König } 77668e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König 77768e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König /* we are protected by the ring mutex */ 77868e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König src = &fence->rdev->fence_drv[fence->ring]; 77968e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König dst = &fence->rdev->fence_drv[dst_ring]; 78068e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König for (i = 0; i < RADEON_NUM_RINGS; ++i) { 78168e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König if (i == dst_ring) { 78268e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König continue; 78368e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König } 78468e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König dst->sync_seq[i] = max(dst->sync_seq[i], src->sync_seq[i]); 78568e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König } 78668e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König} 78768e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König 788d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher/** 789d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * radeon_fence_driver_start_ring - make the fence driver 790d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * ready for use on the requested ring. 791d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * 792d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * @rdev: radeon device pointer 793d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * @ring: ring index to start the fence driver on 794d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * 795d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * Make the fence driver ready for processing (all asics). 796d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * Not all asics have all rings, so each asic will only 797d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * start the fence driver on the rings it has. 798d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * Returns 0 for success, errors for failure. 799d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher */ 80030eb77f4e6ba20f797af4ff79807fae7cb67429eJerome Glisseint radeon_fence_driver_start_ring(struct radeon_device *rdev, int ring) 801771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse{ 80230eb77f4e6ba20f797af4ff79807fae7cb67429eJerome Glisse uint64_t index; 80330eb77f4e6ba20f797af4ff79807fae7cb67429eJerome Glisse int r; 804771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 80530eb77f4e6ba20f797af4ff79807fae7cb67429eJerome Glisse radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg); 80686a1881d08f65a42c17071a59c0088dbe2870246Jerome Glisse if (rdev->wb.use_event || !radeon_ring_supports_scratch_reg(rdev, &rdev->ring[ring])) { 807581bc3a9f628dad6221536d9f37af0472c66bb92Christian König rdev->fence_drv[ring].scratch_reg = 0; 808f2ba57b5eab8817d86d0f108fdf1878e51dc0a37Christian König if (ring != R600_RING_TYPE_UVD_INDEX) { 809f2ba57b5eab8817d86d0f108fdf1878e51dc0a37Christian König index = R600_WB_EVENT_OFFSET + ring * 4; 810f2ba57b5eab8817d86d0f108fdf1878e51dc0a37Christian König rdev->fence_drv[ring].cpu_addr = &rdev->wb.wb[index/4]; 811f2ba57b5eab8817d86d0f108fdf1878e51dc0a37Christian König rdev->fence_drv[ring].gpu_addr = rdev->wb.gpu_addr + 812f2ba57b5eab8817d86d0f108fdf1878e51dc0a37Christian König index; 813f2ba57b5eab8817d86d0f108fdf1878e51dc0a37Christian König 814f2ba57b5eab8817d86d0f108fdf1878e51dc0a37Christian König } else { 815f2ba57b5eab8817d86d0f108fdf1878e51dc0a37Christian König /* put fence directly behind firmware */ 8164ad9c1c774c2af152283f510062094e768876f55Christian König index = ALIGN(rdev->uvd_fw->size, 8); 817d7c605a20ee86a4e8f19ca7b33f7c4f7ba0468bcChristian König rdev->fence_drv[ring].cpu_addr = rdev->uvd.cpu_addr + index; 818d7c605a20ee86a4e8f19ca7b33f7c4f7ba0468bcChristian König rdev->fence_drv[ring].gpu_addr = rdev->uvd.gpu_addr + index; 819f2ba57b5eab8817d86d0f108fdf1878e51dc0a37Christian König } 820f2ba57b5eab8817d86d0f108fdf1878e51dc0a37Christian König 82130eb77f4e6ba20f797af4ff79807fae7cb67429eJerome Glisse } else { 8227465280c076d6440e5908c158c83b542dc063a30Alex Deucher r = radeon_scratch_get(rdev, &rdev->fence_drv[ring].scratch_reg); 8237465280c076d6440e5908c158c83b542dc063a30Alex Deucher if (r) { 8247465280c076d6440e5908c158c83b542dc063a30Alex Deucher dev_err(rdev->dev, "fence failed to get scratch register\n"); 8257465280c076d6440e5908c158c83b542dc063a30Alex Deucher return r; 8267465280c076d6440e5908c158c83b542dc063a30Alex Deucher } 82730eb77f4e6ba20f797af4ff79807fae7cb67429eJerome Glisse index = RADEON_WB_SCRATCH_OFFSET + 82830eb77f4e6ba20f797af4ff79807fae7cb67429eJerome Glisse rdev->fence_drv[ring].scratch_reg - 82930eb77f4e6ba20f797af4ff79807fae7cb67429eJerome Glisse rdev->scratch.reg_base; 830f2ba57b5eab8817d86d0f108fdf1878e51dc0a37Christian König rdev->fence_drv[ring].cpu_addr = &rdev->wb.wb[index/4]; 831f2ba57b5eab8817d86d0f108fdf1878e51dc0a37Christian König rdev->fence_drv[ring].gpu_addr = rdev->wb.gpu_addr + index; 8327465280c076d6440e5908c158c83b542dc063a30Alex Deucher } 83331be6183d53e89ff95ec7cb70aac67fa5588e0dfChristian König radeon_fence_write(rdev, atomic64_read(&rdev->fence_drv[ring].last_seq), ring); 83430eb77f4e6ba20f797af4ff79807fae7cb67429eJerome Glisse rdev->fence_drv[ring].initialized = true; 8353b7a2b24ea2b703b3af595d0d4ee233ab0b36377Jerome Glisse dev_info(rdev->dev, "fence driver on ring %d use gpu addr 0x%016llx and cpu addr 0x%p\n", 83630eb77f4e6ba20f797af4ff79807fae7cb67429eJerome Glisse ring, rdev->fence_drv[ring].gpu_addr, rdev->fence_drv[ring].cpu_addr); 83730eb77f4e6ba20f797af4ff79807fae7cb67429eJerome Glisse return 0; 83830eb77f4e6ba20f797af4ff79807fae7cb67429eJerome Glisse} 83930eb77f4e6ba20f797af4ff79807fae7cb67429eJerome Glisse 840d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher/** 841d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * radeon_fence_driver_init_ring - init the fence driver 842d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * for the requested ring. 843d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * 844d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * @rdev: radeon device pointer 845d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * @ring: ring index to start the fence driver on 846d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * 847d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * Init the fence driver for the requested ring (all asics). 848d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * Helper function for radeon_fence_driver_init(). 849d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher */ 85030eb77f4e6ba20f797af4ff79807fae7cb67429eJerome Glissestatic void radeon_fence_driver_init_ring(struct radeon_device *rdev, int ring) 85130eb77f4e6ba20f797af4ff79807fae7cb67429eJerome Glisse{ 85268e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König int i; 85368e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König 85430eb77f4e6ba20f797af4ff79807fae7cb67429eJerome Glisse rdev->fence_drv[ring].scratch_reg = -1; 85530eb77f4e6ba20f797af4ff79807fae7cb67429eJerome Glisse rdev->fence_drv[ring].cpu_addr = NULL; 85630eb77f4e6ba20f797af4ff79807fae7cb67429eJerome Glisse rdev->fence_drv[ring].gpu_addr = 0; 85768e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König for (i = 0; i < RADEON_NUM_RINGS; ++i) 85868e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König rdev->fence_drv[ring].sync_seq[i] = 0; 859bb635567291482a87e4cc46e6683419c1f365ddfJerome Glisse atomic64_set(&rdev->fence_drv[ring].last_seq, 0); 86030eb77f4e6ba20f797af4ff79807fae7cb67429eJerome Glisse rdev->fence_drv[ring].initialized = false; 8610bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König INIT_DELAYED_WORK(&rdev->fence_drv[ring].lockup_work, 8620bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König radeon_fence_check_lockup); 8630bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König rdev->fence_drv[ring].rdev = rdev; 86430eb77f4e6ba20f797af4ff79807fae7cb67429eJerome Glisse} 86530eb77f4e6ba20f797af4ff79807fae7cb67429eJerome Glisse 866d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher/** 867d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * radeon_fence_driver_init - init the fence driver 868d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * for all possible rings. 869d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * 870d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * @rdev: radeon device pointer 871d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * 872d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * Init the fence driver for all possible rings (all asics). 873d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * Not all asics have all rings, so each asic will only 874d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * start the fence driver on the rings it has using 875d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * radeon_fence_driver_start_ring(). 876d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * Returns 0 for success. 877d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher */ 87830eb77f4e6ba20f797af4ff79807fae7cb67429eJerome Glisseint radeon_fence_driver_init(struct radeon_device *rdev) 87930eb77f4e6ba20f797af4ff79807fae7cb67429eJerome Glisse{ 88030eb77f4e6ba20f797af4ff79807fae7cb67429eJerome Glisse int ring; 88130eb77f4e6ba20f797af4ff79807fae7cb67429eJerome Glisse 8820085c95061e836f3ed489d042b502733c094e7e4Jerome Glisse init_waitqueue_head(&rdev->fence_queue); 88330eb77f4e6ba20f797af4ff79807fae7cb67429eJerome Glisse for (ring = 0; ring < RADEON_NUM_RINGS; ring++) { 88430eb77f4e6ba20f797af4ff79807fae7cb67429eJerome Glisse radeon_fence_driver_init_ring(rdev, ring); 885771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 886771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse if (radeon_debugfs_fence_init(rdev)) { 8870a0c7596c643239e8d4c3eaaba43b74a96f2411eJerome Glisse dev_err(rdev->dev, "fence debugfs file creation failed\n"); 888771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 889771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return 0; 890771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse} 891771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 892d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher/** 893d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * radeon_fence_driver_fini - tear down the fence driver 894d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * for all possible rings. 895d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * 896d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * @rdev: radeon device pointer 897d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * 898d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher * Tear down the fence driver for all possible rings (all asics). 899d66b7ec24adce867890c57bf2a54323accc73a7eAlex Deucher */ 900771fe6b912fca54f03e8a72eb63058b582775362Jerome Glissevoid radeon_fence_driver_fini(struct radeon_device *rdev) 901771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse{ 9025f8f635edd8ad5a6416bff4c5ff486500357f473Jerome Glisse int ring, r; 9037465280c076d6440e5908c158c83b542dc063a30Alex Deucher 9048a47cc9ec1249eefd600adb273148c62879a560dChristian König mutex_lock(&rdev->ring_lock); 9057465280c076d6440e5908c158c83b542dc063a30Alex Deucher for (ring = 0; ring < RADEON_NUM_RINGS; ring++) { 9067465280c076d6440e5908c158c83b542dc063a30Alex Deucher if (!rdev->fence_drv[ring].initialized) 9077465280c076d6440e5908c158c83b542dc063a30Alex Deucher continue; 90837615527c5669f0c332534a797e5aaa175b6f3cbChristian König r = radeon_fence_wait_empty(rdev, ring); 9095f8f635edd8ad5a6416bff4c5ff486500357f473Jerome Glisse if (r) { 9105f8f635edd8ad5a6416bff4c5ff486500357f473Jerome Glisse /* no need to trigger GPU reset as we are unloading */ 911eb98c709907c7a78b9cd0d18642477d47d348f9fChristian König radeon_fence_driver_force_completion(rdev, ring); 9125f8f635edd8ad5a6416bff4c5ff486500357f473Jerome Glisse } 9130bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König cancel_delayed_work_sync(&rdev->fence_drv[ring].lockup_work); 9140085c95061e836f3ed489d042b502733c094e7e4Jerome Glisse wake_up_all(&rdev->fence_queue); 9157465280c076d6440e5908c158c83b542dc063a30Alex Deucher radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg); 9167465280c076d6440e5908c158c83b542dc063a30Alex Deucher rdev->fence_drv[ring].initialized = false; 9177465280c076d6440e5908c158c83b542dc063a30Alex Deucher } 9188a47cc9ec1249eefd600adb273148c62879a560dChristian König mutex_unlock(&rdev->ring_lock); 919771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse} 920771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 92176903b96adbfbb38b049765add21e02e44c387a5Jerome Glisse/** 92276903b96adbfbb38b049765add21e02e44c387a5Jerome Glisse * radeon_fence_driver_force_completion - force all fence waiter to complete 92376903b96adbfbb38b049765add21e02e44c387a5Jerome Glisse * 92476903b96adbfbb38b049765add21e02e44c387a5Jerome Glisse * @rdev: radeon device pointer 925eb98c709907c7a78b9cd0d18642477d47d348f9fChristian König * @ring: the ring to complete 92676903b96adbfbb38b049765add21e02e44c387a5Jerome Glisse * 92776903b96adbfbb38b049765add21e02e44c387a5Jerome Glisse * In case of GPU reset failure make sure no process keep waiting on fence 92876903b96adbfbb38b049765add21e02e44c387a5Jerome Glisse * that will never complete. 92976903b96adbfbb38b049765add21e02e44c387a5Jerome Glisse */ 930eb98c709907c7a78b9cd0d18642477d47d348f9fChristian Königvoid radeon_fence_driver_force_completion(struct radeon_device *rdev, int ring) 93176903b96adbfbb38b049765add21e02e44c387a5Jerome Glisse{ 9320bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König if (rdev->fence_drv[ring].initialized) { 93376903b96adbfbb38b049765add21e02e44c387a5Jerome Glisse radeon_fence_write(rdev, rdev->fence_drv[ring].sync_seq[ring], ring); 9340bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König cancel_delayed_work_sync(&rdev->fence_drv[ring].lockup_work); 9350bfa4b41268ad5fd741f16f484e4fee190822ec6Christian König } 93676903b96adbfbb38b049765add21e02e44c387a5Jerome Glisse} 93776903b96adbfbb38b049765add21e02e44c387a5Jerome Glisse 938771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 939771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse/* 940771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse * Fence debugfs 941771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse */ 942771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse#if defined(CONFIG_DEBUG_FS) 943771fe6b912fca54f03e8a72eb63058b582775362Jerome Glissestatic int radeon_debugfs_fence_info(struct seq_file *m, void *data) 944771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse{ 945771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse struct drm_info_node *node = (struct drm_info_node *)m->private; 946771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse struct drm_device *dev = node->minor->dev; 947771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse struct radeon_device *rdev = dev->dev_private; 94868e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König int i, j; 9497465280c076d6440e5908c158c83b542dc063a30Alex Deucher 9507465280c076d6440e5908c158c83b542dc063a30Alex Deucher for (i = 0; i < RADEON_NUM_RINGS; ++i) { 9517465280c076d6440e5908c158c83b542dc063a30Alex Deucher if (!rdev->fence_drv[i].initialized) 9527465280c076d6440e5908c158c83b542dc063a30Alex Deucher continue; 9537465280c076d6440e5908c158c83b542dc063a30Alex Deucher 954e290b63475a85f188556dd2a870d439e09a87fd5Christian König radeon_fence_process(rdev, i); 955e290b63475a85f188556dd2a870d439e09a87fd5Christian König 9567465280c076d6440e5908c158c83b542dc063a30Alex Deucher seq_printf(m, "--- ring %d ---\n", i); 957d3029b4e0358e8d3519a290d7e61686ad7e58830Dave Airlie seq_printf(m, "Last signaled fence 0x%016llx\n", 958d3029b4e0358e8d3519a290d7e61686ad7e58830Dave Airlie (unsigned long long)atomic64_read(&rdev->fence_drv[i].last_seq)); 95968e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König seq_printf(m, "Last emitted 0x%016llx\n", 96068e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König rdev->fence_drv[i].sync_seq[i]); 96168e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König 96268e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König for (j = 0; j < RADEON_NUM_RINGS; ++j) { 96368e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König if (i != j && rdev->fence_drv[j].initialized) 96468e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König seq_printf(m, "Last sync to ring %d 0x%016llx\n", 96568e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König j, rdev->fence_drv[i].sync_seq[j]); 96668e250b7c281dbb75ea2a892a7d4ca27f974fc91Christian König } 967771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse } 968771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return 0; 969771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse} 970771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 971478b6e72721807953bc3513fc5895d5f007614e3Christian König/** 972478b6e72721807953bc3513fc5895d5f007614e3Christian König * radeon_debugfs_gpu_reset - manually trigger a gpu reset 973478b6e72721807953bc3513fc5895d5f007614e3Christian König * 974478b6e72721807953bc3513fc5895d5f007614e3Christian König * Manually trigger a gpu reset at the next fence wait. 975478b6e72721807953bc3513fc5895d5f007614e3Christian König */ 976478b6e72721807953bc3513fc5895d5f007614e3Christian Königstatic int radeon_debugfs_gpu_reset(struct seq_file *m, void *data) 977478b6e72721807953bc3513fc5895d5f007614e3Christian König{ 978478b6e72721807953bc3513fc5895d5f007614e3Christian König struct drm_info_node *node = (struct drm_info_node *) m->private; 979478b6e72721807953bc3513fc5895d5f007614e3Christian König struct drm_device *dev = node->minor->dev; 980478b6e72721807953bc3513fc5895d5f007614e3Christian König struct radeon_device *rdev = dev->dev_private; 981478b6e72721807953bc3513fc5895d5f007614e3Christian König 982478b6e72721807953bc3513fc5895d5f007614e3Christian König down_read(&rdev->exclusive_lock); 983478b6e72721807953bc3513fc5895d5f007614e3Christian König seq_printf(m, "%d\n", rdev->needs_reset); 984478b6e72721807953bc3513fc5895d5f007614e3Christian König rdev->needs_reset = true; 985f0d970b4fd05cb7af89307bb17689c18c835d739Christian König wake_up_all(&rdev->fence_queue); 986478b6e72721807953bc3513fc5895d5f007614e3Christian König up_read(&rdev->exclusive_lock); 987478b6e72721807953bc3513fc5895d5f007614e3Christian König 988478b6e72721807953bc3513fc5895d5f007614e3Christian König return 0; 989478b6e72721807953bc3513fc5895d5f007614e3Christian König} 990478b6e72721807953bc3513fc5895d5f007614e3Christian König 991771fe6b912fca54f03e8a72eb63058b582775362Jerome Glissestatic struct drm_info_list radeon_debugfs_fence_list[] = { 992771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse {"radeon_fence_info", &radeon_debugfs_fence_info, 0, NULL}, 993478b6e72721807953bc3513fc5895d5f007614e3Christian König {"radeon_gpu_reset", &radeon_debugfs_gpu_reset, 0, NULL} 994771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse}; 995771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse#endif 996771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse 997771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisseint radeon_debugfs_fence_init(struct radeon_device *rdev) 998771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse{ 999771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse#if defined(CONFIG_DEBUG_FS) 1000478b6e72721807953bc3513fc5895d5f007614e3Christian König return radeon_debugfs_add_files(rdev, radeon_debugfs_fence_list, 2); 1001771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse#else 1002771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse return 0; 1003771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse#endif 1004771fe6b912fca54f03e8a72eb63058b582775362Jerome Glisse} 1005954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst 1006954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorststatic const char *radeon_fence_get_driver_name(struct fence *fence) 1007954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst{ 1008954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst return "radeon"; 1009954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst} 1010954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst 1011954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorststatic const char *radeon_fence_get_timeline_name(struct fence *f) 1012954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst{ 1013954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst struct radeon_fence *fence = to_radeon_fence(f); 1014954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst switch (fence->ring) { 1015954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst case RADEON_RING_TYPE_GFX_INDEX: return "radeon.gfx"; 1016954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst case CAYMAN_RING_TYPE_CP1_INDEX: return "radeon.cp1"; 1017954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst case CAYMAN_RING_TYPE_CP2_INDEX: return "radeon.cp2"; 1018954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst case R600_RING_TYPE_DMA_INDEX: return "radeon.dma"; 1019954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst case CAYMAN_RING_TYPE_DMA1_INDEX: return "radeon.dma1"; 1020954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst case R600_RING_TYPE_UVD_INDEX: return "radeon.uvd"; 1021954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst case TN_RING_TYPE_VCE1_INDEX: return "radeon.vce1"; 1022954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst case TN_RING_TYPE_VCE2_INDEX: return "radeon.vce2"; 1023954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst default: WARN_ON_ONCE(1); return "radeon.unk"; 1024954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst } 1025954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst} 1026954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst 1027954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorststatic inline bool radeon_test_signaled(struct radeon_fence *fence) 1028954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst{ 1029954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst return test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->base.flags); 1030954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst} 1031954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst 1032954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorststatic signed long radeon_fence_default_wait(struct fence *f, bool intr, 1033954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst signed long t) 1034954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst{ 1035954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst struct radeon_fence *fence = to_radeon_fence(f); 1036954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst struct radeon_device *rdev = fence->rdev; 1037954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst bool signaled; 1038954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst 1039954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst fence_enable_sw_signaling(&fence->base); 1040954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst 1041954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst /* 1042954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst * This function has to return -EDEADLK, but cannot hold 1043954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst * exclusive_lock during the wait because some callers 1044954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst * may already hold it. This means checking needs_reset without 1045954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst * lock, and not fiddling with any gpu internals. 1046954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst * 1047954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst * The callback installed with fence_enable_sw_signaling will 1048954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst * run before our wait_event_*timeout call, so we will see 1049954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst * both the signaled fence and the changes to needs_reset. 1050954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst */ 1051954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst 1052954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst if (intr) 1053954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst t = wait_event_interruptible_timeout(rdev->fence_queue, 1054954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst ((signaled = radeon_test_signaled(fence)) || 1055954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst rdev->needs_reset), t); 1056954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst else 1057954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst t = wait_event_timeout(rdev->fence_queue, 1058954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst ((signaled = radeon_test_signaled(fence)) || 1059954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst rdev->needs_reset), t); 1060954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst 1061954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst if (t > 0 && !signaled) 1062954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst return -EDEADLK; 1063954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst return t; 1064954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst} 1065954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst 1066954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorstconst struct fence_ops radeon_fence_ops = { 1067954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst .get_driver_name = radeon_fence_get_driver_name, 1068954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst .get_timeline_name = radeon_fence_get_timeline_name, 1069954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst .enable_signaling = radeon_fence_enable_signaling, 1070954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst .signaled = radeon_fence_is_signaled, 1071954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst .wait = radeon_fence_default_wait, 1072954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst .release = NULL, 1073954605ca3f897ad617123279eb3404a404cce5abMaarten Lankhorst}; 1074