171254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider/* 271254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider * Copyright 2010 Jerome Glisse <glisse@freedesktop.org> 371254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider * 471254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider * Permission is hereby granted, free of charge, to any person obtaining a 571254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider * copy of this software and associated documentation files (the "Software"), 671254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider * to deal in the Software without restriction, including without limitation 771254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider * on the rights to use, copy, modify, merge, publish, distribute, sub 871254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider * license, and/or sell copies of the Software, and to permit persons to whom 971254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider * the Software is furnished to do so, subject to the following conditions: 1071254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider * 1171254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider * The above copyright notice and this permission notice (including the next 1271254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider * paragraph) shall be included in all copies or substantial portions of the 1371254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider * Software. 1471254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider * 1571254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 1671254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 1771254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 1871254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM, 1971254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 2071254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 2171254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider * USE OR OTHER DEALINGS IN THE SOFTWARE. 2271254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider * 2371254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider * Authors: 2471254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider * Jerome Glisse 2571254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider */ 2671254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider 2771254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider#include "sid.h" 2871254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider#include "si_pipe.h" 2971254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider 3071254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider#include "util/u_format.h" 3171254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider 3271254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheiderstatic void si_dma_copy_buffer(struct si_context *ctx, 3371254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider struct pipe_resource *dst, 3471254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider struct pipe_resource *src, 3571254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider uint64_t dst_offset, 3671254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider uint64_t src_offset, 3771254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider uint64_t size) 3871254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider{ 396cc8f6c6a72b1aab7bb506deb220e04ae50d8c2bMarek Olšák struct radeon_winsys_cs *cs = ctx->b.dma.cs; 4029d6a367a6508a6814f44039166a65a4b69f53b1Marek Olšák unsigned i, ncopy, count, max_size, sub_cmd, shift; 4171254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider struct r600_resource *rdst = (struct r600_resource*)dst; 4271254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider struct r600_resource *rsrc = (struct r600_resource*)src; 4371254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider 4471254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider /* Mark the buffer range of destination as valid (initialized), 4571254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider * so that transfer_map knows it should wait for the GPU when mapping 4671254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider * that range. */ 4771254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider util_range_add(&rdst->valid_buffer_range, dst_offset, 4871254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider dst_offset + size); 4971254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider 501c03a690bfc3265c7fefa7f87e69782a6672a9b2Marek Olšák dst_offset += rdst->gpu_address; 511c03a690bfc3265c7fefa7f87e69782a6672a9b2Marek Olšák src_offset += rsrc->gpu_address; 5271254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider 5329d6a367a6508a6814f44039166a65a4b69f53b1Marek Olšák /* see whether we should use the dword-aligned or byte-aligned copy */ 5471254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider if (!(dst_offset % 4) && !(src_offset % 4) && !(size % 4)) { 5571254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider sub_cmd = SI_DMA_COPY_DWORD_ALIGNED; 5671254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider shift = 2; 5729d6a367a6508a6814f44039166a65a4b69f53b1Marek Olšák max_size = SI_DMA_COPY_MAX_DWORD_ALIGNED_SIZE; 5871254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider } else { 5971254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider sub_cmd = SI_DMA_COPY_BYTE_ALIGNED; 6071254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider shift = 0; 6129d6a367a6508a6814f44039166a65a4b69f53b1Marek Olšák max_size = SI_DMA_COPY_MAX_BYTE_ALIGNED_SIZE; 6271254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider } 6371254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider 6429d6a367a6508a6814f44039166a65a4b69f53b1Marek Olšák ncopy = DIV_ROUND_UP(size, max_size); 65bb74152597de44ee877b8928587b1cece8b49656Marek Olšák r600_need_dma_space(&ctx->b, ncopy * 5, rdst, rsrc); 66c8011c1885003b79c9f0c6530e46ae6cb0e69575Christian König 6771254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider for (i = 0; i < ncopy; i++) { 6829d6a367a6508a6814f44039166a65a4b69f53b1Marek Olšák count = MIN2(size, max_size); 6929d6a367a6508a6814f44039166a65a4b69f53b1Marek Olšák radeon_emit(cs, SI_DMA_PACKET(SI_DMA_PACKET_COPY, sub_cmd, 7029d6a367a6508a6814f44039166a65a4b69f53b1Marek Olšák count >> shift)); 71c23273532e711f3f0263bfff8bf8a0e733b90e12Nicolai Hähnle radeon_emit(cs, dst_offset); 72c23273532e711f3f0263bfff8bf8a0e733b90e12Nicolai Hähnle radeon_emit(cs, src_offset); 73c23273532e711f3f0263bfff8bf8a0e733b90e12Nicolai Hähnle radeon_emit(cs, (dst_offset >> 32UL) & 0xff); 74c23273532e711f3f0263bfff8bf8a0e733b90e12Nicolai Hähnle radeon_emit(cs, (src_offset >> 32UL) & 0xff); 7529d6a367a6508a6814f44039166a65a4b69f53b1Marek Olšák dst_offset += count; 7629d6a367a6508a6814f44039166a65a4b69f53b1Marek Olšák src_offset += count; 7729d6a367a6508a6814f44039166a65a4b69f53b1Marek Olšák size -= count; 7871254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider } 7971254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider} 8071254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider 81cba9d59362130e1b44cd9cfc4f38ad3773111442Marek Olšákstatic void si_dma_clear_buffer(struct pipe_context *ctx, 82cba9d59362130e1b44cd9cfc4f38ad3773111442Marek Olšák struct pipe_resource *dst, 83cba9d59362130e1b44cd9cfc4f38ad3773111442Marek Olšák uint64_t offset, 84cba9d59362130e1b44cd9cfc4f38ad3773111442Marek Olšák uint64_t size, 85cba9d59362130e1b44cd9cfc4f38ad3773111442Marek Olšák unsigned clear_value) 86cba9d59362130e1b44cd9cfc4f38ad3773111442Marek Olšák{ 87cba9d59362130e1b44cd9cfc4f38ad3773111442Marek Olšák struct si_context *sctx = (struct si_context *)ctx; 88cba9d59362130e1b44cd9cfc4f38ad3773111442Marek Olšák struct radeon_winsys_cs *cs = sctx->b.dma.cs; 89cba9d59362130e1b44cd9cfc4f38ad3773111442Marek Olšák unsigned i, ncopy, csize; 90cba9d59362130e1b44cd9cfc4f38ad3773111442Marek Olšák struct r600_resource *rdst = r600_resource(dst); 91cba9d59362130e1b44cd9cfc4f38ad3773111442Marek Olšák 92cba9d59362130e1b44cd9cfc4f38ad3773111442Marek Olšák if (!cs || offset % 4 != 0 || size % 4 != 0) { 93cba9d59362130e1b44cd9cfc4f38ad3773111442Marek Olšák ctx->clear_buffer(ctx, dst, offset, size, &clear_value, 4); 94cba9d59362130e1b44cd9cfc4f38ad3773111442Marek Olšák return; 95cba9d59362130e1b44cd9cfc4f38ad3773111442Marek Olšák } 96cba9d59362130e1b44cd9cfc4f38ad3773111442Marek Olšák 97cba9d59362130e1b44cd9cfc4f38ad3773111442Marek Olšák /* Mark the buffer range of destination as valid (initialized), 98cba9d59362130e1b44cd9cfc4f38ad3773111442Marek Olšák * so that transfer_map knows it should wait for the GPU when mapping 99cba9d59362130e1b44cd9cfc4f38ad3773111442Marek Olšák * that range. */ 100cba9d59362130e1b44cd9cfc4f38ad3773111442Marek Olšák util_range_add(&rdst->valid_buffer_range, offset, offset + size); 101cba9d59362130e1b44cd9cfc4f38ad3773111442Marek Olšák 102cba9d59362130e1b44cd9cfc4f38ad3773111442Marek Olšák offset += rdst->gpu_address; 103cba9d59362130e1b44cd9cfc4f38ad3773111442Marek Olšák 104cba9d59362130e1b44cd9cfc4f38ad3773111442Marek Olšák /* the same maximum size as for copying */ 105cba9d59362130e1b44cd9cfc4f38ad3773111442Marek Olšák ncopy = DIV_ROUND_UP(size, SI_DMA_COPY_MAX_DWORD_ALIGNED_SIZE); 106cba9d59362130e1b44cd9cfc4f38ad3773111442Marek Olšák r600_need_dma_space(&sctx->b, ncopy * 4, rdst, NULL); 107cba9d59362130e1b44cd9cfc4f38ad3773111442Marek Olšák 108cba9d59362130e1b44cd9cfc4f38ad3773111442Marek Olšák for (i = 0; i < ncopy; i++) { 109cba9d59362130e1b44cd9cfc4f38ad3773111442Marek Olšák csize = MIN2(size, SI_DMA_COPY_MAX_DWORD_ALIGNED_SIZE); 110cba9d59362130e1b44cd9cfc4f38ad3773111442Marek Olšák radeon_emit(cs, SI_DMA_PACKET(SI_DMA_PACKET_CONSTANT_FILL, 0, 111cba9d59362130e1b44cd9cfc4f38ad3773111442Marek Olšák csize / 4)); 112cba9d59362130e1b44cd9cfc4f38ad3773111442Marek Olšák radeon_emit(cs, offset); 113cba9d59362130e1b44cd9cfc4f38ad3773111442Marek Olšák radeon_emit(cs, clear_value); 114cba9d59362130e1b44cd9cfc4f38ad3773111442Marek Olšák radeon_emit(cs, (offset >> 32) << 16); 115cba9d59362130e1b44cd9cfc4f38ad3773111442Marek Olšák offset += csize; 116cba9d59362130e1b44cd9cfc4f38ad3773111442Marek Olšák size -= csize; 117cba9d59362130e1b44cd9cfc4f38ad3773111442Marek Olšák } 118cba9d59362130e1b44cd9cfc4f38ad3773111442Marek Olšák} 119cba9d59362130e1b44cd9cfc4f38ad3773111442Marek Olšák 12071254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheiderstatic void si_dma_copy_tile(struct si_context *ctx, 12171254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider struct pipe_resource *dst, 12271254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider unsigned dst_level, 12371254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider unsigned dst_x, 12471254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider unsigned dst_y, 12571254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider unsigned dst_z, 12671254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider struct pipe_resource *src, 12771254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider unsigned src_level, 12871254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider unsigned src_x, 12971254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider unsigned src_y, 13071254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider unsigned src_z, 13171254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider unsigned copy_height, 13271254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider unsigned pitch, 13371254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider unsigned bpp) 13471254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider{ 1356cc8f6c6a72b1aab7bb506deb220e04ae50d8c2bMarek Olšák struct radeon_winsys_cs *cs = ctx->b.dma.cs; 13671254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider struct r600_texture *rsrc = (struct r600_texture*)src; 13771254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider struct r600_texture *rdst = (struct r600_texture*)dst; 1380c2cba1ec66485c441157fcaa72eb40efe6a8aefMarek Olšák unsigned dst_mode = rdst->surface.level[dst_level].mode; 1390c2cba1ec66485c441157fcaa72eb40efe6a8aefMarek Olšák bool detile = dst_mode == RADEON_SURF_MODE_LINEAR_ALIGNED; 1400c2cba1ec66485c441157fcaa72eb40efe6a8aefMarek Olšák struct r600_texture *rlinear = detile ? rdst : rsrc; 1410c2cba1ec66485c441157fcaa72eb40efe6a8aefMarek Olšák struct r600_texture *rtiled = detile ? rsrc : rdst; 1420c2cba1ec66485c441157fcaa72eb40efe6a8aefMarek Olšák unsigned linear_lvl = detile ? dst_level : src_level; 1430c2cba1ec66485c441157fcaa72eb40efe6a8aefMarek Olšák unsigned tiled_lvl = detile ? src_level : dst_level; 1440c2cba1ec66485c441157fcaa72eb40efe6a8aefMarek Olšák struct radeon_info *info = &ctx->screen->b.info; 1450c2cba1ec66485c441157fcaa72eb40efe6a8aefMarek Olšák unsigned index = rtiled->surface.tiling_index[tiled_lvl]; 1460c2cba1ec66485c441157fcaa72eb40efe6a8aefMarek Olšák unsigned tile_mode = info->si_tile_mode_array[index]; 14771254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider unsigned array_mode, lbpp, pitch_tile_max, slice_tile_max, size; 1480c2cba1ec66485c441157fcaa72eb40efe6a8aefMarek Olšák unsigned ncopy, height, cheight, i; 14982edcb918be3fdff12af3123269552d0829316edMichel Dänzer unsigned linear_x, linear_y, linear_z, tiled_x, tiled_y, tiled_z; 15071254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider unsigned sub_cmd, bank_h, bank_w, mt_aspect, nbanks, tile_split, mt; 15171254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider uint64_t base, addr; 1520c2cba1ec66485c441157fcaa72eb40efe6a8aefMarek Olšák unsigned pipe_config; 15371254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider 154b21912e2e92234d66c90b0f6f014a83698ef5ecbNicolai Hähnle assert(dst_mode != rsrc->surface.level[src_level].mode); 15571254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider 15671254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider sub_cmd = SI_DMA_COPY_TILED; 15771254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider lbpp = util_logbase2(bpp); 15871254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider pitch_tile_max = ((pitch / bpp) / 8) - 1; 15971254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider 16082edcb918be3fdff12af3123269552d0829316edMichel Dänzer linear_x = detile ? dst_x : src_x; 16182edcb918be3fdff12af3123269552d0829316edMichel Dänzer linear_y = detile ? dst_y : src_y; 16282edcb918be3fdff12af3123269552d0829316edMichel Dänzer linear_z = detile ? dst_z : src_z; 16382edcb918be3fdff12af3123269552d0829316edMichel Dänzer tiled_x = detile ? src_x : dst_x; 16482edcb918be3fdff12af3123269552d0829316edMichel Dänzer tiled_y = detile ? src_y : dst_y; 16582edcb918be3fdff12af3123269552d0829316edMichel Dänzer tiled_z = detile ? src_z : dst_z; 16682edcb918be3fdff12af3123269552d0829316edMichel Dänzer 167761d80ddab9ce854dc964b2023bc4fbc734fafc7Michel Dänzer assert(!util_format_is_depth_and_stencil(rtiled->resource.b.b.format)); 168761d80ddab9ce854dc964b2023bc4fbc734fafc7Michel Dänzer 1690c2cba1ec66485c441157fcaa72eb40efe6a8aefMarek Olšák array_mode = G_009910_ARRAY_MODE(tile_mode); 17082edcb918be3fdff12af3123269552d0829316edMichel Dänzer slice_tile_max = (rtiled->surface.level[tiled_lvl].nblk_x * 17182edcb918be3fdff12af3123269552d0829316edMichel Dänzer rtiled->surface.level[tiled_lvl].nblk_y) / (8*8) - 1; 17282edcb918be3fdff12af3123269552d0829316edMichel Dänzer /* linear height must be the same as the slice tile max height, it's ok even 17382edcb918be3fdff12af3123269552d0829316edMichel Dänzer * if the linear destination/source have smaller heigh as the size of the 17482edcb918be3fdff12af3123269552d0829316edMichel Dänzer * dma packet will be using the copy_height which is always smaller or equal 17582edcb918be3fdff12af3123269552d0829316edMichel Dänzer * to the linear height 17682edcb918be3fdff12af3123269552d0829316edMichel Dänzer */ 177d17b85524dfd74824a2135d5d4112a1fae86ed17Michel Dänzer height = rtiled->surface.level[tiled_lvl].nblk_y; 17882edcb918be3fdff12af3123269552d0829316edMichel Dänzer base = rtiled->surface.level[tiled_lvl].offset; 17982edcb918be3fdff12af3123269552d0829316edMichel Dänzer addr = rlinear->surface.level[linear_lvl].offset; 18082edcb918be3fdff12af3123269552d0829316edMichel Dänzer addr += rlinear->surface.level[linear_lvl].slice_size * linear_z; 18182edcb918be3fdff12af3123269552d0829316edMichel Dänzer addr += linear_y * pitch + linear_x * bpp; 1820c2cba1ec66485c441157fcaa72eb40efe6a8aefMarek Olšák bank_h = G_009910_BANK_HEIGHT(tile_mode); 1830c2cba1ec66485c441157fcaa72eb40efe6a8aefMarek Olšák bank_w = G_009910_BANK_WIDTH(tile_mode); 1840c2cba1ec66485c441157fcaa72eb40efe6a8aefMarek Olšák mt_aspect = G_009910_MACRO_TILE_ASPECT(tile_mode); 1850c2cba1ec66485c441157fcaa72eb40efe6a8aefMarek Olšák /* Non-depth modes don't have TILE_SPLIT set. */ 1860c2cba1ec66485c441157fcaa72eb40efe6a8aefMarek Olšák tile_split = util_logbase2(rtiled->surface.tile_split >> 6); 1870c2cba1ec66485c441157fcaa72eb40efe6a8aefMarek Olšák nbanks = G_009910_NUM_BANKS(tile_mode); 18882edcb918be3fdff12af3123269552d0829316edMichel Dänzer base += rtiled->resource.gpu_address; 18982edcb918be3fdff12af3123269552d0829316edMichel Dänzer addr += rlinear->resource.gpu_address; 19071254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider 1910c2cba1ec66485c441157fcaa72eb40efe6a8aefMarek Olšák pipe_config = G_009910_PIPE_CONFIG(tile_mode); 1920c2cba1ec66485c441157fcaa72eb40efe6a8aefMarek Olšák mt = G_009910_MICRO_TILE_MODE(tile_mode); 19329d6a367a6508a6814f44039166a65a4b69f53b1Marek Olšák size = copy_height * pitch; 19429d6a367a6508a6814f44039166a65a4b69f53b1Marek Olšák ncopy = DIV_ROUND_UP(size, SI_DMA_COPY_MAX_DWORD_ALIGNED_SIZE); 195bb74152597de44ee877b8928587b1cece8b49656Marek Olšák r600_need_dma_space(&ctx->b, ncopy * 9, &rdst->resource, &rsrc->resource); 19671254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider 19771254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider for (i = 0; i < ncopy; i++) { 19871254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider cheight = copy_height; 19929d6a367a6508a6814f44039166a65a4b69f53b1Marek Olšák if (cheight * pitch > SI_DMA_COPY_MAX_DWORD_ALIGNED_SIZE) { 20029d6a367a6508a6814f44039166a65a4b69f53b1Marek Olšák cheight = SI_DMA_COPY_MAX_DWORD_ALIGNED_SIZE / pitch; 20171254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider } 20229d6a367a6508a6814f44039166a65a4b69f53b1Marek Olšák size = cheight * pitch; 20329d6a367a6508a6814f44039166a65a4b69f53b1Marek Olšák radeon_emit(cs, SI_DMA_PACKET(SI_DMA_PACKET_COPY, sub_cmd, size / 4)); 204c23273532e711f3f0263bfff8bf8a0e733b90e12Nicolai Hähnle radeon_emit(cs, base >> 8); 205c23273532e711f3f0263bfff8bf8a0e733b90e12Nicolai Hähnle radeon_emit(cs, (detile << 31) | (array_mode << 27) | 206c23273532e711f3f0263bfff8bf8a0e733b90e12Nicolai Hähnle (lbpp << 24) | (bank_h << 21) | 207c23273532e711f3f0263bfff8bf8a0e733b90e12Nicolai Hähnle (bank_w << 18) | (mt_aspect << 16)); 208c23273532e711f3f0263bfff8bf8a0e733b90e12Nicolai Hähnle radeon_emit(cs, (pitch_tile_max << 0) | ((height - 1) << 16)); 209c23273532e711f3f0263bfff8bf8a0e733b90e12Nicolai Hähnle radeon_emit(cs, (slice_tile_max << 0) | (pipe_config << 26)); 210c23273532e711f3f0263bfff8bf8a0e733b90e12Nicolai Hähnle radeon_emit(cs, (tiled_x << 0) | (tiled_z << 18)); 211c23273532e711f3f0263bfff8bf8a0e733b90e12Nicolai Hähnle radeon_emit(cs, (tiled_y << 0) | (tile_split << 21) | (nbanks << 25) | (mt << 27)); 212c23273532e711f3f0263bfff8bf8a0e733b90e12Nicolai Hähnle radeon_emit(cs, addr & 0xfffffffc); 213c23273532e711f3f0263bfff8bf8a0e733b90e12Nicolai Hähnle radeon_emit(cs, (addr >> 32UL) & 0xff); 21471254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider copy_height -= cheight; 21571254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider addr += cheight * pitch; 21682edcb918be3fdff12af3123269552d0829316edMichel Dänzer tiled_y += cheight; 21771254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider } 21871254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider} 21971254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider 220498a40cae8023a8461e7cfe95cb837a0aa459337Marek Olšákstatic void si_dma_copy(struct pipe_context *ctx, 221498a40cae8023a8461e7cfe95cb837a0aa459337Marek Olšák struct pipe_resource *dst, 222498a40cae8023a8461e7cfe95cb837a0aa459337Marek Olšák unsigned dst_level, 223498a40cae8023a8461e7cfe95cb837a0aa459337Marek Olšák unsigned dstx, unsigned dsty, unsigned dstz, 224498a40cae8023a8461e7cfe95cb837a0aa459337Marek Olšák struct pipe_resource *src, 225498a40cae8023a8461e7cfe95cb837a0aa459337Marek Olšák unsigned src_level, 226498a40cae8023a8461e7cfe95cb837a0aa459337Marek Olšák const struct pipe_box *src_box) 22771254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider{ 22871254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider struct si_context *sctx = (struct si_context *)ctx; 22971254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider struct r600_texture *rsrc = (struct r600_texture*)src; 23071254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider struct r600_texture *rdst = (struct r600_texture*)dst; 23161128d750789fa5e3947cb50b165bac5ebb3265eMichel Dänzer unsigned dst_pitch, src_pitch, bpp, dst_mode, src_mode; 23271254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider unsigned src_w, dst_w; 23371254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider unsigned src_x, src_y; 23471254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider unsigned dst_x = dstx, dst_y = dsty, dst_z = dstz; 23571254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider 2366cc8f6c6a72b1aab7bb506deb220e04ae50d8c2bMarek Olšák if (sctx->b.dma.cs == NULL) { 23771254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider goto fallback; 23871254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider } 23971254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider 24071254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider if (dst->target == PIPE_BUFFER && src->target == PIPE_BUFFER) { 24171254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider si_dma_copy_buffer(sctx, dst, src, dst_x, src_box->x, src_box->width); 24271254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider return; 24371254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider } 24471254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider 245ae4536b4f71cbe76230ea7edc7eb4d6041e651b4Michel Dänzer /* XXX: Using the asynchronous DMA engine for multi-dimensional 246ae4536b4f71cbe76230ea7edc7eb4d6041e651b4Michel Dänzer * operations seems to cause random GPU lockups for various people. 247ae4536b4f71cbe76230ea7edc7eb4d6041e651b4Michel Dänzer * While the root cause for this might need to be fixed in the kernel, 248ae4536b4f71cbe76230ea7edc7eb4d6041e651b4Michel Dänzer * let's disable it for now. 249ae4536b4f71cbe76230ea7edc7eb4d6041e651b4Michel Dänzer * 250ae4536b4f71cbe76230ea7edc7eb4d6041e651b4Michel Dänzer * Before re-enabling this, please make sure you can hit all newly 251ae4536b4f71cbe76230ea7edc7eb4d6041e651b4Michel Dänzer * enabled paths in your testing, preferably with both piglit and real 252ae4536b4f71cbe76230ea7edc7eb4d6041e651b4Michel Dänzer * world apps, and get in touch with people on the bug reports below 253ae4536b4f71cbe76230ea7edc7eb4d6041e651b4Michel Dänzer * for stability testing. 254ae4536b4f71cbe76230ea7edc7eb4d6041e651b4Michel Dänzer * 255ae4536b4f71cbe76230ea7edc7eb4d6041e651b4Michel Dänzer * https://bugs.freedesktop.org/show_bug.cgi?id=85647 256ae4536b4f71cbe76230ea7edc7eb4d6041e651b4Michel Dänzer * https://bugs.freedesktop.org/show_bug.cgi?id=83500 257ae4536b4f71cbe76230ea7edc7eb4d6041e651b4Michel Dänzer */ 258ae4536b4f71cbe76230ea7edc7eb4d6041e651b4Michel Dänzer goto fallback; 259ae4536b4f71cbe76230ea7edc7eb4d6041e651b4Michel Dänzer 2602f173b8e13308bea0690684f841fad28ccc2e40eMarek Olšák if (src_box->depth > 1 || 2612f173b8e13308bea0690684f841fad28ccc2e40eMarek Olšák !r600_prepare_for_dma_blit(&sctx->b, rdst, dst_level, dstx, dsty, 2622f173b8e13308bea0690684f841fad28ccc2e40eMarek Olšák dstz, rsrc, src_level, src_box)) 26371254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider goto fallback; 26471254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider 26571254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider src_x = util_format_get_nblocksx(src->format, src_box->x); 26671254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider dst_x = util_format_get_nblocksx(src->format, dst_x); 26771254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider src_y = util_format_get_nblocksy(src->format, src_box->y); 26871254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider dst_y = util_format_get_nblocksy(src->format, dst_y); 26971254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider 27071254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider bpp = rdst->surface.bpe; 271e9c76eeeaa673331fec6056a4baa30095de42f5eMarek Olšák dst_pitch = rdst->surface.level[dst_level].nblk_x * rdst->surface.bpe; 272e9c76eeeaa673331fec6056a4baa30095de42f5eMarek Olšák src_pitch = rsrc->surface.level[src_level].nblk_x * rsrc->surface.bpe; 2737e73ff87c0255a8e0498a47991b640cdece35928Marek Olšák src_w = u_minify(rsrc->resource.b.b.width0, src_level); 2747e73ff87c0255a8e0498a47991b640cdece35928Marek Olšák dst_w = u_minify(rdst->resource.b.b.width0, dst_level); 27571254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider 27671254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider dst_mode = rdst->surface.level[dst_level].mode; 27771254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider src_mode = rsrc->surface.level[src_level].mode; 27871254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider 27974aeccd701c13851e69120d562fe5e899b10fb93Michel Dänzer if (src_pitch != dst_pitch || src_box->x || dst_x || src_w != dst_w || 28074aeccd701c13851e69120d562fe5e899b10fb93Michel Dänzer src_box->width != src_w || 2817e73ff87c0255a8e0498a47991b640cdece35928Marek Olšák src_box->height != u_minify(rsrc->resource.b.b.height0, src_level) || 2827e73ff87c0255a8e0498a47991b640cdece35928Marek Olšák src_box->height != u_minify(rdst->resource.b.b.height0, dst_level) || 28374aeccd701c13851e69120d562fe5e899b10fb93Michel Dänzer rsrc->surface.level[src_level].nblk_y != 28474aeccd701c13851e69120d562fe5e899b10fb93Michel Dänzer rdst->surface.level[dst_level].nblk_y) { 28571254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider /* FIXME si can do partial blit */ 28671254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider goto fallback; 28771254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider } 28871254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider /* the x test here are currently useless (because we don't support partial blit) 28971254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider * but keep them around so we don't forget about those 29071254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider */ 29174aeccd701c13851e69120d562fe5e899b10fb93Michel Dänzer if ((src_pitch % 8) || (src_box->x % 8) || (dst_x % 8) || 29274aeccd701c13851e69120d562fe5e899b10fb93Michel Dänzer (src_box->y % 8) || (dst_y % 8) || (src_box->height % 8)) { 29371254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider goto fallback; 29471254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider } 29571254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider 29671254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider if (src_mode == dst_mode) { 29771254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider uint64_t dst_offset, src_offset; 29871254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider /* simple dma blit would do NOTE code here assume : 29971254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider * src_box.x/y == 0 30071254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider * dst_x/y == 0 30171254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider * dst_pitch == src_pitch 30271254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider */ 30371254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider src_offset= rsrc->surface.level[src_level].offset; 30471254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider src_offset += rsrc->surface.level[src_level].slice_size * src_box->z; 30571254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider src_offset += src_y * src_pitch + src_x * bpp; 30671254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider dst_offset = rdst->surface.level[dst_level].offset; 30771254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider dst_offset += rdst->surface.level[dst_level].slice_size * dst_z; 30871254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider dst_offset += dst_y * dst_pitch + dst_x * bpp; 30971254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider si_dma_copy_buffer(sctx, dst, src, dst_offset, src_offset, 31061128d750789fa5e3947cb50b165bac5ebb3265eMichel Dänzer rsrc->surface.level[src_level].slice_size); 31171254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider } else { 31271254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider si_dma_copy_tile(sctx, dst, dst_level, dst_x, dst_y, dst_z, 31371254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider src, src_level, src_x, src_y, src_box->z, 31461128d750789fa5e3947cb50b165bac5ebb3265eMichel Dänzer src_box->height / rsrc->surface.blk_h, 31561128d750789fa5e3947cb50b165bac5ebb3265eMichel Dänzer dst_pitch, bpp); 31671254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider } 31771254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider return; 31871254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider 31971254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheiderfallback: 320d13d2fd16132f351ec7c8184f165faeac3b31bb4Marek Olšák si_resource_copy_region(ctx, dst, dst_level, dstx, dsty, dstz, 321d13d2fd16132f351ec7c8184f165faeac3b31bb4Marek Olšák src, src_level, src_box); 32271254732db12c8813c002425b5c1b7c31bf56f65Niels Ole Salscheider} 323498a40cae8023a8461e7cfe95cb837a0aa459337Marek Olšák 324498a40cae8023a8461e7cfe95cb837a0aa459337Marek Olšákvoid si_init_dma_functions(struct si_context *sctx) 325498a40cae8023a8461e7cfe95cb837a0aa459337Marek Olšák{ 326498a40cae8023a8461e7cfe95cb837a0aa459337Marek Olšák sctx->b.dma_copy = si_dma_copy; 327cba9d59362130e1b44cd9cfc4f38ad3773111442Marek Olšák sctx->b.dma_clear_buffer = si_dma_clear_buffer; 328498a40cae8023a8461e7cfe95cb837a0aa459337Marek Olšák} 329