1/*
2 * Copyright © 2016 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24#include "radv_meta.h"
25#include "vk_format.h"
26
27static VkExtent3D
28meta_image_block_size(const struct radv_image *image)
29{
30	const struct vk_format_description *desc = vk_format_description(image->vk_format);
31	return (VkExtent3D) { desc->block.width, desc->block.height, 1 };
32}
33
34/* Returns the user-provided VkBufferImageCopy::imageExtent in units of
35 * elements rather than texels. One element equals one texel or one block
36 * if Image is uncompressed or compressed, respectively.
37 */
38static struct VkExtent3D
39meta_region_extent_el(const struct radv_image *image,
40                      const struct VkExtent3D *extent)
41{
42	const VkExtent3D block = meta_image_block_size(image);
43	return radv_sanitize_image_extent(image->type, (VkExtent3D) {
44			.width  = DIV_ROUND_UP(extent->width , block.width),
45				.height = DIV_ROUND_UP(extent->height, block.height),
46				.depth  = DIV_ROUND_UP(extent->depth , block.depth),
47				});
48}
49
50/* Returns the user-provided VkBufferImageCopy::imageOffset in units of
51 * elements rather than texels. One element equals one texel or one block
52 * if Image is uncompressed or compressed, respectively.
53 */
54static struct VkOffset3D
55meta_region_offset_el(const struct radv_image *image,
56                      const struct VkOffset3D *offset)
57{
58	const VkExtent3D block = meta_image_block_size(image);
59	return radv_sanitize_image_offset(image->type, (VkOffset3D) {
60			.x = offset->x / block.width,
61				.y = offset->y / block.height,
62				.z = offset->z / block.depth,
63				});
64}
65
66static VkFormat
67vk_format_for_size(int bs)
68{
69	switch (bs) {
70	case 1: return VK_FORMAT_R8_UINT;
71	case 2: return VK_FORMAT_R8G8_UINT;
72	case 4: return VK_FORMAT_R8G8B8A8_UINT;
73	case 8: return VK_FORMAT_R16G16B16A16_UINT;
74	case 16: return VK_FORMAT_R32G32B32A32_UINT;
75	default:
76		unreachable("Invalid format block size");
77	}
78}
79
80static struct radv_meta_blit2d_surf
81blit_surf_for_image_level_layer(struct radv_image *image,
82				const VkImageSubresourceLayers *subres)
83{
84	VkFormat format = image->vk_format;
85	if (subres->aspectMask & VK_IMAGE_ASPECT_DEPTH_BIT)
86		format = vk_format_depth_only(format);
87	else if (subres->aspectMask & VK_IMAGE_ASPECT_STENCIL_BIT)
88		format = vk_format_stencil_only(format);
89
90	if (!image->surface.dcc_size)
91		format = vk_format_for_size(vk_format_get_blocksize(format));
92
93	return (struct radv_meta_blit2d_surf) {
94		.format = format,
95		.bs = vk_format_get_blocksize(format),
96		.level = subres->mipLevel,
97		.layer = subres->baseArrayLayer,
98		.image = image,
99		.aspect_mask = subres->aspectMask,
100	};
101}
102
103union meta_saved_state {
104	struct radv_meta_saved_state gfx;
105	struct radv_meta_saved_compute_state compute;
106};
107
108static void
109meta_copy_buffer_to_image(struct radv_cmd_buffer *cmd_buffer,
110                          struct radv_buffer* buffer,
111                          struct radv_image* image,
112                          uint32_t regionCount,
113                          const VkBufferImageCopy* pRegions)
114{
115	bool cs = cmd_buffer->queue_family_index == RADV_QUEUE_COMPUTE;
116	union meta_saved_state saved_state;
117
118	/* The Vulkan 1.0 spec says "dstImage must have a sample count equal to
119	 * VK_SAMPLE_COUNT_1_BIT."
120	 */
121	assert(image->samples == 1);
122
123	if (cs)
124		radv_meta_begin_bufimage(cmd_buffer, &saved_state.compute);
125	else
126		radv_meta_save_graphics_reset_vport_scissor(&saved_state.gfx, cmd_buffer);
127
128	for (unsigned r = 0; r < regionCount; r++) {
129
130		/**
131		 * From the Vulkan 1.0.6 spec: 18.3 Copying Data Between Images
132		 *    extent is the size in texels of the source image to copy in width,
133		 *    height and depth. 1D images use only x and width. 2D images use x, y,
134		 *    width and height. 3D images use x, y, z, width, height and depth.
135		 *
136		 *
137		 * Also, convert the offsets and extent from units of texels to units of
138		 * blocks - which is the highest resolution accessible in this command.
139		 */
140		const VkOffset3D img_offset_el =
141			meta_region_offset_el(image, &pRegions[r].imageOffset);
142		const VkExtent3D bufferExtent = {
143			.width  = pRegions[r].bufferRowLength ?
144			pRegions[r].bufferRowLength : pRegions[r].imageExtent.width,
145			.height = pRegions[r].bufferImageHeight ?
146			pRegions[r].bufferImageHeight : pRegions[r].imageExtent.height,
147		};
148		const VkExtent3D buf_extent_el =
149			meta_region_extent_el(image, &bufferExtent);
150
151		/* Start creating blit rect */
152		const VkExtent3D img_extent_el =
153			meta_region_extent_el(image, &pRegions[r].imageExtent);
154		struct radv_meta_blit2d_rect rect = {
155			.width = img_extent_el.width,
156			.height =  img_extent_el.height,
157		};
158
159		/* Create blit surfaces */
160		struct radv_meta_blit2d_surf img_bsurf =
161			blit_surf_for_image_level_layer(image,
162							&pRegions[r].imageSubresource);
163
164		struct radv_meta_blit2d_buffer buf_bsurf = {
165			.bs = img_bsurf.bs,
166			.format = img_bsurf.format,
167			.buffer = buffer,
168			.offset = pRegions[r].bufferOffset,
169			.pitch = buf_extent_el.width,
170		};
171
172		/* Loop through each 3D or array slice */
173		unsigned num_slices_3d = img_extent_el.depth;
174		unsigned num_slices_array = pRegions[r].imageSubresource.layerCount;
175		unsigned slice_3d = 0;
176		unsigned slice_array = 0;
177		while (slice_3d < num_slices_3d && slice_array < num_slices_array) {
178
179			rect.dst_x = img_offset_el.x;
180			rect.dst_y = img_offset_el.y;
181
182
183			/* Perform Blit */
184			if (cs)
185				radv_meta_buffer_to_image_cs(cmd_buffer, &buf_bsurf, &img_bsurf, 1, &rect);
186			else
187				radv_meta_blit2d(cmd_buffer, NULL, &buf_bsurf, &img_bsurf, 1, &rect);
188
189			/* Once we've done the blit, all of the actual information about
190			 * the image is embedded in the command buffer so we can just
191			 * increment the offset directly in the image effectively
192			 * re-binding it to different backing memory.
193			 */
194			buf_bsurf.offset += buf_extent_el.width *
195			                    buf_extent_el.height * buf_bsurf.bs;
196			img_bsurf.layer++;
197			if (image->type == VK_IMAGE_TYPE_3D)
198				slice_3d++;
199			else
200				slice_array++;
201		}
202	}
203	if (cs)
204		radv_meta_end_bufimage(cmd_buffer, &saved_state.compute);
205	else
206		radv_meta_restore(&saved_state.gfx, cmd_buffer);
207}
208
209void radv_CmdCopyBufferToImage(
210	VkCommandBuffer                             commandBuffer,
211	VkBuffer                                    srcBuffer,
212	VkImage                                     destImage,
213	VkImageLayout                               destImageLayout,
214	uint32_t                                    regionCount,
215	const VkBufferImageCopy*                    pRegions)
216{
217	RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
218	RADV_FROM_HANDLE(radv_image, dest_image, destImage);
219	RADV_FROM_HANDLE(radv_buffer, src_buffer, srcBuffer);
220
221	meta_copy_buffer_to_image(cmd_buffer, src_buffer, dest_image,
222				  regionCount, pRegions);
223}
224
225static void
226meta_copy_image_to_buffer(struct radv_cmd_buffer *cmd_buffer,
227                          struct radv_buffer* buffer,
228                          struct radv_image* image,
229                          uint32_t regionCount,
230                          const VkBufferImageCopy* pRegions)
231{
232	struct radv_meta_saved_compute_state saved_state;
233
234	radv_meta_begin_bufimage(cmd_buffer, &saved_state);
235	for (unsigned r = 0; r < regionCount; r++) {
236
237		/**
238		 * From the Vulkan 1.0.6 spec: 18.3 Copying Data Between Images
239		 *    extent is the size in texels of the source image to copy in width,
240		 *    height and depth. 1D images use only x and width. 2D images use x, y,
241		 *    width and height. 3D images use x, y, z, width, height and depth.
242		 *
243		 *
244		 * Also, convert the offsets and extent from units of texels to units of
245		 * blocks - which is the highest resolution accessible in this command.
246		 */
247		const VkOffset3D img_offset_el =
248			meta_region_offset_el(image, &pRegions[r].imageOffset);
249		const VkExtent3D bufferExtent = {
250			.width  = pRegions[r].bufferRowLength ?
251			pRegions[r].bufferRowLength : pRegions[r].imageExtent.width,
252			.height = pRegions[r].bufferImageHeight ?
253			pRegions[r].bufferImageHeight : pRegions[r].imageExtent.height,
254		};
255		const VkExtent3D buf_extent_el =
256			meta_region_extent_el(image, &bufferExtent);
257
258		/* Start creating blit rect */
259		const VkExtent3D img_extent_el =
260			meta_region_extent_el(image, &pRegions[r].imageExtent);
261		struct radv_meta_blit2d_rect rect = {
262			.width = img_extent_el.width,
263			.height =  img_extent_el.height,
264		};
265
266		/* Create blit surfaces */
267		struct radv_meta_blit2d_surf img_info =
268			blit_surf_for_image_level_layer(image,
269							&pRegions[r].imageSubresource);
270
271		struct radv_meta_blit2d_buffer buf_info = {
272			.bs = img_info.bs,
273			.format = img_info.format,
274			.buffer = buffer,
275			.offset = pRegions[r].bufferOffset,
276			.pitch = buf_extent_el.width,
277		};
278
279		/* Loop through each 3D or array slice */
280		unsigned num_slices_3d = img_extent_el.depth;
281		unsigned num_slices_array = pRegions[r].imageSubresource.layerCount;
282		unsigned slice_3d = 0;
283		unsigned slice_array = 0;
284		while (slice_3d < num_slices_3d && slice_array < num_slices_array) {
285
286			rect.src_x = img_offset_el.x;
287			rect.src_y = img_offset_el.y;
288
289
290			/* Perform Blit */
291			radv_meta_image_to_buffer(cmd_buffer, &img_info, &buf_info, 1, &rect);
292
293			buf_info.offset += buf_extent_el.width *
294			                    buf_extent_el.height * buf_info.bs;
295			img_info.layer++;
296			if (image->type == VK_IMAGE_TYPE_3D)
297				slice_3d++;
298			else
299				slice_array++;
300		}
301	}
302	radv_meta_end_bufimage(cmd_buffer, &saved_state);
303}
304
305void radv_CmdCopyImageToBuffer(
306	VkCommandBuffer                             commandBuffer,
307	VkImage                                     srcImage,
308	VkImageLayout                               srcImageLayout,
309	VkBuffer                                    destBuffer,
310	uint32_t                                    regionCount,
311	const VkBufferImageCopy*                    pRegions)
312{
313	RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
314	RADV_FROM_HANDLE(radv_image, src_image, srcImage);
315	RADV_FROM_HANDLE(radv_buffer, dst_buffer, destBuffer);
316
317	meta_copy_image_to_buffer(cmd_buffer, dst_buffer, src_image,
318				  regionCount, pRegions);
319}
320
321static void
322meta_copy_image(struct radv_cmd_buffer *cmd_buffer,
323		struct radv_image *src_image,
324		struct radv_image *dest_image,
325		uint32_t regionCount,
326		const VkImageCopy *pRegions)
327{
328	bool cs = cmd_buffer->queue_family_index == RADV_QUEUE_COMPUTE;
329	union meta_saved_state saved_state;
330
331	/* From the Vulkan 1.0 spec:
332	 *
333	 *    vkCmdCopyImage can be used to copy image data between multisample
334	 *    images, but both images must have the same number of samples.
335	 */
336	assert(src_image->samples == dest_image->samples);
337	if (cs)
338		radv_meta_begin_itoi(cmd_buffer, &saved_state.compute);
339	else
340		radv_meta_save_graphics_reset_vport_scissor(&saved_state.gfx, cmd_buffer);
341
342	for (unsigned r = 0; r < regionCount; r++) {
343		assert(pRegions[r].srcSubresource.aspectMask ==
344		       pRegions[r].dstSubresource.aspectMask);
345
346		/* Create blit surfaces */
347		struct radv_meta_blit2d_surf b_src =
348			blit_surf_for_image_level_layer(src_image,
349							&pRegions[r].srcSubresource);
350
351		struct radv_meta_blit2d_surf b_dst =
352			blit_surf_for_image_level_layer(dest_image,
353							&pRegions[r].dstSubresource);
354
355		/* for DCC */
356		b_src.format = b_dst.format;
357
358		/**
359		 * From the Vulkan 1.0.6 spec: 18.4 Copying Data Between Buffers and Images
360		 *    imageExtent is the size in texels of the image to copy in width, height
361		 *    and depth. 1D images use only x and width. 2D images use x, y, width
362		 *    and height. 3D images use x, y, z, width, height and depth.
363		 *
364		 * Also, convert the offsets and extent from units of texels to units of
365		 * blocks - which is the highest resolution accessible in this command.
366		 */
367		const VkOffset3D dst_offset_el =
368			meta_region_offset_el(dest_image, &pRegions[r].dstOffset);
369		const VkOffset3D src_offset_el =
370			meta_region_offset_el(src_image, &pRegions[r].srcOffset);
371		const VkExtent3D img_extent_el =
372			meta_region_extent_el(src_image, &pRegions[r].extent);
373
374		/* Start creating blit rect */
375		struct radv_meta_blit2d_rect rect = {
376			.width = img_extent_el.width,
377			.height = img_extent_el.height,
378		};
379
380		/* Loop through each 3D or array slice */
381		unsigned num_slices_3d = img_extent_el.depth;
382		unsigned num_slices_array = pRegions[r].dstSubresource.layerCount;
383		unsigned slice_3d = 0;
384		unsigned slice_array = 0;
385		while (slice_3d < num_slices_3d && slice_array < num_slices_array) {
386
387			/* Finish creating blit rect */
388			rect.dst_x = dst_offset_el.x;
389			rect.dst_y = dst_offset_el.y;
390			rect.src_x = src_offset_el.x;
391			rect.src_y = src_offset_el.y;
392
393			/* Perform Blit */
394			if (cs)
395				radv_meta_image_to_image_cs(cmd_buffer, &b_src, &b_dst, 1, &rect);
396			else
397				radv_meta_blit2d(cmd_buffer, &b_src, NULL, &b_dst, 1, &rect);
398
399			b_src.layer++;
400			b_dst.layer++;
401			if (dest_image->type == VK_IMAGE_TYPE_3D)
402				slice_3d++;
403			else
404				slice_array++;
405		}
406	}
407
408	if (cs)
409		radv_meta_end_itoi(cmd_buffer, &saved_state.compute);
410	else
411		radv_meta_restore(&saved_state.gfx, cmd_buffer);
412}
413
414void radv_CmdCopyImage(
415	VkCommandBuffer                             commandBuffer,
416	VkImage                                     srcImage,
417	VkImageLayout                               srcImageLayout,
418	VkImage                                     destImage,
419	VkImageLayout                               destImageLayout,
420	uint32_t                                    regionCount,
421	const VkImageCopy*                          pRegions)
422{
423	RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
424	RADV_FROM_HANDLE(radv_image, src_image, srcImage);
425	RADV_FROM_HANDLE(radv_image, dest_image, destImage);
426
427	meta_copy_image(cmd_buffer, src_image, dest_image,
428			regionCount, pRegions);
429}
430