nvk: enable rendering to DRM_FORMAT_MOD_LINEAR images

Reviewed-by: Dave Airlie <airlied@redhat.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/24795>

src/nouveau/vulkan/nvk_cmd_buffer.h

@@ -10,6 +10,7 @@
 #include "nv_push.h"
 #include "nvk_cmd_pool.h"
 #include "nvk_descriptor_set.h"
+#include "nvk_image.h"
 
 #include "util/u_dynarray.h"
 
@@ -83,6 +84,10 @@ struct nvk_attachment {
 
    VkResolveModeFlagBits resolve_mode;
    struct nvk_image_view *resolve_iview;
+
+   /* Needed to track the value of storeOp in case we need to copy images for
+    * the DRM_FORMAT_MOD_LINEAR case */
+   VkAttachmentStoreOp store_op;
 };
 
 struct nvk_rendering_state {
@@ -278,4 +283,9 @@ void nvk_meta_resolve_rendering(struct nvk_cmd_buffer *cmd,
 
 void nvk_cmd_buffer_dump(struct nvk_cmd_buffer *cmd, FILE *fp);
 
+void nvk_linear_render_copy(struct nvk_cmd_buffer *cmd,
+                            const struct nvk_image_view *iview,
+                            VkRect2D copy_rect,
+                            bool copy_to_tiled_shadow);
+
 #endif

src/nouveau/vulkan/nvk_cmd_copy.c

@@ -10,6 +10,7 @@
 #include "nvk_entrypoints.h"
 #include "nvk_format.h"
 #include "nvk_image.h"
+#include "nvk_image_view.h"
 #include "nvk_physical_device.h"
 
 #include "vk_format.h"
@@ -88,8 +89,8 @@ vk_to_nil_extent(VkExtent3D extent, uint32_t array_layers)
 }
 
 static struct nouveau_copy_buffer
-nouveau_copy_rect_image(struct nvk_image *img,
-                        struct nvk_image_plane *plane,
+nouveau_copy_rect_image(const struct nvk_image *img,
+                        const struct nvk_image_plane *plane,
                         VkOffset3D offset_px,
                         const VkImageSubresourceLayers *sub_res)
 {
@@ -593,6 +594,54 @@ nvk_CmdCopyImageToBuffer2(VkCommandBuffer commandBuffer,
    }
 }
 
+void
+nvk_linear_render_copy(struct nvk_cmd_buffer *cmd,
+                       const struct nvk_image_view *iview,
+                       VkRect2D copy_rect,
+                       bool copy_to_tiled_shadow)
+{
+   const struct nvk_image *image = (struct nvk_image *)iview->vk.image;
+
+   const uint8_t ip = iview->planes[0].image_plane;
+   const struct nvk_image_plane *src_plane = NULL, *dst_plane = NULL;
+   if (copy_to_tiled_shadow) {
+      src_plane = &image->planes[ip];
+      dst_plane = &image->linear_tiled_shadow;
+   } else {
+      src_plane = &image->linear_tiled_shadow;
+      dst_plane = &image->planes[ip];
+   }
+
+   const struct VkImageSubresourceLayers subres = {
+      .aspectMask = iview->vk.aspects,
+      .baseArrayLayer = iview->vk.base_array_layer,
+      .layerCount = iview->vk.layer_count,
+      .mipLevel = iview->vk.base_mip_level,
+   };
+
+   const VkOffset3D offset_px = {
+      .x = copy_rect.offset.x,
+      .y = copy_rect.offset.y,
+      .z = 0,
+   };
+   const struct nil_Extent4D_Pixels extent4d_px = {
+      .width = copy_rect.extent.width,
+      .height = copy_rect.extent.height,
+      .depth = 1,
+      .array_len = 1,
+   };
+
+   struct nouveau_copy copy = {
+      .src = nouveau_copy_rect_image(image, src_plane, offset_px, &subres),
+      .dst = nouveau_copy_rect_image(image, dst_plane, offset_px, &subres),
+      .extent_el = nil_extent4d_px_to_el(extent4d_px, src_plane->nil.format,
+                                         src_plane->nil.sample_layout),
+   };
+
+   copy.remap = nouveau_copy_remap_format(image->vk.format);
+   nouveau_copy_rect(cmd, &copy);
+}
+
 VKAPI_ATTR void VKAPI_CALL
 nvk_CmdCopyImage2(VkCommandBuffer commandBuffer,
                   const VkCopyImageInfo2 *pCopyImageInfo)

src/nouveau/vulkan/nvk_cmd_draw.c

@@ -608,6 +608,8 @@ nvk_attachment_init(struct nvk_attachment *att,
       att->resolve_mode = info->resolveMode;
       att->resolve_iview = res_iview;
    }
+
+   att->store_op = info->storeOp;
 }
 
 static uint32_t
@@ -636,6 +638,30 @@ nvk_GetRenderingAreaGranularityKHR(
    *pGranularity = (VkExtent2D) { .width = 1, .height = 1 };
 }
 
+static bool
+nvk_rendering_all_linear(const struct nvk_rendering_state *render)
+{
+   /* Depth and stencil are never linear */
+   if (render->depth_att.iview || render->stencil_att.iview)
+      return false;
+
+   for (uint32_t i = 0; i < render->color_att_count; i++) {
+      const struct nvk_image_view *iview = render->color_att[i].iview;
+      if (iview == NULL)
+         continue;
+
+      const struct nvk_image *image = (struct nvk_image *)iview->vk.image;
+      const uint8_t ip = iview->planes[0].image_plane;
+      const struct nil_image_level *level =
+         &image->planes[ip].nil.levels[iview->vk.base_mip_level];
+
+      if (level->tiling.is_tiled)
+         return false;
+   }
+
+   return true;
+}
+
 VKAPI_ATTR void VKAPI_CALL
 nvk_CmdBeginRendering(VkCommandBuffer commandBuffer,
                       const VkRenderingInfo *pRenderingInfo)
@@ -685,6 +711,8 @@ nvk_CmdBeginRendering(VkCommandBuffer commandBuffer,
       .height  = render->area.extent.height,
    });
 
+   const bool all_linear = nvk_rendering_all_linear(render);
+
    enum nil_sample_layout sample_layout = NIL_SAMPLE_LAYOUT_INVALID;
    for (uint32_t i = 0; i < color_att_count; i++) {
       if (render->color_att[i].iview) {
@@ -696,7 +724,18 @@ nvk_CmdBeginRendering(VkCommandBuffer commandBuffer,
           */
          assert(iview->plane_count == 1);
          const uint8_t ip = iview->planes[0].image_plane;
-         const struct nil_image *nil_image = &image->planes[ip].nil;
+         const struct nvk_image_plane *plane = &image->planes[ip];
+
+         const VkAttachmentLoadOp load_op =
+            pRenderingInfo->pColorAttachments[i].loadOp;
+         if (!all_linear && !plane->nil.levels[0].tiling.is_tiled) {
+            if (load_op == VK_ATTACHMENT_LOAD_OP_LOAD)
+               nvk_linear_render_copy(cmd, iview, render->area, true);
+
+            plane = &image->linear_tiled_shadow;
+         }
+
+         const struct nil_image *nil_image = &plane->nil;
          const struct nil_image_level *level =
             &nil_image->levels[iview->vk.base_mip_level];
          struct nil_Extent4D_Samples level_extent_sa =
@@ -707,7 +746,7 @@ nvk_CmdBeginRendering(VkCommandBuffer commandBuffer,
          sample_layout = nil_image->sample_layout;
          render->samples = image->vk.samples;
 
-         uint64_t addr = nvk_image_base_address(image, ip) + level->offset_B;
+         uint64_t addr = nvk_image_plane_base_address(plane) + level->offset_B;
 
          if (nil_image->dim == NIL_IMAGE_DIM_3D) {
             addr += nil_image_level_z_offset_B(nil_image,
@@ -853,6 +892,7 @@ nvk_CmdBeginRendering(VkCommandBuffer commandBuffer,
          vk_format_to_pipe_format(iview->vk.format);
       const uint8_t zs_format = nil_format_to_depth_stencil(p_format);
       P_NV9097_SET_ZT_FORMAT(p, zs_format);
+      assert(level->tiling.is_tiled);
       assert(level->tiling.z_log2 == 0);
       P_NV9097_SET_ZT_BLOCK_SIZE(p, {
          .width = WIDTH_ONE_GOB,
@@ -980,6 +1020,20 @@ nvk_CmdEndRendering(VkCommandBuffer commandBuffer)
    VK_FROM_HANDLE(nvk_cmd_buffer, cmd, commandBuffer);
    struct nvk_rendering_state *render = &cmd->state.gfx.render;
 
+   const bool all_linear = nvk_rendering_all_linear(render);
+   for (uint32_t i = 0; i < render->color_att_count; i++) {
+      struct nvk_image_view *iview = render->color_att[i].iview;
+      if (iview == NULL)
+         continue;
+
+      struct nvk_image *image = (struct nvk_image *)iview->vk.image;
+      const uint8_t ip = iview->planes[0].image_plane;
+      const struct nvk_image_plane *plane = &image->planes[ip];
+      if (!all_linear && !plane->nil.levels[0].tiling.is_tiled &&
+          render->color_att[i].store_op == VK_ATTACHMENT_STORE_OP_STORE)
+         nvk_linear_render_copy(cmd, iview, render->area, false);
+   }
+
    bool need_resolve = false;
 
    /* Translate render state back to VK for meta */

src/nouveau/vulkan/nvk_device_memory.c

@@ -168,7 +168,6 @@ nvk_AllocateMemory(VkDevice device,
    if (!mem)
       return vk_error(dev, VK_ERROR_OUT_OF_HOST_MEMORY);
 
-
    mem->map = NULL;
    if (fd_info && fd_info->handleType) {
       assert(fd_info->handleType ==

src/nouveau/vulkan/nvk_image.c

@@ -730,6 +730,28 @@ nvk_image_init(struct nvk_device *dev,
                                            mod_list_info->pDrmFormatModifiers);
          assert(image->vk.drm_format_mod != DRM_FORMAT_MOD_INVALID);
       }
+
+      if (image->vk.drm_format_mod == DRM_FORMAT_MOD_LINEAR) {
+         /* We only have one shadow plane per nvk_image */
+         assert(image->plane_count == 1);
+
+         struct nil_image_init_info tiled_shadow_nil_info = {
+            .dim = vk_image_type_to_nil_dim(pCreateInfo->imageType),
+            .format = nil_format(vk_format_to_pipe_format(image->vk.format)),
+            .modifier = DRM_FORMAT_MOD_INVALID,
+            .extent_px = {
+               .width = pCreateInfo->extent.width,
+               .height = pCreateInfo->extent.height,
+               .depth = pCreateInfo->extent.depth,
+               .array_len = pCreateInfo->arrayLayers,
+            },
+            .levels = pCreateInfo->mipLevels,
+            .samples = pCreateInfo->samples,
+            .usage = usage & ~NIL_IMAGE_USAGE_LINEAR_BIT,
+         };
+         image->linear_tiled_shadow.nil =
+            nil_image_new(&pdev->info, &tiled_shadow_nil_info);
+      }
    }
 
    const struct vk_format_ycbcr_info *ycbcr_info =
@@ -838,6 +860,11 @@ nvk_image_finish(struct nvk_device *dev, struct nvk_image *image,
                              image->vk.create_flags, pAllocator);
    }
 
+   if (image->linear_tiled_shadow.nil.size_B > 0) {
+      assert(image->linear_tiled_shadow.vma_size_B == 0);
+      nouveau_ws_bo_destroy(image->linear_tiled_shadow_bo);
+   }
+
    vk_image_finish(&image->vk);
 }
 
@@ -898,6 +925,22 @@ nvk_CreateImage(VkDevice _device,
       }
    }
 
+   if (image->linear_tiled_shadow.nil.size_B > 0) {
+      struct nvk_image_plane *shadow = &image->linear_tiled_shadow;
+      image->linear_tiled_shadow_bo =
+         nouveau_ws_bo_new_tiled(dev->ws_dev,
+                                 shadow->nil.size_B, shadow->nil.align_B,
+                                 shadow->nil.pte_kind, shadow->nil.tile_mode,
+                                 NOUVEAU_WS_BO_LOCAL);
+      if (image->linear_tiled_shadow_bo == NULL) {
+         nvk_image_finish(dev, image, pAllocator);
+         vk_free2(&dev->vk.alloc, pAllocator, image);
+         return vk_errorf(pdev, VK_ERROR_OUT_OF_DEVICE_MEMORY,
+                          "Failed to allocate tiled shadow image");
+      }
+      shadow->addr = image->linear_tiled_shadow_bo->offset;
+   }
+
    *pImage = nvk_image_to_handle(image);
 
    return VK_SUCCESS;

src/nouveau/vulkan/nvk_image.h

@@ -6,6 +6,7 @@
 #define NVK_IMAGE_H 1
 
 #include "nvk_private.h"
+#include "nvk_device_memory.h"
 
 #include "vk_image.h"
 
@@ -78,6 +79,14 @@ struct nvk_image {
     * copied again down to the 8-bit result.
     */
    struct nvk_image_plane stencil_copy_temp;
+
+   /* The hardware doesn't support rendering to linear images except
+    * under certain conditions, so to support DRM_FORMAT_MOD_LINEAR
+    * rendering in the general case, we need to keep a tiled copy, which would
+    * be used to fake support if the conditions aren't satisfied.
+    */
+   struct nvk_image_plane linear_tiled_shadow;
+   struct nouveau_ws_bo *linear_tiled_shadow_bo;
 };
 
 VK_DEFINE_NONDISP_HANDLE_CASTS(nvk_image, vk.base, VkImage, VK_OBJECT_TYPE_IMAGE)