diff --git a/src/amd/vulkan/radv_descriptor_set.c b/src/amd/vulkan/radv_descriptor_set.c
index 81df56438e7..6c167d81797 100644
--- a/src/amd/vulkan/radv_descriptor_set.c
+++ b/src/amd/vulkan/radv_descriptor_set.c
@@ -679,8 +679,11 @@ radv_descriptor_set_create(struct radv_device *device, struct radv_descriptor_po
    layout_size = align_u32(layout_size, 32);
    set->header.size = layout_size;
 
-   if (!pool->host_memory_base && pool->entry_count == pool->max_entry_count) {
-      vk_free2(&device->vk.alloc, NULL, set);
+   if (pool->entry_count == pool->max_entry_count) {
+      if (!pool->host_memory_base) {
+         vk_free2(&device->vk.alloc, NULL, set);
+      }
+
       return VK_ERROR_OUT_OF_POOL_MEMORY;
    }