st/nine: Retry allocations after freeing some space

Managed resources can be released whenever we need.
When we have an allocation failure, free them and retry.
Try also to flush the csmt queue in case there were some things released.

Signed-off-by: Axel Davy <davyaxel0@gmail.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/5015>

src/gallium/frontends/nine/basetexture9.c

@@ -446,7 +446,7 @@ NineBaseTexture9_CreatePipeResource( struct NineBaseTexture9 *This,
             return D3D_OK;
     }
 
-    res = screen->resource_create(screen, &templ);
+    res = nine_resource_create_with_retry(This->base.base.device, screen, &templ);
     if (!res)
         return D3DERR_OUTOFVIDEOMEMORY;
     This->base.resource = res;
@@ -597,6 +597,7 @@ NineBaseTexture9_UnLoad( struct NineBaseTexture9 *This )
         This->managed.lod_resident == -1)
         return;
 
+    DBG("This=%p, releasing resource\n", This);
     pipe_resource_reference(&This->base.resource, NULL);
     This->managed.lod_resident = -1;
     This->managed.dirty = TRUE;

src/gallium/frontends/nine/buffer9.c

@@ -385,7 +385,7 @@ NineBuffer9_Lock( struct NineBuffer9 *This,
      * with the secondary pipe, instead of waiting on the main pipe. */
     if (Flags & D3DLOCK_DISCARD && device->csmt_active) {
         struct pipe_screen *screen = NineDevice9_GetScreen(device);
-        struct pipe_resource *new_res = screen->resource_create(screen, &This->base.info);
+        struct pipe_resource *new_res = nine_resource_create_with_retry(device, screen, &This->base.info);
         if (new_res) {
             /* Use the new resource */
             pipe_resource_reference(&This->base.resource, new_res);

src/gallium/frontends/nine/device9.c

@@ -83,6 +83,28 @@ static void nine_setup_fpu(void)
 
 #endif
 
+struct pipe_resource *
+nine_resource_create_with_retry( struct NineDevice9 *This,
+                                 struct pipe_screen *screen,
+                                 const struct pipe_resource *templat )
+{
+    struct pipe_resource *res;
+    res = screen->resource_create(screen, templat);
+    if (res)
+        return res;
+    /* Allocation failed, retry after freeing some resources
+     * Note: Shouldn't be called from the worker thread */
+    if (!This)
+        return NULL;
+    /* Evict resources we can evict */
+    NineDevice9_EvictManagedResourcesInternal(This);
+    /* Execute anything pending, such that some
+     * deleted resources can be actually freed */
+    nine_csmt_process(This);
+    /* We could also finish the context, if needed */
+    return screen->resource_create(screen, templat);
+}
+
 void
 NineDevice9_SetDefaultState( struct NineDevice9 *This, boolean is_reset )
 {
@@ -652,6 +674,25 @@ NineDevice9_GetAvailableTextureMem( struct NineDevice9 *This )
     return This->available_texture_mem;
 }
 
+void
+NineDevice9_EvictManagedResourcesInternal( struct NineDevice9 *This )
+{
+    struct NineBaseTexture9 *tex;
+
+    DBG("This=%p\n", This);
+
+    /* This function is called internally when an allocation fails.
+     * We are supposed to release old unused managed textures/buffers,
+     * until we have enough space for the allocation.
+     * For now just release everything, except the bound textures,
+     * as this function can be called when uploading bound textures.
+     */
+    LIST_FOR_EACH_ENTRY(tex, &This->managed_textures, list2) {
+        if (!tex->bind_count)
+            NineBaseTexture9_UnLoad(tex);
+    }
+}
+
 HRESULT NINE_WINAPI
 NineDevice9_EvictManagedResources( struct NineDevice9 *This )
 {

src/gallium/frontends/nine/device9.h

@@ -198,6 +198,11 @@ void
 NineDevice9_dtor( struct NineDevice9 *This );
 
 /*** Nine private ***/
+struct pipe_resource *
+nine_resource_create_with_retry( struct NineDevice9 *This,
+                                 struct pipe_screen *screen,
+                                 const struct pipe_resource *templat );
+
 void
 NineDevice9_SetDefaultState( struct NineDevice9 *This, boolean is_reset );
 
@@ -210,6 +215,9 @@ NineDevice9_GetPipe( struct NineDevice9 *This );
 const D3DCAPS9 *
 NineDevice9_GetCaps( struct NineDevice9 *This );
 
+void
+NineDevice9_EvictManagedResourcesInternal( struct NineDevice9 *This );
+
 /*** Direct3D public ***/
 
 HRESULT NINE_WINAPI

src/gallium/frontends/nine/resource9.c

@@ -80,18 +80,28 @@ NineResource9_ctor( struct NineResource9 *This,
             This->size = util_resource_size(&This->info);
 
             p_atomic_add(&This->base.device->available_texture_mem, -This->size);
-            if (This->base.device->available_texture_mem <=
+            /* Before failing allocation, evict MANAGED memory */
+            if (This->base.device &&
+                p_atomic_read(&This->base.device->available_texture_mem) <=
+                    This->base.device->available_texture_limit)
+                NineDevice9_EvictManagedResourcesInternal(This->base.device);
+            if (p_atomic_read(&This->base.device->available_texture_mem) <=
                     This->base.device->available_texture_limit) {
+                DBG("Memory allocation failure: software limit\n");
                 return D3DERR_OUTOFVIDEOMEMORY;
             }
         }
 
         DBG("(%p) Creating pipe_resource.\n", This);
-        This->resource = screen->resource_create(screen, &This->info);
+        This->resource = nine_resource_create_with_retry(This->base.device, screen, &This->info);
         if (!This->resource)
             return D3DERR_OUTOFVIDEOMEMORY;
     }
 
+    DBG("Current texture memory count: (%d/%d)KB\n",
+        (int)(This->base.device->available_texture_mem >> 10),
+        (int)(This->base.device->available_texture_limit >> 10));
+
     This->type = Type;
     This->pool = Pool;
     This->usage = Usage;

src/gallium/frontends/nine/swapchain9.c

@@ -363,7 +363,7 @@ NineSwapChain9_Resize( struct NineSwapChain9 *This,
                                                    tmplt.bind, FALSE, FALSE);
         if (tmplt.format == PIPE_FORMAT_NONE)
             return D3DERR_INVALIDCALL;
-        resource = This->screen->resource_create(This->screen, &tmplt);
+        resource = nine_resource_create_with_retry(pDevice, This->screen, &tmplt);
         if (!resource) {
             DBG("Failed to create pipe_resource.\n");
             return D3DERR_OUTOFVIDEOMEMORY;
@@ -397,7 +397,7 @@ NineSwapChain9_Resize( struct NineSwapChain9 *This,
                 tmplt.bind |= PIPE_BIND_LINEAR;
             if (pParams->SwapEffect != D3DSWAPEFFECT_DISCARD)
                 tmplt.bind |= PIPE_BIND_RENDER_TARGET;
-            resource = This->screen->resource_create(This->screen, &tmplt);
+            resource = nine_resource_create_with_retry(pDevice, This->screen, &tmplt);
             pipe_resource_reference(&(This->present_buffers[i]), resource);
         }
         This->present_handles[i] = D3DWindowBuffer_create(This, resource, depth, false);
@@ -421,7 +421,7 @@ NineSwapChain9_Resize( struct NineSwapChain9 *This,
             return D3DERR_INVALIDCALL;
 
         if (This->zsbuf) {
-            resource = This->screen->resource_create(This->screen, &tmplt);
+            resource = nine_resource_create_with_retry(pDevice, This->screen, &tmplt);
             if (!resource) {
                 DBG("Failed to create pipe_resource for depth buffer.\n");
                 return D3DERR_OUTOFVIDEOMEMORY;
@@ -606,7 +606,7 @@ create_present_buffer( struct NineSwapChain9 *This,
     tmplt.nr_samples = 0;
     if (This->actx->linear_framebuffer)
         tmplt.bind |= PIPE_BIND_LINEAR;
-    *resource = This->screen->resource_create(This->screen, &tmplt);
+    *resource = nine_resource_create_with_retry(This->base.device, This->screen, &tmplt);
 
     *present_handle = D3DWindowBuffer_create(This, *resource, 24, true);