util/fossilize_db: Add extra flock mutex.

The flock is per-fd, not per thread, and we do it outside of the main mutex. This was
done to avoid having to wait in the mutex, but we can get a case where one ends up running
the body with the flock unlocked.

Fix this by adding a mutex that doesn't need to be locked for reads.

Fixes: 4f0f8133a3 "util/fossilize_db: Do not lock the fossilize db permanently."
Reviewed-by: Timothy Arceri <tarceri@itsqueeze.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/12266>

src/util/fossilize_db.c

@@ -286,6 +286,7 @@ foz_prepare(struct foz_db *foz_db, char *cache_path)
       return false;
 
    simple_mtx_init(&foz_db->mtx, mtx_plain);
+   simple_mtx_init(&foz_db->flock_mtx, mtx_plain);
    foz_db->mem_ctx = ralloc_context(NULL);
    foz_db->index_db = _mesa_hash_table_u64_create(NULL);
 
@@ -348,6 +349,7 @@ foz_destroy(struct foz_db *foz_db)
    if (foz_db->mem_ctx) {
       _mesa_hash_table_u64_destroy(foz_db->index_db);
       ralloc_free(foz_db->mem_ctx);
+      simple_mtx_destroy(&foz_db->flock_mtx);
       simple_mtx_destroy(&foz_db->mtx);
    }
 }
@@ -434,7 +436,12 @@ foz_write_entry(struct foz_db *foz_db, const uint8_t *cache_key_160bit,
    if (!foz_db->alive)
       return false;
 
-   /* Wait for 1 second. This is done outside of the mutex as I believe there is more potential
+   /* The flock is per-fd, not per thread, we do it outside of the main mutex to avoid having to
+    * wait in the mutex potentially blocking reads. We use the secondary flock_mtx to stop race
+    * conditions between the write threads sharing the same file descriptor. */
+   simple_mtx_lock(&foz_db->flock_mtx);
+
+   /* Wait for 1 second. This is done outside of the main mutex as I believe there is more potential
     * for file contention than mtx contention of significant length. */
    int err = lock_file_with_timeout(foz_db->file[0], 1000000000);
    if (err == -1)
@@ -449,6 +456,7 @@ foz_write_entry(struct foz_db *foz_db, const uint8_t *cache_key_160bit,
    if (entry) {
       simple_mtx_unlock(&foz_db->mtx);
       flock(fileno(foz_db->file[0]), LOCK_UN);
+      simple_mtx_unlock(&foz_db->flock_mtx);
       return NULL;
    }
 
@@ -511,6 +519,7 @@ foz_write_entry(struct foz_db *foz_db, const uint8_t *cache_key_160bit,
 
    simple_mtx_unlock(&foz_db->mtx);
    flock(fileno(foz_db->file[0]), LOCK_UN);
+   simple_mtx_unlock(&foz_db->flock_mtx);
 
    return true;
 
@@ -518,6 +527,7 @@ fail:
    simple_mtx_unlock(&foz_db->mtx);
 fail_file:
    flock(fileno(foz_db->file[0]), LOCK_UN);
+   simple_mtx_unlock(&foz_db->flock_mtx);
    return false;
 }
 #else

src/util/fossilize_db.h

@@ -76,6 +76,7 @@ struct foz_db {
    FILE *file[FOZ_MAX_DBS];          /* An array of all foz dbs */
    FILE *db_idx;                     /* The default writable foz db idx */
    simple_mtx_t mtx;                 /* Mutex for file/hash table read/writes */
+   simple_mtx_t flock_mtx;           /* Mutex for flocking the file for writes */
    void *mem_ctx;
    struct hash_table_u64 *index_db;  /* Hash table of all foz db entries */
    bool alive;