util/format: Add some NEON intrinsics-based u_format_unpack.

In looking at the profile of dEQP, GLES3 was spending 5-10% of its time in
ReadPixels, and almost all of that is b8g8r8a8_unorm8.  It's really slow
because we're getting about 47MB/s by doing uncached reads 32 bits at a
time in the code-generated unpack.  If we use NEON to generate larger bus
transactions, we can speed things up to 136MB/s.  In comparison, raw
ldr/str read/writes with no byte swapping can hit a max of 216MB/sec.

Reviewed-by: Jesse Natalie <jenatali@microsoft.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/10014>

src/util/Android.mk

@@ -50,6 +50,8 @@ LOCAL_C_INCLUDES := \
 	$(intermediates)/util/format \
 	$(intermediates)
 
+LOCAL_CFLAGS := -DNO_FORMAT_ASM
+
 # If Android version >=8 MESA should static link libexpat else should dynamic link
 ifeq ($(shell test $(PLATFORM_SDK_VERSION) -ge 27; echo $$?), 0)
 LOCAL_STATIC_LIBRARIES := \

src/util/format/meson.build

@@ -28,6 +28,7 @@ files_mesa_format = [
   'u_format_rgtc.c',
   'u_format_s3tc.c',
   'u_format_tests.c',
+  'u_format_unpack_neon.c',
   'u_format_yuv.c',
   'u_format_zs.c',
 ]

src/util/format/u_format.c

@@ -34,6 +34,7 @@
 
 #include "util/format/u_format.h"
 #include "util/format/u_format_s3tc.h"
+#include "util/u_cpu_detect.h"
 #include "util/u_math.h"
 
 #include "pipe/p_defines.h"
@@ -1130,3 +1131,30 @@ util_format_rgb_to_bgr(enum pipe_format format)
       return PIPE_FORMAT_NONE;
    }
 }
+
+static const struct util_format_unpack_description *util_format_unpack_table[PIPE_FORMAT_COUNT];
+
+static void
+util_format_unpack_table_init(void)
+{
+   for (enum pipe_format format = PIPE_FORMAT_NONE; format < PIPE_FORMAT_COUNT; format++) {
+#if (defined(PIPE_ARCH_AARCH64) || defined(PIPE_ARCH_ARM)) && !defined NO_FORMAT_ASM
+      const struct util_format_unpack_description *unpack = util_format_unpack_description_neon(format);
+      if (unpack) {
+         util_format_unpack_table[format] = unpack;
+         continue;
+      }
+#endif
+
+      util_format_unpack_table[format] = util_format_unpack_description_generic(format);
+   }
+}
+
+const struct util_format_unpack_description *
+util_format_unpack_description(enum pipe_format format)
+{
+   static once_flag flag = ONCE_FLAG_INIT;
+   call_once(&flag, util_format_unpack_table_init);
+
+   return util_format_unpack_table[format];
+}

src/util/format/u_format.h

@@ -415,8 +415,17 @@ util_format_description(enum pipe_format format) ATTRIBUTE_CONST;
 const struct util_format_pack_description *
 util_format_pack_description(enum pipe_format format) ATTRIBUTE_CONST;
 
+/* Lookup with CPU detection for choosing optimized paths. */
 const struct util_format_unpack_description *
 util_format_unpack_description(enum pipe_format format) ATTRIBUTE_CONST;
+
+/* Codegenned table of CPU-agnostic unpack code. */
+const struct util_format_unpack_description *
+util_format_unpack_description_generic(enum pipe_format format) ATTRIBUTE_CONST;
+
+const struct util_format_unpack_description *
+util_format_unpack_description_neon(enum pipe_format format) ATTRIBUTE_CONST;
+
 #ifdef __GNUC__
 #pragma GCC diagnostic pop
 #endif

src/util/format/u_format_table.py

@@ -166,8 +166,11 @@ def write_format_table(formats):
         print("   },")
 
     def generate_table_getter(type):
+        suffix = ""
+        if type == "unpack_":
+            suffix = "_generic"
         print("const struct util_format_%sdescription *" % type)
-        print("util_format_%sdescription(enum pipe_format format)" % type)
+        print("util_format_%sdescription%s(enum pipe_format format)" % (type, suffix))
         print("{")
         print("   if (format >= ARRAY_SIZE(util_format_%sdescriptions))" % (type))
         print("      return NULL;")
@@ -242,7 +245,6 @@ def write_format_table(formats):
     print("};")
     print()
     generate_table_getter("pack_")
-
     print('static const struct util_format_unpack_description')
     print('util_format_unpack_descriptions[] = {')
     for format in formats:

src/util/format/u_format_unpack_neon.c

@@ -0,0 +1,79 @@
+/*
+ * Copyright © 2021 Google LLC
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#include <u_format.h>
+
+#if (defined(PIPE_ARCH_AARCH64) || defined(PIPE_ARCH_ARM)) && !defined NO_FORMAT_ASM
+
+/* armhf builds default to vfp, not neon, and refuses to compile neon intrinsics
+ * unless you tell it "no really".
+ */
+#ifdef PIPE_ARCH_ARM
+#pragma GCC target ("fpu=neon")
+#endif
+
+#include <arm_neon.h>
+#include "u_format_pack.h"
+#include "util/u_cpu_detect.h"
+
+static void
+util_format_b8g8r8a8_unorm_unpack_rgba_8unorm_neon(uint8_t *restrict dst, const uint8_t *restrict src, unsigned width)
+{
+   while (width >= 16) {
+      uint8x16x4_t load = vld4q_u8(src);
+      uint8x16x4_t swap = { .val = { load.val[2], load.val[1], load.val[0], load.val[3] } };
+      vst4q_u8(dst, swap);
+      width -= 16;
+      dst += 16 * 4;
+      src += 16 * 4;
+   }
+   if (width)
+      util_format_b8g8r8a8_unorm_unpack_rgba_8unorm(dst, src, width);
+}
+
+static const struct util_format_unpack_description util_format_unpack_descriptions_neon[] = {
+   [PIPE_FORMAT_B8G8R8A8_UNORM] = {
+      .unpack_rgba_8unorm = &util_format_b8g8r8a8_unorm_unpack_rgba_8unorm_neon,
+      .unpack_rgba = &util_format_b8g8r8a8_unorm_unpack_rgba_float,
+   },
+};
+
+const struct util_format_unpack_description *
+util_format_unpack_description_neon(enum pipe_format format)
+{
+   /* CPU detect for NEON support.  On arm64, it's implied. */
+#ifdef PIPE_ARCH_ARM
+   if (!util_get_cpu_caps()->has_neon)
+      return NULL;
+#endif
+
+   if (format >= ARRAY_SIZE(util_format_unpack_descriptions_neon))
+      return NULL;
+
+   if (!util_format_unpack_descriptions_neon[format].unpack_rgba)
+      return NULL;
+
+   return &util_format_unpack_descriptions_neon[format];
+}
+
+#endif /* PIPE_ARCH_AARCH64 | PIPE_ARCH_ARM */