util/format: Add some NEON intrinsics-based u_format_unpack.
In looking at the profile of dEQP, GLES3 was spending 5-10% of its time in ReadPixels, and almost all of that is b8g8r8a8_unorm8. It's really slow because we're getting about 47MB/s by doing uncached reads 32 bits at a time in the code-generated unpack. If we use NEON to generate larger bus transactions, we can speed things up to 136MB/s. In comparison, raw ldr/str read/writes with no byte swapping can hit a max of 216MB/sec. Reviewed-by: Jesse Natalie <jenatali@microsoft.com> Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/10014>
This commit is contained in:
parent
2b5178ee48
commit
80923e8d58
|
@ -50,6 +50,8 @@ LOCAL_C_INCLUDES := \
|
||||||
$(intermediates)/util/format \
|
$(intermediates)/util/format \
|
||||||
$(intermediates)
|
$(intermediates)
|
||||||
|
|
||||||
|
LOCAL_CFLAGS := -DNO_FORMAT_ASM
|
||||||
|
|
||||||
# If Android version >=8 MESA should static link libexpat else should dynamic link
|
# If Android version >=8 MESA should static link libexpat else should dynamic link
|
||||||
ifeq ($(shell test $(PLATFORM_SDK_VERSION) -ge 27; echo $$?), 0)
|
ifeq ($(shell test $(PLATFORM_SDK_VERSION) -ge 27; echo $$?), 0)
|
||||||
LOCAL_STATIC_LIBRARIES := \
|
LOCAL_STATIC_LIBRARIES := \
|
||||||
|
|
|
@ -28,6 +28,7 @@ files_mesa_format = [
|
||||||
'u_format_rgtc.c',
|
'u_format_rgtc.c',
|
||||||
'u_format_s3tc.c',
|
'u_format_s3tc.c',
|
||||||
'u_format_tests.c',
|
'u_format_tests.c',
|
||||||
|
'u_format_unpack_neon.c',
|
||||||
'u_format_yuv.c',
|
'u_format_yuv.c',
|
||||||
'u_format_zs.c',
|
'u_format_zs.c',
|
||||||
]
|
]
|
||||||
|
|
|
@ -34,6 +34,7 @@
|
||||||
|
|
||||||
#include "util/format/u_format.h"
|
#include "util/format/u_format.h"
|
||||||
#include "util/format/u_format_s3tc.h"
|
#include "util/format/u_format_s3tc.h"
|
||||||
|
#include "util/u_cpu_detect.h"
|
||||||
#include "util/u_math.h"
|
#include "util/u_math.h"
|
||||||
|
|
||||||
#include "pipe/p_defines.h"
|
#include "pipe/p_defines.h"
|
||||||
|
@ -1130,3 +1131,30 @@ util_format_rgb_to_bgr(enum pipe_format format)
|
||||||
return PIPE_FORMAT_NONE;
|
return PIPE_FORMAT_NONE;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static const struct util_format_unpack_description *util_format_unpack_table[PIPE_FORMAT_COUNT];
|
||||||
|
|
||||||
|
static void
|
||||||
|
util_format_unpack_table_init(void)
|
||||||
|
{
|
||||||
|
for (enum pipe_format format = PIPE_FORMAT_NONE; format < PIPE_FORMAT_COUNT; format++) {
|
||||||
|
#if (defined(PIPE_ARCH_AARCH64) || defined(PIPE_ARCH_ARM)) && !defined NO_FORMAT_ASM
|
||||||
|
const struct util_format_unpack_description *unpack = util_format_unpack_description_neon(format);
|
||||||
|
if (unpack) {
|
||||||
|
util_format_unpack_table[format] = unpack;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
|
util_format_unpack_table[format] = util_format_unpack_description_generic(format);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const struct util_format_unpack_description *
|
||||||
|
util_format_unpack_description(enum pipe_format format)
|
||||||
|
{
|
||||||
|
static once_flag flag = ONCE_FLAG_INIT;
|
||||||
|
call_once(&flag, util_format_unpack_table_init);
|
||||||
|
|
||||||
|
return util_format_unpack_table[format];
|
||||||
|
}
|
||||||
|
|
|
@ -415,8 +415,17 @@ util_format_description(enum pipe_format format) ATTRIBUTE_CONST;
|
||||||
const struct util_format_pack_description *
|
const struct util_format_pack_description *
|
||||||
util_format_pack_description(enum pipe_format format) ATTRIBUTE_CONST;
|
util_format_pack_description(enum pipe_format format) ATTRIBUTE_CONST;
|
||||||
|
|
||||||
|
/* Lookup with CPU detection for choosing optimized paths. */
|
||||||
const struct util_format_unpack_description *
|
const struct util_format_unpack_description *
|
||||||
util_format_unpack_description(enum pipe_format format) ATTRIBUTE_CONST;
|
util_format_unpack_description(enum pipe_format format) ATTRIBUTE_CONST;
|
||||||
|
|
||||||
|
/* Codegenned table of CPU-agnostic unpack code. */
|
||||||
|
const struct util_format_unpack_description *
|
||||||
|
util_format_unpack_description_generic(enum pipe_format format) ATTRIBUTE_CONST;
|
||||||
|
|
||||||
|
const struct util_format_unpack_description *
|
||||||
|
util_format_unpack_description_neon(enum pipe_format format) ATTRIBUTE_CONST;
|
||||||
|
|
||||||
#ifdef __GNUC__
|
#ifdef __GNUC__
|
||||||
#pragma GCC diagnostic pop
|
#pragma GCC diagnostic pop
|
||||||
#endif
|
#endif
|
||||||
|
|
|
@ -166,8 +166,11 @@ def write_format_table(formats):
|
||||||
print(" },")
|
print(" },")
|
||||||
|
|
||||||
def generate_table_getter(type):
|
def generate_table_getter(type):
|
||||||
|
suffix = ""
|
||||||
|
if type == "unpack_":
|
||||||
|
suffix = "_generic"
|
||||||
print("const struct util_format_%sdescription *" % type)
|
print("const struct util_format_%sdescription *" % type)
|
||||||
print("util_format_%sdescription(enum pipe_format format)" % type)
|
print("util_format_%sdescription%s(enum pipe_format format)" % (type, suffix))
|
||||||
print("{")
|
print("{")
|
||||||
print(" if (format >= ARRAY_SIZE(util_format_%sdescriptions))" % (type))
|
print(" if (format >= ARRAY_SIZE(util_format_%sdescriptions))" % (type))
|
||||||
print(" return NULL;")
|
print(" return NULL;")
|
||||||
|
@ -242,7 +245,6 @@ def write_format_table(formats):
|
||||||
print("};")
|
print("};")
|
||||||
print()
|
print()
|
||||||
generate_table_getter("pack_")
|
generate_table_getter("pack_")
|
||||||
|
|
||||||
print('static const struct util_format_unpack_description')
|
print('static const struct util_format_unpack_description')
|
||||||
print('util_format_unpack_descriptions[] = {')
|
print('util_format_unpack_descriptions[] = {')
|
||||||
for format in formats:
|
for format in formats:
|
||||||
|
|
|
@ -0,0 +1,79 @@
|
||||||
|
/*
|
||||||
|
* Copyright © 2021 Google LLC
|
||||||
|
*
|
||||||
|
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||||
|
* copy of this software and associated documentation files (the "Software"),
|
||||||
|
* to deal in the Software without restriction, including without limitation
|
||||||
|
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||||
|
* and/or sell copies of the Software, and to permit persons to whom the
|
||||||
|
* Software is furnished to do so, subject to the following conditions:
|
||||||
|
*
|
||||||
|
* The above copyright notice and this permission notice (including the next
|
||||||
|
* paragraph) shall be included in all copies or substantial portions of the
|
||||||
|
* Software.
|
||||||
|
*
|
||||||
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||||
|
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||||
|
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||||
|
* IN THE SOFTWARE.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#include <u_format.h>
|
||||||
|
|
||||||
|
#if (defined(PIPE_ARCH_AARCH64) || defined(PIPE_ARCH_ARM)) && !defined NO_FORMAT_ASM
|
||||||
|
|
||||||
|
/* armhf builds default to vfp, not neon, and refuses to compile neon intrinsics
|
||||||
|
* unless you tell it "no really".
|
||||||
|
*/
|
||||||
|
#ifdef PIPE_ARCH_ARM
|
||||||
|
#pragma GCC target ("fpu=neon")
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#include <arm_neon.h>
|
||||||
|
#include "u_format_pack.h"
|
||||||
|
#include "util/u_cpu_detect.h"
|
||||||
|
|
||||||
|
static void
|
||||||
|
util_format_b8g8r8a8_unorm_unpack_rgba_8unorm_neon(uint8_t *restrict dst, const uint8_t *restrict src, unsigned width)
|
||||||
|
{
|
||||||
|
while (width >= 16) {
|
||||||
|
uint8x16x4_t load = vld4q_u8(src);
|
||||||
|
uint8x16x4_t swap = { .val = { load.val[2], load.val[1], load.val[0], load.val[3] } };
|
||||||
|
vst4q_u8(dst, swap);
|
||||||
|
width -= 16;
|
||||||
|
dst += 16 * 4;
|
||||||
|
src += 16 * 4;
|
||||||
|
}
|
||||||
|
if (width)
|
||||||
|
util_format_b8g8r8a8_unorm_unpack_rgba_8unorm(dst, src, width);
|
||||||
|
}
|
||||||
|
|
||||||
|
static const struct util_format_unpack_description util_format_unpack_descriptions_neon[] = {
|
||||||
|
[PIPE_FORMAT_B8G8R8A8_UNORM] = {
|
||||||
|
.unpack_rgba_8unorm = &util_format_b8g8r8a8_unorm_unpack_rgba_8unorm_neon,
|
||||||
|
.unpack_rgba = &util_format_b8g8r8a8_unorm_unpack_rgba_float,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const struct util_format_unpack_description *
|
||||||
|
util_format_unpack_description_neon(enum pipe_format format)
|
||||||
|
{
|
||||||
|
/* CPU detect for NEON support. On arm64, it's implied. */
|
||||||
|
#ifdef PIPE_ARCH_ARM
|
||||||
|
if (!util_get_cpu_caps()->has_neon)
|
||||||
|
return NULL;
|
||||||
|
#endif
|
||||||
|
|
||||||
|
if (format >= ARRAY_SIZE(util_format_unpack_descriptions_neon))
|
||||||
|
return NULL;
|
||||||
|
|
||||||
|
if (!util_format_unpack_descriptions_neon[format].unpack_rgba)
|
||||||
|
return NULL;
|
||||||
|
|
||||||
|
return &util_format_unpack_descriptions_neon[format];
|
||||||
|
}
|
||||||
|
|
||||||
|
#endif /* PIPE_ARCH_AARCH64 | PIPE_ARCH_ARM */
|
Loading…
Reference in New Issue