llvmpipe: Proper control flow builders.
New control flow helper functions which keep track of all variables and generate the correct Phi functions. This re-enables skipping the fs execution of quads masked out by the rasterizer, early z testing, and kill opcode. This yields a performance improvement of around 20%.
This commit is contained in:
José Fonseca
parent
bd3b59da63
commit
8e6b925d2a
|
|
@ -32,59 +32,261 @@
|
|||
*/
|
||||
|
||||
#include "util/u_debug.h"
|
||||
#include "util/u_memory.h"
|
||||
|
||||
#include "lp_bld_type.h"
|
||||
#include "lp_bld_flow.h"
|
||||
|
||||
|
||||
void
|
||||
lp_build_mask_begin(struct lp_build_mask_context *mask,
|
||||
LLVMBuilderRef builder,
|
||||
union lp_type type,
|
||||
LLVMValueRef value)
|
||||
{
|
||||
memset(mask, 0, sizeof *mask);
|
||||
#define LP_BUILD_FLOW_MAX_VARIABLES 32
|
||||
#define LP_BUILD_FLOW_MAX_DEPTH 32
|
||||
|
||||
mask->builder = builder;
|
||||
mask->reg_type = LLVMIntType(type.width * type.length);
|
||||
mask->value = value;
|
||||
|
||||
/**
|
||||
* Enumeration of all possible flow constructs.
|
||||
*/
|
||||
enum lp_build_flow_construct_kind {
|
||||
lP_BUILD_FLOW_SCOPE,
|
||||
LP_BUILD_FLOW_SKIP,
|
||||
};
|
||||
|
||||
|
||||
/**
|
||||
* Variable declaration scope.
|
||||
*/
|
||||
struct lp_build_flow_scope
|
||||
{
|
||||
/** Number of variables declared in this scope */
|
||||
unsigned num_variables;
|
||||
};
|
||||
|
||||
|
||||
/**
|
||||
* Early exit. Useful to skip to the end of a function or block when
|
||||
* the execution mask becomes zero or when there is an error condition.
|
||||
*/
|
||||
struct lp_build_flow_skip
|
||||
{
|
||||
/** Block to skip to */
|
||||
LLVMBasicBlockRef block;
|
||||
|
||||
/** Number of variables declared at the beginning */
|
||||
unsigned num_variables;
|
||||
|
||||
LLVMValueRef *phi;
|
||||
};
|
||||
|
||||
|
||||
/**
|
||||
* Union of all possible flow constructs' data
|
||||
*/
|
||||
union lp_build_flow_construct_data
|
||||
{
|
||||
struct lp_build_flow_scope scope;
|
||||
struct lp_build_flow_skip skip;
|
||||
};
|
||||
|
||||
|
||||
/**
|
||||
* Element of the flow construct stack.
|
||||
*/
|
||||
struct lp_build_flow_construct
|
||||
{
|
||||
enum lp_build_flow_construct_kind kind;
|
||||
union lp_build_flow_construct_data data;
|
||||
};
|
||||
|
||||
|
||||
/**
|
||||
* All necessary data to generate LLVM control flow constructs.
|
||||
*
|
||||
* Besides keeping track of the control flow construct themselves we also
|
||||
* need to keep track of variables in order to generate SSA Phi values.
|
||||
*/
|
||||
struct lp_build_flow_context
|
||||
{
|
||||
LLVMBuilderRef builder;
|
||||
|
||||
/**
|
||||
* Control flow stack.
|
||||
*/
|
||||
struct lp_build_flow_construct constructs[LP_BUILD_FLOW_MAX_DEPTH];
|
||||
unsigned num_constructs;
|
||||
|
||||
/**
|
||||
* Variable stack
|
||||
*/
|
||||
LLVMValueRef *variables[LP_BUILD_FLOW_MAX_VARIABLES];
|
||||
unsigned num_variables;
|
||||
};
|
||||
|
||||
|
||||
struct lp_build_flow_context *
|
||||
lp_build_flow_create(LLVMBuilderRef builder)
|
||||
{
|
||||
struct lp_build_flow_context *flow;
|
||||
|
||||
flow = CALLOC_STRUCT(lp_build_flow_context);
|
||||
if(!flow)
|
||||
return NULL;
|
||||
|
||||
flow->builder = builder;
|
||||
|
||||
return flow;
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
lp_build_mask_update(struct lp_build_mask_context *mask,
|
||||
LLVMValueRef value)
|
||||
lp_build_flow_destroy(struct lp_build_flow_context *flow)
|
||||
{
|
||||
assert(flow->num_constructs == 0);
|
||||
assert(flow->num_variables == 0);
|
||||
FREE(flow);
|
||||
}
|
||||
|
||||
LLVMValueRef cond;
|
||||
|
||||
static union lp_build_flow_construct_data *
|
||||
lp_build_flow_push(struct lp_build_flow_context *flow,
|
||||
enum lp_build_flow_construct_kind kind)
|
||||
{
|
||||
assert(flow->num_constructs < LP_BUILD_FLOW_MAX_DEPTH);
|
||||
if(flow->num_constructs >= LP_BUILD_FLOW_MAX_DEPTH)
|
||||
return NULL;
|
||||
|
||||
flow->constructs[flow->num_constructs].kind = kind;
|
||||
return &flow->constructs[flow->num_constructs++].data;
|
||||
}
|
||||
|
||||
|
||||
static union lp_build_flow_construct_data *
|
||||
lp_build_flow_peek(struct lp_build_flow_context *flow,
|
||||
enum lp_build_flow_construct_kind kind)
|
||||
{
|
||||
assert(flow->num_constructs);
|
||||
if(!flow->num_constructs)
|
||||
return NULL;
|
||||
|
||||
assert(flow->constructs[flow->num_constructs - 1].kind == kind);
|
||||
if(flow->constructs[flow->num_constructs - 1].kind != kind)
|
||||
return NULL;
|
||||
|
||||
return &flow->constructs[flow->num_constructs - 1].data;
|
||||
}
|
||||
|
||||
|
||||
static union lp_build_flow_construct_data *
|
||||
lp_build_flow_pop(struct lp_build_flow_context *flow,
|
||||
enum lp_build_flow_construct_kind kind)
|
||||
{
|
||||
assert(flow->num_constructs);
|
||||
if(!flow->num_constructs)
|
||||
return NULL;
|
||||
|
||||
assert(flow->constructs[flow->num_constructs - 1].kind == kind);
|
||||
if(flow->constructs[flow->num_constructs - 1].kind != kind)
|
||||
return NULL;
|
||||
|
||||
return &flow->constructs[--flow->num_constructs].data;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Begin a variable scope.
|
||||
*
|
||||
*
|
||||
*/
|
||||
void
|
||||
lp_build_flow_scope_begin(struct lp_build_flow_context *flow)
|
||||
{
|
||||
struct lp_build_flow_scope *scope;
|
||||
|
||||
scope = &lp_build_flow_push(flow, lP_BUILD_FLOW_SCOPE)->scope;
|
||||
if(!scope)
|
||||
return;
|
||||
|
||||
scope->num_variables = 0;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Declare a variable.
|
||||
*
|
||||
* A variable is a named entity which can have different LLVMValueRef's at
|
||||
* different points of the program. This is relevant for control flow because
|
||||
* when there are mutiple branches to a same location we need to replace
|
||||
* the variable's value with a Phi function as explained in
|
||||
* http://en.wikipedia.org/wiki/Static_single_assignment_form .
|
||||
*
|
||||
* We keep track of variables by keeping around a pointer to where their
|
||||
* current.
|
||||
*
|
||||
* There are a few cautions to observe:
|
||||
*
|
||||
* - Variable's value must not be NULL. If there is no initial value then
|
||||
* LLVMGetUndef() should be used.
|
||||
*
|
||||
* - Variable's value must be kept up-to-date. If the variable is going to be
|
||||
* modified by a function then a pointer should be passed so that its value
|
||||
* is accurate. Failure to do this will cause some of the variables'
|
||||
* transient values to be lost, leading to wrong results.
|
||||
*
|
||||
* - A program should be written from top to bottom, by always appending
|
||||
* instructions to the bottom with a single LLVMBuilderRef. Inserting and/or
|
||||
* modifying existing statements will most likely lead to wrong results.
|
||||
*
|
||||
*/
|
||||
void
|
||||
lp_build_flow_scope_declare(struct lp_build_flow_context *flow,
|
||||
LLVMValueRef *variable)
|
||||
{
|
||||
struct lp_build_flow_scope *scope;
|
||||
|
||||
scope = &lp_build_flow_peek(flow, lP_BUILD_FLOW_SCOPE)->scope;
|
||||
if(!scope)
|
||||
return;
|
||||
|
||||
assert(*variable);
|
||||
if(!*variable)
|
||||
return;
|
||||
|
||||
assert(flow->num_variables < LP_BUILD_FLOW_MAX_VARIABLES);
|
||||
if(flow->num_variables >= LP_BUILD_FLOW_MAX_VARIABLES)
|
||||
return;
|
||||
|
||||
flow->variables[flow->num_variables++] = variable;
|
||||
++scope->num_variables;
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
lp_build_flow_scope_end(struct lp_build_flow_context *flow)
|
||||
{
|
||||
struct lp_build_flow_scope *scope;
|
||||
|
||||
scope = &lp_build_flow_pop(flow, lP_BUILD_FLOW_SCOPE)->scope;
|
||||
if(!scope)
|
||||
return;
|
||||
|
||||
assert(flow->num_variables >= scope->num_variables);
|
||||
if(flow->num_variables < scope->num_variables) {
|
||||
flow->num_variables = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
flow->num_variables -= scope->num_variables;
|
||||
}
|
||||
|
||||
|
||||
static LLVMBasicBlockRef
|
||||
lp_build_flow_insert_block(struct lp_build_flow_context *flow)
|
||||
{
|
||||
LLVMBasicBlockRef current_block;
|
||||
LLVMBasicBlockRef next_block;
|
||||
LLVMBasicBlockRef new_block;
|
||||
|
||||
if(mask->value)
|
||||
mask->value = LLVMBuildAnd(mask->builder, mask->value, value, "");
|
||||
else
|
||||
mask->value = value;
|
||||
|
||||
/* FIXME: disabled until we have proper control flow helpers */
|
||||
#if 0
|
||||
cond = LLVMBuildICmp(mask->builder,
|
||||
LLVMIntEQ,
|
||||
LLVMBuildBitCast(mask->builder, mask->value, mask->reg_type, ""),
|
||||
LLVMConstNull(mask->reg_type),
|
||||
"");
|
||||
|
||||
current_block = LLVMGetInsertBlock(mask->builder);
|
||||
|
||||
if(!mask->skip_block) {
|
||||
LLVMValueRef function = LLVMGetBasicBlockParent(current_block);
|
||||
mask->skip_block = LLVMAppendBasicBlock(function, "skip");
|
||||
|
||||
mask->phi = LLVMBuildPhi(mask->builder, LLVMTypeOf(mask->value), "");
|
||||
}
|
||||
current_block = LLVMGetInsertBlock(flow->builder);
|
||||
|
||||
next_block = LLVMGetNextBasicBlock(current_block);
|
||||
assert(next_block);
|
||||
if(next_block) {
|
||||
new_block = LLVMInsertBasicBlock(next_block, "");
|
||||
}
|
||||
|
|
@ -93,30 +295,139 @@ lp_build_mask_update(struct lp_build_mask_context *mask,
|
|||
new_block = LLVMAppendBasicBlock(function, "");
|
||||
}
|
||||
|
||||
LLVMAddIncoming(mask->phi, &mask->value, ¤t_block, 1);
|
||||
LLVMBuildCondBr(mask->builder, cond, mask->skip_block, new_block);
|
||||
return new_block;
|
||||
}
|
||||
|
||||
LLVMPositionBuilderAtEnd(mask->builder, new_block);
|
||||
#endif
|
||||
void
|
||||
lp_build_flow_skip_begin(struct lp_build_flow_context *flow)
|
||||
{
|
||||
struct lp_build_flow_skip *skip;
|
||||
LLVMBuilderRef builder;
|
||||
unsigned i;
|
||||
|
||||
skip = &lp_build_flow_push(flow, LP_BUILD_FLOW_SKIP)->skip;
|
||||
if(!skip)
|
||||
return;
|
||||
|
||||
skip->block = lp_build_flow_insert_block(flow);
|
||||
skip->num_variables = flow->num_variables;
|
||||
if(!skip->num_variables) {
|
||||
skip->phi = NULL;
|
||||
return;
|
||||
}
|
||||
|
||||
skip->phi = MALLOC(skip->num_variables * sizeof *skip->phi);
|
||||
if(!skip->phi) {
|
||||
skip->num_variables = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
builder = LLVMCreateBuilder();
|
||||
LLVMPositionBuilderAtEnd(builder, skip->block);
|
||||
|
||||
for(i = 0; i < skip->num_variables; ++i)
|
||||
skip->phi[i] = LLVMBuildPhi(builder, LLVMTypeOf(*flow->variables[i]), "");
|
||||
|
||||
LLVMDisposeBuilder(builder);
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
lp_build_flow_skip_cond_break(struct lp_build_flow_context *flow,
|
||||
LLVMValueRef cond)
|
||||
{
|
||||
struct lp_build_flow_skip *skip;
|
||||
LLVMBasicBlockRef current_block;
|
||||
LLVMBasicBlockRef new_block;
|
||||
unsigned i;
|
||||
|
||||
skip = &lp_build_flow_peek(flow, LP_BUILD_FLOW_SKIP)->skip;
|
||||
if(!skip)
|
||||
return;
|
||||
|
||||
current_block = LLVMGetInsertBlock(flow->builder);
|
||||
|
||||
new_block = lp_build_flow_insert_block(flow);
|
||||
|
||||
for(i = 0; i < skip->num_variables; ++i) {
|
||||
assert(*flow->variables[i]);
|
||||
LLVMAddIncoming(skip->phi[i], flow->variables[i], ¤t_block, 1);
|
||||
}
|
||||
|
||||
LLVMBuildCondBr(flow->builder, cond, skip->block, new_block);
|
||||
|
||||
LLVMPositionBuilderAtEnd(flow->builder, new_block);
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
lp_build_flow_skip_end(struct lp_build_flow_context *flow)
|
||||
{
|
||||
struct lp_build_flow_skip *skip;
|
||||
LLVMBasicBlockRef current_block;
|
||||
unsigned i;
|
||||
|
||||
skip = &lp_build_flow_pop(flow, LP_BUILD_FLOW_SKIP)->skip;
|
||||
if(!skip)
|
||||
return;
|
||||
|
||||
current_block = LLVMGetInsertBlock(flow->builder);
|
||||
|
||||
for(i = 0; i < skip->num_variables; ++i) {
|
||||
assert(*flow->variables[i]);
|
||||
LLVMAddIncoming(skip->phi[i], flow->variables[i], ¤t_block, 1);
|
||||
*flow->variables[i] = skip->phi[i];
|
||||
}
|
||||
|
||||
LLVMBuildBr(flow->builder, skip->block);
|
||||
LLVMPositionBuilderAtEnd(flow->builder, skip->block);
|
||||
|
||||
FREE(skip->phi);
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
lp_build_mask_begin(struct lp_build_mask_context *mask,
|
||||
struct lp_build_flow_context *flow,
|
||||
union lp_type type,
|
||||
LLVMValueRef value)
|
||||
{
|
||||
memset(mask, 0, sizeof *mask);
|
||||
|
||||
mask->flow = flow;
|
||||
mask->reg_type = LLVMIntType(type.width * type.length);
|
||||
mask->value = value;
|
||||
|
||||
lp_build_flow_scope_begin(flow);
|
||||
lp_build_flow_scope_declare(flow, &mask->value);
|
||||
lp_build_flow_skip_begin(flow);
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
lp_build_mask_update(struct lp_build_mask_context *mask,
|
||||
LLVMValueRef value)
|
||||
{
|
||||
LLVMBuilderRef builder = mask->flow->builder;
|
||||
LLVMValueRef cond;
|
||||
|
||||
mask->value = LLVMBuildAnd(builder, mask->value, value, "");
|
||||
|
||||
cond = LLVMBuildICmp(builder,
|
||||
LLVMIntEQ,
|
||||
LLVMBuildBitCast(builder, mask->value, mask->reg_type, ""),
|
||||
LLVMConstNull(mask->reg_type),
|
||||
"");
|
||||
|
||||
lp_build_flow_skip_cond_break(mask->flow, cond);
|
||||
}
|
||||
|
||||
|
||||
LLVMValueRef
|
||||
lp_build_mask_end(struct lp_build_mask_context *mask)
|
||||
{
|
||||
if(mask->skip_block) {
|
||||
LLVMBasicBlockRef current_block = LLVMGetInsertBlock(mask->builder);
|
||||
|
||||
LLVMAddIncoming(mask->phi, &mask->value, ¤t_block, 1);
|
||||
LLVMBuildBr(mask->builder, mask->skip_block);
|
||||
|
||||
LLVMPositionBuilderAtEnd(mask->builder, mask->skip_block);
|
||||
|
||||
mask->value = mask->phi;
|
||||
mask->phi = NULL;
|
||||
mask->skip_block = NULL;
|
||||
}
|
||||
|
||||
lp_build_flow_skip_end(mask->flow);
|
||||
lp_build_flow_scope_end(mask->flow);
|
||||
return mask->value;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -41,23 +41,49 @@
|
|||
union lp_type;
|
||||
|
||||
|
||||
struct lp_build_flow_context;
|
||||
|
||||
|
||||
struct lp_build_flow_context *
|
||||
lp_build_flow_create(LLVMBuilderRef builder);
|
||||
|
||||
void
|
||||
lp_build_flow_destroy(struct lp_build_flow_context *flow);
|
||||
|
||||
void
|
||||
lp_build_flow_scope_begin(struct lp_build_flow_context *flow);
|
||||
|
||||
void
|
||||
lp_build_flow_scope_declare(struct lp_build_flow_context *flow,
|
||||
LLVMValueRef *variable);
|
||||
|
||||
void
|
||||
lp_build_flow_scope_end(struct lp_build_flow_context *flow);
|
||||
|
||||
void
|
||||
lp_build_flow_skip_begin(struct lp_build_flow_context *flow);
|
||||
|
||||
void
|
||||
lp_build_flow_skip_cond_break(struct lp_build_flow_context *flow,
|
||||
LLVMValueRef cond);
|
||||
|
||||
void
|
||||
lp_build_flow_skip_end(struct lp_build_flow_context *flow);
|
||||
|
||||
|
||||
struct lp_build_mask_context
|
||||
{
|
||||
LLVMBuilderRef builder;
|
||||
struct lp_build_flow_context *flow;
|
||||
|
||||
LLVMTypeRef reg_type;
|
||||
|
||||
LLVMValueRef value;
|
||||
|
||||
LLVMValueRef phi;
|
||||
|
||||
LLVMBasicBlockRef skip_block;
|
||||
};
|
||||
|
||||
|
||||
void
|
||||
lp_build_mask_begin(struct lp_build_mask_context *mask,
|
||||
LLVMBuilderRef builder,
|
||||
struct lp_build_flow_context *flow,
|
||||
union lp_type type,
|
||||
LLVMValueRef value);
|
||||
|
||||
|
|
|
|||
|
|
@ -197,6 +197,7 @@ generate_fs(struct llvmpipe_context *lp,
|
|||
LLVMValueRef consts_ptr;
|
||||
LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][NUM_CHANNELS];
|
||||
LLVMValueRef z = interp->pos[2];
|
||||
struct lp_build_flow_context *flow;
|
||||
struct lp_build_mask_context mask;
|
||||
boolean early_depth_test;
|
||||
unsigned attrib;
|
||||
|
|
@ -208,7 +209,33 @@ generate_fs(struct llvmpipe_context *lp,
|
|||
|
||||
consts_ptr = lp_jit_context_constants(builder, context_ptr);
|
||||
|
||||
lp_build_mask_begin(&mask, builder, type, *pmask);
|
||||
flow = lp_build_flow_create(builder);
|
||||
|
||||
memset(outputs, 0, sizeof outputs);
|
||||
|
||||
lp_build_flow_scope_begin(flow);
|
||||
|
||||
/* Declare the color and z variables */
|
||||
for (attrib = 0; attrib < shader->info.num_outputs; ++attrib) {
|
||||
for(chan = 0; chan < NUM_CHANNELS; ++chan) {
|
||||
boolean declare = FALSE;
|
||||
switch (shader->info.output_semantic_name[attrib]) {
|
||||
case TGSI_SEMANTIC_COLOR:
|
||||
declare = TRUE;
|
||||
break;
|
||||
case TGSI_SEMANTIC_POSITION:
|
||||
if(chan == 2)
|
||||
declare = TRUE;
|
||||
break;
|
||||
}
|
||||
if(declare) {
|
||||
outputs[attrib][chan] = LLVMGetUndef(vec_type);
|
||||
lp_build_flow_scope_declare(flow, &outputs[attrib][chan]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
lp_build_mask_begin(&mask, flow, type, *pmask);
|
||||
|
||||
early_depth_test =
|
||||
key->depth.enabled &&
|
||||
|
|
@ -221,12 +248,19 @@ generate_fs(struct llvmpipe_context *lp,
|
|||
type, &mask,
|
||||
z, depth_ptr);
|
||||
|
||||
memset(outputs, 0, sizeof outputs);
|
||||
|
||||
lp_build_tgsi_soa(builder, tokens, type, &mask,
|
||||
consts_ptr, interp->pos, interp->inputs,
|
||||
outputs, sampler);
|
||||
|
||||
if(!early_depth_test)
|
||||
generate_depth(builder, key,
|
||||
type, &mask,
|
||||
z, depth_ptr);
|
||||
|
||||
lp_build_mask_end(&mask);
|
||||
|
||||
lp_build_flow_scope_end(flow);
|
||||
|
||||
for (attrib = 0; attrib < shader->info.num_outputs; ++attrib) {
|
||||
for(chan = 0; chan < NUM_CHANNELS; ++chan) {
|
||||
if(outputs[attrib][chan]) {
|
||||
|
|
@ -265,12 +299,7 @@ generate_fs(struct llvmpipe_context *lp,
|
|||
}
|
||||
}
|
||||
|
||||
if(!early_depth_test)
|
||||
generate_depth(builder, key,
|
||||
type, &mask,
|
||||
z, depth_ptr);
|
||||
|
||||
lp_build_mask_end(&mask);
|
||||
lp_build_flow_destroy(flow);
|
||||
|
||||
*pmask = mask.value;
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue