glsl: add preprocessor #include support

Reviewed-by: Witold Baryluk <witold.baryluk@gmail.com>

src/compiler/Makefile.sources

@@ -186,7 +186,8 @@ LIBGLSL_GENERATED_FILES = \
 
 LIBGLCPP_FILES = \
 	glsl/glcpp/glcpp.h \
-	glsl/glcpp/pp.c
+	glsl/glcpp/pp.c \
+	glsl/glcpp/pp_standalone_scaffolding.c
 
 LIBGLCPP_GENERATED_FILES = \
 	glsl/glcpp/glcpp-lex.c \

src/compiler/glsl/glcpp/glcpp-lex.l

@@ -322,6 +322,11 @@ PATH			["][]^./ _A-Za-z0-9+*%[(){}|&~=!:;,?-]*["]
 	RETURN_STRING_TOKEN (PRAGMA);
 }
 
+<HASH>include{HSPACE}+["<][]^./ _A-Za-z0-9+*%[(){}|&~=!:;,?-]+[">] {
+	BEGIN INITIAL;
+	RETURN_STRING_TOKEN (INCLUDE);
+}
+
 <HASH>line{HSPACE}+ {
 	BEGIN INITIAL;
 	RETURN_TOKEN (LINE);

src/compiler/glsl/glcpp/glcpp-parse.y

@@ -30,6 +30,11 @@
 
 #include "glcpp.h"
 #include "main/mtypes.h"
+#include "util/strndup.h"
+
+const char *
+_mesa_lookup_shader_include(struct gl_context *ctx, char *path,
+                            bool error_check);
 
 static void
 yyerror(YYLTYPE *locp, glcpp_parser_t *parser, const char *error);
@@ -149,6 +154,14 @@ glcpp_parser_lex(YYSTYPE *yylval, YYLTYPE *yylloc, glcpp_parser_t *parser);
 static void
 glcpp_parser_lex_from(glcpp_parser_t *parser, token_list_t *list);
 
+struct define_include {
+   glcpp_parser_t *parser;
+   YYLTYPE *loc;
+};
+
+static void
+glcpp_parser_copy_defines(const void *key, void *data, void *closure);
+
 static void
 add_builtin_define(glcpp_parser_t *parser, const char *name, int value);
 
@@ -174,11 +187,11 @@ add_builtin_define(glcpp_parser_t *parser, const char *name, int value);
         /* We use HASH_TOKEN, DEFINE_TOKEN and VERSION_TOKEN (as opposed to
          * HASH, DEFINE, and VERSION) to avoid conflicts with other symbols,
          * (such as the <HASH> and <DEFINE> start conditions in the lexer). */
-%token DEFINED ELIF_EXPANDED HASH_TOKEN DEFINE_TOKEN FUNC_IDENTIFIER OBJ_IDENTIFIER ELIF ELSE ENDIF ERROR_TOKEN IF IFDEF IFNDEF LINE PRAGMA UNDEF VERSION_TOKEN GARBAGE IDENTIFIER IF_EXPANDED INTEGER INTEGER_STRING LINE_EXPANDED NEWLINE OTHER PLACEHOLDER SPACE PLUS_PLUS MINUS_MINUS PATH
+%token DEFINED ELIF_EXPANDED HASH_TOKEN DEFINE_TOKEN FUNC_IDENTIFIER OBJ_IDENTIFIER ELIF ELSE ENDIF ERROR_TOKEN IF IFDEF IFNDEF LINE PRAGMA UNDEF VERSION_TOKEN GARBAGE IDENTIFIER IF_EXPANDED INTEGER INTEGER_STRING LINE_EXPANDED NEWLINE OTHER PLACEHOLDER SPACE PLUS_PLUS MINUS_MINUS PATH INCLUDE
 %token PASTE
 %type <ival> INTEGER operator SPACE integer_constant version_constant
 %type <expression_value> expression
-%type <str> IDENTIFIER FUNC_IDENTIFIER OBJ_IDENTIFIER INTEGER_STRING OTHER ERROR_TOKEN PRAGMA PATH
+%type <str> IDENTIFIER FUNC_IDENTIFIER OBJ_IDENTIFIER INTEGER_STRING OTHER ERROR_TOKEN PRAGMA PATH INCLUDE
 %type <string_list> identifier_list
 %type <token> preprocessing_token
 %type <token_list> pp_tokens replacement_list text_line
@@ -330,6 +343,67 @@ control_line_success:
 			_mesa_hash_table_remove (parser->defines, entry);
 		}
 	}
+|	HASH_TOKEN INCLUDE NEWLINE {
+		/* Remove leading and trailing "" or <> */
+		char *start = strchr($2, '"');
+		if (!start)
+			start = strchr($2, '<');
+		char *path = strndup(start + 1, strlen(start + 1) - 1);
+
+		const char *shader =
+			_mesa_lookup_shader_include(parser->gl_ctx, path, false);
+		free(path);
+
+		if (!shader)
+			glcpp_error(&@1, parser, "%s not found", $2);
+		else {
+			/* Create a temporary parser with the same settings */
+			glcpp_parser_t *tmp_parser =
+				glcpp_parser_create(parser->gl_ctx, parser->extensions, parser->state);
+			tmp_parser->version_set = true;
+			tmp_parser->version = parser->version;
+
+			/* Set the shader source and run the lexer */
+			glcpp_lex_set_source_string(tmp_parser, shader);
+
+			/* Copy any existing define macros to the temporary
+			 * shade include parser.
+			 */
+			struct define_include di;
+			di.parser = tmp_parser;
+			di.loc = &@1;
+
+			hash_table_call_foreach(parser->defines,
+						glcpp_parser_copy_defines,
+						&di);
+
+			/* Parse the include string before adding to the
+			 * preprocessor output.
+			 */
+			glcpp_parser_parse(tmp_parser);
+			_mesa_string_buffer_printf(parser->info_log, "%s",
+						   tmp_parser->info_log->buf);
+			_mesa_string_buffer_printf(parser->output, "%s",
+						   tmp_parser->output->buf);
+
+			/* Copy any new define macros to the parent parser
+			 * and steal the memory of our temp parser so we don't
+			 * free these new defines before they are no longer
+			 * needed.
+			 */
+			di.parser = parser;
+			di.loc = &@1;
+			ralloc_steal(parser, tmp_parser);
+
+			hash_table_call_foreach(tmp_parser->defines,
+						glcpp_parser_copy_defines,
+						&di);
+
+			/* Destroy tmp parser memory we no longer need */
+			glcpp_lex_destroy(tmp_parser->scanner);
+			_mesa_hash_table_destroy(tmp_parser->defines, NULL);
+		}
+	}
 |	HASH_TOKEN IF pp_tokens NEWLINE {
 		/* Be careful to only evaluate the 'if' expression if
 		 * we are not skipping. When we are skipping, we
@@ -1403,6 +1477,7 @@ glcpp_parser_create(struct gl_context *gl_ctx,
                                                  INITIAL_PP_OUTPUT_BUF_SIZE);
    parser->error = 0;
 
+   parser->gl_ctx = gl_ctx;
    parser->extensions = extensions;
    parser->extension_list = &gl_ctx->Extensions;
    parser->state = state;
@@ -2420,3 +2495,29 @@ glcpp_parser_resolve_implicit_version(glcpp_parser_t *parser)
    _glcpp_parser_handle_version_declaration(parser, language_version,
                                             NULL, false);
 }
+
+static void
+glcpp_parser_copy_defines(const void *key, void *data, void *closure)
+{
+   struct define_include *di = (struct define_include *) closure;
+   macro_t *macro = (macro_t *) data;
+
+   /* If we hit an error on a previous pass, just return */
+   if (di->parser->error)
+      return;
+
+   const char *identifier =  macro->identifier;
+   struct hash_entry *entry = _mesa_hash_table_search(di->parser->defines,
+                                                      identifier);
+
+   macro_t *previous = entry ? entry->data : NULL;
+   if (previous) {
+      if (_macro_equal(macro, previous)) {
+         return;
+      }
+      glcpp_error(di->loc, di->parser, "Redefinition of macro %s\n",
+                  identifier);
+   }
+
+   _mesa_hash_table_insert(di->parser->defines, identifier, macro);
+}

src/compiler/glsl/glcpp/glcpp.h

@@ -211,6 +211,7 @@ struct glcpp_parser {
 	const struct gl_extensions *extension_list;
 	void *state;
 	gl_api api;
+	struct gl_context *gl_ctx;
 	unsigned version;
 
 	/**

src/compiler/glsl/glcpp/meson.build

@@ -47,7 +47,8 @@ endif
 
 libglcpp = static_library(
   'glcpp',
-  [glcpp_lex, glcpp_parse, files('glcpp.h', 'pp.c')],
+  [glcpp_lex, glcpp_parse, files('glcpp.h', 'pp.c',
+                                 'pp_standalone_scaffolding.c')],
   dependencies : idep_mesautil,
   include_directories : [inc_common],
   c_args : [c_vis_args, no_override_init_args, c_msvc_compat_args, _extra_args],

src/compiler/glsl/glcpp/pp_standalone_scaffolding.c

@@ -0,0 +1,41 @@
+/*
+ * Copyright © 2019 Timothy Arceri
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+/* This file declares stripped-down versions of functions that
+ * normally exist outside of the glsl folder, so that they can be used
+ * when running the GLSL compiler standalone (for unit testing or
+ * compiling builtins).
+ */
+
+#include "pp_standalone_scaffolding.h"
+
+const char *
+_mesa_lookup_shader_include(struct gl_context *ctx, char *path,
+                            bool error_check)
+{
+   (void) ctx;
+   (void) path;
+   (void) error_check;
+
+   return NULL;
+}

src/compiler/glsl/glcpp/pp_standalone_scaffolding.h

@@ -0,0 +1,40 @@
+/*
+ * Copyright © 2019 Timothy Arceri
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+/* This file declares stripped-down versions of functions that
+ * normally exist outside of the glcpp folder, so that they can be used
+ * when running the GLSL compiler standalone (for unit testing or
+ * compiling builtins).
+ */
+
+#ifndef PP_STANDALONE_SCAFFOLDING_H
+#define PP_STANDALONE_SCAFFOLDING_H
+
+#include <stddef.h>
+#include "main/mtypes.h"
+
+const char *
+_mesa_lookup_shader_include(struct gl_context *ctx, char *path,
+                            bool error_check);
+
+#endif /* PP_STANDALONE_SCAFFOLDING_H */