lexer fixes; eval2 resolution qualifiers

This commit is contained in:
Ryan Fleury committed 2026-07-29 11:35:28 -07:00
1 parent b27585476a
commit ba9d36624d
10 files changed
+327 -21

No files matched your search

+42 -5
View File
@@ -1917,12 +1917,46 @@ e2_parse_from_string(Arena *arena, E2_ParseState *state, B32 identifier_is_type,
//- rjf: standalone identifiers (also could be definition or operator keyword)
else if(need_new_expr && e2_try_token(lang, string, E2_TokenKind_Identifier, s(""), &off, &token))
{
String8 identifier = str8_substr(string, token.range);
B32 identifier_mapped = 0;
B32 definitions_are_allowed = (state->top_task->expr_kind != E2_ExprKind_Macro);
// rjf: pick the last identifier as the target identifier, collect the rest as qualifiers
String8List qualifiers = {0};
String8 identifier = {0};
{
E2_Token last_identifier_token = token;
for(;;)
{
U64 next_off_maybe = off;
E2_Token ext_identifier_token = {E2_TokenKind_Null};
if(e2_try_token(lang, string, E2_TokenKind_Symbol, s("!"), &next_off_maybe, 0) ||
(e2_try_token(lang, string, E2_TokenKind_Symbol, s("."), &next_off_maybe, 0) &&
e2_try_token(lang, string, E2_TokenKind_Identifier, s(""), &next_off_maybe, &ext_identifier_token) &&
e2_try_token(lang, string, E2_TokenKind_Symbol, s("!"), &next_off_maybe, 0)))
{
Rng1U64 qualifier_range = last_identifier_token.range;
if(dim_1u64(ext_identifier_token.range) > 0)
{
qualifier_range = union_1u64(qualifier_range, ext_identifier_token.range);
}
String8 qualifier = str8_substr(string, qualifier_range);
str8_list_push(arena, &qualifiers, qualifier);
if(!e2_try_token(lang, string, E2_TokenKind_Identifier, s(""), &next_off_maybe, &last_identifier_token))
{
e2_msgf(arena, &parse.msgs, r1u64(off, off), "Expected identifier after qualifier `%S!`.", qualifier);
}
off = next_off_maybe;
}
else
{
identifier = str8_substr(string, last_identifier_token.range);
break;
}
}
}
// rjf: first try to resolve as a definition
if(definitions_are_allowed && !identifier_mapped && e2_try_token(lang, string, E2_TokenKind_Symbol, s("="), &off, 0))
if(qualifiers.node_count == 0 && definitions_are_allowed && !identifier_mapped && e2_try_token(lang, string, E2_TokenKind_Symbol, s("="), &off, 0))
{
identifier_mapped = 1;
E2_ParseTask *task = state->free_task;
@@ -1944,7 +1978,7 @@ e2_parse_from_string(Arena *arena, E2_ParseState *state, B32 identifier_is_type,
}
// rjf: try to resolve it as a macro definition
if(definitions_are_allowed && !identifier_mapped)
if(qualifiers.node_count == 0 && definitions_are_allowed && !identifier_mapped)
{
U64 macro_off = off;
if(e2_try_token(lang, string, E2_TokenKind_Symbol, s("("), &macro_off, 0))
@@ -2012,7 +2046,7 @@ e2_parse_from_string(Arena *arena, E2_ParseState *state, B32 identifier_is_type,
}
// rjf: first try to resolve as an operator name
if(!identifier_mapped)
if(qualifiers.node_count == 0 && !identifier_mapped)
{
// rjf: string -> op kind
E2_ExprKind expr_kind = E2_ExprKind_Null;
@@ -2055,7 +2089,7 @@ e2_parse_from_string(Arena *arena, E2_ParseState *state, B32 identifier_is_type,
}
// rjf: try to resolve it as a macro argument
if(!identifier_mapped && state->top_task->expr_kind == E2_ExprKind_Macro)
if(qualifiers.node_count == 0 && !identifier_mapped && state->top_task->expr_kind == E2_ExprKind_Macro)
{
U64 macro_arg_num = 0;
{
@@ -2086,6 +2120,7 @@ e2_parse_from_string(Arena *arena, E2_ParseState *state, B32 identifier_is_type,
done = 1;
parse.status = E2_ParseStatus_CheckIdentifierIsType;
parse.identifier = identifier;
parse.qualifiers = qualifiers;
off = start_off;
}
else
@@ -2093,6 +2128,7 @@ e2_parse_from_string(Arena *arena, E2_ParseState *state, B32 identifier_is_type,
identifier_mapped = 1;
expr = e2_expr(arena, identifier_is_type ? E2_ExprKind_TypeIdentifier : E2_ExprKind_Identifier);
expr->string = str8_substr(string, token.range);
expr->qualifiers = qualifiers;
state->caller_request_count = 0;
}
}
@@ -2651,6 +2687,7 @@ e2_compile_from_expr(Arena *arena, E2_CompileState *state, E2_IRNode *resolve_re
done = 1;
compile.status = E2_CompileStatus_MissedIdentifierResolution;
compile.identifier = e->string;
compile.qualifiers = e->qualifiers;
}
}break;
+3
View File
@@ -326,6 +326,7 @@ struct E2_Expr
U32 macro_arg_num;
Rng1U64 src_range;
String8 string;
String8List qualifiers;
};
typedef struct E2_ExprMapNode E2_ExprMapNode;
@@ -400,6 +401,7 @@ struct E2_Parse
{
E2_ParseStatus status;
String8 identifier;
String8List qualifiers;
E2_Expr *expr;
String8 member_name;
E2_Expr *params_expr;
@@ -469,6 +471,7 @@ struct E2_Compile
{
E2_CompileStatus status;
String8 identifier;
String8List qualifiers;
E2_IRNode *irtree;
String8 member_name;
E2_IRNode *params_irtree;
+191
View File
@@ -0,0 +1,191 @@
// Copyright (c) Epic Games Tools
// Licensed under the MIT license (https://opensource.org/license/mit/)
Test(eval2_regressions)
{
E2_ConsTypeMap *cons_types = e2_cons_type_map_alloc();
e2_select_cons_type_map(cons_types);
String8 strings[] =
{
s("int32(*)(int32, int32)"),
s("int32 *(*)(int32, int32)"),
s("int32 **(*)(int32, int32)"),
s("123(1, 2, 3)"),
s("int32 (*) [100]"),
s("(3 * 4) + 2"),
s("cast (float32) 123"),
s("int32 *"),
s("3 * 4 + 2"),
s("int32[100]"),
s("3 * 4"),
s("foo"),
s("foo.bar"),
s("[123]"),
s("123[456]"),
s("2 + (3 * 4)"),
s("int32 == int32"),
s("123, 456"),
s("222.f"),
s("123 as float32"),
s("cast float32 123"),
s("(int32 *)123"),
s("bar(a, b) = a + b"),
s("foo = 123"),
s("1 > 2"),
s("1 ? 123 : 456"),
s("0 ? 123 : 456"),
s("1 ? \"Test\" : 888"),
s("'a'"),
s("'b'"),
s("(float32)222"),
s("(float64)222"),
s("222.0"),
s("123"),
s("123 + 456"),
s("!1"),
s("!0"),
s("1 + 1 + 1"),
s("40 / 0"),
s("123 | 456"),
s("123 + "),
s("-123"),
s("sizeof 123"),
};
for EachElement(idx, strings)
{
Temp scratch = scratch_begin(0, 0);
String8List msgs = {0};
// rjf: string -> expr
E2_Expr *expr = &e2_expr_nil;
{
E2_ParseState state = {0};
B32 identifier_is_type = 0;
for(;;)
{
E2_Parse parse = e2_parse_from_string(scratch.arena, &state, identifier_is_type, E2_LangKind_CLike, strings[idx]);
identifier_is_type = 0;
expr = parse.expr;
for EachNode(n, E2_Msg, parse.msgs.first)
{
str8_list_push(scratch.arena, &msgs, n->string);
}
if(parse.status == E2_ParseStatus_CheckIdentifierIsType)
{
identifier_is_type = identifier_is_type || str8_match(parse.identifier, s("int32"), 0);
identifier_is_type = identifier_is_type || str8_match(parse.identifier, s("float32"), 0);
identifier_is_type = identifier_is_type || str8_match(parse.identifier, s("float64"), 0);
}
if(e2_parse_status_is_terminal(parse.status))
{
break;
}
}
}
// rjf: expr -> ir tree
E2_IRNode *irtree = &e2_irnode_nil;
{
E2_IRNode *resolve_result = &e2_irnode_nil;
E2_Val compile_time_eval_result = {0};
E2_CompileState state = {0};
for(;;)
{
E2_Compile compile = e2_compile_from_expr(scratch.arena, &state, resolve_result, compile_time_eval_result, expr);
irtree = compile.irtree;
resolve_result = &e2_irnode_nil;
if(compile.status == E2_CompileStatus_MissedIdentifierResolution)
{
E2_IRNode *irnode = &e2_irnode_nil;
if(str8_match(compile.identifier, s("float32"), 0))
{
irnode = e2_irnode_type(scratch.arena, e2_type_key_basic(E2_TypeKind_F32));
}
else if(str8_match(compile.identifier, s("float64"), 0))
{
irnode = e2_irnode_type(scratch.arena, e2_type_key_basic(E2_TypeKind_F64));
}
else if(str8_match(compile.identifier, s("int32"), 0))
{
irnode = e2_irnode_type(scratch.arena, e2_type_key_basic(E2_TypeKind_S32));
}
else if(str8_match(compile.identifier, s("int64"), 0))
{
irnode = e2_irnode_type(scratch.arena, e2_type_key_basic(E2_TypeKind_S64));
}
else
{
irnode = e2_irnode_const_u64_or_smaller(scratch.arena, 123);
}
resolve_result = irnode;
}
if(compile.status == E2_CompileStatus_MemberAccess)
{
resolve_result = e2_irnode_const_u64_or_smaller(scratch.arena, 456);
}
if(compile.status == E2_CompileStatus_IndexAccess)
{
resolve_result = e2_irnode_const_u64_or_smaller(scratch.arena, 111);
}
if(compile.status == E2_CompileStatus_Call)
{
resolve_result = e2_irnode_const_u64_or_smaller(scratch.arena, 123456);
}
if(compile.status == E2_CompileStatus_CompileTimeEval)
{
String8 bytecode = e2_bytecode_from_irnode(scratch.arena, irtree);
E2_InterpState interp_state = {0};
E2_SpaceMap space_map = {0};
E2_Interp interp = e2_interp_from_bytecode(scratch.arena, &interp_state, &space_map, bytecode);
compile_time_eval_result = interp.val;
}
if(e2_compile_status_is_terminal(compile.status))
{
break;
}
}
}
// rjf: ir tree -> bytecode
String8 bytecode = e2_bytecode_from_irnode(scratch.arena, irtree);
// rjf: bytecode -> value
E2_Val val = {0};
{
E2_InterpState state = {0};
E2_SpaceMap space_map = {0};
for(;;)
{
E2_Interp interp = e2_interp_from_bytecode(scratch.arena, &state, &space_map, bytecode);
val = interp.val;
if(e2_interp_status_is_terminal(interp.status))
{
if(interp.status != E2_InterpStatus_Good)
{
str8_list_pushf(scratch.arena, &msgs, "Interpretation error (%i).", interp.status);
}
break;
}
}
}
// rjf: message string list -> string
StringJoin join = {.sep = s(" ")};
String8 msgs_string = str8_list_join(scratch.arena, &msgs, &join);
// rjf: log
String8 log = str8f(scratch.arena, "%S -> %I64d (f32: %f) (f64: %f) *%s* (type: %S) %s%S\n",
strings[idx],
val.s64,
val.f32,
val.f64,
irtree->mode == E2_Mode_Type ? "type" :
irtree->mode == E2_Mode_Value ? "value" :
"address",
e2_string_from_type_key(scratch.arena, irtree->type_key),
msgs_string.size != 0 ? " // " : "", msgs_string);
log_info(log);
scratch_end(scratch);
}
}