Compare commits

...

5 Commits

22 changed files with 246 additions and 114 deletions

View File

@ -3,7 +3,7 @@
# include "helpers/push_ignores.inline.hpp"
# include "components/header_start.hpp"
# include "components/types.hpp"
# include "components/gen/ecode.hpp"
# include "components/gen/ecodetypes.hpp"
# include "components/gen/eoperator.hpp"
# include "components/gen/especifier.hpp"
# include "components/ast.hpp"

View File

@ -3,7 +3,7 @@
# include "helpers/push_ignores.inline.hpp"
# include "components/header_start.hpp"
# include "components/types.hpp"
# include "components/gen/ecode.hpp"
# include "components/gen/ecodetypes.hpp"
# include "components/gen/eoperator.hpp"
# include "components/gen/especifier.hpp"
# include "components/ast.hpp"

View File

@ -1,9 +1,6 @@
#ifdef INTELLISENSE_DIRECTIVES
#pragma once
#include "types.hpp"
#include "gen/ecode.hpp"
#include "gen/eoperator.hpp"
#include "gen/especifier.hpp"
#include "parser_types.hpp"
#endif
/*
@ -406,7 +403,8 @@ struct AST
Code PostNameMacro; // Only used with parameters for specifically UE_REQUIRES (Thanks Unreal)
};
};
StrCached Content; // Attributes, Comment, Execution, Include
StrCached Content; // Attributes, Comment, Execution, Include
TokenSlice ContentToks; // TODO(Ed): Use a token slice for content
struct {
Specifier ArrSpecs[AST_ArrSpecs_Cap]; // Specifiers
Code NextSpecs; // Specifiers; If ArrSpecs is full, then NextSpecs is used.
@ -422,7 +420,7 @@ struct AST
Code Next;
Code Back;
};
Token* Token; // Reference to starting token, only available if it was derived from parsing.
Token* Token; // Reference to starting token, only available if it was derived from parsing. // TODO(Ed): Change this to a token slice.
Code Parent;
CodeType Type;
// CodeFlag CodeFlags;

View File

@ -1,6 +1,6 @@
#ifdef INTELLISENSE_DIRECTIVES
#pragma once
#include "interface.hpp"
#include "constants.hpp"
#endif
#pragma region Serialization

View File

@ -71,6 +71,14 @@ void* fallback_allocator_proc( void* allocator_data, AllocType type, ssize size,
return nullptr;
}
internal
void fallback_logger(LogEntry entry)
{
GEN_ASSERT(entry.msg.Len > 0);
GEN_ASSERT(entry.msg.Ptr);
log_fmt("%S: %S", loglevel_to_str(entry.level), entry.msg);
}
internal
void define_constants()
{
@ -292,6 +300,10 @@ void init(Context* ctx)
ctx->InitSize_MacrosTable = kilobytes(8);
}
if (ctx->Logger == nullptr) {
ctx->Logger = & fallback_logger;
}
// Override the current context (user has to put it back if unwanted).
_ctx = ctx;
@ -307,7 +319,7 @@ void init(Context* ctx)
}
// Setup the code pool and code entries arena.
{
Pool code_pool = pool_init( ctx->Allocator_Pool, ctx->CodePool_NumBlocks, sizeof(AST) );
Pool code_pool = pool_init( ctx->Allocator_Pool, ctx->CodePool_NumBlocks, size_of(AST) );
if ( code_pool.PhysicalStart == nullptr )
GEN_FATAL( "gen::init: Failed to initialize the code pool" );
array_append( ctx->CodePools, code_pool );

View File

@ -15,23 +15,37 @@
\ \\ \ \\ \ \ \\ \ \ \ \
*/
#if 0
enum LogLevel : u32
enum LogLevel //: u32
{
Info,
Warning,
Panic,
LL_Null,
LL_Note,
LL_Warning,
LL_Error,
LL_Fatal,
LL_UnderlyingType = GEN_U32_MAX,
};
typedef enum LogLevel LogLevel;
Str loglevel_to_str(LogLevel level)
{
local_persist
Str lookup[] = {
{ "Null", sizeof("Null") - 1 },
{ "Note", sizeof("Note") - 1 },
{ "Warning", sizeof("Info") - 1 },
{ "Error", sizeof("Error") - 1 },
{ "Fatal", sizeof("Fatal") - 1 },
};
return lookup[level];
}
struct LogEntry
{
Str msg;
u32 line_num;
void* data;
Str msg;
LogLevel level;
};
typedef void LoggerCallback(LogEntry entry);
#endif
typedef void LoggerProc(LogEntry entry);
// Note(Ed): This is subject to heavily change
// with upcoming changes to the library's fallback (default) allocations strategy;
@ -70,6 +84,10 @@ struct Context
// TODO(Ed): Symbol Table
// Keep track of all resolved symbols (naemspaced identifiers)
// Logging
LoggerProc* Logger;
// Parser
// Used by the lexer to persistently treat all these identifiers as preprocessor defines.
@ -107,6 +125,45 @@ struct Context
// An implicit context interface will be provided instead as wrapper procedures as convience.
GEN_API extern Context* _ctx;
// By default this library will either crash or exit if an error is detected while generating codes.
// Even if set to not use GEN_FATAL, GEN_FATAL will still be used for memory failures as the library is unusable when they occur.
#ifdef GEN_DONT_USE_FATAL
#define log_failure log_fmt
#else
#define log_failure GEN_FATAL
#endif
// TODO(Ed): Swap all usage of this with logger_fmt (then rename logger_fmt to log_fmt)
inline
ssize log_fmt(char const* fmt, ...)
{
ssize res;
va_list va;
va_start(va, fmt);
res = c_str_fmt_out_va(fmt, va);
va_end(va);
return res;
}
inline
void logger_fmt(Context* ctx, LogLevel level, char const* fmt, ...)
{
local_persist thread_local
PrintF_Buffer buf = struct_zero(PrintF_Buffer);
va_list va;
va_start(va, fmt);
ssize res = c_str_fmt_va(buf, GEN_PRINTF_MAXLEN, fmt, va) -1;
va_end(va);
StrBuilder msg = strbuilder_make_length(ctx->Allocator_Temp, buf, res);
LogEntry entry = { strbuilder_to_str(msg), level };
ctx->Logger(entry);
}
// Initialize the library. There first ctx initialized must exist for lifetime of other contextes that come after as its the one that
GEN_API void init(Context* ctx);
@ -117,7 +174,7 @@ GEN_API void deinit(Context* ctx);
// Retrieves the active context (not usually needed, but here in case...)
GEN_API Context* get_context();
// Clears the allocations, but doesn't free the memoery, then calls init() again.
// Clears the allocations, but doesn't free the memory, then calls init() again.
// Ease of use.
GEN_API void reset(Context* ctx);
@ -337,37 +394,42 @@ forceinline CodeBody def_union_body ( s32 num, Code* codes )
#pragma region Parsing
#if 0
struct StackNode
struct ParseStackNode
{
StackNode* Prev;
ParseStackNode* Prev;
Token Start;
Token Name; // The name of the AST node (if parsed)
Str FailedProc; // The name of the procedure that failed
TokenSlice Tokens;
Token* Start;
Str Name; // The name of the AST node (if parsed)
Str ProcName; // The name of the procedure
Code CodeRel; // Relevant AST node
// TODO(Ed): When an error occurs, the parse stack is not released and instead the scope is left dangling.
};
// Stack nodes are allocated the error's allocator
struct Error
typedef struct ParseMessage ParseMessage;
struct ParseMessage
{
StrBuilder message;
StackNode* context_stack;
ParseMessage* Next;
ParseStackNode* Scope;
Str Log;
LogLevel Level;
};
struct ParseInfo
{
Arena FileMem;
Arena TokMem;
Arena CodeMem;
FileContents FileContent;
Array<Token> Tokens;
Array<Error> Errors;
// Errors are allocated to a dedicated general arena.
ParseMessage* messages;
LexedInfo lexed;
Code result;
};
CodeBody parse_file( Str path );
#endif
struct ParseOpts
{
AllocatorInfo backing_msgs;
AllocatorInfo backing_tokens;
AllocatorInfo backing_ast;
};
ParseInfo wip_parse_str( LexedInfo lexed, ParseOpts* opts GEN_PARAM_DEFAULT );
GEN_API CodeClass parse_class ( Str class_def );
GEN_API CodeConstructor parse_constructor ( Str constructor_def );
@ -398,9 +460,10 @@ GEN_API ssize token_fmt_va( char* buf, usize buf_size, s32 num_tokens, va_list v
//! Do not use directly. Use the token_fmt macro instead.
Str token_fmt_impl( ssize, ... );
GEN_API Code untyped_str( Str content);
GEN_API Code untyped_str ( Str content);
GEN_API Code untyped_fmt ( char const* fmt, ... );
GEN_API Code untyped_token_fmt( s32 num_tokens, char const* fmt, ... );
GEN_API Code untyped_toks ( TokenSlice tokens );
#pragma endregion Untyped text

View File

@ -1,6 +1,6 @@
#ifdef INTELLISENSE_DIRECTIVES
#pragma once
#include "gen/etoktype.cpp"
#include "gen/etoktype.hpp"
#include "interface.upfront.cpp"
#include "lexer.cpp"
#include "parser.cpp"
@ -8,6 +8,29 @@
// Publically Exposed Interface
ParseInfo wip_parse_str(LexedInfo lexed, ParseOpts* opts)
{
TokArray toks;
if (lexed.tokens.Num == 0 && lexed.tokens.Ptr == nullptr) {
check_parse_args(lexed.text);
toks = lex(lexed.text);
TokenSlice slice = { toks.Arr, scast(s32, array_num(toks.Arr)) };
lexed.tokens = slice;
}
ParseInfo info = struct_zero(ParseInfo);
info.lexed = lexed;
// TODO(Ed): ParseInfo should be set to the parser context.
_ctx->parser.Tokens = toks;
push_scope();
CodeBody result = parse_global_nspace(CT_Global_Body);
parser_pop(& _ctx->parser);
return info;
}
CodeClass parse_class( Str def )
{
check_parse_args( def );

View File

@ -176,3 +176,16 @@ Code untyped_token_fmt( s32 num_tokens, char const* fmt, ... )
return result;
}
Code untyped_toks( TokenSlice tokens )
{
if ( tokens.Num == 0 ) {
log_failure( "untyped_toks: empty token slice" );
return InvalidCode;
}
Code
result = make_code();
result->Type = CT_Untyped;
result->ContentToks = tokens;
return result;
}

View File

@ -1,7 +1,7 @@
#ifdef INTELLISENSE_DIRECTIVES
#pragma once
#include "interface.upfront.cpp"
#include "gen/etoktype.cpp"
#include "gen/etoktype.hpp"
#endif
StrBuilder tok_to_strbuilder(Token tok)
@ -564,9 +564,12 @@ void lex_found_token( LexContext* ctx )
array_append( _ctx->Lexer_Tokens, ctx->token );
}
// TODO(Ed): We should dynamically allocate the lexer's array in Allocator_DyanmicContainers.
// TODO(Ed): We need to to attempt to recover from a lex failure?
neverinline
// TokArray lex( Array<Token> tokens, Str content )
// void lex( Array<Token> tokens, Str content )
TokArray lex( Str content )
{
LexContext c; LexContext* ctx = & c;

View File

@ -1,6 +1,6 @@
#ifdef INTELLISENSE_DIRECTIVES
#pragma once
#include "gen/etoktype.cpp"
#include "gen/etoktype.hpp"
#include "parser_case_macros.cpp"
#include "interface.upfront.cpp"
#include "lexer.cpp"
@ -11,7 +11,7 @@
constexpr bool lex_dont_skip_formatting = false;
constexpr bool lex_skip_formatting = true;
void parser_push( ParseContext* ctx, StackNode* node )
void parser_push( ParseContext* ctx, ParseStackNode* node )
{
node->Prev = ctx->Scope;
ctx->Scope = node;
@ -60,7 +60,7 @@ StrBuilder parser_to_strbuilder(ParseContext ctx)
else
strbuilder_append_fmt(& result, "\t(%d, %d)\n", last_valid.Line, last_valid.Column );
StackNode* curr_scope = ctx.Scope;
ParseStackNode* curr_scope = ctx.Scope;
s32 level = 0;
do
{
@ -181,16 +181,17 @@ bool _check_parse_args( Str def, char const* func_name )
# define check_noskip( Type_ ) ( left && currtok_noskip.Type == Type_ )
# define check( Type_ ) ( left && currtok.Type == Type_ )
# define push_scope() \
Str null_name = {}; \
StackNode scope = { nullptr, lex_current( & _ctx->parser.Tokens, lex_dont_skip_formatting ), null_name, txt( __func__ ) }; \
// TODO(Ed): Don't do this anymore, we need a better initializer.
# define push_scope() \
Str null_name = {}; \
ParseStackNode scope = { nullptr, {nullptr, 0}, lex_current( & _ctx->parser.Tokens, lex_dont_skip_formatting ), null_name, txt( __func__ ), { nullptr} }; \
parser_push( & _ctx->parser, & scope )
#pragma endregion Helper Macros
// Procedure Forwards ( Entire parser internal parser interface )
internal Code parse_array_decl ();
internal Code parse_array_decl (ParseContext* ctx);
internal CodeAttributes parse_attributes ();
internal CodeComment parse_comment ();
internal Code parse_complicated_definition ( TokType which );
@ -490,8 +491,15 @@ StrBuilder parser_strip_formatting( Str raw_text, bool preserve_newlines )
return content;
}
StrBuilder parser_strip_formatting_2(TokenSlice tokens)
{
// TODO(Ed): Use this to produce strings for validation purposes. We shouldn't serialize down from tokens once we start storing token slices for content.
StrBuilder result = struct_zero(StrBuilder);
return result;
}
internal
Code parse_array_decl()
Code parse_array_decl(ParseContext* ctx)
{
push_scope();
@ -524,16 +532,20 @@ Code parse_array_decl()
return InvalidCode;
}
TokenSlice tokens = { & currtok, 1 };
Token untyped_tok = currtok;
while ( left && currtok.Type != Tok_BraceSquare_Close )
{
eat( currtok.Type );
++ tokens.Num;
}
untyped_tok.Text.Len = ( (sptr)prevtok.Text.Ptr + prevtok.Text.Len ) - (sptr)untyped_tok.Text.Ptr;
// untyped_tok.Text.Len = ( (sptr)prevtok.Text.Ptr + prevtok.Text.Len ) - (sptr)untyped_tok.Text.Ptr;
untyped_tok.Text = token_range_to_str(untyped_tok, prevtok);
Code array_expr = untyped_str( untyped_tok.Text );
// Code array_expr = untyped_toks( tokens ); // TODO(Ed): Use token slice instead of untyped strings.
// [ <Content>
if ( left == 0 )
@ -556,7 +568,7 @@ Code parse_array_decl()
// Its a multi-dimensional array
if ( check( Tok_BraceSquare_Open ))
{
Code adjacent_arr_expr = parse_array_decl();
Code adjacent_arr_expr = parse_array_decl(ctx);
// [ <Content> ][ <Content> ]...
array_expr->Next = adjacent_arr_expr;
@ -3290,7 +3302,7 @@ CodeVar parse_variable_after_name(
{
push_scope();
Code array_expr = parse_array_decl();
Code array_expr = parse_array_decl(& _ctx->parser);
Code expr = NullCode;
Code bitfield_expr = NullCode;
@ -5344,7 +5356,7 @@ CodeTypedef parser_parse_typedef()
return InvalidCode;
}
array_expr = parse_array_decl();
array_expr = parse_array_decl(& _ctx->parser);
// <UnderlyingType> + <ArrayExpr>
}
@ -5590,7 +5602,7 @@ CodeUsing parser_parse_using()
type = parser_parse_type(parser_not_from_template, nullptr);
// <ModuleFlags> using <Name> <Attributes> = <UnderlyingType>
array_expr = parse_array_decl();
array_expr = parse_array_decl(& _ctx->parser);
// <UnderlyingType> + <ArrExpr>
}
}

View File

@ -1,7 +1,7 @@
#ifdef INTELLISENSE_DIRECTIVES
#pragma once
#include "types.hpp"
#include "gen/ecode.hpp"
#include "gen/ecodetypes.hpp"
#include "gen/eoperator.hpp"
#include "gen/especifier.hpp"
#include "gen/etoktype.hpp"
@ -91,6 +91,27 @@ bool tok_is_end_definition(Token tok) {
StrBuilder tok_to_strbuilder(Token tok);
struct TokenSlice
{
Token* Ptr;
s32 Num;
#if GEN_COMPILER_CPP
forceinline operator Token* () const { return Ptr; }
forceinline Token& operator[]( ssize index ) const { return Ptr[index]; }
#endif
};
forceinline
Str token_range_to_str(Token start, Token end)
{
Str result = {
start.Text.Ptr,
(scast(sptr, rcast(uptr, end.Text.Ptr)) + end.Text.Len) - scast(sptr, rcast(uptr, start.Text.Ptr))
};
return result;
}
struct TokArray
{
Array(Token) Arr;
@ -104,23 +125,21 @@ struct LexContext
char const* scanner;
s32 line;
s32 column;
// StringTable defines;
Token token;
};
struct StackNode
struct LexedInfo
{
StackNode* Prev;
Token* Start;
Str Name; // The name of the AST node (if parsed)
Str ProcName; // The name of the procedure
Str text;
TokenSlice tokens;
};
typedef struct ParseStackNode ParseStackNode;
struct ParseContext
{
TokArray Tokens;
StackNode* Scope;
TokArray Tokens;
ParseStackNode* Scope;
};
enum MacroType : u16

View File

@ -1,6 +1,6 @@
#ifdef INTELLISENSE_DIRECTIVES
#pragma once
#include "../gen.hpp"
#include "interface.hpp"
#endif
#pragma region StaticData

View File

@ -1,6 +1,18 @@
#ifdef INTELLISENSE_DIRECTIVES
#pragma once
#include "header_start.hpp"
#include "dependencies/platform.hpp"
#include "dependencies/macros.hpp"
#include "dependencies/basic_types.hpp"
#include "dependencies/debug.hpp"
#include "dependencies/memory.hpp"
#include "dependencies/string_ops.hpp"
#include "dependencies/printing.hpp"
#include "dependencies/containers.hpp"
#include "dependencies/hashing.hpp"
#include "dependencies/strings.hpp"
#include "dependencies/filesystem.hpp"
#include "dependencies/timing.hpp"
#include "dependencies/parsing.hpp"
#endif
/*
@ -19,16 +31,6 @@
*/
using LogFailType = ssize(*)(char const*, ...);
// By default this library will either crash or exit if an error is detected while generating codes.
// Even if set to not use GEN_FATAL, GEN_FATAL will still be used for memory failures as the library is unusable when they occur.
#ifdef GEN_DONT_USE_FATAL
#define log_failure log_fmt
#else
#define log_failure GEN_FATAL
#endif
enum AccessSpec : u32
{
AccessSpec_Default,

View File

@ -1,9 +1,6 @@
#ifdef INTELLISENSE_DIRECTIVES
# pragma once
# include "dependencies/platform.hpp"
# include "dependencies/macros.hpp"
# include "basic_types.hpp"
# include "macros.hpp"
#endif
#pragma region Debug

View File

@ -187,7 +187,7 @@ struct FileContents
{
AllocatorInfo allocator;
void* data;
ssize size;
ssize size;
};
constexpr b32 file_zero_terminate = true;

View File

@ -298,10 +298,20 @@
# define GEN_PARAM_DEFAULT
#endif
#if GEN_COMPILER_CPP
#define struct_init(type, value) {value}
#else
#define struct_init(type, value) {value}
#ifndef struct_init
# if GEN_COMPILER_CPP
# define struct_init(type, value) value
# else
# define struct_init(type, value) (type) value
# endif
#endif
#ifndef struct_zero
# if GEN_COMPILER_CPP
# define struct_zero(type) {}
# else
# define struct_zero(type) {0}
# endif
#endif
#if 0

View File

@ -134,12 +134,6 @@ GEN_API void* heap_allocator_proc( void* allocator_data, AllocType type, ssize s
//! The heap allocator backed by operating system's memory manager.
constexpr AllocatorInfo heap( void ) { AllocatorInfo allocator = { heap_allocator_proc, nullptr }; return allocator; }
//! Helper to allocate memory using heap allocator.
#define malloc( sz ) alloc( heap(), sz )
//! Helper to free memory allocated by heap allocator.
#define mfree( ptr ) allocator_free( heap(), ptr )
struct VirtualMemory
{
void* data;
@ -185,6 +179,8 @@ void arena_check (Arena* arena);
void arena_free (Arena* arena);
ssize arena_size_remaining(Arena* arena, ssize alignment);
// TODO(Ed): Add arena_pos, arena_pop, and arena_pop_to
struct Arena
{
AllocatorInfo Backing;

View File

@ -1,6 +1,6 @@
#ifdef INTELLISENSE_DIRECTIVES
# pragma once
# include "strbuilder_ops.cpp"
# include "string_ops.cpp"
#endif
#pragma region Printing

View File

@ -1,6 +1,6 @@
#ifdef INTELLISENSE_DIRECTIVES
# pragma once
# include "strbuilder_ops.hpp"
# include "string_ops.hpp"
#endif
#pragma region Printing
@ -26,17 +26,4 @@ GEN_API ssize c_str_fmt_file_va ( FileInfo* f, char const* fmt, va_list va );
constexpr
char const* Msg_Invalid_Value = "INVALID VALUE PROVIDED";
inline
ssize log_fmt(char const* fmt, ...)
{
ssize res;
va_list va;
va_start(va, fmt);
res = c_str_fmt_out_va(fmt, va);
va_end(va);
return res;
}
#pragma endregion Printing

View File

@ -320,7 +320,7 @@ inline
StrBuilder strbuilder_fmt_buf(AllocatorInfo allocator, char const* fmt, ...)
{
local_persist thread_local
PrintF_Buffer buf = struct_init(PrintF_Buffer, {0});
PrintF_Buffer buf = struct_zero(PrintF_Buffer);
va_list va;
va_start(va, fmt);

View File

@ -11,9 +11,6 @@
#include "helpers/push_ignores.inline.hpp"
#include "components/header_start.hpp"
// Has container defines pushed
#include "gen.dep.hpp"
GEN_NS_BEGIN
#include "components/types.hpp"

View File

@ -535,7 +535,7 @@ namespace Lexer_, gen_Lexer_
word LexContext, gen_LexContext
word lex, gen_lex
word StackNode, gen_StackNode
word ParseStackNode, gen_ParseStackNode
word ParseContext, gen_ParseContext
// namespace parse_, gen_parse_