From 231c893c6b008e2a42635237305d386ee39f042b Mon Sep 17 00:00:00 2001 From: Ed_ Date: Fri, 17 Mar 2023 02:09:19 -0400 Subject: [PATCH 1/9] Rework of project implementation For include and multi-file support. I still need to debug it, Test will be adjusted as well; I want to get all the files not just zpl refactored using a powershell script. I dropped the idea of semantically identifiying macros. While it may be possible, I don't see the utility vs the regular idendentifier distinction. I want to keep the refactoring as simple as possible, where it just takes one pass to go through a file without any context to other files. So far the ignores behave as a good guard filter for unwanted refactors and the only true weak area was the includes (which should be aleviated with the coming support for it. --- Readme.md | 30 +- Test/bloat.refactored.hpp | 2 +- project/Bloat.cpp | 37 ++ project/IO.cpp | 159 +++++++ project/IO.hpp | 25 ++ project/Spec.cpp | 315 ++++++++++++++ project/Spec.hpp | 73 ++++ project/_Docs.md | 41 ++ project/bloat.hpp | 63 ++- project/refactor.cpp | 853 ++++++++++++++------------------------ refactor.10x | 63 +++ 11 files changed, 1074 insertions(+), 587 deletions(-) create mode 100644 project/Bloat.cpp create mode 100644 project/IO.cpp create mode 100644 project/IO.hpp create mode 100644 project/Spec.cpp create mode 100644 project/Spec.hpp create mode 100644 project/_Docs.md create mode 100644 refactor.10x diff --git a/Readme.md b/Readme.md index af0376c..f592b90 100644 --- a/Readme.md +++ b/Readme.md @@ -1,9 +1,10 @@ # refactor -A code identifier refactoring app. Intended for c/c++ like identifiers. +Refactor c/c++ files (and problably others) with ease. Parameters : +* `-num` : Used if more than one source file is provided (if used, number of destination files provided MUST MATCH). * `-src` : Source file to refactor * `-dst` : Destination file after the refactor (omit to use the same as source) * `-spec` : Specification containing rules to use for the refactor. @@ -11,18 +12,33 @@ Parameters : Syntax : * `not` Omit word or namespace. +* `include` Preprocessor include related identifiers. * `word` Fixed sized identifier. * `namespace` Variable sized identifiers, mainly intended to redefine c-namespace of an identifier. * `,` is used to delimit arguments to word or namespace. * `L-Value` is the signature to modify. * `R-Value` is the substitute ( only available if rule does not use `not` keyword ) +The only keyword here excluisve to c/c++ is the `include` as it does search specifically for `#include `. +However, the rest of the categorical keywords (word, namespace), can really be used for any langauge. + +There is no semantic awareness this is truely just a simple find and replace, but with some filters specifiable, and +words/namespaces only being restricted to the rules for C/C++ identifiers (alphanumeric or underscores only) + +The main benefit for using this over other stuff is its faster and more ergonomic for large refactors on libraries that +you may want to have automated in a script. + +There are other programs more robust for doing that sort of thing but I was not able to find something this simple. + +**Note** +* Building for debug provides some nice output with context on a per-line basis. +* Release will only show errors for asserts (that will kill the refactor early). +* If the refactor crashes, the files previously written to will retain their changes. +Make sure to have the code backed up on a VCS or in some other way. +* This was compiled using meson with ninja and clang on windows 11. The ZPL library used however should work fine on the other major os platforms and compiler venders. +* The scripts used for building and otherwise are in the scripts directory and are all in powershell (with exception to the meson.build). Techncially there should be a powershell package available on other platorms but worst case it should be pretty easily to port these scripts to w/e shell script you'd perfer. TODO: * Possibly come up with a better name. -* Cleanup memory usage (it hogs quite a bit for what it does..) -* Split lines of file and refactor it that way instead (better debug, problably negligable performance loss, worst case can have both depending on build type) -* Accept multiple files at once `-files` -* Add support for `macro` keyword (single out macro identifiers) -* Add support for `include` keyword (single out include definitions) -* Add support for auto-translating a namespace in a macro to a cpp namespace +* Test to see how much needs to be ported for other platforms (if at all) +* Provide binaries in the release page for github. (debug and release builds) diff --git a/Test/bloat.refactored.hpp b/Test/bloat.refactored.hpp index 3c390ea..ec79e74 100644 --- a/Test/bloat.refactored.hpp +++ b/Test/bloat.refactored.hpp @@ -85,7 +85,7 @@ namespace Memory void setup() { - arena_init_from_allocator( & Global_Arena, heap(), megabytes(10) ); + arena_init_from_allocator( & Global_Arena, heap(), megabytes(1) ); if ( Global_Arena.total_size == 0 ) { diff --git a/project/Bloat.cpp b/project/Bloat.cpp new file mode 100644 index 0000000..648d780 --- /dev/null +++ b/project/Bloat.cpp @@ -0,0 +1,37 @@ +#define BLOAT_IMPL +#include "bloat.hpp" + + + +namespace Memory +{ + static zpl_arena Global_Arena {}; + + void setup() + { + zpl_arena_init_from_allocator( & Global_Arena, zpl_heap(), zpl_megabytes(2) ); + + if ( Global_Arena.total_size == 0 ) + { + zpl_assert_crash( "Failed to reserve memory for Tests:: Global_Arena" ); + } + } + + void resize( uw new_size ) + { + void* new_memory = zpl_resize( zpl_heap(), Global_Arena.physical_start, Global_Arena.total_size, new_size ); + + if ( new_memory == nullptr ) + { + fatal("Failed to resize global arena!"); + } + + Global_Arena.physical_start = new_memory; + Global_Arena.total_size = new_size; + } + + void cleanup() + { + zpl_arena_free( & Global_Arena); + } +} diff --git a/project/IO.cpp b/project/IO.cpp new file mode 100644 index 0000000..c0dc399 --- /dev/null +++ b/project/IO.cpp @@ -0,0 +1,159 @@ +#include "IO.hpp" + + +namespace IO +{ + using array_string = zpl_array( zpl_string ); + + namespace StaticData + { + array_string Sources = nullptr; + array_string Destinations = nullptr; + zpl_string Specification = nullptr; + + // Current source and destination index. + // Used to keep track of which file get_next_source or write refer to. + uw Current = 0; + char* Current_Content = nullptr; + uw Current_Size = 0; + uw Largest_Src_Size = 0; + + /* + Will persist throughout loading different file content. + Should hold a bit more than the largest source file's content, + As an array of lines. + */ + zpl_arena MemPerist; + + /* + Temporary memory held while procesisng files to get their content. + zpl_files are stored here + */ + // zpl_arena MemTransient; + } + using namespace StaticData; + + + void prepare() + { + const sw num_srcs = zpl_array_count( Sources ); + + // Determine the largest content size. + sw left = num_srcs; + zpl_string* path = Sources; + do + { + zpl_file src = {}; + zpl_file_error error = zpl_file_open( & src, *path ); + + if ( error != ZPL_FILE_ERROR_NONE ) + { + fatal("Could not open source file: %s", *path ); + } + + const sw fsize = zpl_file_size( & src ); + + if ( fsize > Largest_Src_Size ) + { + Largest_Src_Size = fsize; + } + + zpl_file_close( & src ); + } + while ( --left ); + + uw persist_size = ZPL_ARRAY_GROW_FORMULA( Largest_Src_Size ); + + zpl_arena_init_from_allocator( & MemPerist, zpl_heap(), persist_size ); + // zpl_arena_init_from_allocator( & MemTransient, zpl_heap(), Largest_Src_Size ); + } + + void cleanup() + { + zpl_arena_free( & MemPerist ); + // zpl_arena_free( & MemTransient ); + } + + Array_Line get_specification() + { + zpl_file file {}; + zpl_file_error error = zpl_file_open( & file, Specification); + + if ( error != ZPL_FILE_ERROR_NONE ) + { + fatal("Could not open the specification file: %s", Specification); + } + + sw fsize = scast( sw, zpl_file_size( & file ) ); + + if ( fsize <= 0 ) + { + fatal("No content in specificaiton to process"); + } + + char* content = rcast( char*, zpl_alloc( zpl_arena_allocator( & MemPerist), fsize + 1) ); + + zpl_file_read( & file, content, fsize); + zpl_file_close( & file ); + + content[fsize] = 0; + + Array_Line lines = zpl_str_split_lines( zpl_arena_allocator( & MemPerist ), content, false ); + return lines; + } + + Array_Line get_next_source() + { + // zpl_memset( MemTransient.physical_start, 0, MemTransient.total_allocated); + // MemTransient.total_allocated = 0; + // MemTransient.temp_count = 0; + + zpl_memset( MemPerist.physical_start, 0, MemPerist.total_allocated); + MemPerist.total_allocated = 0; + MemPerist.temp_count = 0; + + zpl_file file {}; + zpl_file_error error = zpl_file_open( & file, Specification); + + if ( error != ZPL_FILE_ERROR_NONE ) + { + fatal("Could not open the source file: %s", Sources[Current]); + } + + Current_Size = scast( sw, zpl_file_size( & file ) ); + + if ( Current_Size <= 0 ) + return nullptr; + + Current_Content = rcast( char* , zpl_alloc( zpl_arena_allocator( & MemPerist), Current_Size + 1) ); + + zpl_file_read( & file, Current_Content, Current_Size ); + zpl_file_close( & file ); + + Current_Content[Current_Size] = 0; + Current_Size++; + + Array_Line lines = zpl_str_split_lines( zpl_arena_allocator( & MemPerist), Current_Content, ' ' ); + return lines; + } + + void write( zpl_string refacotred ) + { + if ( refacotred == nullptr) + return; + + zpl_string dst = Destinations[Current]; + + zpl_file file_dest {}; + zpl_file_error error = zpl_file_create( & file_dest, dst ); + + if ( error != ZPL_FILE_ERROR_NONE ) + { + fatal( "Unable to open destination file: %s\n", dst ); + } + + zpl_file_write( & file_dest, refacotred, zpl_string_length(refacotred) ); + + zpl_file_close( & file_dest ); + } +} diff --git a/project/IO.hpp b/project/IO.hpp new file mode 100644 index 0000000..051a6a3 --- /dev/null +++ b/project/IO.hpp @@ -0,0 +1,25 @@ +#pragma once + +#include "bloat.hpp" + + +namespace IO +{ + ct uw Path_Size_Largest = zpl_kilobytes(1); + + // Preps the IO by loading all the files and checking to see what the largest size is. + // The file with the largest size is used to determine the size of the persistent memory. + void prepare(); + + // Frees the persistent and transient memory arenas. + void cleanup(); + + // Provides the content of the specification. + Array_Line get_specification(); + + // Provides the content of the next source, broken up as a series of lines. + Array_Line get_next_source(); + + // Writes the refactored content ot the current corresponding destination. + void write( zpl_string refactored ); +} diff --git a/project/Spec.cpp b/project/Spec.cpp new file mode 100644 index 0000000..6ef1d98 --- /dev/null +++ b/project/Spec.cpp @@ -0,0 +1,315 @@ +#include "Spec.hpp" + +#include "IO.hpp" + + + +namespace Spec +{ + ct uw Array_Reserve_Num = zpl_kilobytes(4); + ct uw Token_Max_Length = zpl_kilobytes(1); + + namespace StaticData + { + Array_Entry Ignore_Includes; + Array_Entry Ignore_Words; + Array_Entry Ignore_Regexes; + Array_Entry Ignore_Namespaces; + + Array_Entry Includes; + Array_Entry Words; + Array_Entry Regexes; + Array_Entry Namespaces; + + u32 Sig_Smallest = Token_Max_Length; + } + using namespace StaticData; + + void cleanup() + { + zpl_array_free( Ignore_Includes ); + zpl_array_free( Ignore_Words ); + zpl_array_free( Ignore_Namespaces ); + zpl_array_free( Includes ); + zpl_array_free( Words ); + zpl_array_free( Namespaces ); + } + + + // Helper function for process(). + forceinline + void find_next_token( zpl_string& token, char*& line, u32& length ) + { + zpl_string_clear( token ); + length = 0; + + while ( zpl_char_is_alphanumeric( line[length] ) || line[length] == '_' ) + { + length++; + } + + if ( length == 0 ) + { + fatal("Failed to find valid initial token"); + } + + token = zpl_string_append_length( token, line, length ); + line += length; + } + + void parse() + { + static zpl_string token = zpl_string_make_reserve( g_allocator, zpl_kilobytes(1)); + + static bool Done = false; + if (Done) + { + zpl_array_clear( Ignore_Includes ); + zpl_array_clear( Ignore_Words ); + zpl_array_clear( Ignore_Namespaces ); + zpl_array_clear( Includes ); + zpl_array_clear( Words ); + zpl_array_clear( Namespaces ); + } + else + { + Done = true; + + zpl_array_init_reserve( Ignore_Includes, zpl_heap(), Array_Reserve_Num ); + zpl_array_init_reserve( Ignore_Words, zpl_heap(), Array_Reserve_Num ); + zpl_array_init_reserve( Ignore_Namespaces, zpl_heap(), Array_Reserve_Num ); + zpl_array_init_reserve( Includes, zpl_heap(), Array_Reserve_Num ); + zpl_array_init_reserve( Words, zpl_heap(), Array_Reserve_Num ); + zpl_array_init_reserve( Namespaces, zpl_heap(), Array_Reserve_Num ); + } + + Array_Line lines = IO::get_specification(); + + sw left = zpl_array_count( lines ); + + if ( left == 0 ) + { + fatal("Spec::parse: lines array imporoperly setup"); + } + + // Skip the first line as its the version number and we only support __VERSION 1. + left--; + lines++; + + char token[ Token_Max_Length ]; + + do + { + char* line = * lines; + + // Ignore line if its a comment + if ( line[0] == '/' && line[1] == '/') + { + lines++; + continue; + } + + // Remove indent + { + while ( zpl_char_is_space( line[0] ) ) + line++; + + if ( line[0] == '\0' ) + { + lines++; + continue; + } + } + + u32 length = 0; + + // Find a valid token + find_next_token( token, line, length ); + + Tok type = Tok::Num_Tok; + bool ignore = false; + Entry entry {}; + + // Will be reguarded as an ignore. + if ( is_tok( Tok::Not, token, length )) + { + ignore = true; + + while ( zpl_char_is_space( line[0] ) ) + line++; + + if ( line[0] == '\0' ) + { + lines++; + continue; + } + + // Find the category token + find_next_token( token, line, length ); + } + + if ( is_tok( Tok::Word, token, length ) ) + { + type = Tok::Word; + } + else if ( is_tok( Tok::Namespace, token, length ) ) + { + type = Tok::Namespace; + } + else if ( is_tok( Tok::Include, token, length )) + { + type = Tok::Include; + } + else + { + log_fmt( "Sec::Parse - Unsupported keyword: %s on line: %d", token, zpl_array_count(lines) - left ); + lines++; + continue; + } + + // Find the first argument + while ( zpl_char_is_space( line[0] ) ) + line++; + + if ( line[0] == '\0' ) + { + lines++; + continue; + } + + find_next_token( token, line, length ); + + // First argument is signature. + entry.Sig = zpl_string_make_length( g_allocator, token, length ); + + if ( length < StaticData::Sig_Smallest ) + StaticData::Sig_Smallest = length; + + if ( line[0] == '\0' || ignore ) + { + switch ( type ) + { + case Tok::Word: + if ( ignore) + zpl_array_append( Ignore_Words, entry ); + + else + zpl_array_append( Words, entry ); + break; + + case Tok::Namespace: + if ( ignore) + zpl_array_append( Ignore_Namespaces, entry ); + + else + zpl_array_append( Namespaces, entry ); + break; + + case Tok::Include: + if ( ignore) + zpl_array_append( Ignore_Includes, entry ); + + else + zpl_array_append( Includes, entry ); + break; + } + + lines++; + continue; + } + + // Look for second argument indicator + { + bool bSkip = false; + + while ( line[0] != ',' ) + { + if ( line[0] == '\0' ) + { + switch ( type ) + { + case Tok::Word: + zpl_array_append( Words, entry ); + break; + + case Tok::Namespace: + zpl_array_append( Namespaces, entry ); + break; + + case Tok::Include: + zpl_array_append( Includes, entry ); + break; + } + + bSkip = true; + break; + } + + line++; + } + + if ( bSkip ) + { + lines++; + continue; + } + } + + // Eat the argument delimiter. + line++; + + // Remove spacing + { + bool bSkip = true; + + while ( zpl_char_is_space( line[0] ) ) + line++; + + if ( line[0] == '\0' ) + { + switch ( type ) + { + case Tok::Word: + zpl_array_append( Words, entry ); + break; + + case Tok::Namespace: + zpl_array_append( Namespaces, entry ); + break; + + case Tok::Include: + zpl_array_append( Includes, entry ); + break; + } + + lines++; + continue; + } + } + + find_next_token( token, line, length ); + + // Second argument is substitute. + entry.Sub = zpl_string_make_length( g_allocator, token, length ); + + switch ( type ) + { + case Tok::Word: + zpl_array_append( Words, entry ); + lines++; + continue; + + case Tok::Namespace: + zpl_array_append( Namespaces, entry ); + lines++; + continue; + + case Tok::Include: + zpl_array_append( Includes, entry ); + lines++; + continue; + } + } + while ( --left ); + } +} diff --git a/project/Spec.hpp b/project/Spec.hpp new file mode 100644 index 0000000..c3e158d --- /dev/null +++ b/project/Spec.hpp @@ -0,0 +1,73 @@ +#pragma once +#include "Bloat.hpp" + + +namespace Spec +{ + enum Tok + { + Not, + Include, + Namespace, + Word, + + Num_Tok + }; + + forceinline + char const* str_tok( Tok tok ) + { + static + char const* tok_to_str[ Tok::Num_Tok ] = + { + "not", + "include", + "namespace", + "word", + }; + + return tok_to_str[ tok ]; + } + + forceinline + char strlen_tok( Tok tok ) + { + static + const u8 tok_to_len[ Tok::Num_Tok ] = + { + 3, + 7, + 9, + 4, + }; + + return tok_to_len[ tok ]; + } + + forceinline + bool is_tok( Tok tok, zpl_string str, u32 length ) + { + char const* tok_str = str_tok(tok); + const u8 tok_len = strlen_tok(tok); + + if ( tok_len != length) + return false; + + s32 result = zpl_strncmp( tok_str, str, tok_len ); + + return result == 0; + } + + struct Entry + { + zpl_string Sig = nullptr; // Signature + zpl_string Sub = nullptr; // Substitute + }; + + using Array_Entry = zpl_array( Entry ); + + void cleanup(); + + // Extract the specificication from the provided file. + void parse(); +} diff --git a/project/_Docs.md b/project/_Docs.md new file mode 100644 index 0000000..dc7194a --- /dev/null +++ b/project/_Docs.md @@ -0,0 +1,41 @@ +# Documentation + +The current implementation is divided into 4 parts: + +* Bloat : General library provider. +* IO : File I/O processing. +* Spec : Specification parsing. +* Refactor : Entrypoint, argument parsing, and refactoring process. + +The files are setup to compile as one unit. As such the source files for Bloat, IO, and Spec are located within `refactor.cpp`. + +Bloat contains some aliasing of some C++ keywords and does not use the standard library. Instead a library called ZPL is used (Single header replacement). + +The program has pretty much no optimizations made to it, its just regular loops with no threading. +Just tried to keep the memory at reasonable size of what it does. + +The program execution is pretty much outlined quite clearly in `int main()`. + +1. Setup initial reserve of global memory in an arena. +2. Parse the arguments provided. +3. Prepare IO's memory for retreviing content. +4. Reserve memory for the refactor buffer. +5. Parse the specification file +6. Iterate through all provided files to refactor and write the refactored content to the specificed destintation files. +7. Cleanup all reserves of memory`*` + + +`*` This technically can be skipped on windows, may be worth doing to reduce latency of process shutdown. + +There are constraints of specific sizes of variables; + +* `Path_Size_Largest` : Longest path size is set to 1 KB of characters. +* `Token_Max_Length` : Set to 1 KB characters as well. +* `Array_Reserve_Num` : Is set to 4 KB. +* Initial Global arena size : Set to 2 megabytes. + +The `Path_Size_Largest` and `Token_Max_Length` are compile-time constraints that the runtime will not have a fallback for, if 1 KB is not enough it will need to be changed for your use case. + +`Array_Reserve_Num` is used to dictate the assumed amount of tokens will be held in total for any of spec's arrays holding ignores and refactor entries. If any of the array's exceed 4 KB they will grow trigigng a resize which will bog down the speed of the refactor. Adjust if you think you can increase or lower for use case. + +Initial Global arena size is a finicy thing, its most likely going to be custom allocator at one point so that it can handle growing properly, right now its just grows if the amount of memory file paths will need for sources is greater than 1 MB. diff --git a/project/bloat.hpp b/project/bloat.hpp index 2be7815..1cfd188 100644 --- a/project/bloat.hpp +++ b/project/bloat.hpp @@ -1,13 +1,13 @@ /* BLOAT. - - ZPL requires ZPL_IMPLEMENTATION whereever this library is included. - - This file assumes it will be included in one compilation unit. */ #pragma once +#ifdef BLOAT_IMPL +# define ZPL_IMPLEMEntATION +#endif + #if __clang__ # pragma clang diagnostic ignored "-Wunused-const-variable" # pragma clang diagnostic ignored "-Wswitch" @@ -33,14 +33,13 @@ // # define ZPL_MODULE_REGEX // # define ZPL_MODULE_EVENT // # define ZPL_MODULE_DLL -# define ZPL_MODULE_OPTS +# define ZPL_MODULE_OPTS // # define ZPL_MODULE_PROCESS // # define ZPL_MODULE_MAT // # define ZPL_MODULE_THREADING // # define ZPL_MODULE_JOBS // # define ZPL_MODULE_PARSER #include "zpl.h" -// } #if __clang__ # pragma clang diagnostic pop @@ -59,16 +58,16 @@ #define rcast( Type_, Value_ ) reinterpret_cast< Type_ >( Value_ ) #define pcast( Type_, Value_ ) ( * (Type_*)( & Value_ ) ) -#define do_once() \ -do \ -{ \ - static \ - bool Done = true; \ - if ( Done ) \ - return; \ - Done = false; \ -} \ -while(0) \ +#define do_once() \ +do \ +{ \ + static \ + bool Done = false; \ + if ( Done ) \ + return; \ + Done = true; \ +} \ +while(0) \ using s8 = zpl_i8; @@ -81,39 +80,34 @@ using f64 = zpl_f64; using uw = zpl_usize; using sw = zpl_isize; +using Line = char*; +using Array_Line = zpl_array( Line ); + ct char const* Msg_Invalid_Value = "INVALID VALUE PROVIDED"; namespace Memory { - zpl_arena Global_Arena {}; + extern zpl_arena Global_Arena; #define g_allocator zpl_arena_allocator( & Memory::Global_Arena) - void setup() - { - zpl_arena_init_from_allocator( & Global_Arena, zpl_heap(), zpl_megabytes(10) ); - - if ( Global_Arena.total_size == 0 ) - { - zpl_assert_crash( "Failed to reserve memory for Tests:: Global_Arena" ); - } - } - - void cleanup() - { - zpl_arena_free( & Global_Arena); - } + void setup(); + void resize( uw new_size ); + void cleanup(); } - sw log_fmt(char const *fmt, ...) - { +inline +sw log_fmt(char const *fmt, ...) +{ #if Build_Debug sw res; va_list va; + va_start(va, fmt); res = zpl_printf_va(fmt, va); va_end(va); + return res; #else @@ -121,6 +115,7 @@ namespace Memory #endif } +inline void fatal(char const *fmt, ...) { zpl_local_persist zpl_thread_local @@ -139,6 +134,6 @@ void fatal(char const *fmt, ...) zpl_printf_err_va( fmt, va); va_end(va); - exit(1); + zpl_exit(1); #endif } diff --git a/project/refactor.cpp b/project/refactor.cpp index 5a79920..a9fdd48 100644 --- a/project/refactor.cpp +++ b/project/refactor.cpp @@ -1,467 +1,238 @@ -#define ZPL_IMPLEMENTATION +#define BLOAT_IMPL #include "bloat.hpp" +#include "IO.cpp" +#include "Spec.cpp" -namespace File +#define Build_Debug 1 + + +void parse_options( int num, char** arguments ) { - zpl_string Source = nullptr; - zpl_string Destination = nullptr; - zpl_file_contents Content {}; + zpl_opts opts; + zpl_opts_init( & opts, g_allocator, "refactor"); + zpl_opts_add( & opts, "num", "num" , "Number of files to refactor" , ZPL_OPTS_INT ); + zpl_opts_add( & opts, "src" , "src" , "File/s to refactor" , ZPL_OPTS_STRING); + zpl_opts_add( & opts, "dst" , "dst" , "File/s post refactor" , ZPL_OPTS_STRING); + zpl_opts_add( & opts, "spec", "spec", "Specification for refactoring", ZPL_OPTS_STRING); - zpl_arena Buffer; - - void cleanup() + if (zpl_opts_compile( & opts, num, arguments)) { - zpl_arena_free( & Buffer ); - } - - void read() - { - zpl_file file_src = {}; - - Content.allocator = g_allocator; - - zpl_file_error error_src = zpl_file_open( & file_src, Source ); - - if ( error_src == ZPL_FILE_ERROR_NONE ) + sw num = 0; + + if ( zpl_opts_has_arg( & opts, "num" ) ) { - zpl_isize fsize = cast(zpl_isize) zpl_file_size( & file_src); - - if ( fsize > 0 ) + num = zpl_opts_integer( & opts, "num", -1 ); + uw global_reserve = num * sizeof(zpl_string) * IO::Path_Size_Largest * 2 + 8; + + if ( global_reserve > zpl_megabytes(1) ) { - zpl_arena_init_from_allocator( & Buffer, zpl_heap(), (fsize + fsize % 64) * 4 ); - - Content.data = zpl_alloc( zpl_arena_allocator( & Buffer), fsize); - Content.size = fsize; - - zpl_file_read_at ( & file_src, Content.data, Content.size, 0); + Memory::resize( global_reserve + zpl_megabytes(2) ); } - - zpl_file_close( & file_src); + + zpl_array_init_reserve( IO::Sources, g_allocator, num ); + zpl_array_init_reserve( IO::Destinations, g_allocator, num ); } - - if ( Content.data == nullptr ) + else { - fatal( "Unable to open source file: %s\n", Source ); - } - } - - void write(zpl_string refactored) - { - if ( refactored == nullptr) - return; - - zpl_file file_dest {}; - zpl_file_error error = zpl_file_create( & file_dest, Destination ); - - if ( error != ZPL_FILE_ERROR_NONE ) - { - fatal( "Unable to open destination file: %s\n", Destination ); - } - - zpl_file_write( & file_dest, refactored, zpl_string_length(refactored) ); - } -} - -namespace Spec -{ - zpl_string File; - - enum Tok - { - Not, - Namespace, - Word, - - Num_Tok - }; - - ct - char const* str_tok( Tok tok ) - { - ct - char const* tok_to_str[ Tok::Num_Tok ] = - { - "not", - "namespace", - "word", - }; - - return tok_to_str[ tok ]; - } - - ct - char strlen_tok( Tok tok ) - { - ct - const u8 tok_to_len[ Tok::Num_Tok ] = - { - 3, - 9, - 4, - }; - - return tok_to_len[ tok ]; - } - - forceinline - bool is_tok( Tok tok, zpl_string str, u32 length ) - { - char const* tok_str = str_tok(tok); - const u8 tok_len = strlen_tok(tok); - - if ( tok_len != length) - return false; - - s32 result = zpl_strncmp( tok_str, str, tok_len ); - - return result == 0; - } - - struct Entry - { - zpl_string Sig = nullptr; // Signature - zpl_string Sub = nullptr; // Substitute - }; - - zpl_arena Buffer {}; - zpl_array(Entry) Word_Ignores; - zpl_array(Entry) Namespace_Ignores; - zpl_array(Entry) Words; - zpl_array(Entry) Namespaces; - - u32 Sig_Smallest = zpl_kilobytes(1); - - forceinline - void find_next_token( zpl_string& token, char*& line, u32& length ) - { - zpl_string_clear( token ); - length = 0; - while ( zpl_char_is_alphanumeric( line[length] ) || line[length] == '_' ) - { - length++; - } - - if ( length == 0 ) - { - fatal("Failed to find valid initial token"); - } - - token = zpl_string_append_length( token, line, length ); - line += length; - } - - void process() - { - char* content; - - zpl_array(char*) lines; - - // Get the contents of the file. - { - zpl_file file {}; - zpl_file_error error = zpl_file_open( & file, File); - - if ( error != ZPL_FILE_ERROR_NONE ) - { - fatal("Could not open the specification file: %s", File); - } - - sw fsize = scast( sw, zpl_file_size( & file ) ); - - if ( fsize <= 0 ) - { - fatal("No content in specificaiton to process"); - } - - zpl_arena_init_from_allocator( & Buffer, zpl_heap(), (fsize + fsize % 64) * 10 + zpl_kilobytes(1) ); - - char* content = rcast( char*, zpl_alloc( zpl_arena_allocator( & Buffer), fsize + 1) ); - - zpl_file_read( & file, content, fsize); - - content[fsize] = 0; - - lines = zpl_str_split_lines( zpl_arena_allocator( & Buffer ), content, false ); - - zpl_file_close( & file ); + num = 1; } - sw left = zpl_array_count( lines ); - - if ( left == 0 ) + if ( zpl_opts_has_arg( & opts, "src" ) ) { - fatal("Spec::process: lines array imporoperly setup"); + zpl_string opt = zpl_opts_string( & opts, "src", "INVALID SRC ARGUMENT" ); + + if ( num == 1 ) + { + IO::Sources[0] = zpl_string_make_length( g_allocator, opt, zpl_string_length( opt) ); + } + else + { + char buffer[ IO::Path_Size_Largest ]; + + uw left = num; + do + { + char* path = buffer; + sw length = 0; + + do + { + path[length] = *opt; + } + while ( length++, opt++, *opt != ' ' ); + + IO::Sources[num - left] = zpl_string_make_length( g_allocator, path, length ); + + opt++; + } + while ( --left ); + } + } + else + { + fatal( "-source not provided\n" ); } - // Skip the first line as its the version number and we only support __VERSION 1. - left--; - lines++; - - zpl_array_init( Word_Ignores, zpl_arena_allocator( & Buffer)); - zpl_array_init( Namespace_Ignores, zpl_arena_allocator( & Buffer)); - zpl_array_init( Words, zpl_arena_allocator( & Buffer)); - zpl_array_init( Namespaces, zpl_arena_allocator( & Buffer)); - - // Limiting the maximum output of a token to 1 KB - zpl_string token = zpl_string_make_reserve( zpl_arena_allocator( & Buffer), zpl_kilobytes(1)); - - while ( left-- ) + if ( zpl_opts_has_arg( & opts, "dst" ) ) { - char* line = * lines; + zpl_string opt = zpl_opts_string( & opts, "dst", "INVALID DST ARGUMENT" ); - // Ignore line if its a comment - if ( line[0] == '/' && line[1] == '/') + if ( num == 1 ) { - lines++; - continue; + IO::Destinations[0] = zpl_string_make_length( g_allocator, opt, zpl_string_length( opt) ); } - - // Remove indent + else { - while ( zpl_char_is_space( line[0] ) ) - line++; + char buffer[ IO::Path_Size_Largest ]; - if ( line[0] == '\0' ) + uw left = num; + do { - lines++; - continue; - } - } + char* path = buffer; + sw length = 0; - u32 length = 0; - - // Find a valid token - find_next_token( token, line, length ); - - Tok type = Tok::Num_Tok; - bool ignore = false; - Entry entry {}; - - // Will be reguarded as an ignore. - if ( is_tok( Tok::Not, token, length )) - { - ignore = true; - - while ( zpl_char_is_space( line[0] ) ) - line++; - - if ( line[0] == '\0' ) - { - lines++; - continue; - } - - // Find the category token - find_next_token( token, line, length ); - } - - if ( is_tok( Tok::Namespace, token, length ) ) - { - type = Tok::Namespace; - } - else if ( is_tok( Tok::Word, token, length ) ) - { - type = Tok::Word; - } - - // Parse line. - { - // Find first argument - { - - while ( zpl_char_is_space( line[0] ) ) - line++; - - if ( line[0] == '\0' ) + do { - lines++; - continue; - } - } + path[length] = *opt; + } + while ( length++, opt++, *opt != ' ' ); + + IO::Destinations[num - left] = zpl_string_make_length( g_allocator, path, length ); - find_next_token( token, line, length ); - - // First argument is signature. - entry.Sig = zpl_string_make_length( g_allocator, token, length ); - - if ( length < Sig_Smallest ) - Sig_Smallest = length; - - if ( line[0] == '\0' || ignore ) - { - switch ( type ) - { - case Tok::Namespace: - if ( ignore) - zpl_array_append( Namespace_Ignores, entry ); - - else - zpl_array_append( Namespaces, entry ); - break; - - case Tok::Word: - if ( ignore) - { - zpl_array_append( Word_Ignores, entry ); - u32 test = zpl_array_count( Word_Ignores ); - } - - - else - zpl_array_append( Words, entry ); - break; - } - - lines++; - continue; - } - - // Look for second argument indicator - { - bool bSkip = false; - - while ( line[0] != ',' ) - { - if ( line[0] == '\0' ) - { - switch ( type ) - { - case Tok::Namespace: - zpl_array_append( Namespaces, entry ); - break; - - case Tok::Word: - zpl_array_append( Words, entry ); - break; - } - - bSkip = true; - break; - } - - line++; - } - - if ( bSkip ) - { - lines++; - continue; - } - } - - // Eat the argument delimiter. - line++; - - // Remove spacing - { - bool bSkip = true; - - while ( zpl_char_is_space( line[0] ) ) - line++; - - if ( line[0] == '\0' ) - { - switch ( type ) - { - case Tok::Namespace: - zpl_array_append( Namespaces, entry ); - break; - - case Tok::Word: - zpl_array_append( Words, entry ); - break; - } - - lines++; - continue; - } - } - - find_next_token( token, line, length ); - - // Second argument is substitute. - entry.Sub = zpl_string_make_length( g_allocator, token, length ); - - switch ( type ) - { - case Tok::Namespace: - zpl_array_append( Namespaces, entry ); - lines++; - continue; - - case Tok::Word: - zpl_array_append( Words, entry ); - lines++; - continue; - } + opt++; + } + while ( --left ); } + } - log_fmt("Specification Line: %d is missing valid keyword", zpl_array_count(lines) - left); - lines++; + if ( zpl_opts_has_arg( & opts, "spec" ) ) + { + zpl_string opt = zpl_opts_string( & opts, "spec", "INVALID PATH" ); + + IO::Specification = zpl_string_make( g_allocator, "" ); + IO::Specification = zpl_string_append( IO::Specification, opt ); } } - - void cleanup() + else { - zpl_arena_free( & Buffer ); + fatal( "Failed to parse arguments\n" ); } + + zpl_opts_free( & opts); } -struct Token -{ - u32 Start; - u32 End; - - zpl_string Sig; - zpl_string Sub; -}; +zpl_arena Refactor_Buffer; void refactor() { - sw buffer_size = File::Content.size; - - zpl_array(Token) tokens; - zpl_array_init( tokens, g_allocator); - - char* content = rcast( char*, File::Content.data ); - - zpl_string current = zpl_string_make( g_allocator, ""); - zpl_string preview = zpl_string_make( g_allocator, ""); - - sw left = File::Content.size; - sw line = 0; - - while ( left ) + ct char const* include_sig = "#include \""; + + struct Token { - if ( content[0] == '\n' ) + u32 Start; + u32 End; + + zpl_string Sig; + zpl_string Sub; + }; + + static zpl_array(Token) tokens = nullptr; + static zpl_string current = zpl_string_make( g_allocator, ""); + +#if Build_Debug + static zpl_string preview = zpl_string_make( g_allocator, ""); +#endif + + static bool Done = false; + if (! Done) + { + zpl_array_init( tokens, g_allocator ); + Done = true; + } + else + { + zpl_array_clear( tokens ); + } + + // Prepare data and trackers. + Array_Line src = IO::get_next_source(); + Array_Line lines = src; + + if ( src == nullptr ) + return; + + const sw num_lines = zpl_array_count( lines); + + sw buffer_size = IO::Current_Size; + + sw left = num_lines; + Line line = *lines; + uw pos = 0; + + do + { + Continue_Line: + + // Includes to ignore { - line++; + Spec::Entry* ignore = Spec::Ignore_Words; + sw ignores_left = zpl_array_count( Spec::Ignore_Words); + + do + { + if ( include_sig[0] != line[0] ) + continue; + + u32 sig_length = zpl_string_length( ignore->Sig ); + + current = zpl_string_set( current, include_sig ); + current = zpl_string_append_length( current, line, sig_length ); + current = zpl_string_append_length( current, "\"", 2 ); + // Formats current into: #include "Sig>" + + if ( zpl_string_are_equal( ignore->Sig, current ) ) + { + log_fmt("\nIgnored %-81s line %d", current, num_lines - left ); + + const sw length = zpl_string_length( current ); + + line += length; + pos += length; + + // Force end of line. + while ( line != '\0' ) + { + line++; + pos++; + } + + goto Skip; + } + } + while ( ignore++, --ignores_left ); } // Word Ignores { - Spec::Entry* ignore = Spec::Word_Ignores; - - sw ignores_left = zpl_array_count( Spec::Word_Ignores); + Spec::Entry* ignore = Spec::Ignore_Words; + sw ignores_left = zpl_array_count( Spec::Ignore_Words); do { - if ( ignore->Sig[0] != content[0] ) - { + if ( ignore->Sig[0] != line[0] ) continue; - } zpl_string_clear( current ); u32 sig_length = zpl_string_length( ignore->Sig ); - current = zpl_string_append_length( current, content, sig_length ); + current = zpl_string_append_length( current, line, sig_length ); if ( zpl_string_are_equal( ignore->Sig, current ) ) { - char before = content[-1]; - char after = content[sig_length]; + char before = line[-1]; + char after = line[sig_length]; if ( zpl_char_is_alphanumeric( before ) || before == '_' || zpl_char_is_alphanumeric( after ) || after == '_' ) @@ -469,10 +240,10 @@ void refactor() continue; } - log_fmt("\nIgnored %-81s line %d", current, line ); + log_fmt("\nIgnored %-81s line %d", current, num_lines - left ); - content += sig_length; - left -= sig_length; + line += sig_length; + pos += sig_length; goto Skip; } } @@ -481,26 +252,23 @@ void refactor() // Namespace Ignores { - Spec::Entry* ignore = Spec::Namespace_Ignores; - - sw ignores_left = zpl_array_count( Spec::Namespace_Ignores); + Spec::Entry* ignore = Spec::Ignore_Namespaces; + sw ignores_left = zpl_array_count( Spec::Ignore_Namespaces); do { - if ( ignore->Sig[0] != content[0] ) - { + if ( ignore->Sig[0] != line[0] ) continue; - } zpl_string_clear( current ); u32 sig_length = zpl_string_length( ignore->Sig ); - current = zpl_string_append_length( current, content, sig_length ); + current = zpl_string_append_length( current, line, sig_length ); if ( zpl_string_are_equal( ignore->Sig, current ) ) { u32 length = sig_length; - char* ns_content = content + sig_length; + char* ns_content = line + sig_length; while ( zpl_char_is_alphanumeric( ns_content[0] ) || ns_content[0] == '_' ) { @@ -508,40 +276,93 @@ void refactor() ns_content++; } + #if Build_Debug zpl_string_clear( preview ); - preview = zpl_string_append_length( preview, content, length ); - log_fmt("\nIgnored %-40s %-40s line %d", preview, ignore->Sig, line); + preview = zpl_string_append_length( preview, line, length ); + log_fmt("\nIgnored %-40s %-40s line %d", preview, ignore->Sig, - left); + #endif - content += length; - left -= length; + line += length; + pos += length; goto Skip; } } while ( ignore++, --ignores_left ); } - // Words to match + // Includes to match { - Spec::Entry* word = Spec::Words; + Spec::Entry* include = Spec::Includes; - sw words_left = zpl_array_count ( Spec::Words); + sw includes_left = zpl_array_count ( Spec::Includes); do { - if ( word->Sig[0] != content[0] ) - { + if ( include_sig[0] != line[0] ) continue; + + u32 sig_length = zpl_string_length( include->Sig ); + + current = zpl_string_set( current, include_sig ); + current = zpl_string_append_length( current, line, sig_length ); + current = zpl_string_append_length( current, "\"", 2 ); + // Formats current into: #include "Sig>" + + if ( zpl_string_are_equal( include->Sig, current ) ) + { + Token entry {}; + + const sw length = zpl_string_length( current ); + + entry.Start = pos; + entry.End = pos + length; + entry.Sig = include->Sig; + + if ( include->Sub != nullptr ) + { + entry.Sub = include->Sub; + buffer_size += zpl_string_length( entry.Sub) - sig_length; + } + + zpl_array_append( tokens, entry ); + + log_fmt("\nFound %-81s line %d", current, num_lines - left); + + line += length; + pos += length; + + // Force end of line. + while ( line != '\0' ) + { + line++; + pos++; + } + + goto Skip; } + } + while ( include++, --includes_left ); + } + + // Words to match + { + Spec::Entry* word = Spec::Words; + sw words_left = zpl_array_count ( Spec::Words); + + do + { + if ( word->Sig[0] != line[0] ) + continue; zpl_string_clear( current ); sw sig_length = zpl_string_length( word->Sig); - current = zpl_string_append_length( current, content, sig_length ); + current = zpl_string_append_length( current, line, sig_length ); if ( zpl_string_are_equal( word->Sig, current ) ) { - char before = content[-1]; - char after = content[sig_length]; + char before = line[-1]; + char after = line[sig_length]; if ( zpl_char_is_alphanumeric( before ) || before == '_' || zpl_char_is_alphanumeric( after ) || after == '_' ) @@ -551,8 +372,8 @@ void refactor() Token entry {}; - entry.Start = File::Content.size - left; - entry.End = entry.Start + sig_length; + entry.Start = pos; + entry.End = pos + sig_length; entry.Sig = word->Sig; if ( word->Sub != nullptr ) @@ -563,10 +384,10 @@ void refactor() zpl_array_append( tokens, entry ); - log_fmt("\nFound %-81s line %d", current, line); + log_fmt("\nFound %-81s line %d", current, num_lines - left); - content += sig_length; - left -= sig_length; + line += sig_length; + pos += sig_length; goto Skip; } } @@ -581,20 +402,18 @@ void refactor() do { - if ( nspace->Sig[0] != content[0] ) - { + if ( nspace->Sig[0] != line[0] ) continue; - } zpl_string_clear( current ); u32 sig_length = zpl_string_length( nspace->Sig ); - current = zpl_string_append_length( current, content, sig_length ); + current = zpl_string_append_length( current, line, sig_length ); if ( zpl_string_are_equal( nspace->Sig, current ) ) { u32 length = sig_length; - char* ns_content = content + sig_length; + char* ns_content = line + sig_length; while ( zpl_char_is_alphanumeric( ns_content[0] ) || ns_content[0] == '_' ) { @@ -604,8 +423,8 @@ void refactor() Token entry {}; - entry.Start = File::Content.size - left; - entry.End = entry.Start + length; + entry.Start = pos; + entry.End = pos + length; entry.Sig = nspace->Sig; buffer_size += sig_length; @@ -618,53 +437,46 @@ void refactor() zpl_array_append( tokens, entry ); + #if Build_Debug zpl_string_clear( preview ); - preview = zpl_string_append_length( preview, content, length); - log_fmt("\nFound %-40s %-40s line %d", preview, nspace->Sig, line); + preview = zpl_string_append_length( preview, line, length); + log_fmt("\nFound %-40s %-40s line %d", preview, nspace->Sig, num_lines - left); + #endif - content += length; - left -= length; + line += length; + pos += length; + goto Skip; } } while ( nspace++, --nspaces_left ); } - content++; - left--; - Skip: - continue; - } + if ( line != '\0' ) + goto Continue_Line; + } + while ( lines++, line = *lines, left ); + + // Prep data for building the content + left = IO::Current_Size; + + char* content = IO::Current_Content; - left = zpl_array_count( tokens); - content = rcast( char*, File::Content.data); - // Generate the refactored file content. - zpl_arena buffer; - zpl_string refactored = nullptr; + static zpl_string + refactored = zpl_string_make_reserve( zpl_arena_allocator( & Refactor_Buffer ), buffer_size ); { - Token* entry = tokens; + Token* entry = tokens; + sw previous_end = 0; - if ( entry == nullptr) - return; - - zpl_arena_init_from_allocator( & buffer, zpl_heap(), buffer_size * 2 ); - - zpl_string - new_string = zpl_string_make_reserve( zpl_arena_allocator( & buffer), zpl_kilobytes(1) ); - refactored = zpl_string_make_reserve( zpl_arena_allocator( & buffer), buffer_size ); - - sw previous_end = 0; - - while ( left-- ) + do { sw segment_length = entry->Start - previous_end; - - sw sig_length = zpl_string_length( entry->Sig ); + sw sig_length = zpl_string_length( entry->Sig ); // Append between tokens - refactored = zpl_string_append_length( refactored, content, segment_length ); - content += segment_length + sig_length; + refactored = zpl_string_append_length( refactored, line, segment_length ); + line += segment_length + sig_length; segment_length = entry->End - entry->Start - sig_length; @@ -672,97 +484,48 @@ void refactor() if ( entry->Sub ) refactored = zpl_string_append( refactored, entry->Sub ); - refactored = zpl_string_append_length( refactored, content, segment_length ); - content += segment_length; + refactored = zpl_string_append_length( refactored, line, segment_length ); + line += segment_length; previous_end = entry->End; entry++; } + while ( --left ); entry--; - if ( entry->End < File::Content.size ) + if ( entry->End < IO::Current_Size ) { - refactored = zpl_string_append_length( refactored, content, File::Content.size - entry->End ); + refactored = zpl_string_append_length( refactored, line, IO::Current_Size - entry->End ); } } - - // Write refactored content to destination. - File::write( refactored ); - zpl_arena_free( & buffer ); + IO::write( refactored ); } - -inline -void parse_options( int num, char** arguments ) -{ - zpl_opts opts; - zpl_opts_init( & opts, g_allocator, "refactor"); - zpl_opts_add( & opts, "src" , "src" , "File to refactor" , ZPL_OPTS_STRING); - zpl_opts_add( & opts, "dst" , "dst" , "File post refactor" , ZPL_OPTS_STRING); - zpl_opts_add( & opts, "spec", "spec", "Specification for refactoring", ZPL_OPTS_STRING); - - if (zpl_opts_compile( & opts, num, arguments)) - { - if ( zpl_opts_has_arg( & opts, "src" ) ) - { - zpl_string opt = zpl_opts_string( & opts, "src", "INVALID PATH" ); - - File::Source = zpl_string_make( g_allocator, "" ); - File::Source = zpl_string_append( File::Source, opt ); - } - else - { - fatal( "-source not provided\n" ); - } - - if ( zpl_opts_has_arg( & opts, "dst" ) ) - { - zpl_string opt = zpl_opts_string( & opts, "dst", "INVALID PATH" ); - - File::Destination = zpl_string_make( g_allocator, "" ); - File::Destination = zpl_string_append( File::Destination, opt ); - } - else if ( File::Source ) - { - File::Destination = zpl_string_make( g_allocator, "" ); - File::Destination = zpl_string_append( File::Destination, File::Source ); - } - - if ( zpl_opts_has_arg( & opts, "spec" ) ) - { - zpl_string opt = zpl_opts_string( & opts, "spec", "INVALID PATH" ); - - Spec::File = zpl_string_make( g_allocator, "" ); - Spec::File = zpl_string_append( Spec::File, opt ); - } - } - else - { - fatal( "Failed to parse arguments\n" ); - } - - zpl_opts_free( & opts); -} - -int main( int num, char** arguments) +int main( int num, char** arguments ) { Memory::setup(); + + parse_options( num, arguments); - parse_options( num, arguments ); + IO::prepare(); - if ( Spec::File ) - Spec::process(); + // Just reserving more than we'll ever problably need. + zpl_arena_init_from_allocator( & Refactor_Buffer, zpl_heap(), IO::Largest_Src_Size * 4 + 8); - File::read(); + Spec::parse(); - refactor(); + sw left = zpl_array_count( IO::Sources ); + do + { + refactor(); + } + while ( --left ); - log_fmt("\n"); - zpl_printf("Refacotred: %s using %s\n", File::Source, Spec::File); + zpl_arena_free( & Refactor_Buffer ); Spec:: cleanup(); - File:: cleanup(); + IO:: cleanup(); Memory::cleanup(); } diff --git a/refactor.10x b/refactor.10x new file mode 100644 index 0000000..fc30d17 --- /dev/null +++ b/refactor.10x @@ -0,0 +1,63 @@ + + + + *.*, + *.obj,*.lib,*.pch,*.dll,*.pdb,.vs,Debug,Release,x64,obj,*.user,Intermediate, + true + true + true + false + false + + + + + + + + + + + false + + Debug + Release + + + x64 + + + C:\Program Files\Microsoft Visual Studio\2022\Professional\VC\Tools\MSVC\14.35.32215\include + C:\Program Files\Microsoft Visual Studio\2022\Professional\VC\Tools\MSVC\14.35.32215\ATLMFC\include + C:\Program Files\Microsoft Visual Studio\2022\Professional\VC\Auxiliary\VS\include + C:\Program Files (x86)\Windows Kits\10\include\10.0.22621.0\ucrt + C:\Program Files (x86)\Windows Kits\10\\include\10.0.22621.0\\um + C:\Program Files (x86)\Windows Kits\10\\include\10.0.22621.0\\shared + C:\Program Files (x86)\Windows Kits\10\\include\10.0.22621.0\\winrt + C:\Program Files (x86)\Windows Kits\10\\include\10.0.22621.0\\cppwinrt + C:\Program Files (x86)\Windows Kits\NETFXSDK\4.8\include\um + .\thirdparty + + + ZPL_IMPLEMENTATION + + + + Debug:x64 + + + C:\projects\refactor\thirdparty + + + + Debug + + + + x64 + + + + + + From 7e120ae5e9e144b4cd605e684c57356b83745609 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Fri, 17 Mar 2023 02:19:32 -0400 Subject: [PATCH 2/9] fixing grammatical mistakes --- Readme.md | 7 ++++--- project/{_Docs.md => Readme.md} | 8 ++++---- 2 files changed, 8 insertions(+), 7 deletions(-) rename project/{_Docs.md => Readme.md} (87%) diff --git a/Readme.md b/Readme.md index f592b90..2235791 100644 --- a/Readme.md +++ b/Readme.md @@ -15,7 +15,7 @@ Syntax : * `include` Preprocessor include related identifiers. * `word` Fixed sized identifier. * `namespace` Variable sized identifiers, mainly intended to redefine c-namespace of an identifier. -* `,` is used to delimit arguments to word or namespace. +* `,` is used to delimit arguments (if doing a find and replace). * `L-Value` is the signature to modify. * `R-Value` is the substitute ( only available if rule does not use `not` keyword ) @@ -25,18 +25,19 @@ However, the rest of the categorical keywords (word, namespace), can really be u There is no semantic awareness this is truely just a simple find and replace, but with some filters specifiable, and words/namespaces only being restricted to the rules for C/C++ identifiers (alphanumeric or underscores only) -The main benefit for using this over other stuff is its faster and more ergonomic for large refactors on libraries that +The main benefit for using this over alts is its problably more ergonomic and performant for large refactors on libraries that you may want to have automated in a script. There are other programs more robust for doing that sort of thing but I was not able to find something this simple. **Note** + * Building for debug provides some nice output with context on a per-line basis. * Release will only show errors for asserts (that will kill the refactor early). * If the refactor crashes, the files previously written to will retain their changes. Make sure to have the code backed up on a VCS or in some other way. * This was compiled using meson with ninja and clang on windows 11. The ZPL library used however should work fine on the other major os platforms and compiler venders. -* The scripts used for building and otherwise are in the scripts directory and are all in powershell (with exception to the meson.build). Techncially there should be a powershell package available on other platorms but worst case it should be pretty easily to port these scripts to w/e shell script you'd perfer. +* The scripts used for building and otherwise are in the scripts directory and are all in powershell (with exception to the meson.build). Techncially there should be a powershell package available on other platorms but worst case it should be pretty easy to port these scripts to w/e shell script you'd perfer. TODO: * Possibly come up with a better name. diff --git a/project/_Docs.md b/project/Readme.md similarity index 87% rename from project/_Docs.md rename to project/Readme.md index dc7194a..81cd490 100644 --- a/project/_Docs.md +++ b/project/Readme.md @@ -11,8 +11,8 @@ The files are setup to compile as one unit. As such the source files for Bloat, Bloat contains some aliasing of some C++ keywords and does not use the standard library. Instead a library called ZPL is used (Single header replacement). -The program has pretty much no optimizations made to it, its just regular loops with no threading. -Just tried to keep the memory at reasonable size of what it does. +The program has pretty much no optimizations made to it, its just regular loops with no threading. +Just tried to keep the memory at a reasonable size for what it does. The program execution is pretty much outlined quite clearly in `int main()`. @@ -27,7 +27,7 @@ The program execution is pretty much outlined quite clearly in `int main()`. `*` This technically can be skipped on windows, may be worth doing to reduce latency of process shutdown. -There are constraints of specific sizes of variables; +There are constraints for specific variables; * `Path_Size_Largest` : Longest path size is set to 1 KB of characters. * `Token_Max_Length` : Set to 1 KB characters as well. @@ -36,6 +36,6 @@ There are constraints of specific sizes of variables; The `Path_Size_Largest` and `Token_Max_Length` are compile-time constraints that the runtime will not have a fallback for, if 1 KB is not enough it will need to be changed for your use case. -`Array_Reserve_Num` is used to dictate the assumed amount of tokens will be held in total for any of spec's arrays holding ignores and refactor entries. If any of the array's exceed 4 KB they will grow trigigng a resize which will bog down the speed of the refactor. Adjust if you think you can increase or lower for use case. +`Array_Reserve_Num` is used to dictate the assumed amount of tokens will be held in total for any of spec's arrays holding ignores and refactor entries. If any of the array's exceed 4 KB they will grow triggering a resize which will bog down the speed of the refactor. Adjust if you think you can increase or lower for use case. Initial Global arena size is a finicy thing, its most likely going to be custom allocator at one point so that it can handle growing properly, right now its just grows if the amount of memory file paths will need for sources is greater than 1 MB. From 97967e56d93ab1a6ecfbd8a7f683b407567623f2 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Fri, 17 Mar 2023 18:12:20 -0400 Subject: [PATCH 3/9] WIP - Got it somewhat working, still have issues with multiple files. --- .vscode/launch.json | 27 +- Readme.md | 8 +- Test/bloat.refactored.hpp | 137 ------- Test/project.refactor | 0 Test/refactor.refactored.cpp | 770 ----------------------------------- Test/zpl.refactor | 40 +- project/Bloat.cpp | 132 +++++- project/IO.cpp | 37 +- project/IO.hpp | 4 +- project/Readme.md | 2 +- project/Spec.cpp | 2 - project/bloat.hpp | 6 +- project/refactor.cpp | 217 +++++----- scripts/build.ps1 | 59 ++- scripts/get_sources.ps1 | 4 +- 15 files changed, 356 insertions(+), 1089 deletions(-) delete mode 100644 Test/bloat.refactored.hpp create mode 100644 Test/project.refactor delete mode 100644 Test/refactor.refactored.cpp diff --git a/.vscode/launch.json b/.vscode/launch.json index 6d7d5cb..4e50ea8 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -13,13 +13,34 @@ "-src=./thirdparty/zpl.h", "-dst=./Test/zpl.refactored.h", - "-spec=./Test/zpl.h.refactor" + "-spec=./Test/zpl.refactor" ], "stopAtEntry": false, "cwd": "${workspaceRoot}", "environment": [], "console": "integratedTerminal" - } + }, + // { + // "name": "Refactor ZPL files", + // "type": "cppvsdbg", + // "request": "launch", + // "program": "${workspaceFolder}/build/refactor.exe", + // "args": [ + // "-num=2", + + // "-src=./thirdparty/zpl.h", + // "./thirdparty/file.h", + + // "-dst=./Test/zpl.refactored.h", + // "./thirdparty/file.refactored.h", + + // "-spec=./Test/zpl.refactor" + // ], + // "stopAtEntry": false, + // "cwd": "${workspaceRoot}", + // "environment": [], + // "console": "integratedTerminal" + // }, { "name": "Refactor refactor.c", "type": "cppvsdbg", @@ -29,7 +50,7 @@ "-source=./refactor.cpp", "-destination=./Test/refactor.cpp", - "-specification=./Test/zpl.h.refactor" + "-specification=./Test/zpl.refactor" ], "stopAtEntry": false, "cwd": "${workspaceRoot}", diff --git a/Readme.md b/Readme.md index 2235791..b6baadb 100644 --- a/Readme.md +++ b/Readme.md @@ -2,14 +2,14 @@ Refactor c/c++ files (and problably others) with ease. -Parameters : +## Parameters : * `-num` : Used if more than one source file is provided (if used, number of destination files provided MUST MATCH). * `-src` : Source file to refactor * `-dst` : Destination file after the refactor (omit to use the same as source) * `-spec` : Specification containing rules to use for the refactor. -Syntax : +## Syntax : * `not` Omit word or namespace. * `include` Preprocessor include related identifiers. @@ -30,7 +30,7 @@ you may want to have automated in a script. There are other programs more robust for doing that sort of thing but I was not able to find something this simple. -**Note** +### Note * Building for debug provides some nice output with context on a per-line basis. * Release will only show errors for asserts (that will kill the refactor early). @@ -40,6 +40,8 @@ Make sure to have the code backed up on a VCS or in some other way. * The scripts used for building and otherwise are in the scripts directory and are all in powershell (with exception to the meson.build). Techncially there should be a powershell package available on other platorms but worst case it should be pretty easy to port these scripts to w/e shell script you'd perfer. TODO: + * Possibly come up with a better name. * Test to see how much needs to be ported for other platforms (if at all) +* Setup as api. * Provide binaries in the release page for github. (debug and release builds) diff --git a/Test/bloat.refactored.hpp b/Test/bloat.refactored.hpp deleted file mode 100644 index ec79e74..0000000 --- a/Test/bloat.refactored.hpp +++ /dev/null @@ -1,137 +0,0 @@ -/* - BLOAT. - - ZPL requires ZPL_IMPLEMENTATION whereever this library is included. - - This file assumes it will be included in one compilation unit. -*/ - -#pragma once - -#if __clang__ -# pragma clang diagnostic ignored "-Wunused-const-variable" -# pragma clang diagnostic ignored "-Wswitch" -# pragma clang diagnostic ignored "-Wunused-variable" -#endif - - - -#pragma region ZPL INCLUDE -#if __clang__ -# pragma clang diagnostic push -# pragma clang diagnostic ignored "-Wmissing-braces" -# pragma clang diagnostic ignored "-Wbraced-scalar-init" -#endif - -// # define ZPL_HEAP_ANALYSIS -# define ZPL_NO_MATH_H -# define ZPL_DISABLE_C_DECLS -# define ZPL_WRAP_IN_NAMESPACE -# define ZPL_CUSTOM_MODULES -# define ZPL_MODULE_ESSENTIALS -# define ZPL_MODULE_CORE -# define ZPL_MODULE_TIMER -// # define ZPL_MODULE_HASHING -// # define ZPL_MODULE_REGEX -// # define ZPL_MODULE_EVENT -// # define ZPL_MODULE_DLL -# define ZPL_MODULE_OPTS -// # define ZPL_MODULE_PROCESS -// # define ZPL_MODULE_MATH -// # define ZPL_MODULE_THREADING -// # define ZPL_MODULE_JOBS -// # define ZPL_MODULE_PARSER -// extern "C" { -#include "zpl.refactored.h" -// } - -#if __clang__ -# pragma clang diagnostic pop -#endif -#pragma endregion ZPL INCLUDE - - - -#define bit( Value_ ) ( 1 << Value_ ) -#define bitfield_is_equal( Field_, Mask_ ) ( ( Mask_ & Field_ ) == Mask_ ) -#define ct constexpr -#define gen( ... ) template< __VA_ARGS__ > -#define forceinline ZPL_ALWAYS_INLINE -#define print_nl( _) zpl_printf("\n") -#define cast( Type_, Value_ ) ( ( Type_ ), ( Value_ ) ) -#define scast( Type_, Value_ ) static_cast< Type_ >( Value_ ) -#define rcast( Type_, Value_ ) reinterpret_cast< Type_ >( Value_ ) -#define pcast( Type_, Value_ ) ( * (Type_*)( & Value_ ) ) - -#define do_once() \ -do \ -{ \ - static \ - bool Done = true; \ - if ( Done ) \ - return; \ - Done = false; \ -} \ -while(0) \ - - -ct char const* Msg_Invalid_Value = "INVALID VALUE PROVIDED"; - - -namespace Memory -{ - zpl::arena Global_Arena {}; - #define g_allocator arena_allocator( & Memory::Global_Arena) - - void setup() - { - arena_init_from_allocator( & Global_Arena, heap(), megabytes(1) ); - - if ( Global_Arena.total_size == 0 ) - { - zpl::assert_crash( "Failed to reserve memory for Tests:: Global_Arena" ); - } - } - - void cleanup() - { - arena_free( & Global_Arena); - } -} - -zpl::sw log_fmt(char const *fmt, ...) -{ -#if Build_Debug - zpl::sw res; - va_list va; - va_start(va, fmt); - res = zpl::printf_va(fmt, va); - va_end(va); - return res; - -#else - return 0; -#endif -} - -void fatal(char const *fmt, ...) -{ - local_persist thread_local - char buf[ZPL_PRINTF_MAXLEN] = { 0 }; - - va_list va; - -#if Build_Debug - va_start(va, fmt); - zpl::snprintf_va(buf, ZPL_PRINTF_MAXLEN, fmt, va); - va_end(va); - - zpl::assert_crash(buf); -#else - va_start(va, fmt); - zpl::printf_err_va( fmt, va); - va_end(va); - - zpl::exit(1); -#endif -} diff --git a/Test/project.refactor b/Test/project.refactor new file mode 100644 index 0000000..e69de29 diff --git a/Test/refactor.refactored.cpp b/Test/refactor.refactored.cpp deleted file mode 100644 index 65887e0..0000000 --- a/Test/refactor.refactored.cpp +++ /dev/null @@ -1,770 +0,0 @@ -#define ZPL_IMPLEMENTATION -#include "bloat.refactored.hpp" - - -namespace File -{ - using namespace zpl; - - string Source = nullptr; - string Destination = nullptr; - file_contents Content {}; - - arena Buffer; - - void cleanup() - { - arena_free( & Buffer ); - } - - void read() - { - file file_src = {}; - - Content.allocator = g_allocator; - - file_error error_src = file_open( & file_src, Source ); - - if ( error_src == ZPL_FILE_ERROR_NONE ) - { - sw fsize = zpl_cast(sw) file_size( & file_src); - - if ( fsize > 0 ) - { - arena_init_from_allocator( & Buffer, heap(), (fsize + fsize % 64) * 4 ); - - Content.data = alloc( arena_allocator( & Buffer), fsize); - Content.size = fsize; - - file_read_at ( & file_src, Content.data, Content.size, 0); - } - - file_close( & file_src); - } - - if ( Content.data == nullptr ) - { - fatal( "Unable to open source file: %s\n", Source ); - } - } - - void write(string refactored) - { - if ( refactored == nullptr) - return; - - file file_dest {}; - file_error error = file_create( & file_dest, Destination ); - - if ( error != ZPL_FILE_ERROR_NONE ) - { - fatal( "Unable to open destination file: %s\n", Destination ); - } - - file_write( & file_dest, refactored, string_length(refactored) ); - } -} - -namespace Spec -{ - using namespace zpl; - - string File; - - enum Tok - { - Not, - Namespace, - Word, - - Num_Tok - }; - - ct - char const* str_tok( Tok tok ) - { - ct - char const* tok_to_str[ Tok::Num_Tok ] = - { - "not", - "namespace", - "word", - }; - - return tok_to_str[ tok ]; - } - - ct - char strlen_tok( Tok tok ) - { - ct - const u8 tok_to_len[ Tok::Num_Tok ] = - { - 3, - 9, - 4, - }; - - return tok_to_len[ tok ]; - } - - forceinline - bool is_tok( Tok tok, string str, u32 length ) - { - char const* tok_str = str_tok(tok); - const u8 tok_len = strlen_tok(tok); - - if ( tok_len != length) - return false; - - s32 result = str_compare( tok_str, str, tok_len ); - - return result == 0; - } - - struct Entry - { - string Sig = nullptr; // Signature - string Sub = nullptr; // Substitute - }; - - arena Buffer {}; - zpl_array(Entry) Word_Ignores; - zpl_array(Entry) Namespace_Ignores; - zpl_array(Entry) Words; - zpl_array(Entry) Namespaces; - - u32 Sig_Smallest = kilobytes(1); - - forceinline - void find_next_token( string& token, char*& line, u32& length ) - { - string_clear( token ); - length = 0; - while ( char_is_alphanumeric( line[length] ) || line[length] == '_' ) - { - length++; - } - - if ( length == 0 ) - { - fatal("Failed to find valid initial token"); - } - - token = string_append_length( token, line, length ); - line += length; - } - - void process() - { - char* content; - - zpl_array(char*) lines; - - // Get the contents of the file. - { - file file {}; - file_error error = file_open( & file, File); - - if ( error != ZPL_FILE_ERROR_NONE ) - { - fatal("Could not open the specification file: %s", File); - } - - sw fsize = scast( sw, file_size( & file ) ); - - if ( fsize <= 0 ) - { - fatal("No content in specificaiton to process"); - } - - arena_init_from_allocator( & Buffer, heap(), (fsize + fsize % 64) * 10 + kilobytes(1) ); - - char* content = rcast( char*, alloc( arena_allocator( & Buffer), fsize + 1) ); - - file_read( & file, content, fsize); - - content[fsize] = 0; - - lines = str_split_lines( arena_allocator( & Buffer ), content, false ); - - file_close( & file ); - } - - sw left = array_count( lines ); - - if ( left == 0 ) - { - fatal("Spec::process: lines array imporoperly setup"); - } - - // Skip the first line as its the version number and we only support __VERSION 1. - left--; - lines++; - - array_init( Word_Ignores, arena_allocator( & Buffer)); - array_init( Namespace_Ignores, arena_allocator( & Buffer)); - array_init( Words, arena_allocator( & Buffer)); - array_init( Namespaces, arena_allocator( & Buffer)); - - // Limiting the maximum output of a token to 1 KB - string token = string_make_reserve( arena_allocator( & Buffer), kilobytes(1)); - - while ( left-- ) - { - char* line = * lines; - - // Ignore line if its a comment - if ( line[0] == '/' && line[1] == '/') - { - lines++; - continue; - } - - // Remove indent - { - while ( char_is_space( line[0] ) ) - line++; - - if ( line[0] == '\0' ) - { - lines++; - continue; - } - } - - u32 length = 0; - - // Find a valid token - find_next_token( token, line, length ); - - Tok type = Tok::Num_Tok; - bool ignore = false; - Entry entry {}; - - // Will be reguarded as an ignore. - if ( is_tok( Tok::Not, token, length )) - { - ignore = true; - - while ( char_is_space( line[0] ) ) - line++; - - if ( line[0] == '\0' ) - { - lines++; - continue; - } - - // Find the category token - find_next_token( token, line, length ); - } - - if ( is_tok( Tok::Namespace, token, length ) ) - { - type = Tok::Namespace; - } - else if ( is_tok( Tok::Word, token, length ) ) - { - type = Tok::Word; - } - - // Parse line. - { - // Find first argument - { - - while ( char_is_space( line[0] ) ) - line++; - - if ( line[0] == '\0' ) - { - lines++; - continue; - } - } - - find_next_token( token, line, length ); - - // First argument is signature. - entry.Sig = string_make_length( g_allocator, token, length ); - - if ( length < Sig_Smallest ) - Sig_Smallest = length; - - if ( line[0] == '\0' || ignore ) - { - switch ( type ) - { - case Tok::Namespace: - if ( ignore) - array_append( Namespace_Ignores, entry ); - - else - array_append( Namespaces, entry ); - break; - - case Tok::Word: - if ( ignore) - { - array_append( Word_Ignores, entry ); - u32 test = array_count( Word_Ignores ); - } - - - else - array_append( Words, entry ); - break; - } - - lines++; - continue; - } - - // Look for second argument indicator - { - bool bSkip = false; - - while ( line[0] != ',' ) - { - if ( line[0] == '\0' ) - { - switch ( type ) - { - case Tok::Namespace: - array_append( Namespaces, entry ); - break; - - case Tok::Word: - array_append( Words, entry ); - break; - } - - bSkip = true; - break; - } - - line++; - } - - if ( bSkip ) - { - lines++; - continue; - } - } - - // Eat the argument delimiter. - line++; - - // Remove spacing - { - bool bSkip = true; - - while ( char_is_space( line[0] ) ) - line++; - - if ( line[0] == '\0' ) - { - switch ( type ) - { - case Tok::Namespace: - array_append( Namespaces, entry ); - break; - - case Tok::Word: - array_append( Words, entry ); - break; - } - - lines++; - continue; - } - } - - find_next_token( token, line, length ); - - // Second argument is substitute. - entry.Sub = string_make_length( g_allocator, token, length ); - - switch ( type ) - { - case Tok::Namespace: - array_append( Namespaces, entry ); - lines++; - continue; - - case Tok::Word: - array_append( Words, entry ); - lines++; - continue; - } - } - - log_fmt("Specification Line: %d is missing valid keyword", array_count(lines) - left); - lines++; - } - } - - void cleanup() - { - arena_free( & Buffer ); - } -} - -using namespace zpl; - -struct Token -{ - u32 Start; - u32 End; - - string Sig; - string Sub; -}; - -void refactor() -{ - sw buffer_size = File::Content.size; - - zpl_array(Token) tokens; - array_init( tokens, g_allocator); - - char* content = rcast( char*, File::Content.data ); - - string current = string_make( g_allocator, ""); - string preview = string_make( g_allocator, ""); - - sw left = File::Content.size; - sw line = 0; - - while ( left ) - { - if ( content[0] == '\n' ) - { - line++; - } - - // Word Ignores - { - Spec::Entry* ignore = Spec::Word_Ignores; - - sw ignores_left = array_count( Spec::Word_Ignores); - - do - { - if ( ignore->Sig[0] != content[0] ) - { - continue; - } - - string_clear( current ); - - u32 sig_length = string_length( ignore->Sig ); - current = string_append_length( current, content, sig_length ); - - if ( string_are_equal( ignore->Sig, current ) ) - { - char before = content[-1]; - char after = content[sig_length]; - - if ( char_is_alphanumeric( before ) || before == '_' - || char_is_alphanumeric( after ) || after == '_' ) - { - continue; - } - - log_fmt("\nIgnored %-81s line %d", current, line ); - - content += sig_length; - left -= sig_length; - goto Skip; - } - } - while ( ignore++, --ignores_left ); - } - - // Namespace Ignores - { - Spec::Entry* ignore = Spec::Namespace_Ignores; - - sw ignores_left = array_count( Spec::Namespace_Ignores); - - do - { - if ( ignore->Sig[0] != content[0] ) - { - continue; - } - - string_clear( current ); - - u32 sig_length = string_length( ignore->Sig ); - current = string_append_length( current, content, sig_length ); - - if ( string_are_equal( ignore->Sig, current ) ) - { - u32 length = sig_length; - char* ns_content = content + sig_length; - - while ( char_is_alphanumeric( ns_content[0] ) || ns_content[0] == '_' ) - { - length++; - ns_content++; - } - - string_clear( preview ); - preview = string_append_length( preview, content, length ); - log_fmt("\nIgnored %-40s %-40s line %d", preview, ignore->Sig, line); - - content += length; - left -= length; - goto Skip; - } - } - while ( ignore++, --ignores_left ); - } - - // Words to match - { - Spec::Entry* word = Spec::Words; - - sw words_left = array_count ( Spec::Words); - - do - { - if ( word->Sig[0] != content[0] ) - { - continue; - } - - string_clear( current ); - - sw sig_length = string_length( word->Sig); - current = string_append_length( current, content, sig_length ); - - if ( string_are_equal( word->Sig, current ) ) - { - char before = content[-1]; - char after = content[sig_length]; - - if ( char_is_alphanumeric( before ) || before == '_' - || char_is_alphanumeric( after ) || after == '_' ) - { - continue; - } - - Token entry {}; - - entry.Start = File::Content.size - left; - entry.End = entry.Start + sig_length; - entry.Sig = word->Sig; - - if ( word->Sub != nullptr ) - { - entry.Sub = word->Sub; - buffer_size += string_length( entry.Sub) - sig_length; - } - - array_append( tokens, entry ); - - log_fmt("\nFound %-81s line %d", current, line); - - content += sig_length; - left -= sig_length; - goto Skip; - } - } - while ( word++, --words_left ); - } - - // Namespaces to match - { - Spec::Entry* nspace = Spec::Namespaces; - - sw nspaces_left = array_count( Spec::Namespaces); - - do - { - if ( nspace->Sig[0] != content[0] ) - { - continue; - } - - string_clear( current ); - - u32 sig_length = string_length( nspace->Sig ); - current = string_append_length( current, content, sig_length ); - - if ( string_are_equal( nspace->Sig, current ) ) - { - u32 length = sig_length; - char* ns_content = content + sig_length; - - while ( char_is_alphanumeric( ns_content[0] ) || ns_content[0] == '_' ) - { - length++; - ns_content++; - } - - Token entry {}; - - entry.Start = File::Content.size - left; - entry.End = entry.Start + length; - entry.Sig = nspace->Sig; - - buffer_size += sig_length; - - if ( nspace->Sub != nullptr ) - { - entry.Sub = nspace->Sub; - buffer_size += string_length( entry.Sub ) - length; - } - - array_append( tokens, entry ); - - string_clear( preview ); - preview = string_append_length( preview, content, length); - log_fmt("\nFound %-40s %-40s line %d", preview, nspace->Sig, line); - - content += length; - left -= length; - } - } - while ( nspace++, --nspaces_left ); - } - - content++; - left--; - - Skip: - continue; - } - - left = array_count( tokens); - content = rcast( char*, File::Content.data); - - // Generate the refactored file content. - arena buffer; - string refactored = nullptr; - { - Token* entry = tokens; - - if ( entry == nullptr) - return; - - arena_init_from_allocator( & buffer, heap(), buffer_size * 2 ); - - string - new_string = string_make_reserve( arena_allocator( & buffer), kilobytes(1) ); - refactored = string_make_reserve( arena_allocator( & buffer), buffer_size ); - - sw previous_end = 0; - - while ( left-- ) - { - sw segment_length = entry->Start - previous_end; - - sw sig_length = string_length( entry->Sig ); - - // Append between tokens - refactored = string_append_length( refactored, content, segment_length ); - content += segment_length + sig_length; - - segment_length = entry->End - entry->Start - sig_length; - - // Append token - if ( entry->Sub ) - refactored = string_append( refactored, entry->Sub ); - - refactored = string_append_length( refactored, content, segment_length ); - content += segment_length; - - previous_end = entry->End; - entry++; - } - - entry--; - - if ( entry->End < File::Content.size ) - { - refactored = string_append_length( refactored, content, File::Content.size - entry->End ); - } - } - - // Write refactored content to destination. - File::write( refactored ); - - arena_free( & buffer ); -} - - -inline -void parse_options( int num, char** arguments ) -{ - opts opts; - opts_init( & opts, g_allocator, "refactor"); - opts_add( & opts, "source" , "src" , "File to refactor" , ZPL_OPTS_STRING); - opts_add( & opts, "destination" , "dst" , "File post refactor" , ZPL_OPTS_STRING); - opts_add( & opts, "specification", "spec", "Specification for refactoring", ZPL_OPTS_STRING); - - if (opts_compile( & opts, num, arguments)) - { - if ( opts_has_arg( & opts, "src" ) ) - { - string opt = opts_string( & opts, "src", "INVALID PATH" ); - - File::Source = string_make( g_allocator, "" ); - File::Source = string_append( File::Source, opt ); - } - else - { - fatal( "-source not provided\n" ); - } - - if ( opts_has_arg( & opts, "dst" ) ) - { - string opt = opts_string( & opts, "dst", "INVALID PATH" ); - - File::Destination = string_make( g_allocator, "" ); - File::Destination = string_append( File::Destination, opt ); - } - else if ( File::Source ) - { - File::Destination = string_make( g_allocator, "" ); - File::Destination = string_append( File::Destination, File::Source ); - } - - if ( opts_has_arg( & opts, "spec" ) ) - { - string opt = opts_string( & opts, "spec", "INVALID PATH" ); - - Spec::File = string_make( g_allocator, "" ); - Spec::File = string_append( Spec::File, opt ); - } - } - else - { - fatal( "Failed to parse arguments\n" ); - } - - opts_free( & opts); -} - -int main( int num, char** arguments) -{ - Memory::setup(); - - parse_options( num, arguments ); - - if ( Spec::File ) - Spec::process(); - - File::read(); - - refactor(); - - Spec:: cleanup(); - File:: cleanup(); - Memory::cleanup(); -} diff --git a/Test/zpl.refactor b/Test/zpl.refactor index 44a6f20..c5172bb 100644 --- a/Test/zpl.refactor +++ b/Test/zpl.refactor @@ -1,14 +1,17 @@ __VERSION 1 -// not : Ignore -// word : Alphanumeric or underscore -// regex : Unavailable in __VERSION 1. +// not : Ignore +// include : #includes +// word : Alphanumeric or underscore +// namespace : Prefix search and replace (c-namspaces). +// regex : Unavailable in __VERSION 1. // Precedence (highest to lowest): // word, namespace, regex // Header files -not word zpl_hedley +not include zpl_hedley +//not word zpl_hedley // Removes the namespace. namespace zpl_ @@ -16,30 +19,32 @@ namespace zpl_ // Don't expose internals not namespace zpl__ -not word ZPL_IMPLEMENTATION +// Macro exposure +//namespace ZPL_ +//not word ZPL_IMPLEMENTATION word cast, zpl_cast word zpl_strncmp, str_compare word zpl_strcmp, str_compare // Undesired typedefs -word zpl_i8, s8 -word zpl_i16, s16 -word zpl_i32, s32 -word zpl_i64, s64 -word zpl_u8, u8 -word zpl_u16, u16 -word zpl_u32, u32 -word zpl_u64, u64 -word zpl_intptr, sptr +word zpl_i8, s8 +word zpl_i16, s16 +word zpl_i32, s32 +word zpl_i64, s64 +word zpl_u8, u8 +word zpl_u16, u16 +word zpl_u32, u32 +word zpl_u64, u64 +word zpl_intptr, sptr word zpl_uintptr, uptr -word zpl_usize, uw -word zpl_isize, sw +word zpl_usize, uw +word zpl_isize, sw // Undesired exposures. //not word zpl_allocator //not word zpl_arena -not word zpl_array +//not word zpl_array //not word zpl_file //not word zpl_list //not word zpl_pool @@ -47,6 +52,7 @@ not word zpl_array // Conflicts with refactor word arena, a_arena +word array, a_array word alloc, a_allocator word file, a_file word file_size, fsize diff --git a/project/Bloat.cpp b/project/Bloat.cpp index 648d780..208b716 100644 --- a/project/Bloat.cpp +++ b/project/Bloat.cpp @@ -1,5 +1,5 @@ #define BLOAT_IMPL -#include "bloat.hpp" +#include "Bloat.hpp" @@ -35,3 +35,133 @@ namespace Memory zpl_arena_free( & Global_Arena); } } + + +bool opts_custom_add(zpl_opts* opts, zpl_opts_entry *t, char* b) +{ + if (t->type != ZPL_OPTS_STRING) + { + return false; + } + + t->text = zpl_string_append_length(t->text, " ", 1); + t->text = zpl_string_appendc( t->text, b ); + + return true; +} + +b32 opts_custom_compile(zpl_opts *opts, int argc, char **argv) +{ + zpl_b32 had_errors = false; + + for (int i = 1; i < argc; ++i) + { + char* arg = argv[i]; + + if (*arg) + { + arg = cast(char*)zpl_str_trim(arg, false); + + if (*arg == '-') + { + zpl_opts_entry* entry = 0; + zpl_b32 checkln = false; + if ( *(arg + 1) == '-') + { + checkln = true; + ++arg; + } + + char *b = arg + 1, *e = b; + + while (zpl_char_is_alphanumeric(*e) || *e == '-' || *e == '_') { + ++e; + } + + entry = zpl__opts_find(opts, b, (e - b), checkln); + + if (entry) + { + char *ob = b; + b = e; + + /**/ + if (*e == '=') + { + if (entry->type == ZPL_OPTS_FLAG) + { + *e = '\0'; + zpl__opts_push_error(opts, ob, ZPL_OPTS_ERR_EXTRA_VALUE); + had_errors = true; + + continue; + } + + b = e = e + 1; + } + else if (*e == '\0') + { + char *sp = argv[i+1]; + + if (sp && *sp != '-' && (zpl_array_count(opts->positioned) < 1 || entry->type != ZPL_OPTS_FLAG)) + { + if (entry->type == ZPL_OPTS_FLAG) + { + zpl__opts_push_error(opts, b, ZPL_OPTS_ERR_EXTRA_VALUE); + had_errors = true; + + continue; + } + + arg = sp; + b = e = sp; + ++i; + } + else + { + if (entry->type != ZPL_OPTS_FLAG) + { + zpl__opts_push_error(opts, ob, ZPL_OPTS_ERR_MISSING_VALUE); + had_errors = true; + continue; + } + + entry->met = true; + + continue; + } + } + + e = cast(char *)zpl_str_control_skip(e, '\0'); + zpl__opts_set_value(opts, entry, b); + + if ( (i + 1) < argc ) + { + for ( b = argv[i + 1]; i < argc && b[0] != '-'; i++, b = argv[i + 1] ) + { + opts_custom_add(opts, entry, b ); + } + } + } + else + { + zpl__opts_push_error(opts, b, ZPL_OPTS_ERR_OPTION); + had_errors = true; + } + } + else if (zpl_array_count(opts->positioned)) + { + zpl_opts_entry *l = zpl_array_back(opts->positioned); + zpl_array_pop(opts->positioned); + zpl__opts_set_value(opts, l, arg); + } + else + { + zpl__opts_push_error(opts, arg, ZPL_OPTS_ERR_VALUE); + had_errors = true; + } + } + } + + return !had_errors; +} diff --git a/project/IO.cpp b/project/IO.cpp index c0dc399..f129dde 100644 --- a/project/IO.cpp +++ b/project/IO.cpp @@ -13,23 +13,12 @@ namespace IO // Current source and destination index. // Used to keep track of which file get_next_source or write refer to. - uw Current = 0; + sw Current = -1; char* Current_Content = nullptr; uw Current_Size = 0; uw Largest_Src_Size = 0; - /* - Will persist throughout loading different file content. - Should hold a bit more than the largest source file's content, - As an array of lines. - */ zpl_arena MemPerist; - - /* - Temporary memory held while procesisng files to get their content. - zpl_files are stored here - */ - // zpl_arena MemTransient; } using namespace StaticData; @@ -48,7 +37,7 @@ namespace IO if ( error != ZPL_FILE_ERROR_NONE ) { - fatal("Could not open source file: %s", *path ); + fatal("IO::Prepare - Could not open source file: %s", *path ); } const sw fsize = zpl_file_size( & src ); @@ -60,18 +49,16 @@ namespace IO zpl_file_close( & src ); } - while ( --left ); + while ( left--, left > 1 ); uw persist_size = ZPL_ARRAY_GROW_FORMULA( Largest_Src_Size ); zpl_arena_init_from_allocator( & MemPerist, zpl_heap(), persist_size ); - // zpl_arena_init_from_allocator( & MemTransient, zpl_heap(), Largest_Src_Size ); } void cleanup() { zpl_arena_free( & MemPerist ); - // zpl_arena_free( & MemTransient ); } Array_Line get_specification() @@ -102,25 +89,24 @@ namespace IO return lines; } - Array_Line get_next_source() + char* get_next_source() { - // zpl_memset( MemTransient.physical_start, 0, MemTransient.total_allocated); - // MemTransient.total_allocated = 0; - // MemTransient.temp_count = 0; - zpl_memset( MemPerist.physical_start, 0, MemPerist.total_allocated); MemPerist.total_allocated = 0; MemPerist.temp_count = 0; + Current++; + zpl_file file {}; - zpl_file_error error = zpl_file_open( & file, Specification); + zpl_file_error error = zpl_file_open( & file, Sources[Current]); if ( error != ZPL_FILE_ERROR_NONE ) { - fatal("Could not open the source file: %s", Sources[Current]); + fatal("IO::get_next_source - Could not open the source file: %s", Sources[Current]); } - Current_Size = scast( sw, zpl_file_size( & file ) ); + auto size = zpl_file_size( & file ); + Current_Size = scast( sw, size ); if ( Current_Size <= 0 ) return nullptr; @@ -133,8 +119,7 @@ namespace IO Current_Content[Current_Size] = 0; Current_Size++; - Array_Line lines = zpl_str_split_lines( zpl_arena_allocator( & MemPerist), Current_Content, ' ' ); - return lines; + return Current_Content; } void write( zpl_string refacotred ) diff --git a/project/IO.hpp b/project/IO.hpp index 051a6a3..9ef038e 100644 --- a/project/IO.hpp +++ b/project/IO.hpp @@ -1,6 +1,6 @@ #pragma once -#include "bloat.hpp" +#include "Bloat.hpp" namespace IO @@ -18,7 +18,7 @@ namespace IO Array_Line get_specification(); // Provides the content of the next source, broken up as a series of lines. - Array_Line get_next_source(); + char* get_next_source(); // Writes the refactored content ot the current corresponding destination. void write( zpl_string refactored ); diff --git a/project/Readme.md b/project/Readme.md index 81cd490..4fe853c 100644 --- a/project/Readme.md +++ b/project/Readme.md @@ -38,4 +38,4 @@ The `Path_Size_Largest` and `Token_Max_Length` are compile-time constraints that `Array_Reserve_Num` is used to dictate the assumed amount of tokens will be held in total for any of spec's arrays holding ignores and refactor entries. If any of the array's exceed 4 KB they will grow triggering a resize which will bog down the speed of the refactor. Adjust if you think you can increase or lower for use case. -Initial Global arena size is a finicy thing, its most likely going to be custom allocator at one point so that it can handle growing properly, right now its just grows if the amount of memory file paths will need for sources is greater than 1 MB. +Initial Global arena size is a finicy thing, its most likely going to be custom allocator at one point so that it can handle growing properly, right now its just grows if the amount of memory file paths will need for sources is greater than 1 MB. diff --git a/project/Spec.cpp b/project/Spec.cpp index 6ef1d98..8fb61ca 100644 --- a/project/Spec.cpp +++ b/project/Spec.cpp @@ -96,8 +96,6 @@ namespace Spec left--; lines++; - char token[ Token_Max_Length ]; - do { char* line = * lines; diff --git a/project/bloat.hpp b/project/bloat.hpp index 1cfd188..94c29b8 100644 --- a/project/bloat.hpp +++ b/project/bloat.hpp @@ -5,7 +5,7 @@ #pragma once #ifdef BLOAT_IMPL -# define ZPL_IMPLEMEntATION +# define ZPL_IMPLEMENTATION #endif #if __clang__ @@ -70,6 +70,7 @@ do \ while(0) \ +using b32 = zpl_b32; using s8 = zpl_i8; using s32 = zpl_i32; using s64 = zpl_i64; @@ -97,6 +98,9 @@ namespace Memory void cleanup(); } +// Had to be made to support multiple sub-arguments per "opt" argument. +b32 opts_custom_compile(zpl_opts *opts, int argc, char **argv); + inline sw log_fmt(char const *fmt, ...) { diff --git a/project/refactor.cpp b/project/refactor.cpp index a9fdd48..1318ea9 100644 --- a/project/refactor.cpp +++ b/project/refactor.cpp @@ -1,5 +1,4 @@ -#define BLOAT_IMPL -#include "bloat.hpp" +#include "Bloat.cpp" #include "IO.cpp" #include "Spec.cpp" @@ -16,7 +15,7 @@ void parse_options( int num, char** arguments ) zpl_opts_add( & opts, "dst" , "dst" , "File/s post refactor" , ZPL_OPTS_STRING); zpl_opts_add( & opts, "spec", "spec", "Specification for refactoring", ZPL_OPTS_STRING); - if (zpl_opts_compile( & opts, num, arguments)) + if (opts_custom_compile( & opts, num, arguments)) { sw num = 0; @@ -36,7 +35,12 @@ void parse_options( int num, char** arguments ) else { num = 1; + + zpl_array_init_reserve( IO::Sources, g_allocator, 1 ); + zpl_array_init_reserve( IO::Destinations, g_allocator, 1 ); } + + zpl_printf("NUM IS: %d", num); if ( zpl_opts_has_arg( & opts, "src" ) ) { @@ -44,7 +48,8 @@ void parse_options( int num, char** arguments ) if ( num == 1 ) { - IO::Sources[0] = zpl_string_make_length( g_allocator, opt, zpl_string_length( opt) ); + zpl_string path = zpl_string_make_length( g_allocator, opt, zpl_string_length( opt )); + zpl_array_append( IO::Sources, path ); } else { @@ -60,9 +65,10 @@ void parse_options( int num, char** arguments ) { path[length] = *opt; } - while ( length++, opt++, *opt != ' ' ); + while ( length++, opt++, *opt != ' ' && *opt != '\0' ); - IO::Sources[num - left] = zpl_string_make_length( g_allocator, path, length ); + zpl_string path_string = zpl_string_make_length( g_allocator, path, length ); + zpl_array_append( IO::Sources, path_string ); opt++; } @@ -71,7 +77,7 @@ void parse_options( int num, char** arguments ) } else { - fatal( "-source not provided\n" ); + fatal( "-src not provided\n" ); } if ( zpl_opts_has_arg( & opts, "dst" ) ) @@ -80,7 +86,8 @@ void parse_options( int num, char** arguments ) if ( num == 1 ) { - IO::Destinations[0] = zpl_string_make_length( g_allocator, opt, zpl_string_length( opt) ); + zpl_string path = zpl_string_make_length( g_allocator, opt, zpl_string_length( opt) ); + zpl_array_append( IO::Destinations, path ); } else { @@ -96,15 +103,30 @@ void parse_options( int num, char** arguments ) { path[length] = *opt; } - while ( length++, opt++, *opt != ' ' ); + while ( length++, opt++, *opt != ' ' && *opt != '\0' ); - IO::Destinations[num - left] = zpl_string_make_length( g_allocator, path, length ); + zpl_string path_string = zpl_string_make_length( g_allocator, path, length ); + zpl_array_append( IO::Destinations, path_string ); opt++; } while ( --left ); + + if ( zpl_array_count(IO::Destinations) != zpl_array_count( IO::Sources ) ) + { + fatal("-dst count must match -src count"); + } } } + else + { + uw left = num; + do + { + zpl_array_append( IO::Destinations, IO::Sources[num - left] ); + } + while ( --left ); + } if ( zpl_opts_has_arg( & opts, "spec" ) ) { @@ -113,9 +135,18 @@ void parse_options( int num, char** arguments ) IO::Specification = zpl_string_make( g_allocator, "" ); IO::Specification = zpl_string_append( IO::Specification, opt ); } + else + { + fatal( "-spec not provided\n" ); + } } else { + zpl_printf("\nArguments: "); + for ( int index = 0; index < num; index++) + { + zpl_printf("\nArg[%d]: %s", index, arguments[index]); + } fatal( "Failed to parse arguments\n" ); } @@ -127,7 +158,7 @@ zpl_arena Refactor_Buffer; void refactor() { - ct char const* include_sig = "#include \""; + ct static char const* include_sig = "#include \""; struct Token { @@ -157,61 +188,62 @@ void refactor() } // Prepare data and trackers. - Array_Line src = IO::get_next_source(); - Array_Line lines = src; + char const* src = IO::get_next_source(); if ( src == nullptr ) return; - const sw num_lines = zpl_array_count( lines); + log_fmt("\n\nRefactoring: %s", IO::Sources[IO::Current]); sw buffer_size = IO::Current_Size; - sw left = num_lines; - Line line = *lines; - uw pos = 0; + sw left = buffer_size; + uw col = 0; + uw line = 0; + + #define pos (IO::Current_Size - left) + + #define move_forward( Amount_ ) \ + left -= Amount_; \ + col += Amount_; \ + src += Amount_ \ do { - Continue_Line: - // Includes to ignore { - Spec::Entry* ignore = Spec::Ignore_Words; - sw ignores_left = zpl_array_count( Spec::Ignore_Words); + Spec::Entry* ignore = Spec::Ignore_Includes; + sw ignores_left = zpl_array_count( Spec::Ignore_Includes); - do + for ( ; ignores_left; ignores_left--, ignore++ ) { - if ( include_sig[0] != line[0] ) + if ( include_sig[0] != src[0] ) continue; u32 sig_length = zpl_string_length( ignore->Sig ); current = zpl_string_set( current, include_sig ); - current = zpl_string_append_length( current, line, sig_length ); - current = zpl_string_append_length( current, "\"", 2 ); + current = zpl_string_append_length( current, src, sig_length ); + current = zpl_string_append_length( current, "\"", 1 ); // Formats current into: #include "Sig>" if ( zpl_string_are_equal( ignore->Sig, current ) ) { - log_fmt("\nIgnored %-81s line %d", current, num_lines - left ); + log_fmt("\nIgnored %-81s line %d, col %d", current, line, col ); const sw length = zpl_string_length( current ); - line += length; - pos += length; + move_forward( length ); // Force end of line. - while ( line != '\0' ) + while ( src[0] != '\n' ) { - line++; - pos++; + move_forward( 1 ); } goto Skip; } } - while ( ignore++, --ignores_left ); } // Word Ignores @@ -219,20 +251,20 @@ void refactor() Spec::Entry* ignore = Spec::Ignore_Words; sw ignores_left = zpl_array_count( Spec::Ignore_Words); - do + for ( ; ignores_left; ignores_left--, ignore++ ) { - if ( ignore->Sig[0] != line[0] ) + if ( ignore->Sig[0] != src[0] ) continue; zpl_string_clear( current ); u32 sig_length = zpl_string_length( ignore->Sig ); - current = zpl_string_append_length( current, line, sig_length ); + current = zpl_string_append_length( current, src, sig_length ); if ( zpl_string_are_equal( ignore->Sig, current ) ) { - char before = line[-1]; - char after = line[sig_length]; + char before = src[-1]; + char after = src[sig_length]; if ( zpl_char_is_alphanumeric( before ) || before == '_' || zpl_char_is_alphanumeric( after ) || after == '_' ) @@ -240,14 +272,12 @@ void refactor() continue; } - log_fmt("\nIgnored %-81s line %d", current, num_lines - left ); + log_fmt("\nIgnored %-81s line %d, col %d", current, line, col ); - line += sig_length; - pos += sig_length; + move_forward( sig_length ); goto Skip; } } - while ( ignore++, --ignores_left ); } // Namespace Ignores @@ -255,20 +285,23 @@ void refactor() Spec::Entry* ignore = Spec::Ignore_Namespaces; sw ignores_left = zpl_array_count( Spec::Ignore_Namespaces); - do + for ( ; ignores_left; ignores_left--, ignore++ ) { - if ( ignore->Sig[0] != line[0] ) + if ( ignore->Sig[0] != src[0] ) + { + ignore++; continue; + } zpl_string_clear( current ); u32 sig_length = zpl_string_length( ignore->Sig ); - current = zpl_string_append_length( current, line, sig_length ); + current = zpl_string_append_length( current, src, sig_length ); if ( zpl_string_are_equal( ignore->Sig, current ) ) { - u32 length = sig_length; - char* ns_content = line + sig_length; + u32 length = sig_length; + char const* ns_content = src + sig_length; while ( zpl_char_is_alphanumeric( ns_content[0] ) || ns_content[0] == '_' ) { @@ -278,16 +311,14 @@ void refactor() #if Build_Debug zpl_string_clear( preview ); - preview = zpl_string_append_length( preview, line, length ); - log_fmt("\nIgnored %-40s %-40s line %d", preview, ignore->Sig, - left); + preview = zpl_string_append_length( preview, src, length ); + log_fmt("\nIgnored %-40s %-40s line %d, column %d", preview, ignore->Sig, line, col ); #endif - line += length; - pos += length; + move_forward( length ); goto Skip; } } - while ( ignore++, --ignores_left ); } // Includes to match @@ -296,16 +327,16 @@ void refactor() sw includes_left = zpl_array_count ( Spec::Includes); - do + for ( ; includes_left; includes_left--, include++ ) { - if ( include_sig[0] != line[0] ) + if ( include_sig[0] != src[0] ) continue; u32 sig_length = zpl_string_length( include->Sig ); current = zpl_string_set( current, include_sig ); - current = zpl_string_append_length( current, line, sig_length ); - current = zpl_string_append_length( current, "\"", 2 ); + current = zpl_string_append_length( current, src, sig_length ); + current = zpl_string_append_length( current, "\"", 1 ); // Formats current into: #include "Sig>" if ( zpl_string_are_equal( include->Sig, current ) ) @@ -326,22 +357,19 @@ void refactor() zpl_array_append( tokens, entry ); - log_fmt("\nFound %-81s line %d", current, num_lines - left); + log_fmt("\nFound %-81s line %d, column %d", current, line, col ); - line += length; - pos += length; + move_forward( length ); // Force end of line. - while ( line != '\0' ) + while ( src[0] != '\0' ) { - line++; - pos++; + move_forward( 1 ); } goto Skip; } } - while ( include++, --includes_left ); } // Words to match @@ -349,20 +377,20 @@ void refactor() Spec::Entry* word = Spec::Words; sw words_left = zpl_array_count ( Spec::Words); - do + for ( ; words_left; words_left--, word++ ) { - if ( word->Sig[0] != line[0] ) + if ( word->Sig[0] != src[0] ) continue; zpl_string_clear( current ); sw sig_length = zpl_string_length( word->Sig); - current = zpl_string_append_length( current, line, sig_length ); + current = zpl_string_append_length( current, src, sig_length ); if ( zpl_string_are_equal( word->Sig, current ) ) { - char before = line[-1]; - char after = line[sig_length]; + char before = src[-1]; + char after = src[sig_length]; if ( zpl_char_is_alphanumeric( before ) || before == '_' || zpl_char_is_alphanumeric( after ) || after == '_' ) @@ -384,14 +412,12 @@ void refactor() zpl_array_append( tokens, entry ); - log_fmt("\nFound %-81s line %d", current, num_lines - left); + log_fmt("\nFound %-81s line %d, column %d", current, line, col ); - line += sig_length; - pos += sig_length; + move_forward( sig_length ); goto Skip; } } - while ( word++, --words_left ); } // Namespaces to match @@ -400,20 +426,20 @@ void refactor() sw nspaces_left = zpl_array_count( Spec::Namespaces); - do + for ( ; nspaces_left; nspaces_left--, nspace++ ) { - if ( nspace->Sig[0] != line[0] ) + if ( nspace->Sig[0] != src[0] ) continue; zpl_string_clear( current ); u32 sig_length = zpl_string_length( nspace->Sig ); - current = zpl_string_append_length( current, line, sig_length ); + current = zpl_string_append_length( current, src, sig_length ); if ( zpl_string_are_equal( nspace->Sig, current ) ) { - u32 length = sig_length; - char* ns_content = line + sig_length; + u32 length = sig_length; + char const* ns_content = src + sig_length; while ( zpl_char_is_alphanumeric( ns_content[0] ) || ns_content[0] == '_' ) { @@ -439,26 +465,29 @@ void refactor() #if Build_Debug zpl_string_clear( preview ); - preview = zpl_string_append_length( preview, line, length); - log_fmt("\nFound %-40s %-40s line %d", preview, nspace->Sig, num_lines - left); + preview = zpl_string_append_length( preview, src, length); + log_fmt("\nFound %-40s %-40s line %d, column %d", preview, nspace->Sig, line, col ); #endif - line += length; - pos += length; + move_forward( length ); goto Skip; } } - while ( nspace++, --nspaces_left ); } Skip: - if ( line != '\0' ) - goto Continue_Line; + if ( src[0] == '\n' ) + { + line++; + col = 0; + } + + src++; } - while ( lines++, line = *lines, left ); + while ( --left ); // Prep data for building the content - left = IO::Current_Size; + left = zpl_array_count( tokens); char* content = IO::Current_Content; @@ -475,17 +504,19 @@ void refactor() sw sig_length = zpl_string_length( entry->Sig ); // Append between tokens - refactored = zpl_string_append_length( refactored, line, segment_length ); - line += segment_length + sig_length; + refactored = zpl_string_append_length( refactored, content, segment_length ); + content += segment_length + sig_length; segment_length = entry->End - entry->Start - sig_length; // Append token if ( entry->Sub ) - refactored = zpl_string_append( refactored, entry->Sub ); + { + refactored = zpl_string_append( refactored, entry->Sub ); + } - refactored = zpl_string_append_length( refactored, line, segment_length ); - line += segment_length; + refactored = zpl_string_append_length( refactored, content, segment_length ); + content += segment_length; previous_end = entry->End; entry++; @@ -496,11 +527,13 @@ void refactor() if ( entry->End < IO::Current_Size ) { - refactored = zpl_string_append_length( refactored, line, IO::Current_Size - entry->End ); + refactored = zpl_string_append_length( refactored, content, IO::Current_Size - 1 - entry->End ); } } IO::write( refactored ); + + zpl_string_clear( refactored ); } int main( int num, char** arguments ) @@ -520,6 +553,8 @@ int main( int num, char** arguments ) do { refactor(); + + zpl_printf("\nRefactored: %s", IO::Sources[IO::Current]); } while ( --left ); diff --git a/scripts/build.ps1 b/scripts/build.ps1 index 970777c..af216d5 100644 --- a/scripts/build.ps1 +++ b/scripts/build.ps1 @@ -42,46 +42,39 @@ if ( $type ) Start-Process meson $args_meson -NoNewWindow -Wait -WorkingDirectory $path_scripts } -#endregion Regular Build - - -#region Test Build -write-host "`n`nBuilding Test`n" - -if ( -not( Test-Path $path_build ) ) -{ - return; -} $args_ninja = @() $args_ninja += "-C" $args_ninja += $path_build Start-Process ninja $args_ninja -Wait -NoNewWindow -WorkingDirectory $path_root +#endregion Regular Build -# Refactor thirdparty libraries -& .\refactor_and_format.ps1 -if ( $test -eq $false ) +if ( $test -eq $true -and (Test-Path $path_build) ) { - return; + #region Test Build + write-host "`n`nBuilding Test`n" + + # Refactor thirdparty libraries + & .\refactor_and_format.ps1 + + $path_test = Join-Path $path_root test + $path_test_build = Join-Path $path_test build + + if ( -not( Test-Path $path_test_build ) ) + { + $args_meson = @() + $args_meson += "setup" + $args_meson += $path_test_build + + Start-Process meson $args_meson -NoNewWindow -Wait -WorkingDirectory $path_scripts + } + + $args_ninja = @() + $args_ninja += "-C" + $args_ninja += $path_build + + Start-Process ninja $args_ninja -Wait -NoNewWindow -WorkingDirectory $path_test + #endregion Test Build } - -$path_test = Join-Path $path_root test -$path_test_build = Join-Path $path_test build - -if ( -not( Test-Path $path_test_build ) ) -{ - $args_meson = @() - $args_meson += "setup" - $args_meson += $path_test_build - - Start-Process meson $args_meson -NoNewWindow -Wait -WorkingDirectory $path_scripts -} - -$args_ninja = @() -$args_ninja += "-C" -$args_ninja += $path_build - -Start-Process ninja $args_ninja -Wait -NoNewWindow -WorkingDirectory $path_test -#endregion Test Build diff --git a/scripts/get_sources.ps1 b/scripts/get_sources.ps1 index 4fc592e..03b9889 100644 --- a/scripts/get_sources.ps1 +++ b/scripts/get_sources.ps1 @@ -1,5 +1,5 @@ -[string[]] $include = '*.c', '*.cc', '*.cpp' -# [stirng[]] $exclude = +[string[]] $include = 'refactor.cpp' #'*.c', '*.cc', '*.cpp' +# [stirng[]] $exclude = $path_root = git rev-parse --show-toplevel $path_proj = Join-Path $path_root project From d0fad572bcf43e277761b77ef62c2200081377fa Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sat, 18 Mar 2023 03:10:43 -0400 Subject: [PATCH 4/9] Non-include refactors work. Multi-file works. --- Readme.md | 2 ++ project/Bloat.cpp | 2 +- project/IO.cpp | 26 ++++++++------- project/Spec.cpp | 39 +++++++++++++++------- project/bloat.hpp | 2 ++ project/refactor.cpp | 78 ++++++++++++++++++++++++++++++++------------ 6 files changed, 104 insertions(+), 45 deletions(-) diff --git a/Readme.md b/Readme.md index b6baadb..21f2482 100644 --- a/Readme.md +++ b/Readme.md @@ -8,6 +8,7 @@ Refactor c/c++ files (and problably others) with ease. * `-src` : Source file to refactor * `-dst` : Destination file after the refactor (omit to use the same as source) * `-spec` : Specification containing rules to use for the refactor. +* `-debug` : Use only if on debug build and desire to attach to process. ## Syntax : @@ -45,3 +46,4 @@ TODO: * Test to see how much needs to be ported for other platforms (if at all) * Setup as api. * Provide binaries in the release page for github. (debug and release builds) +* Directive to ignore comments (with a way to specify the comment signature). Right now comments that meet the signature of words or namespaces are refactored. diff --git a/project/Bloat.cpp b/project/Bloat.cpp index 208b716..675b45c 100644 --- a/project/Bloat.cpp +++ b/project/Bloat.cpp @@ -9,7 +9,7 @@ namespace Memory void setup() { - zpl_arena_init_from_allocator( & Global_Arena, zpl_heap(), zpl_megabytes(2) ); + zpl_arena_init_from_allocator( & Global_Arena, zpl_heap(), Initial_Reserve ); if ( Global_Arena.total_size == 0 ) { diff --git a/project/IO.cpp b/project/IO.cpp index f129dde..cb439d6 100644 --- a/project/IO.cpp +++ b/project/IO.cpp @@ -18,7 +18,8 @@ namespace IO uw Current_Size = 0; uw Largest_Src_Size = 0; - zpl_arena MemPerist; + zpl_arena MemSpec; + zpl_arena MemSrc; } using namespace StaticData; @@ -49,16 +50,17 @@ namespace IO zpl_file_close( & src ); } - while ( left--, left > 1 ); + while ( path++, left--, left > 1 ); - uw persist_size = ZPL_ARRAY_GROW_FORMULA( Largest_Src_Size ); + uw persist_size = Largest_Src_Size * 2 + 8; - zpl_arena_init_from_allocator( & MemPerist, zpl_heap(), persist_size ); + zpl_arena_init_from_allocator( & MemSrc, zpl_heap(), persist_size ); } void cleanup() { - zpl_arena_free( & MemPerist ); + zpl_arena_free( & MemSpec ); + zpl_arena_free( & MemSrc ); } Array_Line get_specification() @@ -78,22 +80,23 @@ namespace IO fatal("No content in specificaiton to process"); } - char* content = rcast( char*, zpl_alloc( zpl_arena_allocator( & MemPerist), fsize + 1) ); + zpl_arena_init_from_allocator( & MemSpec, zpl_heap(), fsize * 2 + 8 ); + + char* content = rcast( char*, zpl_alloc( zpl_arena_allocator( & MemSpec), fsize + 1) ); zpl_file_read( & file, content, fsize); zpl_file_close( & file ); content[fsize] = 0; - Array_Line lines = zpl_str_split_lines( zpl_arena_allocator( & MemPerist ), content, false ); + Array_Line lines = zpl_str_split_lines( zpl_arena_allocator( & MemSpec ), content, false ); return lines; } char* get_next_source() { - zpl_memset( MemPerist.physical_start, 0, MemPerist.total_allocated); - MemPerist.total_allocated = 0; - MemPerist.temp_count = 0; + zpl_memset( MemSrc.physical_start, 0, MemSrc.total_allocated); + zpl_free_all( zpl_arena_allocator( & MemSrc) ); Current++; @@ -111,7 +114,7 @@ namespace IO if ( Current_Size <= 0 ) return nullptr; - Current_Content = rcast( char* , zpl_alloc( zpl_arena_allocator( & MemPerist), Current_Size + 1) ); + Current_Content = rcast( char* , zpl_alloc( zpl_arena_allocator( & MemSrc), Current_Size + 1) ); zpl_file_read( & file, Current_Content, Current_Size ); zpl_file_close( & file ); @@ -138,7 +141,6 @@ namespace IO } zpl_file_write( & file_dest, refacotred, zpl_string_length(refacotred) ); - zpl_file_close( & file_dest ); } } diff --git a/project/Spec.cpp b/project/Spec.cpp index 8fb61ca..3009758 100644 --- a/project/Spec.cpp +++ b/project/Spec.cpp @@ -38,15 +38,30 @@ namespace Spec // Helper function for process(). forceinline - void find_next_token( zpl_string& token, char*& line, u32& length ) + void find_next_token( Tok& type, zpl_string& token, char*& line, u32& length ) { zpl_string_clear( token ); length = 0; - while ( zpl_char_is_alphanumeric( line[length] ) || line[length] == '_' ) + #define current line[length] + if (type == Tok::Include) { - length++; + // Allows for '.' + while ( zpl_char_is_alphanumeric( current ) + || current == '_' + || current == '.' ) + { + length++; + } } + else + { + while ( zpl_char_is_alphanumeric( current ) || current == '_' ) + { + length++; + } + } + #undef current if ( length == 0 ) { @@ -119,15 +134,15 @@ namespace Spec } } - u32 length = 0; - - // Find a valid token - find_next_token( token, line, length ); - + u32 length = 0; Tok type = Tok::Num_Tok; bool ignore = false; Entry entry {}; + + // Find a valid token + find_next_token( type, token, line, length ); + // Will be reguarded as an ignore. if ( is_tok( Tok::Not, token, length )) { @@ -143,7 +158,7 @@ namespace Spec } // Find the category token - find_next_token( token, line, length ); + find_next_token( type, token, line, length ); } if ( is_tok( Tok::Word, token, length ) ) @@ -174,8 +189,8 @@ namespace Spec lines++; continue; } - - find_next_token( token, line, length ); + + find_next_token( type, token, line, length ); // First argument is signature. entry.Sig = zpl_string_make_length( g_allocator, token, length ); @@ -285,7 +300,7 @@ namespace Spec } } - find_next_token( token, line, length ); + find_next_token( type, token, line, length ); // Second argument is substitute. entry.Sub = zpl_string_make_length( g_allocator, token, length ); diff --git a/project/bloat.hpp b/project/bloat.hpp index 94c29b8..b4437cd 100644 --- a/project/bloat.hpp +++ b/project/bloat.hpp @@ -90,6 +90,8 @@ ct char const* Msg_Invalid_Value = "INVALID VALUE PROVIDED"; namespace Memory { + ct uw Initial_Reserve = zpl_megabytes(2); + extern zpl_arena Global_Arena; #define g_allocator zpl_arena_allocator( & Memory::Global_Arena) diff --git a/project/refactor.cpp b/project/refactor.cpp index 1318ea9..b601e98 100644 --- a/project/refactor.cpp +++ b/project/refactor.cpp @@ -9,15 +9,27 @@ void parse_options( int num, char** arguments ) { zpl_opts opts; - zpl_opts_init( & opts, g_allocator, "refactor"); + zpl_opts_init( & opts, zpl_heap(), "refactor"); zpl_opts_add( & opts, "num", "num" , "Number of files to refactor" , ZPL_OPTS_INT ); zpl_opts_add( & opts, "src" , "src" , "File/s to refactor" , ZPL_OPTS_STRING); zpl_opts_add( & opts, "dst" , "dst" , "File/s post refactor" , ZPL_OPTS_STRING); zpl_opts_add( & opts, "spec", "spec", "Specification for refactoring", ZPL_OPTS_STRING); +#if Build_Debug + zpl_opts_add( & opts, "debug", "debug", "Allows for wait to attach", ZPL_OPTS_FLAG); +#endif + if (opts_custom_compile( & opts, num, arguments)) { sw num = 0; + + #if Build_Debug + if ( zpl_opts_has_arg( & opts, "debug" ) ) + { + zpl_printf("Will wait (pause available for attachment)"); + char pause = getchar(); + } + #endif if ( zpl_opts_has_arg( & opts, "num" ) ) { @@ -158,7 +170,7 @@ zpl_arena Refactor_Buffer; void refactor() { - ct static char const* include_sig = "#include \""; + ct static char const* include_sig = "include"; struct Token { @@ -220,12 +232,23 @@ void refactor() if ( include_sig[0] != src[0] ) continue; + if ( zpl_strncmp( include_sig, src, sizeof(include_sig) - 1 ) != 0 ) + { + break; + } + + src += sizeof(include_sig) - 1; + + // Ignore whitespace + while ( zpl_char_is_space( src[0] ) || src[0] == '\"' || src[0] == '<' ) + { + src++; + } + u32 sig_length = zpl_string_length( ignore->Sig ); - current = zpl_string_set( current, include_sig ); - current = zpl_string_append_length( current, src, sig_length ); - current = zpl_string_append_length( current, "\"", 1 ); - // Formats current into: #include "Sig>" + zpl_string_clear( current ); + current = zpl_string_append_length( current, ignore->Sig, sig_length ); if ( zpl_string_are_equal( ignore->Sig, current ) ) { @@ -233,7 +256,8 @@ void refactor() const sw length = zpl_string_length( current ); - move_forward( length ); + // The + 1 is for the closing " or > of the include + move_forward( length + 1 ); // Force end of line. while ( src[0] != '\n' ) @@ -332,21 +356,30 @@ void refactor() if ( include_sig[0] != src[0] ) continue; + if ( zpl_strncmp( include_sig, src, sizeof(include_sig) - 1 ) != 0 ) + { + break; + } + + src += sizeof(include_sig) - 1; + + // Ignore whitespace + while ( zpl_char_is_space( src[0] ) || src[0] == '\"' || src[0] == '<' ) + { + src++; + } + u32 sig_length = zpl_string_length( include->Sig ); - current = zpl_string_set( current, include_sig ); - current = zpl_string_append_length( current, src, sig_length ); - current = zpl_string_append_length( current, "\"", 1 ); - // Formats current into: #include "Sig>" + zpl_string_clear( current ); + current = zpl_string_append_length( current, include->Sig, sig_length ); if ( zpl_string_are_equal( include->Sig, current ) ) { Token entry {}; - const sw length = zpl_string_length( current ); - entry.Start = pos; - entry.End = pos + length; + entry.End = pos + sig_length; entry.Sig = include->Sig; if ( include->Sub != nullptr ) @@ -359,10 +392,10 @@ void refactor() log_fmt("\nFound %-81s line %d, column %d", current, line, col ); - move_forward( length ); + move_forward( sig_length ); // Force end of line. - while ( src[0] != '\0' ) + while ( src[0] != '\n' ) { move_forward( 1 ); } @@ -486,14 +519,19 @@ void refactor() } while ( --left ); + if (zpl_array_count( tokens ) == 0) + { + return; + } + // Prep data for building the content left = zpl_array_count( tokens); char* content = IO::Current_Content; + zpl_string refactored = zpl_string_make_reserve( zpl_arena_allocator( & Refactor_Buffer ), buffer_size ); + // Generate the refactored file content. - static zpl_string - refactored = zpl_string_make_reserve( zpl_arena_allocator( & Refactor_Buffer ), buffer_size ); { Token* entry = tokens; sw previous_end = 0; @@ -521,7 +559,7 @@ void refactor() previous_end = entry->End; entry++; } - while ( --left ); + while ( --left > 0 ); entry--; @@ -533,7 +571,7 @@ void refactor() IO::write( refactored ); - zpl_string_clear( refactored ); + zpl_free_all( zpl_arena_allocator( & Refactor_Buffer )); } int main( int num, char** arguments ) From 87c939e2b60ce8424009413dfda72f8f86d3f870 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sat, 25 Mar 2023 22:44:36 -0400 Subject: [PATCH 5/9] Got include ignores to work, comment ignores work Not sure yet if include renames work just yet (need to test) Comment signatures are currently hardcoded for C/C++. --- Test/zpl.refactor | 12 +++++-- project/Spec.cpp | 27 ++++++++++++-- project/Spec.hpp | 3 ++ project/refactor.cpp | 86 ++++++++++++++++++++++++++++++++++---------- refactor.10x | 8 ++--- 5 files changed, 110 insertions(+), 26 deletions(-) diff --git a/Test/zpl.refactor b/Test/zpl.refactor index c5172bb..d8737fe 100644 --- a/Test/zpl.refactor +++ b/Test/zpl.refactor @@ -9,9 +9,17 @@ __VERSION 1 // Precedence (highest to lowest): // word, namespace, regex +// Comments +not comments + // Header files -not include zpl_hedley -//not word zpl_hedley +not include zpl_hedley.h +not include allocator.h +not include array.h +not include header/essentials/collections/array.h +not include header/essentials/collections/list.h +not include header/core/file.h +not include header/opts.h // Removes the namespace. namespace zpl_ diff --git a/project/Spec.cpp b/project/Spec.cpp index 3009758..83d9015 100644 --- a/project/Spec.cpp +++ b/project/Spec.cpp @@ -11,6 +11,9 @@ namespace Spec namespace StaticData { + // Custom comment signatures not supported yet (only C/C++ comments for now) + bool Ignore_Comments = false; + Array_Entry Ignore_Includes; Array_Entry Ignore_Words; Array_Entry Ignore_Regexes; @@ -49,7 +52,9 @@ namespace Spec // Allows for '.' while ( zpl_char_is_alphanumeric( current ) || current == '_' - || current == '.' ) + || current == '.' + || current == '/' + || current == '\\' ) { length++; } @@ -161,7 +166,15 @@ namespace Spec find_next_token( type, token, line, length ); } - if ( is_tok( Tok::Word, token, length ) ) + if ( is_tok( Tok::Comment, token, length ) ) + { + // Custom comment signatures not supported yet (only C/C++ comments for now) + Ignore_Comments = true; + + lines++; + continue; + } + else if ( is_tok( Tok::Word, token, length ) ) { type = Tok::Word; } @@ -324,5 +337,15 @@ namespace Spec } } while ( --left ); + + Spec::Entry* ignore = Spec::Ignore_Includes; + sw ignores_left = zpl_array_count( Spec::Ignore_Includes); + + zpl_printf("\nIgnores: "); + for ( ; ignores_left; ignores_left--, ignore++ ) + { + zpl_printf("\n%s", ignore->Sig); + } + zpl_printf("\n"); } } diff --git a/project/Spec.hpp b/project/Spec.hpp index c3e158d..1e855d0 100644 --- a/project/Spec.hpp +++ b/project/Spec.hpp @@ -7,6 +7,7 @@ namespace Spec enum Tok { Not, + Comment, Include, Namespace, Word, @@ -21,6 +22,7 @@ namespace Spec char const* tok_to_str[ Tok::Num_Tok ] = { "not", + "comments", "include", "namespace", "word", @@ -36,6 +38,7 @@ namespace Spec const u8 tok_to_len[ Tok::Num_Tok ] = { 3, + 8, 7, 9, 4, diff --git a/project/refactor.cpp b/project/refactor.cpp index b601e98..b1ab0db 100644 --- a/project/refactor.cpp +++ b/project/refactor.cpp @@ -216,12 +216,41 @@ void refactor() #define pos (IO::Current_Size - left) #define move_forward( Amount_ ) \ - left -= Amount_; \ - col += Amount_; \ - src += Amount_ \ + if ( left - Amount_ <= 0 ) \ + goto End_Search; \ + \ + left -= Amount_; \ + col += Amount_; \ + src += Amount_ \ do { + if ( Spec::Ignore_Comments && src[0] == '/' && left - 2 > 0 ) + { + if ( src[1] == '/' ) + { + move_forward( 2 ); + + // Force end of line. + while ( src[0] != '\n' ) + { + move_forward( 1 ); + } + + goto Skip; + } + else if ( src[1] == '*' ) + { + do + { + move_forward( 2 ); + } + while ( (left - 2) > 0 && !( src[0] == '*' && src[1] == '/' ) ); + + move_forward( 2 ); + } + } + // Includes to ignore { Spec::Entry* ignore = Spec::Ignore_Includes; @@ -229,26 +258,34 @@ void refactor() for ( ; ignores_left; ignores_left--, ignore++ ) { - if ( include_sig[0] != src[0] ) + if ( '#' != src[0] ) continue; + move_forward( 1 ); + + // Ignore whitespace + while ( zpl_char_is_space( src[0] ) ) + { + move_forward( 1 ); + } + if ( zpl_strncmp( include_sig, src, sizeof(include_sig) - 1 ) != 0 ) { break; } - src += sizeof(include_sig) - 1; + move_forward( sizeof(include_sig) - 1 ); // Ignore whitespace while ( zpl_char_is_space( src[0] ) || src[0] == '\"' || src[0] == '<' ) { - src++; + move_forward(1); } - u32 sig_length = zpl_string_length( ignore->Sig ); - zpl_string_clear( current ); - current = zpl_string_append_length( current, ignore->Sig, sig_length ); + + u32 sig_length = zpl_string_length( ignore->Sig ); + current = zpl_string_append_length( current, ignore->Sig, sig_length ); if ( zpl_string_are_equal( ignore->Sig, current ) ) { @@ -353,9 +390,17 @@ void refactor() for ( ; includes_left; includes_left--, include++ ) { - if ( include_sig[0] != src[0] ) + if ( '#' != src[0] ) continue; + move_forward( 1 ); + + // Ignore whitespace + while ( zpl_char_is_space( src[0] ) ) + { + move_forward( 1 ); + } + if ( zpl_strncmp( include_sig, src, sizeof(include_sig) - 1 ) != 0 ) { break; @@ -366,13 +411,13 @@ void refactor() // Ignore whitespace while ( zpl_char_is_space( src[0] ) || src[0] == '\"' || src[0] == '<' ) { - src++; + move_forward( 1 ); } - u32 sig_length = zpl_string_length( include->Sig ); - zpl_string_clear( current ); - current = zpl_string_append_length( current, include->Sig, sig_length ); + + u32 sig_length = zpl_string_length( include->Sig ); + current = zpl_string_append_length( current, include->Sig, sig_length ); if ( zpl_string_are_equal( include->Sig, current ) ) { @@ -392,7 +437,8 @@ void refactor() log_fmt("\nFound %-81s line %d, column %d", current, line, col ); - move_forward( sig_length ); + // The + 1 is for the closing " or > of the include + move_forward( sig_length + 1 ); // Force end of line. while ( src[0] != '\n' ) @@ -515,9 +561,10 @@ void refactor() col = 0; } - src++; + move_forward( 1 ); } - while ( --left ); + while ( left ); +End_Search: if (zpl_array_count( tokens ) == 0) { @@ -550,7 +597,7 @@ void refactor() // Append token if ( entry->Sub ) { - refactored = zpl_string_append( refactored, entry->Sub ); + refactored = zpl_string_append( refactored, entry->Sub ); } refactored = zpl_string_append_length( refactored, content, segment_length ); @@ -572,6 +619,9 @@ void refactor() IO::write( refactored ); zpl_free_all( zpl_arena_allocator( & Refactor_Buffer )); + + #undef pos + #undef move_forward } int main( int num, char** arguments ) diff --git a/refactor.10x b/refactor.10x index fc30d17..c3febc7 100644 --- a/refactor.10x +++ b/refactor.10x @@ -2,7 +2,7 @@ *.*, - *.obj,*.lib,*.pch,*.dll,*.pdb,.vs,Debug,Release,x64,obj,*.user,Intermediate, + *.obj,*.lib,*.pch,*.dll,*.pdb,.vs,Debug,Release,x64,obj,*.user,Intermediate,.git,.idea,.vscode, true true true @@ -36,17 +36,17 @@ C:\Program Files (x86)\Windows Kits\10\\include\10.0.22621.0\\winrt C:\Program Files (x86)\Windows Kits\10\\include\10.0.22621.0\\cppwinrt C:\Program Files (x86)\Windows Kits\NETFXSDK\4.8\include\um - .\thirdparty - ZPL_IMPLEMENTATION + Debug:x64 - C:\projects\refactor\thirdparty + ./project + ./thirdparty From 1cd4287eb29d396c2e83ae078db398c05bb926bb Mon Sep 17 00:00:00 2001 From: Ed_ Date: Mon, 27 Mar 2023 20:14:04 -0400 Subject: [PATCH 6/9] Fixed issue with ignoring multi-line comments. --- project/refactor.cpp | 6 ++++-- scripts/refactor_and_format.ps1 | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/project/refactor.cpp b/project/refactor.cpp index b1ab0db..90deca7 100644 --- a/project/refactor.cpp +++ b/project/refactor.cpp @@ -243,11 +243,13 @@ void refactor() { do { - move_forward( 2 ); + move_forward( 1 ); } while ( (left - 2) > 0 && !( src[0] == '*' && src[1] == '/' ) ); - move_forward( 2 ); + move_forward( 1 ); + + goto Skip; } } diff --git a/scripts/refactor_and_format.ps1 b/scripts/refactor_and_format.ps1 index 36b278e..515e3a4 100644 --- a/scripts/refactor_and_format.ps1 +++ b/scripts/refactor_and_format.ps1 @@ -18,6 +18,8 @@ write-host "Beginning refactor...`n" $refactors = @(@()) +# TODO: Change this to support refactoring the other files in the project directory. +# It needs two runs, one for the regular files, one for the zpl header. foreach ( $file in $targetFiles ) { $destination = Join-Path $path_test (Split-Path $file -leaf) From 17458b4b4cb3caf471787697503cc9b44c30fb21 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Thu, 30 Mar 2023 13:59:49 -0400 Subject: [PATCH 7/9] Got include refactors (sig to sub) to work. Just need to make sure the test case works with the refactored zpl file --- project/refactor.cpp | 140 ++++++++++++++++++++++++++----------------- 1 file changed, 86 insertions(+), 54 deletions(-) diff --git a/project/refactor.cpp b/project/refactor.cpp index 90deca7..8d22bbc 100644 --- a/project/refactor.cpp +++ b/project/refactor.cpp @@ -223,6 +223,11 @@ void refactor() col += Amount_; \ src += Amount_ \ + #define move_back( Amount_ ) \ + left += Amount_; \ + col -= Amount_; \ + src -= Amount_ \ + do { if ( Spec::Ignore_Comments && src[0] == '/' && left - 2 > 0 ) @@ -254,40 +259,49 @@ void refactor() } // Includes to ignore + do { Spec::Entry* ignore = Spec::Ignore_Includes; sw ignores_left = zpl_array_count( Spec::Ignore_Includes); + sw rewind = 0; + + if ( '#' != src[0] ) + break; + + move_forward( 1 ); + rewind++; + + // Ignore whitespace + while ( zpl_char_is_space( src[0] ) ) + { + move_forward( 1 ); + rewind++; + } + + if ( zpl_strncmp( include_sig, src, sizeof(include_sig) - 1 ) != 0 ) + { + move_back( rewind ); + break; + } + + const u32 sig_size = sizeof(include_sig) - 1; + + move_forward( sig_size ); + rewind += sig_size; + + // Ignore whitespace + while ( zpl_char_is_space( src[0] ) || src[0] == '\"' || src[0] == '<' ) + { + move_forward(1); + rewind++; + } for ( ; ignores_left; ignores_left--, ignore++ ) { - if ( '#' != src[0] ) - continue; - - move_forward( 1 ); - - // Ignore whitespace - while ( zpl_char_is_space( src[0] ) ) - { - move_forward( 1 ); - } - - if ( zpl_strncmp( include_sig, src, sizeof(include_sig) - 1 ) != 0 ) - { - break; - } - - move_forward( sizeof(include_sig) - 1 ); - - // Ignore whitespace - while ( zpl_char_is_space( src[0] ) || src[0] == '\"' || src[0] == '<' ) - { - move_forward(1); - } - zpl_string_clear( current ); u32 sig_length = zpl_string_length( ignore->Sig ); - current = zpl_string_append_length( current, ignore->Sig, sig_length ); + current = zpl_string_append_length( current, src, sig_length ); if ( zpl_string_are_equal( ignore->Sig, current ) ) { @@ -307,7 +321,10 @@ void refactor() goto Skip; } } - } + + move_back( rewind ); + } + while (false); // Word Ignores { @@ -385,41 +402,49 @@ void refactor() } // Includes to match + do { - Spec::Entry* include = Spec::Includes; + Spec::Entry* include = Spec::Includes; + sw includes_left = zpl_array_count ( Spec::Includes); + sw rewind = 0; - sw includes_left = zpl_array_count ( Spec::Includes); + if ( '#' != src[0] ) + break; + + move_forward( 1 ); + rewind++; + + // Ignore whitespace + while ( zpl_char_is_space( src[0] ) ) + { + move_forward( 1 ); + rewind++; + } + + if ( zpl_strncmp( include_sig, src, sizeof(include_sig) - 1 ) != 0 ) + { + move_back( rewind ); + break; + } + + const u32 sig_size = sizeof(include_sig) - 1; + + move_forward( sig_size ); + rewind += sig_size; + + // Ignore whitespace + while ( zpl_char_is_space( src[0] ) || src[0] == '\"' || src[0] == '<' ) + { + move_forward( 1 ); + rewind++; + } for ( ; includes_left; includes_left--, include++ ) { - if ( '#' != src[0] ) - continue; - - move_forward( 1 ); - - // Ignore whitespace - while ( zpl_char_is_space( src[0] ) ) - { - move_forward( 1 ); - } - - if ( zpl_strncmp( include_sig, src, sizeof(include_sig) - 1 ) != 0 ) - { - break; - } - - src += sizeof(include_sig) - 1; - - // Ignore whitespace - while ( zpl_char_is_space( src[0] ) || src[0] == '\"' || src[0] == '<' ) - { - move_forward( 1 ); - } - zpl_string_clear( current ); u32 sig_length = zpl_string_length( include->Sig ); - current = zpl_string_append_length( current, include->Sig, sig_length ); + current = zpl_string_append_length( current, src, sig_length ); if ( zpl_string_are_equal( include->Sig, current ) ) { @@ -451,7 +476,10 @@ void refactor() goto Skip; } } - } + + move_back( rewind ); + } + while (false); // Words to match { @@ -564,6 +592,10 @@ void refactor() } move_forward( 1 ); + + // zpl_string_clear( preview ); + // preview = zpl_string_append_length( preview, src, 100); + // log_fmt( "__PREVIEW: %d \nn%s\n\n__PREVIEW_END", left, preview ); } while ( left ); End_Search: From c4f0b35a30eff116134dc548e8255dc25cd1c2e9 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Thu, 30 Mar 2023 17:35:57 -0400 Subject: [PATCH 8/9] Fixed bugs with spec file parsing related to word ignores. --- project/Bloat.cpp | 4 ++-- project/IO.cpp | 2 +- project/Spec.cpp | 20 +++++--------------- project/refactor.cpp | 3 --- 4 files changed, 8 insertions(+), 21 deletions(-) diff --git a/project/Bloat.cpp b/project/Bloat.cpp index 675b45c..0423a8e 100644 --- a/project/Bloat.cpp +++ b/project/Bloat.cpp @@ -60,7 +60,7 @@ b32 opts_custom_compile(zpl_opts *opts, int argc, char **argv) if (*arg) { - arg = cast(char*)zpl_str_trim(arg, false); + arg = (char*)zpl_str_trim(arg, false); if (*arg == '-') { @@ -132,7 +132,7 @@ b32 opts_custom_compile(zpl_opts *opts, int argc, char **argv) } } - e = cast(char *)zpl_str_control_skip(e, '\0'); + e = (char *)zpl_str_control_skip(e, '\0'); zpl__opts_set_value(opts, entry, b); if ( (i + 1) < argc ) diff --git a/project/IO.cpp b/project/IO.cpp index cb439d6..4e5ba68 100644 --- a/project/IO.cpp +++ b/project/IO.cpp @@ -80,7 +80,7 @@ namespace IO fatal("No content in specificaiton to process"); } - zpl_arena_init_from_allocator( & MemSpec, zpl_heap(), fsize * 2 + 8 ); + zpl_arena_init_from_allocator( & MemSpec, zpl_heap(), fsize * 3 + 8 ); char* content = rcast( char*, zpl_alloc( zpl_arena_allocator( & MemSpec), fsize + 1) ); diff --git a/project/Spec.cpp b/project/Spec.cpp index 83d9015..eb38dd0 100644 --- a/project/Spec.cpp +++ b/project/Spec.cpp @@ -123,7 +123,6 @@ namespace Spec // Ignore line if its a comment if ( line[0] == '/' && line[1] == '/') { - lines++; continue; } @@ -134,7 +133,6 @@ namespace Spec if ( line[0] == '\0' ) { - lines++; continue; } } @@ -144,6 +142,8 @@ namespace Spec bool ignore = false; Entry entry {}; + log_fmt("\nIGNORE WORD COUNT: %d", zpl_array_count(Ignore_Words)); + // Find a valid token find_next_token( type, token, line, length ); @@ -158,7 +158,6 @@ namespace Spec if ( line[0] == '\0' ) { - lines++; continue; } @@ -171,7 +170,6 @@ namespace Spec // Custom comment signatures not supported yet (only C/C++ comments for now) Ignore_Comments = true; - lines++; continue; } else if ( is_tok( Tok::Word, token, length ) ) @@ -189,7 +187,6 @@ namespace Spec else { log_fmt( "Sec::Parse - Unsupported keyword: %s on line: %d", token, zpl_array_count(lines) - left ); - lines++; continue; } @@ -199,7 +196,6 @@ namespace Spec if ( line[0] == '\0' ) { - lines++; continue; } @@ -240,7 +236,6 @@ namespace Spec break; } - lines++; continue; } @@ -276,7 +271,6 @@ namespace Spec if ( bSkip ) { - lines++; continue; } } @@ -308,7 +302,6 @@ namespace Spec break; } - lines++; continue; } } @@ -322,24 +315,21 @@ namespace Spec { case Tok::Word: zpl_array_append( Words, entry ); - lines++; continue; case Tok::Namespace: zpl_array_append( Namespaces, entry ); - lines++; continue; case Tok::Include: zpl_array_append( Includes, entry ); - lines++; continue; } } - while ( --left ); + while ( lines++, left--, left > 0 ); - Spec::Entry* ignore = Spec::Ignore_Includes; - sw ignores_left = zpl_array_count( Spec::Ignore_Includes); + Spec::Entry* ignore = Spec::Ignore_Words; + sw ignores_left = zpl_array_count( Spec::Ignore_Words); zpl_printf("\nIgnores: "); for ( ; ignores_left; ignores_left--, ignore++ ) diff --git a/project/refactor.cpp b/project/refactor.cpp index 8d22bbc..0c6149f 100644 --- a/project/refactor.cpp +++ b/project/refactor.cpp @@ -3,9 +3,6 @@ #include "Spec.cpp" -#define Build_Debug 1 - - void parse_options( int num, char** arguments ) { zpl_opts opts; From 503df85733c76af4d7242382b41b1866e6167450 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Thu, 30 Mar 2023 17:36:36 -0400 Subject: [PATCH 9/9] Setup new test and sucessfuly compiles! --- .gitignore | 4 +- Test/meson.build | 2 +- Test/project.refactor | 71 ++++++++++++++++++++++++++++ Test/zpl.refactor | 36 ++++++++------- scripts/build.ps1 | 6 +-- scripts/clean.ps1 | 7 +++ scripts/get_sources.ps1 | 2 +- scripts/meson.build | 2 +- scripts/refactor_and_format.ps1 | 82 +++++++++++++++++++++++++++------ 9 files changed, 173 insertions(+), 39 deletions(-) diff --git a/.gitignore b/.gitignore index abc3917..2dd8404 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,6 @@ build/* -zpl.refactored.h \ No newline at end of file +Test/*.h +Test/*.hpp +Test/*.cpp diff --git a/Test/meson.build b/Test/meson.build index 0363157..26f9877 100644 --- a/Test/meson.build +++ b/Test/meson.build @@ -4,7 +4,7 @@ project( 'refactor', 'c', 'cpp', default_options : ['buildtype=debug'] ) if get_option('buildtype').startswith('debug') - add_project_arguments('-DBuild_Debug', language : 'cpp') + add_project_arguments('-DBuild_Debug', language : ['c', 'cpp']) endif diff --git a/Test/project.refactor b/Test/project.refactor index e69de29..fdfc469 100644 --- a/Test/project.refactor +++ b/Test/project.refactor @@ -0,0 +1,71 @@ +__VERSION 1 + +// not : Ignore +// include : #includes +// word : Alphanumeric or underscore +// namespace : Prefix search and replace (c-namspaces). +// regex : Unavailable in __VERSION 1. + +// Precedence (highest to lowest): +// word, namespace, regex + +// Comments +not comments + +// Includes +include zpl.h, zpl.refactored.h +include Bloat.hpp, Bloat.refactored.hpp +include Bloat.cpp, Bloat.refactored.cpp +include IO.hpp, IO.refactored.hpp +include IO.cpp, IO.refactored.cpp +include Spec.hpp, Spec.refactored.hpp +include Spec.cpp, Spec.refactored.cpp + +// Remove the zpl namespace. +namespace zpl_ + +// Don't expose zpl internals +not namespace zpl__ + +// Macro exposure +//namespace ZPL_ +//not word ZPL_IMPLEMENTATION + +// Name conflicts +word opts, options + +word zpl_strncmp, str_compare +word zpl_strcmp, str_compare + +// Undesired typedefs +word zpl_i8, s8 +word zpl_i16, s16 +word zpl_i32, s32 +word zpl_i64, s64 +word zpl_u8, u8 +word zpl_u16, u16 +word zpl_u32, u32 +word zpl_u64, u64 +word zpl_intptr, sptr +word zpl_uintptr, uptr +word zpl_usize, uw +word zpl_isize, sw + +// Conflicts with std. (Uncomment if using c externs) +not word zpl_memchr +not word zpl_memmove +not word zpl_memset +not word zpl_memswap +not word zpl_memcopy +not word zpl_printf +not word zpl_printf_va +not word zpl_printf_err +not word zpl_printf_err_va +not word zpl_fprintf +not word zpl_fprintf_va +not word zpl_snprintf +not word zpl_snprintf_va +not word zpl_strchr +not word zpl_strlen +not word zpl_strnlen +not word zpl_exit diff --git a/Test/zpl.refactor b/Test/zpl.refactor index d8737fe..55c76d1 100644 --- a/Test/zpl.refactor +++ b/Test/zpl.refactor @@ -20,6 +20,8 @@ not include header/essentials/collections/array.h not include header/essentials/collections/list.h not include header/core/file.h not include header/opts.h +not include source/core/file.c +not include source/opts.c // Removes the namespace. namespace zpl_ @@ -69,20 +71,20 @@ word opts, a_opts word pool, a_pool // Conflicts with std. (Uncomment if using c externs) -//not word zpl_memchr -//not word zpl_memmove -//not word zpl_memset -//not word zpl_memswap -//not word zpl_memcopy -//not word zpl_printf -//not word zpl_printf_va -//not word zpl_printf_err -//not word zpl_printf_err_va -//not word zpl_fprintf -//not word zpl_fprintf_va -//not word zpl_snprintf -//not word zpl_snprintf_va -//not word zpl_strchr -//not word zpl_strlen -//not word zpl_strnlen -//not word zpl_exit +not word zpl_memchr +not word zpl_memmove +not word zpl_memset +not word zpl_memswap +not word zpl_memcopy +not word zpl_printf +not word zpl_printf_va +not word zpl_printf_err +not word zpl_printf_err_va +not word zpl_fprintf +not word zpl_fprintf_va +not word zpl_snprintf +not word zpl_snprintf_va +not word zpl_strchr +not word zpl_strlen +not word zpl_strnlen +not word zpl_exit diff --git a/scripts/build.ps1 b/scripts/build.ps1 index af216d5..0815802 100644 --- a/scripts/build.ps1 +++ b/scripts/build.ps1 @@ -51,7 +51,7 @@ Start-Process ninja $args_ninja -Wait -NoNewWindow -WorkingDirectory $path_root #endregion Regular Build -if ( $test -eq $true -and (Test-Path $path_build) ) +if ( $test -eq $true ) { #region Test Build write-host "`n`nBuilding Test`n" @@ -68,12 +68,12 @@ if ( $test -eq $true -and (Test-Path $path_build) ) $args_meson += "setup" $args_meson += $path_test_build - Start-Process meson $args_meson -NoNewWindow -Wait -WorkingDirectory $path_scripts + Start-Process meson $args_meson -NoNewWindow -Wait -WorkingDirectory $path_test } $args_ninja = @() $args_ninja += "-C" - $args_ninja += $path_build + $args_ninja += $path_test_build Start-Process ninja $args_ninja -Wait -NoNewWindow -WorkingDirectory $path_test #endregion Test Build diff --git a/scripts/clean.ps1 b/scripts/clean.ps1 index d93fef2..365ca97 100644 --- a/scripts/clean.ps1 +++ b/scripts/clean.ps1 @@ -12,3 +12,10 @@ if ( Test-Path $path_test_build ) { Remove-Item $path_test_build -Recurse } + +[string[]] $include = '*.h', '*.hpp', '*.cpp' +[string[]] $exclude = + +$files = Get-ChildItem -Recurse -Path $path_test -Include $include -Exclude $exclude + +Remove-Item $files diff --git a/scripts/get_sources.ps1 b/scripts/get_sources.ps1 index 03b9889..f9d7d24 100644 --- a/scripts/get_sources.ps1 +++ b/scripts/get_sources.ps1 @@ -1,5 +1,5 @@ [string[]] $include = 'refactor.cpp' #'*.c', '*.cc', '*.cpp' -# [stirng[]] $exclude = +# [string[]] $exclude = $path_root = git rev-parse --show-toplevel $path_proj = Join-Path $path_root project diff --git a/scripts/meson.build b/scripts/meson.build index 8262cd2..a83b1e3 100644 --- a/scripts/meson.build +++ b/scripts/meson.build @@ -1,4 +1,4 @@ -project( 'refactor', 'c', 'cpp', default_options : ['buildtype=debug'] ) +project( 'refactor', 'c', 'cpp', default_options : ['buildtype=release'] ) # add_global_arguments('-E', language : 'cpp') diff --git a/scripts/refactor_and_format.ps1 b/scripts/refactor_and_format.ps1 index 515e3a4..c6465d2 100644 --- a/scripts/refactor_and_format.ps1 +++ b/scripts/refactor_and_format.ps1 @@ -1,9 +1,10 @@ [string[]] $include = '*.h', '*.hh', '*.hpp', '*.c', '*.cc', '*.cpp' -[string[]] $exclude = '*.g.*', '*.refactor', 'bloat.refactored.hpp', 'refactor.refactored.cpp' +[string[]] $exclude = '*.g.*', '*.refactor' $path_root = git rev-parse --show-toplevel $path_build = Join-Path $path_root build +$path_project = Join-Path $path_root project $path_test = Join-Path $path_root test $path_thirdparty = Join-Path $path_root thirdparty @@ -11,27 +12,48 @@ $file_spec = Join-Path $path_test zpl.refactor $refactor = Join-Path $path_build refactor.exe # Gather the files to be formatted. -$targetFiles = @(Get-ChildItem -Recurse -Path $path_thirdparty -Include $include -Exclude $exclude | Select-Object -ExpandProperty FullName) +$targetFiles = @(Get-ChildItem -Recurse -Path $path_thirdparty -Include $include -Exclude $exclude | Select-Object -ExpandProperty FullName) +$refactoredFiles = @() - -write-host "Beginning refactor...`n" - -$refactors = @(@()) - -# TODO: Change this to support refactoring the other files in the project directory. -# It needs two runs, one for the regular files, one for the zpl header. foreach ( $file in $targetFiles ) { $destination = Join-Path $path_test (Split-Path $file -leaf) $destination = $destination.Replace( '.h', '.refactored.h' ) - + $destination = $destination.Replace( '.c', '.refactored.c' ) + + $refactoredFiles += $destination +} + + +write-host "Beginning thirdpary refactor...`n" + +$refactors = @(@()) + +if ( $false ){ + foreach ( $file in $targetFiles ) + { + $destination = Join-Path $path_test (Split-Path $file -leaf) + $destination = $destination.Replace( '.h', '.refactored.h' ) + + $refactorParams = @( + "-src=$($file)", + "-dst=$($destination)" + "-spec=$($file_spec)" + ) + + $refactors += (Start-Process $refactor $refactorParams -NoNewWindow -PassThru) + } +} +else { $refactorParams = @( - "-src=$($file)", - "-dst=$($destination)" + "-debug", + "-num=$($targetFiles.Count)" + "-src=$($targetFiles)", + "-dst=$($refactoredFiles)", "-spec=$($file_spec)" ) - $refactors += (Start-Process $refactor $refactorParams -NoNewWindow -PassThru) + Start-Process $refactor $refactorParams -NoNewWindow -PassThru -Wait } foreach ( $process in $refactors ) @@ -44,10 +66,40 @@ foreach ( $process in $refactors ) Write-Host "`nRefactoring complete`n`n" +write-host "Beginning project refactor...`n" + +# Gather the files to be formatted. +$targetFiles = @(Get-ChildItem -Recurse -Path $path_project -Include $include -Exclude $exclude | Select-Object -ExpandProperty FullName) +$refactoredFiles = @() + +$file_spec = Join-Path $path_test project.refactor + +write-host "FILE SPEC:" $file_spec + +foreach ( $file in $targetFiles ) +{ + $destination = Join-Path $path_test (Split-Path $file -leaf) + $destination = $destination.Replace( '.hpp', '.refactored.hpp' ) + $destination = $destination.Replace( '.cpp', '.refactored.cpp' ) + + $refactoredFiles += $destination +} + +$refactorParams = @( + "-debug", + "-num=$($targetFiles.Count)" + "-src=$($targetFiles)", + "-dst=$($refactoredFiles)", + "-spec=$($file_spec)" +) + +Start-Process $refactor $refactorParams -NoNewWindow -PassThru -Wait + +write-host "`nRefactoring complete`n`n" + # Can't format zpl library... (It hangs clang format) -if ( $false ) -{ +if ( $false ) { Write-Host "Beginning format...`n" # Format the files.