ed/manual_slop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(mcp): Validate C++ tools against real-world gencpp components and improve enum support

Browse Source
This commit is contained in:
Edward R. Gonzalez
2026-05-05 20:40:21 -04:00
parent a809a6e213
commit 904dabe6a1
12 changed files with 6643 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
extract_samples.py
+47
View File
@@ -0,0 +1,47 @@
import os
log_path = 'logs/20260325_103727/outputs/output_0013.txt'
with open(log_path, 'r', encoding='utf-8') as f:
content = f.read()
def extract(start_pat, end_pat, out_name):
start_marker = f'### `{start_pat}`'
end_marker = f'### `{end_pat}`'
start_pos = content.find(start_marker)
if start_pos == -1:
print(f"Start marker not found for {out_name}")
return
end_pos = content.find(end_marker, start_pos)
if end_pos == -1:
print(f"End marker not found for {out_name}")
return
block = content[start_pos:end_pos]
code_start = block.find('```cpp\r\n')
if code_start == -1:
code_start = block.find('```hpp\r\n')
if code_start == -1:
code_start = block.find('```cpp\n')
if code_start == -1:
code_start = block.find('```hpp\n')
if code_start == -1:
print(f"Code start marker not found for {out_name}")
return
code_start_offset = block.find('\n', code_start) + 1
code_end = block.rfind('```')
final_code = block[code_start_offset:code_end].strip()
out_path = os.path.join('tests/assets/gencpp_samples', out_name)
os.makedirs(os.path.dirname(out_path), exist_ok=True)
with open(out_path, 'w', encoding='utf-8', newline='\r\n') as f_out:
f_out.write(final_code)
print(f"Extracted {out_name}")
extract('C:/projects/gencpp/base/components/parser.cpp', 'C:/projects/gencpp/base/components/lexer.cpp', 'parser.cpp')
extract('C:/projects/gencpp/base/components/ast.hpp', 'C:/projects/gencpp/base/components/interface.parsing.cpp', 'ast.hpp')
extract('C:/projects/gencpp/base/components/types.hpp', 'C:/projects/gencpp/base/components/interface.hpp', 'types.hpp')
src/file_cache.py
+58 -51
View File
@@ -94,15 +94,15 @@ class ASTParser:
_ast_cache[path] = (mtime, tree) _ast_cache[path] = (mtime, tree)
return tree return tree
def _get_name(self, node: tree_sitter.Node, code: str) -> str: def _get_name(self, node: tree_sitter.Node, code_bytes: bytes) -> str:
name_node = node.child_by_field_name("name") name_node = node.child_by_field_name("name")
if name_node: if name_node:
return code[name_node.start_byte:name_node.end_byte] return code_bytes[name_node.start_byte:name_node.end_byte].decode("utf8", errors="replace")
if node.type in ("function_definition", "field_declaration"): if node.type in ("function_definition", "field_declaration"):
def find_id(n: tree_sitter.Node) -> str: def find_id(n: tree_sitter.Node) -> str:
if n.type in ("identifier", "field_identifier", "qualified_identifier", "destructor_name"): if n.type in ("identifier", "field_identifier", "qualified_identifier", "destructor_name"):
return code[n.start_byte:n.end_byte] return code_bytes[n.start_byte:n.end_byte].decode("utf8", errors="replace")
# Try field name 'declarator' first # Try field name 'declarator' first
d = n.child_by_field_name("declarator") d = n.child_by_field_name("declarator")
if d: if d:
@@ -118,19 +118,20 @@ class ASTParser:
if node.type == "template_declaration": if node.type == "template_declaration":
for child in node.children: for child in node.children:
if child.type in ("function_definition", "class_definition", "class_specifier", "struct_specifier", "field_declaration"): if child.type in ("function_definition", "class_definition", "class_specifier", "struct_specifier", "enum_specifier", "enum_definition", "field_declaration"):
return self._get_name(child, code) return self._get_name(child, code_bytes)
if node.type in ("struct_specifier", "class_specifier", "class_definition", "namespace_definition"): if node.type in ("struct_specifier", "class_specifier", "class_definition", "enum_specifier", "enum_definition", "namespace_definition"):
for child in node.children: for child in node.children:
if child.type in ("type_identifier", "identifier", "namespace_identifier"): if child.type in ("type_identifier", "identifier", "namespace_identifier"):
return code[child.start_byte:child.end_byte] return code_bytes[child.start_byte:child.end_byte].decode("utf8", errors="replace")
return "" return ""
def get_skeleton(self, code: str, path: Optional[str] = None) -> str: def get_skeleton(self, code: str, path: Optional[str] = None) -> str:
""" """
Returns a skeleton of a Python file (preserving docstrings, stripping function bodies). Returns a skeleton of a Python file (preserving docstrings, stripping function bodies).
""" """
code_bytes = code.encode("utf8")
tree = self.get_cached_tree(path, code) tree = self.get_cached_tree(path, code)
edits: List[Tuple[int, int, str]] = [] edits: List[Tuple[int, int, str]] = []
@@ -170,10 +171,10 @@ class ASTParser:
walk(tree.root_node) walk(tree.root_node)
# Apply edits in reverse to maintain byte offsets # Apply edits in reverse to maintain byte offsets
edits.sort(key=lambda x: x[0], reverse=True) edits.sort(key=lambda x: x[0], reverse=True)
code_bytes = bytearray(code, "utf8") code_bytearray = bytearray(code_bytes)
for start, end, replacement in edits: for start, end, replacement in edits:
code_bytes[start:end] = bytes(replacement, "utf8") code_bytearray[start:end] = bytes(replacement, "utf8")
return code_bytes.decode("utf8") return code_bytearray.decode("utf8")
def get_curated_view(self, code: str, path: Optional[str] = None) -> str: def get_curated_view(self, code: str, path: Optional[str] = None) -> str:
""" """
@@ -181,6 +182,7 @@ class ASTParser:
Preserves function bodies if they have @core_logic decorator or # [HOT] comment. Preserves function bodies if they have @core_logic decorator or # [HOT] comment.
Otherwise strips bodies but preserves docstrings. Otherwise strips bodies but preserves docstrings.
""" """
code_bytes = code.encode("utf8")
tree = self.get_cached_tree(path, code) tree = self.get_cached_tree(path, code)
edits: List[Tuple[int, int, str]] = [] edits: List[Tuple[int, int, str]] = []
@@ -197,7 +199,7 @@ class ASTParser:
for child in parent.children: for child in parent.children:
if child.type == "decorator": if child.type == "decorator":
# decorator -> ( '@', identifier ) or ( '@', call ) # decorator -> ( '@', identifier ) or ( '@', call )
if "@core_logic" in code[child.start_byte:child.end_byte]: if b"@core_logic" in code_bytes[child.start_byte:child.end_byte]:
return True return True
return False return False
@@ -207,8 +209,8 @@ class ASTParser:
while stack: while stack:
curr = stack.pop() curr = stack.pop()
if curr.type == "comment": if curr.type == "comment":
comment_text = code[curr.start_byte:curr.end_byte] comment_bytes = code_bytes[curr.start_byte:curr.end_byte]
if "[HOT]" in comment_text: if b"[HOT]" in comment_bytes:
return True return True
for child in curr.children: for child in curr.children:
stack.append(child) stack.append(child)
@@ -241,16 +243,17 @@ class ASTParser:
walk(tree.root_node) walk(tree.root_node)
# Apply edits in reverse to maintain byte offsets # Apply edits in reverse to maintain byte offsets
edits.sort(key=lambda x: x[0], reverse=True) edits.sort(key=lambda x: x[0], reverse=True)
code_bytes = bytearray(code, "utf8") code_bytearray = bytearray(code_bytes)
for start, end, replacement in edits: for start, end, replacement in edits:
code_bytes[start:end] = bytes(replacement, "utf8") code_bytearray[start:end] = bytes(replacement, "utf8")
return code_bytes.decode("utf8") return code_bytearray.decode("utf8")
def get_targeted_view(self, code: str, function_names: List[str], path: Optional[str] = None) -> str: def get_targeted_view(self, code: str, function_names: List[str], path: Optional[str] = None) -> str:
""" """
Returns a targeted view of the code including only the specified functions Returns a targeted view of the code including only the specified functions
and their dependencies up to depth 2. and their dependencies up to depth 2.
""" """
code_bytes = code.encode("utf8")
tree = self.get_cached_tree(path, code) tree = self.get_cached_tree(path, code)
all_functions = {} all_functions = {}
@@ -258,13 +261,13 @@ class ASTParser:
if node.type == "function_definition": if node.type == "function_definition":
name_node = node.child_by_field_name("name") name_node = node.child_by_field_name("name")
if name_node: if name_node:
func_name = code[name_node.start_byte:name_node.end_byte] func_name = code_bytes[name_node.start_byte:name_node.end_byte].decode("utf8", errors="replace")
full_name = f"{class_name}.{func_name}" if class_name else func_name full_name = f"{class_name}.{func_name}" if class_name else func_name
all_functions[full_name] = node all_functions[full_name] = node
elif node.type == "class_definition": elif node.type == "class_definition":
name_node = node.child_by_field_name("name") name_node = node.child_by_field_name("name")
if name_node: if name_node:
cname = code[name_node.start_byte:name_node.end_byte] cname = code_bytes[name_node.start_byte:name_node.end_byte].decode("utf8", errors="replace")
full_cname = f"{class_name}.{cname}" if class_name else cname full_cname = f"{class_name}.{cname}" if class_name else cname
body = node.child_by_field_name("body") body = node.child_by_field_name("body")
if body: if body:
@@ -282,11 +285,11 @@ class ASTParser:
func_node = n.child_by_field_name("function") func_node = n.child_by_field_name("function")
if func_node: if func_node:
if func_node.type == "identifier": if func_node.type == "identifier":
calls.add(code[func_node.start_byte:func_node.end_byte]) calls.add(code_bytes[func_node.start_byte:func_node.end_byte].decode("utf8", errors="replace"))
elif func_node.type == "attribute": elif func_node.type == "attribute":
attr_node = func_node.child_by_field_name("attribute") attr_node = func_node.child_by_field_name("attribute")
if attr_node: if attr_node:
calls.add(code[attr_node.start_byte:attr_node.end_byte]) calls.add(code_bytes[attr_node.start_byte:attr_node.end_byte].decode("utf8", errors="replace"))
for child in n.children: for child in n.children:
walk_calls(child) walk_calls(child)
walk_calls(node) walk_calls(node)
@@ -329,12 +332,12 @@ class ASTParser:
def check_for_targeted(node, parent_class=None): def check_for_targeted(node, parent_class=None):
if node.type == "function_definition": if node.type == "function_definition":
name_node = node.child_by_field_name("name") name_node = node.child_by_field_name("name")
fname = code[name_node.start_byte:name_node.end_byte] if name_node else "" fname = code_bytes[name_node.start_byte:name_node.end_byte].decode("utf8", errors="replace") if name_node else ""
fullname = f"{parent_class}.{fname}" if parent_class else fname fullname = f"{parent_class}.{fname}" if parent_class else fname
return fullname in all_found return fullname in all_found
if node.type == "class_definition": if node.type == "class_definition":
name_node = node.child_by_field_name("name") name_node = node.child_by_field_name("name")
cname = code[name_node.start_byte:name_node.end_byte] if name_node else "" cname = code_bytes[name_node.start_byte:name_node.end_byte].decode("utf8", errors="replace") if name_node else ""
full_cname = f"{parent_class}.{cname}" if parent_class else cname full_cname = f"{parent_class}.{cname}" if parent_class else cname
body = node.child_by_field_name("body") body = node.child_by_field_name("body")
if body: if body:
@@ -350,7 +353,7 @@ class ASTParser:
def walk_edits(node, parent_class=None): def walk_edits(node, parent_class=None):
if node.type == "function_definition": if node.type == "function_definition":
name_node = node.child_by_field_name("name") name_node = node.child_by_field_name("name")
fname = code[name_node.start_byte:name_node.end_byte] if name_node else "" fname = code_bytes[name_node.start_byte:name_node.end_byte].decode("utf8", errors="replace") if name_node else ""
fullname = f"{parent_class}.{fname}" if parent_class else fname fullname = f"{parent_class}.{fname}" if parent_class else fname
if fullname in all_found: if fullname in all_found:
body = node.child_by_field_name("body") body = node.child_by_field_name("body")
@@ -376,7 +379,7 @@ class ASTParser:
if node.type == "class_definition": if node.type == "class_definition":
if check_for_targeted(node, parent_class): if check_for_targeted(node, parent_class):
name_node = node.child_by_field_name("name") name_node = node.child_by_field_name("name")
cname = code[name_node.start_byte:name_node.end_byte] if name_node else "" cname = code_bytes[name_node.start_byte:name_node.end_byte].decode("utf8", errors="replace") if name_node else ""
full_cname = f"{parent_class}.{cname}" if parent_class else cname full_cname = f"{parent_class}.{cname}" if parent_class else cname
body = node.child_by_field_name("body") body = node.child_by_field_name("body")
if body: if body:
@@ -400,10 +403,10 @@ class ASTParser:
walk_edits(tree.root_node) walk_edits(tree.root_node)
edits.sort(key=lambda x: x[0], reverse=True) edits.sort(key=lambda x: x[0], reverse=True)
code_bytes = bytearray(code, "utf8") code_bytearray = bytearray(code_bytes)
for start, end, replacement in edits: for start, end, replacement in edits:
code_bytes[start:end] = bytes(replacement, "utf8") code_bytearray[start:end] = bytes(replacement, "utf8")
result = code_bytes.decode("utf8") result = code_bytearray.decode("utf8")
result = re.sub(r'\n\s*\n\s*\n+', '\n\n', result) result = re.sub(r'\n\s*\n\s*\n+', '\n\n', result)
return result.strip() + "\n" return result.strip() + "\n"
@@ -412,6 +415,7 @@ class ASTParser:
Returns the full source code for a specific definition by name. Returns the full source code for a specific definition by name.
Supports 'ClassName::method' or 'method' for C++. Supports 'ClassName::method' or 'method' for C++.
""" """
code_bytes = code.encode("utf8")
tree = self.get_cached_tree(path, code) tree = self.get_cached_tree(path, code)
parts = re.split(r'::|\.', name) parts = re.split(r'::|\.', name)
@@ -429,9 +433,9 @@ class ASTParser:
check_node = sub check_node = sub
break break
is_interesting = check_node.type in ("function_definition", "class_definition", "class_specifier", "struct_specifier", "namespace_definition", "template_declaration", "field_declaration") is_interesting = check_node.type in ("function_definition", "class_definition", "class_specifier", "struct_specifier", "enum_specifier", "enum_definition", "namespace_definition", "template_declaration", "field_declaration")
if is_interesting: if is_interesting:
node_name = self._get_name(check_node, code) node_name = self._get_name(check_node, code_bytes)
if node_name == target: if node_name == target:
if len(target_parts) == 1: if len(target_parts) == 1:
return check_node if child.type != "field_declaration" else child return check_node if child.type != "field_declaration" else child
@@ -442,14 +446,14 @@ class ASTParser:
body = check_node.child_by_field_name("body") body = check_node.child_by_field_name("body")
if not body and check_node.type == "template_declaration": if not body and check_node.type == "template_declaration":
for sub in check_node.children: for sub in check_node.children:
if sub.type in ("function_definition", "class_definition", "class_specifier", "struct_specifier"): if sub.type in ("function_definition", "class_definition", "class_specifier", "struct_specifier", "enum_specifier", "enum_definition"):
body = sub.child_by_field_name("body") body = sub.child_by_field_name("body")
break break
if body: if body:
found = walk(body, next_parts) found = walk(body, next_parts)
if found: return found if found: return found
for sub in check_node.children: for sub in check_node.children:
if sub.type in ("field_declaration_list", "class_body", "declaration_list"): if sub.type in ("field_declaration_list", "class_body", "declaration_list", "enum_body"):
found = walk(sub, next_parts) found = walk(sub, next_parts)
if found: return found if found: return found
elif child.type in ("module", "translation_unit", "namespace_definition", "declaration_list", "field_declaration_list", "class_body"): elif child.type in ("module", "translation_unit", "namespace_definition", "declaration_list", "field_declaration_list", "class_body"):
@@ -458,8 +462,8 @@ class ASTParser:
return None return None
def deep_search(node: tree_sitter.Node, target: str) -> Optional[tree_sitter.Node]: def deep_search(node: tree_sitter.Node, target: str) -> Optional[tree_sitter.Node]:
if node.type in ("function_definition", "class_definition", "class_specifier", "struct_specifier", "namespace_definition", "template_declaration"): if node.type in ("function_definition", "class_definition", "class_specifier", "struct_specifier", "enum_specifier", "enum_definition", "namespace_definition", "template_declaration"):
if self._get_name(node, code) == target: if self._get_name(node, code_bytes) == target:
return node return node
for child in node.children: for child in node.children:
res = deep_search(child, target) res = deep_search(child, target)
@@ -471,7 +475,7 @@ class ASTParser:
found_node = deep_search(tree.root_node, name) found_node = deep_search(tree.root_node, name)
if found_node: if found_node:
return code[found_node.start_byte:found_node.end_byte] return code_bytes[found_node.start_byte:found_node.end_byte].decode("utf8", errors="replace")
return f"ERROR: definition '{name}' not found" return f"ERROR: definition '{name}' not found"
def get_signature(self, code: str, name: str, path: Optional[str] = None) -> str: def get_signature(self, code: str, name: str, path: Optional[str] = None) -> str:
@@ -479,6 +483,7 @@ class ASTParser:
Returns only the signature part of a function or method. Returns only the signature part of a function or method.
For C/C++, this is the code from the start of the definition until the block start '{'. For C/C++, this is the code from the start of the definition until the block start '{'.
""" """
code_bytes = code.encode("utf8")
tree = self.get_cached_tree(path, code) tree = self.get_cached_tree(path, code)
parts = re.split(r'::|\.', name) parts = re.split(r'::|\.', name)
@@ -496,9 +501,9 @@ class ASTParser:
check_node = sub check_node = sub
break break
is_interesting = check_node.type in ("function_definition", "class_definition", "class_specifier", "struct_specifier", "namespace_definition", "template_declaration", "field_declaration") is_interesting = check_node.type in ("function_definition", "class_definition", "class_specifier", "struct_specifier", "enum_specifier", "enum_definition", "namespace_definition", "template_declaration", "field_declaration")
if is_interesting: if is_interesting:
node_name = self._get_name(check_node, code) node_name = self._get_name(check_node, code_bytes)
if node_name == target: if node_name == target:
if len(target_parts) == 1: if len(target_parts) == 1:
return check_node if child.type != "field_declaration" else child return check_node if child.type != "field_declaration" else child
@@ -509,14 +514,14 @@ class ASTParser:
body = check_node.child_by_field_name("body") body = check_node.child_by_field_name("body")
if not body and check_node.type == "template_declaration": if not body and check_node.type == "template_declaration":
for sub in check_node.children: for sub in check_node.children:
if sub.type in ("function_definition", "class_definition", "class_specifier", "struct_specifier"): if sub.type in ("function_definition", "class_definition", "class_specifier", "struct_specifier", "enum_specifier", "enum_definition"):
body = sub.child_by_field_name("body") body = sub.child_by_field_name("body")
break break
if body: if body:
found = walk(body, next_parts) found = walk(body, next_parts)
if found: return found if found: return found
for sub in check_node.children: for sub in check_node.children:
if sub.type in ("field_declaration_list", "class_body", "declaration_list"): if sub.type in ("field_declaration_list", "class_body", "declaration_list", "enum_body"):
found = walk(sub, next_parts) found = walk(sub, next_parts)
if found: return found if found: return found
elif child.type in ("module", "translation_unit", "namespace_definition", "declaration_list", "field_declaration_list", "class_body"): elif child.type in ("module", "translation_unit", "namespace_definition", "declaration_list", "field_declaration_list", "class_body"):
@@ -526,7 +531,7 @@ class ASTParser:
def deep_search(node: tree_sitter.Node, target: str) -> Optional[tree_sitter.Node]: def deep_search(node: tree_sitter.Node, target: str) -> Optional[tree_sitter.Node]:
if node.type in ("function_definition", "template_declaration"): if node.type in ("function_definition", "template_declaration"):
if self._get_name(node, code) == target: if self._get_name(node, code_bytes) == target:
return node return node
for child in node.children: for child in node.children:
res = deep_search(child, target) res = deep_search(child, target)
@@ -547,8 +552,8 @@ class ASTParser:
body = target_node.child_by_field_name("body") body = target_node.child_by_field_name("body")
if body: if body:
return code[found_node.start_byte:body.start_byte].strip() return code_bytes[found_node.start_byte:body.start_byte].decode("utf8", errors="replace").strip()
return code[found_node.start_byte:found_node.end_byte].strip() return code_bytes[found_node.start_byte:found_node.end_byte].decode("utf8", errors="replace").strip()
return f"ERROR: signature for '{name}' not found" return f"ERROR: signature for '{name}' not found"
@@ -556,6 +561,7 @@ class ASTParser:
""" """
Returns a hierarchical outline of the code (classes, structs, functions, methods). Returns a hierarchical outline of the code (classes, structs, functions, methods).
""" """
code_bytes = code.encode("utf8")
tree = self.get_cached_tree(path, code) tree = self.get_cached_tree(path, code)
output = [] output = []
@@ -570,7 +576,7 @@ class ASTParser:
label = "[Method]" if indent > 0 else "[Func]" label = "[Method]" if indent > 0 else "[Func]"
if label: if label:
name = self._get_name(node, code) name = self._get_name(node, code_bytes)
if name: if name:
start = node.start_point.row + 1 start = node.start_point.row + 1
end = node.end_point.row + 1 end = node.end_point.row + 1
@@ -591,6 +597,7 @@ class ASTParser:
""" """
Surgically replace the definition of a class or function by name. Surgically replace the definition of a class or function by name.
""" """
code_bytes = code.encode("utf8")
tree = self.get_cached_tree(path, code) tree = self.get_cached_tree(path, code)
parts = re.split(r'::|\.', name) parts = re.split(r'::|\.', name)
@@ -608,9 +615,9 @@ class ASTParser:
check_node = sub check_node = sub
break break
is_interesting = check_node.type in ("function_definition", "class_definition", "class_specifier", "struct_specifier", "namespace_definition", "template_declaration", "field_declaration") is_interesting = check_node.type in ("function_definition", "class_definition", "class_specifier", "struct_specifier", "enum_specifier", "enum_definition", "namespace_definition", "template_declaration", "field_declaration")
if is_interesting: if is_interesting:
node_name = self._get_name(check_node, code) node_name = self._get_name(check_node, code_bytes)
if node_name == target: if node_name == target:
if len(target_parts) == 1: if len(target_parts) == 1:
return check_node if child.type != "field_declaration" else child return check_node if child.type != "field_declaration" else child
@@ -621,14 +628,14 @@ class ASTParser:
body = check_node.child_by_field_name("body") body = check_node.child_by_field_name("body")
if not body and check_node.type == "template_declaration": if not body and check_node.type == "template_declaration":
for sub in check_node.children: for sub in check_node.children:
if sub.type in ("function_definition", "class_definition", "class_specifier", "struct_specifier"): if sub.type in ("function_definition", "class_definition", "class_specifier", "struct_specifier", "enum_specifier", "enum_definition"):
body = sub.child_by_field_name("body") body = sub.child_by_field_name("body")
break break
if body: if body:
found = walk(body, next_parts) found = walk(body, next_parts)
if found: return found if found: return found
for sub in check_node.children: for sub in check_node.children:
if sub.type in ("field_declaration_list", "class_body", "declaration_list"): if sub.type in ("field_declaration_list", "class_body", "declaration_list", "enum_body"):
found = walk(sub, next_parts) found = walk(sub, next_parts)
if found: return found if found: return found
elif child.type in ("module", "translation_unit", "namespace_definition", "declaration_list", "field_declaration_list", "class_body"): elif child.type in ("module", "translation_unit", "namespace_definition", "declaration_list", "field_declaration_list", "class_body"):
@@ -637,8 +644,8 @@ class ASTParser:
return None return None
def deep_search(node: tree_sitter.Node, target: str) -> Optional[tree_sitter.Node]: def deep_search(node: tree_sitter.Node, target: str) -> Optional[tree_sitter.Node]:
if node.type in ("function_definition", "class_definition", "class_specifier", "struct_specifier", "namespace_definition", "template_declaration"): if node.type in ("function_definition", "class_definition", "class_specifier", "struct_specifier", "enum_specifier", "enum_definition", "namespace_definition", "template_declaration"):
if self._get_name(node, code) == target: if self._get_name(node, code_bytes) == target:
return node return node
for child in node.children: for child in node.children:
res = deep_search(child, target) res = deep_search(child, target)
@@ -650,9 +657,9 @@ class ASTParser:
found_node = deep_search(tree.root_node, name) found_node = deep_search(tree.root_node, name)
if found_node: if found_node:
code_bytes = bytearray(code, "utf8") code_bytearray = bytearray(code_bytes)
code_bytes[found_node.start_byte:found_node.end_byte] = bytes(new_content, "utf8") code_bytearray[found_node.start_byte:found_node.end_byte] = bytes(new_content, "utf8")
return code_bytes.decode("utf8") return code_bytearray.decode("utf8")
return f"ERROR: definition '{name}' not found" return f"ERROR: definition '{name}' not found"
def reset_client() -> None: def reset_client() -> None:
tests/assets/gencpp_samples/base_component.h → tests/assets/cpp_samples/base_component.h
View File
tests/assets/gencpp_samples/complex_template.h → tests/assets/cpp_samples/complex_template.h
View File
tests/assets/gencpp_samples/component_registry.cpp → tests/assets/cpp_samples/component_registry.cpp
View File
tests/assets/gencpp_samples/component_registry.h → tests/assets/cpp_samples/component_registry.h
View File
tests/assets/gencpp_samples/verify_samples.py → tests/assets/cpp_samples/verify_samples.py
View File
tests/assets/gencpp_samples/ast.hpp
+457
View File
@@ -0,0 +1,457 @@
#ifdef INTELLISENSE_DIRECTIVES
#pragma once
#include "parser_types.hpp"
#endif
/*
______ ______ ________ __ __ ______ __
/ \ / \| \ | \ | \ / \ | \
| ▓▓▓▓▓▓\ ▓▓▓▓▓▓\\▓▓▓▓▓▓▓▓ | ▓▓\ | ▓▓ | ▓▓▓▓▓▓\ ______ ____| ▓▓ ______
| ▓▓__| ▓▓ ▓▓___\▓▓ | ▓▓ | ▓▓▓\| ▓▓ | ▓▓ \▓▓/ \ / ▓▓/ \
| ▓▓ ▓▓\▓▓ \ | ▓▓ | ▓▓▓▓\ ▓▓ | ▓▓ | ▓▓▓▓▓▓\ ▓▓▓▓▓▓▓ ▓▓▓▓▓▓\
| ▓▓▓▓▓▓▓▓_\▓▓▓▓▓▓\ | ▓▓ | ▓▓\▓▓ ▓▓ | ▓▓ __| ▓▓ | ▓▓ ▓▓ | ▓▓ ▓▓ ▓▓
| ▓▓ | ▓▓ \__| ▓▓ | ▓▓ | ▓▓ \▓▓▓▓ | ▓▓__/ \ ▓▓__/ ▓▓ ▓▓__| ▓▓ ▓▓▓▓▓▓▓▓
| ▓▓ | ▓▓\▓▓ ▓▓ | ▓▓ | ▓▓ \▓▓▓ \▓▓ ▓▓\▓▓ ▓▓\▓▓ ▓▓\▓▓ \
\▓▓ \▓▓ \▓▓▓▓▓▓ \▓▓ \▓▓ \▓▓ \▓▓▓▓▓▓ \▓▓▓▓▓▓ \▓▓▓▓▓▓▓ \▓▓▓▓▓▓▓
*/
struct AST;
struct AST_Body;
struct AST_Attributes;
struct AST_Comment;
struct AST_Constructor;
// struct AST_BaseClass;
struct AST_Class;
struct AST_Define;
struct AST_DefineParams;
struct AST_Destructor;
struct AST_Enum;
struct AST_Exec;
struct AST_Extern;
struct AST_Include;
struct AST_Friend;
struct AST_Fn;
struct AST_Module;
struct AST_NS;
struct AST_Operator;
struct AST_OpCast;
struct AST_Params;
struct AST_Pragma;
struct AST_PreprocessCond;
struct AST_Specifiers;
#ifdef GEN_EXECUTION_EXPRESSION_SUPPORT
struct AST_Expr;
struct AST_Expr_Assign;
struct AST_Expr_Alignof;
struct AST_Expr_Binary;
struct AST_Expr_CStyleCast;
struct AST_Expr_FunctionalCast;
struct AST_Expr_CppCast;
struct AST_Expr_ProcCall;
struct AST_Expr_Decltype;
struct AST_Expr_Comma; // TODO(Ed) : This is a binary op not sure if it needs its own AST...
struct AST_Expr_AMS; // Access Member Symbol
struct AST_Expr_Sizeof;
struct AST_Expr_Subscript;
struct AST_Expr_Ternary;
struct AST_Expr_UnaryPrefix;
struct AST_Expr_UnaryPostfix;
struct AST_Expr_Element;
struct AST_Stmt;
struct AST_Stmt_Break;
struct AST_Stmt_Case;
struct AST_Stmt_Continue;
struct AST_Stmt_Decl;
struct AST_Stmt_Do;
struct AST_Stmt_Expr; // TODO(Ed) : Is this distinction needed? (Should it be a flag instead?)
struct AST_Stmt_Else;
struct AST_Stmt_If;
struct AST_Stmt_For;
struct AST_Stmt_Goto;
struct AST_Stmt_Label;
struct AST_Stmt_Switch;
struct AST_Stmt_While;
#endif
struct AST_Struct;
struct AST_Template;
struct AST_Typename;
struct AST_Typedef;
struct AST_Union;
struct AST_Using;
struct AST_Var;
#if GEN_COMPILER_C
typedef AST* Code;
#else
struct Code;
#endif
#if GEN_COMPILER_C
typedef AST_Body* CodeBody;
typedef AST_Attributes* CodeAttributes;
typedef AST_Comment* CodeComment;
typedef AST_Class* CodeClass;
typedef AST_Constructor* CodeConstructor;
typedef AST_Define* CodeDefine;
typedef AST_DefineParams* CodeDefineParams;
typedef AST_Destructor* CodeDestructor;
typedef AST_Enum* CodeEnum;
typedef AST_Exec* CodeExec;
typedef AST_Extern* CodeExtern;
typedef AST_Include* CodeInclude;
typedef AST_Friend* CodeFriend;
typedef AST_Fn* CodeFn;
typedef AST_Module* CodeModule;
typedef AST_NS* CodeNS;
typedef AST_Operator* CodeOperator;
typedef AST_OpCast* CodeOpCast;
typedef AST_Params* CodeParams;
typedef AST_PreprocessCond* CodePreprocessCond;
typedef AST_Pragma* CodePragma;
typedef AST_Specifiers* CodeSpecifiers;
#else
struct CodeBody;
struct CodeAttributes;
struct CodeComment;
struct CodeClass;
struct CodeConstructor;
struct CodeDefine;
struct CodeDefineParams;
struct CodeDestructor;
struct CodeEnum;
struct CodeExec;
struct CodeExtern;
struct CodeInclude;
struct CodeFriend;
struct CodeFn;
struct CodeModule;
struct CodeNS;
struct CodeOperator;
struct CodeOpCast;
struct CodeParams;
struct CodePreprocessCond;
struct CodePragma;
struct CodeSpecifiers;
#endif
#ifdef GEN_EXECUTION_EXPRESSION_SUPPORT
#if GEN_COMPILER_C
typedef AST_Expr* CodeExpr;
typedef AST_Expr_Assign* CodeExpr_Assign;
typedef AST_Expr_Alignof* CodeExpr_Alignof;
typedef AST_Expr_Binary* CodeExpr_Binary;
typedef AST_Expr_CStyleCast* CodeExpr_CStyleCast;
typedef AST_Expr_FunctionalCast* CodeExpr_FunctionalCast;
typedef AST_Expr_CppCast* CodeExpr_CppCast;
typedef AST_Expr_Element* CodeExpr_Element;
typedef AST_Expr_ProcCall* CodeExpr_ProcCall;
typedef AST_Expr_Decltype* CodeExpr_Decltype;
typedef AST_Expr_Comma* CodeExpr_Comma;
typedef AST_Expr_AMS* CodeExpr_AMS; // Access Member Symbol
typedef AST_Expr_Sizeof* CodeExpr_Sizeof;
typedef AST_Expr_Subscript* CodeExpr_Subscript;
typedef AST_Expr_Ternary* CodeExpr_Ternary;
typedef AST_Expr_UnaryPrefix* CodeExpr_UnaryPrefix;
typedef AST_Expr_UnaryPostfix* CodeExpr_UnaryPostfix;
#else
struct CodeExpr;
struct CodeExpr_Assign;
struct CodeExpr_Alignof;
struct CodeExpr_Binary;
struct CodeExpr_CStyleCast;
struct CodeExpr_FunctionalCast;
struct CodeExpr_CppCast;
struct CodeExpr_Element;
struct CodeExpr_ProcCall;
struct CodeExpr_Decltype;
struct CodeExpr_Comma;
struct CodeExpr_AMS; // Access Member Symbol
struct CodeExpr_Sizeof;
struct CodeExpr_Subscript;
struct CodeExpr_Ternary;
struct CodeExpr_UnaryPrefix;
struct CodeExpr_UnaryPostfix;
#endif
#if GEN_COMPILER_C
typedef AST_Stmt* CodeStmt;
typedef AST_Stmt_Break* CodeStmt_Break;
typedef AST_Stmt_Case* CodeStmt_Case;
typedef AST_Stmt_Continue* CodeStmt_Continue;
typedef AST_Stmt_Decl* CodeStmt_Decl;
typedef AST_Stmt_Do* CodeStmt_Do;
typedef AST_Stmt_Expr* CodeStmt_Expr;
typedef AST_Stmt_Else* CodeStmt_Else;
typedef AST_Stmt_If* CodeStmt_If;
typedef AST_Stmt_For* CodeStmt_For;
typedef AST_Stmt_Goto* CodeStmt_Goto;
typedef AST_Stmt_Label* CodeStmt_Label;
typedef AST_Stmt_Lambda* CodeStmt_Lambda;
typedef AST_Stmt_Switch* CodeStmt_Switch;
typedef AST_Stmt_While* CodeStmt_While;
#else
struct CodeStmt;
struct CodeStmt_Break;
struct CodeStmt_Case;
struct CodeStmt_Continue;
struct CodeStmt_Decl;
struct CodeStmt_Do;
struct CodeStmt_Expr;
struct CodeStmt_Else;
struct CodeStmt_If;
struct CodeStmt_For;
struct CodeStmt_Goto;
struct CodeStmt_Label;
struct CodeStmt_Lambda;
struct CodeStmt_Switch;
struct CodeStmt_While;
#endif
// GEN_EXECUTION_EXPRESSION_SUPPORT
#endif
#if GEN_COMPILER_C
typedef AST_Struct* CodeStruct;
typedef AST_Template* CodeTemplate;
typedef AST_Typename* CodeTypename;
typedef AST_Typedef* CodeTypedef;
typedef AST_Union* CodeUnion;
typedef AST_Using* CodeUsing;
typedef AST_Var* CodeVar;
#else
struct CodeStruct;
struct CodeTemplate;
struct CodeTypename;
struct CodeTypedef;
struct CodeUnion;
struct CodeUsing;
struct CodeVar;
#endif
#if GEN_COMPILER_CPP
template< class Type> forceinline Type tmpl_cast( Code self ) { return * rcast( Type*, & self ); }
#endif
#pragma region Code C-Interface
void code_append (Code code, Code other );
GEN_API Str code_debug_str (Code code);
GEN_API Code code_duplicate (Code code);
Code* code_entry (Code code, u32 idx );
bool code_has_entries (Code code);
bool code_is_body (Code code);
GEN_API bool code_is_equal (Code code, Code other);
bool code_is_valid (Code code);
void code_set_global (Code code);
GEN_API StrBuilder code_to_strbuilder (Code self );
GEN_API void code_to_strbuilder_ref(Code self, StrBuilder* result );
Str code_type_str (Code self );
GEN_API bool code_validate_body (Code self );
#pragma endregion Code C-Interface
#if GEN_COMPILER_CPP
/*
AST* wrapper
- Not constantly have to append the '*' as this is written often..
- Allows for implicit conversion to any of the ASTs (raw or filtered).
*/
struct Code
{
AST* ast;
# define Using_Code( Typename ) \
forceinline Str debug_str() { return code_debug_str(* this); } \
forceinline Code duplicate() { return code_duplicate(* this); } \
forceinline bool is_equal( Code other ) { return code_is_equal(* this, other); } \
forceinline bool is_body() { return code_is_body(* this); } \
forceinline bool is_valid() { return code_is_valid(* this); } \
forceinline void set_global() { return code_set_global(* this); }
# define Using_CodeOps( Typename ) \
forceinline Typename& operator = ( Code other ); \
forceinline bool operator ==( Code other ) { return (AST*)ast == other.ast; } \
forceinline bool operator !=( Code other ) { return (AST*)ast != other.ast; } \
forceinline bool operator ==(std::nullptr_t) const { return ast == nullptr; } \
forceinline bool operator !=(std::nullptr_t) const { return ast != nullptr; } \
operator bool();
#if ! GEN_C_LIKE_CPP
Using_Code( Code );
forceinline void append(Code other) { return code_append(* this, other); }
forceinline Code* entry(u32 idx) { return code_entry(* this, idx); }
forceinline bool has_entries() { return code_has_entries(* this); }
forceinline StrBuilder to_strbuilder() { return code_to_strbuilder(* this); }
forceinline void to_strbuilder(StrBuilder& result) { return code_to_strbuilder_ref(* this, & result); }
forceinline Str type_str() { return code_type_str(* this); }
forceinline bool validate_body() { return code_validate_body(*this); }
#endif
Using_CodeOps( Code );
forceinline Code operator *() { return * this; } // Required to support for-range iteration.
forceinline AST* operator ->() { return ast; }
Code& operator ++();
#ifdef GEN_ENFORCE_STRONG_CODE_TYPES
# define operator explicit operator
#endif
operator CodeBody() const;
operator CodeAttributes() const;
// operator CodeBaseClass() const;
operator CodeComment() const;
operator CodeClass() const;
operator CodeConstructor() const;
operator CodeDefine() const;
operator CodeDefineParams() const;
operator CodeDestructor() const;
operator CodeExec() const;
operator CodeEnum() const;
operator CodeExtern() const;
operator CodeInclude() const;
operator CodeFriend() const;
operator CodeFn() const;
operator CodeModule() const;
operator CodeNS() const;
operator CodeOperator() const;
operator CodeOpCast() const;
operator CodeParams() const;
operator CodePragma() const;
operator CodePreprocessCond() const;
operator CodeSpecifiers() const;
operator CodeStruct() const;
operator CodeTemplate() const;
operator CodeTypename() const;
operator CodeTypedef() const;
operator CodeUnion() const;
operator CodeUsing() const;
operator CodeVar() const;
#undef operator
};
#endif
#pragma region Statics
// Used to identify ASTs that should always be duplicated. (Global constant ASTs)
GEN_API extern Code Code_Global;
// Used to identify invalid generated code.
GEN_API extern Code Code_Invalid;
#pragma endregion Statics
struct Code_POD
{
AST* ast;
};
static_assert( sizeof(Code) == sizeof(Code_POD), "ERROR: Code is not POD" );
// Desired width of the AST data structure.
constexpr int const AST_POD_Size = 128;
constexpr static
int AST_ArrSpecs_Cap =
(
AST_POD_Size
- sizeof(Code)
- sizeof(StrCached)
- sizeof(Code) * 2
- sizeof(Token*)
- sizeof(Code)
- sizeof(CodeType)
- sizeof(ModuleFlag)
- sizeof(u32)
)
/ sizeof(Specifier) - 1;
/*
Simple AST POD with functionality to seralize into C++ syntax.
TODO(Ed): Eventually haven't a transparent AST like this will longer be viable once statements & expressions are in (most likely....)
*/
struct AST
{
union {
struct
{
Code InlineCmt; // Class, Constructor, Destructor, Enum, Friend, Functon, Operator, OpCast, Struct, Typedef, Using, Variable
Code Attributes; // Class, Enum, Function, Struct, Typedef, Union, Using, Variable // TODO(Ed): Parameters can have attributes
Code Specs; // Class, Destructor, Function, Operator, Struct, Typename, Variable
union {
Code InitializerList; // Constructor
Code ParentType; // Class, Struct, ParentType->Next has a possible list of interfaces.
Code ReturnType; // Function, Operator, Typename
Code UnderlyingType; // Enum, Typedef
Code ValueType; // Parameter, Variable
};
union {
Code Macro; // Parameter
Code BitfieldSize; // Variable (Class/Struct Data Member)
Code Params; // Constructor, Define, Function, Operator, Template, Typename
Code UnderlyingTypeMacro; // Enum
};
union {
Code ArrExpr; // Typename
Code Body; // Class, Constructor, Define, Destructor, Enum, Friend, Function, Namespace, Struct, Union
Code Declaration; // Friend, Template
Code Value; // Parameter, Variable
};
union {
Code NextVar; // Variable
Code SuffixSpecs; // Typename, Function (Thanks Unreal)
Code PostNameMacro; // Only used with parameters for specifically UE_REQUIRES (Thanks Unreal)
};
};
StrCached Content; // Attributes, Comment, Execution, Include
TokenSlice ContentToks; // TODO(Ed): Use a token slice for content
struct {
Specifier ArrSpecs[AST_ArrSpecs_Cap]; // Specifiers
Code NextSpecs; // Specifiers; If ArrSpecs is full, then NextSpecs is used.
};
};
StrCached Name;
union {
Code Prev;
Code Front;
Code Last;
};
union {
Code Next;
Code Back;
};
Token* Token; // Reference to starting token, only available if it was derived from parsing. // TODO(Ed): Change this to a token slice.
Code Parent;
CodeType Type;
// CodeFlag CodeFlags;
ModuleFlag ModuleFlags;
union {
b32 IsFunction; // Used by typedef to not serialize the name field.
struct {
b16 IsParamPack; // Used by typename to know if type should be considered a parameter pack.
ETypenameTag TypeTag; // Used by typename to keep track of explicitly declared tags for the identifier (enum, struct, union)
};
Operator Op;
AccessSpec ParentAccess;
s32 NumEntries;
s32 VarParenthesizedInit; // Used by variables to know that initialization is using a constructor expression instead of an assignment expression.
};
};
static_assert( sizeof(AST) == AST_POD_Size, "ERROR: AST is not size of AST_POD_Size" );
#if GEN_COMPILER_CPP
// Uses an implicitly overloaded cast from the AST to the desired code type.
// Necessary if the user wants GEN_ENFORCE_STRONG_CODE_TYPES
struct InvalidCode_ImplictCaster;
#define InvalidCode (InvalidCode_ImplictCaster{})
#else
#define InvalidCode (void*){ (void*)Code_Invalid }
#endif
#if GEN_COMPILER_CPP
struct NullCode_ImplicitCaster;
// Used when the its desired when omission is allowed in a definition.
#define NullCode (NullCode_ImplicitCaster{})
#else
#define NullCode nullptr
#endif
tests/assets/gencpp_samples/parser.cpp
+5784
View File
File diff suppressed because it is too large Load Diff
tests/assets/gencpp_samples/types.hpp
+181
View File
@@ -0,0 +1,181 @@
#ifdef INTELLISENSE_DIRECTIVES
#pragma once
#include "dependencies/platform.hpp"
#include "dependencies/macros.hpp"
#include "dependencies/basic_types.hpp"
#include "dependencies/debug.hpp"
#include "dependencies/memory.hpp"
#include "dependencies/string_ops.hpp"
#include "dependencies/printing.hpp"
#include "dependencies/containers.hpp"
#include "dependencies/hashing.hpp"
#include "dependencies/strings.hpp"
#include "dependencies/filesystem.hpp"
#include "dependencies/timing.hpp"
#include "dependencies/parsing.hpp"
#endif
/*
________ __ __ ________
| \ | \ | \ | \
| ▓▓▓▓▓▓▓▓_______ __ __ ______ ____ _______ | ▓▓\ | ▓▓ \▓▓▓▓▓▓▓▓__ __ ______ ______ _______
| ▓▓__ | \| \ | \ \ \ / \ | ▓▓▓\| ▓▓ | ▓▓ | \ | \/ \ / \ / \
| ▓▓ \ | ▓▓▓▓▓▓▓\ ▓▓ | ▓▓ ▓▓▓▓▓▓\▓▓▓▓\ ▓▓▓▓▓▓▓ | ▓▓▓▓\ ▓▓ | ▓▓ | ▓▓ | ▓▓ ▓▓▓▓▓▓\ ▓▓▓▓▓▓\ ▓▓▓▓▓▓▓
| ▓▓▓▓▓ | ▓▓ | ▓▓ ▓▓ | ▓▓ ▓▓ | ▓▓ | ▓▓\▓▓ \ | ▓▓\▓▓ ▓▓ | ▓▓ | ▓▓ | ▓▓ ▓▓ | ▓▓ ▓▓ ▓▓\▓▓ \
| ▓▓_____| ▓▓ | ▓▓ ▓▓__/ ▓▓ ▓▓ | ▓▓ | ▓▓_\▓▓▓▓▓▓\ | ▓▓ \▓▓▓▓ | ▓▓ | ▓▓__/ ▓▓ ▓▓__/ ▓▓ ▓▓▓▓▓▓▓▓_\▓▓▓▓▓▓\
| ▓▓ \ ▓▓ | ▓▓\▓▓ ▓▓ ▓▓ | ▓▓ | ▓▓ ▓▓ | ▓▓ \▓▓▓ | ▓▓ \▓▓ ▓▓ ▓▓ ▓▓\▓▓ \ ▓▓
\▓▓▓▓▓▓▓▓\▓▓ \▓▓ \▓▓▓▓▓▓ \▓▓ \▓▓ \▓▓\▓▓▓▓▓▓▓ \▓▓ \▓▓ \▓▓ _\▓▓▓▓▓▓▓ ▓▓▓▓▓▓▓ \▓▓▓▓▓▓▓\▓▓▓▓▓▓▓
| \__| ▓▓ ▓▓
\▓▓ ▓▓ ▓▓
\▓▓▓▓▓▓ \▓▓
*/
enum LogLevel //: u32
{
LL_Null,
LL_Note,
LL_Warning,
LL_Error,
LL_Fatal,
LL_UnderlyingType = GEN_U32_MAX,
};
typedef enum LogLevel LogLevel;
Str loglevel_to_str(LogLevel level)
{
local_persist
Str lookup[] = {
{ "Null", sizeof("Null") - 1 },
{ "Note", sizeof("Note") - 1 },
{ "Warning", sizeof("Info") - 1 },
{ "Error", sizeof("Error") - 1 },
{ "Fatal", sizeof("Fatal") - 1 },
};
return lookup[level];
}
typedef struct LogEntry LogEntry;
struct LogEntry
{
Str msg;
LogLevel level;
};
typedef void LoggerProc(LogEntry entry);
// By default this library will either crash or exit if an error is detected while generating codes.
// Even if set to not use GEN_FATAL, GEN_FATAL will still be used for memory failures as the library is unusable when they occur.
#ifdef GEN_DONT_USE_FATAL
#define log_failure log_fmt
#else
#define log_failure GEN_FATAL
#endif
enum AccessSpec : u32
{
AccessSpec_Default,
AccessSpec_Private,
AccessSpec_Protected,
AccessSpec_Public,
AccessSpec_Num_AccessSpec,
AccessSpec_Invalid,
AccessSpec_SizeDef = GEN_U32_MAX,
};
static_assert( size_of(AccessSpec) == size_of(u32), "AccessSpec not u32 size" );
inline
Str access_spec_to_str( AccessSpec type )
{
local_persist
Str lookup[ (u32)AccessSpec_Num_AccessSpec ] = {
{ "", sizeof( "" ) - 1 },
{ "private", sizeof("prviate") - 1 },
{ "private", sizeof("protected") - 1 },
{ "public", sizeof("public") - 1 },
};
Str invalid = { "Invalid", sizeof("Invalid") - 1 };
if ( type > AccessSpec_Public )
return invalid;
return lookup[ (u32)type ];
}
enum CodeFlag : u32
{
CodeFlag_None = 0,
CodeFlag_FunctionType = bit(0),
CodeFlag_ParamPack = bit(1),
CodeFlag_Module_Export = bit(2),
CodeFlag_Module_Import = bit(3),
CodeFlag_SizeDef = GEN_U32_MAX,
};
static_assert( size_of(CodeFlag) == size_of(u32), "CodeFlag not u32 size" );
// Used to indicate if enum definitoin is an enum class or regular enum.
enum EnumDecl : u8
{
EnumDecl_Regular,
EnumDecl_Class,
EnumT_SizeDef = GEN_U8_MAX,
};
typedef u8 EnumT;
enum ModuleFlag : u32
{
ModuleFlag_None = 0,
ModuleFlag_Export = bit(0),
ModuleFlag_Import = bit(1),
Num_ModuleFlags,
ModuleFlag_Invalid,
ModuleFlag_SizeDef = GEN_U32_MAX,
};
static_assert( size_of(ModuleFlag) == size_of(u32), "ModuleFlag not u32 size" );
inline
Str module_flag_to_str( ModuleFlag flag )
{
local_persist
Str lookup[ (u32)Num_ModuleFlags ] = {
{ "__none__", sizeof("__none__") - 1 },
{ "export", sizeof("export") - 1 },
{ "import", sizeof("import") - 1 },
};
local_persist
Str invalid_flag = { "invalid", sizeof("invalid") };
if ( flag > ModuleFlag_Import )
return invalid_flag;
return lookup[ (u32)flag ];
}
enum EPreprocessCond : u32
{
PreprocessCond_If,
PreprocessCond_IfDef,
PreprocessCond_IfNotDef,
PreprocessCond_ElIf,
EPreprocessCond_SizeDef = GEN_U32_MAX,
};
static_assert( size_of(EPreprocessCond) == size_of(u32), "EPreprocessCond not u32 size" );
enum ETypenameTag : u16
{
Tag_None,
Tag_Class,
Tag_Enum,
Tag_Struct,
Tag_Union,
Tag_UnderlyingType = GEN_U16_MAX,
};
static_assert( size_of(ETypenameTag) == size_of(u16), "ETypenameTag is not u16 size");
tests/repro_bug.py
+31
View File
@@ -0,0 +1,31 @@
import sys
import os
sys.path.insert(0, os.getcwd())
from src.file_cache import ASTParser
def test_multibyte_definition_lookup():
# String with a multi-byte character (3 bytes)
# The shade character ▓ is 3 bytes in UTF-8.
# 3 shade characters = 9 bytes. In Python str they are 3 chars.
# Offset drift will be 6 bytes.
code = """
/* ▓▓▓ */
struct Target {
int x;
};
"""
parser = ASTParser("cpp")
# This should find Target and return its definition
definition = parser.get_definition(code, "Target")
print(f"Definition found: '{definition}'")
if "struct Target" not in definition:
print("FAILURE: 'struct Target' not in definition")
sys.exit(1)
if "int x;" not in definition:
print("FAILURE: 'int x;' not in definition")
sys.exit(1)
print("SUCCESS")
if __name__ == "__main__":
test_multibyte_definition_lookup()
tests/test_ts_cpp_tools.py
+85 -2
View File
@@ -71,7 +71,7 @@ void templateFunc(T t) {
mcp_client._resolve_and_check = original_resolve mcp_client._resolve_and_check = original_resolve
def test_exhaustive_gencpp_corpus(): def test_exhaustive_gencpp_corpus():
base_dir = Path(__file__).parent / "assets" / "gencpp_samples" base_dir = Path(__file__).parent / "assets" / "cpp_samples"
files_to_test = { files_to_test = {
"base_component.h": [ "base_component.h": [
"BaseComponent", "BaseComponent",
@@ -126,7 +126,7 @@ def test_exhaustive_gencpp_corpus():
mcp_client._resolve_and_check = original_resolve mcp_client._resolve_and_check = original_resolve
def test_ts_cpp_update_definition(tmp_path): def test_ts_cpp_update_definition(tmp_path):
asset_path = Path(__file__).parent / "assets" / "gencpp_samples" / "component_registry.cpp" asset_path = Path(__file__).parent / "assets" / "cpp_samples" / "component_registry.cpp"
cpp_content = asset_path.read_text(encoding="utf-8") cpp_content = asset_path.read_text(encoding="utf-8")
cpp_file = tmp_path / "component_registry.cpp" cpp_file = tmp_path / "component_registry.cpp"
cpp_file.write_text(cpp_content, encoding="utf-8") cpp_file.write_text(cpp_content, encoding="utf-8")
@@ -157,3 +157,86 @@ def test_ts_cpp_update_definition(tmp_path):
assert "DEBUG: Registering" in definition assert "DEBUG: Registering" in definition
finally: finally:
mcp_client._resolve_and_check = original_resolve mcp_client._resolve_and_check = original_resolve
def test_exhaustive_gencpp_samples():
base_dir = Path(__file__).parent / "assets" / "gencpp_samples"
files_to_test = {
"ast.hpp": [
"AST",
"AST_Body",
"AST_Class",
"AST_Fn",
"AST_Struct"
],
"parser.cpp": [
"parser_push",
"parser_pop",
"parser_to_strbuilder",
"lex__eat"
],
"types.hpp": [
"LogLevel",
"loglevel_to_str",
"AccessSpec",
"access_spec_to_str",
"ModuleFlag"
]
}
from src import mcp_client
original_resolve = mcp_client._resolve_and_check
mcp_client._resolve_and_check = lambda path: (Path(path), None)
try:
for filename, symbols in files_to_test.items():
path = base_dir / filename
assert path.exists(), f"{path} does not exist"
# 1. Verify skeleton executes without errors
skeleton = ts_cpp_get_skeleton(str(path))
assert skeleton and not skeleton.startswith("ERROR")
# 2. Verify code outline executes without errors
outline = ts_cpp_get_code_outline(str(path))
assert outline and not outline.startswith("ERROR")
# 3. Verify specific complex symbols can be retrieved via ts_cpp_get_definition
for symbol in symbols:
definition = ts_cpp_get_definition(str(path), symbol)
assert definition and not definition.startswith("ERROR"), f"Failed for {symbol} in {filename}: {definition}"
simple_name = symbol.split("::")[-1]
assert simple_name in definition
finally:
mcp_client._resolve_and_check = original_resolve
def test_ts_cpp_update_definition_gencpp(tmp_path):
asset_path = Path(__file__).parent / "assets" / "gencpp_samples" / "parser.cpp"
cpp_content = asset_path.read_text(encoding="utf-8")
cpp_file = tmp_path / "parser.cpp"
cpp_file.write_text(cpp_content, encoding="utf-8")
from src import mcp_client
original_resolve = mcp_client._resolve_and_check
mcp_client._resolve_and_check = lambda path: (Path(path), None)
try:
# Update parser_pop
new_pop_body = """void parser_pop(ParseContext* ctx)
{
#if 0 && GEN_BUILD_DEBUG
log_fmt("\\tPopping parser: %.*s\\n", Scope->ProcName.Len, Scope->ProcName.Ptr );
#endif
// HELLO FROM TEST
ctx->scope = ctx->scope->prev;
}"""
result = ts_cpp_update_definition(str(cpp_file), "parser_pop", new_pop_body)
assert "Successfully updated" in result
updated_content = cpp_file.read_text(encoding="utf-8")
assert "// HELLO FROM TEST" in updated_content
# Verify retrieval
definition = ts_cpp_get_definition(str(cpp_file), "parser_pop")
assert "// HELLO FROM TEST" in definition
finally:
mcp_client._resolve_and_check = original_resolve