Add -doc-format command for the new .odin-doc file format (to be used to generate documentation tools)

core/odin/doc-format/doc_format.odin

@@ -0,0 +1,251 @@
+package odin_doc_format
+
+import "core:mem"
+
+Array :: struct($T: typeid) {
+	offset: u32,
+	length: u32,
+}
+
+String :: distinct Array(byte);
+
+Version_Type_Major :: 0;
+Version_Type_Minor :: 1;
+Version_Type_Patch :: 0;
+
+Version_Type :: struct {
+	major, minor, patch: u8,
+	_: u8,
+};
+
+Version_Type_Default :: Version_Type{
+	major=Version_Type_Major,
+	minor=Version_Type_Minor,
+	patch=Version_Type_Patch,
+};
+
+Magic_String :: "odindoc\x00";
+
+Header_Base :: struct {
+	magic: [8]byte,
+	_: u32,
+	version:     Version_Type,
+	total_size:  u32,
+	header_size: u32,
+	hash:        u32,
+}
+
+Header :: struct {
+	using base: Header_Base,
+
+	// NOTE: These arrays reserve the zero element as a sentinel value
+	files:    Array(File),
+	pkgs:     Array(Pkg),
+	entities: Array(Entity),
+	types:    Array(Type),
+}
+
+File_Index   :: distinct u32;
+Pkg_Index    :: distinct u32;
+Entity_Index :: distinct u32;
+Type_Index   :: distinct u32;
+
+
+Position :: struct {
+	file:   File_Index,
+	line:   u32,
+	column: u32,
+	offset: u32,
+};
+
+File :: struct {
+	pkg:  Pkg_Index,
+	name: String,
+}
+
+Pkg :: struct {
+	fullpath: String,
+	name:     String,
+	docs:     String,
+	files:    Array(File_Index),
+	entities: Array(Entity_Index),
+}
+
+Entity_Kind :: enum u32 {
+	Invalid      = 0,
+	Constant     = 1,
+	Variable     = 2,
+	Type_Name    = 3,
+	Procedure    = 4,
+	Proc_Group   = 5,
+	Import_Name  = 6,
+	Library_Name = 7,
+}
+
+Entity_Flag :: enum u32 {
+	Foreign = 0,
+	Export  = 1,
+
+	Param_Using     = 2,
+	Param_Const     = 3,
+	Param_Auto_Cast = 4,
+	Param_Ellipsis  = 5,
+	Param_CVararg   = 6,
+	Param_No_Alias  = 7,
+
+	Type_Alias = 8,
+
+	Var_Thread_Local = 9,
+}
+
+Entity_Flags :: distinct bit_set[Entity_Flag; u32];
+
+Entity :: struct {
+	kind:             Entity_Kind,
+	flags:            Entity_Flags,
+	pos:              Position,
+	name:             String,
+	type:             Type_Index,
+	init_string:      String,
+	_:                u32,
+	comment:          String,
+	docs:             String,
+	foreign_library:  Entity_Index,
+	link_name:        String,
+	attributes:       Array(Attribute),
+	grouped_entities: Array(Entity_Index), // Procedure Groups
+	where_clauses:    Array(String), // Procedures
+}
+
+Attribute :: struct {
+	name:  String,
+	value: String,
+}
+
+Type_Kind :: enum u32 {
+	Invalid            = 0,
+	Basic              = 1,
+	Named              = 2,
+	Generic            = 3,
+	Pointer            = 4,
+	Array              = 5,
+	Enumerated_Array   = 6,
+	Slice              = 7,
+	Dynamic_Array      = 8,
+	Map                = 9,
+	Struct             = 10,
+	Union              = 11,
+	Enum               = 12,
+	Tuple              = 13,
+	Proc               = 14,
+	Bit_Set            = 15,
+	Simd_Vector        = 16,
+	SOA_Struct_Fixed   = 17,
+	SOA_Struct_Slice   = 18,
+	SOA_Struct_Dynamic = 19,
+	Relative_Pointer   = 20,
+	Relative_Slice     = 21,
+}
+
+Type_Elems_Cap :: 4;
+
+Type :: struct {
+	kind:         Type_Kind,
+	flags:        u32, // Type_Kind specific
+	name:         String,
+	custom_align: String,
+
+	// Used by some types
+	elem_count_len: u32,
+	elem_counts:    [Type_Elems_Cap]u64,
+
+	// Each of these is esed by some types, not all
+	types:         Array(Type_Index),
+	entities:      Array(Entity_Index),
+	polymorphic_params: Type_Index, // Struct, Union
+	where_clauses: Array(String), // Struct, Union
+}
+
+Type_Flags_Basic :: distinct bit_set[Type_Flag_Basic; u32];
+Type_Flag_Basic :: enum u32 {
+	Untyped = 1,
+}
+
+Type_Flags_Struct :: distinct bit_set[Type_Flag_Struct; u32];
+Type_Flag_Struct :: enum u32 {
+	Polymorphic = 0,
+	Packed      = 1,
+	Raw_Union   = 2,
+}
+
+Type_Flags_Union :: distinct bit_set[Type_Flag_Union; u32];
+Type_Flag_Union :: enum u32 {
+	Polymorphic = 0,
+	No_Nil      = 1,
+	Maybe       = 2,
+}
+
+Type_Flags_Proc :: distinct bit_set[Type_Flag_Proc; u32];
+Type_Flag_Proc :: enum u32 {
+	Polymorphic = 0,
+	Diverging   = 1,
+	Optional_Ok = 2,
+	Variadic    = 3,
+	C_Vararg    = 4,
+}
+
+Type_Flags_BitSet :: distinct bit_set[Type_Flag_BitSet; u32];
+Type_Flag_BitSet :: enum u32 {
+	Range            = 1,
+	Op_Lt            = 2,
+	Op_Lt_Eq         = 3,
+	Underlying_Type  = 4,
+}
+
+Type_Flags_SimdVector :: distinct bit_set[Type_Flag_SimdVector; u32];
+Type_Flag_SimdVector :: enum u32 {
+	x86_mmx = 1,
+}
+
+from_array :: proc(base: ^Header_Base, a: $A/Array($T)) -> []T {
+	s: mem.Raw_Slice;
+	s.data = rawptr(uintptr(base) + uintptr(a.offset));
+	s.len = int(a.length);
+	return transmute([]T)s;
+}
+from_string :: proc(base: ^Header_Base, s: String) -> string {
+	return string(from_array(base, s));
+}
+
+
+
+
+Reader_Error :: enum {
+	None,
+	Header_Too_Small,
+	Invalid_Magic,
+	Data_Too_Small,
+	Invalid_Version,
+}
+
+read_from_bytes :: proc(data: []byte) -> (h: ^Header, err: Reader_Error) {
+	if len(data) < size_of(Header_Base) {
+		err = .Header_Too_Small;
+		return;
+	}
+	header_base := (^Header_Base)(raw_data(data));
+	if header_base.magic != Magic_String {
+		err = .Invalid_Magic;
+		return;
+	}
+	if len(data) < int(header_base.total_size) {
+		err = .Data_Too_Small;
+		return;
+	}
+	if header_base.version != Version_Type_Default {
+		err = .Invalid_Version;
+		return;
+	}
+	h = (^Header)(header_base);
+	return;
+}

src/build_settings.cpp

@@ -135,6 +135,7 @@ char const *odin_command_strings[32] = {
 enum CmdDocFlag : u32 {
 	CmdDocFlag_Short       = 1<<0,
 	CmdDocFlag_AllPackages = 1<<1,
+	CmdDocFlag_DocFormat   = 1<<2,
 };
 
 

src/check_type.cpp

@@ -2461,7 +2461,6 @@ bool check_procedure_type(CheckerContext *ctx, Type *type, Ast *proc_type_node,
 	type->Proc.specialization_count = specialization_count;
 	type->Proc.diverging            = pt->diverging;
 	type->Proc.optional_ok          = optional_ok;
-	type->Proc.tags                 = pt->tags;
 
 	if (param_count > 0) {
 		Entity *end = params->Tuple.variables[param_count-1];

src/docs.cpp

@@ -34,9 +34,17 @@ GB_COMPARE_PROC(cmp_entities_for_printing) {
 	Entity *x = *cast(Entity **)a;
 	Entity *y = *cast(Entity **)b;
 	int res = 0;
-	res = string_compare(x->pkg->name, y->pkg->name);
+	if (x->pkg != y->pkg) {
-	if (res != 0) {
+		if (x->pkg == nullptr) {
-		return res;
+			return -1;
+		}
+		if (y->pkg == nullptr) {
+			return +1;
+		}
+		res = string_compare(x->pkg->name, y->pkg->name);
+		if (res != 0) {
+			return res;
+		}
 	}
 	int ox = print_entity_kind_ordering[x->kind];
 	int oy = print_entity_kind_ordering[y->kind];
@@ -56,6 +64,9 @@ GB_COMPARE_PROC(cmp_ast_package_by_name) {
 	return string_compare(x->name, y->name);
 }
 
+#include "docs_format.cpp"
+#include "docs_writer.cpp"
+
 void print_doc_line(i32 indent, char const *fmt, ...) {
 	while (indent --> 0) {
 		gb_printf("\t");
@@ -297,23 +308,59 @@ void print_doc_package(CheckerInfo *info, AstPackage *pkg) {
 void generate_documentation(Checker *c) {
 	CheckerInfo *info = &c->info;
 
-	auto pkgs = array_make<AstPackage *>(permanent_allocator(), 0, info->packages.entries.count);
+	if (build_context.cmd_doc_flags & CmdDocFlag_DocFormat) {
-	for_array(i, info->packages.entries) {
+		String init_fullpath = c->parser->init_fullpath;
-		AstPackage *pkg = info->packages.entries[i].value;
+		String output_name = {};
-		if (build_context.cmd_doc_flags & CmdDocFlag_AllPackages) {
+		String output_base = {};
-			array_add(&pkgs, pkg);
+
+		if (build_context.out_filepath.len == 0) {
+			output_name = remove_directory_from_path(init_fullpath);
+			output_name = remove_extension_from_path(output_name);
+			output_name = string_trim_whitespace(output_name);
+			if (output_name.len == 0) {
+				output_name = info->init_scope->pkg->name;
+			}
+			output_base = output_name;
 		} else {
-			if (pkg->kind == Package_Init) {
+			output_name = build_context.out_filepath;
-				array_add(&pkgs, pkg);
+			output_name = string_trim_whitespace(output_name);
-			} else if (pkg->is_extra) {
+			if (output_name.len == 0) {
-				array_add(&pkgs, pkg);
+				output_name = info->init_scope->pkg->name;
+			}
+			isize pos = string_extension_position(output_name);
+			if (pos < 0) {
+				output_base = output_name;
+			} else {
+				output_base = substring(output_name, 0, pos);
 			}
 		}
-	}
 
-	gb_sort_array(pkgs.data, pkgs.count, cmp_ast_package_by_name);
+		output_base = path_to_full_path(permanent_allocator(), output_base);
 
-	for_array(i, pkgs) {
+		gbString output_file_path = gb_string_make_length(heap_allocator(), output_base.text, output_base.len);
-		print_doc_package(info, pkgs[i]);
+		output_file_path = gb_string_appendc(output_file_path, ".odin-doc");
+		defer (gb_string_free(output_file_path));
+
+		odin_doc_write(info, output_file_path);
+	} else {
+		auto pkgs = array_make<AstPackage *>(permanent_allocator(), 0, info->packages.entries.count);
+		for_array(i, info->packages.entries) {
+			AstPackage *pkg = info->packages.entries[i].value;
+			if (build_context.cmd_doc_flags & CmdDocFlag_AllPackages) {
+				array_add(&pkgs, pkg);
+			} else {
+				if (pkg->kind == Package_Init) {
+					array_add(&pkgs, pkg);
+				} else if (pkg->is_extra) {
+					array_add(&pkgs, pkg);
+				}
+			}
+		}
+
+		gb_sort_array(pkgs.data, pkgs.count, cmp_ast_package_by_name);
+
+		for_array(i, pkgs) {
+			print_doc_package(info, pkgs[i]);
+		}
 	}
 }

src/docs_format.cpp

@@ -0,0 +1,208 @@
+#define OdinDocHeader_MagicString "odindoc\0"
+
+template <typename T>
+struct OdinDocArray {
+	u32 offset;
+	u32 length;
+};
+
+using OdinDocString = OdinDocArray<u8>;
+
+struct OdinDocVersionType {
+	u8 major, minor, patch;
+	u8 pad0;
+};
+
+#define OdinDocVersionType_Major 0
+#define OdinDocVersionType_Minor 1
+#define OdinDocVersionType_Patch 0
+
+struct OdinDocHeaderBase {
+	u8                 magic[8];
+	u32                padding0;
+	OdinDocVersionType version;
+	u32                total_size;
+	u32                header_size;
+	u32                hash; // after header
+};
+
+template <typename T>
+Slice<T> from_array(OdinDocHeaderBase *base, OdinDocArray<T> const &a) {
+	Slice<T> s = {};
+	s.data  = cast(T *)(cast(uintptr)base + cast(uintptr)a.offset);
+	s.count = cast(isize)a.length;
+	return s;
+}
+
+String from_string(OdinDocHeaderBase *base, OdinDocString const &s) {
+	String str = {};
+	str.text = cast(u8 *)(cast(uintptr)base + cast(uintptr)s.offset);
+	str.len  = cast(isize)s.length;
+	return str;
+}
+
+typedef u32 OdinDocFileIndex;
+typedef u32 OdinDocPkgIndex;
+typedef u32 OdinDocEntityIndex;
+typedef u32 OdinDocTypeIndex;
+
+struct OdinDocFile {
+	OdinDocPkgIndex pkg;
+	OdinDocString   name;
+};
+
+struct OdinDocPosition {
+	OdinDocFileIndex file;
+	u32              line;
+	u32              column;
+	u32              offset;
+};
+
+enum OdinDocTypeKind : u32 {
+	OdinDocType_Invalid          = 0,
+	OdinDocType_Basic            = 1,
+	OdinDocType_Named            = 2,
+	OdinDocType_Generic          = 3,
+	OdinDocType_Pointer          = 4,
+	OdinDocType_Array            = 5,
+	OdinDocType_EnumeratedArray  = 6,
+	OdinDocType_Slice            = 7,
+	OdinDocType_DynamicArray     = 8,
+	OdinDocType_Map              = 9,
+	OdinDocType_Struct           = 10,
+	OdinDocType_Union            = 11,
+	OdinDocType_Enum             = 12,
+	OdinDocType_Tuple            = 13,
+	OdinDocType_Proc             = 14,
+	OdinDocType_BitSet           = 15,
+	OdinDocType_SimdVector       = 16,
+	OdinDocType_SOAStructFixed   = 17,
+	OdinDocType_SOAStructSlice   = 18,
+	OdinDocType_SOAStructDynamic = 19,
+	OdinDocType_RelativePointer  = 20,
+	OdinDocType_RelativeSlice    = 21,
+};
+
+enum OdinDocTypeFlag_Basic : u32 {
+	OdinDocTypeFlag_Basic_untyped = 1<<1,
+};
+
+enum OdinDocTypeFlag_Struct : u32 {
+	OdinDocTypeFlag_Struct_polymorphic = 1<<0,
+	OdinDocTypeFlag_Struct_packed      = 1<<1,
+	OdinDocTypeFlag_Struct_raw_union   = 1<<2,
+};
+
+enum OdinDocTypeFlag_Union : u32 {
+	OdinDocTypeFlag_Union_polymorphic = 1<<0,
+	OdinDocTypeFlag_Union_no_nil      = 1<<1,
+	OdinDocTypeFlag_Union_maybe       = 1<<2,
+};
+
+enum OdinDocTypeFlag_Proc : u32 {
+	OdinDocTypeFlag_Proc_polymorphic = 1<<0,
+	OdinDocTypeFlag_Proc_diverging   = 1<<1,
+	OdinDocTypeFlag_Proc_optional_ok = 1<<2,
+	OdinDocTypeFlag_Proc_variadic    = 1<<3,
+	OdinDocTypeFlag_Proc_c_vararg    = 1<<4,
+};
+
+enum OdinDocTypeFlag_BitSet : u32 {
+	OdinDocTypeFlag_BitSet_Range          = 1<<1,
+	OdinDocTypeFlag_BitSet_OpLt           = 1<<2,
+	OdinDocTypeFlag_BitSet_OpLtEq         = 1<<3,
+	OdinDocTypeFlag_BitSet_UnderlyingType = 1<<4,
+};
+
+enum OdinDocTypeFlag_SimdVector : u32 {
+	OdinDocTypeFlag_BitSet_x86_mmx = 1<<1,
+};
+
+enum {
+	// constants
+	OdinDocType_ElemsCap = 4,
+};
+
+struct OdinDocType {
+	OdinDocTypeKind kind;
+	u32             flags;
+	OdinDocString   name;
+	OdinDocString   custom_align;
+
+	// Used by some types
+	u32 elem_count_len;
+	u64 elem_counts[OdinDocType_ElemsCap];
+
+	// Each of these is esed by some types, not all
+	OdinDocArray<OdinDocTypeIndex> types;
+	OdinDocArray<OdinDocEntityIndex> entities;
+	OdinDocTypeIndex polmorphic_params;
+	OdinDocArray<OdinDocString> where_clauses;
+};
+
+struct OdinDocAttribute {
+	OdinDocString name;
+	OdinDocString value;
+};
+
+enum OdinDocEntityKind : u32 {
+	OdinDocEntity_Invalid     = 0,
+	OdinDocEntity_Constant    = 1,
+	OdinDocEntity_Variable    = 2,
+	OdinDocEntity_TypeName    = 3,
+	OdinDocEntity_Procedure   = 4,
+	OdinDocEntity_ProcGroup   = 5,
+	OdinDocEntity_ImportName  = 6,
+	OdinDocEntity_LibraryName = 7,
+};
+
+enum OdinDocEntityFlag : u32 {
+	OdinDocEntityFlag_Foreign = 1<<0,
+	OdinDocEntityFlag_Export  = 1<<1,
+
+	OdinDocEntityFlag_Param_Using    = 1<<2,
+	OdinDocEntityFlag_Param_Const    = 1<<3,
+	OdinDocEntityFlag_Param_AutoCast = 1<<4,
+	OdinDocEntityFlag_Param_Ellipsis = 1<<5,
+	OdinDocEntityFlag_Param_CVararg  = 1<<6,
+	OdinDocEntityFlag_Param_NoAlias  = 1<<7,
+
+	OdinDocEntityFlag_Type_Alias = 1<<8,
+
+	OdinDocEntityFlag_Var_Thread_Local = 1<<9,
+};
+
+struct OdinDocEntity {
+	OdinDocEntityKind  kind;
+	u32                flags;
+	OdinDocPosition    pos;
+	OdinDocString      name;
+	OdinDocTypeIndex   type;
+	OdinDocString      init_string;
+	u32                reserved_for_init;
+	OdinDocString      comment;
+	OdinDocString      docs;
+	OdinDocEntityIndex foreign_library;
+	OdinDocString      link_name;
+	OdinDocArray<OdinDocAttribute> attributes;
+	OdinDocArray<OdinDocEntityIndex> grouped_entities; // Procedure Groups
+	OdinDocArray<OdinDocString>      where_clauses; // Procedures
+};
+
+struct OdinDocPkg {
+	OdinDocString fullpath;
+	OdinDocString name;
+	OdinDocString docs;
+	OdinDocArray<OdinDocFileIndex>   files;
+	OdinDocArray<OdinDocEntityIndex> entities;
+};
+
+struct OdinDocHeader {
+	OdinDocHeaderBase base;
+
+	OdinDocArray<OdinDocFile>   files;
+	OdinDocArray<OdinDocPkg>    pkgs;
+	OdinDocArray<OdinDocEntity> entities;
+	OdinDocArray<OdinDocType>   types;
+};
+

src/llvm_backend.cpp

@@ -2523,7 +2523,6 @@ lbProcedure *lb_create_procedure(lbModule *m, Entity *entity) {
 	p->type           = entity->type;
 	p->type_expr      = decl->type_expr;
 	p->body           = pl->body;
-	p->tags           = pt->Proc.tags;
 	p->inlining       = ProcInlining_none;
 	p->is_foreign     = entity->Procedure.is_foreign;
 	p->is_export      = entity->Procedure.is_export;

src/main.cpp

@@ -607,6 +607,7 @@ enum BuildFlagKind {
 
 	BuildFlag_Short,
 	BuildFlag_AllPackages,
+	BuildFlag_DocFormat,
 
 	BuildFlag_IgnoreWarnings,
 	BuildFlag_WarningsAsErrors,
@@ -721,6 +722,7 @@ bool parse_build_flags(Array<String> args) {
 
 	add_flag(&build_flags, BuildFlag_Short,         str_lit("short"),        BuildFlagParam_None, Command_doc);
 	add_flag(&build_flags, BuildFlag_AllPackages,   str_lit("all-packages"), BuildFlagParam_None, Command_doc);
+	add_flag(&build_flags, BuildFlag_DocFormat,     str_lit("doc-format"),   BuildFlagParam_None, Command_doc);
 
 	add_flag(&build_flags, BuildFlag_IgnoreWarnings,   str_lit("ignore-warnings"),    BuildFlagParam_None, Command_all);
 	add_flag(&build_flags, BuildFlag_WarningsAsErrors, str_lit("warnings-as-errors"), BuildFlagParam_None, Command_all);
@@ -1227,6 +1229,9 @@ bool parse_build_flags(Array<String> args) {
 						case BuildFlag_AllPackages:
 							build_context.cmd_doc_flags |= CmdDocFlag_AllPackages;
 							break;
+						case BuildFlag_DocFormat:
+							build_context.cmd_doc_flags |= CmdDocFlag_DocFormat;
+							break;
 						case BuildFlag_IgnoreWarnings:
 							if (build_context.warnings_as_errors) {
 								gb_printf_err("-ignore-warnings cannot be used with -warnings-as-errors\n");

src/types.cpp

@@ -195,7 +195,6 @@ struct TypeProc {
 	Type *   results; // Type_Tuple
 	i32      param_count;
 	i32      result_count;
-	u64      tags;
 	isize    specialization_count;
 	ProcCallingConvention calling_convention;
 	i32      variadic_index;