core/odin: Added new file tag syntax as token. parse_file stores a list of tags that the file tag parser can use later.

core/odin/tokenizer/token.odin

@@ -32,6 +32,7 @@ Token_Kind :: enum u32 {
 	Invalid,
 	EOF,
 	Comment,
+	File_Tag,
 
 	B_Literal_Begin,
 		Ident,   // main
@@ -166,6 +167,7 @@ tokens := [Token_Kind.COUNT]string {
 	"Invalid",
 	"EOF",
 	"Comment",
+	"FileTag",
 
 	"",
 	"identifier",

core/odin/tokenizer/tokenizer.odin

@@ -206,6 +206,23 @@ scan_comment :: proc(t: ^Tokenizer) -> string {
 	return string(lit)
 }
 
+scan_file_tag :: proc(t: ^Tokenizer) -> string {
+	offset := t.offset - 1
+
+	for t.ch != '\n' {
+		if t.ch == '/' {
+			next := peek_byte(t, 0)
+
+			if next == '/' || next == '*' {
+				break
+			}
+		} 
+		advance_rune(t)
+	}
+
+	return string(t.src[offset : t.offset])
+}
+
 scan_identifier :: proc(t: ^Tokenizer) -> string {
 	offset := t.offset
 
@@ -636,6 +653,9 @@ scan :: proc(t: ^Tokenizer) -> Token {
 			if t.ch == '!' {
 				kind = .Comment
 				lit = scan_comment(t)
+			} else if t.ch == '+' {
+				kind = .File_Tag
+				lit = scan_file_tag(t)
 			}
 		case '/':
 			kind = .Quo