Add support for backslash \ to consume a newline

src/parser.cpp

@@ -1262,11 +1262,6 @@ Token peek_token(AstFile *f) {
 	return {};
 }
 
-
-bool token_is_newline(Token const &tok) {
-	return tok.kind == Token_Semicolon && tok.string == "\n";
-}
-
 bool skip_possible_newline(AstFile *f) {
 	if ((f->tokenizer.flags & TokenizerFlag_InsertSemicolon) == 0) {
 		return false;

src/tokenizer.cpp

@@ -224,6 +224,13 @@ bool operator<=(TokenPos const &a, TokenPos const &b) { return token_pos_cmp(a,
 bool operator> (TokenPos const &a, TokenPos const &b) { return token_pos_cmp(a, b) >  0; }
 bool operator>=(TokenPos const &a, TokenPos const &b) { return token_pos_cmp(a, b) >= 0; }
 
+
+TokenPos token_pos_add_column(TokenPos pos) {
+	pos.column += 1;
+	pos.offset += 1;
+	return pos;
+}
+
 struct Token {
 	TokenKind kind;
 	String    string;
@@ -242,6 +249,10 @@ Token make_token_ident(char const *s) {
 	return t;
 }
 
+bool token_is_newline(Token const &tok) {
+	return tok.kind == Token_Semicolon && tok.string == "\n";
+}
+
 
 struct ErrorCollector {
 	TokenPos prev;
@@ -653,6 +664,22 @@ void tokenizer_err(Tokenizer *t, char const *msg, ...) {
 	t->error_count++;
 }
 
+void tokenizer_err(Tokenizer *t, TokenPos const &pos, char const *msg, ...) {
+	va_list va;
+	isize column = t->read_curr - t->line+1;
+	if (column < 1) {
+		column = 1;
+	}
+	Token token = {};
+	token.pos = pos;
+
+	va_start(va, msg);
+	syntax_error_va(token, msg, va);
+	va_end(va);
+
+	t->error_count++;
+}
+
 void advance_to_next_rune(Tokenizer *t) {
 	if (t->read_curr < t->end) {
 		Rune rune;
@@ -958,7 +985,7 @@ bool scan_escape(Tokenizer *t) {
 }
 
 
-void tokenizer_get_token(Tokenizer *t, Token *token) {
+void tokenizer_get_token(Tokenizer *t, Token *token, int repeat=0) {
 	// Skip whitespace
 	for (;;) {
 		switch (t->curr_rune) {
@@ -984,6 +1011,8 @@ void tokenizer_get_token(Tokenizer *t, Token *token) {
 	token->pos.offset = cast(i32)(t->curr - t->start);
 	token->pos.column = cast(i32)(t->curr - t->line + 1);
 
+	TokenPos current_pos = token->pos;
+
 	bool insert_semicolon = false;
 
 	Rune curr_rune = t->curr_rune;
@@ -1050,6 +1079,17 @@ void tokenizer_get_token(Tokenizer *t, Token *token) {
 			token->kind = Token_Semicolon;
 			return;
 
+		case '\\':
+			if (t->flags & TokenizerFlag_InsertSemicolon) {
+				t->insert_semicolon = false;
+			}
+			tokenizer_get_token(t, token);
+			if (token->pos.line == current_pos.line) {
+				tokenizer_err(t, token_pos_add_column(current_pos), "Expected a newline after \\");
+			}
+			// NOTE(bill): tokenizer_get_token has been called already, return early
+			return;
+
 		case '\'': // Rune Literal
 		{
 			insert_semicolon = true;
@@ -1197,7 +1237,6 @@ void tokenizer_get_token(Tokenizer *t, Token *token) {
 			insert_semicolon = true;
 			token->kind = Token_CloseBrace;
 			break;
-		case '\\': token->kind = Token_BackSlash;    break;
 
 		case '%':
 			token->kind = Token_Mod;