From 7b967b2e595bf4ac34e4db928e0d1b96213d91c6 Mon Sep 17 00:00:00 2001
From: DrMaxNix <contact@drmaxnix.de>
Date: Thu, 8 Sep 2022 13:36:00 +0200
Subject: [PATCH] :sparkles: tokenize newline and `;` as delimiters

---
 doc/concept.sh    | 6 +++++-
 src/lexer.js      | 8 +++++++-
 src/main.js       | 3 +++
 test/comments.jce | 1 -
 4 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/doc/concept.sh b/doc/concept.sh
index 935a7ca..35d11cc 100644
--- a/doc/concept.sh
+++ b/doc/concept.sh
@@ -2,7 +2,6 @@
 # one-line comments (from prefix to EOL)
 mov $1 42 # comment
 mov $1 42 // comment
-mov $1 42 ; comment
 
 # block comments
 /*
@@ -14,6 +13,11 @@ mov $1 42 ; comment
 mov /* Inline block comment */ $1 42
 
 
+## DELIMITERS ##
+# `;` acts like a fake newline
+mov $1 42; typ $1; echo $1
+
+
 ## ESCAPE SEQUENCES AND INTEGER PREFIXES ##
 # quote escaping
 "Here comes a quote: \"Lorem ipsum\". This had to be escaped!"
diff --git a/src/lexer.js b/src/lexer.js
index 13a293e..a4efd78 100644
--- a/src/lexer.js
+++ b/src/lexer.js
@@ -63,8 +63,15 @@ class Juicescript_lexer {
 				break;
 			
 			
+			// DELIMITER //
+			case ";":
+				this.token_add({type: Juicescript.token_type.DELIMITER});
+				break;
+			
+			
 			// NEWLINE //
 			case "\n":
+				this.token_add({type: Juicescript.token_type.DELIMITER});
 				this.line++;
 				break;
 			
@@ -113,7 +120,6 @@ class Juicescript_lexer {
 			
 			// COMMENTS //
 			case "#":
-			case ";":
 			case "/":
 				// block comment
 				if(char === "/" && this.match("*")){
diff --git a/src/main.js b/src/main.js
index f5d3d44..47fb636 100644
--- a/src/main.js
+++ b/src/main.js
@@ -26,6 +26,9 @@ class Juicescript {
 		// special chars
 		"AMPERSAND",
 		
+		// delimiters
+		"DELIMITER",
+		
 		// meta stuff
 		"EOF"
 	);
diff --git a/test/comments.jce b/test/comments.jce
index 7b293e0..b9fca94 100644
--- a/test/comments.jce
+++ b/test/comments.jce
@@ -1,6 +1,5 @@
 mov $1 42 # comment
 mov $1 42 // comment
-mov $1 42 ; comment
 
 # block comments
 /*