From 57501b99a78d149d2afe1b067a0951448d283127 Mon Sep 17 00:00:00 2001
From: Saffage <04coder@gmail.com>
Date: Thu, 9 May 2024 16:38:10 +0300
Subject: [PATCH] chore: a `Tab` token

---
 internal/jet/jet.go       |   2 +-
 parser/handle_tokens.go   |   3 +-
 scanner/scanner.go        |   9 ++-
 token/kind.go             |   1 +
 token/kind_string.go      | 129 +++++++++++++++++++-------------------
 token/kind_user_string.go | 129 +++++++++++++++++++-------------------
 6 files changed, 142 insertions(+), 131 deletions(-)

diff --git a/internal/jet/jet.go b/internal/jet/jet.go
index 2b4a18e..64dd8e8 100644
--- a/internal/jet/jet.go
+++ b/internal/jet/jet.go
@@ -69,7 +69,7 @@ func process(
 	fileid config.FileID,
 	isRepl bool,
 ) {
-	toks, scanErrors := scanner.Scan(buffer, fileid, scanner.SkipWhitespace)
+	toks, scanErrors := scanner.Scan(buffer, fileid, scanner.SkipWhitespace|scanner.SkipComments)
 
 	if len(scanErrors) > 0 {
 		for _, err := range scanErrors {
diff --git a/parser/handle_tokens.go b/parser/handle_tokens.go
index 0f7b1bc..057bfe0 100644
--- a/parser/handle_tokens.go
+++ b/parser/handle_tokens.go
@@ -39,7 +39,8 @@ func (p *Parser) next() {
 		p.next()
 	}
 
-	if p.flags&SkipWhitespace != 0 && p.tok.Kind == token.Whitespace {
+	if p.flags&SkipWhitespace != 0 &&
+		(p.tok.Kind == token.Whitespace || p.tok.Kind == token.Tab) {
 		p.next()
 	}
 
diff --git a/scanner/scanner.go b/scanner/scanner.go
index dbbb710..fb11b39 100644
--- a/scanner/scanner.go
+++ b/scanner/scanner.go
@@ -83,6 +83,12 @@ func (s *Scanner) Next() token.Token {
 				Data: s.TakeWhile(func(c byte) bool { return c == ' ' }),
 			}
 
+		case s.Match('\t'):
+			tok = token.Token{
+				Kind: token.Tab,
+				Data: string(s.Advance()),
+			}
+
 		case token.IsIdentifierStartChar(s.Peek()):
 			identifier := s.TakeWhile(token.IsIdentifierChar)
 
@@ -199,7 +205,8 @@ func (s *Scanner) Next() token.Token {
 			tok.End = s.PrevPos()
 		}
 
-		if s.flags&SkipWhitespace != 0 && tok.Kind == token.Whitespace {
+		if s.flags&SkipWhitespace != 0 &&
+			(tok.Kind == token.Whitespace || tok.Kind == token.Tab) {
 			return s.Next()
 		} else if s.flags&SkipIllegal != 0 && tok.Kind == token.Illegal {
 			return s.Next()
diff --git a/token/kind.go b/token/kind.go
index ccc1aa4..476c5e4 100644
--- a/token/kind.go
+++ b/token/kind.go
@@ -17,6 +17,7 @@ const (
 	EOF        // end of file
 	Comment    // comment
 	Whitespace // whitespace
+	Tab        // horizontal tabulation
 	NewLine    // new line
 
 	Ident  // identifier
diff --git a/token/kind_string.go b/token/kind_string.go
index ffc52f1..98b39ab 100644
--- a/token/kind_string.go
+++ b/token/kind_string.go
@@ -12,73 +12,74 @@ func _() {
 	_ = x[EOF-1]
 	_ = x[Comment-2]
 	_ = x[Whitespace-3]
-	_ = x[NewLine-4]
-	_ = x[Ident-5]
-	_ = x[Int-6]
-	_ = x[Float-7]
-	_ = x[String-8]
-	_ = x[LParen-9]
-	_ = x[RParen-10]
-	_ = x[LCurly-11]
-	_ = x[RCurly-12]
-	_ = x[LBracket-13]
-	_ = x[RBracket-14]
-	_ = x[Comma-15]
-	_ = x[Colon-16]
-	_ = x[Semicolon-17]
-	_ = x[Eq-18]
-	_ = x[EqOp-19]
-	_ = x[Bang-20]
-	_ = x[NeOp-21]
-	_ = x[LtOp-22]
-	_ = x[LeOp-23]
-	_ = x[GtOp-24]
-	_ = x[GeOp-25]
-	_ = x[Plus-26]
-	_ = x[PlusEq-27]
-	_ = x[Minus-28]
-	_ = x[MinusEq-29]
-	_ = x[Asterisk-30]
-	_ = x[AsteriskEq-31]
-	_ = x[Slash-32]
-	_ = x[SlashEq-33]
-	_ = x[Percent-34]
-	_ = x[PercentEq-35]
-	_ = x[Amp-36]
-	_ = x[Pipe-37]
-	_ = x[Caret-38]
-	_ = x[At-39]
-	_ = x[QuestionMark-40]
-	_ = x[Arrow-41]
-	_ = x[FatArrow-42]
-	_ = x[Shl-43]
-	_ = x[Shr-44]
-	_ = x[Dot-45]
-	_ = x[Dot2-46]
-	_ = x[Dot2Less-47]
-	_ = x[Ellipsis-48]
-	_ = x[KwAnd-49]
-	_ = x[KwOr-50]
-	_ = x[KwModule-51]
-	_ = x[KwAlias-52]
-	_ = x[KwStruct-53]
-	_ = x[KwEnum-54]
-	_ = x[KwFunc-55]
-	_ = x[KwVal-56]
-	_ = x[KwVar-57]
-	_ = x[KwConst-58]
-	_ = x[KwOf-59]
-	_ = x[KwIf-60]
-	_ = x[KwElse-61]
-	_ = x[KwWhile-62]
-	_ = x[KwReturn-63]
-	_ = x[KwBreak-64]
-	_ = x[KwContinue-65]
+	_ = x[Tab-4]
+	_ = x[NewLine-5]
+	_ = x[Ident-6]
+	_ = x[Int-7]
+	_ = x[Float-8]
+	_ = x[String-9]
+	_ = x[LParen-10]
+	_ = x[RParen-11]
+	_ = x[LCurly-12]
+	_ = x[RCurly-13]
+	_ = x[LBracket-14]
+	_ = x[RBracket-15]
+	_ = x[Comma-16]
+	_ = x[Colon-17]
+	_ = x[Semicolon-18]
+	_ = x[Eq-19]
+	_ = x[EqOp-20]
+	_ = x[Bang-21]
+	_ = x[NeOp-22]
+	_ = x[LtOp-23]
+	_ = x[LeOp-24]
+	_ = x[GtOp-25]
+	_ = x[GeOp-26]
+	_ = x[Plus-27]
+	_ = x[PlusEq-28]
+	_ = x[Minus-29]
+	_ = x[MinusEq-30]
+	_ = x[Asterisk-31]
+	_ = x[AsteriskEq-32]
+	_ = x[Slash-33]
+	_ = x[SlashEq-34]
+	_ = x[Percent-35]
+	_ = x[PercentEq-36]
+	_ = x[Amp-37]
+	_ = x[Pipe-38]
+	_ = x[Caret-39]
+	_ = x[At-40]
+	_ = x[QuestionMark-41]
+	_ = x[Arrow-42]
+	_ = x[FatArrow-43]
+	_ = x[Shl-44]
+	_ = x[Shr-45]
+	_ = x[Dot-46]
+	_ = x[Dot2-47]
+	_ = x[Dot2Less-48]
+	_ = x[Ellipsis-49]
+	_ = x[KwAnd-50]
+	_ = x[KwOr-51]
+	_ = x[KwModule-52]
+	_ = x[KwAlias-53]
+	_ = x[KwStruct-54]
+	_ = x[KwEnum-55]
+	_ = x[KwFunc-56]
+	_ = x[KwVal-57]
+	_ = x[KwVar-58]
+	_ = x[KwConst-59]
+	_ = x[KwOf-60]
+	_ = x[KwIf-61]
+	_ = x[KwElse-62]
+	_ = x[KwWhile-63]
+	_ = x[KwReturn-64]
+	_ = x[KwBreak-65]
+	_ = x[KwContinue-66]
 }
 
-const _Kind_name = "IllegalEOFCommentWhitespaceNewLineIdentIntFloatStringLParenRParenLCurlyRCurlyLBracketRBracketCommaColonSemicolonEqEqOpBangNeOpLtOpLeOpGtOpGeOpPlusPlusEqMinusMinusEqAsteriskAsteriskEqSlashSlashEqPercentPercentEqAmpPipeCaretAtQuestionMarkArrowFatArrowShlShrDotDot2Dot2LessEllipsisKwAndKwOrKwModuleKwAliasKwStructKwEnumKwFuncKwValKwVarKwConstKwOfKwIfKwElseKwWhileKwReturnKwBreakKwContinue"
+const _Kind_name = "IllegalEOFCommentWhitespaceTabNewLineIdentIntFloatStringLParenRParenLCurlyRCurlyLBracketRBracketCommaColonSemicolonEqEqOpBangNeOpLtOpLeOpGtOpGeOpPlusPlusEqMinusMinusEqAsteriskAsteriskEqSlashSlashEqPercentPercentEqAmpPipeCaretAtQuestionMarkArrowFatArrowShlShrDotDot2Dot2LessEllipsisKwAndKwOrKwModuleKwAliasKwStructKwEnumKwFuncKwValKwVarKwConstKwOfKwIfKwElseKwWhileKwReturnKwBreakKwContinue"
 
-var _Kind_index = [...]uint16{0, 7, 10, 17, 27, 34, 39, 42, 47, 53, 59, 65, 71, 77, 85, 93, 98, 103, 112, 114, 118, 122, 126, 130, 134, 138, 142, 146, 152, 157, 164, 172, 182, 187, 194, 201, 210, 213, 217, 222, 224, 236, 241, 249, 252, 255, 258, 262, 270, 278, 283, 287, 295, 302, 310, 316, 322, 327, 332, 339, 343, 347, 353, 360, 368, 375, 385}
+var _Kind_index = [...]uint16{0, 7, 10, 17, 27, 30, 37, 42, 45, 50, 56, 62, 68, 74, 80, 88, 96, 101, 106, 115, 117, 121, 125, 129, 133, 137, 141, 145, 149, 155, 160, 167, 175, 185, 190, 197, 204, 213, 216, 220, 225, 227, 239, 244, 252, 255, 258, 261, 265, 273, 281, 286, 290, 298, 305, 313, 319, 325, 330, 335, 342, 346, 350, 356, 363, 371, 378, 388}
 
 func (i Kind) String() string {
 	if i >= Kind(len(_Kind_index)-1) {
diff --git a/token/kind_user_string.go b/token/kind_user_string.go
index 0077cb9..915e5d7 100644
--- a/token/kind_user_string.go
+++ b/token/kind_user_string.go
@@ -12,73 +12,74 @@ func _() {
 	_ = x[EOF-1]
 	_ = x[Comment-2]
 	_ = x[Whitespace-3]
-	_ = x[NewLine-4]
-	_ = x[Ident-5]
-	_ = x[Int-6]
-	_ = x[Float-7]
-	_ = x[String-8]
-	_ = x[LParen-9]
-	_ = x[RParen-10]
-	_ = x[LCurly-11]
-	_ = x[RCurly-12]
-	_ = x[LBracket-13]
-	_ = x[RBracket-14]
-	_ = x[Comma-15]
-	_ = x[Colon-16]
-	_ = x[Semicolon-17]
-	_ = x[Eq-18]
-	_ = x[EqOp-19]
-	_ = x[Bang-20]
-	_ = x[NeOp-21]
-	_ = x[LtOp-22]
-	_ = x[LeOp-23]
-	_ = x[GtOp-24]
-	_ = x[GeOp-25]
-	_ = x[Plus-26]
-	_ = x[PlusEq-27]
-	_ = x[Minus-28]
-	_ = x[MinusEq-29]
-	_ = x[Asterisk-30]
-	_ = x[AsteriskEq-31]
-	_ = x[Slash-32]
-	_ = x[SlashEq-33]
-	_ = x[Percent-34]
-	_ = x[PercentEq-35]
-	_ = x[Amp-36]
-	_ = x[Pipe-37]
-	_ = x[Caret-38]
-	_ = x[At-39]
-	_ = x[QuestionMark-40]
-	_ = x[Arrow-41]
-	_ = x[FatArrow-42]
-	_ = x[Shl-43]
-	_ = x[Shr-44]
-	_ = x[Dot-45]
-	_ = x[Dot2-46]
-	_ = x[Dot2Less-47]
-	_ = x[Ellipsis-48]
-	_ = x[KwAnd-49]
-	_ = x[KwOr-50]
-	_ = x[KwModule-51]
-	_ = x[KwAlias-52]
-	_ = x[KwStruct-53]
-	_ = x[KwEnum-54]
-	_ = x[KwFunc-55]
-	_ = x[KwVal-56]
-	_ = x[KwVar-57]
-	_ = x[KwConst-58]
-	_ = x[KwOf-59]
-	_ = x[KwIf-60]
-	_ = x[KwElse-61]
-	_ = x[KwWhile-62]
-	_ = x[KwReturn-63]
-	_ = x[KwBreak-64]
-	_ = x[KwContinue-65]
+	_ = x[Tab-4]
+	_ = x[NewLine-5]
+	_ = x[Ident-6]
+	_ = x[Int-7]
+	_ = x[Float-8]
+	_ = x[String-9]
+	_ = x[LParen-10]
+	_ = x[RParen-11]
+	_ = x[LCurly-12]
+	_ = x[RCurly-13]
+	_ = x[LBracket-14]
+	_ = x[RBracket-15]
+	_ = x[Comma-16]
+	_ = x[Colon-17]
+	_ = x[Semicolon-18]
+	_ = x[Eq-19]
+	_ = x[EqOp-20]
+	_ = x[Bang-21]
+	_ = x[NeOp-22]
+	_ = x[LtOp-23]
+	_ = x[LeOp-24]
+	_ = x[GtOp-25]
+	_ = x[GeOp-26]
+	_ = x[Plus-27]
+	_ = x[PlusEq-28]
+	_ = x[Minus-29]
+	_ = x[MinusEq-30]
+	_ = x[Asterisk-31]
+	_ = x[AsteriskEq-32]
+	_ = x[Slash-33]
+	_ = x[SlashEq-34]
+	_ = x[Percent-35]
+	_ = x[PercentEq-36]
+	_ = x[Amp-37]
+	_ = x[Pipe-38]
+	_ = x[Caret-39]
+	_ = x[At-40]
+	_ = x[QuestionMark-41]
+	_ = x[Arrow-42]
+	_ = x[FatArrow-43]
+	_ = x[Shl-44]
+	_ = x[Shr-45]
+	_ = x[Dot-46]
+	_ = x[Dot2-47]
+	_ = x[Dot2Less-48]
+	_ = x[Ellipsis-49]
+	_ = x[KwAnd-50]
+	_ = x[KwOr-51]
+	_ = x[KwModule-52]
+	_ = x[KwAlias-53]
+	_ = x[KwStruct-54]
+	_ = x[KwEnum-55]
+	_ = x[KwFunc-56]
+	_ = x[KwVal-57]
+	_ = x[KwVar-58]
+	_ = x[KwConst-59]
+	_ = x[KwOf-60]
+	_ = x[KwIf-61]
+	_ = x[KwElse-62]
+	_ = x[KwWhile-63]
+	_ = x[KwReturn-64]
+	_ = x[KwBreak-65]
+	_ = x[KwContinue-66]
 }
 
-const _Kind_user_name = "illegal characterend of filecommentwhitespacenew lineidentifieruntyped intuntyped floatuntyped string'('')''{''}''['']'','':'';'operator '='operator '=='operator '!'operator '!='operator '<'operator '<='operator '>'operator '>='operator '+'operator '+='operator '-'operator '-='operator '*'operator '*='operator '/'operator '/='operator '%'operator '%='operator '&'operator '|'operator '^'operator '@'operator '?'operator '->'operator '=>'operator '<<'operator '>>'operator '.'operator '..'operator '..<'operator '...'keyword 'and'keyword 'or'keyword 'module'keyword 'alias'keyword 'struct'keyword 'enum'keyword 'func'keyword 'val'keyword 'var'keyword 'const'keyword 'of'keyword 'if'keyword 'else'keyword 'while'keyword 'return'keyword 'break'keyword 'continue'"
+const _Kind_user_name = "illegal characterend of filecommentwhitespacehorizontal tabulationnew lineidentifieruntyped intuntyped floatuntyped string'('')''{''}''['']'','':'';'operator '='operator '=='operator '!'operator '!='operator '<'operator '<='operator '>'operator '>='operator '+'operator '+='operator '-'operator '-='operator '*'operator '*='operator '/'operator '/='operator '%'operator '%='operator '&'operator '|'operator '^'operator '@'operator '?'operator '->'operator '=>'operator '<<'operator '>>'operator '.'operator '..'operator '..<'operator '...'keyword 'and'keyword 'or'keyword 'module'keyword 'alias'keyword 'struct'keyword 'enum'keyword 'func'keyword 'val'keyword 'var'keyword 'const'keyword 'of'keyword 'if'keyword 'else'keyword 'while'keyword 'return'keyword 'break'keyword 'continue'"
 
-var _Kind_user_index = [...]uint16{0, 17, 28, 35, 45, 53, 63, 74, 87, 101, 104, 107, 110, 113, 116, 119, 122, 125, 128, 140, 153, 165, 178, 190, 203, 215, 228, 240, 253, 265, 278, 290, 303, 315, 328, 340, 353, 365, 377, 389, 401, 413, 426, 439, 452, 465, 477, 490, 504, 518, 531, 543, 559, 574, 590, 604, 618, 631, 644, 659, 671, 683, 697, 712, 728, 743, 761}
+var _Kind_user_index = [...]uint16{0, 17, 28, 35, 45, 66, 74, 84, 95, 108, 122, 125, 128, 131, 134, 137, 140, 143, 146, 149, 161, 174, 186, 199, 211, 224, 236, 249, 261, 274, 286, 299, 311, 324, 336, 349, 361, 374, 386, 398, 410, 422, 434, 447, 460, 473, 486, 498, 511, 525, 539, 552, 564, 580, 595, 611, 625, 639, 652, 665, 680, 692, 704, 718, 733, 749, 764, 782}
 
 func (i Kind) UserString() string {
 	if i >= Kind(len(_Kind_user_index)-1) {