chore: a Tab token

saffage · May 9, 2024 · 57501b9 · 57501b9
1 parent 18b1d90
commit 57501b9
Show file tree

Hide file tree

Showing 6 changed files with 142 additions and 131 deletions.
diff --git a/internal/jet/jet.go b/internal/jet/jet.go
@@ -69,7 +69,7 @@ func process(
 	fileid config.FileID,
 	isRepl bool,
 ) {
-	toks, scanErrors := scanner.Scan(buffer, fileid, scanner.SkipWhitespace)
+	toks, scanErrors := scanner.Scan(buffer, fileid, scanner.SkipWhitespace|scanner.SkipComments)
 
 	if len(scanErrors) > 0 {
 		for _, err := range scanErrors {

diff --git a/parser/handle_tokens.go b/parser/handle_tokens.go
@@ -39,7 +39,8 @@ func (p *Parser) next() {
 		p.next()
 	}
 
-	if p.flags&SkipWhitespace != 0 && p.tok.Kind == token.Whitespace {
+	if p.flags&SkipWhitespace != 0 &&
+		(p.tok.Kind == token.Whitespace || p.tok.Kind == token.Tab) {
 		p.next()
 	}
 

diff --git a/scanner/scanner.go b/scanner/scanner.go
@@ -83,6 +83,12 @@ func (s *Scanner) Next() token.Token {
 				Data: s.TakeWhile(func(c byte) bool { return c == ' ' }),
 			}
 
+		case s.Match('\t'):
+			tok = token.Token{
+				Kind: token.Tab,
+				Data: string(s.Advance()),
+			}
+
 		case token.IsIdentifierStartChar(s.Peek()):
 			identifier := s.TakeWhile(token.IsIdentifierChar)
 
@@ -199,7 +205,8 @@ func (s *Scanner) Next() token.Token {
 			tok.End = s.PrevPos()
 		}
 
-		if s.flags&SkipWhitespace != 0 && tok.Kind == token.Whitespace {
+		if s.flags&SkipWhitespace != 0 &&
+			(tok.Kind == token.Whitespace || tok.Kind == token.Tab) {
 			return s.Next()
 		} else if s.flags&SkipIllegal != 0 && tok.Kind == token.Illegal {
 			return s.Next()

diff --git a/token/kind.go b/token/kind.go
@@ -17,6 +17,7 @@ const (
 	EOF        // end of file
 	Comment    // comment
 	Whitespace // whitespace
+	Tab        // horizontal tabulation
 	NewLine    // new line
 
 	Ident  // identifier

diff --git a/token/kind_string.go b/token/kind_string.go
diff --git a/token/kind_user_string.go b/token/kind_user_string.go