diff --git a/src/lexer/mod.rs b/src/lexer/mod.rs index def5c14..705eb05 100644 --- a/src/lexer/mod.rs +++ b/src/lexer/mod.rs @@ -123,7 +123,7 @@ pub mod tests { let tokens_file = fs_file.to_string().replace(".fs", ".tokens.json"); let tokens = std::fs::File::open(tokens_file.clone()).unwrap(); let expected_tokens: Vec = serde_json::from_reader(tokens).unwrap(); - println!("{}", serde_json::to_string(&output_tokens).unwrap()); + // println!("{}", serde_json::to_string(&output_tokens).unwrap()); assert_eq!(output_tokens, expected_tokens); } } @@ -233,7 +233,7 @@ pub mod tests { let fs_file = path.to_str().unwrap(); let tokens_file = fs_file.to_string().replace(".fs", ".tokens.json"); let tokens = std::fs::File::open(tokens_file).unwrap(); - println!("{}", serde_json::to_string(&output_tokens).unwrap()); + // println!("{}", serde_json::to_string(&output_tokens).unwrap()); let expected_tokens: Vec = serde_json::from_reader(tokens).unwrap(); assert_eq!(output_tokens, expected_tokens); } diff --git a/src/parser/ast.rs b/src/parser/ast.rs index 2eb0f02..c78755a 100644 --- a/src/parser/ast.rs +++ b/src/parser/ast.rs @@ -14,8 +14,16 @@ pub struct Block { #[derive(Debug, Deserialize, Serialize, PartialEq)] pub enum Stmt { - Assign { lhs: Expr, type_: Type, rhs: Expr }, + Assign { + lhs: Expr, + type_: Type, + rhs: Expr, + }, Expr(Expr), + Comment { + comment: String, + location: TokenLocation, + }, } #[derive(Debug, Deserialize, Serialize, PartialEq)] diff --git a/src/parser/mod.rs b/src/parser/mod.rs index c4eca4b..f08f31b 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -69,13 +69,20 @@ impl> Parser { info!("Parsed identifier - {:?}", stms); Some(stms) } + Some(Token { + kind: TokenKind::TokenComment, + .. + }) => { + let comment = self.parse_comment_stmt(); + info!("Parsed comment - {:?}", comment); + Some(comment) + } _ => todo!(), } } fn parse_identifier_stmt(&mut self) -> ast::Stmt { - let lhs = self.curr_token.clone().unwrap(); // Safe to unwrap because we checked for Some - // in parse_stmt + let lhs = self.curr_token.clone().unwrap(); // Safe to unwrap self.consume(); match self.curr_token { @@ -88,39 +95,61 @@ impl> Parser { Some(Token { kind: TokenKind::TokenKeyword(_), .. + }) => self.parse_assign_stmt(lhs), + _ => todo!(), // Match `(` and parse a function + } + } + _ => todo!(), + } + } + + fn parse_assign_stmt(&mut self, lhs: Token) -> ast::Stmt { + let type_ = self.parse_type(); + info!("Parsed type - {:?}", type_); + self.consume(); + match self.curr_token { + Some(Token { + kind: TokenKind::TokenAssign, + .. + }) => { + self.consume(); + let rhs = self.parse_expr(); + info!("Parsed expr - {:?}", rhs); + self.consume(); + match self.curr_token { + Some(Token { + kind: TokenKind::TokenNewLine, + .. }) => { - let type_ = self.parse_type(); self.consume(); - match self.curr_token { - Some(Token { - kind: TokenKind::TokenAssign, - .. - }) => { - self.consume(); - let rhs = self.parse_expr(); - self.consume(); - match self.curr_token { - Some(Token { - kind: TokenKind::TokenNewLine, - .. - }) => { - self.consume(); - ast::Stmt::Assign { - lhs: ast::Expr::Identifier { - name: lhs.lexeme, - location: lhs.location, - }, - type_, - rhs, - } - } - _ => todo!(), - } - } - _ => todo!(), + ast::Stmt::Assign { + lhs: ast::Expr::Identifier { + name: lhs.lexeme, + location: lhs.location, + }, + type_, + rhs, } } - _ => todo!(), // Match `(` and parse a function + _ => todo!(), + } + } + _ => todo!(), + } + } + + fn parse_comment_stmt(&mut self) -> ast::Stmt { + let comment = self.curr_token.clone().unwrap(); // Safe to unwrap + self.consume(); + match self.curr_token { + Some(Token { + kind: TokenKind::TokenNewLine, + .. + }) => { + self.consume(); + ast::Stmt::Comment { + comment: comment.lexeme, + location: comment.location, } } _ => todo!(), @@ -222,11 +251,16 @@ pub mod tests { let fs_files = fs_files.iter().filter(|p| { p.ends_with("id_int_assign.fs") + || p.ends_with("id_int_assign_2.fs") + || p.ends_with("id_int_assign_with_len_one.fs") + || p.ends_with("id_int_assign_with_spaces.fs") || p.ends_with("id_float_assign.fs") || p.ends_with("id_bool_true_assign.fs") || p.ends_with("id_bool_false_assign.fs") || p.ends_with("id_str_assign.fs") || p.ends_with("id_str_assign_multiple_words.fs") + || p.ends_with("comment.fs") + || p.ends_with("comment_and_id_int.fs") }); for path in fs_files { @@ -243,7 +277,7 @@ pub mod tests { let output_ast = Parser::new(source.clone(), Lexer::new(&source)).parse(); let ast_file = fs_file.to_string().replace(".fs", ".ast.json"); let ast = std::fs::File::open(ast_file).unwrap(); - // println!("{}", serde_json::to_string(&output_ast.root).unwrap()); + println!("{}", serde_json::to_string(&output_ast.root).unwrap()); let expected_ast = serde_json::from_reader(ast).unwrap(); assert_eq!(output_ast.root, expected_ast); } diff --git a/testdata/native_types/comment.ast.json b/testdata/native_types/comment.ast.json new file mode 100644 index 0000000..f207d10 --- /dev/null +++ b/testdata/native_types/comment.ast.json @@ -0,0 +1,15 @@ +{ + "stmts": [ + { + "Comment": { + "comment": "# this is a comment", + "location": { + "file_path": "", + "line": 0, + "column_start": 0, + "column_end": 19 + } + } + } + ] +} diff --git a/testdata/native_types/comment_and_id_int.ast.json b/testdata/native_types/comment_and_id_int.ast.json new file mode 100644 index 0000000..8c5a6cd --- /dev/null +++ b/testdata/native_types/comment_and_id_int.ast.json @@ -0,0 +1,44 @@ +{ + "stmts": [ + { + "Comment": { + "comment": "# this is a comment", + "location": { + "file_path": "", + "line": 0, + "column_start": 0, + "column_end": 19 + } + } + }, + { + "Assign": { + "lhs": { + "Identifier": { + "name": "x", + "location": { + "file_path": "", + "line": 1, + "column_start": 0, + "column_end": 1 + } + } + }, + "type_": "Int", + "rhs": { + "Literal": { + "literal": { + "Int": 0 + }, + "location": { + "file_path": "", + "line": 1, + "column_start": 9, + "column_end": 10 + } + } + } + } + } + ] +} diff --git a/testdata/native_types/id_int_assign_2.ast.json b/testdata/native_types/id_int_assign_2.ast.json new file mode 100644 index 0000000..6155211 --- /dev/null +++ b/testdata/native_types/id_int_assign_2.ast.json @@ -0,0 +1,33 @@ +{ + "stmts": [ + { + "Assign": { + "lhs": { + "Identifier": { + "name": "_x_int", + "location": { + "file_path": "", + "line": 0, + "column_start": 0, + "column_end": 6 + } + } + }, + "type_": "Int", + "rhs": { + "Literal": { + "literal": { + "Int": 732 + }, + "location": { + "file_path": "", + "line": 0, + "column_start": 14, + "column_end": 17 + } + } + } + } + } + ] +} diff --git a/testdata/native_types/id_int_assign_with_len_one.ast.json b/testdata/native_types/id_int_assign_with_len_one.ast.json new file mode 100644 index 0000000..3dedcf5 --- /dev/null +++ b/testdata/native_types/id_int_assign_with_len_one.ast.json @@ -0,0 +1,33 @@ +{ + "stmts": [ + { + "Assign": { + "lhs": { + "Identifier": { + "name": "i", + "location": { + "file_path": "", + "line": 0, + "column_start": 0, + "column_end": 1 + } + } + }, + "type_": "Int", + "rhs": { + "Literal": { + "literal": { + "Int": 1 + }, + "location": { + "file_path": "", + "line": 0, + "column_start": 9, + "column_end": 10 + } + } + } + } + } + ] +} diff --git a/testdata/native_types/id_int_assign_with_spaces.ast.json b/testdata/native_types/id_int_assign_with_spaces.ast.json new file mode 100644 index 0000000..39defc7 --- /dev/null +++ b/testdata/native_types/id_int_assign_with_spaces.ast.json @@ -0,0 +1,33 @@ +{ + "stmts": [ + { + "Assign": { + "lhs": { + "Identifier": { + "name": "_x_int", + "location": { + "file_path": "", + "line": 0, + "column_start": 4, + "column_end": 10 + } + } + }, + "type_": "Int", + "rhs": { + "Literal": { + "literal": { + "Int": 0 + }, + "location": { + "file_path": "", + "line": 0, + "column_start": 22, + "column_end": 23 + } + } + } + } + } + ] +} diff --git a/testdata/native_types/id_int_assign_with_spaces.fs b/testdata/native_types/id_int_assign_with_spaces.fs index be14f08..84a2236 100644 --- a/testdata/native_types/id_int_assign_with_spaces.fs +++ b/testdata/native_types/id_int_assign_with_spaces.fs @@ -1 +1 @@ - _x_int: int = 0 \ No newline at end of file + _x_int: int = 0 diff --git a/testdata/native_types/id_int_assign_with_spaces.tokens.json b/testdata/native_types/id_int_assign_with_spaces.tokens.json index 9ffa2bf..00d0b57 100644 --- a/testdata/native_types/id_int_assign_with_spaces.tokens.json +++ b/testdata/native_types/id_int_assign_with_spaces.tokens.json @@ -54,13 +54,23 @@ } }, { - "kind": "TokenEOF", - "lexeme": "", + "kind": "TokenNewLine", + "lexeme": "\\n", "location": { "file_path": "", "line": 0, "column_start": 23, "column_end": 23 } + }, + { + "kind": "TokenEOF", + "lexeme": "", + "location": { + "file_path": "", + "line": 1, + "column_start": 0, + "column_end": 0 + } } ]