Skip to content

Commit

Permalink
lex: remove ws and \t as tokens
Browse files Browse the repository at this point in the history 
Signed-off-by: FedericoBruzzone <[email protected]>
  • Loading branch information
@FedericoBruzzone
FedericoBruzzone committed Aug 19, 2024
1 parent 63143fc commit e6053fc
Show file tree
Hide file tree
Showing 40 changed files with 6,787 additions and 1,210 deletions.
145 changes: 145 additions & 0 deletions src/lexer/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,3 +94,148 @@ impl std::fmt::Display for LexerError {
}
}
}

#[cfg(test)]
pub mod tests {
use crate::{
lexer::{token::Token, Lexer},
source::Source,
utils::file_handler::collect_fs_files,
};
use pretty_assertions::assert_eq;
use tracing::info;

#[test]
fn native_types() {
let fs_files = collect_fs_files("./testdata/native_types", true);
assert_eq!(fs_files.len(), 16);

for path in fs_files {
info!("file -> {:?}", path);
eprintln!("file -> {:?}", path);
let input = std::fs::File::open(path.clone()).unwrap();
let content = std::io::read_to_string(input).unwrap();
let source = Source::from(content);
let mut lexer = Lexer::new(&source);
let output_tokens = (&mut lexer).collect::<Vec<Token>>();

let fs_file = path.to_str().unwrap();
let tokens_file = fs_file.to_string().replace(".fs", ".tokens.json");
let tokens = std::fs::File::open(tokens_file.clone()).unwrap();
let expected_tokens: Vec<Token> = serde_json::from_reader(tokens).unwrap();
println!("{}", serde_json::to_string(&output_tokens).unwrap());
assert_eq!(output_tokens, expected_tokens);
}
}

#[test]
fn functions() {
let fs_files = collect_fs_files("./testdata/functions", true);
assert_eq!(fs_files.len(), 9);

for path in fs_files {
info!("file -> {:?}", path);
eprintln!("file -> {:?}", path);
let input = std::fs::File::open(path.clone()).unwrap();
let content = std::io::read_to_string(input).unwrap();
let source = Source::from(content);
let lexer = Lexer::new(&source);
let output_tokens = lexer.collect::<Vec<Token>>();

let fs_file = path.to_str().unwrap();
let tokens_file = fs_file.to_string().replace(".fs", ".tokens.json");
let tokens = std::fs::File::open(tokens_file).unwrap();
let expected_tokens: Vec<Token> = serde_json::from_reader(tokens).unwrap();
assert_eq!(output_tokens, expected_tokens);
}
}

#[test]
fn lists() {
let fs_files = collect_fs_files("./testdata/lists", true);
assert_eq!(fs_files.len(), 3);

for path in fs_files {
info!("file -> {:?}", path);
eprintln!("file -> {:?}", path);
let input = std::fs::File::open(path.clone()).unwrap();
let content = std::io::read_to_string(input).unwrap();
let source = Source::from(content);
let lexer = Lexer::new(&source);
let output_tokens = lexer.collect::<Vec<Token>>();

let fs_file = path.to_str().unwrap();
let tokens_file = fs_file.to_string().replace(".fs", ".tokens.json");
let tokens = std::fs::File::open(tokens_file).unwrap();
let expected_tokens: Vec<Token> = serde_json::from_reader(tokens).unwrap();
assert_eq!(output_tokens, expected_tokens);
}
}

#[test]
fn tuples() {
let fs_files = collect_fs_files("./testdata/tuples", true);
assert_eq!(fs_files.len(), 3);

for path in fs_files {
info!("file -> {:?}", path);
eprintln!("file -> {:?}", path);
let input = std::fs::File::open(path.clone()).unwrap();
let content = std::io::read_to_string(input).unwrap();
let source = Source::from(content);
let lexer = Lexer::new(&source);
let output_tokens = lexer.collect::<Vec<Token>>();

let fs_file = path.to_str().unwrap();
let tokens_file = fs_file.to_string().replace(".fs", ".tokens.json");
let tokens = std::fs::File::open(tokens_file).unwrap();
let expected_tokens: Vec<Token> = serde_json::from_reader(tokens).unwrap();
assert_eq!(output_tokens, expected_tokens);
}
}

#[test]
fn records() {
let fs_files = collect_fs_files("./testdata/records", true);
assert_eq!(fs_files.len(), 3);

for path in fs_files {
info!("file -> {:?}", path);
eprintln!("file -> {:?}", path);
let input = std::fs::File::open(path.clone()).unwrap();
let content = std::io::read_to_string(input).unwrap();
let source = Source::from(content);
let lexer = Lexer::new(&source);
let output_tokens = lexer.collect::<Vec<Token>>();

let fs_file = path.to_str().unwrap();
let tokens_file = fs_file.to_string().replace(".fs", ".tokens.json");
let tokens = std::fs::File::open(tokens_file).unwrap();
let expected_tokens: Vec<Token> = serde_json::from_reader(tokens).unwrap();
assert_eq!(output_tokens, expected_tokens);
}
}

#[test]
fn variants() {
let fs_files = collect_fs_files("./testdata/variants", true);
assert_eq!(fs_files.len(), 1);

for path in fs_files {
info!("file -> {:?}", path);
eprintln!("file -> {:?}", path);
let input = std::fs::File::open(path.clone()).unwrap();
let content = std::io::read_to_string(input).unwrap();
let source = Source::from(content);
let lexer = Lexer::new(&source);
let output_tokens = lexer.collect::<Vec<Token>>();

let fs_file = path.to_str().unwrap();
let tokens_file = fs_file.to_string().replace(".fs", ".tokens.json");
let tokens = std::fs::File::open(tokens_file).unwrap();
println!("{}", serde_json::to_string(&output_tokens).unwrap());
let expected_tokens: Vec<Token> = serde_json::from_reader(tokens).unwrap();
assert_eq!(output_tokens, expected_tokens);
}
}
}
10 changes: 4 additions & 6 deletions src/lexer/states.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,15 +61,13 @@ impl State for StateStart {
fn visit(&self, cursor: &mut Cursor) -> Result<Transition, LexerError> {
match cursor.peek() {
Some(c) if c.eq(&' ') || c.eq(&'\t') => {
cursor.advance_offset();
Ok(Lexer::proceed(
Box::new(StateStart),
TransitionKind::EmitToken(Token::new(
TokenKind::from(&c.to_string()),
c.to_string(),
cursor.location().clone(),
)),
TransitionKind::Consume,
))
// Uncomment to emit whitespace tokens
// cursor.advance_offset();
// Ok(Lexer::proceed(Box::new(StateStart),TransitionKind::EmitToken(Token::new(TokenKind::from(&c.to_string()),c.to_string(),cursor.location().clone(),)),))
}
Some(c) if c.eq(&'\r') => {
cursor.remove_carriage_return();
Expand Down
9 changes: 4 additions & 5 deletions src/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,9 @@ pub mod lexer;
pub mod logger;
pub mod parser;
pub mod source;
pub mod tests;
pub mod utils;

use lexer::Lexer;
use lexer::{token::Token, Lexer};
use logger::Logger;
use parser::Parser;
use source::Source;
Expand All @@ -32,14 +31,14 @@ fn main() {

let file_path: &str = &args[0];
let source = Source::new(file_path);
let lexer = Lexer::new(&source);
// let _tokens = (&mut lexer).collect::<Vec<Token>>();
let mut lexer = Lexer::new(&source);
let _tokens = (&mut lexer).collect::<Vec<Token>>();
// if lexer.errors().is_empty() {
// println!("No errors found");
// } else {
// lexer.emit_errors();
// }
let mut parser = Parser::new(lexer);
// let _ast = (&mut parser).collect::<Vec<parser::ast::Block>>();
let _ast = (&mut parser).collect::<Vec<parser::ast::Block>>();
println!("{:#?}", parser.parse());
}
143 changes: 143 additions & 0 deletions src/parser/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -213,3 +213,146 @@ impl Iterator for Parser<Lexer> {
self.parse_block()
}
}

#[cfg(test)]
pub mod tests {
use crate::{
lexer::Lexer, parser::Parser, source::Source, utils::file_handler::collect_fs_files,
};
use pretty_assertions::assert_eq;
use tracing::info;

#[test]
fn native_types() {
let fs_files = collect_fs_files("./testdata/native_types", true);
assert_eq!(fs_files.len(), 16);

let fs_files = fs_files.iter().filter(|p| p.ends_with("id_int_assign.fs"));

for path in fs_files {
info!("file -> {:?}", path);
eprintln!("file -> {:?}", path);
let input = std::fs::File::open(path.clone()).unwrap();
let content = std::io::read_to_string(input).unwrap();
let source = Source::from(content);

let fs_file = path.to_str().unwrap();
let output_ast = Parser::new(Lexer::new(&source)).parse();
let ast_file = fs_file.to_string().replace(".fs", ".ast.json");
let ast = std::fs::File::open(ast_file).unwrap();
// println!("{}", serde_json::to_string(&output_ast).unwrap());
let expected_ast = serde_json::from_reader(ast).unwrap();
assert_eq!(output_ast, expected_ast);
}
}

// #[test]
// fn functions() {
// let fs_files = collect_fs_files("./testdata/functions", true);
// assert_eq!(fs_files.len(), 9);
//
// for path in fs_files {
// info!("file -> {:?}", path);
// eprintln!("file -> {:?}", path);
// let input = std::fs::File::open(path.clone()).unwrap();
// let content = std::io::read_to_string(input).unwrap();
// let source = Source::from(content);
// let lexer = Lexer::new(&source);
// let output_tokens = lexer.collect::<Vec<Token>>();
//
// let tokens_file = path.to_str().unwrap();
// let tokens_file = tokens_file.to_string().replace(".fs", ".tokens.json");
// let tokens = std::fs::File::open(tokens_file).unwrap();
// let expected_tokens: Vec<Token> = serde_json::from_reader(tokens).unwrap();
// assert_eq!(output_tokens, expected_tokens);
// }
// }
//
// #[test]
// fn lists() {
// let fs_files = collect_fs_files("./testdata/lists", true);
// assert_eq!(fs_files.len(), 3);
//
// for path in fs_files {
// info!("file -> {:?}", path);
// eprintln!("file -> {:?}", path);
// let input = std::fs::File::open(path.clone()).unwrap();
// let content = std::io::read_to_string(input).unwrap();
// let source = Source::from(content);
// let lexer = Lexer::new(&source);
// let output_tokens = lexer.collect::<Vec<Token>>();
//
// let tokens_file = path.to_str().unwrap();
// let tokens_file = tokens_file.to_string().replace(".fs", ".tokens.json");
// let tokens = std::fs::File::open(tokens_file).unwrap();
// let expected_tokens: Vec<Token> = serde_json::from_reader(tokens).unwrap();
// assert_eq!(output_tokens, expected_tokens);
// }
// }
//
// #[test]
// fn tuples() {
// let fs_files = collect_fs_files("./testdata/tuples", true);
// assert_eq!(fs_files.len(), 3);
//
// for path in fs_files {
// info!("file -> {:?}", path);
// eprintln!("file -> {:?}", path);
// let input = std::fs::File::open(path.clone()).unwrap();
// let content = std::io::read_to_string(input).unwrap();
// let source = Source::from(content);
// let lexer = Lexer::new(&source);
// let output_tokens = lexer.collect::<Vec<Token>>();
//
// let tokens_file = path.to_str().unwrap();
// let tokens_file = tokens_file.to_string().replace(".fs", ".tokens.json");
// let tokens = std::fs::File::open(tokens_file).unwrap();
// let expected_tokens: Vec<Token> = serde_json::from_reader(tokens).unwrap();
// assert_eq!(output_tokens, expected_tokens);
// }
// }
//
// #[test]
// fn records() {
// let fs_files = collect_fs_files("./testdata/records", true);
// assert_eq!(fs_files.len(), 3);
//
// for path in fs_files {
// info!("file -> {:?}", path);
// eprintln!("file -> {:?}", path);
// let input = std::fs::File::open(path.clone()).unwrap();
// let content = std::io::read_to_string(input).unwrap();
// let source = Source::from(content);
// let lexer = Lexer::new(&source);
// let output_tokens = lexer.collect::<Vec<Token>>();
//
// let tokens_file = path.to_str().unwrap();
// let tokens_file = tokens_file.to_string().replace(".fs", ".tokens.json");
// let tokens = std::fs::File::open(tokens_file).unwrap();
// let expected_tokens: Vec<Token> = serde_json::from_reader(tokens).unwrap();
// assert_eq!(output_tokens, expected_tokens);
// }
// }
//
// #[test]
// fn variants() {
// let fs_files = collect_fs_files("./testdata/variants", true);
// assert_eq!(fs_files.len(), 1);
//
// for path in fs_files {
// info!("file -> {:?}", path);
// eprintln!("file -> {:?}", path);
// let input = std::fs::File::open(path.clone()).unwrap();
// let content = std::io::read_to_string(input).unwrap();
// let source = Source::from(content);
// let lexer = Lexer::new(&source);
// let output_tokens = lexer.collect::<Vec<Token>>();
//
// let tokens_file = path.to_str().unwrap();
// let tokens_file = tokens_file.to_string().replace(".fs", ".tokens.json");
// let tokens = std::fs::File::open(tokens_file).unwrap();
// let expected_tokens: Vec<Token> = serde_json::from_reader(tokens).unwrap();
// assert_eq!(output_tokens, expected_tokens);
// }
// }
}
Loading

0 comments on commit e6053fc

Please sign in to comment.