diff --git a/src/lexer.rs b/src/lexer.rs index 6d852f2..553c221 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -42,14 +42,14 @@ pub enum Symbol { } impl Lexer { - pub fn new(input: String) -> Lexer { + pub fn new(input: &str) -> Lexer { Lexer { input: input.chars().collect(), pos: 0, } } - pub fn next(&mut self) -> Result, Error> { + pub fn lex_any(&mut self) -> Result, Error> { let Some(&ch) = self.input.get(self.pos) else { return Ok(None); }; @@ -95,7 +95,7 @@ impl Lexer { }, ch if ch.is_whitespace() => { self.pos += 1; - return self.next(); + return self.lex_any(); } _ => self.lex_op()?, }; @@ -345,3 +345,11 @@ impl Lexer { Ok(Token::Symbol(symbol)) } } + +impl Iterator for Lexer { + type Item = Result; + + fn next(&mut self) -> Option { + self.lex_any().transpose() + } +} diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..83c38aa --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,10 @@ +#![allow(dead_code, clippy::single_match, clippy::only_used_in_recursion)] + +pub mod data; +pub mod error; +pub mod generator; +pub mod lexer; +pub mod parser; +pub mod types; + +pub use error::Error; diff --git a/src/main.rs b/src/main.rs index 2237473..f65aef4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,33 +4,20 @@ a second time when generating (so the types are known), there should be a better way */ -#![allow(dead_code, clippy::single_match, clippy::only_used_in_recursion)] - -mod data; -mod error; -mod generator; -mod lexer; -mod parser; -mod types; - -pub use error::Error; +use lang::{generator, lexer, parser}; use generator::Generator; fn main() { let source = std::fs::read_to_string(std::env::args().skip(1).next().expect("no filename!")).expect("no source!"); - let mut lexer = lexer::Lexer::new(source); + let lexer = lexer::Lexer::new(&source); - let mut tokens = vec![]; - loop { - match lexer.next() { - Ok(None) => break, - Ok(Some(token)) => tokens.push(token), - Err(error) => { - eprintln!("error: {:?}", error); - return; - } + let tokens = match lexer.collect::, _>>() { + Ok(tokens) => tokens, + Err(error) => { + eprintln!("error: {:?}", error); + return; } - } + }; // dbg!(&tokens); diff --git a/tests/lexer.rs b/tests/lexer.rs new file mode 100644 index 0000000..de8a531 --- /dev/null +++ b/tests/lexer.rs @@ -0,0 +1,14 @@ +use lang::{lexer::{Lexer, Token, Symbol}, Error}; + +#[test] +fn test_foo() { + let tokens: Result, Error> = Lexer::new("1 * 5 / 3").collect(); + let tokens = tokens.expect("should parse"); + assert_eq!(tokens, vec![ + Token::Number { radix: 10, text: "1".into() }, + Token::Symbol(Symbol::Star), + Token::Number { radix: 10, text: "5".into() }, + Token::Symbol(Symbol::Slash), + Token::Number { radix: 10, text: "3".into() }, + ]) +}