extern crate core; use core::str::Chars; #[derive(Debug, PartialEq)] pub enum Token { Number(i32), Plus, Minus, Multiply, Divide, } pub struct Lexer<'a> { chars: Chars<'a>, } impl<'a> Lexer<'a> { // Constructor pub fn new(input: &'a str) -> Self { Lexer { chars: input.chars() } } fn next_token(&mut self) -> Option { let next_char = self.chars.next()?; match next_char { '+' => Some(Token::Plus), '-' => Some(Token::Minus), '*' => Some(Token::Multiply), '/' => Some(Token::Divide), '1'..='9' => { let mut number = next_char.to_digit(10)? as i32; while let Some(next_char) = self.chars.clone().next() { if let Some(digit) = next_char.to_digit(10) { number = number * 10 + digit as i32; self.chars.next(); } else { break; } } Some(Token::Number(number)) } '0' => { let base = match self.chars.next().unwrap() { 'b' => Some(2), 'o' => Some(8), 'x' => Some(16), _ => None, }; if base.is_none() { return None; } let mut number = next_char.to_digit(base.clone().unwrap())?; while let Some(next_char) = self.chars.clone().next() { if let Some(digit) = next_char.to_digit(base.clone().unwrap()) { number = number * base.clone().unwrap() + digit; self.chars.next(); } else { break; } } Some(Token::Number(number as i32)) } _ => None, } } pub fn tokenize(&mut self) -> Vec { let mut tokens = Vec::new(); while let Some(token) = self.next_token() { tokens.push(token); } tokens } } fn main() { let mut lexer = Lexer::new("0x6A+0b001101*0o13"); let tokens = lexer.tokenize(); assert_eq!( tokens, vec![ Token::Number(106), Token::Plus, Token::Number(13), Token::Multiply, Token::Number(11), ] ); }