1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
|
extern crate core;
use core::str::Chars;
#[derive(Debug, PartialEq)]
pub enum Token {
Number(i32),
Plus,
Minus,
Multiply,
Divide,
}
pub struct Lexer<'a> {
chars: Chars<'a>,
}
impl<'a> Lexer<'a> {
// Constructor
pub fn new(input: &'a str) -> Self {
Lexer { chars: input.chars() }
}
fn next_token(&mut self) -> Option<Token> {
let next_char = self.chars.next()?;
match next_char {
'+' => Some(Token::Plus),
'-' => Some(Token::Minus),
'*' => Some(Token::Multiply),
'/' => Some(Token::Divide),
'1'..='9' => {
let mut number = next_char.to_digit(10)? as i32;
while let Some(next_char) = self.chars.clone().next() {
if let Some(digit) = next_char.to_digit(10) {
number = number * 10 + digit as i32;
self.chars.next();
} else {
break;
}
}
Some(Token::Number(number))
}
'0' => {
let base = match self.chars.next().unwrap() {
'b' => Some(2),
'o' => Some(8),
'x' => Some(16),
_ => None,
};
if base.is_none() {
return None;
}
let mut number = next_char.to_digit(base.clone().unwrap())?;
while let Some(next_char) = self.chars.clone().next() {
if let Some(digit) = next_char.to_digit(base.clone().unwrap()) {
number = number * base.clone().unwrap() + digit;
self.chars.next();
} else {
break;
}
}
Some(Token::Number(number as i32))
}
_ => None,
}
}
pub fn tokenize(&mut self) -> Vec<Token> {
let mut tokens = Vec::new();
while let Some(token) = self.next_token() {
tokens.push(token);
}
tokens
}
}
fn main() {
let mut lexer = Lexer::new("0x6A+0b001101*0o13");
let tokens = lexer.tokenize();
assert_eq!(
tokens,
vec![
Token::Number(106),
Token::Plus,
Token::Number(13),
Token::Multiply,
Token::Number(11),
]
);
}
|
