drew/acadia
1
0
Fork 0
Code Issues 6 Pull requests Projects Releases Packages Wiki Activity Actions

Create a first pass at a rust parser for the yunq language.

Browse source
This commit is contained in:
Drew Galbraith 2024-06-11 13:01:58 -07:00
parent 5b1debde54
commit 1cda053758
6 changed files with 793 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

225
yunq/rust/src/lexer.rs Normal file
View file

@ -0,0 +1,225 @@
#[derive(Debug, PartialEq)]
pub enum TokenType {
EndOfFile,
Name,
LeftBrace,
RightBrace,
LeftParen,
RightParen,
Arrow,
Semicolon,
Dot,
Equals,
}
#[derive(Debug)]
pub struct Token {
pub token_type: TokenType,
line: usize,
start: usize,
end: usize,
pub chars: String,
}
impl std::fmt::Display for Token {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"'{}' at line {} position {} ",
self.chars, self.line, self.start
)
}
}
pub fn lex_input(input: &str) -> Result<Vec<Token>, String> {
let mut index = 0;
let mut position = 0;
let mut line = 1;
let mut tokens = Vec::new();
let mut chars = input.chars().peekable();
loop {
match chars.next() {
Some(c) => {
position += 1;
match c {
'\n' => {
position = 0;
line += 1;
}
' ' | '\t' | '\r' => {}
'/' => {
match chars.next() {
Some('/') => {
index += 1;
// TODO: Add a consume until function.
loop {
match chars.next() {
Some('\n') => {
index += 1;
position = 0;
line += 1;
break;
}
Some(_) => {
index += 1;
}
None => break,
}
}
}
_ => {
return Err(format!("Unexpected token '/' at line {} position {}", line, position));
}
}
}
'{' => tokens.push(Token{
token_type: TokenType::LeftBrace,
line,
start: position,
end: position + 1,
chars: input[index..index+1].to_string(),
}),
'}' => tokens.push(Token{
token_type: TokenType::RightBrace,
line,
start: position,
end: position + 1,
chars: input[index..index+1].to_string(),
}),
'(' => tokens.push(Token{
token_type: TokenType::LeftParen,
line,
start: position,
end: position + 1,
chars: input[index..index+1].to_string(),
}),
')' => tokens.push(Token{
token_type: TokenType::RightParen,
line,
start: position,
end: position + 1,
chars: input[index..index+1].to_string(),
}),
';' => tokens.push(Token{
token_type: TokenType::Semicolon,
line,
start: position,
end: position + 1,
chars: input[index..index+1].to_string(),
}),
'.' => tokens.push(Token{
token_type: TokenType::Dot,
line,
start: position,
end: position + 1,
chars: input[index..index+1].to_string(),
}),
'=' => tokens.push(Token{
token_type: TokenType::Equals,
line,
start: position,
end: position + 1,
chars: input[index..index+1].to_string(),
}),
'-' => match chars.next() {
Some('>') => {
position += 1;
tokens.push(Token{
token_type: TokenType::Arrow,
line,
start: position - 1,
end: position + 1,
chars: input[index..index+1].to_string(),
});
index += 1;
}
Some(c) => return Err(format!("Expected > to follow - (to form arrow '->') on line {} at position {}. But got '{}'", line, position + 1, c)),
None => return Err(format!("Expected > to follow - but got end of input instead on line {} position {}", line, position))
},
'a'..='z' | 'A'..='Z' => {
let name_start = index;
let pos_start = position;
loop {
match chars.peek() {
Some(c) => {
match c {
'a'..='z' | 'A'..='Z' | '0'..='9' | '_' => {
position += 1;
chars.next();
index += 1;
},
'\n' => {
chars.next();
tokens.push(Token{
token_type: TokenType::Name,
line,
start: pos_start,
end: position + 1,
chars: input[name_start..index+1].to_string(),
});
position = 0;
index += 1;
line += 1;
break;
},
_ => {
tokens.push(Token{
token_type: TokenType::Name,
line,
start: pos_start,
end: position + 1,
chars: input[name_start..index+1].to_string()
});
break;
},
}
}
None => {
tokens.push(Token{
token_type: TokenType::Name,
line,
start: pos_start,
end: position + 1,
chars: input[name_start..index].to_string(),
});
break;
}
}
}
},
_ => {
return Err(format!(
"Unexpected token on line {} character {}: {}",
line, position, c
))
}
}
}
None => {
tokens.push(Token {
token_type: TokenType::EndOfFile,
line,
start: position,
end: position,
chars: "EOF".to_string(),
});
break;
}
}
index += 1;
}
Ok(tokens)
}