lex: fix errors and cleanup

main
anna 2 years ago
parent 50b6da41dd
commit 73d1e8aede
Signed by: fef
GPG Key ID: EC22E476DC2D3D84

@ -42,7 +42,7 @@ impl Iterator for Lexer<'_> {
fn next(&mut self) -> Option<Result<Token, SyntaxError>> { fn next(&mut self) -> Option<Result<Token, SyntaxError>> {
if self.offset > 0 { if self.offset > 0 {
let tmp = self.history[self.history.len() - self.offset]; let tmp = self.history[self.history.len() - self.offset].clone();
self.offset -= 1; self.offset -= 1;
return Some(Ok(tmp)); return Some(Ok(tmp));
} }
@ -68,7 +68,7 @@ impl Iterator for Lexer<'_> {
'%' => self.token_ok(token::Kind::Percent), '%' => self.token_ok(token::Kind::Percent),
'#' => { '#' => {
self.read_comment(); self.read_comment().unwrap(); // this can't fail
// we don't need comments for now and they would // we don't need comments for now and they would
// only confuse the parser so let's just Not // only confuse the parser so let's just Not
self.next()? self.next()?
@ -81,8 +81,8 @@ impl Iterator for Lexer<'_> {
c => self.syntax_error(format!("Unexpected character '{}'", c)), c => self.syntax_error(format!("Unexpected character '{}'", c)),
}; };
if let Ok(token) = result { if let Ok(token) = &result {
self.history.push(token); self.history.push(token.clone());
} }
Some(result) Some(result)
} }
@ -99,6 +99,14 @@ impl<'a> Lexer<'a> {
} }
} }
pub fn current(&self) -> Option<&Token> {
if self.history.len() > 0 {
Some(&self.history[self.history.len() - self.offset - 1])
} else {
None
}
}
pub fn peek(&mut self) -> Option<Result<Token, SyntaxError>> { pub fn peek(&mut self) -> Option<Result<Token, SyntaxError>> {
let t = self.next()?; let t = self.next()?;
self.prev(); self.prev();
@ -113,19 +121,20 @@ impl<'a> Lexer<'a> {
pub fn expect_kind(&mut self, kind: token::Kind) -> Result<Token, SyntaxError> { pub fn expect_kind(&mut self, kind: token::Kind) -> Result<Token, SyntaxError> {
match self.next() { match self.next() {
Some(t) => if t?.kind == kind { Some(Ok(t)) => if t.kind == kind {
Ok(t?) Ok(t)
} else { } else {
self.syntax_error(format!("Expected {}, got {}", kind, t?.kind)) self.syntax_error(format!("Expected {}, got {}", kind, t.kind))
} },
None => self.syntax_error("Unexpected EOF"), Some(Err(e)) => Err(e),
None => self.syntax_error(String::from("Unexpected EOF")),
} }
} }
pub fn require_next(&mut self) -> Result<Token, SyntaxError> { pub fn require_next(&mut self) -> Result<Token, SyntaxError> {
match self.next() { match self.next() {
Some(t) => t, Some(t) => t,
None => self.syntax_error("Unexpected EOF"), None => self.syntax_error(String::from("Unexpected EOF")),
} }
} }
@ -154,12 +163,13 @@ impl<'a> Lexer<'a> {
} }
fn read_comment(&mut self) -> Result<Token, SyntaxError> { fn read_comment(&mut self) -> Result<Token, SyntaxError> {
assert_eq!(self.cursor.current(), Some('#'));
self.cursor.seek_while(|c| c != '\n'); self.cursor.seek_while(|c| c != '\n');
self.token_ok(token::Kind::Comment) self.token_ok(token::Kind::Comment)
} }
fn read_string_literal(&mut self) -> Result<Token, SyntaxError> { fn read_string_literal(&mut self) -> Result<Token, SyntaxError> {
assert!(self.cursor.current() == Some('"')); assert_eq!(self.cursor.current(), Some('"'));
self.cursor.chop(); self.cursor.chop();
let mut raw = String::new(); let mut raw = String::new();
for c in &mut self.cursor { for c in &mut self.cursor {
@ -176,13 +186,13 @@ impl<'a> Lexer<'a> {
} }
fn read_prefix_int_literal(&mut self) -> Result<Token, SyntaxError> { fn read_prefix_int_literal(&mut self) -> Result<Token, SyntaxError> {
assert!(self.cursor.next() == Some('0')); assert_eq!(self.cursor.next(), Some('0'));
match self.cursor.next() { match self.cursor.next() {
Some('x') => self.read_int_literal(16), Some('x') => self.read_int_literal(16),
Some('o') => self.read_int_literal(8), Some('o') => self.read_int_literal(8),
Some('b') => self.read_int_literal(2), Some('b') => self.read_int_literal(2),
Some(c) => self.syntax_error(format!("Unexpected character '{}'", c)), Some(c) => self.syntax_error(format!("Unexpected character '{}'", c)),
None => self.syntax_error("Unexpected end-of-file"), None => self.syntax_error(String::from("Unexpected end-of-file")),
} }
} }
@ -204,7 +214,7 @@ impl<'a> Lexer<'a> {
kind, kind,
line: self.token_line, line: self.token_line,
col: self.token_col, col: self.token_col,
raw: raw, raw,
}; };
self.token_line = self.cursor.line(); self.token_line = self.cursor.line();
self.token_col = self.cursor.col(); self.token_col = self.cursor.col();
@ -233,11 +243,11 @@ impl<'a> Lexer<'a> {
true true
} }
fn syntax_error<T>(&mut self, msg: &str) -> Result<T, SyntaxError> { fn syntax_error<T>(&mut self, msg: String) -> Result<T, SyntaxError> {
Err(SyntaxError { Err(SyntaxError {
line: self.cursor.line(), line: self.cursor.line(),
col: self.cursor.col(), col: self.cursor.col(),
msg: String::from(msg), msg,
}) })
} }
} }

@ -1,7 +1,7 @@
use std::fmt; use std::fmt;
/// A single syntactic element. /// A single syntactic element.
#[derive(Debug)] #[derive(Debug, Clone)]
pub struct Token { pub struct Token {
pub kind: Kind, pub kind: Kind,
/// line of the first character (starting from 1) /// line of the first character (starting from 1)

Loading…
Cancel
Save