lex: fix errors and cleanup

src/lex/mod.rs

@@ -42,7 +42,7 @@ impl Iterator for Lexer<'_> {
 
     fn next(&mut self) -> Option<Result<Token, SyntaxError>> {
         if self.offset > 0 {
-            let tmp = self.history[self.history.len() - self.offset];
+            let tmp = self.history[self.history.len() - self.offset].clone();
             self.offset -= 1;
             return Some(Ok(tmp));
         }
@@ -68,7 +68,7 @@ impl Iterator for Lexer<'_> {
             '%' => self.token_ok(token::Kind::Percent),
 
             '#' => {
-                self.read_comment();
+                self.read_comment().unwrap(); // this can't fail
                 // we don't need comments for now and they would
                 // only confuse the parser so let's just Not
                 self.next()?
@@ -81,8 +81,8 @@ impl Iterator for Lexer<'_> {
             c => self.syntax_error(format!("Unexpected character '{}'", c)),
         };
 
-        if let Ok(token) = result {
-            self.history.push(token);
+        if let Ok(token) = &result {
+            self.history.push(token.clone());
         }
         Some(result)
     }
@@ -99,6 +99,14 @@ impl<'a> Lexer<'a> {
         }
     }
 
+    pub fn current(&self) -> Option<&Token> {
+        if self.history.len() > 0 {
+            Some(&self.history[self.history.len() - self.offset - 1])
+        } else {
+            None
+        }
+    }
+
     pub fn peek(&mut self) -> Option<Result<Token, SyntaxError>> {
         let t = self.next()?;
         self.prev();
@@ -113,19 +121,20 @@ impl<'a> Lexer<'a> {
 
     pub fn expect_kind(&mut self, kind: token::Kind) -> Result<Token, SyntaxError> {
         match self.next() {
-            Some(t) => if t?.kind == kind {
-                Ok(t?)
+            Some(Ok(t)) => if t.kind == kind {
+                Ok(t)
             } else {
-                self.syntax_error(format!("Expected {}, got {}", kind, t?.kind))
-            }
-            None => self.syntax_error("Unexpected EOF"),
+                self.syntax_error(format!("Expected {}, got {}", kind, t.kind))
+            },
+            Some(Err(e)) => Err(e),
+            None => self.syntax_error(String::from("Unexpected EOF")),
         }
     }
 
     pub fn require_next(&mut self) -> Result<Token, SyntaxError> {
         match self.next() {
             Some(t) => t,
-            None => self.syntax_error("Unexpected EOF"),
+            None => self.syntax_error(String::from("Unexpected EOF")),
         }
     }
 
@@ -154,12 +163,13 @@ impl<'a> Lexer<'a> {
     }
 
     fn read_comment(&mut self) -> Result<Token, SyntaxError> {
+        assert_eq!(self.cursor.current(), Some('#'));
         self.cursor.seek_while(|c| c != '\n');
         self.token_ok(token::Kind::Comment)
     }
 
     fn read_string_literal(&mut self) -> Result<Token, SyntaxError> {
-        assert!(self.cursor.current() == Some('"'));
+        assert_eq!(self.cursor.current(), Some('"'));
         self.cursor.chop();
         let mut raw = String::new();
         for c in &mut self.cursor {
@@ -176,13 +186,13 @@ impl<'a> Lexer<'a> {
     }
 
     fn read_prefix_int_literal(&mut self) -> Result<Token, SyntaxError> {
-        assert!(self.cursor.next() == Some('0'));
+        assert_eq!(self.cursor.next(), Some('0'));
         match self.cursor.next() {
             Some('x') => self.read_int_literal(16),
             Some('o') => self.read_int_literal(8),
             Some('b') => self.read_int_literal(2),
             Some(c) => self.syntax_error(format!("Unexpected character '{}'", c)),
-            None => self.syntax_error("Unexpected end-of-file"),
+            None => self.syntax_error(String::from("Unexpected end-of-file")),
         }
     }
 
@@ -204,7 +214,7 @@ impl<'a> Lexer<'a> {
             kind,
             line: self.token_line,
             col: self.token_col,
-            raw: raw,
+            raw,
         };
         self.token_line = self.cursor.line();
         self.token_col = self.cursor.col();
@@ -233,11 +243,11 @@ impl<'a> Lexer<'a> {
         true
     }
 
-    fn syntax_error<T>(&mut self, msg: &str) -> Result<T, SyntaxError> {
+    fn syntax_error<T>(&mut self, msg: String) -> Result<T, SyntaxError> {
         Err(SyntaxError {
             line: self.cursor.line(),
             col: self.cursor.col(),
-            msg: String::from(msg),
+            msg,
         })
     }
 }

src/lex/token.rs

@@ -1,7 +1,7 @@
 use std::fmt;
 
 /// A single syntactic element.
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct Token {
     pub kind: Kind,
     /// line of the first character (starting from 1)