add lexer and test file
There are most likely gonna be a lot of changes until we reach the final syntax, but this is a good start i think.main
parent
0557369397
commit
8ae3bb2f57
@ -1 +1,204 @@
|
||||
use std::fmt;
|
||||
use std::str::Chars;
|
||||
|
||||
mod cursor;
|
||||
use cursor::Cursor;
|
||||
|
||||
mod token;
|
||||
use token::Token;
|
||||
|
||||
pub struct Lexer<'a> {
|
||||
cursor: Cursor<'a>,
|
||||
token_line: usize,
|
||||
token_col: usize,
|
||||
}
|
||||
|
||||
static NUMERALS: [char; 16] = [
|
||||
'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f',
|
||||
];
|
||||
|
||||
struct KeywordMap {
|
||||
raw: &'static str,
|
||||
kind: token::Kind,
|
||||
}
|
||||
|
||||
const fn kw(raw: &'static str, kind: token::Kind) -> KeywordMap {
|
||||
KeywordMap { raw, kind }
|
||||
}
|
||||
|
||||
static KEYWORDS: [KeywordMap; 6] = [
|
||||
kw("depend", token::Kind::DependKeyword),
|
||||
kw("include", token::Kind::IncludeKeyword),
|
||||
kw("module", token::Kind::ModuleKeyword),
|
||||
kw("set", token::Kind::SetKeyword),
|
||||
kw("source", token::Kind::SourceKeyword),
|
||||
kw("type", token::Kind::TypeKeyword),
|
||||
];
|
||||
|
||||
impl Iterator for Lexer<'_> {
|
||||
type Item = Result<Token, SyntaxError>;
|
||||
|
||||
fn next(&mut self) -> Option<Result<Token, SyntaxError>> {
|
||||
Some(match self.cursor.next()? {
|
||||
c if c.is_ascii_whitespace() => {
|
||||
self.cursor.skip_whitespace();
|
||||
self.cursor.chop();
|
||||
self.next()?
|
||||
}
|
||||
',' => self.token_ok(token::Kind::Comma),
|
||||
';' => self.token_ok(token::Kind::Semi),
|
||||
'=' => self.token_ok(token::Kind::Eq),
|
||||
'{' => self.token_ok(token::Kind::OBrace),
|
||||
'}' => self.token_ok(token::Kind::CBrace),
|
||||
'[' => self.token_ok(token::Kind::OBracket),
|
||||
']' => self.token_ok(token::Kind::CBracket),
|
||||
'#' => self.read_comment(),
|
||||
'"' => self.read_string_literal(),
|
||||
'0' => self.read_prefix_int_literal(),
|
||||
_c @ '1'..='9' => self.read_int_literal(10),
|
||||
_c @ 'A'..='Z' => self.read_ident(),
|
||||
_c @ 'a'..='z' => self.read_keyword_or_ident(), // keywords are always lowercase
|
||||
c => self.syntax_error(format!("Unexpected character '{}'", c)),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> Lexer<'a> {
|
||||
pub fn new(stream: Chars<'a>) -> Lexer<'a> {
|
||||
Lexer {
|
||||
cursor: Cursor::new(stream),
|
||||
token_line: 1,
|
||||
token_col: 1,
|
||||
}
|
||||
}
|
||||
|
||||
fn read_keyword_or_ident(&mut self) -> Result<Token, SyntaxError> {
|
||||
let current = self.cursor.current().unwrap();
|
||||
for kw in &KEYWORDS {
|
||||
// keywords are always at least 2 characters long as per the language spec
|
||||
let first_char = kw.raw.chars().next().unwrap();
|
||||
if current == first_char && self.skip_if_match(&kw.raw[1..]) {
|
||||
return self.token_ok(kw.kind);
|
||||
}
|
||||
}
|
||||
|
||||
self.read_ident()
|
||||
}
|
||||
|
||||
fn read_ident(&mut self) -> Result<Token, SyntaxError> {
|
||||
for c in &mut self.cursor {
|
||||
if !c.is_ascii_alphanumeric() && c != '_' {
|
||||
self.cursor.prev();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
self.token_ok(token::Kind::Ident)
|
||||
}
|
||||
|
||||
fn read_comment(&mut self) -> Result<Token, SyntaxError> {
|
||||
self.cursor.seek_while(|c| c != '\n');
|
||||
self.token_ok(token::Kind::Comment)
|
||||
}
|
||||
|
||||
fn read_string_literal(&mut self) -> Result<Token, SyntaxError> {
|
||||
assert!(self.cursor.current() == Some('"'));
|
||||
self.cursor.chop();
|
||||
let mut raw = String::new();
|
||||
for c in &mut self.cursor {
|
||||
if c == '"' {
|
||||
self.cursor.chop();
|
||||
return self.token_raw_ok(token::Kind::StringLiteral, raw);
|
||||
} else {
|
||||
raw.push(c);
|
||||
}
|
||||
}
|
||||
|
||||
self.token_ok(token::Kind::StringLiteral)
|
||||
}
|
||||
|
||||
fn read_prefix_int_literal(&mut self) -> Result<Token, SyntaxError> {
|
||||
assert!(self.cursor.next() == Some('0'));
|
||||
match self.cursor.next() {
|
||||
Some('x') => self.read_int_literal(16),
|
||||
Some('o') => self.read_int_literal(8),
|
||||
Some('b') => self.read_int_literal(2),
|
||||
Some(c) => self.syntax_error(format!("Unexpected character '{}'", c)),
|
||||
None => self.syntax_error(String::from("Unexpected end-of-file")),
|
||||
}
|
||||
}
|
||||
|
||||
fn read_int_literal(&mut self, base: usize) -> Result<Token, SyntaxError> {
|
||||
assert!(base >= 2 && base <= 16);
|
||||
|
||||
for c in &mut self.cursor {
|
||||
if !NUMERALS[0..base].contains(&c.to_ascii_lowercase()) {
|
||||
self.cursor.prev();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
self.token_ok(token::Kind::IntLiteral)
|
||||
}
|
||||
|
||||
fn token(&mut self, kind: token::Kind, raw: String) -> Token {
|
||||
let t = Token {
|
||||
kind,
|
||||
line: self.token_line,
|
||||
col: self.token_col,
|
||||
raw: raw,
|
||||
};
|
||||
self.token_line = self.cursor.line();
|
||||
self.token_col = self.cursor.col();
|
||||
t
|
||||
}
|
||||
|
||||
fn token_ok<T>(&mut self, kind: token::Kind) -> Result<Token, T> {
|
||||
let raw = self.cursor.chop();
|
||||
Ok(self.token(kind, raw))
|
||||
}
|
||||
|
||||
fn token_raw_ok<T>(&mut self, kind: token::Kind, raw: String) -> Result<Token, T> {
|
||||
Ok(self.token(kind, raw))
|
||||
}
|
||||
|
||||
fn skip_if_match(&mut self, s: &str) -> bool {
|
||||
let mut n: usize = 0;
|
||||
for c in s.chars() {
|
||||
if self.cursor.next() == Some(c) {
|
||||
n += 1;
|
||||
} else {
|
||||
self.cursor.seek_back(n);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
fn syntax_error<T>(&mut self, msg: String) -> Result<T, SyntaxError> {
|
||||
Err(SyntaxError {
|
||||
line: self.cursor.line(),
|
||||
col: self.cursor.col(),
|
||||
msg: String::from(msg),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct SyntaxError {
|
||||
pub line: usize,
|
||||
pub col: usize,
|
||||
pub msg: String,
|
||||
}
|
||||
|
||||
impl fmt::Display for SyntaxError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
write!(
|
||||
f,
|
||||
"SyntaxError in {}:{}: {}",
|
||||
self.line,
|
||||
self.col,
|
||||
self.msg.as_str()
|
||||
)
|
||||
}
|
||||
}
|
||||
|
@ -0,0 +1,72 @@
|
||||
use std::fmt;
|
||||
|
||||
/// A single syntactic element.
|
||||
#[derive(Debug)]
|
||||
pub struct Token {
|
||||
pub kind: Kind,
|
||||
/// line of the first character (starting from 1)
|
||||
pub line: usize,
|
||||
/// column of the first character (starting from 1)
|
||||
pub col: usize,
|
||||
/// raw text
|
||||
pub raw: String,
|
||||
}
|
||||
|
||||
impl fmt::Display for Token {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
write!(f, "{}: \"{}\"", self.kind, self.raw)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub enum Kind {
|
||||
Ident,
|
||||
OBrace,
|
||||
CBrace,
|
||||
OBracket,
|
||||
CBracket,
|
||||
Eq,
|
||||
Comma,
|
||||
Semi,
|
||||
|
||||
DependKeyword,
|
||||
IncludeKeyword,
|
||||
ModuleKeyword,
|
||||
SetKeyword,
|
||||
SourceKeyword,
|
||||
TypeKeyword,
|
||||
|
||||
StringLiteral,
|
||||
IntLiteral,
|
||||
Comment,
|
||||
}
|
||||
|
||||
impl fmt::Display for Kind {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
write!(
|
||||
f,
|
||||
"{}",
|
||||
match self {
|
||||
Kind::Ident => "ident",
|
||||
Kind::OBrace => "obrace",
|
||||
Kind::CBrace => "cbrace",
|
||||
Kind::OBracket => "obracket",
|
||||
Kind::CBracket => "cbracket",
|
||||
Kind::Eq => "eq",
|
||||
Kind::Comma => "comma",
|
||||
Kind::Semi => "semi",
|
||||
|
||||
Kind::DependKeyword => "keyword",
|
||||
Kind::IncludeKeyword => "keyword",
|
||||
Kind::ModuleKeyword => "keyword",
|
||||
Kind::SetKeyword => "keyword",
|
||||
Kind::SourceKeyword => "keyword",
|
||||
Kind::TypeKeyword => "keyword",
|
||||
|
||||
Kind::StringLiteral => "string",
|
||||
Kind::IntLiteral => "int",
|
||||
Kind::Comment => "comment",
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
@ -1,3 +1,12 @@
|
||||
use std::fs;
|
||||
|
||||
mod lex;
|
||||
use lex::Lexer;
|
||||
|
||||
fn main() {
|
||||
println!("Hello, world!");
|
||||
let s = fs::read_to_string("test.gaybuild").unwrap();
|
||||
let lexer = Lexer::new(s.chars());
|
||||
for token in lexer {
|
||||
println!("{}", token.unwrap());
|
||||
}
|
||||
}
|
||||
|
@ -0,0 +1,28 @@
|
||||
set RUSTC_EXE = "rustc";
|
||||
set ASM_EXE = "clang";
|
||||
set CC_EXE = "clang";
|
||||
set LINK_EXE = "ld.lld";
|
||||
set BUILD_PREFIX = "build";
|
||||
|
||||
module kern {
|
||||
type exe;
|
||||
depend [
|
||||
libk,
|
||||
arch,
|
||||
];
|
||||
source "kern/lib.rs";
|
||||
}
|
||||
|
||||
module libk {
|
||||
type static; # static library
|
||||
depend arch;
|
||||
source "libk/lib.rs";
|
||||
}
|
||||
|
||||
module arch {
|
||||
type static;
|
||||
source [
|
||||
"arch/lib.rs",
|
||||
"arch/**.nasm",
|
||||
];
|
||||
}
|
Loading…
Reference in New Issue