From 383da4ae052030c5c630d21a7289c1d271bdc47b Mon Sep 17 00:00:00 2001 From: Daniel M Date: Wed, 9 Feb 2022 16:54:06 +0100 Subject: [PATCH] Rewrite declaration as statement instead of binop - Declarations are now separate statements - Generate unknown var errors when vars are not declared - Replace Peekable by new custom PutBackIter type that allows for unlimited putback and therefore look-ahead --- src/ast.rs | 5 +- src/astoptimizer.rs | 3 +- src/interpreter.rs | 11 ++-- src/lib.rs | 1 + src/parser.rs | 57 +++++++++++------- src/token.rs | 1 - src/util.rs | 137 ++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 183 insertions(+), 32 deletions(-) create mode 100644 src/util.rs diff --git a/src/ast.rs b/src/ast.rs index a4f12ab..4e5edc4 100644 --- a/src/ast.rs +++ b/src/ast.rs @@ -59,9 +59,6 @@ pub enum BinOpType { /// Assign value to variable Assign, - - /// Declare new variable with value - Declare, } #[derive(Debug, PartialEq, Eq, Clone)] @@ -119,6 +116,7 @@ pub struct If { #[derive(Debug, PartialEq, Eq, Clone)] pub enum Statement { + Declaration(Sid, usize, Expression), Expr(Expression), Block(BlockScope), Loop(Loop), @@ -144,7 +142,6 @@ impl BinOpType { pub fn precedence(&self) -> u8 { match self { - BinOpType::Declare => 0, BinOpType::Assign => 1, BinOpType::LOr => 2, BinOpType::LAnd => 3, diff --git a/src/astoptimizer.rs b/src/astoptimizer.rs index 7d90d89..c97d25e 100644 --- a/src/astoptimizer.rs +++ b/src/astoptimizer.rs @@ -40,6 +40,7 @@ impl SimpleAstOptimizer { Self::optimize_block(body_false); } Statement::Print(expr) => Self::optimize_expr(expr), + Statement::Declaration(_, _, expr) => Self::optimize_expr(expr), } } } @@ -74,7 +75,7 @@ impl SimpleAstOptimizer { BinOpType::Greater => Expression::I64(if lhs > rhs { 1 } else { 0 }), BinOpType::GreaterEqu => Expression::I64(if lhs >= rhs { 1 } else { 0 }), - BinOpType::Declare | BinOpType::Assign => unreachable!(), + BinOpType::Assign => unreachable!(), }; *expr = new_expr; }, diff --git a/src/interpreter.rs b/src/interpreter.rs index edac6db..08141ad 100644 --- a/src/interpreter.rs +++ b/src/interpreter.rs @@ -81,6 +81,11 @@ impl Interpreter { self.resolve_expr(expr); } + Statement::Declaration(_sid, _idx, rhs) => { + let rhs = self.resolve_expr(rhs); + self.vartable.push(rhs); + } + Statement::Block(block) => { self.run_block(block); } @@ -187,10 +192,6 @@ impl Interpreter { let rhs = self.resolve_expr(rhs); match (&bo, &lhs) { - (BinOpType::Declare, Expression::Var(_name, _idx)) => { - self.vartable.push(rhs.clone()); - return rhs; - } (BinOpType::Assign, Expression::Var(name, idx)) => { match self.get_var_mut(*idx) { Some(val) => *val = rhs.clone(), @@ -242,7 +243,7 @@ impl Interpreter { BinOpType::Greater => Value::I64(if lhs > rhs { 1 } else { 0 }), BinOpType::GreaterEqu => Value::I64(if lhs >= rhs { 1 } else { 0 }), - BinOpType::Declare | BinOpType::Assign => unreachable!(), + BinOpType::Assign => unreachable!(), }, _ => panic!("Value types are not compatible"), } diff --git a/src/lib.rs b/src/lib.rs index 2afaf75..427bf8b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,6 +5,7 @@ pub mod parser; pub mod token; pub mod stringstore; pub mod astoptimizer; +pub mod util; #[cfg(test)] mod tests { diff --git a/src/parser.rs b/src/parser.rs index f2bd3ec..47afd2b 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -1,10 +1,10 @@ -use std::iter::Peekable; use thiserror::Error; use crate::{ - ast::{Ast, BinOpType, BlockScope, Expression, If, Loop, Statement}, + ast::{Ast, BlockScope, Expression, If, Loop, Statement}, stringstore::{Sid, StringStore}, token::Token, + util::{PutBackIter, PutBackableExt}, T, }; @@ -14,6 +14,8 @@ pub enum ParseErr { UnexpectedToken(Token, String), #[error("Left hand side of declaration is not a variable")] DeclarationOfNonVar, + #[error("Use of undefined variable \"{0}\"")] + UseOfUndeclaredVar(String), } type ResPE = Result; @@ -34,7 +36,7 @@ pub fn parse, A: IntoIterator>(tokens: A } struct Parser> { - tokens: Peekable, + tokens: PutBackIter, string_store: StringStore, var_stack: Vec, } @@ -42,7 +44,7 @@ struct Parser> { impl> Parser { /// Create a new parser to parse the given Token Stream pub fn new>(tokens: A) -> Self { - let tokens = tokens.into_iter().peekable(); + let tokens = tokens.into_iter().putbackable(); let string_store = StringStore::new(); let var_stack = Vec::new(); Self { @@ -71,6 +73,7 @@ impl> Parser { T![;] => { self.next(); } + T![EoF] | T!['}'] => break, T!['{'] => { @@ -108,22 +111,25 @@ impl> Parser { T![if] => Statement::If(self.parse_if()?), - // If it is not a loop, try to lex as an expression _ => { - let mut expr = self.parse_expr()?; + let first = self.next(); - match &mut expr { - Expression::BinOp(BinOpType::Declare, lhs, _) => match lhs.as_mut() { - Expression::Var(sid, sp) => { - *sp = self.var_stack.len(); - self.var_stack.push(*sid); - } - _ => return Err(ParseErr::DeclarationOfNonVar), - }, - _ => (), - } + let stmt = match (first, self.peek()) { + (T![ident(name)], T![<-]) => { + self.next(); - let stmt = Statement::Expr(expr); + let sid = self.string_store.intern_or_lookup(&name); + let sp = self.var_stack.len(); + self.var_stack.push(sid); + + let rhs = self.parse_expr()?; + Statement::Declaration(sid, sp, rhs) + } + (first, _) => { + self.putback(first); + Statement::Expr(self.parse_expr()?) + } + }; // After a statement, there must be a semicolon validate_next!(self, T![;], ";"); @@ -255,7 +261,7 @@ impl> Parser { // index as an expression T![ident(name)] if self.peek() == &T!['['] => { let sid = self.string_store.intern_or_lookup(&name); - let stackpos = self.get_stackpos(sid); + let stackpos = self.get_stackpos(sid)?; self.next(); @@ -268,7 +274,7 @@ impl> Parser { T![ident(name)] => { let sid = self.string_store.intern_or_lookup(&name); - let stackpos = self.get_stackpos(sid); + let stackpos = self.get_stackpos(sid)?; Expression::Var(sid, stackpos) } @@ -292,13 +298,18 @@ impl> Parser { Ok(primary) } - fn get_stackpos(&self, varid: Sid) -> usize { + fn get_stackpos(&self, varid: Sid) -> ResPE { self.var_stack .iter() .rev() .position(|it| *it == varid) .map(|it| self.var_stack.len() - it - 1) - .unwrap_or(usize::MAX) + .ok_or(ParseErr::UseOfUndeclaredVar( + self.string_store + .lookup(varid) + .map(String::from) + .unwrap_or("".to_string()), + )) } /// Get the next Token without removing it @@ -306,6 +317,10 @@ impl> Parser { self.tokens.peek().unwrap_or(&T![EoF]) } + fn putback(&mut self, tok: Token) { + self.tokens.putback(tok); + } + /// Advance to next Token and return the removed Token fn next(&mut self) -> Token { self.tokens.next().unwrap_or(T![EoF]) diff --git a/src/token.rs b/src/token.rs index fe15f60..33cb5b7 100644 --- a/src/token.rs +++ b/src/token.rs @@ -162,7 +162,6 @@ impl Token { T![>] => BinOpType::Greater, T![>=] => BinOpType::GreaterEqu, - T![<-] => BinOpType::Declare, T![=] => BinOpType::Assign, _ => return None, diff --git a/src/util.rs b/src/util.rs new file mode 100644 index 0000000..30e7bb3 --- /dev/null +++ b/src/util.rs @@ -0,0 +1,137 @@ +/// The PutBackIter allows for items to be put back back and to be peeked. Putting an item back +/// will cause it to be the next item returned by `next`. Peeking an item will get a reference to +/// the next item in the iterator without removing it. +/// +/// The whole PutBackIter behaves analogous to `std::iter::Peekable` with the addition of the +/// `putback` function. This is slightly slower than `Peekable`, but allows for an unlimited number +/// of putbacks and therefore an unlimited look-ahead range. +pub struct PutBackIter { + iter: T, + putback_stack: Vec, +} + +impl PutBackIter +where + T: Iterator, +{ + /// Make the given iterator putbackable, wrapping it in the PutBackIter type. This effectively + /// adds the `peek` and `putback` functions. + pub fn new(iter: T) -> Self { + Self { + iter, + putback_stack: Vec::new(), + } + } + + /// Put the given item back into the iterator. This causes the putbacked items to be returned by + /// next in last-in-first-out order (aka. stack order). Only after all previously putback items + /// have been returned, the actual underlying iterator is used to get items. + /// The number of items that can be put back is unlimited. + pub fn putback(&mut self, it: T::Item) { + self.putback_stack.push(it); + } + + /// Peek the next item, getting a reference to it without removing it from the iterator. This + /// also includes items that were previsouly put back and not yet removed. + pub fn peek(&mut self) -> Option<&T::Item> { + if self.putback_stack.is_empty() { + let it = self.next()?; + self.putback(it); + } + + self.putback_stack.last() + } +} + +impl Iterator for PutBackIter +where + T: Iterator, +{ + type Item = T::Item; + + fn next(&mut self) -> Option { + match self.putback_stack.pop() { + Some(it) => Some(it), + None => self.iter.next(), + } + } +} + +pub trait PutBackableExt { + /// Make the iterator putbackable, wrapping it in the PutBackIter type. This effectively + /// adds the `peek` and `putback` functions. + fn putbackable(self) -> PutBackIter + where + Self: Iterator + Sized, + { + PutBackIter::new(self) + } +} + +impl PutBackableExt for T {} + +#[cfg(test)] +mod tests { + use super::PutBackableExt; + + #[test] + fn putback_iter_next() { + let mut iter = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10].into_iter(); + let mut pb_iter = iter.clone().putbackable(); + + // Check if next works + for _ in 0..iter.len() { + assert_eq!(pb_iter.next(), iter.next()); + } + } + + #[test] + fn putback_iter_peek() { + let mut iter_orig = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10].into_iter(); + let mut iter = iter_orig.clone(); + let mut pb_iter = iter.clone().putbackable(); + + for _ in 0..iter.len() { + // Check if peek gives a preview of the actual next element + assert_eq!(pb_iter.peek(), iter.next().as_ref()); + // Check if next still returns the next (just peeked) element and not the one after + assert_eq!(pb_iter.next(), iter_orig.next()); + } + } + + #[test] + fn putback_iter_putback() { + let mut iter_orig = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10].into_iter(); + let mut iter = iter_orig.clone(); + let mut pb_iter = iter.clone().putbackable(); + + // Get the first 5 items with next and check if they match + let it0 = pb_iter.next(); + assert_eq!(it0, iter.next()); + let it1 = pb_iter.next(); + assert_eq!(it1, iter.next()); + let it2 = pb_iter.next(); + assert_eq!(it2, iter.next()); + let it3 = pb_iter.next(); + assert_eq!(it3, iter.next()); + let it4 = pb_iter.next(); + assert_eq!(it4, iter.next()); + + // Put one value back and check if `next` works as expected, returning the just put back + // item + pb_iter.putback(it0.unwrap()); + assert_eq!(pb_iter.next(), it0); + + // Put all values back + pb_iter.putback(it4.unwrap()); + pb_iter.putback(it3.unwrap()); + pb_iter.putback(it2.unwrap()); + pb_iter.putback(it1.unwrap()); + pb_iter.putback(it0.unwrap()); + + // After all values have been put back, the iter should match the original again + for _ in 0..iter.len() { + assert_eq!(pb_iter.next(), iter_orig.next()); + } + } +}