Rewrite declaration as statement instead of binop

- Declarations are now separate statements
- Generate unknown var errors when vars are not declared
- Replace Peekable by new custom PutBackIter type that allows for
  unlimited putback and therefore look-ahead

src/ast.rs

@@ -59,9 +59,6 @@ pub enum BinOpType {
 
     /// Assign value to variable
     Assign,
-
-    /// Declare new variable with value
-    Declare,
 }
 
 #[derive(Debug, PartialEq, Eq, Clone)]
@@ -119,6 +116,7 @@ pub struct If {
 
 #[derive(Debug, PartialEq, Eq, Clone)]
 pub enum Statement {
+    Declaration(Sid, usize, Expression),
     Expr(Expression),
     Block(BlockScope),
     Loop(Loop),
@@ -144,7 +142,6 @@ impl BinOpType {
 
     pub fn precedence(&self) -> u8 {
         match self {
-            BinOpType::Declare => 0,
             BinOpType::Assign => 1,
             BinOpType::LOr => 2,
             BinOpType::LAnd => 3,

src/astoptimizer.rs

@@ -40,6 +40,7 @@ impl SimpleAstOptimizer {
                     Self::optimize_block(body_false);
                 }
                 Statement::Print(expr) => Self::optimize_expr(expr),
+                Statement::Declaration(_, _, expr) => Self::optimize_expr(expr),
             }
         }
     }
@@ -74,7 +75,7 @@ impl SimpleAstOptimizer {
                             BinOpType::Greater => Expression::I64(if lhs > rhs { 1 } else { 0 }),
                             BinOpType::GreaterEqu => Expression::I64(if lhs >= rhs { 1 } else { 0 }),
             
-                            BinOpType::Declare | BinOpType::Assign => unreachable!(),
+                            BinOpType::Assign => unreachable!(),
                         };
                         *expr = new_expr;
                     },

src/interpreter.rs

@@ -81,6 +81,11 @@ impl Interpreter {
                     self.resolve_expr(expr);
                 }
 
+                Statement::Declaration(_sid, _idx, rhs) => {
+                    let rhs = self.resolve_expr(rhs);
+                    self.vartable.push(rhs);
+                }
+
                 Statement::Block(block) => {
                     self.run_block(block);
                 }
@@ -187,10 +192,6 @@ impl Interpreter {
         let rhs = self.resolve_expr(rhs);
 
         match (&bo, &lhs) {
-            (BinOpType::Declare, Expression::Var(_name, _idx)) => {
-                self.vartable.push(rhs.clone());
-                return rhs;
-            }
             (BinOpType::Assign, Expression::Var(name, idx)) => {
                 match self.get_var_mut(*idx) {
                     Some(val) => *val = rhs.clone(),
@@ -242,7 +243,7 @@ impl Interpreter {
                 BinOpType::Greater => Value::I64(if lhs > rhs { 1 } else { 0 }),
                 BinOpType::GreaterEqu => Value::I64(if lhs >= rhs { 1 } else { 0 }),
 
-                BinOpType::Declare | BinOpType::Assign => unreachable!(),
+                BinOpType::Assign => unreachable!(),
             },
             _ => panic!("Value types are not compatible"),
         }

src/lib.rs

@@ -5,6 +5,7 @@ pub mod parser;
 pub mod token;
 pub mod stringstore;
 pub mod astoptimizer;
+pub mod util;
 
 #[cfg(test)]
 mod tests {

src/parser.rs

@@ -1,10 +1,10 @@
-use std::iter::Peekable;
 use thiserror::Error;
 
 use crate::{
-    ast::{Ast, BinOpType, BlockScope, Expression, If, Loop, Statement},
+    ast::{Ast, BlockScope, Expression, If, Loop, Statement},
     stringstore::{Sid, StringStore},
     token::Token,
+    util::{PutBackIter, PutBackableExt},
     T,
 };
 
@@ -14,6 +14,8 @@ pub enum ParseErr {
     UnexpectedToken(Token, String),
     #[error("Left hand side of declaration is not a variable")]
     DeclarationOfNonVar,
+    #[error("Use of undefined variable \"{0}\"")]
+    UseOfUndeclaredVar(String),
 }
 
 type ResPE<T> = Result<T, ParseErr>;
@@ -34,7 +36,7 @@ pub fn parse<T: Iterator<Item = Token>, A: IntoIterator<IntoIter = T>>(tokens: A
 }
 
 struct Parser<T: Iterator<Item = Token>> {
-    tokens: Peekable<T>,
+    tokens: PutBackIter<T>,
     string_store: StringStore,
     var_stack: Vec<Sid>,
 }
@@ -42,7 +44,7 @@ struct Parser<T: Iterator<Item = Token>> {
 impl<T: Iterator<Item = Token>> Parser<T> {
     /// Create a new parser to parse the given Token Stream
     pub fn new<A: IntoIterator<IntoIter = T>>(tokens: A) -> Self {
-        let tokens = tokens.into_iter().peekable();
+        let tokens = tokens.into_iter().putbackable();
         let string_store = StringStore::new();
         let var_stack = Vec::new();
         Self {
@@ -71,6 +73,7 @@ impl<T: Iterator<Item = Token>> Parser<T> {
                 T![;] => {
                     self.next();
                 }
+
                 T![EoF] | T!['}'] => break,
 
                 T!['{'] => {
@@ -108,22 +111,25 @@ impl<T: Iterator<Item = Token>> Parser<T> {
 
             T![if] => Statement::If(self.parse_if()?),
 
-            // If it is not a loop, try to lex as an expression
             _ => {
-                let mut expr = self.parse_expr()?;
+                let first = self.next();
 
-                match &mut expr {
+                let stmt = match (first, self.peek()) {
-                    Expression::BinOp(BinOpType::Declare, lhs, _) => match lhs.as_mut() {
+                    (T![ident(name)], T![<-]) => {
-                        Expression::Var(sid, sp) => {
+                        self.next();
-                            *sp = self.var_stack.len();
-                            self.var_stack.push(*sid);
-                        }
-                        _ => return Err(ParseErr::DeclarationOfNonVar),
-                    },
-                    _ => (),
-                }
 
-                let stmt = Statement::Expr(expr);
+                        let sid = self.string_store.intern_or_lookup(&name);
+                        let sp = self.var_stack.len();
+                        self.var_stack.push(sid);
+
+                        let rhs = self.parse_expr()?;
+                        Statement::Declaration(sid, sp, rhs)
+                    }
+                    (first, _) => {
+                        self.putback(first);
+                        Statement::Expr(self.parse_expr()?)
+                    }
+                };
 
                 // After a statement, there must be a semicolon
                 validate_next!(self, T![;], ";");
@@ -255,7 +261,7 @@ impl<T: Iterator<Item = Token>> Parser<T> {
             // index as an expression
             T![ident(name)] if self.peek() == &T!['['] => {
                 let sid = self.string_store.intern_or_lookup(&name);
-                let stackpos = self.get_stackpos(sid);
+                let stackpos = self.get_stackpos(sid)?;
 
                 self.next();
 
@@ -268,7 +274,7 @@ impl<T: Iterator<Item = Token>> Parser<T> {
 
             T![ident(name)] => {
                 let sid = self.string_store.intern_or_lookup(&name);
-                let stackpos = self.get_stackpos(sid);
+                let stackpos = self.get_stackpos(sid)?;
                 Expression::Var(sid, stackpos)
             }
 
@@ -292,13 +298,18 @@ impl<T: Iterator<Item = Token>> Parser<T> {
         Ok(primary)
     }
 
-    fn get_stackpos(&self, varid: Sid) -> usize {
+    fn get_stackpos(&self, varid: Sid) -> ResPE<usize> {
         self.var_stack
             .iter()
             .rev()
             .position(|it| *it == varid)
             .map(|it| self.var_stack.len() - it - 1)
-            .unwrap_or(usize::MAX)
+            .ok_or(ParseErr::UseOfUndeclaredVar(
+                self.string_store
+                    .lookup(varid)
+                    .map(String::from)
+                    .unwrap_or("<unknown>".to_string()),
+            ))
     }
 
     /// Get the next Token without removing it
@@ -306,6 +317,10 @@ impl<T: Iterator<Item = Token>> Parser<T> {
         self.tokens.peek().unwrap_or(&T![EoF])
     }
 
+    fn putback(&mut self, tok: Token) {
+        self.tokens.putback(tok);
+    }
+
     /// Advance to next Token and return the removed Token
     fn next(&mut self) -> Token {
         self.tokens.next().unwrap_or(T![EoF])

src/token.rs

@@ -162,7 +162,6 @@ impl Token {
             T![>] => BinOpType::Greater,
             T![>=] => BinOpType::GreaterEqu,
 
-            T![<-] => BinOpType::Declare,
             T![=] => BinOpType::Assign,
 
             _ => return None,

src/util.rs

@@ -0,0 +1,137 @@
+/// The PutBackIter allows for items to be put back back and to be peeked. Putting an item back
+/// will cause it to be the next item returned by `next`. Peeking an item will get a reference to
+/// the next item in the iterator without removing it.
+///
+/// The whole PutBackIter behaves analogous to `std::iter::Peekable` with the addition of the
+/// `putback` function. This is slightly slower than `Peekable`, but allows for an unlimited number
+/// of putbacks and therefore an unlimited look-ahead range.
+pub struct PutBackIter<T: Iterator> {
+    iter: T,
+    putback_stack: Vec<T::Item>,
+}
+
+impl<T> PutBackIter<T>
+where
+    T: Iterator,
+{
+    /// Make the given iterator putbackable, wrapping it in the PutBackIter type. This effectively 
+    /// adds the `peek` and `putback` functions.
+    pub fn new(iter: T) -> Self {
+        Self {
+            iter,
+            putback_stack: Vec::new(),
+        }
+    }
+
+    /// Put the given item back into the iterator. This causes the putbacked items to be returned by
+    /// next in last-in-first-out order (aka. stack order). Only after all previously putback items
+    /// have been returned, the actual underlying iterator is used to get items.
+    /// The number of items that can be put back is unlimited.
+    pub fn putback(&mut self, it: T::Item) {
+        self.putback_stack.push(it);
+    }
+
+    /// Peek the next item, getting a reference to it without removing it from the iterator. This 
+    /// also includes items that were previsouly put back and not yet removed.
+    pub fn peek(&mut self) -> Option<&T::Item> {
+        if self.putback_stack.is_empty() {
+            let it = self.next()?;
+            self.putback(it);
+        }
+
+        self.putback_stack.last()
+    }
+}
+
+impl<T> Iterator for PutBackIter<T>
+where
+    T: Iterator,
+{
+    type Item = T::Item;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        match self.putback_stack.pop() {
+            Some(it) => Some(it),
+            None => self.iter.next(),
+        }
+    }
+}
+
+pub trait PutBackableExt {
+    /// Make the iterator putbackable, wrapping it in the PutBackIter type. This effectively 
+    /// adds the `peek` and `putback` functions.
+    fn putbackable(self) -> PutBackIter<Self>
+    where
+        Self: Iterator + Sized,
+    {
+        PutBackIter::new(self)
+    }
+}
+
+impl<T: Iterator> PutBackableExt for T {}
+
+#[cfg(test)]
+mod tests {
+    use super::PutBackableExt;
+
+    #[test]
+    fn putback_iter_next() {
+        let mut iter = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10].into_iter();
+        let mut pb_iter = iter.clone().putbackable();
+
+        // Check if next works
+        for _ in 0..iter.len() {
+            assert_eq!(pb_iter.next(), iter.next());
+        }
+    }
+
+    #[test]
+    fn putback_iter_peek() {
+        let mut iter_orig = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10].into_iter();
+        let mut iter = iter_orig.clone();
+        let mut pb_iter = iter.clone().putbackable();
+
+        for _ in 0..iter.len() {
+            // Check if peek gives a preview of the actual next element
+            assert_eq!(pb_iter.peek(), iter.next().as_ref());
+            // Check if next still returns the next (just peeked) element and not the one after
+            assert_eq!(pb_iter.next(), iter_orig.next());
+        }
+    }
+
+    #[test]
+    fn putback_iter_putback() {
+        let mut iter_orig = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10].into_iter();
+        let mut iter = iter_orig.clone();
+        let mut pb_iter = iter.clone().putbackable();
+
+        // Get the first 5 items with next and check if they match
+        let it0 = pb_iter.next();
+        assert_eq!(it0, iter.next());
+        let it1 = pb_iter.next();
+        assert_eq!(it1, iter.next());
+        let it2 = pb_iter.next();
+        assert_eq!(it2, iter.next());
+        let it3 = pb_iter.next();
+        assert_eq!(it3, iter.next());
+        let it4 = pb_iter.next();
+        assert_eq!(it4, iter.next());
+
+        // Put one value back and check if `next` works as expected, returning the just put back 
+        // item
+        pb_iter.putback(it0.unwrap());
+        assert_eq!(pb_iter.next(), it0);
+
+        // Put all values back
+        pb_iter.putback(it4.unwrap());
+        pb_iter.putback(it3.unwrap());
+        pb_iter.putback(it2.unwrap());
+        pb_iter.putback(it1.unwrap());
+        pb_iter.putback(it0.unwrap());
+
+        // After all values have been put back, the iter should match the original again
+        for _ in 0..iter.len() {
+            assert_eq!(pb_iter.next(), iter_orig.next());
+        }
+    }
+}