[Bootstra] Rethink the eval loop to rewrite evaluation tree

Implement a new algorithm to eliminate TCO while preventing
premature execution exit by rewriting the evaluation tree and
replacing old nodes with new set of forms, for example the result of
some old form's evaluation, a body of a function or anything in
that nature. Couple the new forms with a possible new execution scope.

bootstrap/examples/hello-world.srn

@@ -1,9 +1,10 @@
 (ns examples.hello-world)
 
+
 (def hello-world
-     (fn ()
-         "helloworld"))
+     (fn (name)
+         (println "hello " name)))
 
 (def main
      (fn (&args)
-         (println "sameer")))
+         (hello-world)))

bootstrap/pkg/core/block.go

@@ -31,6 +31,7 @@ import (
 // brackets and indentation.
 // Blocks in serene are just a group of forms and nothing more.
 type Block struct {
+	ExecutionScope
 	body []IExpr
 }
 

bootstrap/pkg/core/builtins.go

@@ -31,23 +31,23 @@ var BUILTINS = map[string]NativeFunction{
 
 func PrNativeFn(rt *Runtime, scope IScope, n Node, args *List) (IExpr, IError) {
 	Pr(rt, toRepresentables(args.Rest().(IColl))...)
-	return &Nil, nil
+	return MakeNil(n), nil
 }
 
 func PrnNativeFn(rt *Runtime, scope IScope, n Node, args *List) (IExpr, IError) {
 
 	Prn(rt, toRepresentables(args.Rest().(IColl))...)
-	return &Nil, nil
+	return MakeNil(n), nil
 }
 
 func PrintNativeFn(rt *Runtime, scope IScope, n Node, args *List) (IExpr, IError) {
 	Print(rt, toRepresentables(args.Rest().(IColl))...)
-	return &Nil, nil
+	return MakeNil(n), nil
 }
 
 func PrintlnNativeFn(rt *Runtime, scope IScope, n Node, args *List) (IExpr, IError) {
 	Println(rt, toRepresentables(args.Rest().(IColl))...)
-	return &Nil, nil
+	return MakeNil(n), nil
 }
 
 func RequireNativeFn(rt *Runtime, scope IScope, n Node, args *List) (IExpr, IError) {

bootstrap/pkg/core/core.go

@@ -151,7 +151,6 @@ func Run(debug bool, args []string) {
 	mainBinding := loadedNS.GetRootScope().Lookup(rt, "main")
 
 	if mainBinding == nil {
-		fmt.Printf(">>> %w\n", loadedNS.GetRootScope())
 		PrintError(rt, MakePlainError(fmt.Sprintf("can't find the 'main' function in '%s' namespace", ns)))
 		os.Exit(1)
 	}

bootstrap/pkg/core/eval.go

@@ -24,6 +24,13 @@ import (
 	"serene-lang.org/bootstrap/pkg/ast"
 )
 
+func restOfExprs(es []IExpr, i int) []IExpr {
+	if len(es)-1 > i {
+		return es[i+1:]
+	}
+	return []IExpr{}
+}
+
 // evalForm evaluates the given expression `form` by a slightly different
 // evaluation rules. For example if `form` is a list instead of the formal
 // evaluation of a list it will evaluate all the elements and return the
@@ -66,11 +73,11 @@ func evalForm(rt *Runtime, scope IScope, form IExpr) (IExpr, IError) {
 
 		switch symbolName {
 		case "true":
-			return &True, nil
+			return MakeTrue(MakeNodeFromExpr(form)), nil
 		case "false":
-			return &False, nil
+			return MakeFalse(MakeNodeFromExpr(form)), nil
 		case "nil":
-			return &Nil, nil
+			return MakeNil(MakeNodeFromExpr(form)), nil
 		default:
 			var expr *Binding
 			if sym.IsNSQualified() {
@@ -120,7 +127,7 @@ func evalForm(rt *Runtime, scope IScope, form IExpr) (IExpr, IError) {
 		}
 		return MakeList(result), nil
 	}
-	panic("Asd")
+
 	// Default case
 	return nil, MakeError(rt, fmt.Sprintf("support for '%d' is not implemented", form.GetType()))
 }
@@ -134,6 +141,11 @@ func EvalForms(rt *Runtime, scope IScope, expressions IExpr) (IExpr, IError) {
 	// In order to avoid stackoverflows and implement TCO ( which is a must for
 	// a functional language we need to avoid unnecessary calls and keep as much
 	// as possible in a loop.
+	//
+	// `expressions` is argument is basically a tree of expressions which
+	// this function walks over and rewrite it as necessary. The main purpose
+	// of rewriting the tree is to eliminate any unnecessary function call.
+	// This way we can eliminate tail calls and run everything faster.
 	var ret IExpr
 	var err IError
 
@@ -158,15 +170,25 @@ tco:
 		}
 
 	body:
-		for _, forms := range exprs {
-			// Evaluating forms one by one
-			if rt.IsDebugMode() {
-				fmt.Printf("[DEBUG] Evaluating forms in NS: %s, Forms: %s\n", rt.CurrentNS().GetName(), forms)
+		for i := 0; i < len(exprs); i++ {
+			//for i, forms := range exprs {
+			forms := exprs[i]
+			executionScope := forms.GetExecutionScope()
+			scope := scope
+
+			if executionScope != nil {
+				scope = executionScope
 			}
 
+			if rt.IsDebugMode() {
+				fmt.Printf("[DEBUG] Evaluating forms in NS: %s, Forms: %s\n", rt.CurrentNS().GetName(), forms)
+				fmt.Printf("[DEBUG] -> State: I: %d, Exprs: %s\n", i, exprs)
+			}
+
+			// Evaluating forms one by one
 			if forms.GetType() != ast.List {
 				ret, err = evalForm(rt, scope, forms)
-				break tco // return ret, err
+				continue body
 			}
 
 			// Expand macroes that exists in the given array of expression `forms`.
@@ -219,7 +241,7 @@ tco:
 			// TODO: decide on the syntax and complete the docs
 			case "ns":
 				ret, err = NSForm(rt, scope, list)
-				continue // return
+				continue body // no rewrite
 
 			// `quote` evaluation rules:
 			// * Only takes one argument
@@ -229,7 +251,9 @@ tco:
 				if list.Count() != 2 {
 					return nil, MakeErrorFor(rt, list, "'quote' quote only accepts one argument.")
 				}
-				return list.Rest().First(), nil
+				ret = list.Rest().First()
+				err = nil
+				continue body // no rewrite
 
 			// case "quasiquote-expand":
 			// 	return quasiquote(list.Rest().First()), nil
@@ -243,7 +267,8 @@ tco:
 			// TODO: Implement `list` in serene itself when we have destructuring available
 			// Creates a new list form it's arguments.
 			case "list":
-				return evalForm(rt, scope, list.Rest().(*List))
+				ret, err = evalForm(rt, scope, list.Rest().(*List))
+				continue body // no rewrite
 
 			// TODO: Implement `concat` in serene itself when we have protocols available
 			// Concats all the collections together.
@@ -264,7 +289,8 @@ tco:
 
 					result = append(result, lst.(*List).ToSlice()...)
 				}
-				return MakeList(result), nil
+				ret, err = MakeList(result), nil
+				continue body // no rewrite
 
 			// TODO: Implement `list` in serene itself when we have destructuring available
 			// Calls the `Cons` function on the second argument to cons the first arg to it.
@@ -286,7 +312,8 @@ tco:
 					return nil, MakeErrorFor(rt, list, "second arg of 'cons' has to be a collection")
 				}
 
-				return coll.Cons(evaledForms.(*List).First()), nil
+				ret, err = coll.Cons(evaledForms.(*List).First()), nil
+				continue body // no rewrite
 
 			// `def` evaluation rules
 			// * The first argument has to be a symbol.
@@ -296,7 +323,7 @@ tco:
 			//   the symbol name binded to the value
 			case "def":
 				ret, err = Def(rt, scope, list.Rest().(*List))
-				continue body
+				continue body // no rewrite
 
 			// `defmacro` evaluation rules:
 			// * The first argument has to be a symbol
@@ -305,7 +332,7 @@ tco:
 			//   body of the macro.
 			case "defmacro":
 				ret, err = DefMacro(rt, scope, list.Rest().(*List))
-				continue body
+				continue body // no rewrite
 
 			// `macroexpand` evaluation rules:
 			// * It has to have only one argument
@@ -322,14 +349,14 @@ tco:
 				}
 
 				ret, err = macroexpand(rt, scope, evaledForm.(*List).First())
-				break tco // return
+				continue body // no rewrite
 
 			// `fn` evaluation rules:
 			// * It needs at least a collection of arguments
 			// * Defines an anonymous function.
 			case "fn":
 				ret, err = Fn(rt, scope, list.Rest().(*List))
-				continue body
+				continue body // no rewrite
 
 			// `if` evaluation rules:
 			// * It has to get only 3 arguments: PRED THEN ELSE
@@ -351,21 +378,26 @@ tco:
 
 				if result != ast.False && result != ast.Nil {
 					// Truthy clause
-					expressions = args.Rest().First()
+					exprs = append([]IExpr{args.Rest().First()}, restOfExprs(exprs, i)...)
 				} else {
 
 					// Falsy clause
-					expressions = args.Rest().Rest().First()
+					exprs = append([]IExpr{args.Rest().Rest().First()}, restOfExprs(exprs, i)...)
 				}
-
-				continue tco // Loop over to execute the new expressions
+				i = 0
+				goto body // rewrite
 
 			// `do` evaluation rules:
 			// * Evaluate the body as a new block in the TCO loop
 			//   and return the result of the last expression
 			case "do":
-				expressions = MakeBlock(list.Rest().(*List).ToSlice())
-				continue tco // Loop over to execute the new expressions
+				// create a new slice of expressions by using the
+				// do body and merging it by the remaining expressions
+				// in the old `exprs` value and loop over it
+				doExprs := list.Rest().(*List).ToSlice()
+				exprs = append(doExprs, exprs[i+1:]...)
+				i = 0
+				goto body // rewrite
 
 			// TODO: Implement `eval` as a native function
 			// `eval` evaluation rules:
@@ -385,7 +417,8 @@ tco:
 					return nil, err
 				}
 
-				return EvalForms(rt, scope, form)
+				ret, err = EvalForms(rt, scope, form)
+				continue body // no rewrite
 
 			// `let` evaluation rules:
 			// Let's assume the following:
@@ -444,23 +477,24 @@ tco:
 					bindings = bindings.Rest().Rest().(IColl)
 				}
 
-				expressions = MakeBlock(body)
-				scope = letScope
-				continue tco
+				changeExecutionScope(body, letScope)
+				exprs = append(body, exprs[i+1:]...)
+				i = 0
+				goto body
 
 			// list evaluation rules:
 			// * The first element of the list has to be an expression which is callable
 			// * An empty list evaluates to itself.
 			default:
 				// Evaluating all the elements of the list
-				exprs, e := evalForm(rt, scope, list)
+				listExprs, e := evalForm(rt, scope, list)
 				if e != nil {
 					err = e
 					ret = nil
 					break tco //return
 				}
 
-				f := exprs.(*List).First()
+				f := listExprs.(*List).First()
 
 				switch f.GetType() {
 				case ast.Fn:
@@ -477,27 +511,32 @@ tco:
 
 					}
 
-					argList := exprs.(*List).Rest().(*List)
+					argList := listExprs.(*List).Rest().(*List)
 
-					scope, e = MakeFnScope(rt, fn.GetScope(), fn.GetParams(), argList)
+					fnScope, e := MakeFnScope(rt, fn.GetScope(), fn.GetParams(), argList)
 					if e != nil {
 						err = e
 						ret = nil
 						break body //return
 					}
 
-					expressions = fn.GetBody()
-					continue tco
+					body := fn.GetBody().ToSlice()
+					changeExecutionScope(body, fnScope)
+					exprs = append(body, restOfExprs(exprs, i)...)
+					goto body // rewrite
 
 				// If the function was a native function which is represented
 				// by the `NativeFunction` struct
 				case ast.NativeFn:
 					fn := f.(*NativeFunction)
-					return fn.Apply(
+					ret, err = fn.Apply(
 						rt,
 						scope,
 						MakeNodeFromExpr(fn),
-						exprs.(*List))
+						listExprs.(*List),
+					)
+					continue body // no rewrite
+
 				default:
 					err = MakeError(rt, "don't know how to execute anything beside function")
 					ret = nil

bootstrap/pkg/core/false.go

@@ -23,28 +23,28 @@ import (
 	"serene-lang.org/bootstrap/pkg/hash"
 )
 
-type FalseType struct{}
+type False struct {
+	Node
+	ExecutionScope
+}
 
-// False is just False not `null` or anything
-var False = FalseType{}
-
-func (n FalseType) GetType() ast.NodeType {
+func (f *False) GetType() ast.NodeType {
 	return ast.False
 }
 
-func (n FalseType) GetLocation() ast.Location {
-	return ast.MakeUnknownLocation()
-}
-
-func (n FalseType) String() string {
+func (f *False) String() string {
 	return "false"
 }
 
-func (n FalseType) ToDebugStr() string {
+func (f *False) ToDebugStr() string {
 	return "false"
 }
 
-func (n FalseType) Hash() uint32 {
+func (f *False) Hash() uint32 {
 	bytes := []byte("false")
 	return hash.HashOf(append([]byte{byte(ast.False)}, bytes...))
 }
+
+func MakeFalse(n Node) *False {
+	return &False{Node: n}
+}

bootstrap/pkg/core/function.go

@@ -56,7 +56,7 @@ type Function struct {
 	// Node struct holds the necessary functions to make
 	// Functions locatable
 	Node
-
+	ExecutionScope
 	// Name of the function, it can be empty and it has to be
 	// set via `def`
 	name string
@@ -79,6 +79,7 @@ type NativeFunction struct {
 	// Node struct holds the necessary functions to make
 	// Functions locatable
 	Node
+	ExecutionScope
 	name string
 	fn   nativeFnHandler
 }
@@ -157,10 +158,13 @@ func MakeFnScope(rt *Runtime, parent IScope, bindings IColl, values IColl) (*Sco
 	binds := bindings.ToSlice()
 	exprs := values.ToSlice()
 	numberOfBindings := len(binds)
-	lastBinding := binds[len(binds)-1]
 
-	if lastBinding.GetType() == ast.Symbol && lastBinding.(*Symbol).IsRestable() {
-		numberOfBindings = len(binds) - 1
+	if len(binds) > 0 {
+		lastBinding := binds[len(binds)-1]
+
+		if lastBinding.GetType() == ast.Symbol && lastBinding.(*Symbol).IsRestable() {
+			numberOfBindings = len(binds) - 1
+		}
 	}
 
 	if numberOfBindings > len(exprs) {

bootstrap/pkg/core/keyword.go

@@ -84,6 +84,7 @@ import (
 
 type Keyword struct {
 	Node
+	ExecutionScope
 	name string
 	// nsName is the string that is used as the namespace name. It
 	// might be an ns alias in the current ns or the full namespace

bootstrap/pkg/core/list.go

@@ -35,6 +35,7 @@ link list of cons.
 
 type List struct {
 	Node
+	ExecutionScope
 	exprs []IExpr
 }
 
@@ -62,7 +63,7 @@ func (l *List) ToDebugStr() string {
 
 func (l *List) First() IExpr {
 	if l.Count() == 0 {
-		return Nil
+		return MakeNil(MakeNodeFromExpr(l))
 	}
 	return l.exprs[0]
 }

bootstrap/pkg/core/namespace.go

@@ -68,6 +68,12 @@ func (n *Namespace) ToDebugStr() string {
 	return fmt.Sprintf("<ns: %s at %s>", n.name, n.source)
 }
 
+func (n *Namespace) GetExecutionScope() IScope {
+	return nil
+}
+
+func (n *Namespace) SetExecutionScope(scope IScope) {}
+
 // DefineGlobal inserts the given expr `v` to the root scope of
 // `n`. The `public` parameter determines whether the public
 // value is accessable publicly or not (in other namespaces).

bootstrap/pkg/core/nil.go

@@ -20,27 +20,27 @@ package core
 
 import "serene-lang.org/bootstrap/pkg/ast"
 
-type NilType struct{}
+type Nil struct {
+	Node
+	ExecutionScope
+}
 
-// Nil is just Nil not `null` or anything
-var Nil = NilType{}
-
-func (n NilType) GetType() ast.NodeType {
+func (n *Nil) GetType() ast.NodeType {
 	return ast.Nil
 }
 
-func (n NilType) GetLocation() ast.Location {
-	return ast.MakeUnknownLocation()
-}
-
-func (n NilType) String() string {
+func (n *Nil) String() string {
 	return "nil"
 }
 
-func (n NilType) ToDebugStr() string {
+func (n *Nil) ToDebugStr() string {
 	return "nil"
 }
 
-func (n NilType) Hash() uint32 {
+func (n *Nil) Hash() uint32 {
 	return 0
 }
+
+func MakeNil(n Node) *Nil {
+	return &Nil{Node: n}
+}

bootstrap/pkg/core/nothing.go

@@ -47,3 +47,9 @@ func (n NothingType) String() string {
 func (n NothingType) ToDebugStr() string {
 	return ""
 }
+
+func (n NothingType) GetExecutionScope() IScope {
+	return nil
+}
+
+func (n NothingType) SetExecutionScope(scope IScope) {}

bootstrap/pkg/core/numbers.go

@@ -57,6 +57,10 @@ to begin with
 
 type Integer struct {
 	Node
+	// ExecutionScope checkout IScopable
+	scope IScope
+
+	ExecutionScope
 	value int64
 }
 
@@ -70,6 +74,15 @@ func (i Integer) Hash() uint32 {
 	return hash.HashOf(b)
 }
 
+func (i Integer) GetExecutionScope() IScope {
+	return i.scope
+
+}
+
+func (i Integer) SetExecutionScope(scope IScope) {
+	i.scope = scope
+}
+
 func (i Integer) String() string {
 	return fmt.Sprintf("%d", i.value)
 }
@@ -104,6 +117,8 @@ func MakeInteger(x interface{}) (*Integer, IError) {
 
 type Double struct {
 	Node
+	// ExecutionScope checkout IScopable
+	scope IScope
 	value float64
 }
 
@@ -126,6 +141,15 @@ func (d Double) ToDebugStr() string {
 	return fmt.Sprintf("%#v", d)
 }
 
+func (d Double) GetExecutionScope() IScope {
+	return d.scope
+
+}
+
+func (d Double) SetExecutionScope(scope IScope) {
+	d.scope = scope
+}
+
 func (d Double) I64() int64 {
 	return int64(d.value)
 }

bootstrap/pkg/core/string.go

@@ -28,6 +28,7 @@ import (
 
 type String struct {
 	Node
+	ExecutionScope
 	content string
 }
 
@@ -64,5 +65,8 @@ func (s *String) Escape() string {
 }
 
 func MakeString(n Node, s string) *String {
-	return &String{n, s}
+	return &String{
+		Node:    n,
+		content: s,
+	}
 }

bootstrap/pkg/core/symbol.go

@@ -28,6 +28,7 @@ import (
 
 type Symbol struct {
 	Node
+	ExecutionScope
 	name   string
 	nsPart string
 }

bootstrap/pkg/core/true.go

@@ -23,28 +23,26 @@ import (
 	"serene-lang.org/bootstrap/pkg/hash"
 )
 
-type TrueType struct{}
+type True struct {
+	Node
+	ExecutionScope
+}
 
-// True is just True not `null` or anything
-var True = TrueType{}
-
-func (n TrueType) GetType() ast.NodeType {
+func (t *True) GetType() ast.NodeType {
 	return ast.True
 }
 
-func (n TrueType) GetLocation() ast.Location {
-	return ast.MakeUnknownLocation()
-}
-
-func (n TrueType) String() string {
+func (t *True) String() string {
 	return "true"
 }
 
-func (n TrueType) ToDebugStr() string {
+func (t *True) ToDebugStr() string {
 	return "true"
 }
-
-func (n TrueType) Hash() uint32 {
+func (t *True) Hash() uint32 {
 	bytes := []byte("true")
 	return hash.HashOf(append([]byte{byte(ast.True)}, bytes...))
 }
+func MakeTrue(n Node) *True {
+	return &True{Node: n}
+}

bootstrap/pkg/core/types.go

@@ -41,6 +41,21 @@ type IPrintable interface {
 	PrintToString() string
 }
 
+// IScopable is the interface describing how to get the execution scope of
+// the value. During the evaluation of the forms in Serene we might rewrite
+// the execution tree to eliminate tail calls. In order to do that we should
+// be able to attach the execution scope to any form that we need to rewrite.
+type IScopable interface {
+
+	// GetExecutionScope returns an attached execution scope if there's
+	// any, nil otherwise.
+	GetExecutionScope() IScope
+
+	// SetExecutionScope sets the given scope as the execution scope of
+	// the current implementor
+	SetExecutionScope(scope IScope)
+}
+
 // IDebuggable is the interface designed for converting forms to a string
 // form which are meant to be used as debug data
 type IDebuggable interface {
@@ -56,6 +71,7 @@ type IExpr interface {
 	hash.IHashable
 	IRepresentable
 	IDebuggable
+	IScopable
 }
 
 // Node struct is simply representing a Node in the AST which provides the
@@ -69,8 +85,29 @@ func (n Node) GetLocation() ast.Location {
 	return n.location
 }
 
+type ExecutionScope struct {
+	scope IScope
+}
+
+func (e *ExecutionScope) GetExecutionScope() IScope {
+	return e.scope
+
+}
+
+func (e *ExecutionScope) SetExecutionScope(scope IScope) {
+	e.scope = scope
+}
+
 // Helper functions ===========================================================
 
+// changeExecutionScope sets the execution scope of all the expressions in `es`
+// to the given `scope`
+func changeExecutionScope(es []IExpr, scope IScope) {
+	for _, x := range es {
+		x.SetExecutionScope(scope)
+	}
+}
+
 // toRepresentables converts the given collection of IExprs to an array of
 // IRepresentable. Since golangs type system is weird ( if A is an interface
 // that embeds interface B you []A should be usable as []B but that's not the