initial commit

This commit is contained in:
Daniel Kang 2019-01-08 23:17:42 -08:00
commit 2c3282da21
145 changed files with 12130 additions and 0 deletions

1
.gitignore vendored Normal file
View file

@ -0,0 +1 @@
bin/

8
Makefile Normal file
View file

@ -0,0 +1,8 @@
vet:
go vet ./...
test: vet
go test -race -cover ./...
fmt:
go fmt ./...

302
assert/assert.go Normal file
View file

@ -0,0 +1,302 @@
package assert
import (
"bytes"
"fmt"
"reflect"
"strings"
"testing"
"github.com/d5/tengo/compiler"
"github.com/d5/tengo/objects"
"github.com/d5/tengo/scanner"
"github.com/d5/tengo/token"
)
func NoError(t *testing.T, err error, msg ...interface{}) bool {
t.Helper()
if err == nil {
return true
}
return failExpectedActual(t, "no error", err, msg...)
}
func Error(t *testing.T, err error, msg ...interface{}) bool {
t.Helper()
if err != nil {
return true
}
return failExpectedActual(t, "error", err, msg...)
}
func Nil(t *testing.T, v interface{}, msg ...interface{}) bool {
t.Helper()
if v == nil {
return true
}
return failExpectedActual(t, "nil", v, msg...)
}
func True(t *testing.T, v bool, msg ...interface{}) bool {
t.Helper()
if v {
return true
}
return failExpectedActual(t, "true", v, msg...)
}
func False(t *testing.T, v bool, msg ...interface{}) bool {
t.Helper()
if !v {
return true
}
return failExpectedActual(t, "false", v, msg...)
}
func NotNil(t *testing.T, v interface{}, msg ...interface{}) bool {
t.Helper()
if v != nil {
return true
}
return failExpectedActual(t, "not nil", v, msg...)
}
func IsType(t *testing.T, expected, actual interface{}, msg ...interface{}) bool {
t.Helper()
if reflect.TypeOf(expected) == reflect.TypeOf(actual) {
return true
}
return failExpectedActual(t, reflect.TypeOf(expected), reflect.TypeOf(actual), msg...)
}
func Equal(t *testing.T, expected, actual interface{}, msg ...interface{}) bool {
t.Helper()
if expected == nil {
return Nil(t, actual, "expected nil, but got not nil")
}
if !NotNil(t, actual, "expected not nil, but got nil") {
return false
}
if !IsType(t, expected, actual) {
return false
}
switch expected := expected.(type) {
case int:
if expected != actual.(int) {
return failExpectedActual(t, expected, actual, msg...)
}
case int64:
if expected != actual.(int64) {
return failExpectedActual(t, expected, actual, msg...)
}
case float64:
if expected != actual.(float64) {
return failExpectedActual(t, expected, actual, msg...)
}
case string:
if expected != actual.(string) {
return failExpectedActual(t, expected, actual, msg...)
}
case []byte:
if bytes.Compare(expected, actual.([]byte)) != 0 {
return failExpectedActual(t, expected, actual, msg...)
}
case []int:
if !equalIntSlice(expected, actual.([]int)) {
return failExpectedActual(t, expected, actual, msg...)
}
case bool:
if expected != actual.(bool) {
return failExpectedActual(t, expected, actual, msg...)
}
case rune:
if expected != actual.(rune) {
return failExpectedActual(t, expected, actual, msg...)
}
case compiler.Symbol:
if !equalSymbol(expected, actual.(compiler.Symbol)) {
return failExpectedActual(t, expected, actual, msg...)
}
case scanner.Pos:
if expected != actual.(scanner.Pos) {
return failExpectedActual(t, expected, actual, msg...)
}
case token.Token:
if expected != actual.(token.Token) {
return failExpectedActual(t, expected, actual, msg...)
}
case []objects.Object:
return equalObjectSlice(t, expected, actual.([]objects.Object))
case *objects.Int:
return Equal(t, expected.Value, actual.(*objects.Int).Value)
case *objects.Float:
return Equal(t, expected.Value, actual.(*objects.Float).Value)
case *objects.String:
return Equal(t, expected.Value, actual.(*objects.String).Value)
case *objects.Char:
return Equal(t, expected.Value, actual.(*objects.Char).Value)
case *objects.Bool:
return Equal(t, expected.Value, actual.(*objects.Bool).Value)
case *objects.ReturnValue:
return Equal(t, expected.Value, actual.(objects.ReturnValue).Value)
case *objects.Array:
return equalArray(t, expected, actual.(*objects.Array))
case *objects.Map:
return equalMap(t, expected, actual.(*objects.Map))
case *objects.CompiledFunction:
return equalCompiledFunction(t, expected, actual.(*objects.CompiledFunction))
case *objects.Closure:
return equalClosure(t, expected, actual.(*objects.Closure))
case *objects.Undefined:
return true
default:
panic(fmt.Errorf("type not implemented: %T", expected))
}
return true
}
func Fail(t *testing.T, msg ...interface{}) bool {
t.Helper()
t.Logf("\nError trace:\n\t%s\n%s", strings.Join(errorTrace(), "\n\t"), message(msg...))
t.Fail()
return false
}
func failExpectedActual(t *testing.T, expected, actual interface{}, msg ...interface{}) bool {
t.Helper()
var addMsg string
if len(msg) > 0 {
addMsg = "\nMessage: " + message(msg...)
}
t.Logf("\nError trace:\n\t%s\nExpected: %v\nActual: %v%s",
strings.Join(errorTrace(), "\n\t"),
expected, actual,
addMsg)
t.Fail()
return false
}
func message(formatArgs ...interface{}) string {
var format string
var args []interface{}
if len(formatArgs) > 0 {
format = formatArgs[0].(string)
}
if len(formatArgs) > 1 {
args = formatArgs[1:]
}
return fmt.Sprintf(format, args...)
}
func equalIntSlice(a, b []int) bool {
if len(a) != len(b) {
return false
}
for i := 0; i < len(a); i++ {
if a[i] != b[i] {
return false
}
}
return true
}
func equalSymbol(a, b compiler.Symbol) bool {
return a.Name == b.Name &&
a.Index == b.Index &&
a.Scope == b.Scope
}
func equalArray(t *testing.T, expected, actual objects.Object) bool {
expectedT := expected.(*objects.Array).Value
actualT := actual.(*objects.Array).Value
return equalObjectSlice(t, expectedT, actualT)
}
func equalObjectSlice(t *testing.T, expected, actual []objects.Object) bool {
if !Equal(t, len(expected), len(actual)) {
return false
}
for i := 0; i < len(expected); i++ {
if !Equal(t, expected[i], actual[i]) {
return false
}
}
return true
}
func equalMap(t *testing.T, expected, actual objects.Object) bool {
expectedT := expected.(*objects.Map).Value
actualT := actual.(*objects.Map).Value
if !Equal(t, len(expectedT), len(actualT)) {
return false
}
for key, expectedVal := range expectedT {
actualVal := actualT[key]
if !Equal(t, expectedVal, actualVal) {
return false
}
}
return true
}
func equalCompiledFunction(t *testing.T, expected, actual objects.Object) bool {
expectedT := expected.(*objects.CompiledFunction)
actualT := actual.(*objects.CompiledFunction)
return Equal(t, expectedT.Instructions, actualT.Instructions)
}
func equalClosure(t *testing.T, expected, actual objects.Object) bool {
expectedT := expected.(*objects.Closure)
actualT := actual.(*objects.Closure)
if !Equal(t, expectedT.Fn, actualT.Fn) {
return false
}
if !Equal(t, len(expectedT.Free), len(actualT.Free)) {
return false
}
for i := 0; i < len(expectedT.Free); i++ {
if !Equal(t, *expectedT.Free[i], *actualT.Free[i]) {
return false
}
}
return true
}

72
assert/trace.go Normal file
View file

@ -0,0 +1,72 @@
package assert
import (
"fmt"
"runtime"
"strings"
"unicode"
"unicode/utf8"
)
func errorTrace() []string {
pc := uintptr(0)
file := ""
line := 0
ok := false
name := ""
var callers []string
for i := 0; ; i++ {
pc, file, line, ok = runtime.Caller(i)
if !ok {
break
}
if file == "<autogenerated>" {
break
}
f := runtime.FuncForPC(pc)
if f == nil {
break
}
name = f.Name()
if name == "testing.tRunner" {
break
}
parts := strings.Split(file, "/")
file = parts[len(parts)-1]
if len(parts) > 1 {
dir := parts[len(parts)-2]
if (dir != "assert" && dir != "mock" && dir != "require") || file == "mock_test.go" {
callers = append(callers, fmt.Sprintf("%s:%d", file, line))
}
}
// Drop the package
segments := strings.Split(name, ".")
name = segments[len(segments)-1]
if isTest(name, "Test") ||
isTest(name, "Benchmark") ||
isTest(name, "Example") {
break
}
}
return callers
}
func isTest(name, prefix string) bool {
if !strings.HasPrefix(name, prefix) {
return false
}
if len(name) == len(prefix) { // "Test" is ok
return true
}
rune_, _ := utf8.DecodeRuneInString(name[len(prefix):])
return !unicode.IsLower(rune_)
}

32
ast/array_lit.go Normal file
View file

@ -0,0 +1,32 @@
package ast
import (
"strings"
"github.com/d5/tengo/scanner"
)
type ArrayLit struct {
Elements []Expr
LBrack scanner.Pos
RBrack scanner.Pos
}
func (e *ArrayLit) exprNode() {}
func (e *ArrayLit) Pos() scanner.Pos {
return e.LBrack
}
func (e *ArrayLit) End() scanner.Pos {
return e.RBrack + 1
}
func (e *ArrayLit) String() string {
var elts []string
for _, m := range e.Elements {
elts = append(elts, m.String())
}
return "[" + strings.Join(elts, ", ") + "]"
}

37
ast/assign_stmt.go Normal file
View file

@ -0,0 +1,37 @@
package ast
import (
"strings"
"github.com/d5/tengo/scanner"
"github.com/d5/tengo/token"
)
type AssignStmt struct {
Lhs []Expr
Rhs []Expr
Token token.Token
TokenPos scanner.Pos
}
func (s *AssignStmt) stmtNode() {}
func (s *AssignStmt) Pos() scanner.Pos {
return s.Lhs[0].Pos()
}
func (s *AssignStmt) End() scanner.Pos {
return s.Rhs[len(s.Rhs)-1].End()
}
func (s *AssignStmt) String() string {
var lhs, rhs []string
for _, e := range s.Lhs {
lhs = append(lhs, e.String())
}
for _, e := range s.Rhs {
rhs = append(rhs, e.String())
}
return strings.Join(lhs, ", ") + " " + s.Token.String() + " " + strings.Join(rhs, ", ")
}

5
ast/ast.go Normal file
View file

@ -0,0 +1,5 @@
package ast
const (
nullRep = "<null>"
)

22
ast/bad_expr.go Normal file
View file

@ -0,0 +1,22 @@
package ast
import "github.com/d5/tengo/scanner"
type BadExpr struct {
From scanner.Pos
To scanner.Pos
}
func (e *BadExpr) exprNode() {}
func (e *BadExpr) Pos() scanner.Pos {
return e.From
}
func (e *BadExpr) End() scanner.Pos {
return e.To
}
func (e *BadExpr) String() string {
return "<bad expression>"
}

22
ast/bad_stmt.go Normal file
View file

@ -0,0 +1,22 @@
package ast
import "github.com/d5/tengo/scanner"
type BadStmt struct {
From scanner.Pos
To scanner.Pos
}
func (s *BadStmt) stmtNode() {}
func (s *BadStmt) Pos() scanner.Pos {
return s.From
}
func (s *BadStmt) End() scanner.Pos {
return s.To
}
func (s *BadStmt) String() string {
return "<bad statement>"
}

27
ast/binary_expr.go Normal file
View file

@ -0,0 +1,27 @@
package ast
import (
"github.com/d5/tengo/scanner"
"github.com/d5/tengo/token"
)
type BinaryExpr struct {
Lhs Expr
Rhs Expr
Token token.Token
TokenPos scanner.Pos
}
func (e *BinaryExpr) exprNode() {}
func (e *BinaryExpr) Pos() scanner.Pos {
return e.Lhs.Pos()
}
func (e *BinaryExpr) End() scanner.Pos {
return e.Rhs.End()
}
func (e *BinaryExpr) String() string {
return "(" + e.Lhs.String() + " " + e.Token.String() + " " + e.Rhs.String() + ")"
}

32
ast/block_stmt.go Normal file
View file

@ -0,0 +1,32 @@
package ast
import (
"strings"
"github.com/d5/tengo/scanner"
)
type BlockStmt struct {
Stmts []Stmt
LBrace scanner.Pos
RBrace scanner.Pos
}
func (s *BlockStmt) stmtNode() {}
func (s *BlockStmt) Pos() scanner.Pos {
return s.LBrace
}
func (s *BlockStmt) End() scanner.Pos {
return s.RBrace + 1
}
func (s *BlockStmt) String() string {
var list []string
for _, e := range s.Stmts {
list = append(list, e.String())
}
return "{" + strings.Join(list, "; ") + "}"
}

23
ast/bool_lit.go Normal file
View file

@ -0,0 +1,23 @@
package ast
import "github.com/d5/tengo/scanner"
type BoolLit struct {
Value bool
ValuePos scanner.Pos
Literal string
}
func (e *BoolLit) exprNode() {}
func (e *BoolLit) Pos() scanner.Pos {
return e.ValuePos
}
func (e *BoolLit) End() scanner.Pos {
return scanner.Pos(int(e.ValuePos) + len(e.Literal))
}
func (e *BoolLit) String() string {
return e.Literal
}

35
ast/branch_stmt.go Normal file
View file

@ -0,0 +1,35 @@
package ast
import (
"github.com/d5/tengo/scanner"
"github.com/d5/tengo/token"
)
type BranchStmt struct {
Token token.Token
TokenPos scanner.Pos
Label *Ident
}
func (s *BranchStmt) stmtNode() {}
func (s *BranchStmt) Pos() scanner.Pos {
return s.TokenPos
}
func (s *BranchStmt) End() scanner.Pos {
if s.Label != nil {
return s.Label.End()
}
return scanner.Pos(int(s.TokenPos) + len(s.Token.String()))
}
func (s *BranchStmt) String() string {
var label string
if s.Label != nil {
label = " " + s.Label.Name
}
return s.Token.String() + label
}

33
ast/call_expr.go Normal file
View file

@ -0,0 +1,33 @@
package ast
import (
"strings"
"github.com/d5/tengo/scanner"
)
type CallExpr struct {
Func Expr
LParen scanner.Pos
Args []Expr
RParen scanner.Pos
}
func (e *CallExpr) exprNode() {}
func (e *CallExpr) Pos() scanner.Pos {
return e.Func.Pos()
}
func (e *CallExpr) End() scanner.Pos {
return e.RParen + 1
}
func (e *CallExpr) String() string {
var args []string
for _, e := range e.Args {
args = append(args, e.String())
}
return e.Func.String() + "(" + strings.Join(args, ", ") + ")"
}

23
ast/char_lit.go Normal file
View file

@ -0,0 +1,23 @@
package ast
import "github.com/d5/tengo/scanner"
type CharLit struct {
Value rune
ValuePos scanner.Pos
Literal string
}
func (e *CharLit) exprNode() {}
func (e *CharLit) Pos() scanner.Pos {
return e.ValuePos
}
func (e *CharLit) End() scanner.Pos {
return scanner.Pos(int(e.ValuePos) + len(e.Literal))
}
func (e *CharLit) String() string {
return e.Literal
}

26
ast/empty_stmt.go Normal file
View file

@ -0,0 +1,26 @@
package ast
import "github.com/d5/tengo/scanner"
type EmptyStmt struct {
Semicolon scanner.Pos
Implicit bool
}
func (s *EmptyStmt) stmtNode() {}
func (s *EmptyStmt) Pos() scanner.Pos {
return s.Semicolon
}
func (s *EmptyStmt) End() scanner.Pos {
if s.Implicit {
return s.Semicolon
}
return s.Semicolon + 1
}
func (s *EmptyStmt) String() string {
return ";"
}

6
ast/expr.go Normal file
View file

@ -0,0 +1,6 @@
package ast
type Expr interface {
Node
exprNode()
}

21
ast/expr_stmt.go Normal file
View file

@ -0,0 +1,21 @@
package ast
import "github.com/d5/tengo/scanner"
type ExprStmt struct {
Expr Expr
}
func (s *ExprStmt) stmtNode() {}
func (s *ExprStmt) Pos() scanner.Pos {
return s.Expr.Pos()
}
func (s *ExprStmt) End() scanner.Pos {
return s.Expr.End()
}
func (s *ExprStmt) String() string {
return s.Expr.String()
}

29
ast/file.go Normal file
View file

@ -0,0 +1,29 @@
package ast
import (
"strings"
"github.com/d5/tengo/scanner"
)
type File struct {
InputFile *scanner.File
Stmts []Stmt
}
func (n *File) Pos() scanner.Pos {
return scanner.Pos(n.InputFile.Base())
}
func (n *File) End() scanner.Pos {
return scanner.Pos(n.InputFile.Base() + n.InputFile.Size())
}
func (n *File) String() string {
var stmts []string
for _, e := range n.Stmts {
stmts = append(stmts, e.String())
}
return strings.Join(stmts, "; ")
}

23
ast/float_lit.go Normal file
View file

@ -0,0 +1,23 @@
package ast
import "github.com/d5/tengo/scanner"
type FloatLit struct {
Value float64
ValuePos scanner.Pos
Literal string
}
func (e *FloatLit) exprNode() {}
func (e *FloatLit) Pos() scanner.Pos {
return e.ValuePos
}
func (e *FloatLit) End() scanner.Pos {
return scanner.Pos(int(e.ValuePos) + len(e.Literal))
}
func (e *FloatLit) String() string {
return e.Literal
}

29
ast/for_in_stmt.go Normal file
View file

@ -0,0 +1,29 @@
package ast
import "github.com/d5/tengo/scanner"
type ForInStmt struct {
ForPos scanner.Pos
Key *Ident
Value *Ident
Iterable Expr
Body *BlockStmt
}
func (s *ForInStmt) stmtNode() {}
func (s *ForInStmt) Pos() scanner.Pos {
return s.ForPos
}
func (s *ForInStmt) End() scanner.Pos {
return s.Body.End()
}
func (s *ForInStmt) String() string {
if s.Value != nil {
return "for " + s.Key.String() + ", " + s.Value.String() + " in " + s.Iterable.String() + " " + s.Body.String()
}
return "for " + s.Key.String() + " in " + s.Iterable.String() + " " + s.Body.String()
}

40
ast/for_stmt.go Normal file
View file

@ -0,0 +1,40 @@
package ast
import "github.com/d5/tengo/scanner"
type ForStmt struct {
ForPos scanner.Pos
Init Stmt
Cond Expr
Post Stmt
Body *BlockStmt
}
func (s *ForStmt) stmtNode() {}
func (s *ForStmt) Pos() scanner.Pos {
return s.ForPos
}
func (s *ForStmt) End() scanner.Pos {
return s.Body.End()
}
func (s *ForStmt) String() string {
var init, cond, post string
if s.Init != nil {
init = s.Init.String()
}
if s.Cond != nil {
cond = s.Cond.String() + " "
}
if s.Post != nil {
post = s.Post.String()
}
if init != "" || post != "" {
return "for " + init + " ; " + cond + " ; " + post + s.Body.String()
}
return "for " + cond + s.Body.String()
}

22
ast/func_lit.go Normal file
View file

@ -0,0 +1,22 @@
package ast
import "github.com/d5/tengo/scanner"
type FuncLit struct {
Type *FuncType
Body *BlockStmt
}
func (e *FuncLit) exprNode() {}
func (e *FuncLit) Pos() scanner.Pos {
return e.Type.Pos()
}
func (e *FuncLit) End() scanner.Pos {
return e.Body.End()
}
func (e *FuncLit) String() string {
return "func" + e.Type.Params.String() + " " + e.Body.String()
}

22
ast/func_type.go Normal file
View file

@ -0,0 +1,22 @@
package ast
import "github.com/d5/tengo/scanner"
type FuncType struct {
FuncPos scanner.Pos
Params *IdentList
}
func (e *FuncType) exprNode() {}
func (e *FuncType) Pos() scanner.Pos {
return e.FuncPos
}
func (e *FuncType) End() scanner.Pos {
return e.Params.End()
}
func (e *FuncType) String() string {
return "func" + e.Params.String()
}

26
ast/ident.go Normal file
View file

@ -0,0 +1,26 @@
package ast
import "github.com/d5/tengo/scanner"
type Ident struct {
Name string
NamePos scanner.Pos
}
func (e *Ident) exprNode() {}
func (e *Ident) Pos() scanner.Pos {
return e.NamePos
}
func (e *Ident) End() scanner.Pos {
return scanner.Pos(int(e.NamePos) + len(e.Name))
}
func (e *Ident) String() string {
if e != nil {
return e.Name
}
return nullRep
}

54
ast/ident_list.go Normal file
View file

@ -0,0 +1,54 @@
package ast
import (
"strings"
"github.com/d5/tengo/scanner"
)
type IdentList struct {
LParen scanner.Pos
List []*Ident
RParen scanner.Pos
}
func (n *IdentList) Pos() scanner.Pos {
if n.LParen.IsValid() {
return n.LParen
}
if len(n.List) > 0 {
return n.List[0].Pos()
}
return scanner.NoPos
}
func (n *IdentList) End() scanner.Pos {
if n.RParen.IsValid() {
return n.RParen + 1
}
if l := len(n.List); l > 0 {
return n.List[l-1].End()
}
return scanner.NoPos
}
func (n *IdentList) NumFields() int {
if n == nil {
return 0
}
return len(n.List)
}
func (n *IdentList) String() string {
var list []string
for _, e := range n.List {
list = append(list, e.String())
}
return "(" + strings.Join(list, ", ") + ")"
}

37
ast/if_stmt.go Normal file
View file

@ -0,0 +1,37 @@
package ast
import "github.com/d5/tengo/scanner"
type IfStmt struct {
IfPos scanner.Pos
Init Stmt
Cond Expr
Body *BlockStmt
Else Stmt // else branch; or nil
}
func (s *IfStmt) stmtNode() {}
func (s *IfStmt) Pos() scanner.Pos {
return s.IfPos
}
func (s *IfStmt) End() scanner.Pos {
if s.Else != nil {
return s.Else.End()
}
return s.Body.End()
}
func (s *IfStmt) String() string {
var initStmt, elseStmt string
if s.Init != nil {
initStmt = s.Init.String() + "; "
}
if s.Else != nil {
elseStmt = " else " + s.Else.String()
}
return "if " + initStmt + s.Cond.String() + " " + s.Body.String() + elseStmt
}

26
ast/inc_dec_stmt.go Normal file
View file

@ -0,0 +1,26 @@
package ast
import (
"github.com/d5/tengo/scanner"
"github.com/d5/tengo/token"
)
type IncDecStmt struct {
Expr Expr
Token token.Token
TokenPos scanner.Pos
}
func (s *IncDecStmt) stmtNode() {}
func (s *IncDecStmt) Pos() scanner.Pos {
return s.Expr.Pos()
}
func (s *IncDecStmt) End() scanner.Pos {
return scanner.Pos(int(s.TokenPos) + 2)
}
func (s *IncDecStmt) String() string {
return s.Expr.String() + s.Token.String()
}

29
ast/index_expr.go Normal file
View file

@ -0,0 +1,29 @@
package ast
import "github.com/d5/tengo/scanner"
type IndexExpr struct {
Expr Expr
LBrack scanner.Pos
Index Expr
RBrack scanner.Pos
}
func (e *IndexExpr) exprNode() {}
func (e *IndexExpr) Pos() scanner.Pos {
return e.Expr.Pos()
}
func (e *IndexExpr) End() scanner.Pos {
return e.RBrack + 1
}
func (e *IndexExpr) String() string {
var index string
if e.Index != nil {
index = e.Index.String()
}
return e.Expr.String() + "[" + index + "]"
}

23
ast/int_lit.go Normal file
View file

@ -0,0 +1,23 @@
package ast
import "github.com/d5/tengo/scanner"
type IntLit struct {
Value int64
ValuePos scanner.Pos
Literal string
}
func (e *IntLit) exprNode() {}
func (e *IntLit) Pos() scanner.Pos {
return e.ValuePos
}
func (e *IntLit) End() scanner.Pos {
return scanner.Pos(int(e.ValuePos) + len(e.Literal))
}
func (e *IntLit) String() string {
return e.Literal
}

24
ast/map_element_lit.go Normal file
View file

@ -0,0 +1,24 @@
package ast
import "github.com/d5/tengo/scanner"
type MapElementLit struct {
Key string
KeyPos scanner.Pos
ColonPos scanner.Pos
Value Expr
}
func (e *MapElementLit) exprNode() {}
func (e *MapElementLit) Pos() scanner.Pos {
return e.KeyPos
}
func (e *MapElementLit) End() scanner.Pos {
return e.Value.End()
}
func (e *MapElementLit) String() string {
return e.Key + ": " + e.Value.String()
}

32
ast/map_lit.go Normal file
View file

@ -0,0 +1,32 @@
package ast
import (
"strings"
"github.com/d5/tengo/scanner"
)
type MapLit struct {
LBrace scanner.Pos
Elements []*MapElementLit
RBrace scanner.Pos
}
func (e *MapLit) exprNode() {}
func (e *MapLit) Pos() scanner.Pos {
return e.LBrace
}
func (e *MapLit) End() scanner.Pos {
return e.RBrace + 1
}
func (e *MapLit) String() string {
var elts []string
for _, m := range e.Elements {
elts = append(elts, m.String())
}
return "{" + strings.Join(elts, ", ") + "}"
}

11
ast/node.go Normal file
View file

@ -0,0 +1,11 @@
package ast
import (
"github.com/d5/tengo/scanner"
)
type Node interface {
Pos() scanner.Pos
End() scanner.Pos
String() string
}

23
ast/paren_expr.go Normal file
View file

@ -0,0 +1,23 @@
package ast
import "github.com/d5/tengo/scanner"
type ParenExpr struct {
Expr Expr
LParen scanner.Pos
RParen scanner.Pos
}
func (e *ParenExpr) exprNode() {}
func (e *ParenExpr) Pos() scanner.Pos {
return e.LParen
}
func (e *ParenExpr) End() scanner.Pos {
return e.RParen + 1
}
func (e *ParenExpr) String() string {
return "(" + e.Expr.String() + ")"
}

39
ast/return_stmt.go Normal file
View file

@ -0,0 +1,39 @@
package ast
import (
"strings"
"github.com/d5/tengo/scanner"
)
type ReturnStmt struct {
ReturnPos scanner.Pos
Results []Expr
}
func (s *ReturnStmt) stmtNode() {}
func (s *ReturnStmt) Pos() scanner.Pos {
return s.ReturnPos
}
func (s *ReturnStmt) End() scanner.Pos {
if n := len(s.Results); n > 0 {
return s.Results[n-1].End()
}
return s.ReturnPos + 6
}
func (s *ReturnStmt) String() string {
if len(s.Results) > 0 {
var res []string
for _, e := range s.Results {
res = append(res, e.String())
}
return "return " + strings.Join(res, ", ")
}
return "return"
}

22
ast/selector_expr.go Normal file
View file

@ -0,0 +1,22 @@
package ast
import "github.com/d5/tengo/scanner"
type SelectorExpr struct {
Expr Expr
Sel Expr
}
func (e *SelectorExpr) exprNode() {}
func (e *SelectorExpr) Pos() scanner.Pos {
return e.Expr.Pos()
}
func (e *SelectorExpr) End() scanner.Pos {
return e.Sel.End()
}
func (e *SelectorExpr) String() string {
return e.Expr.String() + "." + e.Sel.String()
}

33
ast/slice_expr.go Normal file
View file

@ -0,0 +1,33 @@
package ast
import "github.com/d5/tengo/scanner"
type SliceExpr struct {
Expr Expr
LBrack scanner.Pos
Low Expr
High Expr
RBrack scanner.Pos
}
func (e *SliceExpr) exprNode() {}
func (e *SliceExpr) Pos() scanner.Pos {
return e.Expr.Pos()
}
func (e *SliceExpr) End() scanner.Pos {
return e.RBrack + 1
}
func (e *SliceExpr) String() string {
var low, high string
if e.Low != nil {
low = e.Low.String()
}
if e.High != nil {
high = e.High.String()
}
return e.Expr.String() + "[" + low + ":" + high + "]"
}

6
ast/stmt.go Normal file
View file

@ -0,0 +1,6 @@
package ast
type Stmt interface {
Node
stmtNode()
}

23
ast/string_lit.go Normal file
View file

@ -0,0 +1,23 @@
package ast
import "github.com/d5/tengo/scanner"
type StringLit struct {
Value string
ValuePos scanner.Pos
Literal string
}
func (e *StringLit) exprNode() {}
func (e *StringLit) Pos() scanner.Pos {
return e.ValuePos
}
func (e *StringLit) End() scanner.Pos {
return scanner.Pos(int(e.ValuePos) + len(e.Literal))
}
func (e *StringLit) String() string {
return e.Literal
}

26
ast/unary_expr.go Normal file
View file

@ -0,0 +1,26 @@
package ast
import (
"github.com/d5/tengo/scanner"
"github.com/d5/tengo/token"
)
type UnaryExpr struct {
Expr Expr
Token token.Token
TokenPos scanner.Pos
}
func (e *UnaryExpr) exprNode() {}
func (e *UnaryExpr) Pos() scanner.Pos {
return e.Expr.Pos()
}
func (e *UnaryExpr) End() scanner.Pos {
return e.Expr.End()
}
func (e *UnaryExpr) String() string {
return "(" + e.Token.String() + e.Expr.String() + ")"
}

118
cmd/bench/main.go Normal file
View file

@ -0,0 +1,118 @@
package main
import (
"fmt"
"time"
"github.com/d5/tengo/ast"
"github.com/d5/tengo/compiler"
"github.com/d5/tengo/objects"
"github.com/d5/tengo/parser"
"github.com/d5/tengo/scanner"
"github.com/d5/tengo/vm"
)
func main() {
runFib(35)
}
func runFib(n int) {
start := time.Now()
nativeResult := fib(n)
nativeTime := time.Since(start)
input := `
fib := func(x) {
if(x == 0) {
return 0
} else if(x == 1) {
return 1
} else {
return fib(x-1) + fib(x-2)
}
}
` + fmt.Sprintf("out = fib(%d)", n)
parseTime, compileTime, runTime, result, err := runBench([]byte(input))
if err != nil {
panic(err)
}
if nativeResult != int(result.(*objects.Int).Value) {
panic(fmt.Errorf("wrong result: %d != %d", nativeResult, int(result.(*objects.Int).Value)))
}
fmt.Printf("Go: %s\n", nativeTime)
fmt.Printf("Parser: %s\n", parseTime)
fmt.Printf("Compile: %s\n", compileTime)
fmt.Printf("VM: %s\n", runTime)
}
func fib(n int) int {
if n == 0 {
return 0
} else if n == 1 {
return 1
} else {
return fib(n-1) + fib(n-2)
}
}
func runBench(input []byte) (parseTime time.Duration, compileTime time.Duration, runTime time.Duration, result objects.Object, err error) {
var astFile *ast.File
parseTime, astFile, err = parse(input)
if err != nil {
return
}
var bytecode *compiler.Bytecode
compileTime, bytecode, err = compileFile(astFile)
if err != nil {
return
}
runTime, result, err = runVM(bytecode)
return
}
func parse(input []byte) (time.Duration, *ast.File, error) {
fileSet := scanner.NewFileSet()
inputFile := fileSet.AddFile("test", -1, len(input))
start := time.Now()
file, err := parser.ParseFile(inputFile, input, nil)
if err != nil {
return time.Since(start), nil, err
}
return time.Since(start), file, nil
}
func compileFile(file *ast.File) (time.Duration, *compiler.Bytecode, error) {
symTable := compiler.NewSymbolTable()
symTable.Define("out")
start := time.Now()
c := compiler.NewCompiler(symTable, nil)
if err := c.Compile(file); err != nil {
return time.Since(start), nil, err
}
return time.Since(start), c.Bytecode(), nil
}
func runVM(bytecode *compiler.Bytecode) (time.Duration, objects.Object, error) {
globals := make([]*objects.Object, vm.GlobalsSize)
start := time.Now()
v := vm.NewVM(bytecode, globals)
if err := v.Run(); err != nil {
return time.Since(start), nil, err
}
return time.Since(start), *globals[0], nil
}

111
cmd/repl/main.go Normal file
View file

@ -0,0 +1,111 @@
package main
import (
"bufio"
"fmt"
"io"
"os"
"os/user"
"github.com/d5/tengo/ast"
"github.com/d5/tengo/compiler"
"github.com/d5/tengo/objects"
"github.com/d5/tengo/parser"
"github.com/d5/tengo/scanner"
"github.com/d5/tengo/vm"
)
const (
Prompt = ">> "
)
func main() {
currentUser, err := user.Current()
if err != nil {
panic(err)
}
fmt.Printf("Hello, %s! This is the Ghost programming language!\n", currentUser.Name)
startRepl(os.Stdin, os.Stdout)
}
func startRepl(in io.Reader, out io.Writer) {
stdin := bufio.NewScanner(in)
fileSet := scanner.NewFileSet()
globals := make([]*objects.Object, vm.GlobalsSize)
symbolTable := compiler.NewSymbolTable()
for {
_, _ = fmt.Fprintf(out, Prompt)
scanned := stdin.Scan()
if !scanned {
return
}
line := stdin.Text()
file, err := parser.ParseFile(fileSet.AddFile("test", -1, len(line)), []byte(line), nil)
if err != nil {
_, _ = fmt.Fprintf(out, "error: %s\n", err.Error())
continue
}
file = addPrints(file)
c := compiler.NewCompiler(symbolTable, nil)
if err := c.Compile(file); err != nil {
_, _ = fmt.Fprintf(out, "Compilation error:\n %s\n", err.Error())
continue
}
machine := vm.NewVM(c.Bytecode(), globals)
if err != nil {
_, _ = fmt.Fprintf(out, "VM error:\n %s\n", err.Error())
continue
}
if err := machine.Run(); err != nil {
_, _ = fmt.Fprintf(out, "Execution error:\n %s\n", err.Error())
continue
}
}
}
func addPrints(file *ast.File) *ast.File {
var stmts []ast.Stmt
for _, s := range file.Stmts {
switch s := s.(type) {
case *ast.ExprStmt:
stmts = append(stmts, &ast.ExprStmt{
Expr: &ast.CallExpr{
Func: &ast.Ident{
Name: "print",
},
Args: []ast.Expr{s.Expr},
},
})
case *ast.AssignStmt:
stmts = append(stmts, s)
stmts = append(stmts, &ast.ExprStmt{
Expr: &ast.CallExpr{
Func: &ast.Ident{
Name: "print",
},
Args: s.Lhs,
},
})
default:
stmts = append(stmts, s)
}
}
return &ast.File{
InputFile: file.InputFile,
Stmts: stmts,
}
}

53
compiler/bytecode.go Normal file
View file

@ -0,0 +1,53 @@
package compiler
import (
"encoding/gob"
"io"
"github.com/d5/tengo/objects"
)
type Bytecode struct {
Instructions []byte
Constants []objects.Object
}
func (b *Bytecode) Decode(r io.Reader) error {
dec := gob.NewDecoder(r)
if err := dec.Decode(&b.Instructions); err != nil {
return err
}
if err := dec.Decode(&b.Constants); err != nil {
return err
}
return nil
}
func (b *Bytecode) Encode(w io.Writer) error {
enc := gob.NewEncoder(w)
if err := enc.Encode(b.Instructions); err != nil {
return err
}
if err := enc.Encode(b.Constants); err != nil {
return err
}
return nil
}
func init() {
gob.Register(&objects.Int{})
gob.Register(&objects.Float{})
gob.Register(&objects.String{})
gob.Register(&objects.Bool{})
gob.Register(&objects.Char{})
gob.Register(&objects.Array{})
gob.Register(&objects.Map{})
gob.Register(&objects.CompiledFunction{})
gob.Register(&objects.Undefined{})
}

93
compiler/bytecode_test.go Normal file
View file

@ -0,0 +1,93 @@
package compiler_test
import (
"bytes"
"testing"
"github.com/d5/tengo/assert"
"github.com/d5/tengo/compiler"
"github.com/d5/tengo/objects"
)
func TestBytecode(t *testing.T) {
testBytecodeSerialization(t, &compiler.Bytecode{})
testBytecodeSerialization(t, bytecode(
concat(), objectsArray(
&objects.Array{
Value: objectsArray(
&objects.Int{Value: 12},
&objects.String{Value: "foo"},
&objects.Bool{Value: true},
&objects.Float{Value: 93.11},
&objects.Char{Value: 'x'},
),
},
&objects.Bool{Value: false},
&objects.Char{Value: 'y'},
&objects.Float{Value: 93.11},
compiledFunction(1, 0,
compiler.MakeInstruction(compiler.OpConstant, 3),
compiler.MakeInstruction(compiler.OpSetLocal, 0),
compiler.MakeInstruction(compiler.OpGetGlobal, 0),
compiler.MakeInstruction(compiler.OpGetFree, 0)),
&objects.Float{Value: 39.2},
&objects.Int{Value: 192},
&objects.Map{
Value: map[string]objects.Object{
"a": &objects.Float{Value: -93.1},
"b": &objects.Bool{Value: false},
},
},
&objects.String{Value: "bar"},
&objects.Undefined{})))
testBytecodeSerialization(t, bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpSetGlobal, 0),
compiler.MakeInstruction(compiler.OpConstant, 6),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray(
intObject(55),
intObject(66),
intObject(77),
intObject(88),
compiledFunction(1, 0,
compiler.MakeInstruction(compiler.OpConstant, 3),
compiler.MakeInstruction(compiler.OpSetLocal, 0),
compiler.MakeInstruction(compiler.OpGetGlobal, 0),
compiler.MakeInstruction(compiler.OpGetFree, 0),
compiler.MakeInstruction(compiler.OpAdd),
compiler.MakeInstruction(compiler.OpGetFree, 1),
compiler.MakeInstruction(compiler.OpAdd),
compiler.MakeInstruction(compiler.OpGetLocal, 0),
compiler.MakeInstruction(compiler.OpAdd),
compiler.MakeInstruction(compiler.OpReturnValue, 1)),
compiledFunction(1, 0,
compiler.MakeInstruction(compiler.OpConstant, 2),
compiler.MakeInstruction(compiler.OpSetLocal, 0),
compiler.MakeInstruction(compiler.OpGetFree, 0),
compiler.MakeInstruction(compiler.OpGetLocal, 0),
compiler.MakeInstruction(compiler.OpClosure, 4, 2),
compiler.MakeInstruction(compiler.OpReturnValue, 1)),
compiledFunction(1, 0,
compiler.MakeInstruction(compiler.OpConstant, 1),
compiler.MakeInstruction(compiler.OpSetLocal, 0),
compiler.MakeInstruction(compiler.OpGetLocal, 0),
compiler.MakeInstruction(compiler.OpClosure, 5, 1),
compiler.MakeInstruction(compiler.OpReturnValue, 1)))))
}
func testBytecodeSerialization(t *testing.T, b *compiler.Bytecode) {
var buf bytes.Buffer
err := b.Encode(&buf)
assert.NoError(t, err)
r := &compiler.Bytecode{}
err = r.Decode(bytes.NewReader(buf.Bytes()))
assert.NoError(t, err)
assert.Equal(t, b.Instructions, r.Instructions)
assert.Equal(t, b.Constants, r.Constants)
}

View file

@ -0,0 +1,6 @@
package compiler
type CompilationScope struct {
instructions []byte
lastInstructions [2]EmittedInstruction
}

514
compiler/compiler.go Normal file
View file

@ -0,0 +1,514 @@
package compiler
import (
"fmt"
"io"
"reflect"
"github.com/d5/tengo/ast"
"github.com/d5/tengo/objects"
"github.com/d5/tengo/token"
)
type Compiler struct {
constants []objects.Object
symbolTable *SymbolTable
scopes []CompilationScope
scopeIndex int
loops []*Loop
loopIndex int
trace io.Writer
indent int
}
func NewCompiler(symbolTable *SymbolTable, trace io.Writer) *Compiler {
mainScope := CompilationScope{
instructions: make([]byte, 0),
}
if symbolTable == nil {
symbolTable = NewSymbolTable()
}
for idx, fn := range objects.Builtins {
symbolTable.DefineBuiltin(idx, fn.Name)
}
return &Compiler{
symbolTable: symbolTable,
scopes: []CompilationScope{mainScope},
scopeIndex: 0,
loopIndex: -1,
trace: trace,
}
}
func (c *Compiler) Compile(node ast.Node) error {
if c.trace != nil {
if node != nil {
defer un(trace(c, fmt.Sprintf("%s (%s)", node.String(), reflect.TypeOf(node).Elem().Name())))
} else {
defer un(trace(c, "<nil>"))
}
}
switch node := node.(type) {
case *ast.File:
for _, stmt := range node.Stmts {
if err := c.Compile(stmt); err != nil {
return err
}
}
case *ast.ExprStmt:
if err := c.Compile(node.Expr); err != nil {
return err
}
c.emit(OpPop)
case *ast.IncDecStmt:
op := token.AddAssign
if node.Token == token.Dec {
op = token.SubAssign
}
return c.compileAssign([]ast.Expr{node.Expr}, []ast.Expr{&ast.IntLit{Value: 1}}, op)
case *ast.ParenExpr:
if err := c.Compile(node.Expr); err != nil {
return err
}
case *ast.BinaryExpr:
if node.Token == token.LAnd || node.Token == token.LOr {
if err := c.compileLogical(node); err != nil {
return err
}
return nil
}
if node.Token == token.Less {
if err := c.Compile(node.Rhs); err != nil {
return err
}
if err := c.Compile(node.Lhs); err != nil {
return err
}
c.emit(OpGreaterThan)
return nil
} else if node.Token == token.LessEq {
if err := c.Compile(node.Rhs); err != nil {
return err
}
if err := c.Compile(node.Lhs); err != nil {
return err
}
c.emit(OpGreaterThanEqual)
return nil
}
if err := c.Compile(node.Lhs); err != nil {
return err
}
if err := c.Compile(node.Rhs); err != nil {
return err
}
switch node.Token {
case token.Add:
c.emit(OpAdd)
case token.Sub:
c.emit(OpSub)
case token.Mul:
c.emit(OpMul)
case token.Quo:
c.emit(OpDiv)
case token.Rem:
c.emit(OpRem)
case token.Greater:
c.emit(OpGreaterThan)
case token.GreaterEq:
c.emit(OpGreaterThanEqual)
case token.Equal:
c.emit(OpEqual)
case token.NotEqual:
c.emit(OpNotEqual)
case token.And:
c.emit(OpBAnd)
case token.Or:
c.emit(OpBOr)
case token.Xor:
c.emit(OpBXor)
case token.AndNot:
c.emit(OpBAndNot)
case token.Shl:
c.emit(OpBShiftLeft)
case token.Shr:
c.emit(OpBShiftRight)
default:
return fmt.Errorf("unknown operator: %s", node.Token.String())
}
case *ast.IntLit:
c.emit(OpConstant, c.addConstant(&objects.Int{Value: node.Value}))
case *ast.FloatLit:
c.emit(OpConstant, c.addConstant(&objects.Float{Value: node.Value}))
case *ast.BoolLit:
if node.Value {
c.emit(OpTrue)
} else {
c.emit(OpFalse)
}
case *ast.StringLit:
c.emit(OpConstant, c.addConstant(&objects.String{Value: node.Value}))
case *ast.UnaryExpr:
if err := c.Compile(node.Expr); err != nil {
return err
}
switch node.Token {
case token.Not:
c.emit(OpLNot)
case token.Sub:
c.emit(OpMinus)
case token.Xor:
c.emit(OpBComplement)
case token.Add:
// do nothing?
default:
return fmt.Errorf("unknown operator: %s", node.Token.String())
}
case *ast.IfStmt:
// open new symbol table for the statement
c.symbolTable = c.symbolTable.Fork(true)
defer func() {
c.symbolTable = c.symbolTable.Parent(false)
}()
if node.Init != nil {
if err := c.Compile(node.Init); err != nil {
return err
}
}
if err := c.Compile(node.Cond); err != nil {
return err
}
// first jump placeholder
jumpPos1 := c.emit(OpJumpFalsy, 0)
if err := c.Compile(node.Body); err != nil {
return err
}
// second jump placeholder
jumpPos2 := c.emit(OpJump, 0)
// update first jump offset
curPos := len(c.currentInstructions())
c.changeOperand(jumpPos1, curPos)
if node.Else != nil {
if err := c.Compile(node.Else); err != nil {
return err
}
}
// update second jump offset
curPos = len(c.currentInstructions())
c.changeOperand(jumpPos2, curPos)
case *ast.ForStmt:
return c.compileForStmt(node)
case *ast.ForInStmt:
return c.compileForInStmt(node)
case *ast.BranchStmt:
if node.Token == token.Break {
curLoop := c.currentLoop()
if curLoop == nil {
return fmt.Errorf("break statement outside loop")
}
pos := c.emit(OpJump, 0)
curLoop.Breaks = append(curLoop.Breaks, pos)
} else if node.Token == token.Continue {
curLoop := c.currentLoop()
if curLoop == nil {
return fmt.Errorf("continue statement outside loop")
}
pos := c.emit(OpJump, 0)
curLoop.Continues = append(curLoop.Continues, pos)
} else {
return fmt.Errorf("unknown branch statement: %s", node.Token.String())
}
case *ast.BlockStmt:
for _, stmt := range node.Stmts {
if err := c.Compile(stmt); err != nil {
return err
}
}
case *ast.AssignStmt:
if err := c.compileAssign(node.Lhs, node.Rhs, node.Token); err != nil {
return err
}
case *ast.Ident:
symbol, _, ok := c.symbolTable.Resolve(node.Name)
if !ok {
return fmt.Errorf("undefined variable: %s", node.Name)
}
switch symbol.Scope {
case ScopeGlobal:
c.emit(OpGetGlobal, symbol.Index)
case ScopeLocal:
c.emit(OpGetLocal, symbol.Index)
case ScopeBuiltin:
c.emit(OpGetBuiltin, symbol.Index)
case ScopeFree:
c.emit(OpGetFree, symbol.Index)
}
case *ast.ArrayLit:
for _, elem := range node.Elements {
if err := c.Compile(elem); err != nil {
return err
}
}
c.emit(OpArray, len(node.Elements))
case *ast.MapLit:
for _, elt := range node.Elements {
// key
c.emit(OpConstant, c.addConstant(&objects.String{Value: elt.Key}))
// value
if err := c.Compile(elt.Value); err != nil {
return err
}
}
c.emit(OpMap, len(node.Elements)*2)
case *ast.SelectorExpr: // selector on RHS side
if err := c.Compile(node.Expr); err != nil {
return err
}
if err := c.Compile(node.Sel); err != nil {
return err
}
c.emit(OpIndex)
case *ast.IndexExpr:
if err := c.Compile(node.Expr); err != nil {
return err
}
if err := c.Compile(node.Index); err != nil {
return err
}
c.emit(OpIndex)
case *ast.SliceExpr:
if err := c.Compile(node.Expr); err != nil {
return err
}
if node.Low != nil {
if err := c.Compile(node.Low); err != nil {
return err
}
} else {
c.emit(OpNull)
}
if node.High != nil {
if err := c.Compile(node.High); err != nil {
return err
}
} else {
c.emit(OpNull)
}
c.emit(OpSliceIndex)
case *ast.FuncLit:
c.enterScope()
for _, p := range node.Type.Params.List {
c.symbolTable.Define(p.Name)
}
if err := c.Compile(node.Body); err != nil {
return err
}
// add OpReturn if function returns nothing
if !c.lastInstructionIs(OpReturnValue) && !c.lastInstructionIs(OpReturn) {
c.emit(OpReturn)
}
freeSymbols := c.symbolTable.FreeSymbols()
numLocals := c.symbolTable.MaxSymbols()
instructions := c.leaveScope()
for _, s := range freeSymbols {
switch s.Scope {
case ScopeLocal:
c.emit(OpGetLocal, s.Index)
case ScopeFree:
c.emit(OpGetFree, s.Index)
}
}
compiledFunction := &objects.CompiledFunction{
Instructions: instructions,
NumLocals: numLocals,
NumParameters: len(node.Type.Params.List),
}
if len(freeSymbols) > 0 {
c.emit(OpClosure, c.addConstant(compiledFunction), len(freeSymbols))
} else {
c.emit(OpConstant, c.addConstant(compiledFunction))
}
case *ast.ReturnStmt:
if c.symbolTable.Parent(true) == nil {
// outside the function
return fmt.Errorf("return statement outside function")
}
switch len(node.Results) {
case 0:
c.emit(OpReturn)
case 1:
if err := c.Compile(node.Results[0]); err != nil {
return err
}
c.emit(OpReturnValue, 1)
default:
return fmt.Errorf("multi-value return not implemented")
}
case *ast.CallExpr:
if err := c.Compile(node.Func); err != nil {
return err
}
for _, arg := range node.Args {
if err := c.Compile(arg); err != nil {
return err
}
}
c.emit(OpCall, len(node.Args))
}
return nil
}
func (c *Compiler) Bytecode() *Bytecode {
return &Bytecode{
Instructions: c.currentInstructions(),
Constants: c.constants,
}
}
func (c *Compiler) addConstant(o objects.Object) int {
c.constants = append(c.constants, o)
if c.trace != nil {
c.printTrace(fmt.Sprintf("CONST %04d %s", len(c.constants)-1, o))
}
return len(c.constants) - 1
}
func (c *Compiler) addInstruction(b []byte) int {
posNewIns := len(c.currentInstructions())
c.scopes[c.scopeIndex].instructions = append(c.currentInstructions(), b...)
return posNewIns
}
func (c *Compiler) setLastInstruction(op Opcode, pos int) {
c.scopes[c.scopeIndex].lastInstructions[1] = c.scopes[c.scopeIndex].lastInstructions[0]
c.scopes[c.scopeIndex].lastInstructions[0].Opcode = op
c.scopes[c.scopeIndex].lastInstructions[0].Position = pos
}
func (c *Compiler) lastInstructionIs(op Opcode) bool {
if len(c.currentInstructions()) == 0 {
return false
}
return c.scopes[c.scopeIndex].lastInstructions[0].Opcode == op
}
func (c *Compiler) removeLastInstruction() {
lastPos := c.scopes[c.scopeIndex].lastInstructions[0].Position
if c.trace != nil {
c.printTrace(fmt.Sprintf("DELET %s",
FormatInstructions(c.scopes[c.scopeIndex].instructions[lastPos:], lastPos)[0]))
}
c.scopes[c.scopeIndex].instructions = c.currentInstructions()[:lastPos]
c.scopes[c.scopeIndex].lastInstructions[0] = c.scopes[c.scopeIndex].lastInstructions[1]
}
func (c *Compiler) replaceInstruction(pos int, inst []byte) {
copy(c.currentInstructions()[pos:], inst)
if c.trace != nil {
c.printTrace(fmt.Sprintf("REPLC %s",
FormatInstructions(c.scopes[c.scopeIndex].instructions[pos:], pos)[0]))
}
}
func (c *Compiler) changeOperand(opPos int, operand ...int) {
op := Opcode(c.currentInstructions()[opPos])
inst := MakeInstruction(op, operand...)
c.replaceInstruction(opPos, inst)
}
func (c *Compiler) emit(opcode Opcode, operands ...int) int {
inst := MakeInstruction(opcode, operands...)
pos := c.addInstruction(inst)
c.setLastInstruction(opcode, pos)
if c.trace != nil {
c.printTrace(fmt.Sprintf("EMIT %s",
FormatInstructions(c.scopes[c.scopeIndex].instructions[pos:], pos)[0]))
}
return pos
}
func (c *Compiler) printTrace(a ...interface{}) {
const (
dots = ". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . "
n = len(dots)
)
i := 2 * c.indent
for i > n {
_, _ = fmt.Fprint(c.trace, dots)
i -= n
}
_, _ = fmt.Fprint(c.trace, dots[0:i])
_, _ = fmt.Fprintln(c.trace, a...)
}
func trace(c *Compiler, msg string) *Compiler {
c.printTrace(msg, "{")
c.indent++
return c
}
func un(c *Compiler) {
c.indent--
c.printTrace("}")
}

151
compiler/compiler_assign.go Normal file
View file

@ -0,0 +1,151 @@
package compiler
import (
"errors"
"fmt"
"github.com/d5/tengo/ast"
"github.com/d5/tengo/token"
)
func (c *Compiler) compileAssign(lhs, rhs []ast.Expr, op token.Token) error {
numLhs, numRhs := len(lhs), len(rhs)
if numLhs < numRhs {
// # of LHS must be >= # of RHS
return fmt.Errorf("assigntment count error: %d < %d", numLhs, numRhs)
}
if numLhs > 1 {
// TODO: until we fully implement the tuple assignment
return fmt.Errorf("tuple assignment not implemented")
}
//if numLhs > 1 && op != token.Assign && op != token.Define {
// return fmt.Errorf("invalid operator for tuple assignment: %s", op.String())
//}
// resolve and compile left-hand side
ident, selectors, err := resolveAssignLhs(lhs[0])
if err != nil {
return err
}
numSel := len(selectors)
if op == token.Define && numSel > 0 {
// using selector on new variable does not make sense
return errors.New("cannot use selector with ':='")
}
symbol, depth, exists := c.symbolTable.Resolve(ident)
if op == token.Define {
if depth == 0 && exists {
return fmt.Errorf("'%s' redeclared in this block", ident)
}
symbol = c.symbolTable.Define(ident)
} else if op == token.Assign {
if !exists {
symbol = c.symbolTable.Define(ident)
}
} else {
if !exists {
return fmt.Errorf("unresolved reference '%s'", ident)
}
}
// +=, -=, *=, /=
if op != token.Assign && op != token.Define {
if err := c.Compile(lhs[0]); err != nil {
return err
}
}
// compile RHSs
for _, expr := range rhs {
if err := c.Compile(expr); err != nil {
return err
}
}
switch op {
case token.AddAssign:
c.emit(OpAdd)
case token.SubAssign:
c.emit(OpSub)
case token.MulAssign:
c.emit(OpMul)
case token.QuoAssign:
c.emit(OpDiv)
case token.RemAssign:
c.emit(OpRem)
case token.AndAssign:
c.emit(OpBAnd)
case token.OrAssign:
c.emit(OpBOr)
case token.AndNotAssign:
c.emit(OpBAndNot)
case token.XorAssign:
c.emit(OpBXor)
case token.ShlAssign:
c.emit(OpBShiftLeft)
case token.ShrAssign:
c.emit(OpBShiftRight)
}
// compile selector expressions (right to left)
for i := numSel - 1; i >= 0; i-- {
if err := c.Compile(selectors[i]); err != nil {
return err
}
}
switch symbol.Scope {
case ScopeGlobal:
if numSel > 0 {
c.emit(OpSetSelGlobal, symbol.Index, numSel)
} else {
c.emit(OpSetGlobal, symbol.Index)
}
case ScopeLocal:
if numSel > 0 {
c.emit(OpSetSelLocal, symbol.Index, numSel)
} else {
c.emit(OpSetLocal, symbol.Index)
}
case ScopeFree:
if numSel > 0 {
c.emit(OpSetSelFree, symbol.Index, numSel)
} else {
c.emit(OpSetFree, symbol.Index)
}
default:
return fmt.Errorf("invalid assignment variable scope: %s", symbol.Scope)
}
return nil
}
func resolveAssignLhs(expr ast.Expr) (name string, selectors []ast.Expr, err error) {
switch term := expr.(type) {
case *ast.SelectorExpr:
name, selectors, err = resolveAssignLhs(term.Expr)
if err != nil {
return
}
selectors = append(selectors, term.Sel)
return
case *ast.IndexExpr:
name, selectors, err = resolveAssignLhs(term.Expr)
if err != nil {
return
}
selectors = append(selectors, term.Index)
case *ast.Ident:
return term.Name, nil, nil
}
return
}

176
compiler/compiler_for.go Normal file
View file

@ -0,0 +1,176 @@
package compiler
import (
"github.com/d5/tengo/ast"
)
func (c *Compiler) compileForStmt(stmt *ast.ForStmt) error {
c.symbolTable = c.symbolTable.Fork(true)
defer func() {
c.symbolTable = c.symbolTable.Parent(false)
}()
// init statement
if stmt.Init != nil {
if err := c.Compile(stmt.Init); err != nil {
return err
}
}
// pre-condition position
preCondPos := len(c.currentInstructions())
// condition expression
if err := c.Compile(stmt.Cond); err != nil {
return err
}
// condition jump position
postCondPos := c.emit(OpJumpFalsy, 0)
// enter loop
loop := c.enterLoop()
// body statement
if err := c.Compile(stmt.Body); err != nil {
c.leaveLoop()
return err
}
c.leaveLoop()
// post-body position
postBodyPos := len(c.currentInstructions())
// post statement
if stmt.Post != nil {
if err := c.Compile(stmt.Post); err != nil {
return err
}
}
// back to condition
c.emit(OpJump, preCondPos)
// post-statement position
postStmtPos := len(c.currentInstructions())
c.changeOperand(postCondPos, postStmtPos)
// update all break/continue jump positions
for _, pos := range loop.Breaks {
c.changeOperand(pos, postStmtPos)
}
for _, pos := range loop.Continues {
c.changeOperand(pos, postBodyPos)
}
return nil
}
func (c *Compiler) compileForInStmt(stmt *ast.ForInStmt) error {
c.symbolTable = c.symbolTable.Fork(true)
defer func() {
c.symbolTable = c.symbolTable.Parent(false)
}()
// for-in statement is compiled like following:
//
// for :it = iterator(iterable); :it.next(); {
// k, v := :it.get() // DEFINE operator
//
// ... body ...
// }
//
// ":it" is a local variable but will be conflict with other user variables
// because character ":" is not allowed.
// init
// :it = iterator(iterable)
itSymbol := c.symbolTable.Define(":it")
if err := c.Compile(stmt.Iterable); err != nil {
return err
}
c.emit(OpIteratorInit)
if itSymbol.Scope == ScopeGlobal {
c.emit(OpSetGlobal, itSymbol.Index)
} else {
c.emit(OpSetLocal, itSymbol.Index)
}
// pre-condition position
preCondPos := len(c.currentInstructions())
// condition
// :it.HasMore()
if itSymbol.Scope == ScopeGlobal {
c.emit(OpGetGlobal, itSymbol.Index)
} else {
c.emit(OpGetLocal, itSymbol.Index)
}
c.emit(OpIteratorNext)
// condition jump position
postCondPos := c.emit(OpJumpFalsy, 0)
// enter loop
loop := c.enterLoop()
// assign key variable
if stmt.Key.Name != "_" {
keySymbol := c.symbolTable.Define(stmt.Key.Name)
if itSymbol.Scope == ScopeGlobal {
c.emit(OpGetGlobal, itSymbol.Index)
} else {
c.emit(OpGetLocal, itSymbol.Index)
}
c.emit(OpIteratorKey)
if keySymbol.Scope == ScopeGlobal {
c.emit(OpSetGlobal, keySymbol.Index)
} else {
c.emit(OpSetLocal, keySymbol.Index)
}
}
// assign value variable
if stmt.Value.Name != "_" {
valueSymbol := c.symbolTable.Define(stmt.Value.Name)
if itSymbol.Scope == ScopeGlobal {
c.emit(OpGetGlobal, itSymbol.Index)
} else {
c.emit(OpGetLocal, itSymbol.Index)
}
c.emit(OpIteratorValue)
if valueSymbol.Scope == ScopeGlobal {
c.emit(OpSetGlobal, valueSymbol.Index)
} else {
c.emit(OpSetLocal, valueSymbol.Index)
}
}
// body statement
if err := c.Compile(stmt.Body); err != nil {
c.leaveLoop()
return err
}
c.leaveLoop()
// post-body position
postBodyPos := len(c.currentInstructions())
// back to condition
c.emit(OpJump, preCondPos)
// post-statement position
postStmtPos := len(c.currentInstructions())
c.changeOperand(postCondPos, postStmtPos)
// update all break/continue jump positions
for _, pos := range loop.Breaks {
c.changeOperand(pos, postStmtPos)
}
for _, pos := range loop.Continues {
c.changeOperand(pos, postBodyPos)
}
return nil
}

View file

@ -0,0 +1,30 @@
package compiler
import (
"github.com/d5/tengo/ast"
"github.com/d5/tengo/token"
)
func (c *Compiler) compileLogical(node *ast.BinaryExpr) error {
// left side term
if err := c.Compile(node.Lhs); err != nil {
return err
}
// jump position
var jumpPos int
if node.Token == token.LAnd {
jumpPos = c.emit(OpAndJump, 0)
} else {
jumpPos = c.emit(OpOrJump, 0)
}
// right side term
if err := c.Compile(node.Rhs); err != nil {
return err
}
c.changeOperand(jumpPos, len(c.currentInstructions()))
return nil
}

View file

@ -0,0 +1,31 @@
package compiler
func (c *Compiler) enterLoop() *Loop {
loop := &Loop{}
c.loops = append(c.loops, loop)
c.loopIndex++
if c.trace != nil {
c.printTrace("LOOPE", c.loopIndex)
}
return loop
}
func (c *Compiler) leaveLoop() {
if c.trace != nil {
c.printTrace("LOOPL", c.loopIndex)
}
c.loops = c.loops[:len(c.loops)-1]
c.loopIndex--
}
func (c *Compiler) currentLoop() *Loop {
if c.loopIndex >= 0 {
return c.loops[c.loopIndex]
}
return nil
}

View file

@ -0,0 +1,35 @@
package compiler
func (c *Compiler) currentInstructions() []byte {
return c.scopes[c.scopeIndex].instructions
}
func (c *Compiler) enterScope() {
scope := CompilationScope{
instructions: make([]byte, 0),
}
c.scopes = append(c.scopes, scope)
c.scopeIndex++
c.symbolTable = c.symbolTable.Fork(false)
if c.trace != nil {
c.printTrace("SCOPE", c.scopeIndex)
}
}
func (c *Compiler) leaveScope() []byte {
instructions := c.currentInstructions()
c.scopes = c.scopes[:len(c.scopes)-1]
c.scopeIndex--
c.symbolTable = c.symbolTable.Parent(true)
if c.trace != nil {
c.printTrace("SCOPL", c.scopeIndex)
}
return instructions
}

945
compiler/compiler_test.go Normal file
View file

@ -0,0 +1,945 @@
package compiler_test
import (
"fmt"
"reflect"
"strings"
"testing"
"github.com/d5/tengo/assert"
"github.com/d5/tengo/compiler"
"github.com/d5/tengo/objects"
"github.com/d5/tengo/parser"
"github.com/d5/tengo/scanner"
)
func TestCompiler_Compile(t *testing.T) {
expect(t, `1 + 2`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpConstant, 1),
compiler.MakeInstruction(compiler.OpAdd),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray(
intObject(1),
intObject(2))))
expect(t, `1; 2`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpPop),
compiler.MakeInstruction(compiler.OpConstant, 1),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray(
intObject(1),
intObject(2))))
expect(t, `1 - 2`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpConstant, 1),
compiler.MakeInstruction(compiler.OpSub),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray(
intObject(1),
intObject(2))))
expect(t, `1 * 2`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpConstant, 1),
compiler.MakeInstruction(compiler.OpMul),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray(
intObject(1),
intObject(2))))
expect(t, `2 / 1`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpConstant, 1),
compiler.MakeInstruction(compiler.OpDiv),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray(
intObject(2),
intObject(1))))
expect(t, `true`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpTrue),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray()))
expect(t, `false`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpFalse),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray()))
expect(t, `1 > 2`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpConstant, 1),
compiler.MakeInstruction(compiler.OpGreaterThan),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray(
intObject(1),
intObject(2))))
expect(t, `1 < 2`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpConstant, 1),
compiler.MakeInstruction(compiler.OpGreaterThan),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray(
intObject(2),
intObject(1))))
expect(t, `1 == 2`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpConstant, 1),
compiler.MakeInstruction(compiler.OpEqual),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray(
intObject(1),
intObject(2))))
expect(t, `1 != 2`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpConstant, 1),
compiler.MakeInstruction(compiler.OpNotEqual),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray(
intObject(1),
intObject(2))))
expect(t, `true == false`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpTrue),
compiler.MakeInstruction(compiler.OpFalse),
compiler.MakeInstruction(compiler.OpEqual),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray()))
expect(t, `true != false`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpTrue),
compiler.MakeInstruction(compiler.OpFalse),
compiler.MakeInstruction(compiler.OpNotEqual),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray()))
expect(t, `-1`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpMinus),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray(
intObject(1))))
expect(t, `!true`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpTrue),
compiler.MakeInstruction(compiler.OpLNot),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray()))
expect(t, `if true { 10 }; 3333`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpTrue), // 0000
compiler.MakeInstruction(compiler.OpJumpFalsy, 11), // 0001
compiler.MakeInstruction(compiler.OpConstant, 0), // 0004
compiler.MakeInstruction(compiler.OpPop), // 0007
compiler.MakeInstruction(compiler.OpJump, 11), // 0008
compiler.MakeInstruction(compiler.OpConstant, 1), // 0011
compiler.MakeInstruction(compiler.OpPop)), // 0014
objectsArray(
intObject(10),
intObject(3333))))
expect(t, `if (true) { 10 } else { 20 }; 3333;`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpTrue), // 0000
compiler.MakeInstruction(compiler.OpJumpFalsy, 11), // 0001
compiler.MakeInstruction(compiler.OpConstant, 0), // 0004
compiler.MakeInstruction(compiler.OpPop), // 0007
compiler.MakeInstruction(compiler.OpJump, 15), // 0008
compiler.MakeInstruction(compiler.OpConstant, 1), // 0011
compiler.MakeInstruction(compiler.OpPop), // 0014
compiler.MakeInstruction(compiler.OpConstant, 2), // 0015
compiler.MakeInstruction(compiler.OpPop)), // 0018
objectsArray(
intObject(10),
intObject(20),
intObject(3333))))
expect(t, `"kami"`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray(
stringObject("kami"))))
expect(t, `"ka" + "mi"`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpConstant, 1),
compiler.MakeInstruction(compiler.OpAdd),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray(
stringObject("ka"),
stringObject("mi"))))
expect(t, `a := 1; b := 2; a += b`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpSetGlobal, 0),
compiler.MakeInstruction(compiler.OpConstant, 1),
compiler.MakeInstruction(compiler.OpSetGlobal, 1),
compiler.MakeInstruction(compiler.OpGetGlobal, 0),
compiler.MakeInstruction(compiler.OpGetGlobal, 1),
compiler.MakeInstruction(compiler.OpAdd),
compiler.MakeInstruction(compiler.OpSetGlobal, 0)),
objectsArray(
intObject(1),
intObject(2))))
expect(t, `a := 1; b := 2; a /= b`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpSetGlobal, 0),
compiler.MakeInstruction(compiler.OpConstant, 1),
compiler.MakeInstruction(compiler.OpSetGlobal, 1),
compiler.MakeInstruction(compiler.OpGetGlobal, 0),
compiler.MakeInstruction(compiler.OpGetGlobal, 1),
compiler.MakeInstruction(compiler.OpDiv),
compiler.MakeInstruction(compiler.OpSetGlobal, 0)),
objectsArray(
intObject(1),
intObject(2))))
expect(t, `[]`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpArray, 0),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray()))
expect(t, `[1, 2, 3]`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpConstant, 1),
compiler.MakeInstruction(compiler.OpConstant, 2),
compiler.MakeInstruction(compiler.OpArray, 3),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray(
intObject(1),
intObject(2),
intObject(3))))
expect(t, `[1 + 2, 3 - 4, 5 * 6]`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpConstant, 1),
compiler.MakeInstruction(compiler.OpAdd),
compiler.MakeInstruction(compiler.OpConstant, 2),
compiler.MakeInstruction(compiler.OpConstant, 3),
compiler.MakeInstruction(compiler.OpSub),
compiler.MakeInstruction(compiler.OpConstant, 4),
compiler.MakeInstruction(compiler.OpConstant, 5),
compiler.MakeInstruction(compiler.OpMul),
compiler.MakeInstruction(compiler.OpArray, 3),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray(
intObject(1),
intObject(2),
intObject(3),
intObject(4),
intObject(5),
intObject(6))))
expect(t, `{}`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpMap, 0),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray()))
expect(t, `{a: 2, b: 4, c: 6}`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpConstant, 1),
compiler.MakeInstruction(compiler.OpConstant, 2),
compiler.MakeInstruction(compiler.OpConstant, 3),
compiler.MakeInstruction(compiler.OpConstant, 4),
compiler.MakeInstruction(compiler.OpConstant, 5),
compiler.MakeInstruction(compiler.OpMap, 6),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray(
stringObject("a"),
intObject(2),
stringObject("b"),
intObject(4),
stringObject("c"),
intObject(6))))
expect(t, `{a: 2 + 3, b: 5 * 6}`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpConstant, 1),
compiler.MakeInstruction(compiler.OpConstant, 2),
compiler.MakeInstruction(compiler.OpAdd),
compiler.MakeInstruction(compiler.OpConstant, 3),
compiler.MakeInstruction(compiler.OpConstant, 4),
compiler.MakeInstruction(compiler.OpConstant, 5),
compiler.MakeInstruction(compiler.OpMul),
compiler.MakeInstruction(compiler.OpMap, 4),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray(
stringObject("a"),
intObject(2),
intObject(3),
stringObject("b"),
intObject(5),
intObject(6))))
expect(t, `[1, 2, 3][1 + 1]`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpConstant, 1),
compiler.MakeInstruction(compiler.OpConstant, 2),
compiler.MakeInstruction(compiler.OpArray, 3),
compiler.MakeInstruction(compiler.OpConstant, 3),
compiler.MakeInstruction(compiler.OpConstant, 4),
compiler.MakeInstruction(compiler.OpAdd),
compiler.MakeInstruction(compiler.OpIndex),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray(
intObject(1),
intObject(2),
intObject(3),
intObject(1),
intObject(1))))
expect(t, `{a: 2}[2 - 1]`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpConstant, 1),
compiler.MakeInstruction(compiler.OpMap, 2),
compiler.MakeInstruction(compiler.OpConstant, 2),
compiler.MakeInstruction(compiler.OpConstant, 3),
compiler.MakeInstruction(compiler.OpSub),
compiler.MakeInstruction(compiler.OpIndex),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray(
stringObject("a"),
intObject(2),
intObject(2),
intObject(1))))
expect(t, `[1, 2, 3][:]`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpConstant, 1),
compiler.MakeInstruction(compiler.OpConstant, 2),
compiler.MakeInstruction(compiler.OpArray, 3),
compiler.MakeInstruction(compiler.OpNull),
compiler.MakeInstruction(compiler.OpNull),
compiler.MakeInstruction(compiler.OpSliceIndex),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray(
intObject(1),
intObject(2),
intObject(3))))
expect(t, `[1, 2, 3][0 : 2]`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpConstant, 1),
compiler.MakeInstruction(compiler.OpConstant, 2),
compiler.MakeInstruction(compiler.OpArray, 3),
compiler.MakeInstruction(compiler.OpConstant, 3),
compiler.MakeInstruction(compiler.OpConstant, 4),
compiler.MakeInstruction(compiler.OpSliceIndex),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray(
intObject(1),
intObject(2),
intObject(3),
intObject(0),
intObject(2))))
expect(t, `[1, 2, 3][:2]`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpConstant, 1),
compiler.MakeInstruction(compiler.OpConstant, 2),
compiler.MakeInstruction(compiler.OpArray, 3),
compiler.MakeInstruction(compiler.OpNull),
compiler.MakeInstruction(compiler.OpConstant, 3),
compiler.MakeInstruction(compiler.OpSliceIndex),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray(
intObject(1),
intObject(2),
intObject(3),
intObject(2))))
expect(t, `[1, 2, 3][0:]`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpConstant, 1),
compiler.MakeInstruction(compiler.OpConstant, 2),
compiler.MakeInstruction(compiler.OpArray, 3),
compiler.MakeInstruction(compiler.OpConstant, 3),
compiler.MakeInstruction(compiler.OpNull),
compiler.MakeInstruction(compiler.OpSliceIndex),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray(
intObject(1),
intObject(2),
intObject(3),
intObject(0))))
expect(t, `func() { return 5 + 10 }`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 2),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray(
intObject(5),
intObject(10),
compiledFunction(0, 0,
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpConstant, 1),
compiler.MakeInstruction(compiler.OpAdd),
compiler.MakeInstruction(compiler.OpReturnValue, 1)))))
expect(t, `func() { 5 + 10 }`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 2),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray(
intObject(5),
intObject(10),
compiledFunction(0, 0,
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpConstant, 1),
compiler.MakeInstruction(compiler.OpAdd),
compiler.MakeInstruction(compiler.OpPop),
compiler.MakeInstruction(compiler.OpReturn)))))
expect(t, `func() { 1; 2 }`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 2),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray(
intObject(1),
intObject(2),
compiledFunction(0, 0,
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpPop),
compiler.MakeInstruction(compiler.OpConstant, 1),
compiler.MakeInstruction(compiler.OpPop),
compiler.MakeInstruction(compiler.OpReturn)))))
expect(t, `func() { 1; return 2 }`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 2),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray(
intObject(1),
intObject(2),
compiledFunction(0, 0,
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpPop),
compiler.MakeInstruction(compiler.OpConstant, 1),
compiler.MakeInstruction(compiler.OpReturnValue, 1)))))
expect(t, `func() { if(true) { return 1 } else { return 2 } }`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 2),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray(
intObject(1),
intObject(2),
compiledFunction(0, 0,
compiler.MakeInstruction(compiler.OpTrue), // 0000
compiler.MakeInstruction(compiler.OpJumpFalsy, 12), // 0001
compiler.MakeInstruction(compiler.OpConstant, 0), // 0004
compiler.MakeInstruction(compiler.OpReturnValue, 1), // 0007
compiler.MakeInstruction(compiler.OpJump, 17), // 0009
compiler.MakeInstruction(compiler.OpConstant, 1), // 0012
compiler.MakeInstruction(compiler.OpReturnValue, 1))))) // 0014
expect(t, `func() { 1; if(true) { 2 } else { 3 }; 4 }`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 4),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray(
intObject(1),
intObject(2),
intObject(3),
intObject(4),
compiledFunction(0, 0,
compiler.MakeInstruction(compiler.OpConstant, 0), // 0000
compiler.MakeInstruction(compiler.OpPop), // 0003
compiler.MakeInstruction(compiler.OpTrue), // 0004
compiler.MakeInstruction(compiler.OpJumpFalsy, 15), // 0005
compiler.MakeInstruction(compiler.OpConstant, 1), // 0008
compiler.MakeInstruction(compiler.OpPop), // 0011
compiler.MakeInstruction(compiler.OpJump, 19), // 0012
compiler.MakeInstruction(compiler.OpConstant, 2), // 0015
compiler.MakeInstruction(compiler.OpPop), // 0018
compiler.MakeInstruction(compiler.OpConstant, 3), // 0019
compiler.MakeInstruction(compiler.OpPop), // 0022
compiler.MakeInstruction(compiler.OpReturn))))) // 0023
expect(t, `func() { }`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray(
compiledFunction(0, 0, compiler.MakeInstruction(compiler.OpReturn)))))
expect(t, `func() { 24 }()`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 1),
compiler.MakeInstruction(compiler.OpCall, 0),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray(
intObject(24),
compiledFunction(0, 0,
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpPop),
compiler.MakeInstruction(compiler.OpReturn)))))
expect(t, `func() { return 24 }()`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 1),
compiler.MakeInstruction(compiler.OpCall, 0),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray(
intObject(24),
compiledFunction(0, 0,
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpReturnValue, 1)))))
expect(t, `noArg := func() { 24 }; noArg();`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 1),
compiler.MakeInstruction(compiler.OpSetGlobal, 0),
compiler.MakeInstruction(compiler.OpGetGlobal, 0),
compiler.MakeInstruction(compiler.OpCall, 0),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray(
intObject(24),
compiledFunction(0, 0,
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpPop),
compiler.MakeInstruction(compiler.OpReturn)))))
expect(t, `noArg := func() { return 24 }; noArg();`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 1),
compiler.MakeInstruction(compiler.OpSetGlobal, 0),
compiler.MakeInstruction(compiler.OpGetGlobal, 0),
compiler.MakeInstruction(compiler.OpCall, 0),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray(
intObject(24),
compiledFunction(0, 0,
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpReturnValue, 1)))))
expect(t, `n := 55; func() { n };`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpSetGlobal, 0),
compiler.MakeInstruction(compiler.OpConstant, 1),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray(
intObject(55),
compiledFunction(0, 0,
compiler.MakeInstruction(compiler.OpGetGlobal, 0),
compiler.MakeInstruction(compiler.OpPop),
compiler.MakeInstruction(compiler.OpReturn)))))
expect(t, `func() { n := 55; return n }`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 1),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray(
intObject(55),
compiledFunction(1, 0,
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpSetLocal, 0),
compiler.MakeInstruction(compiler.OpGetLocal, 0),
compiler.MakeInstruction(compiler.OpReturnValue, 1)))))
expect(t, `func() { n := 55; n = 23; return n }`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 2),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray(
intObject(55),
intObject(23),
compiledFunction(1, 0,
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpSetLocal, 0),
compiler.MakeInstruction(compiler.OpConstant, 1),
compiler.MakeInstruction(compiler.OpSetLocal, 0),
compiler.MakeInstruction(compiler.OpGetLocal, 0),
compiler.MakeInstruction(compiler.OpReturnValue, 1)))))
expect(t, `func() { a := 55; b := 77; return a + b }`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 2),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray(
intObject(55),
intObject(77),
compiledFunction(2, 0,
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpSetLocal, 0),
compiler.MakeInstruction(compiler.OpConstant, 1),
compiler.MakeInstruction(compiler.OpSetLocal, 1),
compiler.MakeInstruction(compiler.OpGetLocal, 0),
compiler.MakeInstruction(compiler.OpGetLocal, 1),
compiler.MakeInstruction(compiler.OpAdd),
compiler.MakeInstruction(compiler.OpReturnValue, 1)))))
expect(t, `f1 := func(a) { return a }; f1(24);`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpSetGlobal, 0),
compiler.MakeInstruction(compiler.OpGetGlobal, 0),
compiler.MakeInstruction(compiler.OpConstant, 1),
compiler.MakeInstruction(compiler.OpCall, 1),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray(
compiledFunction(1, 1,
compiler.MakeInstruction(compiler.OpGetLocal, 0),
compiler.MakeInstruction(compiler.OpReturnValue, 1)),
intObject(24))))
expect(t, `f1 := func(a, b, c) { a; b; return c; }; f1(24, 25, 26);`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpSetGlobal, 0),
compiler.MakeInstruction(compiler.OpGetGlobal, 0),
compiler.MakeInstruction(compiler.OpConstant, 1),
compiler.MakeInstruction(compiler.OpConstant, 2),
compiler.MakeInstruction(compiler.OpConstant, 3),
compiler.MakeInstruction(compiler.OpCall, 3),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray(
compiledFunction(3, 3,
compiler.MakeInstruction(compiler.OpGetLocal, 0),
compiler.MakeInstruction(compiler.OpPop),
compiler.MakeInstruction(compiler.OpGetLocal, 1),
compiler.MakeInstruction(compiler.OpPop),
compiler.MakeInstruction(compiler.OpGetLocal, 2),
compiler.MakeInstruction(compiler.OpReturnValue, 1)),
intObject(24),
intObject(25),
intObject(26))))
expect(t, `len([]);`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpGetBuiltin, 1),
compiler.MakeInstruction(compiler.OpArray, 0),
compiler.MakeInstruction(compiler.OpCall, 1),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray()))
expect(t, `func() { return len([]) }`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray(
compiledFunction(0, 0,
compiler.MakeInstruction(compiler.OpGetBuiltin, 1),
compiler.MakeInstruction(compiler.OpArray, 0),
compiler.MakeInstruction(compiler.OpCall, 1),
compiler.MakeInstruction(compiler.OpReturnValue, 1)))))
expect(t, `func(a) { func(b) { return a + b } }`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 1),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray(
compiledFunction(1, 1,
compiler.MakeInstruction(compiler.OpGetFree, 0),
compiler.MakeInstruction(compiler.OpGetLocal, 0),
compiler.MakeInstruction(compiler.OpAdd),
compiler.MakeInstruction(compiler.OpReturnValue, 1)),
compiledFunction(1, 1,
compiler.MakeInstruction(compiler.OpGetLocal, 0),
compiler.MakeInstruction(compiler.OpClosure, 0, 1),
compiler.MakeInstruction(compiler.OpPop),
compiler.MakeInstruction(compiler.OpReturn)))))
expect(t, `
func(a) {
return func(b) {
return func(c) {
return a + b + c
}
}
}`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 2),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray(
compiledFunction(1, 1,
compiler.MakeInstruction(compiler.OpGetFree, 0),
compiler.MakeInstruction(compiler.OpGetFree, 1),
compiler.MakeInstruction(compiler.OpAdd),
compiler.MakeInstruction(compiler.OpGetLocal, 0),
compiler.MakeInstruction(compiler.OpAdd),
compiler.MakeInstruction(compiler.OpReturnValue, 1)),
compiledFunction(1, 1,
compiler.MakeInstruction(compiler.OpGetFree, 0),
compiler.MakeInstruction(compiler.OpGetLocal, 0),
compiler.MakeInstruction(compiler.OpClosure, 0, 2),
compiler.MakeInstruction(compiler.OpReturnValue, 1)),
compiledFunction(1, 1,
compiler.MakeInstruction(compiler.OpGetLocal, 0),
compiler.MakeInstruction(compiler.OpClosure, 1, 1),
compiler.MakeInstruction(compiler.OpReturnValue, 1)))))
expect(t, `
g := 55;
func() {
a := 66;
return func() {
b := 77;
return func() {
c := 88;
return g + a + b + c;
}
}
}`,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpSetGlobal, 0),
compiler.MakeInstruction(compiler.OpConstant, 6),
compiler.MakeInstruction(compiler.OpPop)),
objectsArray(
intObject(55),
intObject(66),
intObject(77),
intObject(88),
compiledFunction(1, 0,
compiler.MakeInstruction(compiler.OpConstant, 3),
compiler.MakeInstruction(compiler.OpSetLocal, 0),
compiler.MakeInstruction(compiler.OpGetGlobal, 0),
compiler.MakeInstruction(compiler.OpGetFree, 0),
compiler.MakeInstruction(compiler.OpAdd),
compiler.MakeInstruction(compiler.OpGetFree, 1),
compiler.MakeInstruction(compiler.OpAdd),
compiler.MakeInstruction(compiler.OpGetLocal, 0),
compiler.MakeInstruction(compiler.OpAdd),
compiler.MakeInstruction(compiler.OpReturnValue, 1)),
compiledFunction(1, 0,
compiler.MakeInstruction(compiler.OpConstant, 2),
compiler.MakeInstruction(compiler.OpSetLocal, 0),
compiler.MakeInstruction(compiler.OpGetFree, 0),
compiler.MakeInstruction(compiler.OpGetLocal, 0),
compiler.MakeInstruction(compiler.OpClosure, 4, 2),
compiler.MakeInstruction(compiler.OpReturnValue, 1)),
compiledFunction(1, 0,
compiler.MakeInstruction(compiler.OpConstant, 1),
compiler.MakeInstruction(compiler.OpSetLocal, 0),
compiler.MakeInstruction(compiler.OpGetLocal, 0),
compiler.MakeInstruction(compiler.OpClosure, 5, 1),
compiler.MakeInstruction(compiler.OpReturnValue, 1)))))
}
func concat(insts ...[]byte) []byte {
concat := make([]byte, 0)
for _, i := range insts {
concat = append(concat, i...)
}
return concat
}
func bytecode(instructions []byte, constants []objects.Object) *compiler.Bytecode {
return &compiler.Bytecode{
Instructions: instructions,
Constants: constants,
}
}
func expect(t *testing.T, input string, expected *compiler.Bytecode) (ok bool) {
actual, trace, err := traceCompile(input, nil)
defer func() {
if !ok {
for _, tr := range trace {
t.Log(tr)
}
}
}()
if !assert.NoError(t, err) {
return
}
ok = equalBytecode(t, expected, actual)
return
}
func equalBytecode(t *testing.T, expected, actual *compiler.Bytecode) bool {
expectedInstructions := strings.Join(compiler.FormatInstructions(expected.Instructions, 0), "\n")
actualInstructions := strings.Join(compiler.FormatInstructions(actual.Instructions, 0), "\n")
return assert.Equal(t, expectedInstructions, actualInstructions) &&
equalConstants(t, expected.Constants, actual.Constants)
}
func equalConstants(t *testing.T, expected, actual []objects.Object) bool {
if !assert.Equal(t, len(expected), len(actual)) {
return false
}
for i := 0; i < len(expected); i++ {
if !assert.Equal(t, expected[i], actual[i]) {
return false
}
}
return true
}
type tracer struct {
Out []string
}
func (o *tracer) Write(p []byte) (n int, err error) {
o.Out = append(o.Out, string(p))
return len(p), nil
}
func traceCompile(input string, symbols map[string]objects.Object) (res *compiler.Bytecode, trace []string, err error) {
fileSet := scanner.NewFileSet()
file := fileSet.AddFile("test", -1, len(input))
p := parser.NewParser(file, []byte(input), nil)
symTable := compiler.NewSymbolTable()
for name := range symbols {
symTable.Define(name)
}
tr := &tracer{}
c := compiler.NewCompiler(symTable, tr)
parsed, err := p.ParseFile()
if err != nil {
return
}
err = c.Compile(parsed)
{
trace = append(trace, fmt.Sprintf("Compiler Trace:\n%s", strings.Join(tr.Out, "")))
bytecode := c.Bytecode()
var constStr []string
for cidx, cn := range bytecode.Constants {
if cmFn, ok := cn.(*objects.CompiledFunction); ok {
constStr = append(constStr, fmt.Sprintf("[% 3d] (Compiled Function|%p)", cidx, cn))
for _, l := range compiler.FormatInstructions(cmFn.Instructions, 0) {
constStr = append(constStr, fmt.Sprintf(" %s", l))
}
} else {
constStr = append(constStr, fmt.Sprintf("[% 3d] %s (%s|%p)", cidx, cn, reflect.TypeOf(cn).Name(), cn))
}
}
trace = append(trace, fmt.Sprintf("Compiled Constants:\n%s", strings.Join(constStr, "\n")))
trace = append(trace, fmt.Sprintf("Compiled Instructions:\n%s\n", strings.Join(compiler.FormatInstructions(bytecode.Instructions, 0), "\n")))
}
if err != nil {
return
}
res = c.Bytecode()
return
}
func objectsArray(o ...objects.Object) []objects.Object {
return o
}
func intObject(v int64) *objects.Int {
return &objects.Int{Value: v}
}
func stringObject(v string) *objects.String {
return &objects.String{Value: v}
}
func compiledFunction(numLocals, numParams int, insts ...[]byte) *objects.CompiledFunction {
return &objects.CompiledFunction{Instructions: concat(insts...), NumLocals: numLocals, NumParameters: numParams}
}

64
compiler/definitions.go Normal file
View file

@ -0,0 +1,64 @@
package compiler
type Definition struct {
Name string
Operands []int
}
var definitions = map[Opcode]*Definition{
OpConstant: {Name: "CONST", Operands: []int{2}},
OpPop: {Name: "POP", Operands: []int{}},
OpTrue: {Name: "TRUE", Operands: []int{}},
OpFalse: {Name: "FALSE", Operands: []int{}},
OpAdd: {Name: "ADD", Operands: []int{}},
OpSub: {Name: "SUB", Operands: []int{}},
OpMul: {Name: "MUL", Operands: []int{}},
OpDiv: {Name: "DIV", Operands: []int{}},
OpRem: {Name: "REM", Operands: []int{}},
OpBAnd: {Name: "AND", Operands: []int{}},
OpBOr: {Name: "OR", Operands: []int{}},
OpBXor: {Name: "XOR", Operands: []int{}},
OpBAndNot: {Name: "ANDN", Operands: []int{}},
OpBShiftLeft: {Name: "SHL", Operands: []int{}},
OpBShiftRight: {Name: "SHR", Operands: []int{}},
OpBComplement: {Name: "NEG", Operands: []int{}},
OpEqual: {Name: "EQL", Operands: []int{}},
OpNotEqual: {Name: "NEQ", Operands: []int{}},
OpGreaterThan: {Name: "GTR", Operands: []int{}},
OpGreaterThanEqual: {Name: "GEQ", Operands: []int{}},
OpMinus: {Name: "NEG", Operands: []int{}},
OpLNot: {Name: "NOT", Operands: []int{}},
OpJumpFalsy: {Name: "JMPF", Operands: []int{2}},
OpAndJump: {Name: "ANDJMP", Operands: []int{2}},
OpOrJump: {Name: "ORJMP", Operands: []int{2}},
OpJump: {Name: "JMP", Operands: []int{2}},
OpNull: {Name: "NULL", Operands: []int{}},
OpGetGlobal: {Name: "GETG", Operands: []int{2}},
OpSetGlobal: {Name: "SETG", Operands: []int{2}},
OpSetSelGlobal: {Name: "SETSG", Operands: []int{2, 1}},
OpArray: {Name: "ARR", Operands: []int{2}},
OpMap: {Name: "MAP", Operands: []int{2}},
OpIndex: {Name: "INDEX", Operands: []int{}},
OpSliceIndex: {Name: "SLICE", Operands: []int{}},
OpCall: {Name: "CALL", Operands: []int{1}},
OpReturn: {Name: "RET", Operands: []int{}},
OpReturnValue: {Name: "RETVAL", Operands: []int{1}},
OpGetLocal: {Name: "GETL", Operands: []int{1}},
OpSetLocal: {Name: "SETL", Operands: []int{1}},
OpSetSelLocal: {Name: "SETSL", Operands: []int{1, 1}},
OpGetBuiltin: {Name: "BUILTIN", Operands: []int{1}},
OpClosure: {Name: "CLOSURE", Operands: []int{2, 1}},
OpGetFree: {Name: "GETF", Operands: []int{1}},
OpSetFree: {Name: "SETF", Operands: []int{1}},
OpSetSelFree: {Name: "SETSF", Operands: []int{1, 1}},
OpIteratorInit: {Name: "ITER", Operands: []int{}},
OpIteratorNext: {Name: "ITNXT", Operands: []int{}},
OpIteratorKey: {Name: "ITKEY", Operands: []int{}},
OpIteratorValue: {Name: "ITVAL", Operands: []int{}},
}
func Lookup(opcode Opcode) (def *Definition, ok bool) {
def, ok = definitions[opcode]
return
}

View file

@ -0,0 +1,6 @@
package compiler
type EmittedInstruction struct {
Opcode Opcode
Position int
}

64
compiler/instructions.go Normal file
View file

@ -0,0 +1,64 @@
package compiler
import (
"fmt"
)
func MakeInstruction(opcode Opcode, operands ...int) []byte {
def, ok := Lookup(opcode)
if !ok {
return nil
}
totalLen := 1
for _, w := range def.Operands {
totalLen += w
}
instruction := make([]byte, totalLen, totalLen)
instruction[0] = byte(opcode)
offset := 1
for i, o := range operands {
width := def.Operands[i]
switch width {
case 1:
instruction[offset] = byte(o)
case 2:
n := uint16(o)
instruction[offset] = byte(n >> 8)
instruction[offset+1] = byte(n)
}
offset += width
}
return instruction
}
func FormatInstructions(b []byte, posOffset int) []string {
var out []string
i := 0
for i < len(b) {
def, ok := Lookup(Opcode(b[i]))
if !ok {
out = append(out, fmt.Sprintf("error: unknown Opcode %d", b[i]))
continue
}
operands, read := ReadOperands(def, b[i+1:])
switch len(def.Operands) {
case 0:
out = append(out, fmt.Sprintf("%04d %-7s", posOffset+i, def.Name))
case 1:
out = append(out, fmt.Sprintf("%04d %-7s %-5d", posOffset+i, def.Name, operands[0]))
case 2:
out = append(out, fmt.Sprintf("%04d %-7s %-5d %-5d", posOffset+i, def.Name, operands[0], operands[1]))
}
i += 1 + read
}
return out
}

View file

@ -0,0 +1,69 @@
package compiler_test
import (
"strings"
"testing"
"github.com/d5/tengo/assert"
"github.com/d5/tengo/compiler"
)
func TestInstructions_String(t *testing.T) {
assertInstructionString(t,
[][]byte{
compiler.MakeInstruction(compiler.OpConstant, 1),
compiler.MakeInstruction(compiler.OpConstant, 2),
compiler.MakeInstruction(compiler.OpConstant, 65535),
},
`0000 CONST 1
0003 CONST 2
0006 CONST 65535`)
assertInstructionString(t,
[][]byte{
compiler.MakeInstruction(compiler.OpAdd),
compiler.MakeInstruction(compiler.OpConstant, 2),
compiler.MakeInstruction(compiler.OpConstant, 65535),
},
`0000 ADD
0001 CONST 2
0004 CONST 65535`)
assertInstructionString(t,
[][]byte{
compiler.MakeInstruction(compiler.OpAdd),
compiler.MakeInstruction(compiler.OpGetLocal, 1),
compiler.MakeInstruction(compiler.OpConstant, 2),
compiler.MakeInstruction(compiler.OpConstant, 65535),
},
`0000 ADD
0001 GETL 1
0003 CONST 2
0006 CONST 65535`)
}
func TestMakeInstruction(t *testing.T) {
makeInstruction(t, []byte{byte(compiler.OpConstant), 0, 0}, compiler.OpConstant, 0)
makeInstruction(t, []byte{byte(compiler.OpConstant), 0, 1}, compiler.OpConstant, 1)
makeInstruction(t, []byte{byte(compiler.OpConstant), 255, 254}, compiler.OpConstant, 65534)
makeInstruction(t, []byte{byte(compiler.OpPop)}, compiler.OpPop)
makeInstruction(t, []byte{byte(compiler.OpTrue)}, compiler.OpTrue)
makeInstruction(t, []byte{byte(compiler.OpFalse)}, compiler.OpFalse)
makeInstruction(t, []byte{byte(compiler.OpAdd)}, compiler.OpAdd)
makeInstruction(t, []byte{byte(compiler.OpSub)}, compiler.OpSub)
makeInstruction(t, []byte{byte(compiler.OpMul)}, compiler.OpMul)
makeInstruction(t, []byte{byte(compiler.OpDiv)}, compiler.OpDiv)
}
func assertInstructionString(t *testing.T, instructions [][]byte, expected string) {
concatted := make([]byte, 0)
for _, e := range instructions {
concatted = append(concatted, e...)
}
assert.Equal(t, expected, strings.Join(compiler.FormatInstructions(concatted, 0), "\n"))
}
func makeInstruction(t *testing.T, expected []byte, opcode compiler.Opcode, operands ...int) {
inst := compiler.MakeInstruction(opcode, operands...)
assert.Equal(t, expected, []byte(inst))
}

6
compiler/loop.go Normal file
View file

@ -0,0 +1,6 @@
package compiler
type Loop struct {
Continues []int
Breaks []int
}

55
compiler/opcodes.go Normal file
View file

@ -0,0 +1,55 @@
package compiler
type Opcode byte
const (
OpConstant Opcode = iota
OpAdd
OpSub
OpMul
OpDiv
OpRem
OpBAnd
OpBOr
OpBXor
OpBShiftLeft
OpBShiftRight
OpBAndNot
OpBComplement
OpPop
OpTrue
OpFalse
OpEqual
OpNotEqual
OpGreaterThan
OpGreaterThanEqual
OpMinus
OpLNot
OpJumpFalsy
OpAndJump
OpOrJump
OpJump
OpNull
OpGetGlobal
OpSetGlobal
OpSetSelGlobal
OpArray
OpMap
OpIndex
OpSliceIndex
OpCall
OpReturn
OpReturnValue
OpGetLocal
OpSetLocal
OpSetSelLocal
OpGetBuiltin
OpClosure
OpGetFree
OpSetFree
OpSetSelFree
OpIteratorInit
OpIteratorNext
OpIteratorKey
OpIteratorValue
)

24
compiler/operands.go Normal file
View file

@ -0,0 +1,24 @@
package compiler
func ReadOperands(def *Definition, ins []byte) (operands []int, offset int) {
for _, width := range def.Operands {
switch width {
case 1:
operands = append(operands, int(ReadUint8(ins[offset:])))
case 2:
operands = append(operands, int(ReadUint16(ins[offset:])))
}
offset += width
}
return
}
func ReadUint16(b []byte) uint16 {
return uint16(b[1]) | uint16(b[0])<<8
}
func ReadUint8(b []byte) uint8 {
return uint8(b[0])
}

21
compiler/operands_test.go Normal file
View file

@ -0,0 +1,21 @@
package compiler_test
import (
"testing"
"github.com/d5/tengo/assert"
"github.com/d5/tengo/compiler"
)
func TestReadOperands(t *testing.T) {
assertReadOperand(t, compiler.OpConstant, []int{65535}, 2)
}
func assertReadOperand(t *testing.T, opcode compiler.Opcode, operands []int, expectedBytes int) {
inst := compiler.MakeInstruction(opcode, operands...)
def, ok := compiler.Lookup(opcode)
assert.True(t, ok)
operandsRead, read := compiler.ReadOperands(def, inst[1:])
assert.Equal(t, expectedBytes, read)
assert.Equal(t, operands, operandsRead)
}

7
compiler/symbol.go Normal file
View file

@ -0,0 +1,7 @@
package compiler
type Symbol struct {
Name string
Scope SymbolScope
Index int
}

10
compiler/symbol_scopes.go Normal file
View file

@ -0,0 +1,10 @@
package compiler
type SymbolScope string
const (
ScopeGlobal SymbolScope = "GLOBAL"
ScopeLocal = "LOCAL"
ScopeBuiltin = "BUILTIN"
ScopeFree = "FREE"
)

135
compiler/symbol_table.go Normal file
View file

@ -0,0 +1,135 @@
package compiler
type SymbolTable struct {
parent *SymbolTable
block bool
store map[string]Symbol
numDefinition int
maxDefinition int
freeSymbols []Symbol
}
func NewSymbolTable() *SymbolTable {
return &SymbolTable{
store: make(map[string]Symbol),
}
}
func (t *SymbolTable) Define(name string) Symbol {
symbol := Symbol{Name: name, Index: t.nextIndex()}
t.numDefinition++
if t.Parent(true) == nil {
symbol.Scope = ScopeGlobal
} else {
symbol.Scope = ScopeLocal
}
t.store[name] = symbol
t.updateMaxDefs(symbol.Index + 1)
return symbol
}
func (t *SymbolTable) DefineBuiltin(index int, name string) Symbol {
symbol := Symbol{
Name: name,
Index: index,
Scope: ScopeBuiltin,
}
t.store[name] = symbol
return symbol
}
func (t *SymbolTable) Resolve(name string) (symbol Symbol, depth int, ok bool) {
symbol, ok = t.store[name]
if !ok && t.parent != nil {
symbol, depth, ok = t.parent.Resolve(name)
if !ok {
return
}
if !t.block {
depth += 1
}
// if symbol is defined in parent table and if it's not global/builtin
// then it's free variable.
if depth > 0 && symbol.Scope != ScopeGlobal && symbol.Scope != ScopeBuiltin {
return t.defineFree(symbol), depth, true
}
return
}
return
}
func (t *SymbolTable) Fork(block bool) *SymbolTable {
return &SymbolTable{
store: make(map[string]Symbol),
parent: t,
block: block,
}
}
func (t *SymbolTable) Parent(skipBlock bool) *SymbolTable {
if skipBlock && t.block {
return t.parent.Parent(skipBlock)
}
return t.parent
}
func (t *SymbolTable) MaxSymbols() int {
return t.maxDefinition
}
func (t *SymbolTable) FreeSymbols() []Symbol {
return t.freeSymbols
}
func (t *SymbolTable) Names() []string {
var names []string
for name := range t.store {
names = append(names, name)
}
return names
}
func (t *SymbolTable) nextIndex() int {
if t.block {
return t.parent.nextIndex() + t.numDefinition
}
return t.numDefinition
}
func (t *SymbolTable) updateMaxDefs(numDefs int) {
if numDefs > t.maxDefinition {
t.maxDefinition = numDefs
}
if t.block {
t.parent.updateMaxDefs(numDefs)
}
}
func (t *SymbolTable) defineFree(original Symbol) Symbol {
// TODO: should we check duplicates?
t.freeSymbols = append(t.freeSymbols, original)
symbol := Symbol{
Name: original.Name,
Index: len(t.freeSymbols) - 1,
Scope: ScopeFree,
}
t.store[original.Name] = symbol
return symbol
}

View file

@ -0,0 +1,123 @@
package compiler_test
import (
"testing"
"github.com/d5/tengo/assert"
"github.com/d5/tengo/compiler"
)
func TestSymbolTable(t *testing.T) {
/*
GLOBAL
[0] a
[1] b
LOCAL 1
[0] d
LOCAL 2
[0] e
[1] f
LOCAL 2 BLOCK 1
[2] g
[3] h
LOCAL 2 BLOCK 2
[2] i
[3] j
[4] k
LOCAL 1 BLOCK 1
[1] l
[2] m
[3] n
[4] o
[5] p
LOCAL 3
[0] q
[1] r
*/
global := symbolTable()
assert.Equal(t, globalSymbol("a", 0), global.Define("a"))
assert.Equal(t, globalSymbol("b", 1), global.Define("b"))
local1 := global.Fork(false)
assert.Equal(t, localSymbol("d", 0), local1.Define("d"))
local1Block1 := local1.Fork(true)
assert.Equal(t, localSymbol("l", 1), local1Block1.Define("l"))
assert.Equal(t, localSymbol("m", 2), local1Block1.Define("m"))
assert.Equal(t, localSymbol("n", 3), local1Block1.Define("n"))
assert.Equal(t, localSymbol("o", 4), local1Block1.Define("o"))
assert.Equal(t, localSymbol("p", 5), local1Block1.Define("p"))
local2 := local1.Fork(false)
assert.Equal(t, localSymbol("e", 0), local2.Define("e"))
assert.Equal(t, localSymbol("f", 1), local2.Define("f"))
local2Block1 := local2.Fork(true)
assert.Equal(t, localSymbol("g", 2), local2Block1.Define("g"))
assert.Equal(t, localSymbol("h", 3), local2Block1.Define("h"))
local2Block2 := local2.Fork(true)
assert.Equal(t, localSymbol("i", 2), local2Block2.Define("i"))
assert.Equal(t, localSymbol("j", 3), local2Block2.Define("j"))
assert.Equal(t, localSymbol("k", 4), local2Block2.Define("k"))
local3 := local1Block1.Fork(false)
assert.Equal(t, localSymbol("q", 0), local3.Define("q"))
assert.Equal(t, localSymbol("r", 1), local3.Define("r"))
assert.Equal(t, 2, global.MaxSymbols())
assert.Equal(t, 6, local1.MaxSymbols())
assert.Equal(t, 6, local1Block1.MaxSymbols())
assert.Equal(t, 5, local2.MaxSymbols())
assert.Equal(t, 4, local2Block1.MaxSymbols())
assert.Equal(t, 5, local2Block2.MaxSymbols())
assert.Equal(t, 2, local3.MaxSymbols())
resolveExpect(t, global, "a", globalSymbol("a", 0), 0)
resolveExpect(t, local1, "d", localSymbol("d", 0), 0)
resolveExpect(t, local1, "a", globalSymbol("a", 0), 1)
resolveExpect(t, local3, "a", globalSymbol("a", 0), 2)
resolveExpect(t, local3, "d", freeSymbol("d", 0), 1)
resolveExpect(t, local3, "r", localSymbol("r", 1), 0)
resolveExpect(t, local2Block2, "k", localSymbol("k", 4), 0)
resolveExpect(t, local2Block2, "e", localSymbol("e", 0), 0)
resolveExpect(t, local2Block2, "b", globalSymbol("b", 1), 2)
}
func symbol(name string, scope compiler.SymbolScope, index int) compiler.Symbol {
return compiler.Symbol{
Name: name,
Scope: scope,
Index: index,
}
}
func globalSymbol(name string, index int) compiler.Symbol {
return symbol(name, compiler.ScopeGlobal, index)
}
func localSymbol(name string, index int) compiler.Symbol {
return symbol(name, compiler.ScopeLocal, index)
}
func freeSymbol(name string, index int) compiler.Symbol {
return symbol(name, compiler.ScopeFree, index)
}
func symbolTable() *compiler.SymbolTable {
return compiler.NewSymbolTable()
}
func resolveExpect(t *testing.T, symbolTable *compiler.SymbolTable, name string, expectedSymbol compiler.Symbol, expectedDepth int) {
actualSymbol, actualDepth, ok := symbolTable.Resolve(name)
assert.True(t, ok)
assert.Equal(t, expectedSymbol, actualSymbol)
assert.Equal(t, expectedDepth, actualDepth)
}

87
objects/array.go Normal file
View file

@ -0,0 +1,87 @@
package objects
import (
"errors"
"fmt"
"strings"
"github.com/d5/tengo/token"
)
type Array struct {
Value []Object
}
func (o *Array) TypeName() string {
return "array"
}
func (o *Array) String() string {
var elements []string
for _, e := range o.Value {
elements = append(elements, e.String())
}
return fmt.Sprintf("[%s]", strings.Join(elements, ", "))
}
func (o *Array) BinaryOp(op token.Token, rhs Object) (Object, error) {
if rhs, ok := rhs.(*Array); ok {
switch op {
case token.Add:
return &Array{Value: append(o.Value, rhs.Value...)}, nil
}
}
return nil, ErrInvalidOperator
}
func (o *Array) Copy() Object {
var c []Object
for _, elem := range o.Value {
c = append(c, elem.Copy())
}
return &Array{Value: c}
}
func (o *Array) IsFalsy() bool {
return false
}
func (o *Array) Equals(x Object) bool {
t, ok := x.(*Array)
if !ok {
return false
}
if len(o.Value) != len(t.Value) {
return false
}
for i, e := range o.Value {
if !e.Equals(t.Value[i]) {
return false
}
}
return true
}
func (o *Array) Get(index int) (Object, error) {
if index < 0 || index >= len(o.Value) {
return nil, errors.New("array index out of bounds")
}
return o.Value[index], nil
}
func (o *Array) Set(index int, value Object) error {
if index < 0 || index >= len(o.Value) {
return errors.New("array index out of bounds")
}
o.Value[index] = value
return nil
}

43
objects/bool.go Normal file
View file

@ -0,0 +1,43 @@
package objects
import (
"github.com/d5/tengo/token"
)
type Bool struct {
Value bool
}
func (o *Bool) String() string {
if o.Value {
return "true"
}
return "false"
}
func (o *Bool) TypeName() string {
return "bool"
}
func (o *Bool) BinaryOp(op token.Token, rhs Object) (Object, error) {
return nil, ErrInvalidOperator
}
func (o *Bool) Copy() Object {
v := Bool(*o)
return &v
}
func (o *Bool) IsFalsy() bool {
return !o.Value
}
func (o *Bool) Equals(x Object) bool {
t, ok := x.(*Bool)
if !ok {
return false
}
return o.Value == t.Value
}

29
objects/break.go Normal file
View file

@ -0,0 +1,29 @@
package objects
import "github.com/d5/tengo/token"
type Break struct{}
func (o *Break) TypeName() string {
return "break"
}
func (o *Break) String() string {
return "<break>"
}
func (o *Break) BinaryOp(op token.Token, rhs Object) (Object, error) {
return nil, ErrInvalidOperator
}
func (o *Break) Copy() Object {
return &Break{}
}
func (o *Break) IsFalsy() bool {
return false
}
func (o *Break) Equals(x Object) bool {
return false
}

19
objects/builtin_append.go Normal file
View file

@ -0,0 +1,19 @@
package objects
import (
"fmt"
)
// append(src, items...)
func builtinAppend(args ...Object) (Object, error) {
if len(args) < 2 {
return nil, fmt.Errorf("not enough arguments in call to append")
}
switch arg := args[0].(type) {
case *Array:
return &Array{Value: append(arg.Value, args[1:]...)}, nil
default:
return nil, fmt.Errorf("unsupported type for 'append' function: %s", arg.TypeName())
}
}

12
objects/builtin_copy.go Normal file
View file

@ -0,0 +1,12 @@
package objects
import "errors"
func builtinCopy(args ...Object) (Object, error) {
// TODO: should multi arguments later?
if len(args) != 1 {
return nil, errors.New("wrong number of arguments")
}
return args[0].Copy(), nil
}

View file

@ -0,0 +1,31 @@
package objects
import (
"github.com/d5/tengo/token"
)
type BuiltinFunction BuiltinFunc
func (o BuiltinFunction) TypeName() string {
return "builtin-function"
}
func (o BuiltinFunction) String() string {
return "<builtin-function>"
}
func (o BuiltinFunction) BinaryOp(op token.Token, rhs Object) (Object, error) {
return nil, ErrInvalidOperator
}
func (o BuiltinFunction) Copy() Object {
return BuiltinFunction(o)
}
func (o BuiltinFunction) IsFalsy() bool {
return false
}
func (o BuiltinFunction) Equals(x Object) bool {
return false
}

20
objects/builtin_len.go Normal file
View file

@ -0,0 +1,20 @@
package objects
import (
"fmt"
)
func builtinLen(args ...Object) (Object, error) {
if len(args) != 1 {
return nil, fmt.Errorf("wrong number of arguments (got=%d, want=1)", len(args))
}
switch arg := args[0].(type) {
case *Array:
return &Int{int64(len(arg.Value))}, nil
case *String:
return &Int{int64(len(arg.Value))}, nil
default:
return nil, fmt.Errorf("unsupported type for 'len' function: %s", arg.TypeName())
}
}

13
objects/builtin_print.go Normal file
View file

@ -0,0 +1,13 @@
package objects
import (
"fmt"
)
func builtinPrint(args ...Object) (Object, error) {
for _, arg := range args {
fmt.Println(arg.String())
}
return nil, nil
}

25
objects/builtins.go Normal file
View file

@ -0,0 +1,25 @@
package objects
type BuiltinFunc func(args ...Object) (ret Object, err error)
var Builtins = []struct {
Name string
Func BuiltinFunc
}{
{
Name: "print",
Func: builtinPrint,
},
{
Name: "len",
Func: builtinLen,
},
{
Name: "copy",
Func: builtinCopy,
},
{
Name: "append",
Func: builtinAppend,
},
}

40
objects/char.go Normal file
View file

@ -0,0 +1,40 @@
package objects
import (
"fmt"
"github.com/d5/tengo/token"
)
type Char struct {
Value rune
}
func (o *Char) String() string {
return fmt.Sprintf("%q", string(o.Value))
}
func (o *Char) TypeName() string {
return "char"
}
func (o *Char) BinaryOp(op token.Token, rhs Object) (Object, error) {
return nil, ErrInvalidOperator
}
func (o *Char) Copy() Object {
return &Char{Value: o.Value}
}
func (o *Char) IsFalsy() bool {
return false
}
func (o *Char) Equals(x Object) bool {
t, ok := x.(*Char)
if !ok {
return false
}
return o.Value == t.Value
}

37
objects/closure.go Normal file
View file

@ -0,0 +1,37 @@
package objects
import (
"github.com/d5/tengo/token"
)
type Closure struct {
Fn *CompiledFunction
Free []*Object
}
func (o *Closure) TypeName() string {
return "closure"
}
func (o *Closure) String() string {
return "<closure>"
}
func (o *Closure) BinaryOp(op token.Token, rhs Object) (Object, error) {
return nil, ErrInvalidOperator
}
func (o *Closure) Copy() Object {
return &Closure{
Fn: o.Fn.Copy().(*CompiledFunction),
Free: append([]*Object{}, o.Free...), // DO NOT Copy() of elements; these are variable pointers
}
}
func (o *Closure) IsFalsy() bool {
return false
}
func (o *Closure) Equals(x Object) bool {
return false
}

View file

@ -0,0 +1,39 @@
package objects
import (
"github.com/d5/tengo/token"
)
type CompiledFunction struct {
Instructions []byte
NumLocals int
NumParameters int
}
func (o *CompiledFunction) TypeName() string {
return "compiled-function"
}
func (o *CompiledFunction) String() string {
return "<compiled-function>"
}
func (o *CompiledFunction) BinaryOp(op token.Token, rhs Object) (Object, error) {
return nil, ErrInvalidOperator
}
func (o *CompiledFunction) Copy() Object {
return &CompiledFunction{
Instructions: append([]byte{}, o.Instructions...),
NumLocals: o.NumLocals,
NumParameters: o.NumParameters,
}
}
func (o *CompiledFunction) IsFalsy() bool {
return false
}
func (o *CompiledFunction) Equals(x Object) bool {
return false
}

30
objects/continue.go Normal file
View file

@ -0,0 +1,30 @@
package objects
import "github.com/d5/tengo/token"
type Continue struct {
}
func (o *Continue) TypeName() string {
return "continue"
}
func (o *Continue) String() string {
return "<continue>"
}
func (o *Continue) BinaryOp(op token.Token, rhs Object) (Object, error) {
return nil, ErrInvalidOperator
}
func (o *Continue) Copy() Object {
return &Continue{}
}
func (o *Continue) IsFalsy() bool {
return false
}
func (o *Continue) Equals(x Object) bool {
return false
}

5
objects/errors.go Normal file
View file

@ -0,0 +1,5 @@
package objects
import "errors"
var ErrInvalidOperator = errors.New("invalid operator")

81
objects/float.go Normal file
View file

@ -0,0 +1,81 @@
package objects
import (
"strconv"
"github.com/d5/tengo/token"
)
type Float struct {
Value float64
}
func (o *Float) String() string {
return strconv.FormatFloat(o.Value, 'f', -1, 64)
}
func (o *Float) TypeName() string {
return "float"
}
func (o *Float) BinaryOp(op token.Token, rhs Object) (Object, error) {
switch rhs := rhs.(type) {
case *Float:
switch op {
case token.Add:
return &Float{o.Value + rhs.Value}, nil
case token.Sub:
return &Float{o.Value - rhs.Value}, nil
case token.Mul:
return &Float{o.Value * rhs.Value}, nil
case token.Quo:
return &Float{o.Value / rhs.Value}, nil
case token.Less:
return &Bool{o.Value < rhs.Value}, nil
case token.Greater:
return &Bool{o.Value > rhs.Value}, nil
case token.LessEq:
return &Bool{o.Value <= rhs.Value}, nil
case token.GreaterEq:
return &Bool{o.Value >= rhs.Value}, nil
}
case *Int:
switch op {
case token.Add:
return &Float{o.Value + float64(rhs.Value)}, nil
case token.Sub:
return &Float{o.Value - float64(rhs.Value)}, nil
case token.Mul:
return &Float{o.Value * float64(rhs.Value)}, nil
case token.Quo:
return &Float{o.Value / float64(rhs.Value)}, nil
case token.Less:
return &Bool{o.Value < float64(rhs.Value)}, nil
case token.Greater:
return &Bool{o.Value > float64(rhs.Value)}, nil
case token.LessEq:
return &Bool{o.Value <= float64(rhs.Value)}, nil
case token.GreaterEq:
return &Bool{o.Value >= float64(rhs.Value)}, nil
}
}
return nil, ErrInvalidOperator
}
func (o *Float) Copy() Object {
return &Float{Value: o.Value}
}
func (o *Float) IsFalsy() bool {
return false
}
func (o *Float) Equals(x Object) bool {
t, ok := x.(*Float)
if !ok {
return false
}
return o.Value == t.Value
}

95
objects/int.go Normal file
View file

@ -0,0 +1,95 @@
package objects
import (
"strconv"
"github.com/d5/tengo/token"
)
type Int struct {
Value int64
}
func (o *Int) String() string {
return strconv.FormatInt(o.Value, 10)
}
func (o *Int) TypeName() string {
return "int"
}
func (o *Int) BinaryOp(op token.Token, rhs Object) (Object, error) {
switch rhs := rhs.(type) {
case *Int:
switch op {
case token.Add:
return &Int{o.Value + rhs.Value}, nil
case token.Sub:
return &Int{o.Value - rhs.Value}, nil
case token.Mul:
return &Int{o.Value * rhs.Value}, nil
case token.Quo:
return &Int{o.Value / rhs.Value}, nil
case token.Rem:
return &Int{o.Value % rhs.Value}, nil
case token.And:
return &Int{o.Value & rhs.Value}, nil
case token.Or:
return &Int{o.Value | rhs.Value}, nil
case token.Xor:
return &Int{o.Value ^ rhs.Value}, nil
case token.AndNot:
return &Int{o.Value &^ rhs.Value}, nil
case token.Shl:
return &Int{o.Value << uint(rhs.Value)}, nil
case token.Shr:
return &Int{o.Value >> uint(rhs.Value)}, nil
case token.Less:
return &Bool{o.Value < rhs.Value}, nil
case token.Greater:
return &Bool{o.Value > rhs.Value}, nil
case token.LessEq:
return &Bool{o.Value <= rhs.Value}, nil
case token.GreaterEq:
return &Bool{o.Value >= rhs.Value}, nil
}
case *Float:
switch op {
case token.Add:
return &Float{float64(o.Value) + rhs.Value}, nil
case token.Sub:
return &Float{float64(o.Value) - rhs.Value}, nil
case token.Mul:
return &Float{float64(o.Value) * rhs.Value}, nil
case token.Quo:
return &Float{float64(o.Value) / rhs.Value}, nil
case token.Less:
return &Bool{float64(o.Value) < rhs.Value}, nil
case token.Greater:
return &Bool{float64(o.Value) > rhs.Value}, nil
case token.LessEq:
return &Bool{float64(o.Value) <= rhs.Value}, nil
case token.GreaterEq:
return &Bool{float64(o.Value) >= rhs.Value}, nil
}
}
return nil, ErrInvalidOperator
}
func (o *Int) Copy() Object {
return &Int{o.Value}
}
func (o *Int) IsFalsy() bool {
return o.Value == 0
}
func (o *Int) Equals(x Object) bool {
t, ok := x.(*Int)
if !ok {
return false
}
return o.Value == t.Value
}

173
objects/iterator.go Normal file
View file

@ -0,0 +1,173 @@
package objects
import "github.com/d5/tengo/token"
type Iterator interface {
Object
Next() bool
Key() Object
Value() Object
}
type ArrayIterator struct {
v []Object
i int
l int
}
func NewArrayIterator(v *Array) Iterator {
return &ArrayIterator{
v: v.Value,
l: len(v.Value),
}
}
func (i *ArrayIterator) TypeName() string {
return "array-iterator"
}
func (i *ArrayIterator) String() string {
return "<array-iterator>"
}
func (i *ArrayIterator) BinaryOp(op token.Token, rhs Object) (Object, error) {
return nil, ErrInvalidOperator
}
func (i *ArrayIterator) IsFalsy() bool {
return true
}
func (i *ArrayIterator) Equals(Object) bool {
return false
}
func (i *ArrayIterator) Copy() Object {
return &ArrayIterator{v: i.v, i: i.i, l: i.l}
}
func (i *ArrayIterator) Next() bool {
i.i++
return i.i <= i.l
}
func (i *ArrayIterator) Key() Object {
return &Int{int64(i.i - 1)}
}
func (i *ArrayIterator) Value() Object {
return i.v[i.i-1]
}
type MapIterator struct {
v map[string]Object
k []string
i int
l int
}
func NewMapIterator(v *Map) Iterator {
var keys []string
for k := range v.Value {
keys = append(keys, k)
}
return &MapIterator{
v: v.Value,
k: keys,
l: len(keys),
}
}
func (i *MapIterator) TypeName() string {
return "map-iterator"
}
func (i *MapIterator) String() string {
return "<map-iterator>"
}
func (i *MapIterator) BinaryOp(op token.Token, rhs Object) (Object, error) {
return nil, ErrInvalidOperator
}
func (i *MapIterator) IsFalsy() bool {
return true
}
func (i *MapIterator) Equals(Object) bool {
return false
}
func (i *MapIterator) Copy() Object {
return &MapIterator{v: i.v, k: i.k, i: i.i, l: i.l}
}
func (i *MapIterator) Next() bool {
i.i++
return i.i <= i.l
}
func (i *MapIterator) Key() Object {
k := i.k[i.i-1]
return &String{Value: k}
}
func (i *MapIterator) Value() Object {
k := i.k[i.i-1]
return i.v[k]
}
type StringIterator struct {
v []rune
i int
l int
}
func NewStringIterator(v *String) Iterator {
r := []rune(v.Value)
return &StringIterator{
v: r,
l: len(r),
}
}
func (i *StringIterator) TypeName() string {
return "string-iterator"
}
func (i *StringIterator) String() string {
return "<string-iterator>"
}
func (i *StringIterator) BinaryOp(op token.Token, rhs Object) (Object, error) {
return nil, ErrInvalidOperator
}
func (i *StringIterator) IsFalsy() bool {
return true
}
func (i *StringIterator) Equals(Object) bool {
return false
}
func (i *StringIterator) Copy() Object {
return &StringIterator{v: i.v, i: i.i, l: i.l}
}
func (i *StringIterator) Next() bool {
i.i++
return i.i <= i.l
}
func (i *StringIterator) Key() Object {
return &Int{int64(i.i - 1)}
}
func (i *StringIterator) Value() Object {
return &Char{Value: i.v[i.i-1]}
}

72
objects/map.go Normal file
View file

@ -0,0 +1,72 @@
package objects
import (
"fmt"
"strings"
"github.com/d5/tengo/token"
)
type Map struct {
Value map[string]Object
}
func (o *Map) TypeName() string {
return "map"
}
func (o *Map) String() string {
var pairs []string
for k, v := range o.Value {
pairs = append(pairs, fmt.Sprintf("%s: %s", k, v.String()))
}
return fmt.Sprintf("{%s}", strings.Join(pairs, ", "))
}
func (o *Map) BinaryOp(op token.Token, rhs Object) (Object, error) {
return nil, ErrInvalidOperator
}
func (o *Map) Copy() Object {
c := make(map[string]Object)
for k, v := range o.Value {
c[k] = v.Copy()
}
return &Map{Value: c}
}
func (o *Map) IsFalsy() bool {
return false
}
func (o *Map) Get(key string) (Object, bool) {
val, ok := o.Value[key]
return val, ok
}
func (o *Map) Set(key string, value Object) {
o.Value[key] = value
}
func (o *Map) Equals(x Object) bool {
t, ok := x.(*Map)
if !ok {
return false
}
if len(o.Value) != len(t.Value) {
return false
}
for k, v := range o.Value {
tv := t.Value[k]
if !v.Equals(tv) {
return false
}
}
return true
}

12
objects/object.go Normal file
View file

@ -0,0 +1,12 @@
package objects
import "github.com/d5/tengo/token"
type Object interface {
TypeName() string
String() string
BinaryOp(op token.Token, rhs Object) (Object, error)
IsFalsy() bool
Equals(Object) bool
Copy() Object
}

52
objects/objects.go Normal file
View file

@ -0,0 +1,52 @@
package objects
import "fmt"
func FromValue(v interface{}) (Object, error) {
switch v := v.(type) {
case string:
return &String{Value: v}, nil
case int64:
return &Int{Value: v}, nil
case int:
return &Int{Value: int64(v)}, nil
case bool:
return &Bool{Value: v}, nil
case rune:
return &Char{Value: v}, nil
case byte:
return &Char{Value: rune(v)}, nil
case float64:
return &Float{Value: v}, nil
case map[string]Object:
return &Map{Value: v}, nil
case map[string]interface{}:
kv := make(map[string]Object)
for vk, vv := range v {
vo, err := FromValue(vv)
if err != nil {
return nil, err
}
kv[vk] = vo
}
return &Map{Value: kv}, nil
case []Object:
return &Array{Value: v}, nil
case []interface{}:
arr := make([]Object, len(v), len(v))
for _, e := range v {
vo, err := FromValue(e)
if err != nil {
return nil, err
}
arr = append(arr, vo)
}
return &Array{Value: arr}, nil
case Object:
return v, nil
}
return nil, fmt.Errorf("unsupported value type: %T", v)
}

31
objects/return_value.go Normal file
View file

@ -0,0 +1,31 @@
package objects
import "github.com/d5/tengo/token"
type ReturnValue struct {
Value Object
}
func (o *ReturnValue) TypeName() string {
return "return-value"
}
func (o *ReturnValue) String() string {
return "<return>"
}
func (o *ReturnValue) BinaryOp(op token.Token, rhs Object) (Object, error) {
return nil, ErrInvalidOperator
}
func (o *ReturnValue) Copy() Object {
return &ReturnValue{Value: o.Copy()}
}
func (o *ReturnValue) IsFalsy() bool {
return false
}
func (o *ReturnValue) Equals(x Object) bool {
return false
}

53
objects/string.go Normal file
View file

@ -0,0 +1,53 @@
package objects
import (
"fmt"
"github.com/d5/tengo/token"
)
type String struct {
Value string
}
func (o *String) TypeName() string {
return "string"
}
func (o *String) String() string {
return fmt.Sprintf("%q", o.Value)
}
func (o *String) BinaryOp(op token.Token, rhs Object) (Object, error) {
switch rhs := rhs.(type) {
case *String:
switch op {
case token.Add:
return &String{Value: o.Value + rhs.Value}, nil
}
case *Char:
switch op {
case token.Add:
return &String{Value: o.Value + string(rhs.Value)}, nil
}
}
return nil, ErrInvalidOperator
}
func (o *String) IsFalsy() bool {
return len(o.Value) == 0
}
func (o *String) Copy() Object {
return &String{Value: o.Value}
}
func (o *String) Equals(x Object) bool {
t, ok := x.(*String)
if !ok {
return false
}
return o.Value == t.Value
}

31
objects/undefined.go Normal file
View file

@ -0,0 +1,31 @@
package objects
import "github.com/d5/tengo/token"
type Undefined struct{}
func (o Undefined) TypeName() string {
return "undefined"
}
func (o Undefined) String() string {
return "<undefined>"
}
func (o Undefined) BinaryOp(op token.Token, rhs Object) (Object, error) {
return nil, ErrInvalidOperator
}
func (o Undefined) Copy() Object {
return Undefined{}
}
func (o Undefined) IsFalsy() bool {
return true
}
func (o Undefined) Equals(x Object) bool {
_, ok := x.(Undefined)
return ok
}

16
parser/error.go Normal file
View file

@ -0,0 +1,16 @@
package parser
import "github.com/d5/tengo/scanner"
type Error struct {
Pos scanner.FilePos
Msg string
}
func (e Error) Error() string {
if e.Pos.Filename != "" || e.Pos.IsValid() {
return e.Pos.String() + ": " + e.Msg
}
return e.Msg
}

84
parser/error_list.go Normal file
View file

@ -0,0 +1,84 @@
package parser
import (
"fmt"
"sort"
"github.com/d5/tengo/scanner"
)
type ErrorList []*Error
func (p *ErrorList) Add(pos scanner.FilePos, msg string) {
*p = append(*p, &Error{pos, msg})
}
func (p *ErrorList) Reset() {
*p = (*p)[0:0]
}
func (p ErrorList) Len() int {
return len(p)
}
func (p ErrorList) Swap(i, j int) {
p[i], p[j] = p[j], p[i]
}
func (p ErrorList) Less(i, j int) bool {
e := &p[i].Pos
f := &p[j].Pos
if e.Filename != f.Filename {
return e.Filename < f.Filename
}
if e.Line != f.Line {
return e.Line < f.Line
}
if e.Column != f.Column {
return e.Column < f.Column
}
return p[i].Msg < p[j].Msg
}
func (p ErrorList) Sort() {
sort.Sort(p)
}
func (p *ErrorList) RemoveMultiples() {
sort.Sort(p)
var last scanner.FilePos // initial last.Line is != any legal error line
i := 0
for _, e := range *p {
if e.Pos.Filename != last.Filename || e.Pos.Line != last.Line {
last = e.Pos
(*p)[i] = e
i++
}
}
*p = (*p)[0:i]
}
func (p ErrorList) Error() string {
switch len(p) {
case 0:
return "no errors"
case 1:
return p[0].Error()
}
return fmt.Sprintf("%s (and %d more errors)", p[0], len(p)-1)
}
func (p ErrorList) Err() error {
if len(p) == 0 {
return nil
}
return p
}

27
parser/parse_file.go Normal file
View file

@ -0,0 +1,27 @@
package parser
import (
"io"
"github.com/d5/tengo/ast"
"github.com/d5/tengo/scanner"
)
func ParseFile(file *scanner.File, src []byte, trace io.Writer) (res *ast.File, err error) {
p := NewParser(file, src, trace)
defer func() {
if e := recover(); e != nil {
if _, ok := e.(bailout); !ok {
panic(e)
}
}
p.errors.Sort()
err = p.errors.Err()
}()
res, err = p.ParseFile()
return
}

1028
parser/parser.go Normal file

File diff suppressed because it is too large Load diff

106
parser/parser_array_test.go Normal file
View file

@ -0,0 +1,106 @@
package parser_test
import (
"testing"
"github.com/d5/tengo/ast"
"github.com/d5/tengo/token"
)
func TestArray(t *testing.T) {
expect(t, "[1, 2, 3]", func(p pfn) []ast.Stmt {
return stmts(
exprStmt(
arrayLit(p(1, 1), p(1, 9),
intLit(1, p(1, 2)),
intLit(2, p(1, 5)),
intLit(3, p(1, 8)))))
})
expect(t, `
[
1,
2,
3
]`, func(p pfn) []ast.Stmt {
return stmts(
exprStmt(
arrayLit(p(2, 1), p(6, 1),
intLit(1, p(3, 2)),
intLit(2, p(4, 2)),
intLit(3, p(5, 2)))))
})
expect(t, `
[
1,
2,
3
]`, func(p pfn) []ast.Stmt {
return stmts(
exprStmt(
arrayLit(p(2, 1), p(7, 1),
intLit(1, p(3, 2)),
intLit(2, p(4, 2)),
intLit(3, p(5, 2)))))
})
expect(t, `[1, "foo", 12.34]`, func(p pfn) []ast.Stmt {
return stmts(
exprStmt(
arrayLit(p(1, 1), p(1, 17),
intLit(1, p(1, 2)),
stringLit("foo", p(1, 5)),
floatLit(12.34, p(1, 12)))))
})
expect(t, "a = [1, 2, 3]", func(p pfn) []ast.Stmt {
return stmts(
assignStmt(
exprs(ident("a", p(1, 1))),
exprs(arrayLit(p(1, 5), p(1, 13),
intLit(1, p(1, 6)),
intLit(2, p(1, 9)),
intLit(3, p(1, 12)))),
token.Assign,
p(1, 3)))
})
expect(t, "a = [1 + 2, b * 4, [4, c]]", func(p pfn) []ast.Stmt {
return stmts(
assignStmt(
exprs(ident("a", p(1, 1))),
exprs(arrayLit(p(1, 5), p(1, 26),
binaryExpr(
intLit(1, p(1, 6)),
intLit(2, p(1, 10)),
token.Add,
p(1, 8)),
binaryExpr(
ident("b", p(1, 13)),
intLit(4, p(1, 17)),
token.Mul,
p(1, 15)),
arrayLit(p(1, 20), p(1, 25),
intLit(4, p(1, 21)),
ident("c", p(1, 24))))),
token.Assign,
p(1, 3)))
})
expectError(t, `[1, 2, 3,]`)
expectError(t, `
[
1,
2,
3,
]`)
expectError(t, `
[
1,
2,
3,
]`)
expectError(t, `[1, 2, 3, ,]`)
}

View file

@ -0,0 +1,132 @@
package parser_test
import (
"testing"
"github.com/d5/tengo/ast"
"github.com/d5/tengo/token"
)
func TestAssignment(t *testing.T) {
expect(t, "a = 5", func(p pfn) []ast.Stmt {
return stmts(
assignStmt(
exprs(ident("a", p(1, 1))),
exprs(intLit(5, p(1, 5))),
token.Assign,
p(1, 3)))
})
expect(t, "a := 5", func(p pfn) []ast.Stmt {
return stmts(
assignStmt(
exprs(ident("a", p(1, 1))),
exprs(intLit(5, p(1, 6))),
token.Define,
p(1, 3)))
})
expect(t, "a, b = 5, 10", func(p pfn) []ast.Stmt {
return stmts(
assignStmt(
exprs(
ident("a", p(1, 1)),
ident("b", p(1, 4))),
exprs(
intLit(5, p(1, 8)),
intLit(10, p(1, 11))),
token.Assign,
p(1, 6)))
})
expect(t, "a, b := 5, 10", func(p pfn) []ast.Stmt {
return stmts(
assignStmt(
exprs(
ident("a", p(1, 1)),
ident("b", p(1, 4))),
exprs(
intLit(5, p(1, 9)),
intLit(10, p(1, 12))),
token.Define,
p(1, 6)))
})
expect(t, "a, b = a + 2, b - 8", func(p pfn) []ast.Stmt {
return stmts(
assignStmt(
exprs(
ident("a", p(1, 1)),
ident("b", p(1, 4))),
exprs(
binaryExpr(
ident("a", p(1, 8)),
intLit(2, p(1, 12)),
token.Add,
p(1, 10)),
binaryExpr(
ident("b", p(1, 15)),
intLit(8, p(1, 19)),
token.Sub,
p(1, 17))),
token.Assign,
p(1, 6)))
})
expect(t, "a = [1, 2, 3]", func(p pfn) []ast.Stmt {
return stmts(
assignStmt(
exprs(ident("a", p(1, 1))),
exprs(arrayLit(p(1, 5), p(1, 13),
intLit(1, p(1, 6)),
intLit(2, p(1, 9)),
intLit(3, p(1, 12)))),
token.Assign,
p(1, 3)))
})
expect(t, "a = [1 + 2, b * 4, [4, c]]", func(p pfn) []ast.Stmt {
return stmts(
assignStmt(
exprs(ident("a", p(1, 1))),
exprs(arrayLit(p(1, 5), p(1, 26),
binaryExpr(
intLit(1, p(1, 6)),
intLit(2, p(1, 10)),
token.Add,
p(1, 8)),
binaryExpr(
ident("b", p(1, 13)),
intLit(4, p(1, 17)),
token.Mul,
p(1, 15)),
arrayLit(p(1, 20), p(1, 25),
intLit(4, p(1, 21)),
ident("c", p(1, 24))))),
token.Assign,
p(1, 3)))
})
expect(t, "a += 5", func(p pfn) []ast.Stmt {
return stmts(
assignStmt(
exprs(ident("a", p(1, 1))),
exprs(intLit(5, p(1, 6))),
token.AddAssign,
p(1, 3)))
})
expect(t, "a *= 5 + 10", func(p pfn) []ast.Stmt {
return stmts(
assignStmt(
exprs(ident("a", p(1, 1))),
exprs(
binaryExpr(
intLit(5, p(1, 6)),
intLit(10, p(1, 10)),
token.Add,
p(1, 8))),
token.MulAssign,
p(1, 3)))
})
}

View file

@ -0,0 +1,41 @@
package parser_test
import (
"testing"
"github.com/d5/tengo/ast"
"github.com/d5/tengo/token"
)
func TestBoolean(t *testing.T) {
expect(t, "true", func(p pfn) []ast.Stmt {
return stmts(
exprStmt(
boolLit(true, p(1, 1))))
})
expect(t, "false", func(p pfn) []ast.Stmt {
return stmts(
exprStmt(
boolLit(false, p(1, 1))))
})
expect(t, "true != false", func(p pfn) []ast.Stmt {
return stmts(
exprStmt(
binaryExpr(
boolLit(true, p(1, 1)),
boolLit(false, p(1, 9)),
token.NotEqual,
p(1, 6))))
})
expect(t, "!false", func(p pfn) []ast.Stmt {
return stmts(
exprStmt(
unaryExpr(
boolLit(false, p(1, 2)),
token.Not,
p(1, 1))))
})
}

143
parser/parser_call_test.go Normal file
View file

@ -0,0 +1,143 @@
package parser_test
import (
"testing"
"github.com/d5/tengo/ast"
"github.com/d5/tengo/token"
)
func TestCall(t *testing.T) {
expect(t, "add(1, 2, 3)", func(p pfn) []ast.Stmt {
return stmts(
exprStmt(
callExpr(
ident("add", p(1, 1)),
p(1, 4), p(1, 12),
intLit(1, p(1, 5)),
intLit(2, p(1, 8)),
intLit(3, p(1, 11)))))
})
expect(t, "a = add(1, 2, 3)", func(p pfn) []ast.Stmt {
return stmts(
assignStmt(
exprs(
ident("a", p(1, 1))),
exprs(
callExpr(
ident("add", p(1, 5)),
p(1, 8), p(1, 16),
intLit(1, p(1, 9)),
intLit(2, p(1, 12)),
intLit(3, p(1, 15)))),
token.Assign,
p(1, 3)))
})
expect(t, "a, b = add(1, 2, 3)", func(p pfn) []ast.Stmt {
return stmts(
assignStmt(
exprs(
ident("a", p(1, 1)),
ident("b", p(1, 4))),
exprs(
callExpr(
ident("add", p(1, 8)),
p(1, 11), p(1, 19),
intLit(1, p(1, 12)),
intLit(2, p(1, 15)),
intLit(3, p(1, 18)))),
token.Assign,
p(1, 6)))
})
expect(t, "add(a + 1, 2 * 1, (b + c))", func(p pfn) []ast.Stmt {
return stmts(
exprStmt(
callExpr(
ident("add", p(1, 1)),
p(1, 4), p(1, 26),
binaryExpr(
ident("a", p(1, 5)),
intLit(1, p(1, 9)),
token.Add,
p(1, 7)),
binaryExpr(
intLit(2, p(1, 12)),
intLit(1, p(1, 16)),
token.Mul,
p(1, 14)),
parenExpr(
binaryExpr(
ident("b", p(1, 20)),
ident("c", p(1, 24)),
token.Add,
p(1, 22)),
p(1, 19), p(1, 25)))))
})
expectString(t, "a + add(b * c) + d", "((a + add((b * c))) + d)")
expectString(t, "add(a, b, 1, 2 * 3, 4 + 5, add(6, 7 * 8))", "add(a, b, 1, (2 * 3), (4 + 5), add(6, (7 * 8)))")
expectString(t, "f1(a) + f2(b) * f3(c)", "(f1(a) + (f2(b) * f3(c)))")
expectString(t, "(f1(a) + f2(b)) * f3(c)", "(((f1(a) + f2(b))) * f3(c))")
expect(t, "func(a, b) { a + b }(1, 2)", func(p pfn) []ast.Stmt {
return stmts(
exprStmt(
callExpr(
funcLit(
funcType(
identList(
p(1, 5), p(1, 10),
ident("a", p(1, 6)),
ident("b", p(1, 9))),
p(1, 1)),
blockStmt(
p(1, 12), p(1, 20),
exprStmt(
binaryExpr(
ident("a", p(1, 14)),
ident("b", p(1, 18)),
token.Add,
p(1, 16))))),
p(1, 21), p(1, 26),
intLit(1, p(1, 22)),
intLit(2, p(1, 25)))))
})
expect(t, `a.b()`, func(p pfn) []ast.Stmt {
return stmts(
exprStmt(
callExpr(
selectorExpr(
ident("a", p(1, 1)),
stringLit("b", p(1, 3))),
p(1, 4), p(1, 5))))
})
expect(t, `a.b.c()`, func(p pfn) []ast.Stmt {
return stmts(
exprStmt(
callExpr(
selectorExpr(
selectorExpr(
ident("a", p(1, 1)),
stringLit("b", p(1, 3))),
stringLit("c", p(1, 5))),
p(1, 6), p(1, 7))))
})
expect(t, `a["b"].c()`, func(p pfn) []ast.Stmt {
return stmts(
exprStmt(
callExpr(
selectorExpr(
indexExpr(
ident("a", p(1, 1)),
stringLit("b", p(1, 3)),
p(1, 2), p(1, 6)),
stringLit("c", p(1, 8))),
p(1, 9), p(1, 10))))
})
}

View file

@ -0,0 +1,66 @@
package parser_test
import (
"testing"
"github.com/d5/tengo/ast"
)
func TestForIn(t *testing.T) {
expect(t, "for x in y {}", func(p pfn) []ast.Stmt {
return stmts(
forInStmt(
ident("_", p(1, 5)),
ident("x", p(1, 5)),
ident("y", p(1, 10)),
blockStmt(p(1, 12), p(1, 13)),
p(1, 1)))
})
expect(t, "for _ in y {}", func(p pfn) []ast.Stmt {
return stmts(
forInStmt(
ident("_", p(1, 5)),
ident("_", p(1, 5)),
ident("y", p(1, 10)),
blockStmt(p(1, 12), p(1, 13)),
p(1, 1)))
})
expect(t, "for x in [1, 2, 3] {}", func(p pfn) []ast.Stmt {
return stmts(
forInStmt(
ident("_", p(1, 5)),
ident("x", p(1, 5)),
arrayLit(
p(1, 10), p(1, 18),
intLit(1, p(1, 11)),
intLit(2, p(1, 14)),
intLit(3, p(1, 17))),
blockStmt(p(1, 20), p(1, 21)),
p(1, 1)))
})
expect(t, "for x, y in z {}", func(p pfn) []ast.Stmt {
return stmts(
forInStmt(
ident("x", p(1, 5)),
ident("y", p(1, 8)),
ident("z", p(1, 13)),
blockStmt(p(1, 15), p(1, 16)),
p(1, 1)))
})
expect(t, "for x, y in {k1: 1, k2: 2} {}", func(p pfn) []ast.Stmt {
return stmts(
forInStmt(
ident("x", p(1, 5)),
ident("y", p(1, 8)),
mapLit(
p(1, 13), p(1, 26),
mapElementLit("k1", p(1, 14), p(1, 16), intLit(1, p(1, 18))),
mapElementLit("k2", p(1, 21), p(1, 23), intLit(2, p(1, 25)))),
blockStmt(p(1, 28), p(1, 29)),
p(1, 1)))
})
}

118
parser/parser_for_test.go Normal file
View file

@ -0,0 +1,118 @@
package parser_test
import (
"testing"
"github.com/d5/tengo/ast"
"github.com/d5/tengo/token"
)
func TestFor(t *testing.T) {
expect(t, "for {}", func(p pfn) []ast.Stmt {
return stmts(
forStmt(nil, nil, nil, blockStmt(p(1, 5), p(1, 6)), p(1, 1)))
})
expect(t, "for a == 5 {}", func(p pfn) []ast.Stmt {
return stmts(
forStmt(
nil,
binaryExpr(
ident("a", p(1, 5)),
intLit(5, p(1, 10)),
token.Equal,
p(1, 7)),
nil,
blockStmt(p(1, 12), p(1, 13)),
p(1, 1)))
})
expect(t, "for a := 0; a == 5; {}", func(p pfn) []ast.Stmt {
return stmts(
forStmt(
assignStmt(
exprs(ident("a", p(1, 5))),
exprs(intLit(0, p(1, 10))),
token.Define, p(1, 7)),
binaryExpr(
ident("a", p(1, 13)),
intLit(5, p(1, 18)),
token.Equal,
p(1, 15)),
nil,
blockStmt(p(1, 22), p(1, 23)),
p(1, 1)))
})
expect(t, "for a := 0; a < 5; a++ {}", func(p pfn) []ast.Stmt {
return stmts(
forStmt(
assignStmt(
exprs(ident("a", p(1, 5))),
exprs(intLit(0, p(1, 10))),
token.Define, p(1, 7)),
binaryExpr(
ident("a", p(1, 13)),
intLit(5, p(1, 17)),
token.Less,
p(1, 15)),
incDecStmt(
ident("a", p(1, 20)),
token.Inc, p(1, 21)),
blockStmt(p(1, 24), p(1, 25)),
p(1, 1)))
})
expect(t, "for ; a < 5; a++ {}", func(p pfn) []ast.Stmt {
return stmts(
forStmt(
nil,
binaryExpr(
ident("a", p(1, 7)),
intLit(5, p(1, 11)),
token.Less,
p(1, 9)),
incDecStmt(
ident("a", p(1, 14)),
token.Inc, p(1, 15)),
blockStmt(p(1, 18), p(1, 19)),
p(1, 1)))
})
expect(t, "for a := 0; ; a++ {}", func(p pfn) []ast.Stmt {
return stmts(
forStmt(
assignStmt(
exprs(ident("a", p(1, 5))),
exprs(intLit(0, p(1, 10))),
token.Define, p(1, 7)),
nil,
incDecStmt(
ident("a", p(1, 15)),
token.Inc, p(1, 16)),
blockStmt(p(1, 19), p(1, 20)),
p(1, 1)))
})
expect(t, "for a == 5 && b != 4 {}", func(p pfn) []ast.Stmt {
return stmts(
forStmt(
nil,
binaryExpr(
binaryExpr(
ident("a", p(1, 5)),
intLit(5, p(1, 10)),
token.Equal,
p(1, 7)),
binaryExpr(
ident("b", p(1, 15)),
intLit(4, p(1, 20)),
token.NotEqual,
p(1, 17)),
token.LAnd,
p(1, 12)),
nil,
blockStmt(p(1, 22), p(1, 23)),
p(1, 1)))
})
}

View file

@ -0,0 +1,46 @@
package parser_test
import (
"testing"
"github.com/d5/tengo/ast"
"github.com/d5/tengo/token"
)
func TestFunction(t *testing.T) {
// TODO: function declaration currently not parsed.
// All functions are parsed as function literal instead.
// In Go, function declaration is parsed only at the top level.
//expect(t, "func a(b, c, d) {}", func(p pfn) []ast.Stmt {
// return stmts(
// declStmt(
// funcDecl(
// ident("a", p(1, 6)),
// funcType(
// identList(p(1, 7), p(1, 15),
// ident("b", p(1, 8)),
// ident("c", p(1, 11)),
// ident("d", p(1, 14))),
// p(1, 12)),
// blockStmt(p(1, 17), p(1, 18)))))
//})
expect(t, "a = func(b, c, d) { return d }", func(p pfn) []ast.Stmt {
return stmts(
assignStmt(
exprs(
ident("a", p(1, 1))),
exprs(
funcLit(
funcType(
identList(p(1, 9), p(1, 17),
ident("b", p(1, 10)),
ident("c", p(1, 13)),
ident("d", p(1, 16))),
p(1, 5)),
blockStmt(p(1, 19), p(1, 30),
returnStmt(p(1, 21), ident("d", p(1, 28)))))),
token.Assign,
p(1, 3)))
})
}

215
parser/parser_if_test.go Normal file
View file

@ -0,0 +1,215 @@
package parser_test
import (
"testing"
"github.com/d5/tengo/ast"
"github.com/d5/tengo/token"
)
func TestIf(t *testing.T) {
expect(t, "if a == 5 {}", func(p pfn) []ast.Stmt {
return stmts(
ifStmt(
nil,
binaryExpr(
ident("a", p(1, 4)),
intLit(5, p(1, 9)),
token.Equal,
p(1, 6)),
blockStmt(
p(1, 11), p(1, 12)),
nil,
p(1, 1)))
})
expect(t, "if a == 5 && b != 3 {}", func(p pfn) []ast.Stmt {
return stmts(
ifStmt(
nil,
binaryExpr(
binaryExpr(
ident("a", p(1, 4)),
intLit(5, p(1, 9)),
token.Equal,
p(1, 6)),
binaryExpr(
ident("b", p(1, 14)),
intLit(3, p(1, 19)),
token.NotEqual,
p(1, 16)),
token.LAnd,
p(1, 11)),
blockStmt(
p(1, 21), p(1, 22)),
nil,
p(1, 1)))
})
expect(t, "if a == 5 { a = 3; a = 1 }", func(p pfn) []ast.Stmt {
return stmts(
ifStmt(
nil,
binaryExpr(
ident("a", p(1, 4)),
intLit(5, p(1, 9)),
token.Equal,
p(1, 6)),
blockStmt(
p(1, 11), p(1, 26),
assignStmt(
exprs(ident("a", p(1, 13))),
exprs(intLit(3, p(1, 17))),
token.Assign,
p(1, 15)),
assignStmt(
exprs(ident("a", p(1, 20))),
exprs(intLit(1, p(1, 24))),
token.Assign,
p(1, 22))),
nil,
p(1, 1)))
})
expect(t, "if a == 5 { a = 3; a = 1 } else { a = 2; a = 4 }", func(p pfn) []ast.Stmt {
return stmts(
ifStmt(
nil,
binaryExpr(
ident("a", p(1, 4)),
intLit(5, p(1, 9)),
token.Equal,
p(1, 6)),
blockStmt(
p(1, 11), p(1, 26),
assignStmt(
exprs(ident("a", p(1, 13))),
exprs(intLit(3, p(1, 17))),
token.Assign,
p(1, 15)),
assignStmt(
exprs(ident("a", p(1, 20))),
exprs(intLit(1, p(1, 24))),
token.Assign,
p(1, 22))),
blockStmt(
p(1, 33), p(1, 48),
assignStmt(
exprs(ident("a", p(1, 35))),
exprs(intLit(2, p(1, 39))),
token.Assign,
p(1, 37)),
assignStmt(
exprs(ident("a", p(1, 42))),
exprs(intLit(4, p(1, 46))),
token.Assign,
p(1, 44))),
p(1, 1)))
})
expect(t, `
if a == 5 {
b = 3
c = 1
} else if d == 3 {
e = 8
f = 3
} else {
g = 2
h = 4
}`, func(p pfn) []ast.Stmt {
return stmts(
ifStmt(
nil,
binaryExpr(
ident("a", p(2, 4)),
intLit(5, p(2, 9)),
token.Equal,
p(2, 6)),
blockStmt(
p(2, 11), p(5, 1),
assignStmt(
exprs(ident("b", p(3, 2))),
exprs(intLit(3, p(3, 6))),
token.Assign,
p(3, 4)),
assignStmt(
exprs(ident("c", p(4, 2))),
exprs(intLit(1, p(4, 6))),
token.Assign,
p(4, 4))),
ifStmt(
nil,
binaryExpr(
ident("d", p(5, 11)),
intLit(3, p(5, 16)),
token.Equal,
p(5, 13)),
blockStmt(
p(5, 18), p(8, 1),
assignStmt(
exprs(ident("e", p(6, 2))),
exprs(intLit(8, p(6, 6))),
token.Assign,
p(6, 4)),
assignStmt(
exprs(ident("f", p(7, 2))),
exprs(intLit(3, p(7, 6))),
token.Assign,
p(7, 4))),
blockStmt(
p(8, 8), p(11, 1),
assignStmt(
exprs(ident("g", p(9, 2))),
exprs(intLit(2, p(9, 6))),
token.Assign,
p(9, 4)),
assignStmt(
exprs(ident("h", p(10, 2))),
exprs(intLit(4, p(10, 6))),
token.Assign,
p(10, 4))),
p(5, 8)),
p(2, 1)))
})
expect(t, "if a := 3; a < b {}", func(p pfn) []ast.Stmt {
return stmts(
ifStmt(
assignStmt(
exprs(ident("a", p(1, 4))),
exprs(intLit(3, p(1, 9))),
token.Define, p(1, 6)),
binaryExpr(
ident("a", p(1, 12)),
ident("b", p(1, 16)),
token.Less, p(1, 14)),
blockStmt(
p(1, 18), p(1, 19)),
nil,
p(1, 1)))
})
expect(t, "if a++; a < b {}", func(p pfn) []ast.Stmt {
return stmts(
ifStmt(
incDecStmt(ident("a", p(1, 4)), token.Inc, p(1, 5)),
binaryExpr(
ident("a", p(1, 9)),
ident("b", p(1, 13)),
token.Less, p(1, 11)),
blockStmt(
p(1, 15), p(1, 16)),
nil,
p(1, 1)))
})
expectError(t, `if {}`)
expectError(t, `if a == b { } else a != b { }`)
expectError(t, `if a == b { } else if { }`)
expectError(t, `else { }`)
expectError(t, `if ; {}`)
expectError(t, `if a := 3; {}`)
expectError(t, `if ; a < 3 {}`)
}

View file

@ -0,0 +1,98 @@
package parser_test
import (
"testing"
"github.com/d5/tengo/ast"
"github.com/d5/tengo/token"
)
func TestIndex(t *testing.T) {
expect(t, "[1, 2, 3][1]", func(p pfn) []ast.Stmt {
return stmts(
exprStmt(
indexExpr(
arrayLit(p(1, 1), p(1, 9),
intLit(1, p(1, 2)),
intLit(2, p(1, 5)),
intLit(3, p(1, 8))),
intLit(1, p(1, 11)),
p(1, 10), p(1, 12))))
})
expect(t, "[1, 2, 3][5 - a]", func(p pfn) []ast.Stmt {
return stmts(
exprStmt(
indexExpr(
arrayLit(p(1, 1), p(1, 9),
intLit(1, p(1, 2)),
intLit(2, p(1, 5)),
intLit(3, p(1, 8))),
binaryExpr(
intLit(5, p(1, 11)),
ident("a", p(1, 15)),
token.Sub,
p(1, 13)),
p(1, 10), p(1, 16))))
})
expect(t, "[1, 2, 3][5 : a]", func(p pfn) []ast.Stmt {
return stmts(
exprStmt(
sliceExpr(
arrayLit(p(1, 1), p(1, 9),
intLit(1, p(1, 2)),
intLit(2, p(1, 5)),
intLit(3, p(1, 8))),
intLit(5, p(1, 11)),
ident("a", p(1, 15)),
p(1, 10), p(1, 16))))
})
expect(t, "[1, 2, 3][a + 3 : b - 8]", func(p pfn) []ast.Stmt {
return stmts(
exprStmt(
sliceExpr(
arrayLit(p(1, 1), p(1, 9),
intLit(1, p(1, 2)),
intLit(2, p(1, 5)),
intLit(3, p(1, 8))),
binaryExpr(
ident("a", p(1, 11)),
intLit(3, p(1, 15)),
token.Add,
p(1, 13)),
binaryExpr(
ident("b", p(1, 19)),
intLit(8, p(1, 23)),
token.Sub,
p(1, 21)),
p(1, 10), p(1, 24))))
})
expect(t, `{a: 1, b: 2}["b"]`, func(p pfn) []ast.Stmt {
return stmts(
exprStmt(
indexExpr(
mapLit(p(1, 1), p(1, 12),
mapElementLit("a", p(1, 2), p(1, 3), intLit(1, p(1, 5))),
mapElementLit("b", p(1, 8), p(1, 9), intLit(2, p(1, 11)))),
stringLit("b", p(1, 14)),
p(1, 13), p(1, 17))))
})
expect(t, `{a: 1, b: 2}[a + b]`, func(p pfn) []ast.Stmt {
return stmts(
exprStmt(
indexExpr(
mapLit(p(1, 1), p(1, 12),
mapElementLit("a", p(1, 2), p(1, 3), intLit(1, p(1, 5))),
mapElementLit("b", p(1, 8), p(1, 9), intLit(2, p(1, 11)))),
binaryExpr(
ident("a", p(1, 14)),
ident("b", p(1, 18)),
token.Add,
p(1, 16)),
p(1, 13), p(1, 19))))
})
}

Some files were not shown because too many files have changed in this diff Show more