From 3f55a6b5b058b7839ea18c62f1dda8d93cafe21e Mon Sep 17 00:00:00 2001 From: Daniel Kang Date: Thu, 17 Jan 2019 01:56:05 -0800 Subject: [PATCH] Modules (#5) Implement module imports --- README.md | 29 ++- compiler/ast/array_lit.go | 6 +- compiler/ast/bool_lit.go | 2 +- compiler/ast/for_stmt.go | 2 +- compiler/ast/func_type.go | 2 +- compiler/ast/ident.go | 2 +- compiler/ast/ident_list.go | 2 +- compiler/ast/import_expr.go | 29 +++ compiler/ast/int_lit.go | 2 +- compiler/ast/map_lit.go | 6 +- compiler/compiler.go | 92 ++++++++-- compiler/compiler_module.go | 116 ++++++++++++ compiler/compiler_test.go | 22 ++- compiler/module_loader.go | 4 + compiler/opcodes.go | 3 + compiler/parser/parser.go | 50 ++++- compiler/parser/parser_error_test.go | 2 +- compiler/parser/parser_import_test.go | 46 +++++ compiler/parser/parser_test.go | 8 + compiler/stdmods/func_typedefs.go | 243 +++++++++++++++++++++++++ compiler/stdmods/func_typedefs_test.go | 124 +++++++++++++ compiler/stdmods/math.go | 84 +++++++++ compiler/stdmods/stdmods.go | 8 + compiler/token/tokens.go | 2 + objects/builtin_append.go | 2 +- objects/builtin_convert.go | 11 +- objects/builtin_copy.go | 5 +- objects/builtin_function.go | 2 +- objects/builtin_len.go | 2 +- objects/builtins.go | 5 +- objects/callable.go | 2 +- objects/callable_func.go | 4 + objects/compiled_module.go | 50 +++++ objects/errors.go | 3 + objects/module_map.go | 77 ++++++++ objects/module_map_iterator.go | 76 ++++++++ objects/string.go | 18 +- objects/user_function.go | 46 +++++ runtime/vm.go | 60 +++++- runtime/vm_for_in_test.go | 20 +- runtime/vm_inc_dec_test.go | 11 +- runtime/vm_module_test.go | 82 +++++++++ runtime/vm_string_test.go | 17 ++ runtime/vm_test.go | 29 ++- 44 files changed, 1316 insertions(+), 92 deletions(-) create mode 100644 compiler/ast/import_expr.go create mode 100644 compiler/compiler_module.go create mode 100644 compiler/module_loader.go create mode 100644 compiler/parser/parser_import_test.go create mode 100644 compiler/stdmods/func_typedefs.go create mode 100644 compiler/stdmods/func_typedefs_test.go create mode 100644 compiler/stdmods/math.go create mode 100644 compiler/stdmods/stdmods.go create mode 100644 objects/callable_func.go create mode 100644 objects/compiled_module.go create mode 100644 objects/module_map.go create mode 100644 objects/module_map_iterator.go create mode 100644 objects/user_function.go create mode 100644 runtime/vm_module_test.go diff --git a/README.md b/README.md index 5350318..848bf65 100644 --- a/README.md +++ b/README.md @@ -186,6 +186,26 @@ if is_error(err1) { // 'is_error' builtin function ``` > [Run in Playground](https://tengolang.com/?s=5eaba4289c9d284d97704dd09cb15f4f03ad05c1) +You can load other scripts as import modules using `import` expression. + +Main script: +```golang +mod1 := import("./mod1") // assuming mod1.tengo file exists in the current directory + // same as 'import("./mod1.tengo")' or 'import("mod1")' +mod1.func1(a) // module function +a += mod1.foo // module variable +//mod1.foo = 5 // error: module variables are read-only +``` + +`mod1.tengo` file: + +```golang +func1 := func(x) { print(x) } +foo := 2 +``` + +Basically `import` expression returns all the global variables defined in the module as a Map-like value. One can access the functions or variables defined in the module using `.` selector or `["key"]` indexer, but, module variables are immutable. + ## Embedding Tengo in Go @@ -245,9 +265,9 @@ func main() { } ``` -In the example above, a variable `b` is defined by the user before compiliation using `Script.Add()` function. Then a compiled bytecode `c` is used to execute the bytecode and get the value of global variables. In thie example, the value of global variable `a` is read using `Compiled.Get()` function. +In the example above, a variable `b` is defined by the user before compiliation using `Script.Add()` function. Then a compiled bytecode `c` is used to execute the bytecode and get the value of global variables. In this example, the value of global variable `a` is read using `Compiled.Get()` function. -If you need the custom data types (outside Tengo's primitive types), you can define your own `struct` that implements `objects.Object` interface _(and optinoally `objects.Callable` if you want to make function-like invokable objects)_. +If you need the custom data types (outside Tengo's primitive types), you can define your own `struct` that implements `objects.Object` interface _(and optionally `objects.Callable` if you want to make function-like invokable objects)_. ```golang import ( @@ -380,10 +400,9 @@ tengo Development roadmap for Tengo: -- Module system _(or packages)_ -- Standard libraries +- Standard libraries _(modules)_ - Better documentations -- More language constructs such as error handling, object methods, switch-case statements +- More language constructs such as destructuring assignment, `this` binding for object methods, switch-case statements - Native executables compilation - Performance improvements - Syntax highlighter for IDEs diff --git a/compiler/ast/array_lit.go b/compiler/ast/array_lit.go index f98d69d..9fb4ed6 100644 --- a/compiler/ast/array_lit.go +++ b/compiler/ast/array_lit.go @@ -26,10 +26,10 @@ func (e *ArrayLit) End() source.Pos { } func (e *ArrayLit) String() string { - var elts []string + var elements []string for _, m := range e.Elements { - elts = append(elts, m.String()) + elements = append(elements, m.String()) } - return "[" + strings.Join(elts, ", ") + "]" + return "[" + strings.Join(elements, ", ") + "]" } diff --git a/compiler/ast/bool_lit.go b/compiler/ast/bool_lit.go index c3dfbf8..e667a5c 100644 --- a/compiler/ast/bool_lit.go +++ b/compiler/ast/bool_lit.go @@ -2,7 +2,7 @@ package ast import "github.com/d5/tengo/compiler/source" -// BoolLit represetns a boolean literal. +// BoolLit represents a boolean literal. type BoolLit struct { Value bool ValuePos source.Pos diff --git a/compiler/ast/for_stmt.go b/compiler/ast/for_stmt.go index c6cb5e3..4b5a0a1 100644 --- a/compiler/ast/for_stmt.go +++ b/compiler/ast/for_stmt.go @@ -2,7 +2,7 @@ package ast import "github.com/d5/tengo/compiler/source" -// ForStmt represetns a for statement. +// ForStmt represents a for statement. type ForStmt struct { ForPos source.Pos Init Stmt diff --git a/compiler/ast/func_type.go b/compiler/ast/func_type.go index ba9aa20..2afaabb 100644 --- a/compiler/ast/func_type.go +++ b/compiler/ast/func_type.go @@ -2,7 +2,7 @@ package ast import "github.com/d5/tengo/compiler/source" -// FuncType represetns a function type definition. +// FuncType represents a function type definition. type FuncType struct { FuncPos source.Pos Params *IdentList diff --git a/compiler/ast/ident.go b/compiler/ast/ident.go index 3c7a2f7..33b7ff7 100644 --- a/compiler/ast/ident.go +++ b/compiler/ast/ident.go @@ -2,7 +2,7 @@ package ast import "github.com/d5/tengo/compiler/source" -// Ident represetns an identifier. +// Ident represents an identifier. type Ident struct { Name string NamePos source.Pos diff --git a/compiler/ast/ident_list.go b/compiler/ast/ident_list.go index 604431c..ee8f7db 100644 --- a/compiler/ast/ident_list.go +++ b/compiler/ast/ident_list.go @@ -6,7 +6,7 @@ import ( "github.com/d5/tengo/compiler/source" ) -// IdentList represetns a list of identifiers. +// IdentList represents a list of identifiers. type IdentList struct { LParen source.Pos List []*Ident diff --git a/compiler/ast/import_expr.go b/compiler/ast/import_expr.go new file mode 100644 index 0000000..6eff74a --- /dev/null +++ b/compiler/ast/import_expr.go @@ -0,0 +1,29 @@ +package ast + +import ( + "github.com/d5/tengo/compiler/source" + "github.com/d5/tengo/compiler/token" +) + +// ImportExpr represents an import expression +type ImportExpr struct { + ModuleName string + Token token.Token + TokenPos source.Pos +} + +func (e *ImportExpr) exprNode() {} + +// Pos returns the position of first character belonging to the node. +func (e *ImportExpr) Pos() source.Pos { + return e.TokenPos +} + +// End returns the position of first character immediately after the node. +func (e *ImportExpr) End() source.Pos { + return source.Pos(int(e.TokenPos) + 10 + len(e.ModuleName)) // import("moduleName") +} + +func (e *ImportExpr) String() string { + return `import("` + e.ModuleName + `")"` +} diff --git a/compiler/ast/int_lit.go b/compiler/ast/int_lit.go index 3f52eb3..3e1fd98 100644 --- a/compiler/ast/int_lit.go +++ b/compiler/ast/int_lit.go @@ -2,7 +2,7 @@ package ast import "github.com/d5/tengo/compiler/source" -// IntLit represetns an integer literal. +// IntLit represents an integer literal. type IntLit struct { Value int64 ValuePos source.Pos diff --git a/compiler/ast/map_lit.go b/compiler/ast/map_lit.go index 2a3a542..a228224 100644 --- a/compiler/ast/map_lit.go +++ b/compiler/ast/map_lit.go @@ -26,10 +26,10 @@ func (e *MapLit) End() source.Pos { } func (e *MapLit) String() string { - var elts []string + var elements []string for _, m := range e.Elements { - elts = append(elts, m.String()) + elements = append(elements, m.String()) } - return "{" + strings.Join(elts, ", ") + "}" + return "{" + strings.Join(elements, ", ") + "}" } diff --git a/compiler/compiler.go b/compiler/compiler.go index 27d87b1..f78c302 100644 --- a/compiler/compiler.go +++ b/compiler/compiler.go @@ -6,20 +6,25 @@ import ( "reflect" "github.com/d5/tengo/compiler/ast" + "github.com/d5/tengo/compiler/stdmods" "github.com/d5/tengo/compiler/token" "github.com/d5/tengo/objects" ) // Compiler compiles the AST into a bytecode. type Compiler struct { - constants []objects.Object - symbolTable *SymbolTable - scopes []CompilationScope - scopeIndex int - loops []*Loop - loopIndex int - trace io.Writer - indent int + parent *Compiler + moduleName string + constants []objects.Object + symbolTable *SymbolTable + scopes []CompilationScope + scopeIndex int + moduleLoader ModuleLoader + compiledModules map[string]*objects.CompiledModule + loops []*Loop + loopIndex int + trace io.Writer + indent int } // NewCompiler creates a Compiler. @@ -37,11 +42,12 @@ func NewCompiler(symbolTable *SymbolTable, trace io.Writer) *Compiler { } return &Compiler{ - symbolTable: symbolTable, - scopes: []CompilationScope{mainScope}, - scopeIndex: 0, - loopIndex: -1, - trace: trace, + symbolTable: symbolTable, + scopes: []CompilationScope{mainScope}, + scopeIndex: 0, + loopIndex: -1, + trace: trace, + compiledModules: make(map[string]*objects.CompiledModule), } } @@ -62,11 +68,13 @@ func (c *Compiler) Compile(node ast.Node) error { return err } } + case *ast.ExprStmt: if err := c.Compile(node.Expr); err != nil { return err } c.emit(OpPop) + case *ast.IncDecStmt: op := token.AddAssign if node.Token == token.Dec { @@ -74,10 +82,12 @@ func (c *Compiler) Compile(node ast.Node) error { } return c.compileAssign([]ast.Expr{node.Expr}, []ast.Expr{&ast.IntLit{Value: 1}}, op) + case *ast.ParenExpr: if err := c.Compile(node.Expr); err != nil { return err } + case *ast.BinaryExpr: if node.Token == token.LAnd || node.Token == token.LOr { return c.compileLogical(node) @@ -149,22 +159,29 @@ func (c *Compiler) Compile(node ast.Node) error { default: return fmt.Errorf("unknown operator: %s", node.Token.String()) } + case *ast.IntLit: c.emit(OpConstant, c.addConstant(&objects.Int{Value: node.Value})) + case *ast.FloatLit: c.emit(OpConstant, c.addConstant(&objects.Float{Value: node.Value})) + case *ast.BoolLit: if node.Value { c.emit(OpTrue) } else { c.emit(OpFalse) } + case *ast.StringLit: c.emit(OpConstant, c.addConstant(&objects.String{Value: node.Value})) + case *ast.CharLit: c.emit(OpConstant, c.addConstant(&objects.Char{Value: node.Value})) + case *ast.UndefinedLit: c.emit(OpNull) + case *ast.UnaryExpr: if err := c.Compile(node.Expr); err != nil { return err @@ -182,6 +199,7 @@ func (c *Compiler) Compile(node ast.Node) error { default: return fmt.Errorf("unknown operator: %s", node.Token.String()) } + case *ast.IfStmt: // open new symbol table for the statement c.symbolTable = c.symbolTable.Fork(true) @@ -229,8 +247,10 @@ func (c *Compiler) Compile(node ast.Node) error { case *ast.ForStmt: return c.compileForStmt(node) + case *ast.ForInStmt: return c.compileForInStmt(node) + case *ast.BranchStmt: if node.Token == token.Break { curLoop := c.currentLoop() @@ -249,17 +269,19 @@ func (c *Compiler) Compile(node ast.Node) error { } else { return fmt.Errorf("unknown branch statement: %s", node.Token.String()) } + case *ast.BlockStmt: for _, stmt := range node.Stmts { if err := c.Compile(stmt); err != nil { return err } } - case *ast.AssignStmt: + case *ast.AssignStmt: if err := c.compileAssign(node.LHS, node.RHS, node.Token); err != nil { return err } + case *ast.Ident: symbol, _, ok := c.symbolTable.Resolve(node.Name) if !ok { @@ -276,6 +298,7 @@ func (c *Compiler) Compile(node ast.Node) error { case ScopeFree: c.emit(OpGetFree, symbol.Index) } + case *ast.ArrayLit: for _, elem := range node.Elements { if err := c.Compile(elem); err != nil { @@ -284,6 +307,7 @@ func (c *Compiler) Compile(node ast.Node) error { } c.emit(OpArray, len(node.Elements)) + case *ast.MapLit: for _, elt := range node.Elements { // key @@ -296,6 +320,7 @@ func (c *Compiler) Compile(node ast.Node) error { } c.emit(OpMap, len(node.Elements)*2) + case *ast.SelectorExpr: // selector on RHS side if err := c.Compile(node.Expr); err != nil { return err @@ -306,6 +331,7 @@ func (c *Compiler) Compile(node ast.Node) error { } c.emit(OpIndex) + case *ast.IndexExpr: if err := c.Compile(node.Expr); err != nil { return err @@ -316,6 +342,7 @@ func (c *Compiler) Compile(node ast.Node) error { } c.emit(OpIndex) + case *ast.SliceExpr: if err := c.Compile(node.Expr); err != nil { return err @@ -338,6 +365,7 @@ func (c *Compiler) Compile(node ast.Node) error { } c.emit(OpSliceIndex) + case *ast.FuncLit: c.enterScope() @@ -378,6 +406,7 @@ func (c *Compiler) Compile(node ast.Node) error { } else { c.emit(OpConstant, c.addConstant(compiledFunction)) } + case *ast.ReturnStmt: if c.symbolTable.Parent(true) == nil { // outside the function @@ -396,6 +425,7 @@ func (c *Compiler) Compile(node ast.Node) error { default: return fmt.Errorf("multi-value return not implemented") } + case *ast.CallExpr: if err := c.Compile(node.Func); err != nil { return err @@ -409,6 +439,21 @@ func (c *Compiler) Compile(node ast.Node) error { c.emit(OpCall, len(node.Args)) + case *ast.ImportExpr: + stdMod, ok := stdmods.Modules[node.ModuleName] + if ok { + // standard modules contain only globals with no code. + // so no need to compile anything + c.emit(OpConstant, c.addConstant(stdMod)) + } else { + userMod, err := c.compileModule(node.ModuleName) + if err != nil { + return err + } + + c.emit(OpModule, c.addConstant(userMod)) + } + case *ast.ErrorExpr: if err := c.Compile(node.Expr); err != nil { return err @@ -428,7 +473,26 @@ func (c *Compiler) Bytecode() *Bytecode { } } +// SetModuleLoader sets or replaces the current module loader. +func (c *Compiler) SetModuleLoader(moduleLoader ModuleLoader) { + c.moduleLoader = moduleLoader +} + +func (c *Compiler) fork(moduleName string, symbolTable *SymbolTable) *Compiler { + child := NewCompiler(symbolTable, c.trace) + child.moduleName = moduleName // name of the module to compile + child.parent = c // parent to set to current compiler + child.moduleLoader = c.moduleLoader // share module loader + + return child +} + func (c *Compiler) addConstant(o objects.Object) int { + if c.parent != nil { + // module compilers will use their parent's constants array + return c.parent.addConstant(o) + } + c.constants = append(c.constants, o) if c.trace != nil { diff --git a/compiler/compiler_module.go b/compiler/compiler_module.go new file mode 100644 index 0000000..eda394e --- /dev/null +++ b/compiler/compiler_module.go @@ -0,0 +1,116 @@ +package compiler + +import ( + "fmt" + "io/ioutil" + "strings" + + "github.com/d5/tengo/compiler/parser" + "github.com/d5/tengo/compiler/source" + "github.com/d5/tengo/objects" +) + +var ( + fileSet = source.NewFileSet() +) + +func (c *Compiler) compileModule(moduleName string) (*objects.CompiledModule, error) { + compiledModule, exists := c.loadCompiledModule(moduleName) + if exists { + return compiledModule, nil + } + + // read module source from loader + var moduleSrc []byte + if c.moduleLoader == nil { + // default loader: read from local file + if !strings.HasSuffix(moduleName, ".tengo") { + moduleName += ".tengo" + } + + if err := c.checkCyclicImports(moduleName); err != nil { + return nil, err + } + + var err error + moduleSrc, err = ioutil.ReadFile(moduleName) + if err != nil { + return nil, err + } + } else { + if err := c.checkCyclicImports(moduleName); err != nil { + return nil, err + } + + var err error + moduleSrc, err = c.moduleLoader(moduleName) + if err != nil { + return nil, err + } + } + + compiledModule, err := c.doCompileModule(moduleName, moduleSrc) + if err != nil { + return nil, err + } + + c.storeCompiledModule(moduleName, compiledModule) + + return compiledModule, nil +} + +func (c *Compiler) checkCyclicImports(moduleName string) error { + if c.moduleName == moduleName { + return fmt.Errorf("cyclic module import: %s", moduleName) + } else if c.parent != nil { + return c.parent.checkCyclicImports(moduleName) + } + + return nil +} + +func (c *Compiler) doCompileModule(moduleName string, src []byte) (*objects.CompiledModule, error) { + p := parser.NewParser(fileSet.AddFile(moduleName, -1, len(src)), src, nil) + file, err := p.ParseFile() + if err != nil { + return nil, err + } + + symbolTable := NewSymbolTable() + globals := make(map[string]int) + + moduleCompiler := c.fork(moduleName, symbolTable) + if err := moduleCompiler.Compile(file); err != nil { + return nil, err + } + + for _, name := range symbolTable.Names() { + symbol, _, _ := symbolTable.Resolve(name) + if symbol.Scope == ScopeGlobal { + globals[name] = symbol.Index + } + } + + return &objects.CompiledModule{ + Instructions: moduleCompiler.Bytecode().Instructions, + Globals: globals, + }, nil +} + +func (c *Compiler) loadCompiledModule(moduleName string) (mod *objects.CompiledModule, ok bool) { + if c.parent != nil { + return c.parent.loadCompiledModule(moduleName) + } + + mod, ok = c.compiledModules[moduleName] + + return +} + +func (c *Compiler) storeCompiledModule(moduleName string, module *objects.CompiledModule) { + if c.parent != nil { + c.parent.storeCompiledModule(moduleName, module) + } + + c.compiledModules[moduleName] = module +} diff --git a/compiler/compiler_test.go b/compiler/compiler_test.go index 6ed40ec..c63b053 100644 --- a/compiler/compiler_test.go +++ b/compiler/compiler_test.go @@ -875,11 +875,13 @@ func() { intObject(0), intObject(1), intObject(1)))) + + expectError(t, `import("user1")`) // unknown module name } -func concat(insts ...[]byte) []byte { +func concat(instructions ...[]byte) []byte { concat := make([]byte, 0) - for _, i := range insts { + for _, i := range instructions { concat = append(concat, i...) } @@ -913,6 +915,22 @@ func expect(t *testing.T, input string, expected *compiler.Bytecode) (ok bool) { return } +func expectError(t *testing.T, input string) (ok bool) { + _, trace, err := traceCompile(input, nil) + + defer func() { + if !ok { + for _, tr := range trace { + t.Log(tr) + } + } + }() + + ok = assert.Error(t, err) + + return +} + func equalBytecode(t *testing.T, expected, actual *compiler.Bytecode) bool { expectedInstructions := strings.Join(compiler.FormatInstructions(expected.Instructions, 0), "\n") actualInstructions := strings.Join(compiler.FormatInstructions(actual.Instructions, 0), "\n") diff --git a/compiler/module_loader.go b/compiler/module_loader.go new file mode 100644 index 0000000..b050474 --- /dev/null +++ b/compiler/module_loader.go @@ -0,0 +1,4 @@ +package compiler + +// ModuleLoader should take a module name and return the module data. +type ModuleLoader func(moduleName string) ([]byte, error) diff --git a/compiler/opcodes.go b/compiler/opcodes.go index 19c66d9..ec92f02 100644 --- a/compiler/opcodes.go +++ b/compiler/opcodes.go @@ -56,6 +56,7 @@ const ( OpIteratorNext // Iterator next OpIteratorKey // Iterator key OpIteratorValue // Iterator value + OpModule // Module ) // OpcodeNames is opcode names. @@ -111,6 +112,7 @@ var OpcodeNames = [...]string{ OpIteratorNext: "ITNXT", OpIteratorKey: "ITKEY", OpIteratorValue: "ITVAL", + OpModule: "MODULE", } // OpcodeOperands is the number of operands. @@ -166,6 +168,7 @@ var OpcodeOperands = [...][]int{ OpIteratorNext: {}, OpIteratorKey: {}, OpIteratorValue: {}, + OpModule: {2}, } // ReadOperands reads operands from the bytecode. diff --git a/compiler/parser/parser.go b/compiler/parser/parser.go index 2d8cff9..ae19d12 100644 --- a/compiler/parser/parser.go +++ b/compiler/parser/parser.go @@ -334,6 +334,9 @@ func (p *Parser) parseOperand() ast.Expr { p.next() return x + case token.Import: + return p.parseImportExpr() + case token.LParen: lparen := p.pos p.next() @@ -366,6 +369,35 @@ func (p *Parser) parseOperand() ast.Expr { return &ast.BadExpr{From: pos, To: p.pos} } +func (p *Parser) parseImportExpr() ast.Expr { + pos := p.pos + + p.next() + + p.expect(token.LParen) + + if p.token != token.String { + p.errorExpected(p.pos, "module name") + p.advance(stmtStart) + return &ast.BadExpr{From: pos, To: p.pos} + } + + // module name + moduleName, _ := strconv.Unquote(p.tokenLit) + + expr := &ast.ImportExpr{ + ModuleName: moduleName, + Token: token.Import, + TokenPos: pos, + } + + p.next() + + p.expect(token.RParen) + + return expr +} + func (p *Parser) parseCharLit() ast.Expr { if n := len(p.tokenLit); n >= 3 { if code, _, _, err := strconv.UnquoteChar(p.tokenLit[1:n-1], '\''); err == nil { @@ -413,9 +445,9 @@ func (p *Parser) parseArrayLit() ast.Expr { lbrack := p.expect(token.LBrack) p.exprLevel++ - var elts []ast.Expr + var elements []ast.Expr for p.token != token.RBrack && p.token != token.EOF { - elts = append(elts, p.parseExpr()) + elements = append(elements, p.parseExpr()) if !p.expectComma(token.RBrack, "array element") { break @@ -426,7 +458,7 @@ func (p *Parser) parseArrayLit() ast.Expr { rbrack := p.expect(token.RBrack) return &ast.ArrayLit{ - Elements: elts, + Elements: elements, LBrack: lbrack, RBrack: rbrack, } @@ -540,9 +572,9 @@ func (p *Parser) parseStmt() (stmt ast.Stmt) { switch p.token { case // simple statements - token.Func, token.Error, token.Ident, token.Int, token.Float, token.Char, token.String, token.True, token.False, token.Undefined, token.LParen, // operands - token.LBrace, token.LBrack, // composite types - token.Add, token.Sub, token.Mul, token.And, token.Xor, token.Not: // unary operators + token.Func, token.Error, token.Ident, token.Int, token.Float, token.Char, token.String, token.True, token.False, + token.Undefined, token.Import, token.LParen, token.LBrace, token.LBrack, + token.Add, token.Sub, token.Mul, token.And, token.Xor, token.Not: s := p.parseSimpleStmt(false) p.expectSemi() return s @@ -926,9 +958,9 @@ func (p *Parser) parseMapLit() *ast.MapLit { lbrace := p.expect(token.LBrace) p.exprLevel++ - var elts []*ast.MapElementLit + var elements []*ast.MapElementLit for p.token != token.RBrace && p.token != token.EOF { - elts = append(elts, p.parseMapElementLit()) + elements = append(elements, p.parseMapElementLit()) if !p.expectComma(token.RBrace, "map element") { break @@ -941,7 +973,7 @@ func (p *Parser) parseMapLit() *ast.MapLit { return &ast.MapLit{ LBrace: lbrace, RBrace: rbrace, - Elements: elts, + Elements: elements, } } diff --git a/compiler/parser/parser_error_test.go b/compiler/parser/parser_error_test.go index a9bbb1d..0375e14 100644 --- a/compiler/parser/parser_error_test.go +++ b/compiler/parser/parser_error_test.go @@ -7,7 +7,7 @@ import ( "github.com/d5/tengo/compiler/token" ) -func TestImport(t *testing.T) { +func TestError(t *testing.T) { expect(t, `error(1234)`, func(p pfn) []ast.Stmt { return stmts( exprStmt( diff --git a/compiler/parser/parser_import_test.go b/compiler/parser/parser_import_test.go new file mode 100644 index 0000000..c263350 --- /dev/null +++ b/compiler/parser/parser_import_test.go @@ -0,0 +1,46 @@ +package parser_test + +import ( + "testing" + + "github.com/d5/tengo/compiler/ast" + "github.com/d5/tengo/compiler/token" +) + +func TestImport(t *testing.T) { + expect(t, `a := import("mod1")`, func(p pfn) []ast.Stmt { + return stmts( + assignStmt( + exprs(ident("a", p(1, 1))), + exprs(importExpr("mod1", p(1, 6))), + token.Define, p(1, 3))) + }) + + expect(t, `import("mod1").var1`, func(p pfn) []ast.Stmt { + return stmts( + exprStmt( + selectorExpr( + importExpr("mod1", p(1, 1)), + stringLit("var1", p(1, 16))))) + }) + + expect(t, `import("mod1").func1()`, func(p pfn) []ast.Stmt { + return stmts( + exprStmt( + callExpr( + selectorExpr( + importExpr("mod1", p(1, 1)), + stringLit("func1", p(1, 16))), + p(1, 21), p(1, 22)))) + }) + + expect(t, `for x, y in import("mod1") {}`, func(p pfn) []ast.Stmt { + return stmts( + forInStmt( + ident("x", p(1, 5)), + ident("y", p(1, 8)), + importExpr("mod1", p(1, 13)), + blockStmt(p(1, 28), p(1, 29)), + p(1, 1))) + }) +} diff --git a/compiler/parser/parser_test.go b/compiler/parser/parser_test.go index c2069f2..e39cbcc 100644 --- a/compiler/parser/parser_test.go +++ b/compiler/parser/parser_test.go @@ -188,6 +188,10 @@ func unaryExpr(x ast.Expr, op token.Token, pos source.Pos) *ast.UnaryExpr { return &ast.UnaryExpr{Expr: x, Token: op, TokenPos: pos} } +func importExpr(moduleName string, pos source.Pos) *ast.ImportExpr { + return &ast.ImportExpr{ModuleName: moduleName, Token: token.Import, TokenPos: pos} +} + func exprs(list ...ast.Expr) []ast.Expr { return list } @@ -385,6 +389,10 @@ func equalExpr(t *testing.T, expected, actual ast.Expr) bool { case *ast.SelectorExpr: return equalExpr(t, expected.Expr, actual.(*ast.SelectorExpr).Expr) && equalExpr(t, expected.Sel, actual.(*ast.SelectorExpr).Sel) + case *ast.ImportExpr: + return assert.Equal(t, expected.ModuleName, actual.(*ast.ImportExpr).ModuleName) && + assert.Equal(t, int(expected.TokenPos), int(actual.(*ast.ImportExpr).TokenPos)) && + assert.Equal(t, expected.Token, actual.(*ast.ImportExpr).Token) case *ast.ErrorExpr: return equalExpr(t, expected.Expr, actual.(*ast.ErrorExpr).Expr) && assert.Equal(t, int(expected.ErrorPos), int(actual.(*ast.ErrorExpr).ErrorPos)) && diff --git a/compiler/stdmods/func_typedefs.go b/compiler/stdmods/func_typedefs.go new file mode 100644 index 0000000..cf71edd --- /dev/null +++ b/compiler/stdmods/func_typedefs.go @@ -0,0 +1,243 @@ +package stdmods + +import ( + "fmt" + + "github.com/d5/tengo/objects" +) + +// FuncAFRF transform a function of 'func(float64) float64' signature +// into a user function object. +func FuncAFRF(fn func(float64) float64) *objects.UserFunction { + return &objects.UserFunction{ + Value: func(args ...objects.Object) (ret objects.Object, err error) { + if len(args) != 1 { + return nil, objects.ErrWrongNumArguments + } + + switch arg := args[0].(type) { + case *objects.Int: + return &objects.Float{Value: fn(float64(arg.Value))}, nil + case *objects.Float: + return &objects.Float{Value: fn(arg.Value)}, nil + default: + return nil, fmt.Errorf("invalid argument type: %s", arg.TypeName()) + } + }, + } +} + +// FuncARF transform a function of 'func() float64' signature +// into a user function object. +func FuncARF(fn func() float64) *objects.UserFunction { + return &objects.UserFunction{ + Value: func(args ...objects.Object) (ret objects.Object, err error) { + if len(args) != 0 { + return nil, objects.ErrWrongNumArguments + } + + return &objects.Float{Value: fn()}, nil + }, + } +} + +// FuncAIRF transform a function of 'func(int) float64' signature +// into a user function object. +func FuncAIRF(fn func(int) float64) *objects.UserFunction { + return &objects.UserFunction{ + Value: func(args ...objects.Object) (ret objects.Object, err error) { + if len(args) != 1 { + return nil, objects.ErrWrongNumArguments + } + + switch arg := args[0].(type) { + case *objects.Int: + return &objects.Float{Value: fn(int(arg.Value))}, nil + case *objects.Float: + return &objects.Float{Value: fn(int(arg.Value))}, nil + default: + return nil, fmt.Errorf("invalid argument type: %s", arg.TypeName()) + } + }, + } +} + +// FuncAFRI transform a function of 'func(float64) int' signature +// into a user function object. +func FuncAFRI(fn func(float64) int) *objects.UserFunction { + return &objects.UserFunction{ + Value: func(args ...objects.Object) (ret objects.Object, err error) { + if len(args) != 1 { + return nil, objects.ErrWrongNumArguments + } + + switch arg := args[0].(type) { + case *objects.Int: + return &objects.Int{Value: int64(fn(float64(arg.Value)))}, nil + case *objects.Float: + return &objects.Int{Value: int64(fn(arg.Value))}, nil + default: + return nil, fmt.Errorf("invalid argument type: %s", arg.TypeName()) + } + }, + } +} + +// FuncAFFRF transform a function of 'func(float64, float64) float64' signature +// into a user function object. +func FuncAFFRF(fn func(float64, float64) float64) *objects.UserFunction { + return &objects.UserFunction{ + Value: func(args ...objects.Object) (ret objects.Object, err error) { + if len(args) != 2 { + return nil, objects.ErrWrongNumArguments + } + + var arg0, arg1 float64 + + switch arg := args[0].(type) { + case *objects.Int: + arg0 = float64(arg.Value) + case *objects.Float: + arg0 = arg.Value + default: + return nil, fmt.Errorf("invalid argument type: %s", arg.TypeName()) + } + switch arg := args[1].(type) { + case *objects.Int: + arg1 = float64(arg.Value) + case *objects.Float: + arg1 = arg.Value + default: + return nil, fmt.Errorf("invalid argument type: %s", arg.TypeName()) + } + + return &objects.Float{Value: fn(arg0, arg1)}, nil + }, + } +} + +// FuncAIFRF transform a function of 'func(int, float64) float64' signature +// into a user function object. +func FuncAIFRF(fn func(int, float64) float64) *objects.UserFunction { + return &objects.UserFunction{ + Value: func(args ...objects.Object) (ret objects.Object, err error) { + if len(args) != 2 { + return nil, objects.ErrWrongNumArguments + } + + var arg0 int + var arg1 float64 + + switch arg := args[0].(type) { + case *objects.Int: + arg0 = int(arg.Value) + case *objects.Float: + arg0 = int(arg.Value) + default: + return nil, fmt.Errorf("invalid argument type: %s", arg.TypeName()) + } + switch arg := args[1].(type) { + case *objects.Int: + arg1 = float64(arg.Value) + case *objects.Float: + arg1 = arg.Value + default: + return nil, fmt.Errorf("invalid argument type: %s", arg.TypeName()) + } + + return &objects.Float{Value: fn(arg0, arg1)}, nil + }, + } +} + +// FuncAFIRF transform a function of 'func(float64, int) float64' signature +// into a user function object. +func FuncAFIRF(fn func(float64, int) float64) *objects.UserFunction { + return &objects.UserFunction{ + Value: func(args ...objects.Object) (ret objects.Object, err error) { + if len(args) != 2 { + return nil, objects.ErrWrongNumArguments + } + + var arg0 float64 + var arg1 int + + switch arg := args[0].(type) { + case *objects.Int: + arg0 = float64(arg.Value) + case *objects.Float: + arg0 = float64(arg.Value) + default: + return nil, fmt.Errorf("invalid argument type: %s", arg.TypeName()) + } + switch arg := args[1].(type) { + case *objects.Int: + arg1 = int(arg.Value) + case *objects.Float: + arg1 = int(arg.Value) + default: + return nil, fmt.Errorf("invalid argument type: %s", arg.TypeName()) + } + + return &objects.Float{Value: fn(arg0, arg1)}, nil + }, + } +} + +// FuncAFIRB transform a function of 'func(float64, int) bool' signature +// into a user function object. +func FuncAFIRB(fn func(float64, int) bool) *objects.UserFunction { + return &objects.UserFunction{ + Value: func(args ...objects.Object) (ret objects.Object, err error) { + if len(args) != 2 { + return nil, objects.ErrWrongNumArguments + } + + var arg0 float64 + var arg1 int + + switch arg := args[0].(type) { + case *objects.Int: + arg0 = float64(arg.Value) + case *objects.Float: + arg0 = arg.Value + default: + return nil, fmt.Errorf("invalid argument type: %s", arg.TypeName()) + } + switch arg := args[1].(type) { + case *objects.Int: + arg1 = int(arg.Value) + case *objects.Float: + arg1 = int(arg.Value) + default: + return nil, fmt.Errorf("invalid argument type: %s", arg.TypeName()) + } + + return &objects.Bool{Value: fn(arg0, arg1)}, nil + }, + } +} + +// FuncAFRB transform a function of 'func(float64) bool' signature +// into a user function object. +func FuncAFRB(fn func(float64) bool) *objects.UserFunction { + return &objects.UserFunction{ + Value: func(args ...objects.Object) (ret objects.Object, err error) { + if len(args) != 1 { + return nil, objects.ErrWrongNumArguments + } + + var arg0 float64 + switch arg := args[0].(type) { + case *objects.Int: + arg0 = float64(arg.Value) + case *objects.Float: + arg0 = arg.Value + default: + return nil, fmt.Errorf("invalid argument type: %s", arg.TypeName()) + } + + return &objects.Bool{Value: fn(arg0)}, nil + }, + } +} diff --git a/compiler/stdmods/func_typedefs_test.go b/compiler/stdmods/func_typedefs_test.go new file mode 100644 index 0000000..c06b930 --- /dev/null +++ b/compiler/stdmods/func_typedefs_test.go @@ -0,0 +1,124 @@ +package stdmods_test + +import ( + "testing" + + "github.com/d5/tengo/assert" + "github.com/d5/tengo/compiler/stdmods" + "github.com/d5/tengo/objects" +) + +func TestFuncARF(t *testing.T) { + uf := stdmods.FuncARF(func() float64 { + return 10.0 + }) + ret, err := uf.Call() + assert.NoError(t, err) + assert.Equal(t, &objects.Float{Value: 10.0}, ret) + ret, err = uf.Call(objects.TrueValue) + assert.Error(t, err) +} + +func TestFuncAFRF(t *testing.T) { + uf := stdmods.FuncAFRF(func(a float64) float64 { + return a + }) + ret, err := uf.Call(&objects.Float{Value: 10.0}) + assert.NoError(t, err) + assert.Equal(t, &objects.Float{Value: 10.0}, ret) + ret, err = uf.Call() + assert.Error(t, err) + ret, err = uf.Call(objects.TrueValue, objects.TrueValue) + assert.Error(t, err) +} + +func TestFuncAIRF(t *testing.T) { + uf := stdmods.FuncAIRF(func(a int) float64 { + return float64(a) + }) + ret, err := uf.Call(&objects.Int{Value: 10.0}) + assert.NoError(t, err) + assert.Equal(t, &objects.Float{Value: 10.0}, ret) + ret, err = uf.Call() + assert.Error(t, err) + ret, err = uf.Call(objects.TrueValue, objects.TrueValue) + assert.Error(t, err) +} + +func TestFuncAFRI(t *testing.T) { + uf := stdmods.FuncAFRI(func(a float64) int { + return int(a) + }) + ret, err := uf.Call(&objects.Float{Value: 10.5}) + assert.NoError(t, err) + assert.Equal(t, &objects.Int{Value: 10}, ret) + ret, err = uf.Call() + assert.Error(t, err) + ret, err = uf.Call(objects.TrueValue, objects.TrueValue) + assert.Error(t, err) +} + +func TestFuncAFRB(t *testing.T) { + uf := stdmods.FuncAFRB(func(a float64) bool { + return a > 0.0 + }) + ret, err := uf.Call(&objects.Float{Value: 0.1}) + assert.NoError(t, err) + assert.Equal(t, &objects.Bool{Value: true}, ret) + ret, err = uf.Call() + assert.Error(t, err) + ret, err = uf.Call(objects.TrueValue, objects.TrueValue) + assert.Error(t, err) +} + +func TestFuncAFFRF(t *testing.T) { + uf := stdmods.FuncAFFRF(func(a, b float64) float64 { + return a + b + }) + ret, err := uf.Call(&objects.Float{Value: 10.0}, &objects.Float{Value: 20.0}) + assert.NoError(t, err) + assert.Equal(t, &objects.Float{Value: 30.0}, ret) + ret, err = uf.Call() + assert.Error(t, err) + ret, err = uf.Call(objects.TrueValue) + assert.Error(t, err) +} + +func TestFuncAIFRF(t *testing.T) { + uf := stdmods.FuncAIFRF(func(a int, b float64) float64 { + return float64(a) + b + }) + ret, err := uf.Call(&objects.Int{Value: 10}, &objects.Float{Value: 20.0}) + assert.NoError(t, err) + assert.Equal(t, &objects.Float{Value: 30.0}, ret) + ret, err = uf.Call() + assert.Error(t, err) + ret, err = uf.Call(objects.TrueValue) + assert.Error(t, err) +} + +func TestFuncAFIRF(t *testing.T) { + uf := stdmods.FuncAFIRF(func(a float64, b int) float64 { + return a + float64(b) + }) + ret, err := uf.Call(&objects.Float{Value: 10.0}, &objects.Int{Value: 20}) + assert.NoError(t, err) + assert.Equal(t, &objects.Float{Value: 30.0}, ret) + ret, err = uf.Call() + assert.Error(t, err) + ret, err = uf.Call(objects.TrueValue) + assert.Error(t, err) +} + +func TestFuncAFIRB(t *testing.T) { + uf := stdmods.FuncAFIRB(func(a float64, b int) bool { + return a < float64(b) + }) + ret, err := uf.Call(&objects.Float{Value: 10.0}, &objects.Int{Value: 20}) + assert.NoError(t, err) + assert.Equal(t, &objects.Bool{Value: true}, ret) + ret, err = uf.Call() + assert.Error(t, err) + ret, err = uf.Call(objects.TrueValue) + assert.Error(t, err) +} diff --git a/compiler/stdmods/math.go b/compiler/stdmods/math.go new file mode 100644 index 0000000..d2bb075 --- /dev/null +++ b/compiler/stdmods/math.go @@ -0,0 +1,84 @@ +package stdmods + +import ( + "math" + + "github.com/d5/tengo/objects" +) + +var mathModule = map[string]objects.Object{ + "e": &objects.Float{Value: math.E}, + "pi": &objects.Float{Value: math.Pi}, + "phi": &objects.Float{Value: math.Phi}, + "sqrt2": &objects.Float{Value: math.Sqrt2}, + "sqrtE": &objects.Float{Value: math.SqrtE}, + "sqrtPi": &objects.Float{Value: math.SqrtPi}, + "sqrtPhi": &objects.Float{Value: math.SqrtPhi}, + "ln2": &objects.Float{Value: math.Ln2}, + "log2E": &objects.Float{Value: math.Log2E}, + "ln10": &objects.Float{Value: math.Ln10}, + "log10E": &objects.Float{Value: math.Log10E}, + "abs": FuncAFRF(math.Abs), + "acos": FuncAFRF(math.Acos), + "acosh": FuncAFRF(math.Acosh), + "asin": FuncAFRF(math.Asin), + "asinh": FuncAFRF(math.Asinh), + "atan": FuncAFRF(math.Atan), + "atan2": FuncAFFRF(math.Atan2), + "atanh": FuncAFRF(math.Atanh), + "cbrt": FuncAFRF(math.Cbrt), + "ceil": FuncAFRF(math.Ceil), + "copysign": FuncAFFRF(math.Copysign), + "cos": FuncAFRF(math.Cos), + "cosh": FuncAFRF(math.Cosh), + "dim": FuncAFFRF(math.Dim), + "erf": FuncAFRF(math.Erf), + "erfc": FuncAFRF(math.Erfc), + "erfcinv": FuncAFRF(math.Erfcinv), + "erfinv": FuncAFRF(math.Erfinv), + "exp": FuncAFRF(math.Exp), + "exp2": FuncAFRF(math.Exp2), + "expm1": FuncAFRF(math.Expm1), + "floor": FuncAFRF(math.Floor), + "gamma": FuncAFRF(math.Gamma), + "hypot": FuncAFFRF(math.Hypot), + "ilogb": FuncAFRI(math.Ilogb), + "inf": FuncAIRF(math.Inf), + "is_inf": FuncAFIRB(math.IsInf), + "is_nan": FuncAFRB(math.IsNaN), + "j0": FuncAFRF(math.J0), + "j1": FuncAFRF(math.J1), + "jn": FuncAIFRF(math.Jn), + "ldexp": FuncAFIRF(math.Ldexp), + "log": FuncAFRF(math.Log), + "log10": FuncAFRF(math.Log10), + "log1p": FuncAFRF(math.Log1p), + "log2": FuncAFRF(math.Log2), + "logb": FuncAFRF(math.Logb), + "max": FuncAFFRF(math.Max), + "min": FuncAFFRF(math.Min), + "mod": FuncAFFRF(math.Mod), + "nan": FuncARF(math.NaN), + "nextafter": FuncAFFRF(math.Nextafter), + "pow": FuncAFFRF(math.Pow), + "pow10": FuncAIRF(math.Pow10), + "remainder": FuncAFFRF(math.Remainder), + "round": FuncAFRF(math.Round), + "round_to_even": FuncAFRF(math.RoundToEven), + "signbit": FuncAFRB(math.Signbit), + "sin": FuncAFRF(math.Sin), + "sinh": FuncAFRF(math.Sinh), + "sqrt": FuncAFRF(math.Sqrt), + "tan": FuncAFRF(math.Tan), + "tanh": FuncAFRF(math.Tanh), + "runct": FuncAFRF(math.Trunc), + "y0": FuncAFRF(math.Y0), + "y1": FuncAFRF(math.Y1), + "yn": FuncAIFRF(math.Yn), + // TODO: functions that have multiple returns + // Should these be tuple assignment? Or Map return? + //"frexp": nil, + //"lgamma": nil, + //"modf": nil, + //"sincos": nil, +} diff --git a/compiler/stdmods/stdmods.go b/compiler/stdmods/stdmods.go new file mode 100644 index 0000000..9cf5963 --- /dev/null +++ b/compiler/stdmods/stdmods.go @@ -0,0 +1,8 @@ +package stdmods + +import "github.com/d5/tengo/objects" + +// Modules contain the standard modules. +var Modules = map[string]*objects.ModuleMap{ + "math": {Value: mathModule}, +} diff --git a/compiler/token/tokens.go b/compiler/token/tokens.go index 7bf1e72..1721d23 100644 --- a/compiler/token/tokens.go +++ b/compiler/token/tokens.go @@ -82,6 +82,7 @@ const ( False In Undefined + Import _keywordEnd ) @@ -156,6 +157,7 @@ var tokens = [...]string{ False: "false", In: "in", Undefined: "undefined", + Import: "import", } func (tok Token) String() string { diff --git a/objects/builtin_append.go b/objects/builtin_append.go index c891ac9..c4f8c27 100644 --- a/objects/builtin_append.go +++ b/objects/builtin_append.go @@ -7,7 +7,7 @@ import ( // append(src, items...) func builtinAppend(args ...Object) (Object, error) { if len(args) < 2 { - return nil, fmt.Errorf("not enough arguments in call to append") + return nil, ErrWrongNumArguments } switch arg := args[0].(type) { diff --git a/objects/builtin_convert.go b/objects/builtin_convert.go index 8a95ee1..618691d 100644 --- a/objects/builtin_convert.go +++ b/objects/builtin_convert.go @@ -1,13 +1,12 @@ package objects import ( - "errors" "strconv" ) func builtinString(args ...Object) (Object, error) { if len(args) != 1 { - return nil, errors.New("wrong number of arguments") + return nil, ErrWrongNumArguments } switch arg := args[0].(type) { @@ -22,7 +21,7 @@ func builtinString(args ...Object) (Object, error) { func builtinInt(args ...Object) (Object, error) { if len(args) != 1 { - return nil, errors.New("wrong number of arguments") + return nil, ErrWrongNumArguments } switch arg := args[0].(type) { @@ -49,7 +48,7 @@ func builtinInt(args ...Object) (Object, error) { func builtinFloat(args ...Object) (Object, error) { if len(args) != 1 { - return nil, errors.New("wrong number of arguments") + return nil, ErrWrongNumArguments } switch arg := args[0].(type) { @@ -69,7 +68,7 @@ func builtinFloat(args ...Object) (Object, error) { func builtinBool(args ...Object) (Object, error) { if len(args) != 1 { - return nil, errors.New("wrong number of arguments") + return nil, ErrWrongNumArguments } switch arg := args[0].(type) { @@ -82,7 +81,7 @@ func builtinBool(args ...Object) (Object, error) { func builtinChar(args ...Object) (Object, error) { if len(args) != 1 { - return nil, errors.New("wrong number of arguments") + return nil, ErrWrongNumArguments } switch arg := args[0].(type) { diff --git a/objects/builtin_copy.go b/objects/builtin_copy.go index 3272ece..4b254b2 100644 --- a/objects/builtin_copy.go +++ b/objects/builtin_copy.go @@ -1,11 +1,8 @@ package objects -import "errors" - func builtinCopy(args ...Object) (Object, error) { - // TODO: should multi arguments later? if len(args) != 1 { - return nil, errors.New("wrong number of arguments") + return nil, ErrWrongNumArguments } return args[0].Copy(), nil diff --git a/objects/builtin_function.go b/objects/builtin_function.go index 5c084df..56fed40 100644 --- a/objects/builtin_function.go +++ b/objects/builtin_function.go @@ -6,7 +6,7 @@ import ( // BuiltinFunction represents a builtin function. type BuiltinFunction struct { - Value BuiltinFunc + Value CallableFunc } // TypeName returns the name of the type. diff --git a/objects/builtin_len.go b/objects/builtin_len.go index 51a2b10..26d59a9 100644 --- a/objects/builtin_len.go +++ b/objects/builtin_len.go @@ -6,7 +6,7 @@ import ( func builtinLen(args ...Object) (Object, error) { if len(args) != 1 { - return nil, fmt.Errorf("wrong number of arguments (got=%d, want=1)", len(args)) + return nil, ErrWrongNumArguments } switch arg := args[0].(type) { diff --git a/objects/builtins.go b/objects/builtins.go index a2284e8..8ec3b7f 100644 --- a/objects/builtins.go +++ b/objects/builtins.go @@ -1,12 +1,9 @@ package objects -// BuiltinFunc is a function signature for the builtin functions. -type BuiltinFunc func(args ...Object) (ret Object, err error) - // Builtins contains all known builtin functions. var Builtins = []struct { Name string - Func BuiltinFunc + Func CallableFunc }{ { Name: "print", diff --git a/objects/callable.go b/objects/callable.go index 96e3733..a066e1b 100644 --- a/objects/callable.go +++ b/objects/callable.go @@ -1,6 +1,6 @@ package objects -// Callable repesents an object that can be called like a function. +// Callable represents an object that can be called like a function. type Callable interface { // Call should take an arbitrary number of arguments // and returns a return value and/or an error, diff --git a/objects/callable_func.go b/objects/callable_func.go new file mode 100644 index 0000000..cf9b43a --- /dev/null +++ b/objects/callable_func.go @@ -0,0 +1,4 @@ +package objects + +// CallableFunc is a function signature for the callable functions. +type CallableFunc func(args ...Object) (ret Object, err error) diff --git a/objects/compiled_module.go b/objects/compiled_module.go new file mode 100644 index 0000000..d531500 --- /dev/null +++ b/objects/compiled_module.go @@ -0,0 +1,50 @@ +package objects + +import ( + "github.com/d5/tengo/compiler/token" +) + +// CompiledModule represents a compiled module. +type CompiledModule struct { + Instructions []byte // compiled instructions + Globals map[string]int // global variable name-to-index map +} + +// TypeName returns the name of the type. +func (o *CompiledModule) TypeName() string { + return "compiled-module" +} + +func (o *CompiledModule) String() string { + return "" +} + +// BinaryOp returns another object that is the result of +// a given binary operator and a right-hand side object. +func (o *CompiledModule) BinaryOp(op token.Token, rhs Object) (Object, error) { + return nil, ErrInvalidOperator +} + +// Copy returns a copy of the type. +func (o *CompiledModule) Copy() Object { + globals := make(map[string]int, len(o.Globals)) + for name, index := range o.Globals { + globals[name] = index + } + + return &CompiledModule{ + Instructions: append([]byte{}, o.Instructions...), + Globals: globals, + } +} + +// IsFalsy returns true if the value of the type is falsy. +func (o *CompiledModule) IsFalsy() bool { + return false +} + +// Equals returns true if the value of the type +// is equal to the value of another object. +func (o *CompiledModule) Equals(x Object) bool { + return false +} diff --git a/objects/errors.go b/objects/errors.go index 110a769..78d6a40 100644 --- a/objects/errors.go +++ b/objects/errors.go @@ -4,3 +4,6 @@ import "errors" // ErrInvalidOperator represents an error for invalid operator usage. var ErrInvalidOperator = errors.New("invalid operator") + +// ErrWrongNumArguments represents a wrong number of arguments error. +var ErrWrongNumArguments = errors.New("wrong number of arguments") diff --git a/objects/module_map.go b/objects/module_map.go new file mode 100644 index 0000000..baa2f46 --- /dev/null +++ b/objects/module_map.go @@ -0,0 +1,77 @@ +package objects + +import ( + "fmt" + "strings" + + "github.com/d5/tengo/compiler/token" +) + +// ModuleMap represents a module map object. +type ModuleMap struct { + Value map[string]Object +} + +// TypeName returns the name of the type. +func (o *ModuleMap) TypeName() string { + return "module" +} + +func (o *ModuleMap) String() string { + var pairs []string + for k, v := range o.Value { + pairs = append(pairs, fmt.Sprintf("%s: %s", k, v.String())) + } + + return fmt.Sprintf("{%s}", strings.Join(pairs, ", ")) +} + +// BinaryOp returns another object that is the result of +// a given binary operator and a right-hand side object. +func (o *ModuleMap) BinaryOp(op token.Token, rhs Object) (Object, error) { + return nil, ErrInvalidOperator +} + +// Copy returns a copy of the type. +func (o *ModuleMap) Copy() Object { + c := make(map[string]Object) + for k, v := range o.Value { + c[k] = v.Copy() + } + + return &ModuleMap{Value: c} +} + +// IsFalsy returns true if the value of the type is falsy. +func (o *ModuleMap) IsFalsy() bool { + return len(o.Value) == 0 +} + +// Get returns the value for the given key. +func (o *ModuleMap) Get(key string) (Object, bool) { + val, ok := o.Value[key] + + return val, ok +} + +// Equals returns true if the value of the type +// is equal to the value of another object. +func (o *ModuleMap) Equals(x Object) bool { + t, ok := x.(*ModuleMap) + if !ok { + return false + } + + if len(o.Value) != len(t.Value) { + return false + } + + for k, v := range o.Value { + tv := t.Value[k] + if !v.Equals(tv) { + return false + } + } + + return true +} diff --git a/objects/module_map_iterator.go b/objects/module_map_iterator.go new file mode 100644 index 0000000..9aa49a1 --- /dev/null +++ b/objects/module_map_iterator.go @@ -0,0 +1,76 @@ +package objects + +import "github.com/d5/tengo/compiler/token" + +// ModuleMapIterator represents an iterator for the module map. +type ModuleMapIterator struct { + v map[string]Object + k []string + i int + l int +} + +// NewModuleMapIterator creates a module iterator. +func NewModuleMapIterator(v *ModuleMap) Iterator { + var keys []string + for k := range v.Value { + keys = append(keys, k) + } + + return &ModuleMapIterator{ + v: v.Value, + k: keys, + l: len(keys), + } +} + +// TypeName returns the name of the type. +func (i *ModuleMapIterator) TypeName() string { + return "module-iterator" +} + +func (i *ModuleMapIterator) String() string { + return "" +} + +// BinaryOp returns another object that is the result of +// a given binary operator and a right-hand side object. +func (i *ModuleMapIterator) BinaryOp(op token.Token, rhs Object) (Object, error) { + return nil, ErrInvalidOperator +} + +// IsFalsy returns true if the value of the type is falsy. +func (i *ModuleMapIterator) IsFalsy() bool { + return true +} + +// Equals returns true if the value of the type +// is equal to the value of another object. +func (i *ModuleMapIterator) Equals(Object) bool { + return false +} + +// Copy returns a copy of the type. +func (i *ModuleMapIterator) Copy() Object { + return &ModuleMapIterator{v: i.v, k: i.k, i: i.i, l: i.l} +} + +// Next returns true if there are more elements to iterate. +func (i *ModuleMapIterator) Next() bool { + i.i++ + return i.i <= i.l +} + +// Key returns the key or index value of the current element. +func (i *ModuleMapIterator) Key() Object { + k := i.k[i.i-1] + + return &String{Value: k} +} + +// Value returns the value of the current element. +func (i *ModuleMapIterator) Value() Object { + k := i.k[i.i-1] + + return i.v[k] +} diff --git a/objects/string.go b/objects/string.go index deca219..c95de59 100644 --- a/objects/string.go +++ b/objects/string.go @@ -23,19 +23,13 @@ func (o *String) String() string { // BinaryOp returns another object that is the result of // a given binary operator and a right-hand side object. func (o *String) BinaryOp(op token.Token, rhs Object) (Object, error) { - switch rhs := rhs.(type) { - case *String: - switch op { - case token.Add: - if rhs.Value == "" { - return o, nil - } + switch op { + case token.Add: + switch rhs := rhs.(type) { + case *String: return &String{Value: o.Value + rhs.Value}, nil - } - case *Char: - switch op { - case token.Add: - return &String{Value: o.Value + string(rhs.Value)}, nil + default: + return &String{Value: o.Value + rhs.String()}, nil } } diff --git a/objects/user_function.go b/objects/user_function.go new file mode 100644 index 0000000..c83866a --- /dev/null +++ b/objects/user_function.go @@ -0,0 +1,46 @@ +package objects + +import ( + "github.com/d5/tengo/compiler/token" +) + +// UserFunction represents a user function. +type UserFunction struct { + Value CallableFunc +} + +// TypeName returns the name of the type. +func (o *UserFunction) TypeName() string { + return "user-function" +} + +func (o *UserFunction) String() string { + return "" +} + +// BinaryOp returns another object that is the result of +// a given binary operator and a right-hand side object. +func (o *UserFunction) BinaryOp(op token.Token, rhs Object) (Object, error) { + return nil, ErrInvalidOperator +} + +// Copy returns a copy of the type. +func (o *UserFunction) Copy() Object { + return &UserFunction{Value: o.Value} +} + +// IsFalsy returns true if the value of the type is falsy. +func (o *UserFunction) IsFalsy() bool { + return false +} + +// Equals returns true if the value of the type +// is equal to the value of another object. +func (o *UserFunction) Equals(x Object) bool { + return false +} + +// Call executes a builtin function. +func (o *UserFunction) Call(args ...Object) (Object, error) { + return o.Value(args...) +} diff --git a/runtime/vm.go b/runtime/vm.go index d0c2297..74e5bfe 100644 --- a/runtime/vm.go +++ b/runtime/vm.go @@ -625,7 +625,26 @@ func (v *VM) Run() error { case *objects.Map: key, ok := (*index).(*objects.String) if !ok { - return fmt.Errorf("non-string map key: %s", left.TypeName()) + return fmt.Errorf("non-string key: %s", left.TypeName()) + } + + var res = objects.UndefinedValue + val, ok := left.Value[key.Value] + if ok { + res = val + } + + if v.sp >= StackSize { + return ErrStackOverflow + } + + v.stack[v.sp] = &res + v.sp++ + + case *objects.ModuleMap: + key, ok := (*index).(*objects.String) + if !ok { + return fmt.Errorf("non-string key: %s", left.TypeName()) } var res = objects.UndefinedValue @@ -986,6 +1005,8 @@ func (v *VM) Run() error { iterator = objects.NewArrayIterator(dst) case *objects.Map: iterator = objects.NewMapIterator(dst) + case *objects.ModuleMap: + iterator = objects.NewModuleMapIterator(dst) case *objects.String: iterator = objects.NewStringIterator(dst) default: @@ -1041,6 +1062,14 @@ func (v *VM) Run() error { v.stack[v.sp] = &val v.sp++ + case compiler.OpModule: + cidx := compiler.ReadUint16(v.curInsts[ip+1:]) + v.curFrame.ip += 2 + + if err := v.importModule(v.constants[cidx].(*objects.CompiledModule)); err != nil { + return err + } + default: return fmt.Errorf("unknown opcode: %d", v.curInsts[ip]) } @@ -1184,6 +1213,35 @@ func (v *VM) callFunction(fn *objects.CompiledFunction, freeVars []*objects.Obje return nil } +// TODO: should reuse *objects.ModuleMap for the same imports? +func (v *VM) importModule(compiledModule *objects.CompiledModule) error { + // import module is basically to create a new instance of VM + // and run the module code and retrieve all global variables after execution. + moduleVM := NewVM(&compiler.Bytecode{ + Instructions: compiledModule.Instructions, + Constants: v.constants, + }, nil) + if err := moduleVM.Run(); err != nil { + return err + } + + mmValue := make(map[string]objects.Object) + for name, index := range compiledModule.Globals { + mmValue[name] = *moduleVM.globals[index] + } + + var mm objects.Object = &objects.ModuleMap{Value: mmValue} + + if v.sp >= StackSize { + return ErrStackOverflow + } + + v.stack[v.sp] = &mm + v.sp++ + + return nil +} + func selectorAssign(dst, src *objects.Object, selectors []interface{}) error { numSel := len(selectors) diff --git a/runtime/vm_for_in_test.go b/runtime/vm_for_in_test.go index 4bca8b2..cb69f81 100644 --- a/runtime/vm_for_in_test.go +++ b/runtime/vm_for_in_test.go @@ -5,15 +5,21 @@ import ( ) func TestForIn(t *testing.T) { - expect(t, `for i, x in [1, 2, 3] { out += i + x }`, 9) - expect(t, `func() { for i, x in [1, 2, 3] { out += i + x } }()`, 9) + // array + expect(t, `for x in [1, 2, 3] { out += x }`, 6) // value + expect(t, `for i, x in [1, 2, 3] { out += i + x }`, 9) // index, value + expect(t, `func() { for i, x in [1, 2, 3] { out += i + x } }()`, 9) // index, value + expect(t, `for i, _ in [1, 2, 3] { out += i }`, 3) // index, _ + expect(t, `func() { for i, _ in [1, 2, 3] { out += i } }()`, 3) // index, _ - expect(t, `for i, _ in [1, 2, 3] { out += i }`, 3) - expect(t, `func() { for i, _ in [1, 2, 3] { out += i } }()`, 3) - - expect(t, `for k, v in {a:2,b:3,c:4} { out = k; if v==3 { break } }`, "b") - expect(t, `func() { for k, v in {a:2,b:3,c:4} { out = k; if v==3 { break } } }()`, "b") + // map + expect(t, `for v in {a:2,b:3,c:4} { out += v }`, 9) // value + expect(t, `for k, v in {a:2,b:3,c:4} { out = k; if v==3 { break } }`, "b") // key, value + expect(t, `for k, _ in {a:2} { out += k }`, "a") // key, _ + expect(t, `for _, v in {a:2,b:3,c:4} { out += v }`, 9) // _, value + expect(t, `func() { for k, v in {a:2,b:3,c:4} { out = k; if v==3 { break } } }()`, "b") // key, value + // string expect(t, `for c in "abcde" { out += c }`, "abcde") expect(t, `for i, c in "abcde" { if i == 2 { continue }; out += c }`, "abde") } diff --git a/runtime/vm_inc_dec_test.go b/runtime/vm_inc_dec_test.go index 5bf8702..c83c1bb 100644 --- a/runtime/vm_inc_dec_test.go +++ b/runtime/vm_inc_dec_test.go @@ -10,10 +10,13 @@ func TestIncDec(t *testing.T) { expect(t, `a := 0; a++; out = a`, 1) expect(t, `a := 0; a++; a--; out = a`, 0) - expectError(t, `a++`) // not declared - expectError(t, `a--`) // not declared - expectError(t, `a := "foo"; a++`) // invalid operand - //expectError(t, `a := 0; b := a++`) // inc-dec is statement not expression <- parser error + // this seems strange but it works because 'a += b' is + // translated into 'a = a + b' and string type takes other types for + operator. + expect(t, `a := "foo"; a++; out = a`, "foo1") + expectError(t, `a := "foo"; a--`) + expectError(t, `a++`) // not declared + expectError(t, `a--`) // not declared + //expectError(t, `a := 0; b := a++`) // inc-dec is statement not expression <- parser error expectError(t, `4++`) } diff --git a/runtime/vm_module_test.go b/runtime/vm_module_test.go new file mode 100644 index 0000000..5c97afc --- /dev/null +++ b/runtime/vm_module_test.go @@ -0,0 +1,82 @@ +package runtime_test + +import "testing" + +func TestModule(t *testing.T) { + // stdmods + expect(t, `math := import("math"); out = math.abs(1)`, 1.0) + expect(t, `math := import("math"); out = math.abs(-1)`, 1.0) + expect(t, `math := import("math"); out = math.abs(1.0)`, 1.0) + expect(t, `math := import("math"); out = math.abs(-1.0)`, 1.0) + + // user modules + expectWithUserModules(t, `out = import("mod1").bar()`, 5.0, map[string]string{ + "mod1": `bar := func() { return 5.0 }`, + }) + // (main) -> mod1 -> mod2 + expectWithUserModules(t, `out = import("mod1").mod2.bar()`, 5.0, map[string]string{ + "mod1": `mod2 := import("mod2")`, + "mod2": `bar := func() { return 5.0 }`, + }) + // (main) -> mod1 -> mod2 + // -> mod2 + expectWithUserModules(t, `import("mod1"); out = import("mod2").bar()`, 5.0, map[string]string{ + "mod1": `mod2 := import("mod2")`, + "mod2": `bar := func() { return 5.0 }`, + }) + // (main) -> mod1 -> mod2 -> mod3 + // -> mod2 -> mod3 + expectWithUserModules(t, `import("mod1"); out = import("mod2").mod3.bar()`, 5.0, map[string]string{ + "mod1": `mod2 := import("mod2")`, + "mod2": `mod3 := import("mod3")`, + "mod3": `bar := func() { return 5.0 }`, + }) + + // cyclic imports + // (main) -> mod1 -> mod2 -> mod1 + expectErrorWithUserModules(t, `import("mod1")`, map[string]string{ + "mod1": `import("mod2")`, + "mod2": `import("mod1")`, + }) + // (main) -> mod1 -> mod2 -> mod3 -> mod1 + expectErrorWithUserModules(t, `import("mod1")`, map[string]string{ + "mod1": `import("mod2")`, + "mod2": `import("mod3")`, + "mod3": `import("mod1")`, + }) + // (main) -> mod1 -> mod2 -> mod3 -> mod2 + expectErrorWithUserModules(t, `import("mod1")`, map[string]string{ + "mod1": `import("mod2")`, + "mod2": `import("mod3")`, + "mod3": `import("mod2")`, + }) + + // unknown modules + expectErrorWithUserModules(t, `import("mod0")`, map[string]string{ + "mod1": `a := 5`, + }) + expectErrorWithUserModules(t, `import("mod1")`, map[string]string{ + "mod1": `import("mod2")`, + }) + + // for-in + expectWithUserModules(t, `for _, n in import("mod1") { out += n }`, 6, map[string]string{ + "mod1": `a := 1; b := 2; c := 3`, + }) + expectWithUserModules(t, `for k, _ in import("mod1") { out += k }`, "a", map[string]string{ + "mod1": `a := 1`, // only 1 global variable because module map does not sort the keys + }) + + // mutating global variables inside the module does not affect exported values + expectWithUserModules(t, `m1 := import("mod1"); m1.mutate(); out = m1.a`, 3, map[string]string{ + "mod1": `a := 3; mutate := func() { a = 10 }`, + }) + + // module map is immutable + expectErrorWithUserModules(t, `m1 := import("mod1"); m1.a = 5`, map[string]string{ + "mod1": `a := 3`, + }) + expectErrorWithUserModules(t, `m1 := import("mod1"); m1.a.b = 5`, map[string]string{ + "mod1": `a := {b: 3}`, + }) +} diff --git a/runtime/vm_string_test.go b/runtime/vm_string_test.go index 112502e..d3e7985 100644 --- a/runtime/vm_string_test.go +++ b/runtime/vm_string_test.go @@ -42,5 +42,22 @@ func TestString(t *testing.T) { expectError(t, fmt.Sprintf("%s[:%d]", strStr, strLen+1)) expectError(t, fmt.Sprintf("%s[%d:%d]", strStr, 2, 1)) + // string concatenation with other types + expect(t, `out = "foo" + 1`, "foo1") + // Float.String() returns the smallest number of digits + // necessary such that ParseFloat will return f exactly. + expect(t, `out = "foo" + 1.0`, "foo1") // <- note '1' instead of '1.0' + expect(t, `out = "foo" + 1.5`, "foo1.5") + expect(t, `out = "foo" + true`, "footrue") + expect(t, `out = "foo" + 'X'`, "fooX") + expect(t, `out = "foo" + error(5)`, "fooerror: 5") + expect(t, `out = "foo" + undefined`, "foo") + expect(t, `out = "foo" + [1,2,3]`, "foo[1, 2, 3]") + //expect(t, `out = "foo" + {a: 1, b: 2}`, "foo{a: 1, b: 2}") // TODO: commented because order of key is not consistent + // also works with "+=" operator + expect(t, `out = "foo"; out += 1.5`, "foo1.5") + // string concats works only when string is LHS + expectError(t, `1 + "foo"`) + expectError(t, `"foo" - "bar"`) } diff --git a/runtime/vm_test.go b/runtime/vm_test.go index 3c05748..c0b36f2 100644 --- a/runtime/vm_test.go +++ b/runtime/vm_test.go @@ -24,6 +24,10 @@ type MAP = map[string]interface{} type ARR = []interface{} func expect(t *testing.T, input string, expected interface{}) { + expectWithUserModules(t, input, expected, nil) +} + +func expectWithUserModules(t *testing.T, input string, expected interface{}, userModules map[string]string) { // parse file := parse(t, input) if file == nil { @@ -31,10 +35,14 @@ func expect(t *testing.T, input string, expected interface{}) { } // compiler/VM - runVM(t, file, expected) + runVM(t, file, expected, userModules) } func expectError(t *testing.T, input string) { + expectErrorWithUserModules(t, input, nil) +} + +func expectErrorWithUserModules(t *testing.T, input string, userModules map[string]string) { // parse program := parse(t, input) if program == nil { @@ -42,15 +50,15 @@ func expectError(t *testing.T, input string) { } // compiler/VM - runVMError(t, program) + runVMError(t, program, userModules) } -func runVM(t *testing.T, file *ast.File, expected interface{}) (ok bool) { +func runVM(t *testing.T, file *ast.File, expected interface{}, userModules map[string]string) (ok bool) { expectedObj := toObject(expected) res, trace, err := traceCompileRun(file, map[string]objects.Object{ testOut: objectZeroCopy(expectedObj), - }) + }, userModules) defer func() { if !ok { @@ -68,8 +76,8 @@ func runVM(t *testing.T, file *ast.File, expected interface{}) (ok bool) { } // TODO: should differentiate compile-time error, runtime error, and, error object returned -func runVMError(t *testing.T, file *ast.File) (ok bool) { - _, trace, err := traceCompileRun(file, nil) +func runVMError(t *testing.T, file *ast.File, userModules map[string]string) (ok bool) { + _, trace, err := traceCompileRun(file, nil, userModules) defer func() { if !ok { @@ -132,7 +140,7 @@ func (o *tracer) Write(p []byte) (n int, err error) { return len(p), nil } -func traceCompileRun(file *ast.File, symbols map[string]objects.Object) (res map[string]objects.Object, trace []string, err error) { +func traceCompileRun(file *ast.File, symbols map[string]objects.Object, userModules map[string]string) (res map[string]objects.Object, trace []string, err error) { var v *runtime.VM defer func() { @@ -169,6 +177,13 @@ func traceCompileRun(file *ast.File, symbols map[string]objects.Object) (res map tr := &tracer{} c := compiler.NewCompiler(symTable, tr) + c.SetModuleLoader(func(moduleName string) ([]byte, error) { + if src, ok := userModules[moduleName]; ok { + return []byte(src), nil + } + + return nil, fmt.Errorf("module '%s' not found", moduleName) + }) err = c.Compile(file) trace = append(trace, fmt.Sprintf("\n[Compiler Trace]\n\n%s", strings.Join(tr.Out, ""))) if err != nil {