core/xgo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

move file and fileset to source package

Browse source
This commit is contained in:
Daniel Kang 2019-01-11 01:16:34 -08:00
parent b2a29342b1
commit 53af0d6328
54 changed files with 253 additions and 234 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
assert/assert.go
View file

@ -9,7 +9,7 @@ import (
"github.com/d5/tengo/compiler"
"github.com/d5/tengo/objects"
"github.com/d5/tengo/scanner"
"github.com/d5/tengo/source"
"github.com/d5/tengo/token"
)
@ -133,8 +133,8 @@ func Equal(t *testing.T, expected, actual interface{}, msg ...interface{}) bool
if !equalSymbol(expected, actual.(compiler.Symbol)) {
return failExpectedActual(t, expected, actual, msg...)
}
case scanner.Pos:
if expected != actual.(scanner.Pos) {
case source.Pos:
if expected != actual.(source.Pos) {
return failExpectedActual(t, expected, actual, msg...)
}
case token.Token:

10
ast/array_lit.go
View file

@ -3,22 +3,22 @@ package ast
import (
"strings"
"github.com/d5/tengo/scanner"
"github.com/d5/tengo/source"
)
type ArrayLit struct {
Elements []Expr
LBrack scanner.Pos
RBrack scanner.Pos
LBrack source.Pos
RBrack source.Pos
}
func (e *ArrayLit) exprNode() {}
func (e *ArrayLit) Pos() scanner.Pos {
func (e *ArrayLit) Pos() source.Pos {
return e.LBrack
}
func (e *ArrayLit) End() scanner.Pos {
func (e *ArrayLit) End() source.Pos {
return e.RBrack + 1
}

8
ast/assign_stmt.go
View file

@ -3,7 +3,7 @@ package ast
import (
"strings"
"github.com/d5/tengo/scanner"
"github.com/d5/tengo/source"
"github.com/d5/tengo/token"
)
@ -11,16 +11,16 @@ type AssignStmt struct {
Lhs []Expr
Rhs []Expr
Token token.Token
TokenPos scanner.Pos
TokenPos source.Pos
}
func (s *AssignStmt) stmtNode() {}
func (s *AssignStmt) Pos() scanner.Pos {
func (s *AssignStmt) Pos() source.Pos {
return s.Lhs[0].Pos()
}
func (s *AssignStmt) End() scanner.Pos {
func (s *AssignStmt) End() source.Pos {
return s.Rhs[len(s.Rhs)-1].End()
}

10
ast/bad_expr.go
View file

@ -1,19 +1,19 @@
package ast
import "github.com/d5/tengo/scanner"
import "github.com/d5/tengo/source"
type BadExpr struct {
From scanner.Pos
To scanner.Pos
From source.Pos
To source.Pos
}
func (e *BadExpr) exprNode() {}
func (e *BadExpr) Pos() scanner.Pos {
func (e *BadExpr) Pos() source.Pos {
return e.From
}
func (e *BadExpr) End() scanner.Pos {
func (e *BadExpr) End() source.Pos {
return e.To
}

10
ast/bad_stmt.go
View file

@ -1,19 +1,19 @@
package ast
import "github.com/d5/tengo/scanner"
import "github.com/d5/tengo/source"
type BadStmt struct {
From scanner.Pos
To scanner.Pos
From source.Pos
To source.Pos
}
func (s *BadStmt) stmtNode() {}
func (s *BadStmt) Pos() scanner.Pos {
func (s *BadStmt) Pos() source.Pos {
return s.From
}
func (s *BadStmt) End() scanner.Pos {
func (s *BadStmt) End() source.Pos {
return s.To
}

8
ast/binary_expr.go
View file

@ -1,7 +1,7 @@
package ast
import (
"github.com/d5/tengo/scanner"
"github.com/d5/tengo/source"
"github.com/d5/tengo/token"
)
@ -9,16 +9,16 @@ type BinaryExpr struct {
Lhs Expr
Rhs Expr
Token token.Token
TokenPos scanner.Pos
TokenPos source.Pos
}
func (e *BinaryExpr) exprNode() {}
func (e *BinaryExpr) Pos() scanner.Pos {
func (e *BinaryExpr) Pos() source.Pos {
return e.Lhs.Pos()
}
func (e *BinaryExpr) End() scanner.Pos {
func (e *BinaryExpr) End() source.Pos {
return e.Rhs.End()
}

10
ast/block_stmt.go
View file

@ -3,22 +3,22 @@ package ast
import (
"strings"
"github.com/d5/tengo/scanner"
"github.com/d5/tengo/source"
)
type BlockStmt struct {
Stmts []Stmt
LBrace scanner.Pos
RBrace scanner.Pos
LBrace source.Pos
RBrace source.Pos
}
func (s *BlockStmt) stmtNode() {}
func (s *BlockStmt) Pos() scanner.Pos {
func (s *BlockStmt) Pos() source.Pos {
return s.LBrace
}
func (s *BlockStmt) End() scanner.Pos {
func (s *BlockStmt) End() source.Pos {
return s.RBrace + 1
}

10
ast/bool_lit.go
View file

@ -1,21 +1,21 @@
package ast
import "github.com/d5/tengo/scanner"
import "github.com/d5/tengo/source"
type BoolLit struct {
Value bool
ValuePos scanner.Pos
ValuePos source.Pos
Literal string
}
func (e *BoolLit) exprNode() {}
func (e *BoolLit) Pos() scanner.Pos {
func (e *BoolLit) Pos() source.Pos {
return e.ValuePos
}
func (e *BoolLit) End() scanner.Pos {
return scanner.Pos(int(e.ValuePos) + len(e.Literal))
func (e *BoolLit) End() source.Pos {
return source.Pos(int(e.ValuePos) + len(e.Literal))
}
func (e *BoolLit) String() string {

10
ast/branch_stmt.go
View file

@ -1,28 +1,28 @@
package ast
import (
"github.com/d5/tengo/scanner"
"github.com/d5/tengo/source"
"github.com/d5/tengo/token"
)
type BranchStmt struct {
Token token.Token
TokenPos scanner.Pos
TokenPos source.Pos
Label *Ident
}
func (s *BranchStmt) stmtNode() {}
func (s *BranchStmt) Pos() scanner.Pos {
func (s *BranchStmt) Pos() source.Pos {
return s.TokenPos
}
func (s *BranchStmt) End() scanner.Pos {
func (s *BranchStmt) End() source.Pos {
if s.Label != nil {
return s.Label.End()
}
return scanner.Pos(int(s.TokenPos) + len(s.Token.String()))
return source.Pos(int(s.TokenPos) + len(s.Token.String()))
}
func (s *BranchStmt) String() string {

10
ast/call_expr.go
View file

@ -3,23 +3,23 @@ package ast
import (
"strings"
"github.com/d5/tengo/scanner"
"github.com/d5/tengo/source"
)
type CallExpr struct {
Func Expr
LParen scanner.Pos
LParen source.Pos
Args []Expr
RParen scanner.Pos
RParen source.Pos
}
func (e *CallExpr) exprNode() {}
func (e *CallExpr) Pos() scanner.Pos {
func (e *CallExpr) Pos() source.Pos {
return e.Func.Pos()
}
func (e *CallExpr) End() scanner.Pos {
func (e *CallExpr) End() source.Pos {
return e.RParen + 1
}

10
ast/char_lit.go
View file

@ -1,21 +1,21 @@
package ast
import "github.com/d5/tengo/scanner"
import "github.com/d5/tengo/source"
type CharLit struct {
Value rune
ValuePos scanner.Pos
ValuePos source.Pos
Literal string
}
func (e *CharLit) exprNode() {}
func (e *CharLit) Pos() scanner.Pos {
func (e *CharLit) Pos() source.Pos {
return e.ValuePos
}
func (e *CharLit) End() scanner.Pos {
return scanner.Pos(int(e.ValuePos) + len(e.Literal))
func (e *CharLit) End() source.Pos {
return source.Pos(int(e.ValuePos) + len(e.Literal))
}
func (e *CharLit) String() string {

8
ast/empty_stmt.go
View file

@ -1,19 +1,19 @@
package ast
import "github.com/d5/tengo/scanner"
import "github.com/d5/tengo/source"
type EmptyStmt struct {
Semicolon scanner.Pos
Semicolon source.Pos
Implicit bool
}
func (s *EmptyStmt) stmtNode() {}
func (s *EmptyStmt) Pos() scanner.Pos {
func (s *EmptyStmt) Pos() source.Pos {
return s.Semicolon
}
func (s *EmptyStmt) End() scanner.Pos {
func (s *EmptyStmt) End() source.Pos {
if s.Implicit {
return s.Semicolon
}

6
ast/expr_stmt.go
View file

@ -1,6 +1,6 @@
package ast
import "github.com/d5/tengo/scanner"
import "github.com/d5/tengo/source"
type ExprStmt struct {
Expr Expr
@ -8,11 +8,11 @@ type ExprStmt struct {
func (s *ExprStmt) stmtNode() {}
func (s *ExprStmt) Pos() scanner.Pos {
func (s *ExprStmt) Pos() source.Pos {
return s.Expr.Pos()
}
func (s *ExprStmt) End() scanner.Pos {
func (s *ExprStmt) End() source.Pos {
return s.Expr.End()
}

12
ast/file.go
View file

@ -3,20 +3,20 @@ package ast
import (
"strings"
"github.com/d5/tengo/scanner"
"github.com/d5/tengo/source"
)
type File struct {
InputFile *scanner.File
InputFile *source.File
Stmts []Stmt
}
func (n *File) Pos() scanner.Pos {
return scanner.Pos(n.InputFile.Base())
func (n *File) Pos() source.Pos {
return source.Pos(n.InputFile.Base())
}
func (n *File) End() scanner.Pos {
return scanner.Pos(n.InputFile.Base() + n.InputFile.Size())
func (n *File) End() source.Pos {
return source.Pos(n.InputFile.Base() + n.InputFile.Size())
}
func (n *File) String() string {

10
ast/float_lit.go
View file

@ -1,21 +1,21 @@
package ast
import "github.com/d5/tengo/scanner"
import "github.com/d5/tengo/source"
type FloatLit struct {
Value float64
ValuePos scanner.Pos
ValuePos source.Pos
Literal string
}
func (e *FloatLit) exprNode() {}
func (e *FloatLit) Pos() scanner.Pos {
func (e *FloatLit) Pos() source.Pos {
return e.ValuePos
}
func (e *FloatLit) End() scanner.Pos {
return scanner.Pos(int(e.ValuePos) + len(e.Literal))
func (e *FloatLit) End() source.Pos {
return source.Pos(int(e.ValuePos) + len(e.Literal))
}
func (e *FloatLit) String() string {

8
ast/for_in_stmt.go
View file

@ -1,9 +1,9 @@
package ast
import "github.com/d5/tengo/scanner"
import "github.com/d5/tengo/source"
type ForInStmt struct {
ForPos scanner.Pos
ForPos source.Pos
Key *Ident
Value *Ident
Iterable Expr
@ -12,11 +12,11 @@ type ForInStmt struct {
func (s *ForInStmt) stmtNode() {}
func (s *ForInStmt) Pos() scanner.Pos {
func (s *ForInStmt) Pos() source.Pos {
return s.ForPos
}
func (s *ForInStmt) End() scanner.Pos {
func (s *ForInStmt) End() source.Pos {
return s.Body.End()
}

8
ast/for_stmt.go
View file

@ -1,9 +1,9 @@
package ast
import "github.com/d5/tengo/scanner"
import "github.com/d5/tengo/source"
type ForStmt struct {
ForPos scanner.Pos
ForPos source.Pos
Init Stmt
Cond Expr
Post Stmt
@ -12,11 +12,11 @@ type ForStmt struct {
func (s *ForStmt) stmtNode() {}
func (s *ForStmt) Pos() scanner.Pos {
func (s *ForStmt) Pos() source.Pos {
return s.ForPos
}
func (s *ForStmt) End() scanner.Pos {
func (s *ForStmt) End() source.Pos {
return s.Body.End()
}

6
ast/func_lit.go
View file

@ -1,6 +1,6 @@
package ast
import "github.com/d5/tengo/scanner"
import "github.com/d5/tengo/source"
type FuncLit struct {
Type *FuncType
@ -9,11 +9,11 @@ type FuncLit struct {
func (e *FuncLit) exprNode() {}
func (e *FuncLit) Pos() scanner.Pos {
func (e *FuncLit) Pos() source.Pos {
return e.Type.Pos()
}
func (e *FuncLit) End() scanner.Pos {
func (e *FuncLit) End() source.Pos {
return e.Body.End()
}

8
ast/func_type.go
View file

@ -1,19 +1,19 @@
package ast
import "github.com/d5/tengo/scanner"
import "github.com/d5/tengo/source"
type FuncType struct {
FuncPos scanner.Pos
FuncPos source.Pos
Params *IdentList
}
func (e *FuncType) exprNode() {}
func (e *FuncType) Pos() scanner.Pos {
func (e *FuncType) Pos() source.Pos {
return e.FuncPos
}
func (e *FuncType) End() scanner.Pos {
func (e *FuncType) End() source.Pos {
return e.Params.End()
}

10
ast/ident.go
View file

@ -1,20 +1,20 @@
package ast
import "github.com/d5/tengo/scanner"
import "github.com/d5/tengo/source"
type Ident struct {
Name string
NamePos scanner.Pos
NamePos source.Pos
}
func (e *Ident) exprNode() {}
func (e *Ident) Pos() scanner.Pos {
func (e *Ident) Pos() source.Pos {
return e.NamePos
}
func (e *Ident) End() scanner.Pos {
return scanner.Pos(int(e.NamePos) + len(e.Name))
func (e *Ident) End() source.Pos {
return source.Pos(int(e.NamePos) + len(e.Name))
}
func (e *Ident) String() string {

14
ast/ident_list.go
View file

@ -3,16 +3,16 @@ package ast
import (
"strings"
"github.com/d5/tengo/scanner"
"github.com/d5/tengo/source"
)
type IdentList struct {
LParen scanner.Pos
LParen source.Pos
List []*Ident
RParen scanner.Pos
RParen source.Pos
}
func (n *IdentList) Pos() scanner.Pos {
func (n *IdentList) Pos() source.Pos {
if n.LParen.IsValid() {
return n.LParen
}
@ -21,10 +21,10 @@ func (n *IdentList) Pos() scanner.Pos {
return n.List[0].Pos()
}
return scanner.NoPos
return source.NoPos
}
func (n *IdentList) End() scanner.Pos {
func (n *IdentList) End() source.Pos {
if n.RParen.IsValid() {
return n.RParen + 1
}
@ -33,7 +33,7 @@ func (n *IdentList) End() scanner.Pos {
return n.List[l-1].End()
}
return scanner.NoPos
return source.NoPos
}
func (n *IdentList) NumFields() int {

8
ast/if_stmt.go
View file

@ -1,9 +1,9 @@
package ast
import "github.com/d5/tengo/scanner"
import "github.com/d5/tengo/source"
type IfStmt struct {
IfPos scanner.Pos
IfPos source.Pos
Init Stmt
Cond Expr
Body *BlockStmt
@ -12,11 +12,11 @@ type IfStmt struct {
func (s *IfStmt) stmtNode() {}
func (s *IfStmt) Pos() scanner.Pos {
func (s *IfStmt) Pos() source.Pos {
return s.IfPos
}
func (s *IfStmt) End() scanner.Pos {
func (s *IfStmt) End() source.Pos {
if s.Else != nil {
return s.Else.End()
}

10
ast/inc_dec_stmt.go
View file

@ -1,24 +1,24 @@
package ast
import (
"github.com/d5/tengo/scanner"
"github.com/d5/tengo/source"
"github.com/d5/tengo/token"
)
type IncDecStmt struct {
Expr Expr
Token token.Token
TokenPos scanner.Pos
TokenPos source.Pos
}
func (s *IncDecStmt) stmtNode() {}
func (s *IncDecStmt) Pos() scanner.Pos {
func (s *IncDecStmt) Pos() source.Pos {
return s.Expr.Pos()
}
func (s *IncDecStmt) End() scanner.Pos {
return scanner.Pos(int(s.TokenPos) + 2)
func (s *IncDecStmt) End() source.Pos {
return source.Pos(int(s.TokenPos) + 2)
}
func (s *IncDecStmt) String() string {

10
ast/index_expr.go
View file

@ -1,21 +1,21 @@
package ast
import "github.com/d5/tengo/scanner"
import "github.com/d5/tengo/source"
type IndexExpr struct {
Expr Expr
LBrack scanner.Pos
LBrack source.Pos
Index Expr
RBrack scanner.Pos
RBrack source.Pos
}
func (e *IndexExpr) exprNode() {}
func (e *IndexExpr) Pos() scanner.Pos {
func (e *IndexExpr) Pos() source.Pos {
return e.Expr.Pos()
}
func (e *IndexExpr) End() scanner.Pos {
func (e *IndexExpr) End() source.Pos {
return e.RBrack + 1
}

10
ast/int_lit.go
View file

@ -1,21 +1,21 @@
package ast
import "github.com/d5/tengo/scanner"
import "github.com/d5/tengo/source"
type IntLit struct {
Value int64
ValuePos scanner.Pos
ValuePos source.Pos
Literal string
}
func (e *IntLit) exprNode() {}
func (e *IntLit) Pos() scanner.Pos {
func (e *IntLit) Pos() source.Pos {
return e.ValuePos
}
func (e *IntLit) End() scanner.Pos {
return scanner.Pos(int(e.ValuePos) + len(e.Literal))
func (e *IntLit) End() source.Pos {
return source.Pos(int(e.ValuePos) + len(e.Literal))
}
func (e *IntLit) String() string {

10
ast/map_element_lit.go
View file

@ -1,21 +1,21 @@
package ast
import "github.com/d5/tengo/scanner"
import "github.com/d5/tengo/source"
type MapElementLit struct {
Key string
KeyPos scanner.Pos
ColonPos scanner.Pos
KeyPos source.Pos
ColonPos source.Pos
Value Expr
}
func (e *MapElementLit) exprNode() {}
func (e *MapElementLit) Pos() scanner.Pos {
func (e *MapElementLit) Pos() source.Pos {
return e.KeyPos
}
func (e *MapElementLit) End() scanner.Pos {
func (e *MapElementLit) End() source.Pos {
return e.Value.End()
}

10
ast/map_lit.go
View file

@ -3,22 +3,22 @@ package ast
import (
"strings"
"github.com/d5/tengo/scanner"
"github.com/d5/tengo/source"
)
type MapLit struct {
LBrace scanner.Pos
LBrace source.Pos
Elements []*MapElementLit
RBrace scanner.Pos
RBrace source.Pos
}
func (e *MapLit) exprNode() {}
func (e *MapLit) Pos() scanner.Pos {
func (e *MapLit) Pos() source.Pos {
return e.LBrace
}
func (e *MapLit) End() scanner.Pos {
func (e *MapLit) End() source.Pos {
return e.RBrace + 1
}

8
ast/node.go
View file

@ -1,11 +1,9 @@
package ast
import (
"github.com/d5/tengo/scanner"
)
import "github.com/d5/tengo/source"
type Node interface {
Pos() scanner.Pos
End() scanner.Pos
Pos() source.Pos
End() source.Pos
String() string
}

10
ast/paren_expr.go
View file

@ -1,20 +1,20 @@
package ast
import "github.com/d5/tengo/scanner"
import "github.com/d5/tengo/source"
type ParenExpr struct {
Expr Expr
LParen scanner.Pos
RParen scanner.Pos
LParen source.Pos
RParen source.Pos
}
func (e *ParenExpr) exprNode() {}
func (e *ParenExpr) Pos() scanner.Pos {
func (e *ParenExpr) Pos() source.Pos {
return e.LParen
}
func (e *ParenExpr) End() scanner.Pos {
func (e *ParenExpr) End() source.Pos {
return e.RParen + 1
}

8
ast/return_stmt.go
View file

@ -3,21 +3,21 @@ package ast
import (
"strings"
"github.com/d5/tengo/scanner"
"github.com/d5/tengo/source"
)
type ReturnStmt struct {
ReturnPos scanner.Pos
ReturnPos source.Pos
Results []Expr
}
func (s *ReturnStmt) stmtNode() {}
func (s *ReturnStmt) Pos() scanner.Pos {
func (s *ReturnStmt) Pos() source.Pos {
return s.ReturnPos
}
func (s *ReturnStmt) End() scanner.Pos {
func (s *ReturnStmt) End() source.Pos {
if n := len(s.Results); n > 0 {
return s.Results[n-1].End()
}

6
ast/selector_expr.go
View file

@ -1,6 +1,6 @@
package ast
import "github.com/d5/tengo/scanner"
import "github.com/d5/tengo/source"
type SelectorExpr struct {
Expr Expr
@ -9,11 +9,11 @@ type SelectorExpr struct {
func (e *SelectorExpr) exprNode() {}
func (e *SelectorExpr) Pos() scanner.Pos {
func (e *SelectorExpr) Pos() source.Pos {
return e.Expr.Pos()
}
func (e *SelectorExpr) End() scanner.Pos {
func (e *SelectorExpr) End() source.Pos {
return e.Sel.End()
}

10
ast/slice_expr.go
View file

@ -1,22 +1,22 @@
package ast
import "github.com/d5/tengo/scanner"
import "github.com/d5/tengo/source"
type SliceExpr struct {
Expr Expr
LBrack scanner.Pos
LBrack source.Pos
Low Expr
High Expr
RBrack scanner.Pos
RBrack source.Pos
}
func (e *SliceExpr) exprNode() {}
func (e *SliceExpr) Pos() scanner.Pos {
func (e *SliceExpr) Pos() source.Pos {
return e.Expr.Pos()
}
func (e *SliceExpr) End() scanner.Pos {
func (e *SliceExpr) End() source.Pos {
return e.RBrack + 1
}

10
ast/string_lit.go
View file

@ -1,21 +1,21 @@
package ast
import "github.com/d5/tengo/scanner"
import "github.com/d5/tengo/source"
type StringLit struct {
Value string
ValuePos scanner.Pos
ValuePos source.Pos
Literal string
}
func (e *StringLit) exprNode() {}
func (e *StringLit) Pos() scanner.Pos {
func (e *StringLit) Pos() source.Pos {
return e.ValuePos
}
func (e *StringLit) End() scanner.Pos {
return scanner.Pos(int(e.ValuePos) + len(e.Literal))
func (e *StringLit) End() source.Pos {
return source.Pos(int(e.ValuePos) + len(e.Literal))
}
func (e *StringLit) String() string {

8
ast/unary_expr.go
View file

@ -1,23 +1,23 @@
package ast
import (
"github.com/d5/tengo/scanner"
"github.com/d5/tengo/source"
"github.com/d5/tengo/token"
)
type UnaryExpr struct {
Expr Expr
Token token.Token
TokenPos scanner.Pos
TokenPos source.Pos
}
func (e *UnaryExpr) exprNode() {}
func (e *UnaryExpr) Pos() scanner.Pos {
func (e *UnaryExpr) Pos() source.Pos {
return e.Expr.Pos()
}
func (e *UnaryExpr) End() scanner.Pos {
func (e *UnaryExpr) End() source.Pos {
return e.Expr.End()
}

8
ast/undefined_lit.go
View file

@ -1,18 +1,18 @@
package ast
import "github.com/d5/tengo/scanner"
import "github.com/d5/tengo/source"
type UndefinedLit struct {
TokenPos scanner.Pos
TokenPos source.Pos
}
func (e *UndefinedLit) exprNode() {}
func (e *UndefinedLit) Pos() scanner.Pos {
func (e *UndefinedLit) Pos() source.Pos {
return e.TokenPos
}
func (e *UndefinedLit) End() scanner.Pos {
func (e *UndefinedLit) End() source.Pos {
return e.TokenPos + 9 // len(undefined) == 9
}

4
cmd/bench/main.go
View file

@ -8,7 +8,7 @@ import (
"github.com/d5/tengo/compiler"
"github.com/d5/tengo/objects"
"github.com/d5/tengo/parser"
"github.com/d5/tengo/scanner"
"github.com/d5/tengo/source"
"github.com/d5/tengo/vm"
)
@ -126,7 +126,7 @@ func runBench(input []byte) (parseTime time.Duration, compileTime time.Duration,
}
func parse(input []byte) (time.Duration, *ast.File, error) {
fileSet := scanner.NewFileSet()
fileSet := source.NewFileSet()
inputFile := fileSet.AddFile("test", -1, len(input))
start := time.Now()

4
cmd/repl/main.go
View file

@ -11,7 +11,7 @@ import (
"github.com/d5/tengo/compiler"
"github.com/d5/tengo/objects"
"github.com/d5/tengo/parser"
"github.com/d5/tengo/scanner"
"github.com/d5/tengo/source"
"github.com/d5/tengo/vm"
)
@ -33,7 +33,7 @@ func main() {
func startRepl(in io.Reader, out io.Writer) {
stdin := bufio.NewScanner(in)
fileSet := scanner.NewFileSet()
fileSet := source.NewFileSet()
globals := make([]*objects.Object, vm.GlobalsSize)
symbolTable := compiler.NewSymbolTable()

4
compiler/compiler_test.go
View file

@ -10,7 +10,7 @@ import (
"github.com/d5/tengo/compiler"
"github.com/d5/tengo/objects"
"github.com/d5/tengo/parser"
"github.com/d5/tengo/scanner"
"github.com/d5/tengo/source"
)
func TestCompiler_Compile(t *testing.T) {
@ -884,7 +884,7 @@ func (o *tracer) Write(p []byte) (n int, err error) {
}
func traceCompile(input string, symbols map[string]objects.Object) (res *compiler.Bytecode, trace []string, err error) {
fileSet := scanner.NewFileSet()
fileSet := source.NewFileSet()
file := fileSet.AddFile("test", -1, len(input))
p := parser.NewParser(file, []byte(input), nil)

4
parser/error.go
View file

@ -1,9 +1,9 @@
package parser
import "github.com/d5/tengo/scanner"
import "github.com/d5/tengo/source"
type Error struct {
Pos scanner.FilePos
Pos source.FilePos
Msg string
}

6
parser/error_list.go
View file

@ -4,12 +4,12 @@ import (
"fmt"
"sort"
"github.com/d5/tengo/scanner"
"github.com/d5/tengo/source"
)
type ErrorList []*Error
func (p *ErrorList) Add(pos scanner.FilePos, msg string) {
func (p *ErrorList) Add(pos source.FilePos, msg string) {
*p = append(*p, &Error{pos, msg})
}
@ -51,7 +51,7 @@ func (p ErrorList) Sort() {
func (p *ErrorList) RemoveMultiples() {
sort.Sort(p)
var last scanner.FilePos // initial last.Line is != any legal error line
var last source.FilePos // initial last.Line is != any legal error line
i := 0
for _, e := range *p {

4
parser/parse_file.go
View file

@ -4,10 +4,10 @@ import (
"io"
"github.com/d5/tengo/ast"
"github.com/d5/tengo/scanner"
"github.com/d5/tengo/source"
)
func ParseFile(file *scanner.File, src []byte, trace io.Writer) (res *ast.File, err error) {
func ParseFile(file *source.File, src []byte, trace io.Writer) (res *ast.File, err error) {
p := NewParser(file, src, trace)
defer func() {

16
parser/parse_source.go Normal file
View file

@ -0,0 +1,16 @@
package parser
import (
"io"
"github.com/d5/tengo/ast"
"github.com/d5/tengo/source"
)
// ParseSource parses source code 'src' and builds an AST.
func ParseSource(src []byte, trace io.Writer) (res *ast.File, err error) {
fileSet := source.NewFileSet()
file := fileSet.AddFile("", -1, len(src))
return ParseFile(file, src, trace)
}

25
parser/parser.go
View file

@ -7,34 +7,35 @@ import (
"github.com/d5/tengo/ast"
"github.com/d5/tengo/scanner"
"github.com/d5/tengo/source"
"github.com/d5/tengo/token"
)
type bailout struct{}
type Parser struct {
file *scanner.File
file *source.File
errors ErrorList
scanner *scanner.Scanner
pos scanner.Pos
pos source.Pos
token token.Token
tokenLit string
exprLevel int // < 0: in control clause, >= 0: in expression
syncPos scanner.Pos // last sync position
syncCount int // number of advance calls without progress
exprLevel int // < 0: in control clause, >= 0: in expression
syncPos source.Pos // last sync position
syncCount int // number of advance calls without progress
trace bool
indent int
traceOut io.Writer
}
func NewParser(file *scanner.File, src []byte, trace io.Writer) *Parser {
func NewParser(file *source.File, src []byte, trace io.Writer) *Parser {
p := &Parser{
file: file,
trace: trace != nil,
traceOut: trace,
}
p.scanner = scanner.NewScanner(p.file, src, func(pos scanner.FilePos, msg string) {
p.scanner = scanner.NewScanner(p.file, src, func(pos source.FilePos, msg string) {
p.errors.Add(pos, msg)
}, 0)
@ -910,7 +911,7 @@ func (p *Parser) parseMapLit() *ast.MapLit {
}
}
func (p *Parser) expect(token token.Token) scanner.Pos {
func (p *Parser) expect(token token.Token) source.Pos {
pos := p.pos
if p.token != token {
@ -955,7 +956,7 @@ func (p *Parser) advance(to map[token.Token]bool) {
}
}
func (p *Parser) error(pos scanner.Pos, msg string) {
func (p *Parser) error(pos source.Pos, msg string) {
filePos := p.file.Position(pos)
n := len(p.errors)
@ -972,7 +973,7 @@ func (p *Parser) error(pos scanner.Pos, msg string) {
p.errors.Add(filePos, msg)
}
func (p *Parser) errorExpected(pos scanner.Pos, msg string) {
func (p *Parser) errorExpected(pos source.Pos, msg string) {
msg = "expected " + msg
if pos == p.pos {
// error happened at the current position: provide more specific
@ -1023,12 +1024,12 @@ func (p *Parser) printTrace(a ...interface{}) {
_, _ = fmt.Fprintln(p.traceOut, a...)
}
func (p *Parser) safePos(pos scanner.Pos) scanner.Pos {
func (p *Parser) safePos(pos source.Pos) source.Pos {
fileBase := p.file.Base()
fileSize := p.file.Size()
if int(pos) < fileBase || int(pos) > fileBase+fileSize {
return scanner.Pos(fileBase + fileSize)
return source.Pos(fileBase + fileSize)
}
return pos

66
parser/parser_test.go
View file

@ -9,11 +9,11 @@ import (
"github.com/d5/tengo/assert"
"github.com/d5/tengo/ast"
"github.com/d5/tengo/parser"
"github.com/d5/tengo/scanner"
"github.com/d5/tengo/source"
"github.com/d5/tengo/token"
)
type pfn func(int, int) scanner.Pos // position conversion function
type pfn func(int, int) source.Pos // position conversion function
type expectedFn func(pos pfn) []ast.Stmt // callback function to return expected results
type tracer struct {
@ -35,7 +35,7 @@ func (o *tracer) Write(p []byte) (n int, err error) {
//}
func expect(t *testing.T, input string, fn expectedFn) (ok bool) {
testFileSet := scanner.NewFileSet()
testFileSet := source.NewFileSet()
testFile := testFileSet.AddFile("", -1, len(input))
defer func() {
@ -55,8 +55,8 @@ func expect(t *testing.T, input string, fn expectedFn) (ok bool) {
return
}
expected := fn(func(line, column int) scanner.Pos {
return scanner.Pos(int(testFile.LineStart(line)) + (column - 1))
expected := fn(func(line, column int) source.Pos {
return source.Pos(int(testFile.LineStart(line)) + (column - 1))
})
if !assert.Equal(t, len(expected), len(actual.Stmts)) {
@ -75,7 +75,7 @@ func expect(t *testing.T, input string, fn expectedFn) (ok bool) {
}
func expectError(t *testing.T, input string) (ok bool) {
testFileSet := scanner.NewFileSet()
testFileSet := source.NewFileSet()
testFile := testFileSet.AddFile("", -1, len(input))
defer func() {
@ -98,7 +98,7 @@ func expectError(t *testing.T, input string) (ok bool) {
}
func expectString(t *testing.T, input, expected string) (ok bool) {
testFileSet := scanner.NewFileSet()
testFileSet := source.NewFileSet()
testFile := testFileSet.AddFile("", -1, len(input))
defer func() {
@ -125,7 +125,7 @@ func expectString(t *testing.T, input, expected string) (ok bool) {
}
//func printTrace(input string) {
// testFileSet := scanner.NewFileSet()
// testFileSet := source.NewFileSet()
// testFile := testFileSet.AddFile("", -1, len(input))
//
// _, _ = parser.ParseFile(testFile, []byte(input), &slowPrinter{})
@ -139,55 +139,55 @@ func exprStmt(x ast.Expr) *ast.ExprStmt {
return &ast.ExprStmt{Expr: x}
}
func assignStmt(lhs, rhs []ast.Expr, token token.Token, pos scanner.Pos) *ast.AssignStmt {
func assignStmt(lhs, rhs []ast.Expr, token token.Token, pos source.Pos) *ast.AssignStmt {
return &ast.AssignStmt{Lhs: lhs, Rhs: rhs, Token: token, TokenPos: pos}
}
func emptyStmt(implicit bool, pos scanner.Pos) *ast.EmptyStmt {
func emptyStmt(implicit bool, pos source.Pos) *ast.EmptyStmt {
return &ast.EmptyStmt{Implicit: implicit, Semicolon: pos}
}
func returnStmt(pos scanner.Pos, results ...ast.Expr) *ast.ReturnStmt {
func returnStmt(pos source.Pos, results ...ast.Expr) *ast.ReturnStmt {
return &ast.ReturnStmt{Results: results, ReturnPos: pos}
}
func forStmt(init ast.Stmt, cond ast.Expr, post ast.Stmt, body *ast.BlockStmt, pos scanner.Pos) *ast.ForStmt {
func forStmt(init ast.Stmt, cond ast.Expr, post ast.Stmt, body *ast.BlockStmt, pos source.Pos) *ast.ForStmt {
return &ast.ForStmt{Cond: cond, Init: init, Post: post, Body: body, ForPos: pos}
}
func forInStmt(key, value *ast.Ident, seq ast.Expr, body *ast.BlockStmt, pos scanner.Pos) *ast.ForInStmt {
func forInStmt(key, value *ast.Ident, seq ast.Expr, body *ast.BlockStmt, pos source.Pos) *ast.ForInStmt {
return &ast.ForInStmt{Key: key, Value: value, Iterable: seq, Body: body, ForPos: pos}
}
func ifStmt(init ast.Stmt, cond ast.Expr, body *ast.BlockStmt, elseStmt ast.Stmt, pos scanner.Pos) *ast.IfStmt {
func ifStmt(init ast.Stmt, cond ast.Expr, body *ast.BlockStmt, elseStmt ast.Stmt, pos source.Pos) *ast.IfStmt {
return &ast.IfStmt{Init: init, Cond: cond, Body: body, Else: elseStmt, IfPos: pos}
}
func incDecStmt(expr ast.Expr, tok token.Token, pos scanner.Pos) *ast.IncDecStmt {
func incDecStmt(expr ast.Expr, tok token.Token, pos source.Pos) *ast.IncDecStmt {
return &ast.IncDecStmt{Expr: expr, Token: tok, TokenPos: pos}
}
func funcType(params *ast.IdentList, pos scanner.Pos) *ast.FuncType {
func funcType(params *ast.IdentList, pos source.Pos) *ast.FuncType {
return &ast.FuncType{Params: params, FuncPos: pos}
}
func blockStmt(lbrace, rbrace scanner.Pos, list ...ast.Stmt) *ast.BlockStmt {
func blockStmt(lbrace, rbrace source.Pos, list ...ast.Stmt) *ast.BlockStmt {
return &ast.BlockStmt{Stmts: list, LBrace: lbrace, RBrace: rbrace}
}
func ident(name string, pos scanner.Pos) *ast.Ident {
func ident(name string, pos source.Pos) *ast.Ident {
return &ast.Ident{Name: name, NamePos: pos}
}
func identList(opening, closing scanner.Pos, list ...*ast.Ident) *ast.IdentList {
func identList(opening, closing source.Pos, list ...*ast.Ident) *ast.IdentList {
return &ast.IdentList{List: list, LParen: opening, RParen: closing}
}
func binaryExpr(x, y ast.Expr, op token.Token, pos scanner.Pos) *ast.BinaryExpr {
func binaryExpr(x, y ast.Expr, op token.Token, pos source.Pos) *ast.BinaryExpr {
return &ast.BinaryExpr{Lhs: x, Rhs: y, Token: op, TokenPos: pos}
}
func unaryExpr(x ast.Expr, op token.Token, pos scanner.Pos) *ast.UnaryExpr {
func unaryExpr(x ast.Expr, op token.Token, pos source.Pos) *ast.UnaryExpr {
return &ast.UnaryExpr{Expr: x, Token: op, TokenPos: pos}
}
@ -195,35 +195,35 @@ func exprs(list ...ast.Expr) []ast.Expr {
return list
}
func intLit(value int64, pos scanner.Pos) *ast.IntLit {
func intLit(value int64, pos source.Pos) *ast.IntLit {
return &ast.IntLit{Value: value, ValuePos: pos}
}
func floatLit(value float64, pos scanner.Pos) *ast.FloatLit {
func floatLit(value float64, pos source.Pos) *ast.FloatLit {
return &ast.FloatLit{Value: value, ValuePos: pos}
}
func stringLit(value string, pos scanner.Pos) *ast.StringLit {
func stringLit(value string, pos source.Pos) *ast.StringLit {
return &ast.StringLit{Value: value, ValuePos: pos}
}
func charLit(value rune, pos scanner.Pos) *ast.CharLit {
func charLit(value rune, pos source.Pos) *ast.CharLit {
return &ast.CharLit{Value: value, ValuePos: pos, Literal: fmt.Sprintf("'%c'", value)}
}
func boolLit(value bool, pos scanner.Pos) *ast.BoolLit {
func boolLit(value bool, pos source.Pos) *ast.BoolLit {
return &ast.BoolLit{Value: value, ValuePos: pos}
}
func arrayLit(lbracket, rbracket scanner.Pos, list ...ast.Expr) *ast.ArrayLit {
func arrayLit(lbracket, rbracket source.Pos, list ...ast.Expr) *ast.ArrayLit {
return &ast.ArrayLit{LBrack: lbracket, RBrack: rbracket, Elements: list}
}
func mapElementLit(key string, keyPos scanner.Pos, colonPos scanner.Pos, value ast.Expr) *ast.MapElementLit {
func mapElementLit(key string, keyPos source.Pos, colonPos source.Pos, value ast.Expr) *ast.MapElementLit {
return &ast.MapElementLit{Key: key, KeyPos: keyPos, ColonPos: colonPos, Value: value}
}
func mapLit(lbrace, rbrace scanner.Pos, list ...*ast.MapElementLit) *ast.MapLit {
func mapLit(lbrace, rbrace source.Pos, list ...*ast.MapElementLit) *ast.MapLit {
return &ast.MapLit{LBrace: lbrace, RBrace: rbrace, Elements: list}
}
@ -231,19 +231,19 @@ func funcLit(funcType *ast.FuncType, body *ast.BlockStmt) *ast.FuncLit {
return &ast.FuncLit{Type: funcType, Body: body}
}
func parenExpr(x ast.Expr, lparen, rparen scanner.Pos) *ast.ParenExpr {
func parenExpr(x ast.Expr, lparen, rparen source.Pos) *ast.ParenExpr {
return &ast.ParenExpr{Expr: x, LParen: lparen, RParen: rparen}
}
func callExpr(f ast.Expr, lparen, rparen scanner.Pos, args ...ast.Expr) *ast.CallExpr {
func callExpr(f ast.Expr, lparen, rparen source.Pos, args ...ast.Expr) *ast.CallExpr {
return &ast.CallExpr{Func: f, LParen: lparen, RParen: rparen, Args: args}
}
func indexExpr(x, index ast.Expr, lbrack, rbrack scanner.Pos) *ast.IndexExpr {
func indexExpr(x, index ast.Expr, lbrack, rbrack source.Pos) *ast.IndexExpr {
return &ast.IndexExpr{Expr: x, Index: index, LBrack: lbrack, RBrack: rbrack}
}
func sliceExpr(x, low, high ast.Expr, lbrack, rbrack scanner.Pos) *ast.SliceExpr {
func sliceExpr(x, low, high ast.Expr, lbrack, rbrack source.Pos) *ast.SliceExpr {
return &ast.SliceExpr{Expr: x, Low: low, High: high, LBrack: lbrack, RBrack: rbrack}
}

4
scanner/error_handler.go
View file

@ -1,3 +1,5 @@
package scanner
type ErrorHandler func(pos FilePos, msg string)
import "github.com/d5/tengo/source"
type ErrorHandler func(pos source.FilePos, msg string)

7
scanner/scanner.go
View file

@ -5,6 +5,7 @@ import (
"unicode"
"unicode/utf8"
"github.com/d5/tengo/source"
"github.com/d5/tengo/token"
)
@ -12,7 +13,7 @@ import (
const bom = 0xFEFF
type Scanner struct {
file *File // source file handle
file *source.File // source file handle
src []byte // source
ch rune // current character
offset int // character offset
@ -24,7 +25,7 @@ type Scanner struct {
mode Mode
}
func NewScanner(file *File, src []byte, errorHandler ErrorHandler, mode Mode) *Scanner {
func NewScanner(file *source.File, src []byte, errorHandler ErrorHandler, mode Mode) *Scanner {
if file.Size() != len(src) {
panic(fmt.Sprintf("file size (%d) does not match src len (%d)", file.Size(), len(src)))
}
@ -49,7 +50,7 @@ func (s *Scanner) ErrorCount() int {
return s.errorCount
}
func (s *Scanner) Scan() (tok token.Token, literal string, pos Pos) {
func (s *Scanner) Scan() (tok token.Token, literal string, pos source.Pos) {
s.skipWhitespace()
pos = s.file.FileSetPos(s.offset)

5
scanner/scanner_test.go
View file

@ -9,10 +9,11 @@ import (
"github.com/d5/tengo/assert"
"github.com/d5/tengo/scanner"
"github.com/d5/tengo/source"
"github.com/d5/tengo/token"
)
var testFileSet = scanner.NewFileSet()
var testFileSet = source.NewFileSet()
type scanResult struct {
Token token.Token
@ -220,7 +221,7 @@ func scanExpect(t *testing.T, input string, mode scanner.Mode, expected ...scanR
s := scanner.NewScanner(
testFile,
[]byte(input),
func(_ scanner.FilePos, msg string) { assert.Fail(t, msg) },
func(_ source.FilePos, msg string) { assert.Fail(t, msg) },
mode)
for idx, e := range expected {

4
script/script.go
View file

@ -6,7 +6,7 @@ import (
"github.com/d5/tengo/compiler"
"github.com/d5/tengo/objects"
"github.com/d5/tengo/parser"
"github.com/d5/tengo/scanner"
"github.com/d5/tengo/source"
"github.com/d5/tengo/vm"
)
@ -55,7 +55,7 @@ func (s *Script) Remove(name string) bool {
func (s *Script) Compile() (*Compiled, error) {
symbolTable, globals := s.prepCompile()
fileSet := scanner.NewFileSet()
fileSet := source.NewFileSet()
p := parser.NewParser(fileSet.AddFile("", -1, len(s.input)), s.input, nil)
file, err := p.ParseFile()

2
scanner/file.go → source/file.go
View file

@ -1,4 +1,4 @@
package scanner
package source
import (
"sync"

2
scanner/file_pos.go → source/file_pos.go
View file

@ -1,4 +1,4 @@
package scanner
package source
import "fmt"

2
scanner/file_set.go → source/file_set.go
View file

@ -1,4 +1,4 @@
package scanner
package source
import (
"sort"

2
scanner/pos.go → source/pos.go
View file

@ -1,4 +1,4 @@
package scanner
package source
type Pos int

4
tengo.go
View file

@ -3,12 +3,12 @@ package tengo
import (
"github.com/d5/tengo/compiler"
"github.com/d5/tengo/parser"
"github.com/d5/tengo/scanner"
"github.com/d5/tengo/source"
"github.com/d5/tengo/vm"
)
func Compile(input []byte, filename string) (*compiler.Bytecode, error) {
fileSet := scanner.NewFileSet()
fileSet := source.NewFileSet()
p := parser.NewParser(fileSet.AddFile(filename, -1, len(input)), input, nil)
file, err := p.ParseFile()

4
vm/vm_test.go
View file

@ -13,7 +13,7 @@ import (
"github.com/d5/tengo/compiler"
"github.com/d5/tengo/objects"
"github.com/d5/tengo/parser"
"github.com/d5/tengo/scanner"
"github.com/d5/tengo/source"
"github.com/d5/tengo/vm"
)
@ -246,7 +246,7 @@ func traceCompileRun(file *ast.File, symbols map[string]objects.Object) (res map
}
func parse(t *testing.T, input string) *ast.File {
testFileSet := scanner.NewFileSet()
testFileSet := source.NewFileSet()
testFile := testFileSet.AddFile("", -1, len(input))
file, err := parser.ParseFile(testFile, []byte(input), nil)