move file and fileset to source package

This commit is contained in:
Daniel Kang 2019-01-11 01:16:34 -08:00
parent b2a29342b1
commit 53af0d6328
54 changed files with 253 additions and 234 deletions

View file

@ -9,7 +9,7 @@ import (
"github.com/d5/tengo/compiler" "github.com/d5/tengo/compiler"
"github.com/d5/tengo/objects" "github.com/d5/tengo/objects"
"github.com/d5/tengo/scanner" "github.com/d5/tengo/source"
"github.com/d5/tengo/token" "github.com/d5/tengo/token"
) )
@ -133,8 +133,8 @@ func Equal(t *testing.T, expected, actual interface{}, msg ...interface{}) bool
if !equalSymbol(expected, actual.(compiler.Symbol)) { if !equalSymbol(expected, actual.(compiler.Symbol)) {
return failExpectedActual(t, expected, actual, msg...) return failExpectedActual(t, expected, actual, msg...)
} }
case scanner.Pos: case source.Pos:
if expected != actual.(scanner.Pos) { if expected != actual.(source.Pos) {
return failExpectedActual(t, expected, actual, msg...) return failExpectedActual(t, expected, actual, msg...)
} }
case token.Token: case token.Token:

View file

@ -3,22 +3,22 @@ package ast
import ( import (
"strings" "strings"
"github.com/d5/tengo/scanner" "github.com/d5/tengo/source"
) )
type ArrayLit struct { type ArrayLit struct {
Elements []Expr Elements []Expr
LBrack scanner.Pos LBrack source.Pos
RBrack scanner.Pos RBrack source.Pos
} }
func (e *ArrayLit) exprNode() {} func (e *ArrayLit) exprNode() {}
func (e *ArrayLit) Pos() scanner.Pos { func (e *ArrayLit) Pos() source.Pos {
return e.LBrack return e.LBrack
} }
func (e *ArrayLit) End() scanner.Pos { func (e *ArrayLit) End() source.Pos {
return e.RBrack + 1 return e.RBrack + 1
} }

View file

@ -3,7 +3,7 @@ package ast
import ( import (
"strings" "strings"
"github.com/d5/tengo/scanner" "github.com/d5/tengo/source"
"github.com/d5/tengo/token" "github.com/d5/tengo/token"
) )
@ -11,16 +11,16 @@ type AssignStmt struct {
Lhs []Expr Lhs []Expr
Rhs []Expr Rhs []Expr
Token token.Token Token token.Token
TokenPos scanner.Pos TokenPos source.Pos
} }
func (s *AssignStmt) stmtNode() {} func (s *AssignStmt) stmtNode() {}
func (s *AssignStmt) Pos() scanner.Pos { func (s *AssignStmt) Pos() source.Pos {
return s.Lhs[0].Pos() return s.Lhs[0].Pos()
} }
func (s *AssignStmt) End() scanner.Pos { func (s *AssignStmt) End() source.Pos {
return s.Rhs[len(s.Rhs)-1].End() return s.Rhs[len(s.Rhs)-1].End()
} }

View file

@ -1,19 +1,19 @@
package ast package ast
import "github.com/d5/tengo/scanner" import "github.com/d5/tengo/source"
type BadExpr struct { type BadExpr struct {
From scanner.Pos From source.Pos
To scanner.Pos To source.Pos
} }
func (e *BadExpr) exprNode() {} func (e *BadExpr) exprNode() {}
func (e *BadExpr) Pos() scanner.Pos { func (e *BadExpr) Pos() source.Pos {
return e.From return e.From
} }
func (e *BadExpr) End() scanner.Pos { func (e *BadExpr) End() source.Pos {
return e.To return e.To
} }

View file

@ -1,19 +1,19 @@
package ast package ast
import "github.com/d5/tengo/scanner" import "github.com/d5/tengo/source"
type BadStmt struct { type BadStmt struct {
From scanner.Pos From source.Pos
To scanner.Pos To source.Pos
} }
func (s *BadStmt) stmtNode() {} func (s *BadStmt) stmtNode() {}
func (s *BadStmt) Pos() scanner.Pos { func (s *BadStmt) Pos() source.Pos {
return s.From return s.From
} }
func (s *BadStmt) End() scanner.Pos { func (s *BadStmt) End() source.Pos {
return s.To return s.To
} }

View file

@ -1,7 +1,7 @@
package ast package ast
import ( import (
"github.com/d5/tengo/scanner" "github.com/d5/tengo/source"
"github.com/d5/tengo/token" "github.com/d5/tengo/token"
) )
@ -9,16 +9,16 @@ type BinaryExpr struct {
Lhs Expr Lhs Expr
Rhs Expr Rhs Expr
Token token.Token Token token.Token
TokenPos scanner.Pos TokenPos source.Pos
} }
func (e *BinaryExpr) exprNode() {} func (e *BinaryExpr) exprNode() {}
func (e *BinaryExpr) Pos() scanner.Pos { func (e *BinaryExpr) Pos() source.Pos {
return e.Lhs.Pos() return e.Lhs.Pos()
} }
func (e *BinaryExpr) End() scanner.Pos { func (e *BinaryExpr) End() source.Pos {
return e.Rhs.End() return e.Rhs.End()
} }

View file

@ -3,22 +3,22 @@ package ast
import ( import (
"strings" "strings"
"github.com/d5/tengo/scanner" "github.com/d5/tengo/source"
) )
type BlockStmt struct { type BlockStmt struct {
Stmts []Stmt Stmts []Stmt
LBrace scanner.Pos LBrace source.Pos
RBrace scanner.Pos RBrace source.Pos
} }
func (s *BlockStmt) stmtNode() {} func (s *BlockStmt) stmtNode() {}
func (s *BlockStmt) Pos() scanner.Pos { func (s *BlockStmt) Pos() source.Pos {
return s.LBrace return s.LBrace
} }
func (s *BlockStmt) End() scanner.Pos { func (s *BlockStmt) End() source.Pos {
return s.RBrace + 1 return s.RBrace + 1
} }

View file

@ -1,21 +1,21 @@
package ast package ast
import "github.com/d5/tengo/scanner" import "github.com/d5/tengo/source"
type BoolLit struct { type BoolLit struct {
Value bool Value bool
ValuePos scanner.Pos ValuePos source.Pos
Literal string Literal string
} }
func (e *BoolLit) exprNode() {} func (e *BoolLit) exprNode() {}
func (e *BoolLit) Pos() scanner.Pos { func (e *BoolLit) Pos() source.Pos {
return e.ValuePos return e.ValuePos
} }
func (e *BoolLit) End() scanner.Pos { func (e *BoolLit) End() source.Pos {
return scanner.Pos(int(e.ValuePos) + len(e.Literal)) return source.Pos(int(e.ValuePos) + len(e.Literal))
} }
func (e *BoolLit) String() string { func (e *BoolLit) String() string {

View file

@ -1,28 +1,28 @@
package ast package ast
import ( import (
"github.com/d5/tengo/scanner" "github.com/d5/tengo/source"
"github.com/d5/tengo/token" "github.com/d5/tengo/token"
) )
type BranchStmt struct { type BranchStmt struct {
Token token.Token Token token.Token
TokenPos scanner.Pos TokenPos source.Pos
Label *Ident Label *Ident
} }
func (s *BranchStmt) stmtNode() {} func (s *BranchStmt) stmtNode() {}
func (s *BranchStmt) Pos() scanner.Pos { func (s *BranchStmt) Pos() source.Pos {
return s.TokenPos return s.TokenPos
} }
func (s *BranchStmt) End() scanner.Pos { func (s *BranchStmt) End() source.Pos {
if s.Label != nil { if s.Label != nil {
return s.Label.End() return s.Label.End()
} }
return scanner.Pos(int(s.TokenPos) + len(s.Token.String())) return source.Pos(int(s.TokenPos) + len(s.Token.String()))
} }
func (s *BranchStmt) String() string { func (s *BranchStmt) String() string {

View file

@ -3,23 +3,23 @@ package ast
import ( import (
"strings" "strings"
"github.com/d5/tengo/scanner" "github.com/d5/tengo/source"
) )
type CallExpr struct { type CallExpr struct {
Func Expr Func Expr
LParen scanner.Pos LParen source.Pos
Args []Expr Args []Expr
RParen scanner.Pos RParen source.Pos
} }
func (e *CallExpr) exprNode() {} func (e *CallExpr) exprNode() {}
func (e *CallExpr) Pos() scanner.Pos { func (e *CallExpr) Pos() source.Pos {
return e.Func.Pos() return e.Func.Pos()
} }
func (e *CallExpr) End() scanner.Pos { func (e *CallExpr) End() source.Pos {
return e.RParen + 1 return e.RParen + 1
} }

View file

@ -1,21 +1,21 @@
package ast package ast
import "github.com/d5/tengo/scanner" import "github.com/d5/tengo/source"
type CharLit struct { type CharLit struct {
Value rune Value rune
ValuePos scanner.Pos ValuePos source.Pos
Literal string Literal string
} }
func (e *CharLit) exprNode() {} func (e *CharLit) exprNode() {}
func (e *CharLit) Pos() scanner.Pos { func (e *CharLit) Pos() source.Pos {
return e.ValuePos return e.ValuePos
} }
func (e *CharLit) End() scanner.Pos { func (e *CharLit) End() source.Pos {
return scanner.Pos(int(e.ValuePos) + len(e.Literal)) return source.Pos(int(e.ValuePos) + len(e.Literal))
} }
func (e *CharLit) String() string { func (e *CharLit) String() string {

View file

@ -1,19 +1,19 @@
package ast package ast
import "github.com/d5/tengo/scanner" import "github.com/d5/tengo/source"
type EmptyStmt struct { type EmptyStmt struct {
Semicolon scanner.Pos Semicolon source.Pos
Implicit bool Implicit bool
} }
func (s *EmptyStmt) stmtNode() {} func (s *EmptyStmt) stmtNode() {}
func (s *EmptyStmt) Pos() scanner.Pos { func (s *EmptyStmt) Pos() source.Pos {
return s.Semicolon return s.Semicolon
} }
func (s *EmptyStmt) End() scanner.Pos { func (s *EmptyStmt) End() source.Pos {
if s.Implicit { if s.Implicit {
return s.Semicolon return s.Semicolon
} }

View file

@ -1,6 +1,6 @@
package ast package ast
import "github.com/d5/tengo/scanner" import "github.com/d5/tengo/source"
type ExprStmt struct { type ExprStmt struct {
Expr Expr Expr Expr
@ -8,11 +8,11 @@ type ExprStmt struct {
func (s *ExprStmt) stmtNode() {} func (s *ExprStmt) stmtNode() {}
func (s *ExprStmt) Pos() scanner.Pos { func (s *ExprStmt) Pos() source.Pos {
return s.Expr.Pos() return s.Expr.Pos()
} }
func (s *ExprStmt) End() scanner.Pos { func (s *ExprStmt) End() source.Pos {
return s.Expr.End() return s.Expr.End()
} }

View file

@ -3,20 +3,20 @@ package ast
import ( import (
"strings" "strings"
"github.com/d5/tengo/scanner" "github.com/d5/tengo/source"
) )
type File struct { type File struct {
InputFile *scanner.File InputFile *source.File
Stmts []Stmt Stmts []Stmt
} }
func (n *File) Pos() scanner.Pos { func (n *File) Pos() source.Pos {
return scanner.Pos(n.InputFile.Base()) return source.Pos(n.InputFile.Base())
} }
func (n *File) End() scanner.Pos { func (n *File) End() source.Pos {
return scanner.Pos(n.InputFile.Base() + n.InputFile.Size()) return source.Pos(n.InputFile.Base() + n.InputFile.Size())
} }
func (n *File) String() string { func (n *File) String() string {

View file

@ -1,21 +1,21 @@
package ast package ast
import "github.com/d5/tengo/scanner" import "github.com/d5/tengo/source"
type FloatLit struct { type FloatLit struct {
Value float64 Value float64
ValuePos scanner.Pos ValuePos source.Pos
Literal string Literal string
} }
func (e *FloatLit) exprNode() {} func (e *FloatLit) exprNode() {}
func (e *FloatLit) Pos() scanner.Pos { func (e *FloatLit) Pos() source.Pos {
return e.ValuePos return e.ValuePos
} }
func (e *FloatLit) End() scanner.Pos { func (e *FloatLit) End() source.Pos {
return scanner.Pos(int(e.ValuePos) + len(e.Literal)) return source.Pos(int(e.ValuePos) + len(e.Literal))
} }
func (e *FloatLit) String() string { func (e *FloatLit) String() string {

View file

@ -1,9 +1,9 @@
package ast package ast
import "github.com/d5/tengo/scanner" import "github.com/d5/tengo/source"
type ForInStmt struct { type ForInStmt struct {
ForPos scanner.Pos ForPos source.Pos
Key *Ident Key *Ident
Value *Ident Value *Ident
Iterable Expr Iterable Expr
@ -12,11 +12,11 @@ type ForInStmt struct {
func (s *ForInStmt) stmtNode() {} func (s *ForInStmt) stmtNode() {}
func (s *ForInStmt) Pos() scanner.Pos { func (s *ForInStmt) Pos() source.Pos {
return s.ForPos return s.ForPos
} }
func (s *ForInStmt) End() scanner.Pos { func (s *ForInStmt) End() source.Pos {
return s.Body.End() return s.Body.End()
} }

View file

@ -1,9 +1,9 @@
package ast package ast
import "github.com/d5/tengo/scanner" import "github.com/d5/tengo/source"
type ForStmt struct { type ForStmt struct {
ForPos scanner.Pos ForPos source.Pos
Init Stmt Init Stmt
Cond Expr Cond Expr
Post Stmt Post Stmt
@ -12,11 +12,11 @@ type ForStmt struct {
func (s *ForStmt) stmtNode() {} func (s *ForStmt) stmtNode() {}
func (s *ForStmt) Pos() scanner.Pos { func (s *ForStmt) Pos() source.Pos {
return s.ForPos return s.ForPos
} }
func (s *ForStmt) End() scanner.Pos { func (s *ForStmt) End() source.Pos {
return s.Body.End() return s.Body.End()
} }

View file

@ -1,6 +1,6 @@
package ast package ast
import "github.com/d5/tengo/scanner" import "github.com/d5/tengo/source"
type FuncLit struct { type FuncLit struct {
Type *FuncType Type *FuncType
@ -9,11 +9,11 @@ type FuncLit struct {
func (e *FuncLit) exprNode() {} func (e *FuncLit) exprNode() {}
func (e *FuncLit) Pos() scanner.Pos { func (e *FuncLit) Pos() source.Pos {
return e.Type.Pos() return e.Type.Pos()
} }
func (e *FuncLit) End() scanner.Pos { func (e *FuncLit) End() source.Pos {
return e.Body.End() return e.Body.End()
} }

View file

@ -1,19 +1,19 @@
package ast package ast
import "github.com/d5/tengo/scanner" import "github.com/d5/tengo/source"
type FuncType struct { type FuncType struct {
FuncPos scanner.Pos FuncPos source.Pos
Params *IdentList Params *IdentList
} }
func (e *FuncType) exprNode() {} func (e *FuncType) exprNode() {}
func (e *FuncType) Pos() scanner.Pos { func (e *FuncType) Pos() source.Pos {
return e.FuncPos return e.FuncPos
} }
func (e *FuncType) End() scanner.Pos { func (e *FuncType) End() source.Pos {
return e.Params.End() return e.Params.End()
} }

View file

@ -1,20 +1,20 @@
package ast package ast
import "github.com/d5/tengo/scanner" import "github.com/d5/tengo/source"
type Ident struct { type Ident struct {
Name string Name string
NamePos scanner.Pos NamePos source.Pos
} }
func (e *Ident) exprNode() {} func (e *Ident) exprNode() {}
func (e *Ident) Pos() scanner.Pos { func (e *Ident) Pos() source.Pos {
return e.NamePos return e.NamePos
} }
func (e *Ident) End() scanner.Pos { func (e *Ident) End() source.Pos {
return scanner.Pos(int(e.NamePos) + len(e.Name)) return source.Pos(int(e.NamePos) + len(e.Name))
} }
func (e *Ident) String() string { func (e *Ident) String() string {

View file

@ -3,16 +3,16 @@ package ast
import ( import (
"strings" "strings"
"github.com/d5/tengo/scanner" "github.com/d5/tengo/source"
) )
type IdentList struct { type IdentList struct {
LParen scanner.Pos LParen source.Pos
List []*Ident List []*Ident
RParen scanner.Pos RParen source.Pos
} }
func (n *IdentList) Pos() scanner.Pos { func (n *IdentList) Pos() source.Pos {
if n.LParen.IsValid() { if n.LParen.IsValid() {
return n.LParen return n.LParen
} }
@ -21,10 +21,10 @@ func (n *IdentList) Pos() scanner.Pos {
return n.List[0].Pos() return n.List[0].Pos()
} }
return scanner.NoPos return source.NoPos
} }
func (n *IdentList) End() scanner.Pos { func (n *IdentList) End() source.Pos {
if n.RParen.IsValid() { if n.RParen.IsValid() {
return n.RParen + 1 return n.RParen + 1
} }
@ -33,7 +33,7 @@ func (n *IdentList) End() scanner.Pos {
return n.List[l-1].End() return n.List[l-1].End()
} }
return scanner.NoPos return source.NoPos
} }
func (n *IdentList) NumFields() int { func (n *IdentList) NumFields() int {

View file

@ -1,9 +1,9 @@
package ast package ast
import "github.com/d5/tengo/scanner" import "github.com/d5/tengo/source"
type IfStmt struct { type IfStmt struct {
IfPos scanner.Pos IfPos source.Pos
Init Stmt Init Stmt
Cond Expr Cond Expr
Body *BlockStmt Body *BlockStmt
@ -12,11 +12,11 @@ type IfStmt struct {
func (s *IfStmt) stmtNode() {} func (s *IfStmt) stmtNode() {}
func (s *IfStmt) Pos() scanner.Pos { func (s *IfStmt) Pos() source.Pos {
return s.IfPos return s.IfPos
} }
func (s *IfStmt) End() scanner.Pos { func (s *IfStmt) End() source.Pos {
if s.Else != nil { if s.Else != nil {
return s.Else.End() return s.Else.End()
} }

View file

@ -1,24 +1,24 @@
package ast package ast
import ( import (
"github.com/d5/tengo/scanner" "github.com/d5/tengo/source"
"github.com/d5/tengo/token" "github.com/d5/tengo/token"
) )
type IncDecStmt struct { type IncDecStmt struct {
Expr Expr Expr Expr
Token token.Token Token token.Token
TokenPos scanner.Pos TokenPos source.Pos
} }
func (s *IncDecStmt) stmtNode() {} func (s *IncDecStmt) stmtNode() {}
func (s *IncDecStmt) Pos() scanner.Pos { func (s *IncDecStmt) Pos() source.Pos {
return s.Expr.Pos() return s.Expr.Pos()
} }
func (s *IncDecStmt) End() scanner.Pos { func (s *IncDecStmt) End() source.Pos {
return scanner.Pos(int(s.TokenPos) + 2) return source.Pos(int(s.TokenPos) + 2)
} }
func (s *IncDecStmt) String() string { func (s *IncDecStmt) String() string {

View file

@ -1,21 +1,21 @@
package ast package ast
import "github.com/d5/tengo/scanner" import "github.com/d5/tengo/source"
type IndexExpr struct { type IndexExpr struct {
Expr Expr Expr Expr
LBrack scanner.Pos LBrack source.Pos
Index Expr Index Expr
RBrack scanner.Pos RBrack source.Pos
} }
func (e *IndexExpr) exprNode() {} func (e *IndexExpr) exprNode() {}
func (e *IndexExpr) Pos() scanner.Pos { func (e *IndexExpr) Pos() source.Pos {
return e.Expr.Pos() return e.Expr.Pos()
} }
func (e *IndexExpr) End() scanner.Pos { func (e *IndexExpr) End() source.Pos {
return e.RBrack + 1 return e.RBrack + 1
} }

View file

@ -1,21 +1,21 @@
package ast package ast
import "github.com/d5/tengo/scanner" import "github.com/d5/tengo/source"
type IntLit struct { type IntLit struct {
Value int64 Value int64
ValuePos scanner.Pos ValuePos source.Pos
Literal string Literal string
} }
func (e *IntLit) exprNode() {} func (e *IntLit) exprNode() {}
func (e *IntLit) Pos() scanner.Pos { func (e *IntLit) Pos() source.Pos {
return e.ValuePos return e.ValuePos
} }
func (e *IntLit) End() scanner.Pos { func (e *IntLit) End() source.Pos {
return scanner.Pos(int(e.ValuePos) + len(e.Literal)) return source.Pos(int(e.ValuePos) + len(e.Literal))
} }
func (e *IntLit) String() string { func (e *IntLit) String() string {

View file

@ -1,21 +1,21 @@
package ast package ast
import "github.com/d5/tengo/scanner" import "github.com/d5/tengo/source"
type MapElementLit struct { type MapElementLit struct {
Key string Key string
KeyPos scanner.Pos KeyPos source.Pos
ColonPos scanner.Pos ColonPos source.Pos
Value Expr Value Expr
} }
func (e *MapElementLit) exprNode() {} func (e *MapElementLit) exprNode() {}
func (e *MapElementLit) Pos() scanner.Pos { func (e *MapElementLit) Pos() source.Pos {
return e.KeyPos return e.KeyPos
} }
func (e *MapElementLit) End() scanner.Pos { func (e *MapElementLit) End() source.Pos {
return e.Value.End() return e.Value.End()
} }

View file

@ -3,22 +3,22 @@ package ast
import ( import (
"strings" "strings"
"github.com/d5/tengo/scanner" "github.com/d5/tengo/source"
) )
type MapLit struct { type MapLit struct {
LBrace scanner.Pos LBrace source.Pos
Elements []*MapElementLit Elements []*MapElementLit
RBrace scanner.Pos RBrace source.Pos
} }
func (e *MapLit) exprNode() {} func (e *MapLit) exprNode() {}
func (e *MapLit) Pos() scanner.Pos { func (e *MapLit) Pos() source.Pos {
return e.LBrace return e.LBrace
} }
func (e *MapLit) End() scanner.Pos { func (e *MapLit) End() source.Pos {
return e.RBrace + 1 return e.RBrace + 1
} }

View file

@ -1,11 +1,9 @@
package ast package ast
import ( import "github.com/d5/tengo/source"
"github.com/d5/tengo/scanner"
)
type Node interface { type Node interface {
Pos() scanner.Pos Pos() source.Pos
End() scanner.Pos End() source.Pos
String() string String() string
} }

View file

@ -1,20 +1,20 @@
package ast package ast
import "github.com/d5/tengo/scanner" import "github.com/d5/tengo/source"
type ParenExpr struct { type ParenExpr struct {
Expr Expr Expr Expr
LParen scanner.Pos LParen source.Pos
RParen scanner.Pos RParen source.Pos
} }
func (e *ParenExpr) exprNode() {} func (e *ParenExpr) exprNode() {}
func (e *ParenExpr) Pos() scanner.Pos { func (e *ParenExpr) Pos() source.Pos {
return e.LParen return e.LParen
} }
func (e *ParenExpr) End() scanner.Pos { func (e *ParenExpr) End() source.Pos {
return e.RParen + 1 return e.RParen + 1
} }

View file

@ -3,21 +3,21 @@ package ast
import ( import (
"strings" "strings"
"github.com/d5/tengo/scanner" "github.com/d5/tengo/source"
) )
type ReturnStmt struct { type ReturnStmt struct {
ReturnPos scanner.Pos ReturnPos source.Pos
Results []Expr Results []Expr
} }
func (s *ReturnStmt) stmtNode() {} func (s *ReturnStmt) stmtNode() {}
func (s *ReturnStmt) Pos() scanner.Pos { func (s *ReturnStmt) Pos() source.Pos {
return s.ReturnPos return s.ReturnPos
} }
func (s *ReturnStmt) End() scanner.Pos { func (s *ReturnStmt) End() source.Pos {
if n := len(s.Results); n > 0 { if n := len(s.Results); n > 0 {
return s.Results[n-1].End() return s.Results[n-1].End()
} }

View file

@ -1,6 +1,6 @@
package ast package ast
import "github.com/d5/tengo/scanner" import "github.com/d5/tengo/source"
type SelectorExpr struct { type SelectorExpr struct {
Expr Expr Expr Expr
@ -9,11 +9,11 @@ type SelectorExpr struct {
func (e *SelectorExpr) exprNode() {} func (e *SelectorExpr) exprNode() {}
func (e *SelectorExpr) Pos() scanner.Pos { func (e *SelectorExpr) Pos() source.Pos {
return e.Expr.Pos() return e.Expr.Pos()
} }
func (e *SelectorExpr) End() scanner.Pos { func (e *SelectorExpr) End() source.Pos {
return e.Sel.End() return e.Sel.End()
} }

View file

@ -1,22 +1,22 @@
package ast package ast
import "github.com/d5/tengo/scanner" import "github.com/d5/tengo/source"
type SliceExpr struct { type SliceExpr struct {
Expr Expr Expr Expr
LBrack scanner.Pos LBrack source.Pos
Low Expr Low Expr
High Expr High Expr
RBrack scanner.Pos RBrack source.Pos
} }
func (e *SliceExpr) exprNode() {} func (e *SliceExpr) exprNode() {}
func (e *SliceExpr) Pos() scanner.Pos { func (e *SliceExpr) Pos() source.Pos {
return e.Expr.Pos() return e.Expr.Pos()
} }
func (e *SliceExpr) End() scanner.Pos { func (e *SliceExpr) End() source.Pos {
return e.RBrack + 1 return e.RBrack + 1
} }

View file

@ -1,21 +1,21 @@
package ast package ast
import "github.com/d5/tengo/scanner" import "github.com/d5/tengo/source"
type StringLit struct { type StringLit struct {
Value string Value string
ValuePos scanner.Pos ValuePos source.Pos
Literal string Literal string
} }
func (e *StringLit) exprNode() {} func (e *StringLit) exprNode() {}
func (e *StringLit) Pos() scanner.Pos { func (e *StringLit) Pos() source.Pos {
return e.ValuePos return e.ValuePos
} }
func (e *StringLit) End() scanner.Pos { func (e *StringLit) End() source.Pos {
return scanner.Pos(int(e.ValuePos) + len(e.Literal)) return source.Pos(int(e.ValuePos) + len(e.Literal))
} }
func (e *StringLit) String() string { func (e *StringLit) String() string {

View file

@ -1,23 +1,23 @@
package ast package ast
import ( import (
"github.com/d5/tengo/scanner" "github.com/d5/tengo/source"
"github.com/d5/tengo/token" "github.com/d5/tengo/token"
) )
type UnaryExpr struct { type UnaryExpr struct {
Expr Expr Expr Expr
Token token.Token Token token.Token
TokenPos scanner.Pos TokenPos source.Pos
} }
func (e *UnaryExpr) exprNode() {} func (e *UnaryExpr) exprNode() {}
func (e *UnaryExpr) Pos() scanner.Pos { func (e *UnaryExpr) Pos() source.Pos {
return e.Expr.Pos() return e.Expr.Pos()
} }
func (e *UnaryExpr) End() scanner.Pos { func (e *UnaryExpr) End() source.Pos {
return e.Expr.End() return e.Expr.End()
} }

View file

@ -1,18 +1,18 @@
package ast package ast
import "github.com/d5/tengo/scanner" import "github.com/d5/tengo/source"
type UndefinedLit struct { type UndefinedLit struct {
TokenPos scanner.Pos TokenPos source.Pos
} }
func (e *UndefinedLit) exprNode() {} func (e *UndefinedLit) exprNode() {}
func (e *UndefinedLit) Pos() scanner.Pos { func (e *UndefinedLit) Pos() source.Pos {
return e.TokenPos return e.TokenPos
} }
func (e *UndefinedLit) End() scanner.Pos { func (e *UndefinedLit) End() source.Pos {
return e.TokenPos + 9 // len(undefined) == 9 return e.TokenPos + 9 // len(undefined) == 9
} }

View file

@ -8,7 +8,7 @@ import (
"github.com/d5/tengo/compiler" "github.com/d5/tengo/compiler"
"github.com/d5/tengo/objects" "github.com/d5/tengo/objects"
"github.com/d5/tengo/parser" "github.com/d5/tengo/parser"
"github.com/d5/tengo/scanner" "github.com/d5/tengo/source"
"github.com/d5/tengo/vm" "github.com/d5/tengo/vm"
) )
@ -126,7 +126,7 @@ func runBench(input []byte) (parseTime time.Duration, compileTime time.Duration,
} }
func parse(input []byte) (time.Duration, *ast.File, error) { func parse(input []byte) (time.Duration, *ast.File, error) {
fileSet := scanner.NewFileSet() fileSet := source.NewFileSet()
inputFile := fileSet.AddFile("test", -1, len(input)) inputFile := fileSet.AddFile("test", -1, len(input))
start := time.Now() start := time.Now()

View file

@ -11,7 +11,7 @@ import (
"github.com/d5/tengo/compiler" "github.com/d5/tengo/compiler"
"github.com/d5/tengo/objects" "github.com/d5/tengo/objects"
"github.com/d5/tengo/parser" "github.com/d5/tengo/parser"
"github.com/d5/tengo/scanner" "github.com/d5/tengo/source"
"github.com/d5/tengo/vm" "github.com/d5/tengo/vm"
) )
@ -33,7 +33,7 @@ func main() {
func startRepl(in io.Reader, out io.Writer) { func startRepl(in io.Reader, out io.Writer) {
stdin := bufio.NewScanner(in) stdin := bufio.NewScanner(in)
fileSet := scanner.NewFileSet() fileSet := source.NewFileSet()
globals := make([]*objects.Object, vm.GlobalsSize) globals := make([]*objects.Object, vm.GlobalsSize)
symbolTable := compiler.NewSymbolTable() symbolTable := compiler.NewSymbolTable()

View file

@ -10,7 +10,7 @@ import (
"github.com/d5/tengo/compiler" "github.com/d5/tengo/compiler"
"github.com/d5/tengo/objects" "github.com/d5/tengo/objects"
"github.com/d5/tengo/parser" "github.com/d5/tengo/parser"
"github.com/d5/tengo/scanner" "github.com/d5/tengo/source"
) )
func TestCompiler_Compile(t *testing.T) { func TestCompiler_Compile(t *testing.T) {
@ -884,7 +884,7 @@ func (o *tracer) Write(p []byte) (n int, err error) {
} }
func traceCompile(input string, symbols map[string]objects.Object) (res *compiler.Bytecode, trace []string, err error) { func traceCompile(input string, symbols map[string]objects.Object) (res *compiler.Bytecode, trace []string, err error) {
fileSet := scanner.NewFileSet() fileSet := source.NewFileSet()
file := fileSet.AddFile("test", -1, len(input)) file := fileSet.AddFile("test", -1, len(input))
p := parser.NewParser(file, []byte(input), nil) p := parser.NewParser(file, []byte(input), nil)

View file

@ -1,9 +1,9 @@
package parser package parser
import "github.com/d5/tengo/scanner" import "github.com/d5/tengo/source"
type Error struct { type Error struct {
Pos scanner.FilePos Pos source.FilePos
Msg string Msg string
} }

View file

@ -4,12 +4,12 @@ import (
"fmt" "fmt"
"sort" "sort"
"github.com/d5/tengo/scanner" "github.com/d5/tengo/source"
) )
type ErrorList []*Error type ErrorList []*Error
func (p *ErrorList) Add(pos scanner.FilePos, msg string) { func (p *ErrorList) Add(pos source.FilePos, msg string) {
*p = append(*p, &Error{pos, msg}) *p = append(*p, &Error{pos, msg})
} }
@ -51,7 +51,7 @@ func (p ErrorList) Sort() {
func (p *ErrorList) RemoveMultiples() { func (p *ErrorList) RemoveMultiples() {
sort.Sort(p) sort.Sort(p)
var last scanner.FilePos // initial last.Line is != any legal error line var last source.FilePos // initial last.Line is != any legal error line
i := 0 i := 0
for _, e := range *p { for _, e := range *p {

View file

@ -4,10 +4,10 @@ import (
"io" "io"
"github.com/d5/tengo/ast" "github.com/d5/tengo/ast"
"github.com/d5/tengo/scanner" "github.com/d5/tengo/source"
) )
func ParseFile(file *scanner.File, src []byte, trace io.Writer) (res *ast.File, err error) { func ParseFile(file *source.File, src []byte, trace io.Writer) (res *ast.File, err error) {
p := NewParser(file, src, trace) p := NewParser(file, src, trace)
defer func() { defer func() {

16
parser/parse_source.go Normal file
View file

@ -0,0 +1,16 @@
package parser
import (
"io"
"github.com/d5/tengo/ast"
"github.com/d5/tengo/source"
)
// ParseSource parses source code 'src' and builds an AST.
func ParseSource(src []byte, trace io.Writer) (res *ast.File, err error) {
fileSet := source.NewFileSet()
file := fileSet.AddFile("", -1, len(src))
return ParseFile(file, src, trace)
}

View file

@ -7,34 +7,35 @@ import (
"github.com/d5/tengo/ast" "github.com/d5/tengo/ast"
"github.com/d5/tengo/scanner" "github.com/d5/tengo/scanner"
"github.com/d5/tengo/source"
"github.com/d5/tengo/token" "github.com/d5/tengo/token"
) )
type bailout struct{} type bailout struct{}
type Parser struct { type Parser struct {
file *scanner.File file *source.File
errors ErrorList errors ErrorList
scanner *scanner.Scanner scanner *scanner.Scanner
pos scanner.Pos pos source.Pos
token token.Token token token.Token
tokenLit string tokenLit string
exprLevel int // < 0: in control clause, >= 0: in expression exprLevel int // < 0: in control clause, >= 0: in expression
syncPos scanner.Pos // last sync position syncPos source.Pos // last sync position
syncCount int // number of advance calls without progress syncCount int // number of advance calls without progress
trace bool trace bool
indent int indent int
traceOut io.Writer traceOut io.Writer
} }
func NewParser(file *scanner.File, src []byte, trace io.Writer) *Parser { func NewParser(file *source.File, src []byte, trace io.Writer) *Parser {
p := &Parser{ p := &Parser{
file: file, file: file,
trace: trace != nil, trace: trace != nil,
traceOut: trace, traceOut: trace,
} }
p.scanner = scanner.NewScanner(p.file, src, func(pos scanner.FilePos, msg string) { p.scanner = scanner.NewScanner(p.file, src, func(pos source.FilePos, msg string) {
p.errors.Add(pos, msg) p.errors.Add(pos, msg)
}, 0) }, 0)
@ -910,7 +911,7 @@ func (p *Parser) parseMapLit() *ast.MapLit {
} }
} }
func (p *Parser) expect(token token.Token) scanner.Pos { func (p *Parser) expect(token token.Token) source.Pos {
pos := p.pos pos := p.pos
if p.token != token { if p.token != token {
@ -955,7 +956,7 @@ func (p *Parser) advance(to map[token.Token]bool) {
} }
} }
func (p *Parser) error(pos scanner.Pos, msg string) { func (p *Parser) error(pos source.Pos, msg string) {
filePos := p.file.Position(pos) filePos := p.file.Position(pos)
n := len(p.errors) n := len(p.errors)
@ -972,7 +973,7 @@ func (p *Parser) error(pos scanner.Pos, msg string) {
p.errors.Add(filePos, msg) p.errors.Add(filePos, msg)
} }
func (p *Parser) errorExpected(pos scanner.Pos, msg string) { func (p *Parser) errorExpected(pos source.Pos, msg string) {
msg = "expected " + msg msg = "expected " + msg
if pos == p.pos { if pos == p.pos {
// error happened at the current position: provide more specific // error happened at the current position: provide more specific
@ -1023,12 +1024,12 @@ func (p *Parser) printTrace(a ...interface{}) {
_, _ = fmt.Fprintln(p.traceOut, a...) _, _ = fmt.Fprintln(p.traceOut, a...)
} }
func (p *Parser) safePos(pos scanner.Pos) scanner.Pos { func (p *Parser) safePos(pos source.Pos) source.Pos {
fileBase := p.file.Base() fileBase := p.file.Base()
fileSize := p.file.Size() fileSize := p.file.Size()
if int(pos) < fileBase || int(pos) > fileBase+fileSize { if int(pos) < fileBase || int(pos) > fileBase+fileSize {
return scanner.Pos(fileBase + fileSize) return source.Pos(fileBase + fileSize)
} }
return pos return pos

View file

@ -9,11 +9,11 @@ import (
"github.com/d5/tengo/assert" "github.com/d5/tengo/assert"
"github.com/d5/tengo/ast" "github.com/d5/tengo/ast"
"github.com/d5/tengo/parser" "github.com/d5/tengo/parser"
"github.com/d5/tengo/scanner" "github.com/d5/tengo/source"
"github.com/d5/tengo/token" "github.com/d5/tengo/token"
) )
type pfn func(int, int) scanner.Pos // position conversion function type pfn func(int, int) source.Pos // position conversion function
type expectedFn func(pos pfn) []ast.Stmt // callback function to return expected results type expectedFn func(pos pfn) []ast.Stmt // callback function to return expected results
type tracer struct { type tracer struct {
@ -35,7 +35,7 @@ func (o *tracer) Write(p []byte) (n int, err error) {
//} //}
func expect(t *testing.T, input string, fn expectedFn) (ok bool) { func expect(t *testing.T, input string, fn expectedFn) (ok bool) {
testFileSet := scanner.NewFileSet() testFileSet := source.NewFileSet()
testFile := testFileSet.AddFile("", -1, len(input)) testFile := testFileSet.AddFile("", -1, len(input))
defer func() { defer func() {
@ -55,8 +55,8 @@ func expect(t *testing.T, input string, fn expectedFn) (ok bool) {
return return
} }
expected := fn(func(line, column int) scanner.Pos { expected := fn(func(line, column int) source.Pos {
return scanner.Pos(int(testFile.LineStart(line)) + (column - 1)) return source.Pos(int(testFile.LineStart(line)) + (column - 1))
}) })
if !assert.Equal(t, len(expected), len(actual.Stmts)) { if !assert.Equal(t, len(expected), len(actual.Stmts)) {
@ -75,7 +75,7 @@ func expect(t *testing.T, input string, fn expectedFn) (ok bool) {
} }
func expectError(t *testing.T, input string) (ok bool) { func expectError(t *testing.T, input string) (ok bool) {
testFileSet := scanner.NewFileSet() testFileSet := source.NewFileSet()
testFile := testFileSet.AddFile("", -1, len(input)) testFile := testFileSet.AddFile("", -1, len(input))
defer func() { defer func() {
@ -98,7 +98,7 @@ func expectError(t *testing.T, input string) (ok bool) {
} }
func expectString(t *testing.T, input, expected string) (ok bool) { func expectString(t *testing.T, input, expected string) (ok bool) {
testFileSet := scanner.NewFileSet() testFileSet := source.NewFileSet()
testFile := testFileSet.AddFile("", -1, len(input)) testFile := testFileSet.AddFile("", -1, len(input))
defer func() { defer func() {
@ -125,7 +125,7 @@ func expectString(t *testing.T, input, expected string) (ok bool) {
} }
//func printTrace(input string) { //func printTrace(input string) {
// testFileSet := scanner.NewFileSet() // testFileSet := source.NewFileSet()
// testFile := testFileSet.AddFile("", -1, len(input)) // testFile := testFileSet.AddFile("", -1, len(input))
// //
// _, _ = parser.ParseFile(testFile, []byte(input), &slowPrinter{}) // _, _ = parser.ParseFile(testFile, []byte(input), &slowPrinter{})
@ -139,55 +139,55 @@ func exprStmt(x ast.Expr) *ast.ExprStmt {
return &ast.ExprStmt{Expr: x} return &ast.ExprStmt{Expr: x}
} }
func assignStmt(lhs, rhs []ast.Expr, token token.Token, pos scanner.Pos) *ast.AssignStmt { func assignStmt(lhs, rhs []ast.Expr, token token.Token, pos source.Pos) *ast.AssignStmt {
return &ast.AssignStmt{Lhs: lhs, Rhs: rhs, Token: token, TokenPos: pos} return &ast.AssignStmt{Lhs: lhs, Rhs: rhs, Token: token, TokenPos: pos}
} }
func emptyStmt(implicit bool, pos scanner.Pos) *ast.EmptyStmt { func emptyStmt(implicit bool, pos source.Pos) *ast.EmptyStmt {
return &ast.EmptyStmt{Implicit: implicit, Semicolon: pos} return &ast.EmptyStmt{Implicit: implicit, Semicolon: pos}
} }
func returnStmt(pos scanner.Pos, results ...ast.Expr) *ast.ReturnStmt { func returnStmt(pos source.Pos, results ...ast.Expr) *ast.ReturnStmt {
return &ast.ReturnStmt{Results: results, ReturnPos: pos} return &ast.ReturnStmt{Results: results, ReturnPos: pos}
} }
func forStmt(init ast.Stmt, cond ast.Expr, post ast.Stmt, body *ast.BlockStmt, pos scanner.Pos) *ast.ForStmt { func forStmt(init ast.Stmt, cond ast.Expr, post ast.Stmt, body *ast.BlockStmt, pos source.Pos) *ast.ForStmt {
return &ast.ForStmt{Cond: cond, Init: init, Post: post, Body: body, ForPos: pos} return &ast.ForStmt{Cond: cond, Init: init, Post: post, Body: body, ForPos: pos}
} }
func forInStmt(key, value *ast.Ident, seq ast.Expr, body *ast.BlockStmt, pos scanner.Pos) *ast.ForInStmt { func forInStmt(key, value *ast.Ident, seq ast.Expr, body *ast.BlockStmt, pos source.Pos) *ast.ForInStmt {
return &ast.ForInStmt{Key: key, Value: value, Iterable: seq, Body: body, ForPos: pos} return &ast.ForInStmt{Key: key, Value: value, Iterable: seq, Body: body, ForPos: pos}
} }
func ifStmt(init ast.Stmt, cond ast.Expr, body *ast.BlockStmt, elseStmt ast.Stmt, pos scanner.Pos) *ast.IfStmt { func ifStmt(init ast.Stmt, cond ast.Expr, body *ast.BlockStmt, elseStmt ast.Stmt, pos source.Pos) *ast.IfStmt {
return &ast.IfStmt{Init: init, Cond: cond, Body: body, Else: elseStmt, IfPos: pos} return &ast.IfStmt{Init: init, Cond: cond, Body: body, Else: elseStmt, IfPos: pos}
} }
func incDecStmt(expr ast.Expr, tok token.Token, pos scanner.Pos) *ast.IncDecStmt { func incDecStmt(expr ast.Expr, tok token.Token, pos source.Pos) *ast.IncDecStmt {
return &ast.IncDecStmt{Expr: expr, Token: tok, TokenPos: pos} return &ast.IncDecStmt{Expr: expr, Token: tok, TokenPos: pos}
} }
func funcType(params *ast.IdentList, pos scanner.Pos) *ast.FuncType { func funcType(params *ast.IdentList, pos source.Pos) *ast.FuncType {
return &ast.FuncType{Params: params, FuncPos: pos} return &ast.FuncType{Params: params, FuncPos: pos}
} }
func blockStmt(lbrace, rbrace scanner.Pos, list ...ast.Stmt) *ast.BlockStmt { func blockStmt(lbrace, rbrace source.Pos, list ...ast.Stmt) *ast.BlockStmt {
return &ast.BlockStmt{Stmts: list, LBrace: lbrace, RBrace: rbrace} return &ast.BlockStmt{Stmts: list, LBrace: lbrace, RBrace: rbrace}
} }
func ident(name string, pos scanner.Pos) *ast.Ident { func ident(name string, pos source.Pos) *ast.Ident {
return &ast.Ident{Name: name, NamePos: pos} return &ast.Ident{Name: name, NamePos: pos}
} }
func identList(opening, closing scanner.Pos, list ...*ast.Ident) *ast.IdentList { func identList(opening, closing source.Pos, list ...*ast.Ident) *ast.IdentList {
return &ast.IdentList{List: list, LParen: opening, RParen: closing} return &ast.IdentList{List: list, LParen: opening, RParen: closing}
} }
func binaryExpr(x, y ast.Expr, op token.Token, pos scanner.Pos) *ast.BinaryExpr { func binaryExpr(x, y ast.Expr, op token.Token, pos source.Pos) *ast.BinaryExpr {
return &ast.BinaryExpr{Lhs: x, Rhs: y, Token: op, TokenPos: pos} return &ast.BinaryExpr{Lhs: x, Rhs: y, Token: op, TokenPos: pos}
} }
func unaryExpr(x ast.Expr, op token.Token, pos scanner.Pos) *ast.UnaryExpr { func unaryExpr(x ast.Expr, op token.Token, pos source.Pos) *ast.UnaryExpr {
return &ast.UnaryExpr{Expr: x, Token: op, TokenPos: pos} return &ast.UnaryExpr{Expr: x, Token: op, TokenPos: pos}
} }
@ -195,35 +195,35 @@ func exprs(list ...ast.Expr) []ast.Expr {
return list return list
} }
func intLit(value int64, pos scanner.Pos) *ast.IntLit { func intLit(value int64, pos source.Pos) *ast.IntLit {
return &ast.IntLit{Value: value, ValuePos: pos} return &ast.IntLit{Value: value, ValuePos: pos}
} }
func floatLit(value float64, pos scanner.Pos) *ast.FloatLit { func floatLit(value float64, pos source.Pos) *ast.FloatLit {
return &ast.FloatLit{Value: value, ValuePos: pos} return &ast.FloatLit{Value: value, ValuePos: pos}
} }
func stringLit(value string, pos scanner.Pos) *ast.StringLit { func stringLit(value string, pos source.Pos) *ast.StringLit {
return &ast.StringLit{Value: value, ValuePos: pos} return &ast.StringLit{Value: value, ValuePos: pos}
} }
func charLit(value rune, pos scanner.Pos) *ast.CharLit { func charLit(value rune, pos source.Pos) *ast.CharLit {
return &ast.CharLit{Value: value, ValuePos: pos, Literal: fmt.Sprintf("'%c'", value)} return &ast.CharLit{Value: value, ValuePos: pos, Literal: fmt.Sprintf("'%c'", value)}
} }
func boolLit(value bool, pos scanner.Pos) *ast.BoolLit { func boolLit(value bool, pos source.Pos) *ast.BoolLit {
return &ast.BoolLit{Value: value, ValuePos: pos} return &ast.BoolLit{Value: value, ValuePos: pos}
} }
func arrayLit(lbracket, rbracket scanner.Pos, list ...ast.Expr) *ast.ArrayLit { func arrayLit(lbracket, rbracket source.Pos, list ...ast.Expr) *ast.ArrayLit {
return &ast.ArrayLit{LBrack: lbracket, RBrack: rbracket, Elements: list} return &ast.ArrayLit{LBrack: lbracket, RBrack: rbracket, Elements: list}
} }
func mapElementLit(key string, keyPos scanner.Pos, colonPos scanner.Pos, value ast.Expr) *ast.MapElementLit { func mapElementLit(key string, keyPos source.Pos, colonPos source.Pos, value ast.Expr) *ast.MapElementLit {
return &ast.MapElementLit{Key: key, KeyPos: keyPos, ColonPos: colonPos, Value: value} return &ast.MapElementLit{Key: key, KeyPos: keyPos, ColonPos: colonPos, Value: value}
} }
func mapLit(lbrace, rbrace scanner.Pos, list ...*ast.MapElementLit) *ast.MapLit { func mapLit(lbrace, rbrace source.Pos, list ...*ast.MapElementLit) *ast.MapLit {
return &ast.MapLit{LBrace: lbrace, RBrace: rbrace, Elements: list} return &ast.MapLit{LBrace: lbrace, RBrace: rbrace, Elements: list}
} }
@ -231,19 +231,19 @@ func funcLit(funcType *ast.FuncType, body *ast.BlockStmt) *ast.FuncLit {
return &ast.FuncLit{Type: funcType, Body: body} return &ast.FuncLit{Type: funcType, Body: body}
} }
func parenExpr(x ast.Expr, lparen, rparen scanner.Pos) *ast.ParenExpr { func parenExpr(x ast.Expr, lparen, rparen source.Pos) *ast.ParenExpr {
return &ast.ParenExpr{Expr: x, LParen: lparen, RParen: rparen} return &ast.ParenExpr{Expr: x, LParen: lparen, RParen: rparen}
} }
func callExpr(f ast.Expr, lparen, rparen scanner.Pos, args ...ast.Expr) *ast.CallExpr { func callExpr(f ast.Expr, lparen, rparen source.Pos, args ...ast.Expr) *ast.CallExpr {
return &ast.CallExpr{Func: f, LParen: lparen, RParen: rparen, Args: args} return &ast.CallExpr{Func: f, LParen: lparen, RParen: rparen, Args: args}
} }
func indexExpr(x, index ast.Expr, lbrack, rbrack scanner.Pos) *ast.IndexExpr { func indexExpr(x, index ast.Expr, lbrack, rbrack source.Pos) *ast.IndexExpr {
return &ast.IndexExpr{Expr: x, Index: index, LBrack: lbrack, RBrack: rbrack} return &ast.IndexExpr{Expr: x, Index: index, LBrack: lbrack, RBrack: rbrack}
} }
func sliceExpr(x, low, high ast.Expr, lbrack, rbrack scanner.Pos) *ast.SliceExpr { func sliceExpr(x, low, high ast.Expr, lbrack, rbrack source.Pos) *ast.SliceExpr {
return &ast.SliceExpr{Expr: x, Low: low, High: high, LBrack: lbrack, RBrack: rbrack} return &ast.SliceExpr{Expr: x, Low: low, High: high, LBrack: lbrack, RBrack: rbrack}
} }

View file

@ -1,3 +1,5 @@
package scanner package scanner
type ErrorHandler func(pos FilePos, msg string) import "github.com/d5/tengo/source"
type ErrorHandler func(pos source.FilePos, msg string)

View file

@ -5,6 +5,7 @@ import (
"unicode" "unicode"
"unicode/utf8" "unicode/utf8"
"github.com/d5/tengo/source"
"github.com/d5/tengo/token" "github.com/d5/tengo/token"
) )
@ -12,7 +13,7 @@ import (
const bom = 0xFEFF const bom = 0xFEFF
type Scanner struct { type Scanner struct {
file *File // source file handle file *source.File // source file handle
src []byte // source src []byte // source
ch rune // current character ch rune // current character
offset int // character offset offset int // character offset
@ -24,7 +25,7 @@ type Scanner struct {
mode Mode mode Mode
} }
func NewScanner(file *File, src []byte, errorHandler ErrorHandler, mode Mode) *Scanner { func NewScanner(file *source.File, src []byte, errorHandler ErrorHandler, mode Mode) *Scanner {
if file.Size() != len(src) { if file.Size() != len(src) {
panic(fmt.Sprintf("file size (%d) does not match src len (%d)", file.Size(), len(src))) panic(fmt.Sprintf("file size (%d) does not match src len (%d)", file.Size(), len(src)))
} }
@ -49,7 +50,7 @@ func (s *Scanner) ErrorCount() int {
return s.errorCount return s.errorCount
} }
func (s *Scanner) Scan() (tok token.Token, literal string, pos Pos) { func (s *Scanner) Scan() (tok token.Token, literal string, pos source.Pos) {
s.skipWhitespace() s.skipWhitespace()
pos = s.file.FileSetPos(s.offset) pos = s.file.FileSetPos(s.offset)

View file

@ -9,10 +9,11 @@ import (
"github.com/d5/tengo/assert" "github.com/d5/tengo/assert"
"github.com/d5/tengo/scanner" "github.com/d5/tengo/scanner"
"github.com/d5/tengo/source"
"github.com/d5/tengo/token" "github.com/d5/tengo/token"
) )
var testFileSet = scanner.NewFileSet() var testFileSet = source.NewFileSet()
type scanResult struct { type scanResult struct {
Token token.Token Token token.Token
@ -220,7 +221,7 @@ func scanExpect(t *testing.T, input string, mode scanner.Mode, expected ...scanR
s := scanner.NewScanner( s := scanner.NewScanner(
testFile, testFile,
[]byte(input), []byte(input),
func(_ scanner.FilePos, msg string) { assert.Fail(t, msg) }, func(_ source.FilePos, msg string) { assert.Fail(t, msg) },
mode) mode)
for idx, e := range expected { for idx, e := range expected {

View file

@ -6,7 +6,7 @@ import (
"github.com/d5/tengo/compiler" "github.com/d5/tengo/compiler"
"github.com/d5/tengo/objects" "github.com/d5/tengo/objects"
"github.com/d5/tengo/parser" "github.com/d5/tengo/parser"
"github.com/d5/tengo/scanner" "github.com/d5/tengo/source"
"github.com/d5/tengo/vm" "github.com/d5/tengo/vm"
) )
@ -55,7 +55,7 @@ func (s *Script) Remove(name string) bool {
func (s *Script) Compile() (*Compiled, error) { func (s *Script) Compile() (*Compiled, error) {
symbolTable, globals := s.prepCompile() symbolTable, globals := s.prepCompile()
fileSet := scanner.NewFileSet() fileSet := source.NewFileSet()
p := parser.NewParser(fileSet.AddFile("", -1, len(s.input)), s.input, nil) p := parser.NewParser(fileSet.AddFile("", -1, len(s.input)), s.input, nil)
file, err := p.ParseFile() file, err := p.ParseFile()

View file

@ -1,4 +1,4 @@
package scanner package source
import ( import (
"sync" "sync"

View file

@ -1,4 +1,4 @@
package scanner package source
import "fmt" import "fmt"

View file

@ -1,4 +1,4 @@
package scanner package source
import ( import (
"sort" "sort"

View file

@ -1,4 +1,4 @@
package scanner package source
type Pos int type Pos int

View file

@ -3,12 +3,12 @@ package tengo
import ( import (
"github.com/d5/tengo/compiler" "github.com/d5/tengo/compiler"
"github.com/d5/tengo/parser" "github.com/d5/tengo/parser"
"github.com/d5/tengo/scanner" "github.com/d5/tengo/source"
"github.com/d5/tengo/vm" "github.com/d5/tengo/vm"
) )
func Compile(input []byte, filename string) (*compiler.Bytecode, error) { func Compile(input []byte, filename string) (*compiler.Bytecode, error) {
fileSet := scanner.NewFileSet() fileSet := source.NewFileSet()
p := parser.NewParser(fileSet.AddFile(filename, -1, len(input)), input, nil) p := parser.NewParser(fileSet.AddFile(filename, -1, len(input)), input, nil)
file, err := p.ParseFile() file, err := p.ParseFile()

View file

@ -13,7 +13,7 @@ import (
"github.com/d5/tengo/compiler" "github.com/d5/tengo/compiler"
"github.com/d5/tengo/objects" "github.com/d5/tengo/objects"
"github.com/d5/tengo/parser" "github.com/d5/tengo/parser"
"github.com/d5/tengo/scanner" "github.com/d5/tengo/source"
"github.com/d5/tengo/vm" "github.com/d5/tengo/vm"
) )
@ -246,7 +246,7 @@ func traceCompileRun(file *ast.File, symbols map[string]objects.Object) (res map
} }
func parse(t *testing.T, input string) *ast.File { func parse(t *testing.T, input string) *ast.File {
testFileSet := scanner.NewFileSet() testFileSet := source.NewFileSet()
testFile := testFileSet.AddFile("", -1, len(input)) testFile := testFileSet.AddFile("", -1, len(input))
file, err := parser.ParseFile(testFile, []byte(input), nil) file, err := parser.ParseFile(testFile, []byte(input), nil)