limit max object allocations ()

- add object.NumObjects()
- add object allocation limit in VM
- delete objects.Break, objects.Continue, objects.ReturnValue
- add Script.SetMaxAllocs()
- update sandbox documentation
- add some tests
- remove duplicate values in compiled constants (fixes )
- option to limit the maximum number of objects in compiled bytecode constants
This commit is contained in:
Daniel 2019-03-06 17:20:05 -08:00 committed by GitHub
parent 46884c7b25
commit e93f6f6325
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
20 changed files with 663 additions and 257 deletions

View file

@ -109,6 +109,10 @@ func Equal(t *testing.T, expected, actual interface{}, msg ...interface{}) bool
if bytes.Compare(expected, actual.([]byte)) != 0 {
return failExpectedActual(t, string(expected), string(actual.([]byte)), msg...)
}
case []string:
if !equalStringSlice(expected, actual.([]string)) {
return failExpectedActual(t, expected, actual, msg...)
}
case []int:
if !equalIntSlice(expected, actual.([]int)) {
return failExpectedActual(t, expected, actual, msg...)
@ -147,8 +151,6 @@ func Equal(t *testing.T, expected, actual interface{}, msg ...interface{}) bool
if expected != actual {
return failExpectedActual(t, expected, actual, msg...)
}
case *objects.ReturnValue:
return Equal(t, expected.Value, actual.(objects.ReturnValue).Value, msg...)
case *objects.Array:
return equalObjectSlice(t, expected.Value, actual.(*objects.Array).Value, msg...)
case *objects.ImmutableArray:
@ -245,6 +247,20 @@ func equalIntSlice(a, b []int) bool {
return true
}
func equalStringSlice(a, b []string) bool {
if len(a) != len(b) {
return false
}
for i := 0; i < len(a); i++ {
if a[i] != b[i] {
return false
}
}
return true
}
func equalSymbol(a, b *compiler.Symbol) bool {
return a.Name == b.Name &&
a.Index == b.Index &&
@ -299,7 +315,9 @@ func equalCompiledFunction(t *testing.T, expected, actual objects.Object, msg ..
expectedT := expected.(*objects.CompiledFunction)
actualT := actual.(*objects.CompiledFunction)
return Equal(t, expectedT.Instructions, actualT.Instructions, msg...)
return Equal(t,
compiler.FormatInstructions(expectedT.Instructions, 0),
compiler.FormatInstructions(actualT.Instructions, 0), msg...)
}
func equalClosure(t *testing.T, expected, actual objects.Object, msg ...interface{}) bool {

View file

@ -199,7 +199,10 @@ func compileFile(file *ast.File) (time.Duration, *compiler.Bytecode, error) {
return time.Since(start), nil, err
}
return time.Since(start), c.Bytecode(), nil
bytecode := c.Bytecode()
bytecode.RemoveDuplicates()
return time.Since(start), bytecode, nil
}
func runVM(bytecode *compiler.Bytecode) (time.Duration, objects.Object, error) {
@ -207,7 +210,7 @@ func runVM(bytecode *compiler.Bytecode) (time.Duration, objects.Object, error) {
start := time.Now()
v := runtime.NewVM(bytecode, globals, nil, nil)
v := runtime.NewVM(bytecode, globals, nil, nil, -1)
if err := v.Run(); err != nil {
return time.Since(start), nil, err
}

View file

@ -158,7 +158,7 @@ func compileAndRun(data []byte, inputFile string) (err error) {
return
}
machine := runtime.NewVM(bytecode, nil, nil, builtinModules)
machine := runtime.NewVM(bytecode, nil, nil, builtinModules, -1)
err = machine.Run()
if err != nil {
@ -175,7 +175,7 @@ func runCompiled(data []byte) (err error) {
return
}
machine := runtime.NewVM(bytecode, nil, nil, builtinModules)
machine := runtime.NewVM(bytecode, nil, nil, builtinModules, -1)
err = machine.Run()
if err != nil {
@ -226,7 +226,7 @@ func runREPL(in io.Reader, out io.Writer) {
bytecode := c.Bytecode()
machine := runtime.NewVM(bytecode, globals, nil, builtinModules)
machine := runtime.NewVM(bytecode, globals, nil, builtinModules, -1)
if err := machine.Run(); err != nil {
_, _ = fmt.Fprintln(out, err.Error())
continue
@ -251,7 +251,10 @@ func compileSrc(src []byte, filename string) (*compiler.Bytecode, error) {
return nil, err
}
return c.Bytecode(), nil
bytecode := c.Bytecode()
bytecode.RemoveDuplicates()
return bytecode, nil
}
func addPrints(file *ast.File) *ast.File {

View file

@ -148,7 +148,7 @@ func compileAndRun(data []byte, inputFile string) (err error) {
return
}
machine := runtime.NewVM(bytecode, nil, nil, nil)
machine := runtime.NewVM(bytecode, nil, nil, nil, -1)
err = machine.Run()
if err != nil {
@ -165,7 +165,7 @@ func runCompiled(data []byte) (err error) {
return
}
machine := runtime.NewVM(bytecode, nil, nil, nil)
machine := runtime.NewVM(bytecode, nil, nil, nil, -1)
err = machine.Run()
if err != nil {
@ -216,7 +216,7 @@ func runREPL(in io.Reader, out io.Writer) {
bytecode := c.Bytecode()
machine := runtime.NewVM(bytecode, globals, nil, nil)
machine := runtime.NewVM(bytecode, globals, nil, nil, -1)
if err := machine.Run(); err != nil {
_, _ = fmt.Fprintln(out, err.Error())
continue
@ -241,7 +241,10 @@ func compileSrc(src []byte, filename string) (*compiler.Bytecode, error) {
return nil, err
}
return c.Bytecode(), nil
bytecode := c.Bytecode()
bytecode.RemoveDuplicates()
return bytecode, nil
}
func addPrints(file *ast.File) *ast.File {

View file

@ -35,11 +35,6 @@ func (b *Bytecode) Decode(r io.Reader) error {
return err
}
// replace Bool and Undefined with known value
for i, v := range b.Constants {
b.Constants[i] = cleanupObjects(v)
}
return nil
}
@ -59,6 +54,17 @@ func (b *Bytecode) Encode(w io.Writer) error {
return enc.Encode(b.Constants)
}
// CountObjects returns the number of objects found in Constants.
func (b *Bytecode) CountObjects() int {
n := 0
for _, c := range b.Constants {
n += objects.CountObjects(c)
}
return n
}
// FormatInstructions returns human readable string representations of
// compiled instructions.
func (b *Bytecode) FormatInstructions() []string {
@ -83,52 +89,12 @@ func (b *Bytecode) FormatConstants() (output []string) {
return
}
func cleanupObjects(o objects.Object) objects.Object {
switch o := o.(type) {
case *objects.Bool:
if o.IsFalsy() {
return objects.FalseValue
}
return objects.TrueValue
case *objects.Undefined:
return objects.UndefinedValue
case *objects.Array:
for i, v := range o.Value {
o.Value[i] = cleanupObjects(v)
}
case *objects.Map:
for k, v := range o.Value {
o.Value[k] = cleanupObjects(v)
}
}
return o
}
func init() {
gob.Register(&source.FileSet{})
gob.Register(&source.File{})
gob.Register(&objects.Array{})
gob.Register(&objects.ArrayIterator{})
gob.Register(&objects.Bool{})
gob.Register(&objects.Break{})
gob.Register(&objects.BuiltinFunction{})
gob.Register(&objects.Bytes{})
gob.Register(&objects.Char{})
gob.Register(&objects.Closure{})
gob.Register(&objects.CompiledFunction{})
gob.Register(&objects.Continue{})
gob.Register(&objects.Error{})
gob.Register(&objects.Float{})
gob.Register(&objects.ImmutableArray{})
gob.Register(&objects.ImmutableMap{})
gob.Register(&objects.Int{})
gob.Register(&objects.Map{})
gob.Register(&objects.MapIterator{})
gob.Register(&objects.ReturnValue{})
gob.Register(&objects.String{})
gob.Register(&objects.StringIterator{})
gob.Register(&objects.Time{})
gob.Register(&objects.Undefined{})
gob.Register(&objects.UserFunction{})
}

View file

@ -0,0 +1,110 @@
package compiler
import (
"fmt"
"github.com/d5/tengo/objects"
)
// RemoveDuplicates finds and remove the duplicate values in Constants.
// Note this function mutates Bytecode.
func (b *Bytecode) RemoveDuplicates() {
var deduped []objects.Object
indexMap := make(map[int]int) // mapping from old constant index to new index
ints := make(map[int64]int)
strings := make(map[string]int)
floats := make(map[float64]int)
chars := make(map[rune]int)
for curIdx, c := range b.Constants {
switch c := c.(type) {
case *objects.CompiledFunction:
// add to deduped list
indexMap[curIdx] = len(deduped)
deduped = append(deduped, c)
continue
case *objects.Int:
if newIdx, ok := ints[c.Value]; ok {
indexMap[curIdx] = newIdx
} else {
newIdx = len(deduped)
ints[c.Value] = newIdx
indexMap[curIdx] = newIdx
deduped = append(deduped, c)
}
case *objects.String:
if newIdx, ok := strings[c.Value]; ok {
indexMap[curIdx] = newIdx
} else {
newIdx = len(deduped)
strings[c.Value] = newIdx
indexMap[curIdx] = newIdx
deduped = append(deduped, c)
}
case *objects.Float:
if newIdx, ok := floats[c.Value]; ok {
indexMap[curIdx] = newIdx
} else {
newIdx = len(deduped)
floats[c.Value] = newIdx
indexMap[curIdx] = newIdx
deduped = append(deduped, c)
}
case *objects.Char:
if newIdx, ok := chars[c.Value]; ok {
indexMap[curIdx] = newIdx
} else {
newIdx = len(deduped)
chars[c.Value] = newIdx
indexMap[curIdx] = newIdx
deduped = append(deduped, c)
}
default:
panic(fmt.Errorf("invalid constant type: %s", c.TypeName()))
}
}
// replace with de-duplicated constants
b.Constants = deduped
// update CONST instructions with new indexes
// main function
updateConstIndexes(b.MainFunction.Instructions, indexMap)
// other compiled functions in constants
for _, c := range b.Constants {
switch c := c.(type) {
case *objects.CompiledFunction:
updateConstIndexes(c.Instructions, indexMap)
}
}
}
func updateConstIndexes(insts []byte, indexMap map[int]int) {
i := 0
for i < len(insts) {
op := insts[i]
numOperands := OpcodeOperands[op]
_, read := ReadOperands(numOperands, insts[i+1:])
switch op {
case OpConstant:
curIdx := int(insts[i+2]) | int(insts[i+1])<<8
newIdx, ok := indexMap[curIdx]
if !ok {
panic(fmt.Errorf("constant index not found: %d", curIdx))
}
copy(insts[i:], MakeInstruction(op, newIdx))
case OpClosure:
curIdx := int(insts[i+2]) | int(insts[i+1])<<8
numFree := int(insts[i+3])
newIdx, ok := indexMap[curIdx]
if !ok {
panic(fmt.Errorf("constant index not found: %d", curIdx))
}
copy(insts[i:], MakeInstruction(op, newIdx, numFree))
}
i += 1 + read
}
}

View file

@ -3,7 +3,6 @@ package compiler_test
import (
"bytes"
"testing"
"time"
"github.com/d5/tengo/assert"
"github.com/d5/tengo/compiler"
@ -21,20 +20,6 @@ func TestBytecode(t *testing.T) {
testBytecodeSerialization(t, bytecode(
concat(), objectsArray(
objects.UndefinedValue,
&objects.Time{Value: time.Now()},
&objects.Array{
Value: objectsArray(
&objects.Int{Value: 12},
&objects.String{Value: "foo"},
objects.TrueValue,
objects.FalseValue,
&objects.Float{Value: 93.11},
&objects.Char{Value: 'x'},
objects.UndefinedValue,
),
},
objects.FalseValue,
&objects.Char{Value: 'y'},
&objects.Float{Value: 93.11},
compiledFunction(1, 0,
@ -44,15 +29,7 @@ func TestBytecode(t *testing.T) {
compiler.MakeInstruction(compiler.OpGetFree, 0)),
&objects.Float{Value: 39.2},
&objects.Int{Value: 192},
&objects.Map{
Value: map[string]objects.Object{
"a": &objects.Float{Value: -93.1},
"b": objects.FalseValue,
"c": objects.UndefinedValue,
},
},
&objects.String{Value: "bar"},
objects.UndefinedValue)))
&objects.String{Value: "bar"})))
testBytecodeSerialization(t, bytecodeFileSet(
concat(
@ -92,6 +69,132 @@ func TestBytecode(t *testing.T) {
fileSet(srcfile{name: "file1", size: 100}, srcfile{name: "file2", size: 200})))
}
func TestBytecode_RemoveDuplicates(t *testing.T) {
testBytecodeRemoveDuplicates(t,
bytecode(
concat(), objectsArray(
&objects.Char{Value: 'y'},
&objects.Float{Value: 93.11},
compiledFunction(1, 0,
compiler.MakeInstruction(compiler.OpConstant, 3),
compiler.MakeInstruction(compiler.OpSetLocal, 0),
compiler.MakeInstruction(compiler.OpGetGlobal, 0),
compiler.MakeInstruction(compiler.OpGetFree, 0)),
&objects.Float{Value: 39.2},
&objects.Int{Value: 192},
&objects.String{Value: "bar"})),
bytecode(
concat(), objectsArray(
&objects.Char{Value: 'y'},
&objects.Float{Value: 93.11},
compiledFunction(1, 0,
compiler.MakeInstruction(compiler.OpConstant, 3),
compiler.MakeInstruction(compiler.OpSetLocal, 0),
compiler.MakeInstruction(compiler.OpGetGlobal, 0),
compiler.MakeInstruction(compiler.OpGetFree, 0)),
&objects.Float{Value: 39.2},
&objects.Int{Value: 192},
&objects.String{Value: "bar"})))
testBytecodeRemoveDuplicates(t,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpConstant, 1),
compiler.MakeInstruction(compiler.OpConstant, 2),
compiler.MakeInstruction(compiler.OpConstant, 3),
compiler.MakeInstruction(compiler.OpConstant, 4),
compiler.MakeInstruction(compiler.OpConstant, 5),
compiler.MakeInstruction(compiler.OpConstant, 6),
compiler.MakeInstruction(compiler.OpConstant, 7),
compiler.MakeInstruction(compiler.OpConstant, 8),
compiler.MakeInstruction(compiler.OpClosure, 4, 1)),
objectsArray(
&objects.Int{Value: 1},
&objects.Float{Value: 2.0},
&objects.Char{Value: '3'},
&objects.String{Value: "four"},
compiledFunction(1, 0,
compiler.MakeInstruction(compiler.OpConstant, 3),
compiler.MakeInstruction(compiler.OpConstant, 7),
compiler.MakeInstruction(compiler.OpSetLocal, 0),
compiler.MakeInstruction(compiler.OpGetGlobal, 0),
compiler.MakeInstruction(compiler.OpGetFree, 0)),
&objects.Int{Value: 1},
&objects.Float{Value: 2.0},
&objects.Char{Value: '3'},
&objects.String{Value: "four"})),
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpConstant, 1),
compiler.MakeInstruction(compiler.OpConstant, 2),
compiler.MakeInstruction(compiler.OpConstant, 3),
compiler.MakeInstruction(compiler.OpConstant, 4),
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpConstant, 1),
compiler.MakeInstruction(compiler.OpConstant, 2),
compiler.MakeInstruction(compiler.OpConstant, 3),
compiler.MakeInstruction(compiler.OpClosure, 4, 1)),
objectsArray(
&objects.Int{Value: 1},
&objects.Float{Value: 2.0},
&objects.Char{Value: '3'},
&objects.String{Value: "four"},
compiledFunction(1, 0,
compiler.MakeInstruction(compiler.OpConstant, 3),
compiler.MakeInstruction(compiler.OpConstant, 2),
compiler.MakeInstruction(compiler.OpSetLocal, 0),
compiler.MakeInstruction(compiler.OpGetGlobal, 0),
compiler.MakeInstruction(compiler.OpGetFree, 0)))))
testBytecodeRemoveDuplicates(t,
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpConstant, 1),
compiler.MakeInstruction(compiler.OpConstant, 2),
compiler.MakeInstruction(compiler.OpConstant, 3),
compiler.MakeInstruction(compiler.OpConstant, 4)),
objectsArray(
&objects.Int{Value: 1},
&objects.Int{Value: 2},
&objects.Int{Value: 3},
&objects.Int{Value: 1},
&objects.Int{Value: 3})),
bytecode(
concat(
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpConstant, 1),
compiler.MakeInstruction(compiler.OpConstant, 2),
compiler.MakeInstruction(compiler.OpConstant, 0),
compiler.MakeInstruction(compiler.OpConstant, 2)),
objectsArray(
&objects.Int{Value: 1},
&objects.Int{Value: 2},
&objects.Int{Value: 3})))
}
func TestBytecode_CountObjects(t *testing.T) {
b := bytecode(
concat(),
objectsArray(
intObject(55),
intObject(66),
intObject(77),
intObject(88),
compiledFunction(1, 0,
compiler.MakeInstruction(compiler.OpConstant, 3),
compiler.MakeInstruction(compiler.OpReturnValue)),
compiledFunction(1, 0,
compiler.MakeInstruction(compiler.OpConstant, 2),
compiler.MakeInstruction(compiler.OpReturnValue)),
compiledFunction(1, 0,
compiler.MakeInstruction(compiler.OpConstant, 1),
compiler.MakeInstruction(compiler.OpReturnValue))))
assert.Equal(t, 7, b.CountObjects())
}
func fileSet(files ...srcfile) *source.FileSet {
fileSet := source.NewFileSet()
for _, f := range files {
@ -108,6 +211,14 @@ func bytecodeFileSet(instructions []byte, constants []objects.Object, fileSet *s
}
}
func testBytecodeRemoveDuplicates(t *testing.T, input, expected *compiler.Bytecode) {
input.RemoveDuplicates()
assert.Equal(t, expected.FileSet, input.FileSet)
assert.Equal(t, expected.MainFunction, input.MainFunction)
assert.Equal(t, expected.Constants, input.Constants)
}
func testBytecodeSerialization(t *testing.T, b *compiler.Bytecode) {
var buf bytes.Buffer
err := b.Encode(&buf)

View file

@ -176,6 +176,19 @@ s.SetUserModuleLoader(func(moduleName string) ([]byte, error) {
Note that when a script is being added to another script as a module (via `Script.AddModule`), it does not inherit the module loader from the main script.
#### Script.SetMaxAllocs(n int64)
SetMaxAllocs sets the maximum number of object allocations. Note this is a cumulative metric that tracks only the object creations. Set this to a negative number (e.g. `-1`) if you don't need to limit the number of allocations.
#### tengo.MaxStringLen
Sets the maximum byte-length of string values. This limit applies to all running VM instances in the process. Also it's not recommended to set or update this value while any VM is executing.
#### tengo.MaxBytesLen
Sets the maximum length of bytes values. This limit applies to all running VM instances in the process. Also it's not recommended to set or update this value while any VM is executing.
## Compiler and VM
Although it's not recommended, you can directly create and run the Tengo [Parser](https://godoc.org/github.com/d5/tengo/compiler/parser#Parser), [Compiler](https://godoc.org/github.com/d5/tengo/compiler#Compiler), and [VM](https://godoc.org/github.com/d5/tengo/runtime#VM) for yourself instead of using Scripts and Script Variables. It's a bit more involved as you have to manage the symbol tables and global variables between them, but, basically that's what Script and Script Variable is doing internally.

View file

@ -1,37 +0,0 @@
package objects
import "github.com/d5/tengo/compiler/token"
// Break represents a break statement.
type Break struct{}
// TypeName returns the name of the type.
func (o *Break) TypeName() string {
return "break"
}
func (o *Break) String() string {
return "<break>"
}
// BinaryOp returns another object that is the result of
// a given binary operator and a right-hand side object.
func (o *Break) BinaryOp(op token.Token, rhs Object) (Object, error) {
return nil, ErrInvalidOperator
}
// Copy returns a copy of the type.
func (o *Break) Copy() Object {
return &Break{}
}
// IsFalsy returns true if the value of the type is falsy.
func (o *Break) IsFalsy() bool {
return false
}
// Equals returns true if the value of the type
// is equal to the value of another object.
func (o *Break) Equals(x Object) bool {
return false
}

View file

@ -1,38 +0,0 @@
package objects
import "github.com/d5/tengo/compiler/token"
// Continue represents a continue statement.
type Continue struct {
}
// TypeName returns the name of the type.
func (o *Continue) TypeName() string {
return "continue"
}
func (o *Continue) String() string {
return "<continue>"
}
// BinaryOp returns another object that is the result of
// a given binary operator and a right-hand side object.
func (o *Continue) BinaryOp(op token.Token, rhs Object) (Object, error) {
return nil, ErrInvalidOperator
}
// Copy returns a copy of the type.
func (o *Continue) Copy() Object {
return &Continue{}
}
// IsFalsy returns true if the value of the type is falsy.
func (o *Continue) IsFalsy() bool {
return false
}
// Equals returns true if the value of the type
// is equal to the value of another object.
func (o *Continue) Equals(x Object) bool {
return false
}

31
objects/count_objects.go Normal file
View file

@ -0,0 +1,31 @@
package objects
// CountObjects returns the number of objects that a given object o contains.
// For scalar value types, it will always be 1. For compound value types,
// this will include its elements and all of their elements recursively.
func CountObjects(o Object) (c int) {
c = 1
switch o := o.(type) {
case *Array:
for _, v := range o.Value {
c += CountObjects(v)
}
case *ImmutableArray:
for _, v := range o.Value {
c += CountObjects(v)
}
case *Map:
for _, v := range o.Value {
c += CountObjects(v)
}
case *ImmutableMap:
for _, v := range o.Value {
c += CountObjects(v)
}
case *Error:
c += CountObjects(o.Value)
}
return
}

View file

@ -0,0 +1,66 @@
package objects_test
import (
"testing"
"time"
"github.com/d5/tengo/assert"
"github.com/d5/tengo/objects"
)
func TestNumObjects(t *testing.T) {
testCountObjects(t, &objects.Array{}, 1)
testCountObjects(t, &objects.Array{Value: []objects.Object{
&objects.Int{Value: 1},
&objects.Int{Value: 2},
&objects.Array{Value: []objects.Object{
&objects.Int{Value: 3},
&objects.Int{Value: 4},
&objects.Int{Value: 5},
}},
}}, 7)
testCountObjects(t, objects.TrueValue, 1)
testCountObjects(t, objects.FalseValue, 1)
testCountObjects(t, &objects.BuiltinFunction{}, 1)
testCountObjects(t, &objects.Bytes{Value: []byte("foobar")}, 1)
testCountObjects(t, &objects.Char{Value: '가'}, 1)
testCountObjects(t, &objects.Closure{}, 1)
testCountObjects(t, &objects.CompiledFunction{}, 1)
testCountObjects(t, &objects.Error{Value: &objects.Int{Value: 5}}, 2)
testCountObjects(t, &objects.Float{Value: 19.84}, 1)
testCountObjects(t, &objects.ImmutableArray{Value: []objects.Object{
&objects.Int{Value: 1},
&objects.Int{Value: 2},
&objects.ImmutableArray{Value: []objects.Object{
&objects.Int{Value: 3},
&objects.Int{Value: 4},
&objects.Int{Value: 5},
}},
}}, 7)
testCountObjects(t, &objects.ImmutableMap{Value: map[string]objects.Object{
"k1": &objects.Int{Value: 1},
"k2": &objects.Int{Value: 2},
"k3": &objects.Array{Value: []objects.Object{
&objects.Int{Value: 3},
&objects.Int{Value: 4},
&objects.Int{Value: 5},
}},
}}, 7)
testCountObjects(t, &objects.Int{Value: 1984}, 1)
testCountObjects(t, &objects.Map{Value: map[string]objects.Object{
"k1": &objects.Int{Value: 1},
"k2": &objects.Int{Value: 2},
"k3": &objects.Array{Value: []objects.Object{
&objects.Int{Value: 3},
&objects.Int{Value: 4},
&objects.Int{Value: 5},
}},
}}, 7)
testCountObjects(t, &objects.String{Value: "foo bar"}, 1)
testCountObjects(t, &objects.Time{Value: time.Now()}, 1)
testCountObjects(t, objects.UndefinedValue, 1)
}
func testCountObjects(t *testing.T, o objects.Object, expected int) {
assert.Equal(t, expected, objects.CountObjects(o))
}

View file

@ -30,10 +30,6 @@ func TestObject_TypeName(t *testing.T) {
assert.Equal(t, "string-iterator", o.TypeName())
o = &objects.MapIterator{}
assert.Equal(t, "map-iterator", o.TypeName())
o = &objects.Break{}
assert.Equal(t, "break", o.TypeName())
o = &objects.Continue{}
assert.Equal(t, "continue", o.TypeName())
o = &objects.BuiltinFunction{Name: "fn"}
assert.Equal(t, "builtin-function:fn", o.TypeName())
o = &objects.UserFunction{Name: "fn"}
@ -42,8 +38,6 @@ func TestObject_TypeName(t *testing.T) {
assert.Equal(t, "closure", o.TypeName())
o = &objects.CompiledFunction{}
assert.Equal(t, "compiled-function", o.TypeName())
o = &objects.ReturnValue{}
assert.Equal(t, "return-value", o.TypeName())
o = &objects.Undefined{}
assert.Equal(t, "undefined", o.TypeName())
o = &objects.Error{}
@ -84,18 +78,12 @@ func TestObject_IsFalsy(t *testing.T) {
assert.True(t, o.IsFalsy())
o = &objects.MapIterator{}
assert.True(t, o.IsFalsy())
o = &objects.Break{}
assert.False(t, o.IsFalsy())
o = &objects.Continue{}
assert.False(t, o.IsFalsy())
o = &objects.BuiltinFunction{}
assert.False(t, o.IsFalsy())
o = &objects.Closure{}
assert.False(t, o.IsFalsy())
o = &objects.CompiledFunction{}
assert.False(t, o.IsFalsy())
o = &objects.ReturnValue{}
assert.False(t, o.IsFalsy())
o = &objects.Undefined{}
assert.True(t, o.IsFalsy())
o = &objects.Error{}
@ -138,12 +126,6 @@ func TestObject_String(t *testing.T) {
assert.Equal(t, "<array-iterator>", o.String())
o = &objects.MapIterator{}
assert.Equal(t, "<map-iterator>", o.String())
o = &objects.Break{}
assert.Equal(t, "<break>", o.String())
o = &objects.Continue{}
assert.Equal(t, "<continue>", o.String())
o = &objects.ReturnValue{}
assert.Equal(t, "<return-value>", o.String())
o = &objects.Undefined{}
assert.Equal(t, "<undefined>", o.String())
o = &objects.Bytes{}
@ -172,12 +154,6 @@ func TestObject_BinaryOp(t *testing.T) {
o = &objects.MapIterator{}
_, err = o.BinaryOp(token.Add, objects.UndefinedValue)
assert.Error(t, err)
o = &objects.Break{}
_, err = o.BinaryOp(token.Add, objects.UndefinedValue)
assert.Error(t, err)
o = &objects.Continue{}
_, err = o.BinaryOp(token.Add, objects.UndefinedValue)
assert.Error(t, err)
o = &objects.BuiltinFunction{}
_, err = o.BinaryOp(token.Add, objects.UndefinedValue)
assert.Error(t, err)
@ -187,9 +163,6 @@ func TestObject_BinaryOp(t *testing.T) {
o = &objects.CompiledFunction{}
_, err = o.BinaryOp(token.Add, objects.UndefinedValue)
assert.Error(t, err)
o = &objects.ReturnValue{}
_, err = o.BinaryOp(token.Add, objects.UndefinedValue)
assert.Error(t, err)
o = &objects.Undefined{}
_, err = o.BinaryOp(token.Add, objects.UndefinedValue)
assert.Error(t, err)

View file

@ -1,39 +0,0 @@
package objects
import "github.com/d5/tengo/compiler/token"
// ReturnValue represents a value that is being returned.
type ReturnValue struct {
Value Object
}
// TypeName returns the name of the type.
func (o *ReturnValue) TypeName() string {
return "return-value"
}
func (o *ReturnValue) String() string {
return "<return-value>"
}
// BinaryOp returns another object that is the result of
// a given binary operator and a right-hand side object.
func (o *ReturnValue) BinaryOp(op token.Token, rhs Object) (Object, error) {
return nil, ErrInvalidOperator
}
// Copy returns a copy of the type.
func (o *ReturnValue) Copy() Object {
return &ReturnValue{Value: o.Copy()}
}
// IsFalsy returns true if the value of the type is falsy.
func (o *ReturnValue) IsFalsy() bool {
return false
}
// Equals returns true if the value of the type
// is equal to the value of another object.
func (o *ReturnValue) Equals(x Object) bool {
return false
}

View file

@ -6,3 +6,6 @@ import (
// ErrStackOverflow is a stack overflow error.
var ErrStackOverflow = errors.New("stack overflow")
// ErrObjectAllocLimit is an objects allocation limit error.
var ErrObjectAllocLimit = errors.New("object allocation limit exceeded")

View file

@ -43,12 +43,14 @@ type VM struct {
aborting int64
builtinFuncs []objects.Object
builtinModules map[string]*objects.Object
maxAllocs int64
allocs int64
err error
errOffset int
}
// NewVM creates a VM.
func NewVM(bytecode *compiler.Bytecode, globals []*objects.Object, builtinFuncs []objects.Object, builtinModules map[string]*objects.Object) *VM {
func NewVM(bytecode *compiler.Bytecode, globals []*objects.Object, builtinFuncs []objects.Object, builtinModules map[string]*objects.Object, maxAllocs int64) *VM {
if globals == nil {
globals = make([]*objects.Object, GlobalsSize)
}
@ -69,9 +71,7 @@ func NewVM(bytecode *compiler.Bytecode, globals []*objects.Object, builtinFuncs
frames := make([]Frame, MaxFrames)
frames[0].fn = bytecode.MainFunction
frames[0].freeVars = nil
frames[0].ip = -1
frames[0].basePointer = 0
return &VM{
constants: bytecode.Constants,
@ -87,6 +87,7 @@ func NewVM(bytecode *compiler.Bytecode, globals []*objects.Object, builtinFuncs
ip: -1,
builtinFuncs: builtinFuncs,
builtinModules: builtinModules,
maxAllocs: maxAllocs,
}
}
@ -104,6 +105,7 @@ func (v *VM) Run() (err error) {
v.curIPLimit = len(v.curInsts) - 1
v.framesIndex = 1
v.ip = -1
v.allocs = v.maxAllocs + 1
atomic.StoreInt64(&v.aborting, 0)
v.run()
@ -131,7 +133,6 @@ func (v *VM) Run() (err error) {
}
func (v *VM) run() {
mainloop:
for v.ip < v.curIPLimit && (atomic.LoadInt64(&v.aborting) == 0) {
v.ip++
@ -190,6 +191,12 @@ mainloop:
case compiler.OpBShiftRight:
v.binaryOp(token.Shr)
case compiler.OpGreaterThan:
v.binaryOp(token.Greater)
case compiler.OpGreaterThanEqual:
v.binaryOp(token.GreaterEq)
case compiler.OpEqual:
right := v.stack[v.sp-1]
left := v.stack[v.sp-2]
@ -224,12 +231,6 @@ mainloop:
}
v.sp++
case compiler.OpGreaterThan:
v.binaryOp(token.Greater)
case compiler.OpGreaterThanEqual:
v.binaryOp(token.GreaterEq)
case compiler.OpPop:
v.sp--
@ -280,6 +281,12 @@ mainloop:
var res objects.Object = &objects.Int{Value: ^x.Value}
v.allocs--
if v.allocs == 0 {
v.err = ErrObjectAllocLimit
return
}
v.stack[v.sp] = &res
v.sp++
default:
@ -300,6 +307,12 @@ mainloop:
var res objects.Object = &objects.Int{Value: -x.Value}
v.allocs--
if v.allocs == 0 {
v.err = ErrObjectAllocLimit
return
}
v.stack[v.sp] = &res
v.sp++
case *objects.Float:
@ -310,6 +323,12 @@ mainloop:
var res objects.Object = &objects.Float{Value: -x.Value}
v.allocs--
if v.allocs == 0 {
v.err = ErrObjectAllocLimit
return
}
v.stack[v.sp] = &res
v.sp++
default:
@ -404,6 +423,12 @@ mainloop:
var arr objects.Object = &objects.Array{Value: elements}
v.allocs--
if v.allocs == 0 {
v.err = ErrObjectAllocLimit
return
}
if v.sp >= StackSize {
v.err = ErrStackOverflow
return
@ -426,6 +451,12 @@ mainloop:
var m objects.Object = &objects.Map{Value: kv}
v.allocs--
if v.allocs == 0 {
v.err = ErrObjectAllocLimit
return
}
if v.sp >= StackSize {
v.err = ErrStackOverflow
return
@ -441,6 +472,12 @@ mainloop:
Value: *value,
}
v.allocs--
if v.allocs == 0 {
v.err = ErrObjectAllocLimit
return
}
v.stack[v.sp-1] = &e
case compiler.OpImmutable:
@ -451,11 +488,25 @@ mainloop:
var immutableArray objects.Object = &objects.ImmutableArray{
Value: value.Value,
}
v.allocs--
if v.allocs == 0 {
v.err = ErrObjectAllocLimit
return
}
v.stack[v.sp-1] = &immutableArray
case *objects.Map:
var immutableMap objects.Object = &objects.ImmutableMap{
Value: value.Value,
}
v.allocs--
if v.allocs == 0 {
v.err = ErrObjectAllocLimit
return
}
v.stack[v.sp-1] = &immutableMap
}
@ -561,6 +612,13 @@ mainloop:
}
var val objects.Object = &objects.Array{Value: left.Value[lowIdx:highIdx]}
v.allocs--
if v.allocs == 0 {
v.err = ErrObjectAllocLimit
return
}
v.stack[v.sp] = &val
v.sp++
@ -600,6 +658,12 @@ mainloop:
var val objects.Object = &objects.Array{Value: left.Value[lowIdx:highIdx]}
v.allocs--
if v.allocs == 0 {
v.err = ErrObjectAllocLimit
return
}
v.stack[v.sp] = &val
v.sp++
@ -639,6 +703,12 @@ mainloop:
var val objects.Object = &objects.String{Value: left.Value[lowIdx:highIdx]}
v.allocs--
if v.allocs == 0 {
v.err = ErrObjectAllocLimit
return
}
v.stack[v.sp] = &val
v.sp++
@ -678,6 +748,12 @@ mainloop:
var val objects.Object = &objects.Bytes{Value: left.Value[lowIdx:highIdx]}
v.allocs--
if v.allocs == 0 {
v.err = ErrObjectAllocLimit
return
}
v.stack[v.sp] = &val
v.sp++
}
@ -707,7 +783,7 @@ mainloop:
}
v.sp -= numArgs + 1
v.ip = -1 // reset IP to beginning of the frame
continue mainloop
continue
}
}
@ -741,7 +817,7 @@ mainloop:
}
v.sp -= numArgs + 1
v.ip = -1 // reset IP to beginning of the frame
continue mainloop
continue
}
}
@ -791,6 +867,12 @@ mainloop:
ret = objects.UndefinedValue
}
v.allocs--
if v.allocs == 0 {
v.err = ErrObjectAllocLimit
return
}
if v.sp >= StackSize {
v.err = ErrStackOverflow
return
@ -957,6 +1039,12 @@ mainloop:
Free: free,
}
v.allocs--
if v.allocs == 0 {
v.err = ErrObjectAllocLimit
return
}
v.stack[v.sp] = &cl
v.sp++
@ -1013,6 +1101,12 @@ mainloop:
iterator = iterable.Iterate()
v.allocs--
if v.allocs == 0 {
v.err = ErrObjectAllocLimit
return
}
if v.sp >= StackSize {
v.err = ErrStackOverflow
return
@ -1134,6 +1228,12 @@ func (v *VM) binaryOp(tok token.Token) {
return
}
v.allocs--
if v.allocs == 0 {
v.err = ErrObjectAllocLimit
return
}
v.stack[v.sp-2] = &res
v.sp--
}

View file

@ -0,0 +1,49 @@
package runtime_test
import (
"testing"
"github.com/d5/tengo/objects"
)
func TestObjectsLimit(t *testing.T) {
testAllocsLimit(t, `5`, 0)
testAllocsLimit(t, `5 + 5`, 1)
testAllocsLimit(t, `a := [1, 2, 3]`, 1)
testAllocsLimit(t, `a := 1; b := 2; c := 3; d := [a, b, c]`, 1)
testAllocsLimit(t, `a := {foo: 1, bar: 2}`, 1)
testAllocsLimit(t, `a := 1; b := 2; c := {foo: a, bar: b}`, 1)
testAllocsLimit(t, `
f := func() {
return 5 + 5
}
a := f() + 5
`, 2)
testAllocsLimit(t, `
f := func() {
return 5 + 5
}
a := f()
`, 1)
testAllocsLimit(t, `
a := []
f := func() {
a = append(a, 5)
}
f()
f()
f()
`, 4)
}
func testAllocsLimit(t *testing.T, src string, limit int64) {
expectAllocsLimit(t, src, -1, objects.UndefinedValue) // no limit
expectAllocsLimit(t, src, limit, objects.UndefinedValue)
expectAllocsLimit(t, src, limit+1, objects.UndefinedValue)
if limit > 1 {
expectErrorAllocsLimit(t, src, limit-1, "allocation limit exceeded")
}
if limit > 2 {
expectErrorAllocsLimit(t, src, limit-2, "allocation limit exceeded")
}
}

View file

@ -16,9 +16,7 @@ import (
"github.com/d5/tengo/runtime"
)
const (
testOut = "out"
)
const testOut = "out"
type IARR []interface{}
type IMAP map[string]interface{}
@ -27,42 +25,50 @@ type ARR = []interface{}
type SYM = map[string]objects.Object
func expect(t *testing.T, input string, expected interface{}) {
expectWithUserModules(t, input, expected, nil)
runVM(t, input, expected, nil, nil, nil, -1, false)
}
func expectAllocsLimit(t *testing.T, input string, maxAllocs int64, expected interface{}) {
runVM(t, input, expected, nil, nil, nil, maxAllocs, true)
}
func expectNoMod(t *testing.T, input string, expected interface{}) {
runVM(t, input, expected, nil, nil, nil, true)
runVM(t, input, expected, nil, nil, nil, -1, true)
}
func expectWithSymbols(t *testing.T, input string, expected interface{}, symbols map[string]objects.Object) {
runVM(t, input, expected, symbols, nil, nil, true)
runVM(t, input, expected, symbols, nil, nil, -1, true)
}
func expectWithUserModules(t *testing.T, input string, expected interface{}, userModules map[string]string) {
runVM(t, input, expected, nil, userModules, nil, false)
runVM(t, input, expected, nil, userModules, nil, -1, false)
}
func expectWithBuiltinModules(t *testing.T, input string, expected interface{}, builtinModules map[string]*objects.Object) {
runVM(t, input, expected, nil, nil, builtinModules, false)
runVM(t, input, expected, nil, nil, builtinModules, -1, false)
}
func expectWithUserAndBuiltinModules(t *testing.T, input string, expected interface{}, userModules map[string]string, builtinModules map[string]*objects.Object) {
runVM(t, input, expected, nil, userModules, builtinModules, false)
runVM(t, input, expected, nil, userModules, builtinModules, -1, false)
}
func expectError(t *testing.T, input, expected string) {
runVMError(t, input, nil, nil, nil, expected)
runVMError(t, input, nil, nil, nil, -1, expected)
}
func expectErrorAllocsLimit(t *testing.T, input string, maxAllocs int64, expected string) {
runVMError(t, input, nil, nil, nil, maxAllocs, expected)
}
func expectErrorWithUserModules(t *testing.T, input string, userModules map[string]string, expected string) {
runVMError(t, input, nil, userModules, nil, expected)
runVMError(t, input, nil, userModules, nil, -1, expected)
}
func expectErrorWithSymbols(t *testing.T, input string, symbols map[string]objects.Object, expected string) {
runVMError(t, input, symbols, nil, nil, expected)
runVMError(t, input, symbols, nil, nil, -1, expected)
}
func runVM(t *testing.T, input string, expected interface{}, symbols map[string]objects.Object, userModules map[string]string, builtinModules map[string]*objects.Object, skipModuleTest bool) {
func runVM(t *testing.T, input string, expected interface{}, symbols map[string]objects.Object, userModules map[string]string, builtinModules map[string]*objects.Object, maxAllocs int64, skipModuleTest bool) {
expectedObj := toObject(expected)
if symbols == nil {
@ -79,7 +85,7 @@ func runVM(t *testing.T, input string, expected interface{}, symbols map[string]
}
// compiler/VM
res, trace, err := traceCompileRun(file, symbols, userModules, builtinModules)
res, trace, err := traceCompileRun(file, symbols, userModules, builtinModules, maxAllocs)
if !assert.NoError(t, err) ||
!assert.Equal(t, expectedObj, res[testOut]) {
t.Log("\n" + strings.Join(trace, "\n"))
@ -106,7 +112,7 @@ func runVM(t *testing.T, input string, expected interface{}, symbols map[string]
}
userModules["__code__"] = fmt.Sprintf("out := undefined; %s; export out", input)
res, trace, err := traceCompileRun(file, symbols, userModules, builtinModules)
res, trace, err := traceCompileRun(file, symbols, userModules, builtinModules, maxAllocs)
if !assert.NoError(t, err) ||
!assert.Equal(t, expectedObj, res[testOut]) {
t.Log("\n" + strings.Join(trace, "\n"))
@ -114,7 +120,7 @@ func runVM(t *testing.T, input string, expected interface{}, symbols map[string]
}
}
func runVMError(t *testing.T, input string, symbols map[string]objects.Object, userModules map[string]string, builtinModules map[string]*objects.Object, expected string) {
func runVMError(t *testing.T, input string, symbols map[string]objects.Object, userModules map[string]string, builtinModules map[string]*objects.Object, maxAllocs int64, expected string) {
expected = strings.TrimSpace(expected)
if expected == "" {
panic("expected must not be empty")
@ -127,7 +133,7 @@ func runVMError(t *testing.T, input string, symbols map[string]objects.Object, u
}
// compiler/VM
_, trace, err := traceCompileRun(program, symbols, userModules, builtinModules)
_, trace, err := traceCompileRun(program, symbols, userModules, builtinModules, maxAllocs)
if !assert.Error(t, err) ||
!assert.True(t, strings.Contains(err.Error(), expected), "expected error string: %s, got: %s", expected, err.Error()) {
t.Log("\n" + strings.Join(trace, "\n"))
@ -143,7 +149,7 @@ func (o *tracer) Write(p []byte) (n int, err error) {
return len(p), nil
}
func traceCompileRun(file *ast.File, symbols map[string]objects.Object, userModules map[string]string, builtinModules map[string]*objects.Object) (res map[string]objects.Object, trace []string, err error) {
func traceCompileRun(file *ast.File, symbols map[string]objects.Object, userModules map[string]string, builtinModules map[string]*objects.Object, maxAllocs int64) (res map[string]objects.Object, trace []string, err error) {
var v *runtime.VM
defer func() {
@ -200,10 +206,11 @@ func traceCompileRun(file *ast.File, symbols map[string]objects.Object, userModu
}
bytecode := c.Bytecode()
bytecode.RemoveDuplicates()
trace = append(trace, fmt.Sprintf("\n[Compiled Constants]\n\n%s", strings.Join(bytecode.FormatConstants(), "\n")))
trace = append(trace, fmt.Sprintf("\n[Compiled Instructions]\n\n%s\n", strings.Join(bytecode.FormatInstructions(), "\n")))
v = runtime.NewVM(bytecode, globals, nil, builtinModules)
v = runtime.NewVM(bytecode, globals, nil, builtinModules, maxAllocs)
err = v.Run()
{

View file

@ -18,13 +18,17 @@ type Script struct {
builtinModules map[string]*objects.Object
userModuleLoader compiler.ModuleLoader
input []byte
maxAllocs int64
maxConstObjects int
}
// New creates a Script instance with an input script.
func New(input []byte) *Script {
return &Script{
variables: make(map[string]*Variable),
input: input,
variables: make(map[string]*Variable),
input: input,
maxAllocs: -1,
maxConstObjects: -1,
}
}
@ -84,6 +88,17 @@ func (s *Script) SetUserModuleLoader(loader compiler.ModuleLoader) {
s.userModuleLoader = loader
}
// SetMaxAllocs sets the maximum number of objects allocations during the run time.
// Compiled script will return runtime.ErrObjectAllocLimit error if it exceeds this limit.
func (s *Script) SetMaxAllocs(n int64) {
s.maxAllocs = n
}
// SetMaxConstObjects sets the maximum number of objects in the compiled constants.
func (s *Script) SetMaxConstObjects(n int) {
s.maxConstObjects = n
}
// Compile compiles the script with all the defined variables, and, returns Compiled object.
func (s *Script) Compile() (*Compiled, error) {
symbolTable, builtinModules, globals, err := s.prepCompile()
@ -110,9 +125,24 @@ func (s *Script) Compile() (*Compiled, error) {
return nil, err
}
// reduce globals size
globals = globals[:symbolTable.MaxSymbols()+1]
// remove duplicates from constants
bytecode := c.Bytecode()
bytecode.RemoveDuplicates()
// check the constant objects limit
if s.maxConstObjects >= 0 {
cnt := bytecode.CountObjects()
if cnt > s.maxConstObjects {
return nil, fmt.Errorf("exceeding constant objects limit: %d", cnt)
}
}
return &Compiled{
symbolTable: symbolTable,
machine: runtime.NewVM(c.Bytecode(), globals, s.builtinFuncs, s.builtinModules),
machine: runtime.NewVM(bytecode, globals, s.builtinFuncs, s.builtinModules, s.maxAllocs),
}, nil
}

View file

@ -121,6 +121,40 @@ func TestScript_SetBuiltinModules(t *testing.T) {
assert.Error(t, err)
}
func TestScript_SetMaxConstObjects(t *testing.T) {
// one constant '5'
s := script.New([]byte(`a := 5`))
s.SetMaxConstObjects(1) // limit = 1
_, err := s.Compile()
assert.NoError(t, err)
s.SetMaxConstObjects(0) // limit = 0
_, err = s.Compile()
assert.Equal(t, "exceeding constant objects limit: 1", err.Error())
// two constants '5' and '1'
s = script.New([]byte(`a := 5 + 1`))
s.SetMaxConstObjects(2) // limit = 2
_, err = s.Compile()
assert.NoError(t, err)
s.SetMaxConstObjects(1) // limit = 1
_, err = s.Compile()
assert.Equal(t, "exceeding constant objects limit: 2", err.Error())
// duplicates will be removed
s = script.New([]byte(`a := 5 + 5`))
s.SetMaxConstObjects(1) // limit = 1
_, err = s.Compile()
assert.NoError(t, err)
s.SetMaxConstObjects(0) // limit = 0
_, err = s.Compile()
assert.Equal(t, "exceeding constant objects limit: 1", err.Error())
// no limit set
s = script.New([]byte(`a := 1 + 2 + 3 + 4 + 5`))
_, err = s.Compile()
assert.NoError(t, err)
}
func objectPtr(o objects.Object) *objects.ImmutableMap {
return o.(*objects.ImmutableMap)
}