add custom extension support for importing source file (#350)

* chore: add tests for custom extension

* feat: cusom source extension #286

* fix: path to test directory

* add getter + change setter name for file extension

* add tests of source file extension validation

* fix: add validation for file extension names

* fix: property importExt -> importFileExt

* fix: redundant check (no resetting)

* fix: failing test wich did not follow the new spec

* chore: add detailed description of the test

* chore: fix doc comment to be descriptive

* docs: add note about customizing the file extension

compiler.go

@@ -1,9 +1,11 @@
 package tengo
 
 import (
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
+	"os"
 	"path/filepath"
 	"reflect"
 	"strings"
@@ -45,6 +47,7 @@ type Compiler struct {
 	parent          *Compiler
 	modulePath      string
 	importDir       string
+	importFileExt   []string
 	constants       []Object
 	symbolTable     *SymbolTable
 	scopes          []compilationScope
@@ -96,6 +99,7 @@ func NewCompiler(
 		trace:           trace,
 		modules:         modules,
 		compiledModules: make(map[string]*CompiledFunction),
+		importFileExt:   []string{SourceFileExtDefault},
 	}
 }
 
@@ -538,12 +542,8 @@ func (c *Compiler) Compile(node parser.Node) error {
 			}
 		} else if c.allowFileImport {
 			moduleName := node.ModuleName
-			if !strings.HasSuffix(moduleName, ".tengo") {
-				moduleName += ".tengo"
-			}
 
-			modulePath, err := filepath.Abs(
-				filepath.Join(c.importDir, moduleName))
+			modulePath, err := c.getPathModule(moduleName)
 			if err != nil {
 				return c.errorf(node, "module file path error: %s",
 					err.Error())
@@ -640,6 +640,39 @@ func (c *Compiler) SetImportDir(dir string) {
 	c.importDir = dir
 }
 
+// SetImportFileExt sets the extension name of the source file for loading
+// local module files.
+//
+// Use this method if you want other source file extension than ".tengo".
+//
+//     // this will search for *.tengo, *.foo, *.bar
+//     err := c.SetImportFileExt(".tengo", ".foo", ".bar")
+//
+// This function requires at least one argument, since it will replace the
+// current list of extension name.
+func (c *Compiler) SetImportFileExt(exts ...string) error {
+	if len(exts) == 0 {
+		return fmt.Errorf("missing arg: at least one argument is required")
+	}
+
+	for _, ext := range exts {
+		if ext != filepath.Ext(ext) || ext == "" {
+			return fmt.Errorf("invalid file extension: %s", ext)
+		}
+	}
+
+	c.importFileExt = exts // Replace the hole current extension list
+
+	return nil
+}
+
+// GetImportFileExt returns the current list of extension name.
+// Thease are the complementary suffix of the source file to search and load
+// local module files.
+func (c *Compiler) GetImportFileExt() []string {
+	return c.importFileExt
+}
+
 func (c *Compiler) compileAssign(
 	node parser.Node,
 	lhs, rhs []parser.Expr,
@@ -1098,6 +1131,7 @@ func (c *Compiler) fork(
 	child.parent = c              // parent to set to current compiler
 	child.allowFileImport = c.allowFileImport
 	child.importDir = c.importDir
+	child.importFileExt = c.importFileExt
 	if isFile && c.importDir != "" {
 		child.importDir = filepath.Dir(modulePath)
 	}
@@ -1287,6 +1321,28 @@ func (c *Compiler) printTrace(a ...interface{}) {
 	_, _ = fmt.Fprintln(c.trace, a...)
 }
 
+func (c *Compiler) getPathModule(moduleName string) (pathFile string, err error) {
+	for _, ext := range c.importFileExt {
+		nameFile := moduleName
+
+		if !strings.HasSuffix(nameFile, ext) {
+			nameFile += ext
+		}
+
+		pathFile, err = filepath.Abs(filepath.Join(c.importDir, nameFile))
+		if err != nil {
+			continue
+		}
+
+		// Check if file exists
+		if _, err := os.Stat(pathFile); !errors.Is(err, os.ErrNotExist) {
+			return pathFile, nil
+		}
+	}
+
+	return "", fmt.Errorf("module '%s' not found at: %s", moduleName, pathFile)
+}
+
 func resolveAssignLHS(
 	expr parser.Expr,
 ) (name string, selectors []parser.Expr) {

compiler_test.go

@@ -2,12 +2,15 @@ package tengo_test
 
 import (
 	"fmt"
+	"io/ioutil"
+	"path/filepath"
 	"strings"
 	"testing"
 
 	"github.com/d5/tengo/v2"
 	"github.com/d5/tengo/v2/parser"
 	"github.com/d5/tengo/v2/require"
+	"github.com/d5/tengo/v2/stdlib"
 )
 
 func TestCompiler_Compile(t *testing.T) {
@@ -1222,6 +1225,91 @@ func() {
 				tengo.MakeInstruction(parser.OpReturn, 0)))))
 }
 
+func TestCompiler_custom_extension(t *testing.T) {
+	pathFileSource := "./testdata/issue286/test.mshk"
+
+	modules := stdlib.GetModuleMap(stdlib.AllModuleNames()...)
+
+	src, err := ioutil.ReadFile(pathFileSource)
+	require.NoError(t, err)
+
+	// Escape shegang
+	if len(src) > 1 && string(src[:2]) == "#!" {
+		copy(src, "//")
+	}
+
+	fileSet := parser.NewFileSet()
+	srcFile := fileSet.AddFile(filepath.Base(pathFileSource), -1, len(src))
+
+	p := parser.NewParser(srcFile, src, nil)
+	file, err := p.ParseFile()
+	require.NoError(t, err)
+
+	c := tengo.NewCompiler(srcFile, nil, nil, modules, nil)
+	c.EnableFileImport(true)
+	c.SetImportDir(filepath.Dir(pathFileSource))
+
+	// Search for "*.tengo" and ".mshk"(custom extension)
+	c.SetImportFileExt(".tengo", ".mshk")
+
+	err = c.Compile(file)
+	require.NoError(t, err)
+}
+
+func TestCompilerNewCompiler_default_file_extension(t *testing.T) {
+	modules := stdlib.GetModuleMap(stdlib.AllModuleNames()...)
+	input := "{}"
+	fileSet := parser.NewFileSet()
+	file := fileSet.AddFile("test", -1, len(input))
+
+	c := tengo.NewCompiler(file, nil, nil, modules, nil)
+	c.EnableFileImport(true)
+
+	require.Equal(t, []string{".tengo"}, c.GetImportFileExt(),
+		"newly created compiler object must contain the default extension")
+}
+
+func TestCompilerSetImportExt_extension_name_validation(t *testing.T) {
+	c := new(tengo.Compiler) // Instantiate a new compiler object with no initialization
+
+	// Test of empty arg
+	err := c.SetImportFileExt()
+
+	require.Error(t, err, "empty arg should return an error")
+
+	// Test of various arg types
+	for _, test := range []struct {
+		extensions []string
+		expect     []string
+		requireErr bool
+		msgFail    string
+	}{
+		{[]string{".tengo"}, []string{".tengo"}, false,
+			"well-formed extension should not return an error"},
+		{[]string{""}, []string{".tengo"}, true,
+			"empty extension name should return an error"},
+		{[]string{"foo"}, []string{".tengo"}, true,
+			"name without dot prefix should return an error"},
+		{[]string{"foo.bar"}, []string{".tengo"}, true,
+			"malformed extension should return an error"},
+		{[]string{"foo."}, []string{".tengo"}, true,
+			"malformed extension should return an error"},
+		{[]string{".mshk"}, []string{".mshk"}, false,
+			"name with dot prefix should be added"},
+		{[]string{".foo", ".bar"}, []string{".foo", ".bar"}, false,
+			"it should replace instead of appending"},
+	} {
+		err := c.SetImportFileExt(test.extensions...)
+		if test.requireErr {
+			require.Error(t, err, test.msgFail)
+		}
+
+		expect := test.expect
+		actual := c.GetImportFileExt()
+		require.Equal(t, expect, actual, test.msgFail)
+	}
+}
+
 func concatInsts(instructions ...[]byte) []byte {
 	var concat []byte
 	for _, i := range instructions {

docs/tutorial.md

@@ -508,6 +508,16 @@ export func(x) {
 }
 ```
 
+By default, `import` solves the missing extension name of a module file as
+"`.tengo`"[^note].
+Thus, `sum := import("./sum")` is equivalent to `sum := import("./sum.tengo")`.
+
+[^note]:
+    If using Tengo as a library in Go, the file extension name "`.tengo`" can
+    be customized. In that case, use the `SetImportFileExt` function of the
+    `Compiler` type.
+    See the [Go reference](https://pkg.go.dev/github.com/d5/tengo/v2) for details.
+
 In Tengo, modules are very similar to functions.
 
 - `import` expression loads the module code and execute it like a function.

tengo.go

@@ -26,6 +26,9 @@ const (
 
 	// MaxFrames is the maximum number of function frames for a VM.
 	MaxFrames = 1024
+
+	// SourceFileExtDefault is the default extension for source files.
+	SourceFileExtDefault = ".tengo"
 )
 
 // CallableFunc is a function signature for the callable functions.

testdata/issue286/dos/cinco/cinco.mshk

@@ -0,0 +1,8 @@
+export {
+    fn: func(...args) {
+        text := import("text")
+        args = append(args, "cinco")
+
+        return text.join(args, " ")
+    }
+}

testdata/issue286/dos/dos.mshk

@@ -0,0 +1,7 @@
+export {
+    fn: func(a, b) {
+        tres := import("../tres")
+
+        return tres.fn(a, b, "dos")
+    }
+}

testdata/issue286/dos/quatro/quatro.mshk

@@ -0,0 +1,7 @@
+export {
+    fn: func(a, b, c, d) {
+        cinco := import("../cinco/cinco")
+
+        return cinco.fn(a, b, c, d, "quatro")
+    }
+}

testdata/issue286/test.mshk

@@ -0,0 +1,23 @@
+#!/usr/bin/env tengo
+// This is a test of custom extension for issue #286 and PR #350.
+// Which allows the tengo library to use custom extension names for the
+// source files.
+//
+// This test should pass if the interpreter's tengo.Compiler.SetImportExt()
+// was set as `c.SetImportExt(".tengo", ".mshk")`.
+
+os := import("os")
+uno := import("uno") // it will search uno.tengo and uno.mshk
+fmt := import("fmt")
+text := import("text")
+
+expected := ["test", "uno", "dos", "tres", "quatro", "cinco"]
+expected = text.join(expected, " ")
+if v := uno.fn("test"); v != expected {
+    fmt.printf("relative import test error:\n\texpected: %v\n\tgot     : %v\n",
+                expected, v)
+    os.exit(1)
+}
+
+args := text.join(os.args(), " ")
+fmt.println("ok\t", args)

testdata/issue286/tres.tengo

@@ -0,0 +1,6 @@
+export {
+    fn: func(a, b, c) {
+        quatro := import("./dos/quatro/quatro.mshk")
+        return quatro.fn(a, b, c, "tres")
+    }
+}

testdata/issue286/uno.mshk

@@ -0,0 +1,6 @@
+export {
+    fn: func(a) {
+        dos := import("dos/dos")
+        return dos.fn(a, "uno")
+    }
+}