From f60bd728df0fc1132a104e8dbe543c609a862da2 Mon Sep 17 00:00:00 2001 From: surdeus Date: Fri, 2 Aug 2024 19:00:46 +0500 Subject: [PATCH] feat: restruct and and better API for making caching for servers. --- .gitignore | 3 +- build.sh | 4 - cmd/gopp/main.go | 10 ++ cmd/tpp/main.go | 10 -- errors.go | 2 +- eval.go | 6 - go.mod | 9 +- go.sum | 6 +- install.sh | 3 - main.go => pp.go | 83 ++++++--- tengo.go | 237 -------------------------- {tests => testdata}/cat.jpg.tpp | 0 {tests => testdata}/cat.webp | Bin {tests => testdata}/index.md.pp | 4 +- tests/shit.tengo => testdata/shit.xgo | 0 {tests => testdata}/somefile | 0 tool.go | 61 ++++--- xgo.go | 230 +++++++++++++++++++++++++ 18 files changed, 354 insertions(+), 314 deletions(-) delete mode 100755 build.sh create mode 100644 cmd/gopp/main.go delete mode 100644 cmd/tpp/main.go delete mode 100644 eval.go delete mode 100755 install.sh rename main.go => pp.go (54%) delete mode 100644 tengo.go rename {tests => testdata}/cat.jpg.tpp (100%) rename {tests => testdata}/cat.webp (100%) rename {tests => testdata}/index.md.pp (69%) rename tests/shit.tengo => testdata/shit.xgo (100%) rename {tests => testdata}/somefile (100%) create mode 100644 xgo.go diff --git a/.gitignore b/.gitignore index d128777..c1c0c11 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,2 @@ -/tpp -/exe +/gopp Session.vim diff --git a/build.sh b/build.sh deleted file mode 100755 index 98acd9b..0000000 --- a/build.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh -# -go build -o ./exe/ ./cmd/tpp - diff --git a/cmd/gopp/main.go b/cmd/gopp/main.go new file mode 100644 index 0000000..a544eb0 --- /dev/null +++ b/cmd/gopp/main.go @@ -0,0 +1,10 @@ +package main + +import ( + "surdeus.su/util/gopp" + "os" +) + +func main() { + gopp.Tool.Run(os.Args[1:]) +} diff --git a/cmd/tpp/main.go b/cmd/tpp/main.go deleted file mode 100644 index 1349e3d..0000000 --- a/cmd/tpp/main.go +++ /dev/null @@ -1,10 +0,0 @@ -package main - -import ( - "surdeus.su/util/tpp" - "os" -) - -func main() { - tpp.Tool.Run(os.Args[1:]) -} diff --git a/errors.go b/errors.go index 658c65f..7638932 100644 --- a/errors.go +++ b/errors.go @@ -1,4 +1,4 @@ -package tpp +package gopp import ( //"errors" diff --git a/eval.go b/eval.go deleted file mode 100644 index 816b3dc..0000000 --- a/eval.go +++ /dev/null @@ -1,6 +0,0 @@ -package tpp - -type Evaler interface { - Tags() [2]string - Eval(filePath string, pieces []string) ([]string, error) -} diff --git a/go.mod b/go.mod index 22b0b12..db0f820 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,7 @@ -module surdeus.su/util/tpp +module surdeus.su/util/gopp go 1.22.3 -require ( - github.com/d5/tengo/v2 v2.17.0 - surdeus.su/core/cli v0.1.2 -) +require surdeus.su/core/cli v0.5.0 + +require surdeus.su/core/xgo/v2 v2.18.0 diff --git a/go.sum b/go.sum index 2b30525..edf713a 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,6 @@ -github.com/d5/tengo/v2 v2.17.0 h1:BWUN9NoJzw48jZKiYDXDIF3QrIVZRm1uV1gTzeZ2lqM= -github.com/d5/tengo/v2 v2.17.0/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8= surdeus.su/core/cli v0.1.2 h1:qPzjawqPyZsO4Z5SaA1u141recVE65yioA83Qs7Jecs= surdeus.su/core/cli v0.1.2/go.mod h1:r9JtQz3aEJzpYzMaNUNQHJoYkoWKNPi047qhd5uGlmA= +surdeus.su/core/cli v0.5.0 h1:jYvE0JVDikFT9FhWGV3wIAcMgByziAVxTwsVUwWkeHs= +surdeus.su/core/cli v0.5.0/go.mod h1:r9JtQz3aEJzpYzMaNUNQHJoYkoWKNPi047qhd5uGlmA= +surdeus.su/core/xgo/v2 v2.18.0 h1:Ypz2CIiqkfVD9S4Jes5fES9zkcOYppNRvf4Af5D3JXU= +surdeus.su/core/xgo/v2 v2.18.0/go.mod h1:EBQA07DjGd0bqIGbaNQ2HZHQkilbYacI7fpQen5ivao= diff --git a/install.sh b/install.sh deleted file mode 100755 index 11bfb74..0000000 --- a/install.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -go install ./cmd/tpp/ diff --git a/main.go b/pp.go similarity index 54% rename from main.go rename to pp.go index db61f1f..07cd743 100644 --- a/main.go +++ b/pp.go @@ -1,21 +1,27 @@ -package tpp +package gopp import ( "bytes" "context" + "io" ) +type Parsed struct { + Texts [][]byte + PreCode []byte + Codes [][]byte +} type Preprocessor struct { - tengo *Tengo + xgo *XGo tags [2][]byte preTag byte } // Get the new preprocessor with default options. -func New(tengo *Tengo ) *Preprocessor { +func New(xgo *XGo) *Preprocessor { pp := &Preprocessor{} - pp.tengo = tengo + pp.xgo = xgo pp.tags = [2][]byte{ []byte("{{"), []byte("}}"), @@ -24,11 +30,18 @@ func New(tengo *Tengo ) *Preprocessor { return pp } -func (pp *Preprocessor) Process( +func (pp *Preprocessor) XGo() *XGo { + return pp.xgo +} + +func (pp *Preprocessor) Parse( ctx context.Context, - filePath string, - data []byte, -) (*Compiled, []byte, error) { + input io.Reader, +) (*Parsed, error) { + data, err := io.ReadAll(input) + if err != nil { + return nil, err + } //var b bytes.Buffer preCode := []byte(nil) texts := [][]byte{} @@ -37,7 +50,7 @@ func (pp *Preprocessor) Process( if bytes.HasPrefix(data, pref) { idxEnd := bytes.Index(data, pp.tags[1]) if idxEnd < 0 { - return nil, nil, UnexpectedError{ + return nil, UnexpectedError{ What: "pre-code start tag", } } @@ -51,14 +64,14 @@ func (pp *Preprocessor) Process( //fmt.Printf("cock %d %d %d\n", last, idxStart, idxEnd) if idxStart < 0 { if idxEnd >= 0 { - return nil, nil, UnexpectedError{ + return nil, UnexpectedError{ What: "end tag", } } texts = append(texts, data) break } else if idxEnd < 0 { - return nil, nil, UnexpectedError{ + return nil, UnexpectedError{ What: "start tag", } } @@ -72,17 +85,41 @@ func (pp *Preprocessor) Process( data = data[idxEnd + len(pp.tags[1]):] } - compiled, ret, err := pp.tengo.Eval( - ctx, - filePath, - texts, - preCode, - codes, - ) - if err != nil { - return nil, nil, err - } - - return compiled, ret, nil + return &Parsed{ + Texts: texts, + PreCode: preCode, + Codes: codes, + }, nil +} + +func (pp *Preprocessor) Render( + ctx context.Context, + input io.Reader, + output io.Writer, +) error { + parsed, err := pp.Parse(ctx, input) + if err != nil { return err } + + return pp.xgo.Render(ctx, parsed, output) +} + +func (pp *Preprocessor) Compile( + ctx context.Context, + input io.Reader, +) (*Compiled, error) { + parsed, err := pp.Parse(ctx, input) + if err != nil { + return nil, err + } + + compiled, err := pp.xgo.Compile( + ctx, + parsed, + ) + if err != nil { + return nil, err + } + + return compiled, nil } diff --git a/tengo.go b/tengo.go deleted file mode 100644 index c1ea7fd..0000000 --- a/tengo.go +++ /dev/null @@ -1,237 +0,0 @@ -package tpp - -import ( - "github.com/d5/tengo/v2" - "fmt" - "context" - "bytes" - "strconv" -) - -type CompiledMap map[string] *Compiled - -type Script = tengo.Script -type Compiled = tengo.Compiled - -// The type describes -// customizable way to evaluate -// files. -type Tengo struct { - // Functions to modify - // preprocessor for - // more specific purposes. - preCode func(context.Context) []byte - postCode func(context.Context) []byte - preCompile func(context.Context, *Script) -} - -// Returns the new Tengo preprocessor -// with -func NewTengo() *Tengo { - ret := &Tengo{} - return ret -} - -func (tengo *Tengo) SetPreCode(fn func(context.Context) []byte) *Tengo { - tengo.preCode = fn - return tengo -} - -func (tengo *Tengo) SetPostCode(fn func(context.Context) []byte) *Tengo { - tengo.postCode = fn - return tengo -} - -func (tengo *Tengo) SetPreCompile(fn func(context.Context, *Script)) *Tengo { - tengo.preCompile = fn - return tengo -} - -func (pp *Tengo) Eval( - ctx context.Context, - // File path to give it. - filePath string, - // Static text pieces. - texts [][]byte, - // The code that comes before any - // dynamic code. - preCode []byte, - // General code. - codes [][]byte, -) (*Compiled, []byte, error) { - var fullCodeBuf, retBuf bytes.Buffer - const retHead = ` - context := {} - pp := immutable({ - fpath: __pp_filepath__, - printf : __pp_printf__, - print : __pp_print__, - println : __pp_println__, - write_raw : __pp_write_raw__, - include: __pp_include__ - }) - ` - - const retSeparator = ` - __separate__() - ` - - fmt.Fprint(&fullCodeBuf, retHead) - - if preCode != nil { - fmt.Fprintln( - &fullCodeBuf, - string(preCode)+"\n", - ) - } - - if pp.preCode != nil { - fullCodeBuf.Write(pp.preCode(ctx)) - fullCodeBuf.Write([]byte(retSeparator)) - } - - for i, code := range codes { - fmt.Fprintf( - &fullCodeBuf, - "\n__pp_write_raw__(%q)\n", - texts[i], - ) - fmt.Fprintln( - &fullCodeBuf, - "\n" + string(code) + retSeparator, - ) - } - fmt.Fprintf( - &fullCodeBuf, - "\n__pp_write_raw__(%q)\n", - texts[len(texts)-1], - ) - - if pp.postCode != nil { - fullCodeBuf.Write(pp.postCode(ctx)) - fullCodeBuf.Write([]byte(retSeparator)) - } - - script := tengo.NewScript(fullCodeBuf.Bytes()) - if pp.preCompile != nil { - pp.preCompile(ctx, script) - } - - script.Add("__pp_include__", &tengo.UserFunction{ - Value: func(args ...tengo.Object) (tengo.Object, error){ - if len(args) < 1 { - return nil, tengo.ErrWrongNumArguments - } - format, ok := tengo.ToString(args[0]) - if !ok { - return nil, tengo.ErrInvalidArgumentType{ - Expected: "string", - } - } - gargs := make([]any, len(args) - 1) - for i := range gargs { - gargs[i] = tengo.ToInterface(args[i+1]) - //fmt.Printf("shit: %q\n", gargs[i]) - } - fmt.Fprintf(&retBuf, format, gargs...) - return nil, nil - //return tengo.FromInterface([]byte(str)) - }, - }) - script.Add("__pp_filepath__", filePath) - script.Add("__pp_printf__", &tengo.UserFunction{ - Value: func(args ...tengo.Object) (tengo.Object, error){ - if len(args) < 1 { - return nil, tengo.ErrWrongNumArguments - } - format, ok := tengo.ToString(args[0]) - if !ok { - return nil, tengo.ErrInvalidArgumentType{ - Expected: "string", - } - } - gargs := make([]any, len(args) - 1) - for i := range gargs { - gargs[i] = tengo.ToInterface(args[i+1]) - //fmt.Printf("shit: %q\n", gargs[i]) - } - fmt.Fprintf(&retBuf, format, gargs...) - return nil, nil - //return tengo.FromInterface([]byte(str)) - }, - }) - script.Add("__pp_print__", &tengo.UserFunction{ - Value: func(args ...tengo.Object) (tengo.Object, error){ - gargs := make([]any, len(args)) - for i := range gargs { - gargs[i] = tengo.ToInterface(args[i]) - } - fmt.Fprint(&retBuf, gargs...) - return nil, nil - //return tengo.FromInterface([]byte(str)) - }, - }) - script.Add("__pp_println__", &tengo.UserFunction{ - Value: func(args ...tengo.Object) (tengo.Object, error){ - gargs := make([]any, len(args)) - for i := range gargs { - gargs[i] = tengo.ToInterface(args[i]) - } - fmt.Fprintln(&retBuf, gargs...) - return nil, nil - //return tengo.FromInterface([]byte(str)) - }, - }) - script.Add("__pp_write_raw__", &tengo.UserFunction{ - Value: func(args ...tengo.Object) (tengo.Object, error){ - bt := make([][]byte, len(args)) - for i, o := range args { - bts, ok := tengo.ToByteSlice(o) - if !ok { - return nil, tengo.ErrInvalidArgumentType{ - Name: strconv.Itoa(i), - Expected: "string/bytes", - Found: o.TypeName(), - } - } - bt[i] = bts - } - for _, b := range bt { - retBuf.Write(b) - } - return nil, nil - }, - }) - script.Add("__separate__", &tengo.UserFunction{ - Value: func(args ...tengo.Object) (tengo.Object, error){ - return nil, nil - /*if len(args) < 1 { - return nil, tengo.ErrWrongNumArguments - } - bts, ok := tengo.ToByteSlice(args[0]) - if !ok { - return nil, tengo.ErrInvalidArgumentType{ - } - }*/ - //rets = append(rets, retBuf.Bytes()) - //retBuf.Reset() - return nil, nil - }, - }) - - compiled, err := script.Compile() - if err != nil { - return nil, nil, err - } - // To keep everything of the changes. - compiled = compiled.Clone() - - _, err = script.RunContext(ctx) - if err != nil { - return nil, nil, err - } - - return compiled, retBuf.Bytes(), nil -} - - diff --git a/tests/cat.jpg.tpp b/testdata/cat.jpg.tpp similarity index 100% rename from tests/cat.jpg.tpp rename to testdata/cat.jpg.tpp diff --git a/tests/cat.webp b/testdata/cat.webp similarity index 100% rename from tests/cat.webp rename to testdata/cat.webp diff --git a/tests/index.md.pp b/testdata/index.md.pp similarity index 69% rename from tests/index.md.pp rename to testdata/index.md.pp index eba06dd..a5008e7 100644 --- a/tests/index.md.pp +++ b/testdata/index.md.pp @@ -1,6 +1,6 @@ {{# os := import("os") - shit := import("./tests/shit.tengo") + shit := import("./testdata/shit") newVar := "'this is gen shita'" }} Text 0 @@ -13,7 +13,7 @@ Text 1 }} Text 2 {{ - bts := os.read_file("tests/somefile") + bts := os.read_file("testdata/somefile") pp.printf("%v\n%s", bts, bts) }} diff --git a/tests/shit.tengo b/testdata/shit.xgo similarity index 100% rename from tests/shit.tengo rename to testdata/shit.xgo diff --git a/tests/somefile b/testdata/somefile similarity index 100% rename from tests/somefile rename to testdata/somefile diff --git a/tool.go b/tool.go index 9b41082..0e31d47 100644 --- a/tool.go +++ b/tool.go @@ -1,8 +1,8 @@ -package tpp +package gopp import ( //"github.com/d5/tengo/v2" - "github.com/d5/tengo/v2/stdlib" + "surdeus.su/core/xgo/v2/stdlib" "surdeus.su/core/cli/mtool" //"fmt" "os" @@ -11,11 +11,13 @@ import ( "context" ) -var Tool = mtool.T("tpp").Func(func(flags *mtool.Flags){ +var Tool = mtool.T("gopp").Func(func(flags *mtool.Flags){ var ( modDir string + render bool ) + log.SetFlags(0) flags.StringVar( &modDir, "mod", @@ -23,34 +25,55 @@ var Tool = mtool.T("tpp").Func(func(flags *mtool.Flags){ "set the import directory", ) + flags.BoolVar( + &render, + "render", + false, + "render to xgo instead of preprocessing", + ) + filePaths := flags.Parse() - t := NewTengo(). - SetPreCompile(func(ctx context.Context, s *Script){ - s.SetImports(stdlib.GetModuleMap( - stdlib.AllModuleNames()..., - )) - s.EnableFileImport(true) - s.SetImportDir(modDir) - }) + x := NewXGo().SetPreCompile(func( + ctx context.Context, s *Script, + ){ + s.SetImports(stdlib.GetModuleMap( + stdlib.AllModuleNames()..., + )) + s.EnableFileImport(true) + s.SetImportDir(modDir) + }) - pp := New(t) + ctx := context.Background() + + pp := New(x) for _, filePath := range filePaths { pth := filepath.FromSlash(filePath) - bts, err := os.ReadFile(pth) + file, err := os.Open(pth) if err != nil { - log.Println("read error:", err) + log.Printf("os.Open(%s): %s\n", pth, err) continue } - _, out, err := pp.Process( + if render { + err := pp.Render(ctx, file, os.Stdout) + if err != nil { + log.Printf("pp.Render(...): %s\n", err) + } + continue + } + compiled, err := pp.Compile(context.Background(), file) + if err != nil { + log.Printf("pp.Compile(...): %s\n", err) + continue + } + err = pp.XGo().RunContext( context.Background(), - pth, - bts, + compiled.Clone(), + pth, os.Stdout, ) if err != nil { - log.Println("pp error:", err) + log.Printf("pp.Xgo().RunContext(...): %s", err) continue } - os.Stdout.Write(out) } }).Usage("[files]") diff --git a/xgo.go b/xgo.go new file mode 100644 index 0000000..27e3489 --- /dev/null +++ b/xgo.go @@ -0,0 +1,230 @@ +package gopp + +import ( + "surdeus.su/core/xgo/v2" + "fmt" + "io" + "context" + "bytes" + "strconv" +) + +type CompiledMap map[string] *Compiled + +type Script = xgo.Script +type Compiled = xgo.Compiled + +// The type describes +// customizable way to evaluate +// files. +type XGo struct { + // Functions to modify + // preprocessor for + // more specific purposes. + preCode func(context.Context) []byte + postCode func(context.Context) []byte + preCompile func(context.Context, *Script) +} + +// Returns the new XGo preprocessor +// with +func NewXGo() *XGo { + ret := &XGo{} + return ret +} + +func (xgo *XGo) SetPreCode(fn func(context.Context) []byte) *XGo { + xgo.preCode = fn + return xgo +} + +func (xgo *XGo) SetPostCode(fn func(context.Context) []byte) *XGo { + xgo.postCode = fn + return xgo +} + +func (xgo *XGo) SetPreCompile(fn func(context.Context, *Script)) *XGo { + xgo.preCompile = fn + return xgo +} + +const ( + ppFilePath = "__pp_file_path__" + ppPrintf = "__pp_printf__" + ppPrint = "__pp_print__" + ppPrintln = "__pp_println__" + ppWriteRaw = "__pp_write_raw__" + ppHead = `// The main generated head. Exists in every file. +context := {} +pp := immutable({ + fpath:` + ppFilePath + `, + printf :` + ppPrintf + `, + print :` + ppPrint + `, + println :` + ppPrintln + `, + write_raw :` + ppWriteRaw + ` +}) +`) + +// Render to code. +func (pp *XGo) Render( + ctx context.Context, + parsed *Parsed, + output io.Writer, +) (error) { + + fmt.Fprint(output, ppHead) + if parsed.PreCode != nil { + output.Write([]byte("// The parsed precode from file.\n")) + output.Write(parsed.PreCode) + } + + if pp.preCode != nil { + output.Write(pp.preCode(ctx)) + } + + for i, code := range parsed.Codes { + // Text. + fmt.Fprintf( + output, + "// Text %d\n" + + ppWriteRaw+"(%q)\n", + i, parsed.Texts[i], + ) + // Code. + fmt.Fprintf(output, "// Code %d\n", i) + output.Write(code) + + } + + i := len(parsed.Texts)-1 + fmt.Fprintf( + output, + "// Text %d\n" + + ppWriteRaw+"(%q)\n", + i, parsed.Texts[i], + ) + + if pp.postCode != nil { + output.Write(pp.postCode(ctx)) + } + return nil +} + +// b +func (pp *XGo) Compile( + ctx context.Context, + // Static text pieces. + parsed *Parsed, +) (*Compiled, error) { + var buf bytes.Buffer + err := pp.Render( + ctx, parsed, &buf, + ) + fullCode := buf.Bytes() + if err != nil { + return nil, err + } + script := xgo.NewScript(fullCode) + if pp.preCompile != nil { + pp.preCompile(ctx, script) + } + + // Presetting variables before running. + script.Add(ppFilePath, false) + script.Add(ppPrintf, false) + script.Add(ppPrint, false) + script.Add(ppPrintln, false) + script.Add(ppWriteRaw, false) + + compiled, err := script.Compile() + if err != nil { + return nil, err + } + return compiled, nil +} + +func (pp *XGo) RunContext( + ctx context.Context, + compiled *Compiled, + filePath string, + output io.Writer, +) error { + var err error + + err = compiled.Set(ppFilePath, filePath) + if err != nil {return err} + + err = compiled.Set(ppPrintf, &xgo.UserFunction{ + Value: func(args ...xgo.Object) (xgo.Object, error){ + if len(args) < 1 { + return nil, xgo.ErrWrongNumArguments + } + format, ok := xgo.ToString(args[0]) + if !ok { + return nil, xgo.ErrInvalidArgumentType{ + Expected: "string", + } + } + gargs := make([]any, len(args) - 1) + for i := range gargs { + gargs[i] = xgo.ToInterface(args[i+1]) + //fmt.Printf("shit: %q\n", gargs[i]) + } + fmt.Fprintf(output, format, gargs...) + return nil, nil + //return xgo.FromInterface([]byte(str)) + }, + }) + if err != nil {return err} + + err = compiled.Set(ppPrint, &xgo.UserFunction{ + Value: func(args ...xgo.Object) (xgo.Object, error){ + gargs := make([]any, len(args)) + for i := range gargs { + gargs[i] = xgo.ToInterface(args[i]) + } + fmt.Fprint(output, gargs...) + return nil, nil + //return xgo.FromInterface([]byte(str)) + }, + }) + if err != nil {return err} + + err = compiled.Set(ppPrintln, &xgo.UserFunction{ + Value: func(args ...xgo.Object) (xgo.Object, error){ + gargs := make([]any, len(args)) + for i := range gargs { + gargs[i] = xgo.ToInterface(args[i]) + } + fmt.Fprintln(output, gargs...) + return nil, nil + //return xgo.FromInterface([]byte(str)) + }, + }) + if err != nil {return err} + + err = compiled.Set(ppWriteRaw, &xgo.UserFunction{ + Value: func(args ...xgo.Object) (xgo.Object, error){ + bt := make([][]byte, len(args)) + for i, o := range args { + bts, ok := xgo.ToByteSlice(o) + if !ok { + return nil, xgo.ErrInvalidArgumentType{ + Name: strconv.Itoa(i), + Expected: "string/bytes", + Found: o.TypeName(), + } + } + bt[i] = bts + } + for _, b := range bt { + output.Write(b) + } + return nil, nil + }, + }) + if err != nil {return err} + + return compiled.RunContext(ctx) +} +