feat: restruct and and better API for making caching for servers.

This commit is contained in:
Andrey Parhomenko 2024-08-02 19:00:46 +05:00
parent c2f2c344c2
commit f60bd728df
18 changed files with 354 additions and 314 deletions

3
.gitignore vendored
View file

@ -1,3 +1,2 @@
/tpp
/exe
/gopp
Session.vim

View file

@ -1,4 +0,0 @@
#!/bin/sh
#
go build -o ./exe/ ./cmd/tpp

10
cmd/gopp/main.go Normal file
View file

@ -0,0 +1,10 @@
package main
import (
"surdeus.su/util/gopp"
"os"
)
func main() {
gopp.Tool.Run(os.Args[1:])
}

View file

@ -1,10 +0,0 @@
package main
import (
"surdeus.su/util/tpp"
"os"
)
func main() {
tpp.Tool.Run(os.Args[1:])
}

View file

@ -1,4 +1,4 @@
package tpp
package gopp
import (
//"errors"

View file

@ -1,6 +0,0 @@
package tpp
type Evaler interface {
Tags() [2]string
Eval(filePath string, pieces []string) ([]string, error)
}

9
go.mod
View file

@ -1,8 +1,7 @@
module surdeus.su/util/tpp
module surdeus.su/util/gopp
go 1.22.3
require (
github.com/d5/tengo/v2 v2.17.0
surdeus.su/core/cli v0.1.2
)
require surdeus.su/core/cli v0.5.0
require surdeus.su/core/xgo/v2 v2.18.0

6
go.sum
View file

@ -1,4 +1,6 @@
github.com/d5/tengo/v2 v2.17.0 h1:BWUN9NoJzw48jZKiYDXDIF3QrIVZRm1uV1gTzeZ2lqM=
github.com/d5/tengo/v2 v2.17.0/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8=
surdeus.su/core/cli v0.1.2 h1:qPzjawqPyZsO4Z5SaA1u141recVE65yioA83Qs7Jecs=
surdeus.su/core/cli v0.1.2/go.mod h1:r9JtQz3aEJzpYzMaNUNQHJoYkoWKNPi047qhd5uGlmA=
surdeus.su/core/cli v0.5.0 h1:jYvE0JVDikFT9FhWGV3wIAcMgByziAVxTwsVUwWkeHs=
surdeus.su/core/cli v0.5.0/go.mod h1:r9JtQz3aEJzpYzMaNUNQHJoYkoWKNPi047qhd5uGlmA=
surdeus.su/core/xgo/v2 v2.18.0 h1:Ypz2CIiqkfVD9S4Jes5fES9zkcOYppNRvf4Af5D3JXU=
surdeus.su/core/xgo/v2 v2.18.0/go.mod h1:EBQA07DjGd0bqIGbaNQ2HZHQkilbYacI7fpQen5ivao=

View file

@ -1,3 +0,0 @@
#!/bin/sh
go install ./cmd/tpp/

View file

@ -1,21 +1,27 @@
package tpp
package gopp
import (
"bytes"
"context"
"io"
)
type Parsed struct {
Texts [][]byte
PreCode []byte
Codes [][]byte
}
type Preprocessor struct {
tengo *Tengo
xgo *XGo
tags [2][]byte
preTag byte
}
// Get the new preprocessor with default options.
func New(tengo *Tengo ) *Preprocessor {
func New(xgo *XGo) *Preprocessor {
pp := &Preprocessor{}
pp.tengo = tengo
pp.xgo = xgo
pp.tags = [2][]byte{
[]byte("{{"),
[]byte("}}"),
@ -24,11 +30,18 @@ func New(tengo *Tengo ) *Preprocessor {
return pp
}
func (pp *Preprocessor) Process(
func (pp *Preprocessor) XGo() *XGo {
return pp.xgo
}
func (pp *Preprocessor) Parse(
ctx context.Context,
filePath string,
data []byte,
) (*Compiled, []byte, error) {
input io.Reader,
) (*Parsed, error) {
data, err := io.ReadAll(input)
if err != nil {
return nil, err
}
//var b bytes.Buffer
preCode := []byte(nil)
texts := [][]byte{}
@ -37,7 +50,7 @@ func (pp *Preprocessor) Process(
if bytes.HasPrefix(data, pref) {
idxEnd := bytes.Index(data, pp.tags[1])
if idxEnd < 0 {
return nil, nil, UnexpectedError{
return nil, UnexpectedError{
What: "pre-code start tag",
}
}
@ -51,14 +64,14 @@ func (pp *Preprocessor) Process(
//fmt.Printf("cock %d %d %d\n", last, idxStart, idxEnd)
if idxStart < 0 {
if idxEnd >= 0 {
return nil, nil, UnexpectedError{
return nil, UnexpectedError{
What: "end tag",
}
}
texts = append(texts, data)
break
} else if idxEnd < 0 {
return nil, nil, UnexpectedError{
return nil, UnexpectedError{
What: "start tag",
}
}
@ -72,17 +85,41 @@ func (pp *Preprocessor) Process(
data = data[idxEnd + len(pp.tags[1]):]
}
compiled, ret, err := pp.tengo.Eval(
ctx,
filePath,
texts,
preCode,
codes,
)
if err != nil {
return nil, nil, err
}
return compiled, ret, nil
return &Parsed{
Texts: texts,
PreCode: preCode,
Codes: codes,
}, nil
}
func (pp *Preprocessor) Render(
ctx context.Context,
input io.Reader,
output io.Writer,
) error {
parsed, err := pp.Parse(ctx, input)
if err != nil { return err }
return pp.xgo.Render(ctx, parsed, output)
}
func (pp *Preprocessor) Compile(
ctx context.Context,
input io.Reader,
) (*Compiled, error) {
parsed, err := pp.Parse(ctx, input)
if err != nil {
return nil, err
}
compiled, err := pp.xgo.Compile(
ctx,
parsed,
)
if err != nil {
return nil, err
}
return compiled, nil
}

237
tengo.go
View file

@ -1,237 +0,0 @@
package tpp
import (
"github.com/d5/tengo/v2"
"fmt"
"context"
"bytes"
"strconv"
)
type CompiledMap map[string] *Compiled
type Script = tengo.Script
type Compiled = tengo.Compiled
// The type describes
// customizable way to evaluate
// files.
type Tengo struct {
// Functions to modify
// preprocessor for
// more specific purposes.
preCode func(context.Context) []byte
postCode func(context.Context) []byte
preCompile func(context.Context, *Script)
}
// Returns the new Tengo preprocessor
// with
func NewTengo() *Tengo {
ret := &Tengo{}
return ret
}
func (tengo *Tengo) SetPreCode(fn func(context.Context) []byte) *Tengo {
tengo.preCode = fn
return tengo
}
func (tengo *Tengo) SetPostCode(fn func(context.Context) []byte) *Tengo {
tengo.postCode = fn
return tengo
}
func (tengo *Tengo) SetPreCompile(fn func(context.Context, *Script)) *Tengo {
tengo.preCompile = fn
return tengo
}
func (pp *Tengo) Eval(
ctx context.Context,
// File path to give it.
filePath string,
// Static text pieces.
texts [][]byte,
// The code that comes before any
// dynamic code.
preCode []byte,
// General code.
codes [][]byte,
) (*Compiled, []byte, error) {
var fullCodeBuf, retBuf bytes.Buffer
const retHead = `
context := {}
pp := immutable({
fpath: __pp_filepath__,
printf : __pp_printf__,
print : __pp_print__,
println : __pp_println__,
write_raw : __pp_write_raw__,
include: __pp_include__
})
`
const retSeparator = `
__separate__()
`
fmt.Fprint(&fullCodeBuf, retHead)
if preCode != nil {
fmt.Fprintln(
&fullCodeBuf,
string(preCode)+"\n",
)
}
if pp.preCode != nil {
fullCodeBuf.Write(pp.preCode(ctx))
fullCodeBuf.Write([]byte(retSeparator))
}
for i, code := range codes {
fmt.Fprintf(
&fullCodeBuf,
"\n__pp_write_raw__(%q)\n",
texts[i],
)
fmt.Fprintln(
&fullCodeBuf,
"\n" + string(code) + retSeparator,
)
}
fmt.Fprintf(
&fullCodeBuf,
"\n__pp_write_raw__(%q)\n",
texts[len(texts)-1],
)
if pp.postCode != nil {
fullCodeBuf.Write(pp.postCode(ctx))
fullCodeBuf.Write([]byte(retSeparator))
}
script := tengo.NewScript(fullCodeBuf.Bytes())
if pp.preCompile != nil {
pp.preCompile(ctx, script)
}
script.Add("__pp_include__", &tengo.UserFunction{
Value: func(args ...tengo.Object) (tengo.Object, error){
if len(args) < 1 {
return nil, tengo.ErrWrongNumArguments
}
format, ok := tengo.ToString(args[0])
if !ok {
return nil, tengo.ErrInvalidArgumentType{
Expected: "string",
}
}
gargs := make([]any, len(args) - 1)
for i := range gargs {
gargs[i] = tengo.ToInterface(args[i+1])
//fmt.Printf("shit: %q\n", gargs[i])
}
fmt.Fprintf(&retBuf, format, gargs...)
return nil, nil
//return tengo.FromInterface([]byte(str))
},
})
script.Add("__pp_filepath__", filePath)
script.Add("__pp_printf__", &tengo.UserFunction{
Value: func(args ...tengo.Object) (tengo.Object, error){
if len(args) < 1 {
return nil, tengo.ErrWrongNumArguments
}
format, ok := tengo.ToString(args[0])
if !ok {
return nil, tengo.ErrInvalidArgumentType{
Expected: "string",
}
}
gargs := make([]any, len(args) - 1)
for i := range gargs {
gargs[i] = tengo.ToInterface(args[i+1])
//fmt.Printf("shit: %q\n", gargs[i])
}
fmt.Fprintf(&retBuf, format, gargs...)
return nil, nil
//return tengo.FromInterface([]byte(str))
},
})
script.Add("__pp_print__", &tengo.UserFunction{
Value: func(args ...tengo.Object) (tengo.Object, error){
gargs := make([]any, len(args))
for i := range gargs {
gargs[i] = tengo.ToInterface(args[i])
}
fmt.Fprint(&retBuf, gargs...)
return nil, nil
//return tengo.FromInterface([]byte(str))
},
})
script.Add("__pp_println__", &tengo.UserFunction{
Value: func(args ...tengo.Object) (tengo.Object, error){
gargs := make([]any, len(args))
for i := range gargs {
gargs[i] = tengo.ToInterface(args[i])
}
fmt.Fprintln(&retBuf, gargs...)
return nil, nil
//return tengo.FromInterface([]byte(str))
},
})
script.Add("__pp_write_raw__", &tengo.UserFunction{
Value: func(args ...tengo.Object) (tengo.Object, error){
bt := make([][]byte, len(args))
for i, o := range args {
bts, ok := tengo.ToByteSlice(o)
if !ok {
return nil, tengo.ErrInvalidArgumentType{
Name: strconv.Itoa(i),
Expected: "string/bytes",
Found: o.TypeName(),
}
}
bt[i] = bts
}
for _, b := range bt {
retBuf.Write(b)
}
return nil, nil
},
})
script.Add("__separate__", &tengo.UserFunction{
Value: func(args ...tengo.Object) (tengo.Object, error){
return nil, nil
/*if len(args) < 1 {
return nil, tengo.ErrWrongNumArguments
}
bts, ok := tengo.ToByteSlice(args[0])
if !ok {
return nil, tengo.ErrInvalidArgumentType{
}
}*/
//rets = append(rets, retBuf.Bytes())
//retBuf.Reset()
return nil, nil
},
})
compiled, err := script.Compile()
if err != nil {
return nil, nil, err
}
// To keep everything of the changes.
compiled = compiled.Clone()
_, err = script.RunContext(ctx)
if err != nil {
return nil, nil, err
}
return compiled, retBuf.Bytes(), nil
}

View file

Before

Width:  |  Height:  |  Size: 127 KiB

After

Width:  |  Height:  |  Size: 127 KiB

View file

@ -1,6 +1,6 @@
{{#
os := import("os")
shit := import("./tests/shit.tengo")
shit := import("./testdata/shit")
newVar := "'this is gen shita'"
}}
Text 0
@ -13,7 +13,7 @@ Text 1
}}
Text 2
{{
bts := os.read_file("tests/somefile")
bts := os.read_file("testdata/somefile")
pp.printf("%v\n%s", bts, bts)
}}

View file

49
tool.go
View file

@ -1,8 +1,8 @@
package tpp
package gopp
import (
//"github.com/d5/tengo/v2"
"github.com/d5/tengo/v2/stdlib"
"surdeus.su/core/xgo/v2/stdlib"
"surdeus.su/core/cli/mtool"
//"fmt"
"os"
@ -11,11 +11,13 @@ import (
"context"
)
var Tool = mtool.T("tpp").Func(func(flags *mtool.Flags){
var Tool = mtool.T("gopp").Func(func(flags *mtool.Flags){
var (
modDir string
render bool
)
log.SetFlags(0)
flags.StringVar(
&modDir,
"mod",
@ -23,10 +25,18 @@ var Tool = mtool.T("tpp").Func(func(flags *mtool.Flags){
"set the import directory",
)
flags.BoolVar(
&render,
"render",
false,
"render to xgo instead of preprocessing",
)
filePaths := flags.Parse()
t := NewTengo().
SetPreCompile(func(ctx context.Context, s *Script){
x := NewXGo().SetPreCompile(func(
ctx context.Context, s *Script,
){
s.SetImports(stdlib.GetModuleMap(
stdlib.AllModuleNames()...,
))
@ -34,23 +44,36 @@ var Tool = mtool.T("tpp").Func(func(flags *mtool.Flags){
s.SetImportDir(modDir)
})
pp := New(t)
ctx := context.Background()
pp := New(x)
for _, filePath := range filePaths {
pth := filepath.FromSlash(filePath)
bts, err := os.ReadFile(pth)
file, err := os.Open(pth)
if err != nil {
log.Println("read error:", err)
log.Printf("os.Open(%s): %s\n", pth, err)
continue
}
_, out, err := pp.Process(
if render {
err := pp.Render(ctx, file, os.Stdout)
if err != nil {
log.Printf("pp.Render(...): %s\n", err)
}
continue
}
compiled, err := pp.Compile(context.Background(), file)
if err != nil {
log.Printf("pp.Compile(...): %s\n", err)
continue
}
err = pp.XGo().RunContext(
context.Background(),
pth,
bts,
compiled.Clone(),
pth, os.Stdout,
)
if err != nil {
log.Println("pp error:", err)
log.Printf("pp.Xgo().RunContext(...): %s", err)
continue
}
os.Stdout.Write(out)
}
}).Usage("[files]")

230
xgo.go Normal file
View file

@ -0,0 +1,230 @@
package gopp
import (
"surdeus.su/core/xgo/v2"
"fmt"
"io"
"context"
"bytes"
"strconv"
)
type CompiledMap map[string] *Compiled
type Script = xgo.Script
type Compiled = xgo.Compiled
// The type describes
// customizable way to evaluate
// files.
type XGo struct {
// Functions to modify
// preprocessor for
// more specific purposes.
preCode func(context.Context) []byte
postCode func(context.Context) []byte
preCompile func(context.Context, *Script)
}
// Returns the new XGo preprocessor
// with
func NewXGo() *XGo {
ret := &XGo{}
return ret
}
func (xgo *XGo) SetPreCode(fn func(context.Context) []byte) *XGo {
xgo.preCode = fn
return xgo
}
func (xgo *XGo) SetPostCode(fn func(context.Context) []byte) *XGo {
xgo.postCode = fn
return xgo
}
func (xgo *XGo) SetPreCompile(fn func(context.Context, *Script)) *XGo {
xgo.preCompile = fn
return xgo
}
const (
ppFilePath = "__pp_file_path__"
ppPrintf = "__pp_printf__"
ppPrint = "__pp_print__"
ppPrintln = "__pp_println__"
ppWriteRaw = "__pp_write_raw__"
ppHead = `// The main generated head. Exists in every file.
context := {}
pp := immutable({
fpath:` + ppFilePath + `,
printf :` + ppPrintf + `,
print :` + ppPrint + `,
println :` + ppPrintln + `,
write_raw :` + ppWriteRaw + `
})
`)
// Render to code.
func (pp *XGo) Render(
ctx context.Context,
parsed *Parsed,
output io.Writer,
) (error) {
fmt.Fprint(output, ppHead)
if parsed.PreCode != nil {
output.Write([]byte("// The parsed precode from file.\n"))
output.Write(parsed.PreCode)
}
if pp.preCode != nil {
output.Write(pp.preCode(ctx))
}
for i, code := range parsed.Codes {
// Text.
fmt.Fprintf(
output,
"// Text %d\n" +
ppWriteRaw+"(%q)\n",
i, parsed.Texts[i],
)
// Code.
fmt.Fprintf(output, "// Code %d\n", i)
output.Write(code)
}
i := len(parsed.Texts)-1
fmt.Fprintf(
output,
"// Text %d\n" +
ppWriteRaw+"(%q)\n",
i, parsed.Texts[i],
)
if pp.postCode != nil {
output.Write(pp.postCode(ctx))
}
return nil
}
// b
func (pp *XGo) Compile(
ctx context.Context,
// Static text pieces.
parsed *Parsed,
) (*Compiled, error) {
var buf bytes.Buffer
err := pp.Render(
ctx, parsed, &buf,
)
fullCode := buf.Bytes()
if err != nil {
return nil, err
}
script := xgo.NewScript(fullCode)
if pp.preCompile != nil {
pp.preCompile(ctx, script)
}
// Presetting variables before running.
script.Add(ppFilePath, false)
script.Add(ppPrintf, false)
script.Add(ppPrint, false)
script.Add(ppPrintln, false)
script.Add(ppWriteRaw, false)
compiled, err := script.Compile()
if err != nil {
return nil, err
}
return compiled, nil
}
func (pp *XGo) RunContext(
ctx context.Context,
compiled *Compiled,
filePath string,
output io.Writer,
) error {
var err error
err = compiled.Set(ppFilePath, filePath)
if err != nil {return err}
err = compiled.Set(ppPrintf, &xgo.UserFunction{
Value: func(args ...xgo.Object) (xgo.Object, error){
if len(args) < 1 {
return nil, xgo.ErrWrongNumArguments
}
format, ok := xgo.ToString(args[0])
if !ok {
return nil, xgo.ErrInvalidArgumentType{
Expected: "string",
}
}
gargs := make([]any, len(args) - 1)
for i := range gargs {
gargs[i] = xgo.ToInterface(args[i+1])
//fmt.Printf("shit: %q\n", gargs[i])
}
fmt.Fprintf(output, format, gargs...)
return nil, nil
//return xgo.FromInterface([]byte(str))
},
})
if err != nil {return err}
err = compiled.Set(ppPrint, &xgo.UserFunction{
Value: func(args ...xgo.Object) (xgo.Object, error){
gargs := make([]any, len(args))
for i := range gargs {
gargs[i] = xgo.ToInterface(args[i])
}
fmt.Fprint(output, gargs...)
return nil, nil
//return xgo.FromInterface([]byte(str))
},
})
if err != nil {return err}
err = compiled.Set(ppPrintln, &xgo.UserFunction{
Value: func(args ...xgo.Object) (xgo.Object, error){
gargs := make([]any, len(args))
for i := range gargs {
gargs[i] = xgo.ToInterface(args[i])
}
fmt.Fprintln(output, gargs...)
return nil, nil
//return xgo.FromInterface([]byte(str))
},
})
if err != nil {return err}
err = compiled.Set(ppWriteRaw, &xgo.UserFunction{
Value: func(args ...xgo.Object) (xgo.Object, error){
bt := make([][]byte, len(args))
for i, o := range args {
bts, ok := xgo.ToByteSlice(o)
if !ok {
return nil, xgo.ErrInvalidArgumentType{
Name: strconv.Itoa(i),
Expected: "string/bytes",
Found: o.TypeName(),
}
}
bt[i] = bts
}
for _, b := range bt {
output.Write(b)
}
return nil, nil
},
})
if err != nil {return err}
return compiled.RunContext(ctx)
}