Compare commits

...

16 commits
master ... main

27 changed files with 462 additions and 483 deletions

4
.gitignore vendored
View file

@ -1,2 +1,2 @@
/tpp
/exe
/gopp
Session.vim

View file

@ -1,4 +0,0 @@
#!/bin/sh
#
go build -o ./exe/ ./cmd/tpp ./cmd/tht

BIN
cat.webp

Binary file not shown.

Before

Width:  |  Height:  |  Size: 127 KiB

10
cmd/gopp/main.go Normal file
View file

@ -0,0 +1,10 @@
package main
import (
"surdeus.su/util/gopp"
"os"
)
func main() {
gopp.Tool.Run(os.Args[1:])
}

View file

@ -1,11 +0,0 @@
package main
import (
"vultras.su/util/tpp/server"
"os"
)
func main() {
server.Tool.Run(os.Args[1:])
}

View file

@ -1,10 +0,0 @@
package main
import (
"vultras.su/util/tpp"
"os"
)
func main() {
tpp.Tool.Run(os.Args[1:])
}

3
doc/readme.md Normal file
View file

@ -0,0 +1,3 @@
# Intro
Some more intro.

View file

@ -1,4 +1,4 @@
package tpp
package gopp
import (
//"errors"

View file

@ -1,6 +0,0 @@
package tpp
type Evaler interface {
Tags() [2]string
Eval(filePath string, pieces []string) ([]string, error)
}

11
go.mod
View file

@ -1,8 +1,7 @@
module vultras.su/util/tpp
module surdeus.su/util/gopp
go 1.21.7
go 1.22.3
require (
github.com/d5/tengo/v2 v2.16.1 // indirect
vultras.su/core/cli v0.0.0-20240104195345-5d79542278a0 // indirect
)
require surdeus.su/core/cli v0.5.0
require surdeus.su/core/xgo/v2 v2.18.0

10
go.sum
View file

@ -1,4 +1,6 @@
github.com/d5/tengo/v2 v2.16.1 h1:/N6dqiGu9toqANInZEOQMM8I06icdZnmb+81DG/lZdw=
github.com/d5/tengo/v2 v2.16.1/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8=
vultras.su/core/cli v0.0.0-20240104195345-5d79542278a0 h1:acA77oEg7hDuUchyBvw7scG9gayVLFAC9/CGuqyfLSA=
vultras.su/core/cli v0.0.0-20240104195345-5d79542278a0/go.mod h1:rYu/sFWE3vUkDSSJCZt+K1aVaso0MYcZ+tmxQd4whdQ=
surdeus.su/core/cli v0.1.2 h1:qPzjawqPyZsO4Z5SaA1u141recVE65yioA83Qs7Jecs=
surdeus.su/core/cli v0.1.2/go.mod h1:r9JtQz3aEJzpYzMaNUNQHJoYkoWKNPi047qhd5uGlmA=
surdeus.su/core/cli v0.5.0 h1:jYvE0JVDikFT9FhWGV3wIAcMgByziAVxTwsVUwWkeHs=
surdeus.su/core/cli v0.5.0/go.mod h1:r9JtQz3aEJzpYzMaNUNQHJoYkoWKNPi047qhd5uGlmA=
surdeus.su/core/xgo/v2 v2.18.0 h1:Ypz2CIiqkfVD9S4Jes5fES9zkcOYppNRvf4Af5D3JXU=
surdeus.su/core/xgo/v2 v2.18.0/go.mod h1:EBQA07DjGd0bqIGbaNQ2HZHQkilbYacI7fpQen5ivao=

83
main.go
View file

@ -1,83 +0,0 @@
package tpp
import (
//"fmt"
"bytes"
"context"
)
type Preprocessor struct {
tengo *Tengo
tags [2][]byte
}
// Get the new preprocessor with default options.
func New(tengo *Tengo ) *Preprocessor {
pp := &Preprocessor{}
pp.tengo = tengo
pp.tags = [2][]byte{
[]byte("{{"),
[]byte("}}"),
}
return pp
}
func (pp *Preprocessor) Process(
ctx context.Context,
recompile bool,
filePath string,
data []byte,
) ([]byte, error) {
var b bytes.Buffer
last := 0
texts := [][]byte{}
codes := [][]byte{}
for {
idxStart := bytes.Index(data[last:], pp.tags[0])
idxEnd := bytes.Index(data[last:], pp.tags[1])
//fmt.Printf("cock %d %d %d\n", last, idxStart, idxEnd)
if idxStart < 0 {
if idxEnd >= 0 {
return nil, UnexpectedError{
What: "end tag",
}
}
texts = append(texts, data[last:])
break
} else if idxEnd < 0 {
return nil, UnexpectedError{
What: "start tag",
}
}
text := data[last:idxStart]
texts = append(texts, text)
code := data[idxStart+len(pp.tags[0]):idxEnd]
codes = append(codes, code)
data = data[idxEnd + len(pp.tags[1]):]
/*if len(data) > 0 && data[0] == '\n' {
data = data[1:]
}*/
}
codeRets, err := pp.tengo.Eval(
ctx,
recompile,
filePath,
codes,
)
if err != nil {
return nil, err
}
for i, codeRet := range codeRets {
b.Write(texts[i])
b.Write(codeRet)
}
b.Write(texts[len(codeRets)])
return b.Bytes(), nil
}

125
pp.go Normal file
View file

@ -0,0 +1,125 @@
package gopp
import (
"bytes"
"context"
"io"
)
type Parsed struct {
Texts [][]byte
PreCode []byte
Codes [][]byte
}
type Preprocessor struct {
xgo *XGo
tags [2][]byte
preTag byte
}
// Get the new preprocessor with default options.
func New(xgo *XGo) *Preprocessor {
pp := &Preprocessor{}
pp.xgo = xgo
pp.tags = [2][]byte{
[]byte("{{"),
[]byte("}}"),
}
pp.preTag = '#'
return pp
}
func (pp *Preprocessor) XGo() *XGo {
return pp.xgo
}
func (pp *Preprocessor) Parse(
ctx context.Context,
input io.Reader,
) (*Parsed, error) {
data, err := io.ReadAll(input)
if err != nil {
return nil, err
}
//var b bytes.Buffer
preCode := []byte(nil)
texts := [][]byte{}
codes := [][]byte{}
pref := append(pp.tags[0], pp.preTag)
if bytes.HasPrefix(data, pref) {
idxEnd := bytes.Index(data, pp.tags[1])
if idxEnd < 0 {
return nil, UnexpectedError{
What: "pre-code start tag",
}
}
preCode = data[len(pref):idxEnd]
//texts = append(texts, []byte{})
data = data[idxEnd+len(pp.tags[1]):]
}
for {
idxStart := bytes.Index(data, pp.tags[0])
idxEnd := bytes.Index(data, pp.tags[1])
//fmt.Printf("cock %d %d %d\n", last, idxStart, idxEnd)
if idxStart < 0 {
if idxEnd >= 0 {
return nil, UnexpectedError{
What: "end tag",
}
}
texts = append(texts, data)
break
} else if idxEnd < 0 {
return nil, UnexpectedError{
What: "start tag",
}
}
text := data[:idxStart]
texts = append(texts, text)
code := data[idxStart+len(pp.tags[0]):idxEnd]
codes = append(codes, code)
data = data[idxEnd + len(pp.tags[1]):]
}
return &Parsed{
Texts: texts,
PreCode: preCode,
Codes: codes,
}, nil
}
func (pp *Preprocessor) Render(
ctx context.Context,
input io.Reader,
output io.Writer,
) error {
parsed, err := pp.Parse(ctx, input)
if err != nil { return err }
return pp.xgo.Render(ctx, parsed, output)
}
func (pp *Preprocessor) Compile(
ctx context.Context,
input io.Reader,
) (*Compiled, error) {
parsed, err := pp.Parse(ctx, input)
if err != nil {
return nil, err
}
compiled, err := pp.xgo.Compile(
ctx,
parsed,
)
if err != nil {
return nil, err
}
return compiled, nil
}

4
readme.md Normal file
View file

@ -0,0 +1,4 @@
# tpp
Tengo preprocessor library.
See the [docs](./doc/readme.md)

View file

@ -1,76 +0,0 @@
package server
import (
"vultras.su/util/tpp"
"path/filepath"
"net/http"
"path"
"mime"
"log"
"os"
"io"
)
var _ = http.Handler(&Handler{})
// The type describes behaviour
// of handling stuff like in PHP.
type Handler struct {
// THe field represents
// where we store site's
// source files and request handlers.
SourcePath string
// Preprocessor must be set by user
// to be able to bring custom features in.
PP *tpp.Preprocessor
// Aditional extension. ".tpp" by default.
// For example "file.html.tpp" will be
// first preprocessed TPP and sent back as simple HTML.
Ext string
}
func (h *Handler) ServeHTTP(
w http.ResponseWriter,
r *http.Request,
) {
urlPath := r.URL.Path
// Cleaning URL path to prevent injections.
urlPath = path.Clean(urlPath)
urlExt := path.Ext(urlPath)
filePath := filepath.Join(
filepath.FromSlash(h.SourcePath),
filepath.FromSlash(urlPath),
)
filePathTpp := filePath + h.Ext
//log.Println("pth:", urlPath, filePathTpp)
file, err := os.Open(filePathTpp)
if err != nil {
http.NotFound(w, r)
return
}
//process := true
fileData, err := io.ReadAll(file)
if err != nil {
http.NotFound(w, r)
return
}
processedData, err := h.PP.Process(
r.Context(),
true,
filePathTpp,
fileData,
)
if err != nil {
http.NotFound(w, r)
log.Printf("Error: pp.Process(...): %s\n", err)
return
}
contentType := mime.TypeByExtension(urlExt)
w.Header().Set("Content-Type", contentType)
w.Write(processedData)
}

View file

@ -1,42 +0,0 @@
package server
import (
//"github.com/d5/tengo/v2"
"github.com/d5/tengo/v2/stdlib"
"vultras.su/util/tpp"
"vultras.su/core/cli/mtool"
"net/http"
"log"
)
var Tool = mtool.T("tht").Func(func(flags *mtool.Flags){
var (
addr string
handler Handler
)
flags.StringVar(&addr, "addr", ":3000", "address to serve at")
flags.StringVar(&handler.SourcePath, "src", "./src", "directory with source files")
flags.StringVar(&handler.Ext, "ext", ".tpp", "extension for TPP files")
flags.Parse()
t := tpp.NewTengo().SetPreCompile(func(s *tpp.Script){
s.SetImports(stdlib.GetModuleMap(
stdlib.AllModuleNames()...,
))
s.EnableFileImport(true)
s.SetImportDir(handler.SourcePath)
})
handler.PP = tpp.New(t)
srv := &http.Server{
Addr: addr,
Handler: &handler,
}
err := srv.ListenAndServe()
if err != nil {
log.Printf("Error: srv.ListenAndServe(...): %s\n", err)
}
})

View file

@ -1,14 +0,0 @@
{{
list := [1, 2, 3, 4, 123]
}}<!doctype html>
<html><head>
</head><body>
<div>
Hello, Cock!
</div>
<ul>{{
for v in list {
pp.print("<li>", v, "</li>")
}
}}</ul>
</body></html>

View file

@ -1,8 +0,0 @@
{
"error": null,
"data": {
"name": "Andrey",
"surname": "Parhomenko",
"age": 22
}
}

174
tengo.go
View file

@ -1,174 +0,0 @@
package tpp
import (
"github.com/d5/tengo/v2"
"fmt"
"context"
"bytes"
)
type Script = tengo.Script
type Compiled = tengo.Compiled
type Tengo struct {
head string
compiledScripts map[string] *Compiled
// Functions to modify
// preprocessor for
// more specific purposes.
preCode func() []byte
postCode func() []byte
preCompile func(*Script)
}
// Returns the new Tengo preprocessor
// with
func NewTengo() *Tengo {
ret := &Tengo{}
return ret
}
func (tengo *Tengo) SetPreCode(fn func() []byte) *Tengo {
tengo.preCode = fn
return tengo
}
func (tengo *Tengo) SetPostCode(fn func() []byte) *Tengo {
tengo.postCode = fn
return tengo
}
func (tengo *Tengo) SetPreCompile(fn func(*Script)) *Tengo {
tengo.preCompile = fn
return tengo
}
// Simple Evaler implementation for the Tengo language
func (pp *Tengo) Eval(
ctx context.Context,
recompile bool,
filePath string,
codes [][]byte,
) ([][]byte, error) {
var fullCodeBuf bytes.Buffer
const retHead = `
__ret_one__ := bytes("")
pp := {
printf : func(format, ...vals) {
__ret_one__ += __sprintf__(format, vals...)
},
print : func(...vals) {
__ret_one__ += __sprint__(vals...)
},
println : func(...vals) {
__ret_one__ += __sprintln__(vals...)
},
write_raw : func(bts) {
__ret_one__ += bytes(bts)
}
}
`
const retSeparator = `
__separate__(__ret_one__)
__ret_one__ = ""
`
rets := [][]byte{}
fmt.Fprint(&fullCodeBuf, retHead)
if pp.preCode != nil {
fullCodeBuf.Write(pp.preCode())
}
for _, code := range codes {
fmt.Fprintln(&fullCodeBuf, "\n" + string(code) + retSeparator)
}
if pp.postCode != nil {
fullCodeBuf.Write(pp.postCode())
}
script := tengo.NewScript(fullCodeBuf.Bytes())
if pp.preCompile != nil {
pp.preCompile(script)
}
err := script.Add("__sprintf__", &tengo.UserFunction{
Value: func(args ...tengo.Object) (tengo.Object, error){
if len(args) < 1 {
return nil, tengo.ErrWrongNumArguments
}
format, ok := tengo.ToString(args[0])
if !ok {
return nil, tengo.ErrInvalidArgumentType{
Expected: "string",
}
}
gargs := make([]any, len(args) - 1)
for i := range gargs {
gargs[i] = tengo.ToInterface(args[i+1])
//fmt.Printf("shit: %q\n", gargs[i])
}
str := fmt.Sprintf(format, gargs...)
return tengo.FromInterface([]byte(str))
},
},
)
if err != nil {
return nil, err
}
err = script.Add("__sprint__", &tengo.UserFunction{
Value: func(args ...tengo.Object) (tengo.Object, error){
gargs := make([]any, len(args))
for i := range gargs {
gargs[i] = tengo.ToInterface(args[i])
}
str := fmt.Sprint(gargs...)
return tengo.FromInterface([]byte(str))
},
},
)
if err != nil {
return nil, err
}
err = script.Add("__sprintln__", &tengo.UserFunction{
Value: func(args ...tengo.Object) (tengo.Object, error){
gargs := make([]any, len(args))
for i := range gargs {
gargs[i] = tengo.ToInterface(args[i])
}
str := fmt.Sprintln(gargs...)
return tengo.FromInterface([]byte(str))
},
},
)
if err != nil {
return nil, err
}
err = script.Add("__separate__", &tengo.UserFunction{
Value: func(args ...tengo.Object) (tengo.Object, error){
if len(args) < 1 {
return nil, tengo.ErrWrongNumArguments
}
bts, ok := tengo.ToByteSlice(args[0])
if !ok {
return nil, tengo.ErrInvalidArgumentType{
}
}
rets = append(rets, bts)
return nil, nil
},
},
)
if err != nil {
return nil, err
}
_, err = script.RunContext(ctx)
if err != nil {
return nil, err
}
return rets, nil
}

View file

Before

Width:  |  Height:  |  Size: 127 KiB

After

Width:  |  Height:  |  Size: 127 KiB

19
testdata/index.md.pp vendored Normal file
View file

@ -0,0 +1,19 @@
{{#
os := import("os")
shit := import("./testdata/shit")
newVar := "'this is gen shita'"
}}
Text 0
{{
pp.println("Code 0")
}}
Text 1
{{
pp.println("Code 1")
}}
Text 2
{{
bts := os.read_file("testdata/somefile")
pp.printf("%v\n%s", bts, bts)
}}

View file

View file

@ -1,20 +0,0 @@
{{
os := import("os")
shit := import("./tests/shit.tengo")
newVar := "'this is gen shita'"
pp.println(shit.some_func())
}}# The index testing
1 + 1 = {{ pp.print(1+1) }}
cock {{ pp.print(newVar, "and cock") }}
## The shit after
checking {{ pp.printf(newVar) }}
## File contents
{{
bts := os.read_file("tests/somefile")
pp.printf("%v\n%s", bts, bts)
}}

79
tool.go
View file

@ -1,9 +1,9 @@
package tpp
package gopp
import (
//"github.com/d5/tengo/v2"
"github.com/d5/tengo/v2/stdlib"
"vultras.su/core/cli/mtool"
"surdeus.su/core/xgo/v2/stdlib"
"surdeus.su/core/cli/mtool"
//"fmt"
"os"
"log"
@ -11,34 +11,69 @@ import (
"context"
)
var Tool = mtool.T("pp").Func(func(flags *mtool.Flags){
t := NewTengo().
SetPreCompile(func(s *Script){
s.SetImports(stdlib.GetModuleMap(
stdlib.AllModuleNames()...,
))
s.EnableFileImport(true)
s.SetImportDir(".")
})
pp := New(t)
var Tool = mtool.T("gopp").Func(func(flags *mtool.Flags){
var (
modDir string
render bool
)
log.SetFlags(0)
flags.StringVar(
&modDir,
"mod",
".",
"set the import directory",
)
flags.BoolVar(
&render,
"render",
false,
"render to xgo instead of preprocessing",
)
filePaths := flags.Parse()
x := NewXGo().SetPreCompile(func(
ctx context.Context, s *Script,
){
s.SetImports(stdlib.GetModuleMap(
stdlib.AllModuleNames()...,
))
s.EnableFileImport(true)
s.SetImportDir(modDir)
})
ctx := context.Background()
pp := New(x)
for _, filePath := range filePaths {
pth := filepath.FromSlash(filePath)
bts, err := os.ReadFile(pth)
file, err := os.Open(pth)
if err != nil {
log.Println("read error:", err)
log.Printf("os.Open(%s): %s\n", pth, err)
continue
}
out, err := pp.Process(
if render {
err := pp.Render(ctx, file, os.Stdout)
if err != nil {
log.Printf("pp.Render(...): %s\n", err)
}
continue
}
compiled, err := pp.Compile(context.Background(), file)
if err != nil {
log.Printf("pp.Compile(...): %s\n", err)
continue
}
err = pp.XGo().RunContext(
context.Background(),
true,
pth,
bts,
compiled.Clone(),
pth, os.Stdout,
)
if err != nil {
log.Println("pp error:", err)
log.Printf("pp.Xgo().RunContext(...): %s", err)
continue
}
os.Stdout.Write(out)
}
})
}).Usage("[files]")

230
xgo.go Normal file
View file

@ -0,0 +1,230 @@
package gopp
import (
"surdeus.su/core/xgo/v2"
"fmt"
"io"
"context"
"bytes"
"strconv"
)
type CompiledMap map[string] *Compiled
type Script = xgo.Script
type Compiled = xgo.Compiled
// The type describes
// customizable way to evaluate
// files.
type XGo struct {
// Functions to modify
// preprocessor for
// more specific purposes.
preCode func(context.Context) []byte
postCode func(context.Context) []byte
preCompile func(context.Context, *Script)
}
// Returns the new XGo preprocessor
// with
func NewXGo() *XGo {
ret := &XGo{}
return ret
}
func (xgo *XGo) SetPreCode(fn func(context.Context) []byte) *XGo {
xgo.preCode = fn
return xgo
}
func (xgo *XGo) SetPostCode(fn func(context.Context) []byte) *XGo {
xgo.postCode = fn
return xgo
}
func (xgo *XGo) SetPreCompile(fn func(context.Context, *Script)) *XGo {
xgo.preCompile = fn
return xgo
}
const (
ppFilePath = "__pp_file_path__"
ppPrintf = "__pp_printf__"
ppPrint = "__pp_print__"
ppPrintln = "__pp_println__"
ppWriteRaw = "__pp_write_raw__"
ppHead = `// The main generated head. Exists in every file.
context := {}
pp := immutable({
fpath:` + ppFilePath + `,
printf :` + ppPrintf + `,
print :` + ppPrint + `,
println :` + ppPrintln + `,
write_raw :` + ppWriteRaw + `
})
`)
// Render to code.
func (pp *XGo) Render(
ctx context.Context,
parsed *Parsed,
output io.Writer,
) (error) {
fmt.Fprint(output, ppHead)
if parsed.PreCode != nil {
output.Write([]byte("// The parsed precode from file.\n"))
output.Write(parsed.PreCode)
}
if pp.preCode != nil {
output.Write(pp.preCode(ctx))
}
for i, code := range parsed.Codes {
// Text.
fmt.Fprintf(
output,
"// Text %d\n" +
ppWriteRaw+"(%q)\n",
i, parsed.Texts[i],
)
// Code.
fmt.Fprintf(output, "// Code %d\n", i)
output.Write(code)
}
i := len(parsed.Texts)-1
fmt.Fprintf(
output,
"// Text %d\n" +
ppWriteRaw+"(%q)\n",
i, parsed.Texts[i],
)
if pp.postCode != nil {
output.Write(pp.postCode(ctx))
}
return nil
}
// b
func (pp *XGo) Compile(
ctx context.Context,
// Static text pieces.
parsed *Parsed,
) (*Compiled, error) {
var buf bytes.Buffer
err := pp.Render(
ctx, parsed, &buf,
)
fullCode := buf.Bytes()
if err != nil {
return nil, err
}
script := xgo.NewScript(fullCode)
if pp.preCompile != nil {
pp.preCompile(ctx, script)
}
// Presetting variables before running.
script.Add(ppFilePath, false)
script.Add(ppPrintf, false)
script.Add(ppPrint, false)
script.Add(ppPrintln, false)
script.Add(ppWriteRaw, false)
compiled, err := script.Compile()
if err != nil {
return nil, err
}
return compiled, nil
}
func (pp *XGo) RunContext(
ctx context.Context,
compiled *Compiled,
filePath string,
output io.Writer,
) error {
var err error
err = compiled.Set(ppFilePath, filePath)
if err != nil {return err}
err = compiled.Set(ppPrintf, &xgo.UserFunction{
Value: func(args ...xgo.Object) (xgo.Object, error){
if len(args) < 1 {
return nil, xgo.ErrWrongNumArguments
}
format, ok := xgo.ToString(args[0])
if !ok {
return nil, xgo.ErrInvalidArgumentType{
Expected: "string",
}
}
gargs := make([]any, len(args) - 1)
for i := range gargs {
gargs[i] = xgo.ToInterface(args[i+1])
//fmt.Printf("shit: %q\n", gargs[i])
}
fmt.Fprintf(output, format, gargs...)
return nil, nil
//return xgo.FromInterface([]byte(str))
},
})
if err != nil {return err}
err = compiled.Set(ppPrint, &xgo.UserFunction{
Value: func(args ...xgo.Object) (xgo.Object, error){
gargs := make([]any, len(args))
for i := range gargs {
gargs[i] = xgo.ToInterface(args[i])
}
fmt.Fprint(output, gargs...)
return nil, nil
//return xgo.FromInterface([]byte(str))
},
})
if err != nil {return err}
err = compiled.Set(ppPrintln, &xgo.UserFunction{
Value: func(args ...xgo.Object) (xgo.Object, error){
gargs := make([]any, len(args))
for i := range gargs {
gargs[i] = xgo.ToInterface(args[i])
}
fmt.Fprintln(output, gargs...)
return nil, nil
//return xgo.FromInterface([]byte(str))
},
})
if err != nil {return err}
err = compiled.Set(ppWriteRaw, &xgo.UserFunction{
Value: func(args ...xgo.Object) (xgo.Object, error){
bt := make([][]byte, len(args))
for i, o := range args {
bts, ok := xgo.ToByteSlice(o)
if !ok {
return nil, xgo.ErrInvalidArgumentType{
Name: strconv.Itoa(i),
Expected: "string/bytes",
Found: o.TypeName(),
}
}
bt[i] = bts
}
for _, b := range bt {
output.Write(b)
}
return nil, nil
},
})
if err != nil {return err}
return compiled.RunContext(ctx)
}