From 50c9873c2baa5fcb56a71b3ac45a67da887b9840 Mon Sep 17 00:00:00 2001 From: Mechiel Lukkien Date: Sat, 11 Nov 2023 19:40:53 +0100 Subject: [PATCH] cross-referencing code & rfc: todo comments and html pages - the rfc links back to the code now show any "todo" text that appears in the code. helps when looking at an rfc to find any work that may need to be done. - html pages can now be generated to view code and rfc's side by side. clicking on links in one side opens the linked document in the other page, at the correct line number. i'll be publishing the "dev" html version (latest commit on main branch) on the mox website, updated with each commit. the dev pages will also link to the latest released version. --- develop.txt | 1 + rfc/Makefile | 3 + rfc/index.txt | 2 +- rfc/link.go | 23 ++- rfc/xr.go | 422 ++++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 447 insertions(+), 4 deletions(-) create mode 100644 rfc/xr.go diff --git a/develop.txt b/develop.txt index c8f546e..5e76101 100644 --- a/develop.txt +++ b/develop.txt @@ -222,4 +222,5 @@ done - Create git tag, push code. - Publish new docker image. - Publish signed release notes for updates.xmox.nl and update DNS record. +- Publish new cross-referenced code/rfc to www.xmox.nl/xr/. - Create new release on the github page, so watchers get a notification. diff --git a/rfc/Makefile b/rfc/Makefile index 7ee44a9..41dbe91 100644 --- a/rfc/Makefile +++ b/rfc/Makefile @@ -5,3 +5,6 @@ fetch: link: go run -tags link link.go -- ../*.go ../*/*.go + +xr: + go run xr.go -- xr-dev $$(git tag | tail -n1) ../*.go ../*/*.go diff --git a/rfc/index.txt b/rfc/index.txt index b442808..eccc9d4 100644 --- a/rfc/index.txt +++ b/rfc/index.txt @@ -221,7 +221,7 @@ https://www.iana.org/assignments/message-headers/message-headers.xhtml 9208 IMAP QUOTA Extension 9394 IMAP PARTIAL Extension for Paged SEARCH and FETCH -5198 Unicode Format for Network Interchange +5198 Unicode Format for Network Interchange # Lemonade profile 4550 (obsoleted by RFC 5550) Internet Email to Support Diverse Service Environments (Lemonade) Profile diff --git a/rfc/link.go b/rfc/link.go index 2172b88..1f2028f 100644 --- a/rfc/link.go +++ b/rfc/link.go @@ -4,6 +4,8 @@ package main // Read source files and RFC and errata files, and cross-link them. +// todo: also cross-reference typescript and possibly other files. switch from go parser to just reading the source as text. + import ( "bytes" "flag" @@ -40,6 +42,7 @@ func main() { dstlineno int dstisrfc bool dstrfc string // e.g. "5322" or "6376-eid4810" + comment string // e.g. "todo" or "todo spec" } // RFC-file to RFC-line to references to list of file+line (possibly RFCs). @@ -75,6 +78,16 @@ func main() { continue } + var comment string + if strings.HasPrefix(line, "// todo") { + s, _, have := strings.Cut(strings.TrimPrefix(line, "// "), ":") + if have { + comment = s + } else { + comment = "todo" + } + } + srcpath := arg srclineno := fset.Position(c.Pos()).Line + i dir := filepath.Dir(srcpath) @@ -104,9 +117,9 @@ func main() { if _, err := os.Stat(dstpath); err != nil { log.Fatalf("%s:%d: references %s: %v", srcpath, srclineno, dstpath, err) } - r := ref{srcpath, srclineno, dstpath, dstlineno, true, rfc} + r := ref{srcpath, srclineno, dstpath, dstlineno, true, rfc, comment} addRef(sourceLineRFCs, r.srcpath, r.srclineno, r) - addRef(rfcLineSources, r.dstrfc, r.dstlineno, ref{r.dstrfc, r.dstlineno, r.srcpath, r.srclineno, false, ""}) + addRef(rfcLineSources, r.dstrfc, r.dstlineno, ref{r.dstrfc, r.dstlineno, r.srcpath, r.srclineno, false, "", comment}) } } } @@ -162,7 +175,11 @@ func main() { // Add link from rfc to source code. for _, r := range refs { - line += fmt.Sprintf(" %s:%d", r.dstpath, r.dstlineno) + comment := r.comment + if comment != "" { + comment += ": " + } + line += fmt.Sprintf(" %s%s:%d", comment, r.dstpath, r.dstlineno) } if iserrata { line = line[1:] diff --git a/rfc/xr.go b/rfc/xr.go new file mode 100644 index 0000000..29f940c --- /dev/null +++ b/rfc/xr.go @@ -0,0 +1,422 @@ +//go:build xr + +package main + +// xr reads source files and rfc files and generates html versions, a code and +// rfc index file, and an overal index file to view code and rfc side by side. + +import ( + "bytes" + "flag" + "fmt" + htmltemplate "html/template" + "log" + "os" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + + "golang.org/x/exp/maps" +) + +var destdir string + +func xcheckf(err error, format string, args ...any) { + if err != nil { + log.Fatalf("%s: %s", fmt.Sprintf(format, args...), err) + } +} + +func xwritefile(path string, buf []byte) { + p := filepath.Join(destdir, path) + os.MkdirAll(filepath.Dir(p), 0755) + err := os.WriteFile(p, buf, 0644) + xcheckf(err, "writing file", p) +} + +func main() { + log.SetFlags(0) + + flag.Usage = func() { + log.Println("usage: go run xr.go destdir latestrelease ../*.go ../*/*.go") + flag.PrintDefaults() + os.Exit(2) + } + flag.Parse() + args := flag.Args() + if len(args) < 2 { + flag.Usage() + } + + destdir = args[0] + latestRelease := args[1] + srcfiles := args[2:] + + // Generate code.html index. + srcdirs := map[string][]string{} + for _, arg := range srcfiles { + arg = strings.TrimPrefix(arg, "../") + dir := filepath.Dir(arg) + file := filepath.Base(arg) + srcdirs[dir] = append(srcdirs[dir], file) + } + for _, files := range srcdirs { + sort.Strings(files) + } + dirs := maps.Keys(srcdirs) + sort.Strings(dirs) + var codeBuf bytes.Buffer + err := codeTemplate.Execute(&codeBuf, map[string]any{ + "Dirs": srcdirs, + }) + xcheckf(err, "generating code.html") + xwritefile("code.html", codeBuf.Bytes()) + + // Generate code html files. + re := regexp.MustCompile(`(\.\./)?rfc/[0-9]{3,5}(-[^ :]*)?(:[0-9]+)?`) + for dir, files := range srcdirs { + for _, file := range files { + src := filepath.Join("..", dir, file) + dst := filepath.Join(dir, file+".html") + buf, err := os.ReadFile(src) + xcheckf(err, "reading file %s", src) + + var b bytes.Buffer + fmt.Fprint(&b, ` + + + + + + +`) + + for i, line := range strings.Split(string(buf), "\n") { + n := i + 1 + _, err := fmt.Fprintf(&b, `
%d`, n, n, n) + xcheckf(err, "writing source line") + + if line == "" { + b.WriteString("\n") + } else { + for line != "" { + loc := re.FindStringIndex(line) + if loc == nil { + b.WriteString(htmltemplate.HTMLEscapeString(line)) + break + } + s, e := loc[0], loc[1] + b.WriteString(htmltemplate.HTMLEscapeString(line[:s])) + match := line[s:e] + line = line[e:] + t := strings.Split(match, ":") + linenumber := 1 + if len(t) == 2 { + v, err := strconv.ParseInt(t[1], 10, 31) + xcheckf(err, "parsing linenumber %q", t[1]) + linenumber = int(v) + } + fmt.Fprintf(&b, `%s`, t[0], linenumber, htmltemplate.HTMLEscapeString(match)) + } + } + fmt.Fprint(&b, "
\n") + } + + fmt.Fprint(&b, ` + + +`) + + xwritefile(dst, b.Bytes()) + } + } + + // Generate rfc index. + rfctext, err := os.ReadFile("index.txt") + xcheckf(err, "reading rfc index.txt") + type rfc struct { + File string + Title string + } + topics := map[string][]rfc{} + var topic string + for _, line := range strings.Split(string(rfctext), "\n") { + if strings.HasPrefix(line, "# ") { + topic = line[2:] + continue + } + t := strings.Split(line, "\t") + if len(t) != 2 { + continue + } + topics[topic] = append(topics[topic], rfc{strings.TrimSpace(t[0]), t[1]}) + } + var rfcBuf bytes.Buffer + err = rfcTemplate.Execute(&rfcBuf, map[string]any{ + "Topics": topics, + }) + xcheckf(err, "generating rfc.html") + xwritefile("rfc.html", rfcBuf.Bytes()) + + // Process each rfc file into html. + for _, rfcs := range topics { + for _, rfc := range rfcs { + dst := filepath.Join("rfc", rfc.File+".html") + + buf, err := os.ReadFile(rfc.File) + xcheckf(err, "reading rfc %s", rfc.File) + + var b bytes.Buffer + fmt.Fprint(&b, ` + + + + + + +`) + + isRef := func(s string) bool { + return s[0] >= '0' && s[0] <= '9' || strings.HasPrefix(s, "../") + } + + parseRef := func(s string) (string, int, bool) { + t := strings.Split(s, ":") + linenumber := 1 + if len(t) == 2 { + v, err := strconv.ParseInt(t[1], 10, 31) + xcheckf(err, "parsing linenumber") + linenumber = int(v) + } + isCode := strings.HasPrefix(t[0], "../") + return t[0], linenumber, isCode + } + + for i, line := range strings.Split(string(buf), "\n") { + if line == "" { + line = "\n" + } else if len(line) < 80 || strings.Contains(rfc.File, "-") { + line = htmltemplate.HTMLEscapeString(line) + } else { + t := strings.Split(line[80:], " ") + line = htmltemplate.HTMLEscapeString(line[:80]) + for i, s := range t { + if i > 0 { + line += " " + } + if s == "" || !isRef(s) { + line += htmltemplate.HTMLEscapeString(s) + continue + } + file, linenumber, isCode := parseRef(s) + target := "" + if isCode { + target = ` target="code"` + } + line += fmt.Sprintf(` %s:%d`, file, linenumber, target, file, linenumber) + } + } + n := i + 1 + _, err := fmt.Fprintf(&b, `
%d%s
%s`, n, n, n, line, "\n") + xcheckf(err, "writing rfc line") + } + + fmt.Fprint(&b, ` + + + +`) + + xwritefile(dst, b.Bytes()) + } + } + + // Generate overal file. + xwritefile("index.html", []byte(strings.ReplaceAll(indexHTML, "RELEASE", latestRelease))) +} + +var indexHTML = ` + + + + mox cross-referenced code and RFCs + + + +
+
mox, cross-referenced code and RFCs: dev RELEASE
+
+
+
..., index
+ +
+
+
..., index
+ +
+
+
+ + + +` + +var codeTemplate = htmltemplate.Must(htmltemplate.New("code").Parse(` + + + + code index + + + + + +{{- range $dir, $files := .Dirs }} + + + + +{{- end }} +
PackageFiles
{{ $dir }}/ + {{- range $files }} + {{ . }} + {{- end }} +
+ + +`)) + +var rfcTemplate = htmltemplate.Must(htmltemplate.New("rfc").Parse(` + + + + + + + + +{{- range $topic, $rfcs := .Topics }} + + + + +{{- end }} +
TopicRFC
{{ $topic }} + {{- range $rfcs }} + {{ .File }} + {{- end }} +
+ + +`))