From e240cd5ba22945d8fda3fd930a070cf7d7881640 Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Tue, 28 Jul 2015 05:21:09 +0100 Subject: [PATCH] Metadata variables flattened. Fix race condition on parsers. Added page links generator. --- config/setup/markdown.go | 4 + middleware/markdown/markdown.go | 13 +++ middleware/markdown/metadata.go | 75 ++++++---------- middleware/markdown/metadata_test.go | 46 ++++------ middleware/markdown/page.go | 103 ++++++++++++++++++++++ middleware/markdown/process.go | 10 ++- middleware/markdown/renderer.go | 93 +++++++++++++++++++ middleware/markdown/testdata/blog/test.md | 3 +- middleware/markdown/testdata/log/test.md | 3 +- 9 files changed, 268 insertions(+), 82 deletions(-) create mode 100644 middleware/markdown/page.go create mode 100644 middleware/markdown/renderer.go diff --git a/config/setup/markdown.go b/config/setup/markdown.go index bbccf9d5..e5428c59 100644 --- a/config/setup/markdown.go +++ b/config/setup/markdown.go @@ -34,6 +34,10 @@ func Markdown(c *Controller) (middleware.Middleware, error) { continue } + if err := markdown.GenerateLinks(md, &cfg); err != nil { + return err + } + // If generated site already exists, clear it out _, err := os.Stat(cfg.StaticDir) if err == nil { diff --git a/middleware/markdown/markdown.go b/middleware/markdown/markdown.go index 71b18906..7efa3db7 100644 --- a/middleware/markdown/markdown.go +++ b/middleware/markdown/markdown.go @@ -10,6 +10,7 @@ import ( "github.com/mholt/caddy/middleware" "github.com/russross/blackfriday" + // "log" ) // Markdown implements a layer of middleware that serves @@ -64,6 +65,9 @@ type Config struct { // Map of request URL to static files generated StaticFiles map[string]string + // Links to all markdown pages ordered by date. + Links []PageLink + // Directory to store static files StaticDir string } @@ -114,6 +118,15 @@ func (md Markdown) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error } } + if m.StaticDir != "" { + // Markdown modified or new. Update links. + // go func() { + // if err := GenerateLinks(md, &md.Configs[i]); err != nil { + // log.Println(err) + // } + // }() + } + body, err := ioutil.ReadAll(f) if err != nil { return http.StatusInternalServerError, err diff --git a/middleware/markdown/metadata.go b/middleware/markdown/metadata.go index 9dc2cf1b..103b97b2 100644 --- a/middleware/markdown/metadata.go +++ b/middleware/markdown/metadata.go @@ -9,14 +9,7 @@ import ( "github.com/BurntSushi/toml" "gopkg.in/yaml.v2" -) - -var ( - parsers = []MetadataParser{ - &JSONMetadataParser{metadata: Metadata{Variables: make(map[string]string)}}, - &TOMLMetadataParser{metadata: Metadata{Variables: make(map[string]string)}}, - &YAMLMetadataParser{metadata: Metadata{Variables: make(map[string]string)}}, - } + "time" ) // Metadata stores a page's metadata @@ -27,20 +20,31 @@ type Metadata struct { // Page template Template string + // Publish date + Date time.Time + // Variables to be used with Template Variables map[string]string } // load loads parsed values in parsedMap into Metadata func (m *Metadata) load(parsedMap map[string]interface{}) { - if template, ok := parsedMap["title"]; ok { - m.Title, _ = template.(string) + if title, ok := parsedMap["title"]; ok { + m.Title, _ = title.(string) } if template, ok := parsedMap["template"]; ok { m.Template, _ = template.(string) } - if variables, ok := parsedMap["variables"]; ok { - m.Variables, _ = variables.(map[string]string) + if date, ok := parsedMap["date"].(string); ok { + if t, err := time.Parse(timeLayout, date); err == nil { + m.Date = t + } + } + // store everything as a variable + for key, val := range parsedMap { + if v, ok := val.(string); ok { + m.Variables[key] = v + } } } @@ -62,7 +66,7 @@ type MetadataParser interface { Metadata() Metadata } -// JSONMetadataParser is the MetdataParser for JSON +// JSONMetadataParser is the MetadataParser for JSON type JSONMetadataParser struct { metadata Metadata } @@ -76,16 +80,6 @@ func (j *JSONMetadataParser) Parse(b []byte) ([]byte, error) { if err := decoder.Decode(&m); err != nil { return b, err } - if vars, ok := m["variables"].(map[string]interface{}); ok { - vars1 := make(map[string]string) - for k, v := range vars { - if val, ok := v.(string); ok { - vars1[k] = val - } - } - m["variables"] = vars1 - } - j.metadata.load(m) // Retrieve remaining bytes after decoding @@ -129,15 +123,6 @@ func (t *TOMLMetadataParser) Parse(b []byte) ([]byte, error) { if err := toml.Unmarshal(b, &m); err != nil { return markdown, err } - if vars, ok := m["variables"].(map[string]interface{}); ok { - vars1 := make(map[string]string) - for k, v := range vars { - if val, ok := v.(string); ok { - vars1[k] = val - } - } - m["variables"] = vars1 - } t.metadata.load(m) return markdown, nil } @@ -174,21 +159,6 @@ func (y *YAMLMetadataParser) Parse(b []byte) ([]byte, error) { if err := yaml.Unmarshal(b, &m); err != nil { return markdown, err } - - // convert variables (if present) to map[string]interface{} - // to match expected type - if vars, ok := m["variables"].(map[interface{}]interface{}); ok { - vars1 := make(map[string]string) - for k, v := range vars { - if key, ok := k.(string); ok { - if val, ok := v.(string); ok { - vars1[key] = val - } - } - } - m["variables"] = vars1 - } - y.metadata.load(m) return markdown, nil } @@ -260,10 +230,19 @@ func findParser(b []byte) MetadataParser { return nil } line = bytes.TrimSpace(line) - for _, parser := range parsers { + for _, parser := range parsers() { if bytes.Equal(parser.Opening(), line) { return parser } } return nil } + +// parsers returns all available parsers +func parsers() []MetadataParser { + return []MetadataParser{ + &JSONMetadataParser{metadata: Metadata{Variables: make(map[string]string)}}, + &TOMLMetadataParser{metadata: Metadata{Variables: make(map[string]string)}}, + &YAMLMetadataParser{metadata: Metadata{Variables: make(map[string]string)}}, + } +} diff --git a/middleware/markdown/metadata_test.go b/middleware/markdown/metadata_test.go index c1028f29..b93c9ddc 100644 --- a/middleware/markdown/metadata_test.go +++ b/middleware/markdown/metadata_test.go @@ -11,13 +11,11 @@ import ( var TOML = [4]string{` title = "A title" template = "default" -[variables] name = "value" `, `+++ title = "A title" template = "default" -[variables] name = "value" +++ Page content @@ -25,7 +23,6 @@ Page content `+++ title = "A title" template = "default" -[variables] name = "value" `, `title = "A title" template = "default" [variables] name = "value"`, @@ -34,38 +31,31 @@ name = "value" var YAML = [4]string{` title : A title template : default -variables : - name : value +name : value `, `--- title : A title template : default -variables : - name : value +name : value --- Page content `, `--- title : A title template : default -variables : - name : value +name : value `, `title : A title template : default variables : name : value`, } var JSON = [4]string{` "title" : "A title", "template" : "default", - "variables" : { - "name" : "value" - } + "name" : "value" `, `{ "title" : "A title", "template" : "default", - "variables" : { - "name" : "value" - } + "name" : "value" } Page content `, @@ -73,17 +63,13 @@ Page content { "title" : "A title", "template" : "default", - "variables" : { - "name" : "value" - } + "name" : "value" `, ` {{ "title" : "A title", "template" : "default", - "variables" : { - "name" : "value" - } + "name" : "value" } `, } @@ -96,9 +82,13 @@ func check(t *testing.T, err error) { func TestParsers(t *testing.T) { expected := Metadata{ - Title: "A title", - Template: "default", - Variables: map[string]string{"name": "value"}, + Title: "A title", + Template: "default", + Variables: map[string]string{ + "name": "value", + "title": "A title", + "template": "default", + }, } compare := func(m Metadata) bool { if m.Title != expected.Title { @@ -112,7 +102,7 @@ func TestParsers(t *testing.T) { return false } } - return len(m.Variables) == 1 + return len(m.Variables) == len(expected.Variables) } data := []struct { @@ -120,9 +110,9 @@ func TestParsers(t *testing.T) { testData [4]string name string }{ - {&JSONMetadataParser{}, JSON, "json"}, - {&YAMLMetadataParser{}, YAML, "yaml"}, - {&TOMLMetadataParser{}, TOML, "toml"}, + {&JSONMetadataParser{metadata: Metadata{Variables: make(map[string]string)}}, JSON, "json"}, + {&YAMLMetadataParser{metadata: Metadata{Variables: make(map[string]string)}}, YAML, "yaml"}, + {&TOMLMetadataParser{metadata: Metadata{Variables: make(map[string]string)}}, TOML, "toml"}, } for _, v := range data { diff --git a/middleware/markdown/page.go b/middleware/markdown/page.go new file mode 100644 index 00000000..c1c82dcb --- /dev/null +++ b/middleware/markdown/page.go @@ -0,0 +1,103 @@ +package markdown + +import ( + "io/ioutil" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "time" + + "github.com/russross/blackfriday" +) + +var ( + pagesMutex sync.RWMutex + linksGenerating bool +) + +const ( + timeLayout = `2006-01-02 15:04:05` + summaryLen = 150 +) + +// Page represents a statically generated markdown page. +type PageLink struct { + Title string + Summary string + Date time.Time + Url string +} + +// pageLinkSorter sort PageLink by newest date to oldest. +type pageLinkSorter []PageLink + +func (p pageLinkSorter) Len() int { return len(p) } +func (p pageLinkSorter) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p pageLinkSorter) Less(i, j int) bool { return p[i].Date.After(p[j].Date) } + +func GenerateLinks(md Markdown, cfg *Config) error { + if linksGenerating { + return nil + } + + pagesMutex.Lock() + linksGenerating = true + + fp := filepath.Join(md.Root, cfg.PathScope) + + cfg.Links = []PageLink{} + err := filepath.Walk(fp, func(path string, info os.FileInfo, err error) error { + for _, ext := range cfg.Extensions { + if !info.IsDir() && strings.HasSuffix(info.Name(), ext) { + // Load the file + body, err := ioutil.ReadFile(path) + if err != nil { + return err + } + + // Get the relative path as if it were a HTTP request, + // then prepend with "/" (like a real HTTP request) + reqPath, err := filepath.Rel(md.Root, path) + if err != nil { + return err + } + reqPath = "/" + reqPath + + parser := findParser(body) + if parser == nil { + // no metadata, ignore. + continue + } + summary, err := parser.Parse(body) + if err != nil { + return err + } + + if len(summary) > summaryLen { + summary = summary[:summaryLen] + } + + metadata := parser.Metadata() + + cfg.Links = append(cfg.Links, PageLink{ + Title: metadata.Title, + Url: reqPath, + Date: metadata.Date, + Summary: string(blackfriday.Markdown(summary, PlaintextRenderer{}, 0)), + }) + + break // don't try other file extensions + } + } + + // sort by newest date + sort.Sort(pageLinkSorter(cfg.Links)) + return nil + }) + + linksGenerating = false + pagesMutex.Unlock() + return err +} diff --git a/middleware/markdown/process.go b/middleware/markdown/process.go index b97c205a..64ca313c 100644 --- a/middleware/markdown/process.go +++ b/middleware/markdown/process.go @@ -20,7 +20,8 @@ const ( type MarkdownData struct { middleware.Context - Doc map[string]string + Doc map[string]string + Links []PageLink } // Process processes the contents of a page in b. It parses the metadata @@ -97,9 +98,14 @@ func (md Markdown) processTemplate(c Config, requestPath string, tmpl []byte, me mdData := MarkdownData{ Context: ctx, Doc: metadata.Variables, + Links: c.Links, } - if err = t.Execute(b, mdData); err != nil { + pagesMutex.RLock() + err = t.Execute(b, mdData) + pagesMutex.RUnlock() + + if err != nil { return nil, err } diff --git a/middleware/markdown/renderer.go b/middleware/markdown/renderer.go new file mode 100644 index 00000000..8fab8d2a --- /dev/null +++ b/middleware/markdown/renderer.go @@ -0,0 +1,93 @@ +package markdown + +import ( + "bytes" +) + +type PlaintextRenderer struct{} + +// Block-level callbacks + +func (r PlaintextRenderer) BlockCode(out *bytes.Buffer, text []byte, lang string) {} + +func (r PlaintextRenderer) BlockQuote(out *bytes.Buffer, text []byte) {} + +func (r PlaintextRenderer) BlockHtml(out *bytes.Buffer, text []byte) {} + +func (r PlaintextRenderer) Header(out *bytes.Buffer, text func() bool, level int, id string) {} + +func (r PlaintextRenderer) HRule(out *bytes.Buffer) {} + +func (r PlaintextRenderer) List(out *bytes.Buffer, text func() bool, flags int) {} + +func (r PlaintextRenderer) ListItem(out *bytes.Buffer, text []byte, flags int) {} + +func (r PlaintextRenderer) Paragraph(out *bytes.Buffer, text func() bool) { + marker := out.Len() + if !text() { + out.Truncate(marker) + } + out.Write([]byte{' '}) +} + +func (r PlaintextRenderer) Table(out *bytes.Buffer, header []byte, body []byte, columnData []int) {} + +func (r PlaintextRenderer) TableRow(out *bytes.Buffer, text []byte) {} + +func (r PlaintextRenderer) TableHeaderCell(out *bytes.Buffer, text []byte, flags int) {} + +func (r PlaintextRenderer) TableCell(out *bytes.Buffer, text []byte, flags int) {} + +func (r PlaintextRenderer) Footnotes(out *bytes.Buffer, text func() bool) {} + +func (r PlaintextRenderer) FootnoteItem(out *bytes.Buffer, name, text []byte, flags int) {} + +func (r PlaintextRenderer) TitleBlock(out *bytes.Buffer, text []byte) {} + +// Span-level callbacks + +func (r PlaintextRenderer) AutoLink(out *bytes.Buffer, link []byte, kind int) {} + +func (r PlaintextRenderer) CodeSpan(out *bytes.Buffer, text []byte) {} + +func (r PlaintextRenderer) DoubleEmphasis(out *bytes.Buffer, text []byte) { + out.Write(text) +} + +func (r PlaintextRenderer) Emphasis(out *bytes.Buffer, text []byte) { + out.Write(text) +} + +func (r PlaintextRenderer) Image(out *bytes.Buffer, link []byte, title []byte, alt []byte) {} + +func (r PlaintextRenderer) LineBreak(out *bytes.Buffer) {} + +func (r PlaintextRenderer) Link(out *bytes.Buffer, link []byte, title []byte, content []byte) { + out.Write(content) +} +func (r PlaintextRenderer) RawHtmlTag(out *bytes.Buffer, tag []byte) {} + +func (r PlaintextRenderer) TripleEmphasis(out *bytes.Buffer, text []byte) { + out.Write(text) +} +func (r PlaintextRenderer) StrikeThrough(out *bytes.Buffer, text []byte) {} + +func (r PlaintextRenderer) FootnoteRef(out *bytes.Buffer, ref []byte, id int) {} + +// Low-level callbacks + +func (r PlaintextRenderer) Entity(out *bytes.Buffer, entity []byte) { + out.Write(entity) +} + +func (r PlaintextRenderer) NormalText(out *bytes.Buffer, text []byte) { + out.Write(text) +} + +// Header and footer + +func (r PlaintextRenderer) DocumentHeader(out *bytes.Buffer) {} + +func (r PlaintextRenderer) DocumentFooter(out *bytes.Buffer) {} + +func (r PlaintextRenderer) GetFlags() int { return 0 } diff --git a/middleware/markdown/testdata/blog/test.md b/middleware/markdown/testdata/blog/test.md index 7ec76616..3d33ad91 100644 --- a/middleware/markdown/testdata/blog/test.md +++ b/middleware/markdown/testdata/blog/test.md @@ -1,7 +1,6 @@ --- title: Markdown test -variables: - sitename: A Caddy website +sitename: A Caddy website --- ## Welcome on the blog diff --git a/middleware/markdown/testdata/log/test.md b/middleware/markdown/testdata/log/test.md index 7ec76616..3d33ad91 100644 --- a/middleware/markdown/testdata/log/test.md +++ b/middleware/markdown/testdata/log/test.md @@ -1,7 +1,6 @@ --- title: Markdown test -variables: - sitename: A Caddy website +sitename: A Caddy website --- ## Welcome on the blog