Merge pull request #213 from abiosoft/master

markdown: Watch for file changes for links. Removed sitegen requirement for link index.

config/setup/markdown.go

@@ -29,13 +29,20 @@ func Markdown(c *Controller) (middleware.Middleware, error) {
 
 	// For any configs that enabled static site gen, sweep the whole path at startup
 	c.Startup = append(c.Startup, func() error {
-		for _, cfg := range mdconfigs {
+		for i := range mdconfigs {
-			if cfg.StaticDir == "" {
+			cfg := &mdconfigs[i]
-				continue
+
+			// Links generation.
+			if err := markdown.GenerateLinks(md, cfg); err != nil {
+				return err
+			}
+			// Watch file changes for links generation if not in development mode.
+			if !cfg.Development {
+				markdown.Watch(md, cfg, markdown.DefaultInterval)
 			}
 
-			if err := markdown.GenerateLinks(md, &cfg); err != nil {
+			if cfg.StaticDir == "" {
-				return err
+				continue
 			}
 
 			// If generated site already exists, clear it out
@@ -68,7 +75,7 @@ func Markdown(c *Controller) (middleware.Middleware, error) {
 
 						// Generate the static file
 						ctx := middleware.Context{Root: md.FileSys}
-						_, err = md.Process(cfg, reqPath, body, ctx)
+						_, err = md.Process(*cfg, reqPath, body, ctx)
 						if err != nil {
 							return err
 						}
@@ -155,6 +162,16 @@ func markdownParse(c *Controller) ([]markdown.Config, error) {
 					// only 1 argument allowed
 					return mdconfigs, c.ArgErr()
 				}
+			case "dev":
+				if c.NextArg() {
+					md.Development = strings.ToLower(c.Val()) == "true"
+				} else {
+					md.Development = true
+				}
+				if c.NextArg() {
+					// only 1 argument allowed
+					return mdconfigs, c.ArgErr()
+				}
 			default:
 				return mdconfigs, c.Err("Expected valid markdown configuration property")
 			}

middleware/markdown/markdown.go

@@ -69,12 +69,29 @@ type Config struct {
 	// Links to all markdown pages ordered by date.
 	Links []PageLink
 
+	// Stores a directory hash to check for changes.
+	linksHash string
+
 	// Directory to store static files
 	StaticDir string
 
+	// If in development mode. i.e. Actively editing markdown files.
+	Development bool
+
 	sync.RWMutex
 }
 
+// IsValidExt checks to see if an extension is a valid markdown extension
+// for config.
+func (c Config) IsValidExt(ext string) bool {
+	for _, e := range c.Extensions {
+		if e == ext {
+			return true
+		}
+	}
+	return false
+}
+
 // ServeHTTP implements the http.Handler interface.
 func (md Markdown) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) {
 	for i := range md.Configs {
@@ -103,6 +120,13 @@ func (md Markdown) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error
 					return http.StatusNotFound, nil
 				}
 
+				// if development is set, scan directory for file changes for links.
+				if m.Development {
+					if err := GenerateLinks(md, m); err != nil {
+						log.Println(err)
+					}
+				}
+
 				// if static site is generated, attempt to use it
 				if filepath, ok := m.StaticFiles[fpath]; ok {
 					if fs1, err := os.Stat(filepath); err == nil {
@@ -122,13 +146,6 @@ func (md Markdown) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error
 					}
 				}
 
-				if m.StaticDir != "" {
-					// Markdown modified or new. Update links.
-					if err := GenerateLinks(md, m); err != nil {
-						log.Println(err)
-					}
-				}
-
 				body, err := ioutil.ReadAll(f)
 				if err != nil {
 					return http.StatusInternalServerError, err

middleware/markdown/markdown_test.go

@@ -1,6 +1,7 @@
 package markdown
 
 import (
+	"bufio"
 	"log"
 	"net/http"
 	"net/http/httptest"
@@ -102,7 +103,7 @@ func getTrue() bool {
 </body>
 </html>
 `
-	if respBody != expectedBody {
+	if !equalStrings(respBody, expectedBody) {
 		t.Fatalf("Expected body: %v got: %v", expectedBody, respBody)
 	}
 
@@ -143,10 +144,7 @@ func getTrue() bool {
 	</body>
 </html>`
 
-	replacer := strings.NewReplacer("\r", "", "\n", "")
+	if !equalStrings(respBody, expectedBody) {
-	respBody = replacer.Replace(respBody)
-	expectedBody = replacer.Replace(expectedBody)
-	if respBody != expectedBody {
 		t.Fatalf("Expected body: %v got: %v", expectedBody, respBody)
 	}
 
@@ -177,26 +175,31 @@ func getTrue() bool {
 
 </body>
 </html>`
-	respBody = replacer.Replace(respBody)
+
-	expectedBody = replacer.Replace(expectedBody)
+	if !equalStrings(respBody, expectedBody) {
-	if respBody != expectedBody {
 		t.Fatalf("Expected body: %v got: %v", expectedBody, respBody)
 	}
 
 	expectedLinks := []string{
 		"/blog/test.md",
 		"/log/test.md",
-		"/og/first.md",
 	}
 
-	for i, c := range md.Configs {
+	for i := range md.Configs {
+		c := &md.Configs[i]
+		if err := GenerateLinks(md, c); err != nil {
+			t.Fatalf("Error: %v", err)
+		}
+	}
+
+	for i, c := range md.Configs[:2] {
 		log.Printf("Test number: %d, configuration links: %v, config: %v", i, c.Links, c)
 		if c.Links[0].URL != expectedLinks[i] {
 			t.Fatalf("Expected %v got %v", expectedLinks[i], c.Links[0].URL)
 		}
 	}
 
-	// attempt to trigger race condition
+	// attempt to trigger race conditions
 	var w sync.WaitGroup
 	f := func() {
 		req, err := http.NewRequest("GET", "/log/test.md", nil)
@@ -214,8 +217,32 @@ func getTrue() bool {
 	}
 	w.Wait()
 
+	f = func() {
+		GenerateLinks(md, &md.Configs[0])
+		w.Done()
+	}
+	for i := 0; i < 5; i++ {
+		w.Add(1)
+		go f()
+	}
+	w.Wait()
+
 	if err = os.RemoveAll(DefaultStaticDir); err != nil {
 		t.Errorf("Error while removing the generated static files: %v", err)
 	}
 
 }
+
+func equalStrings(s1, s2 string) bool {
+	s1 = strings.TrimSpace(s1)
+	s2 = strings.TrimSpace(s2)
+	in := bufio.NewScanner(strings.NewReader(s1))
+	for in.Scan() {
+		txt := strings.TrimSpace(in.Text())
+		if !strings.HasPrefix(strings.TrimSpace(s2), txt) {
+			return false
+		}
+		s2 = strings.Replace(s2, txt, "", 1)
+	}
+	return true
+}

middleware/markdown/page.go

@@ -2,7 +2,11 @@ package markdown
 
 import (
 	"bytes"
+	"crypto/sha1"
+	"encoding/hex"
+	"fmt"
 	"io/ioutil"
+	"log"
 	"os"
 	"path/filepath"
 	"sort"
@@ -75,10 +79,23 @@ func (l *linkGen) generateLinks(md Markdown, cfg *Config) {
 	if _, err := os.Stat(fp); os.IsNotExist(err) {
 		l.Lock()
 		l.lastErr = err
+		l.generating = false
 		l.Unlock()
 		return
 	}
 
+	hash, err := computeDirHash(md, *cfg)
+
+	// same hash, return.
+	if err == nil && hash == cfg.linksHash {
+		l.Lock()
+		l.generating = false
+		l.Unlock()
+		return
+	} else if err != nil {
+		log.Println("Error:", err)
+	}
+
 	cfg.Links = []PageLink{}
 
 	cfg.Lock()
@@ -138,6 +155,8 @@ func (l *linkGen) generateLinks(md Markdown, cfg *Config) {
 
 	// sort by newest date
 	sort.Sort(byDate(cfg.Links))
+
+	cfg.linksHash = hash
 	cfg.Unlock()
 
 	l.Lock()
@@ -176,3 +195,25 @@ func GenerateLinks(md Markdown, cfg *Config) error {
 	g.discardWaiters()
 	return g.lastErr
 }
+
+// computeDirHash computes an hash on static directory of c.
+func computeDirHash(md Markdown, c Config) (string, error) {
+	dir := filepath.Join(md.Root, c.PathScope)
+	if _, err := os.Stat(dir); err != nil {
+		return "", err
+	}
+
+	hashString := ""
+	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if !info.IsDir() && c.IsValidExt(filepath.Ext(path)) {
+			hashString += fmt.Sprintf("%v%v%v%v", info.ModTime(), info.Name(), info.Size(), path)
+		}
+		return nil
+	})
+	if err != nil {
+		return "", err
+	}
+
+	sum := sha1.Sum([]byte(hashString))
+	return hex.EncodeToString(sum[:]), nil
+}

middleware/markdown/testdata/og/first.md

@@ -1 +1,5 @@
+---
+title: first_post
+sitename: title
+---
 # Test h1

middleware/markdown/testdata/og_static/og/first.md/index.html

@@ -1,7 +1,8 @@
+
 <!DOCTYPE html>
 <html>
 <head>
-<title>first_post</title>
+    <title>first_post</title>
 </head>
 <body>
 <h1>Header title</h1>
@@ -9,4 +10,4 @@
 <h1>Test h1</h1>
 
 </body>
-</html>
+</html>

middleware/markdown/watcher.go

@@ -0,0 +1,35 @@
+package markdown
+
+import "time"
+
+const DefaultInterval = time.Second * 60
+
+// Watch monitors the configured markdown directory for changes. It calls GenerateLinks
+// when there are changes.
+func Watch(md Markdown, c *Config, interval time.Duration) (stopChan chan struct{}) {
+	return TickerFunc(interval, func() {
+		GenerateLinks(md, c)
+	})
+}
+
+// TickerFunc runs f at interval. If interval is <= 0, it loops f. A message to the
+// returned channel will stop the executing goroutine.
+func TickerFunc(interval time.Duration, f func()) chan struct{} {
+	stopChan := make(chan struct{})
+
+	ticker := time.NewTicker(interval)
+	go func() {
+	loop:
+		for {
+			select {
+			case <-ticker.C:
+				f()
+			case <-stopChan:
+				ticker.Stop()
+				break loop
+			}
+		}
+	}()
+
+	return stopChan
+}

middleware/markdown/watcher_test.go

@@ -0,0 +1,34 @@
+package markdown
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestWatcher(t *testing.T) {
+	expected := "12345678"
+	interval := time.Millisecond * 100
+	i := 0
+	out := ""
+	stopChan := TickerFunc(interval, func() {
+		i++
+		out += fmt.Sprint(i)
+	})
+	time.Sleep(interval * 8)
+	stopChan <- struct{}{}
+	if expected != out {
+		t.Fatalf("Expected %v, found %v", expected, out)
+	}
+	out = ""
+	i = 0
+	stopChan = TickerFunc(interval, func() {
+		i++
+		out += fmt.Sprint(i)
+	})
+	time.Sleep(interval * 10)
+	if !strings.HasPrefix(out, expected) || out == expected {
+		t.Fatalf("expected (%v) must be a proper prefix of out(%v).", expected, out)
+	}
+}