Experimentally transform markdown for better presentation in full text search results

For issue #37

assets/search.js

@@ -37,7 +37,7 @@ function debouncer(interval, callback) {
 
         const query = input.value;
         fetch(
-            "_search?snippet_size=4&limit=4&q=" + encodeURIComponent(query),
+            "_search?snippet_size=10&limit=4&q=" + encodeURIComponent(query),
             {
                 headers: {
                     "Accept": "application/json",
@@ -56,7 +56,7 @@ function debouncer(interval, callback) {
                 item.querySelector('.link').href = hit.slug || ".";
                 item.querySelector('.link').setAttribute("data-focusindex", index + 1);
                 item.querySelector('.title').textContent = hit.title;
-                item.querySelector('.snippet').textContent = hit.snippet;
+                item.querySelector('.snippet').innerHTML = hit.snippet;
                 results.appendChild(item);
             })
 

assets/style.css

@@ -305,9 +305,6 @@ article ul.search-results {
 .search-result .title {
     font-weight: bold;
 }
-.snippet {
-    white-space: pre-line;
-}
 .search-result p {
     margin: 0;
 }

build.rs

@@ -11,6 +11,11 @@ use std::io::prelude::*;
 use std::path::Path;
 use walkdir::WalkDir;
 
+use std::ffi::CString;
+fn markdown_to_fts(_: &::diesel::sqlite::Context) -> CString {
+    panic!("Should never be called when running migrations on build.db")
+}
+
 fn main() {
     let out_dir = env::var("OUT_DIR").expect("cargo must set OUT_DIR");
     let db_path = Path::new(&out_dir).join("build.db");
@@ -18,14 +23,21 @@ fn main() {
 
     let _ignore_failure = std::fs::remove_file(db_path);
 
-    let connection = SqliteConnection::establish(db_path)
+    let mut connection = SqliteConnection::establish(db_path)
         .expect(&format!("Error esablishing a database connection to {}", db_path));
 
     // Integer is a dummy placeholder. Compiling fails when passing ().
-    diesel::expression::sql_literal::sql::<(diesel::types::Integer)>("PRAGMA foreign_keys = ON")
+    diesel::expression::sql_literal::sql::<(diesel::sql_types::Integer)>("PRAGMA foreign_keys = ON")
         .execute(&connection)
         .expect("Should be able to enable foreign keys");
 
+    connection.create_scalar_function(
+        "markdown_to_fts",
+        1,
+        true,
+        markdown_to_fts,
+    ).unwrap();
+
     diesel_migrations::run_pending_migrations(&connection).unwrap();
 
     let infer_schema_path = Path::new(&out_dir).join("infer_schema.rs");

migrations/20180119150706_convert_markdown_for_fts/up.sql

@@ -0,0 +1,24 @@
+DROP TRIGGER article_revisions_ai;
+DROP TRIGGER article_revisions_ad;
+DROP TRIGGER article_revisions_au_disable;
+DROP TRIGGER article_revisions_au_enable;
+
+CREATE TRIGGER article_revisions_ai AFTER INSERT ON article_revisions WHEN new.latest = 1 BEGIN
+  DELETE FROM article_search WHERE rowid = new.article_id;
+  INSERT INTO article_search(rowid, title, body, slug) VALUES (new.article_id, new.title, markdown_to_fts(new.body), new.slug);
+END;
+CREATE TRIGGER article_revisions_ad AFTER DELETE ON article_revisions WHEN old.latest = 1 BEGIN
+  DELETE FROM article_search WHERE rowid = old.article_id;
+END;
+
+-- Index unique_latest_revision_per_article_id makes sure the following is sufficient:
+CREATE TRIGGER article_revisions_au_disable AFTER UPDATE ON article_revisions WHEN old.latest = 1 AND new.latest = 0 BEGIN
+  DELETE FROM article_search WHERE rowid = old.article_id;
+END;
+CREATE TRIGGER article_revisions_au_enable AFTER UPDATE ON article_revisions WHEN old.latest = 0 AND new.latest = 1 BEGIN
+  INSERT INTO article_search(rowid, title, body, slug) VALUES (new.article_id, new.title, markdown_to_fts(new.body), new.slug);
+END;
+
+DELETE FROM article_search;
+INSERT INTO article_search(title, body, slug)
+    SELECT title, markdown_to_fts(body), slug FROM article_revisions WHERE latest = 1;

src/db.rs

@@ -9,12 +9,27 @@ embed_migrations!();
 #[derive(Debug)]
 struct SqliteInitializer;
 
+use std::ffi::CString;
+
+fn markdown_to_fts(ctx: &::diesel::sqlite::Context) -> CString {
+    use rendering;
+    CString::new(rendering::render_markdown_for_fts(&ctx.get::<String>(0))).unwrap()
+}
+
 impl CustomizeConnection<SqliteConnection, r2d2_diesel::Error> for SqliteInitializer {
     fn on_acquire(&self, conn: &mut SqliteConnection) -> Result<(), r2d2_diesel::Error> {
         sql::<(Integer)>("PRAGMA foreign_keys = ON")
             .execute(conn)
-            .and(Ok(()))
+            .map_err(|x| r2d2_diesel::Error::QueryError(x))?;
-            .map_err(|x| r2d2_diesel::Error::QueryError(x))
+
+        conn.create_scalar_function(
+            "markdown_to_fts",
+            1,
+            true,
+            markdown_to_fts,
+        ).map_err(|x| r2d2_diesel::Error::QueryError(x))?;
+
+        Ok(())
     }
 }
 

src/rendering.rs

@@ -1,4 +1,5 @@
 use pulldown_cmark::{Parser, html, OPTION_ENABLE_TABLES, OPTION_DISABLE_HTML};
+use pulldown_cmark::Event::Text;
 
 pub fn render_markdown(src: &str) -> String {
     let opts = OPTION_ENABLE_TABLES | OPTION_DISABLE_HTML;
@@ -7,3 +8,22 @@ pub fn render_markdown(src: &str) -> String {
     html::push_html(&mut buf, p);
     buf
 }
+
+pub fn render_markdown_for_fts(src: &str) -> String {
+    let opts = OPTION_ENABLE_TABLES | OPTION_DISABLE_HTML;
+    let p = Parser::new_ext(src, opts);
+    let mut buf = String::new();
+
+    for event in p {
+        match event {
+            Text(text) => buf.push_str(&text),
+            _ => buf.push_str(" "),
+        }
+    }
+
+    buf.replace('&', "");
+    buf.replace('<', "");
+    buf.replace('>', "");
+
+    buf
+}

src/resources/search_resource.rs

@@ -12,7 +12,7 @@ use state::State;
 use web::{Resource, ResponseFuture};
 
 const DEFAULT_LIMIT: u32 = 10;
-const DEFAULT_SNIPPET_SIZE: u32 = 8;
+const DEFAULT_SNIPPET_SIZE: u32 = 25;
 
 type BoxResource = Box<Resource + Sync + Send>;
 

src/state.rs

@@ -386,7 +386,7 @@ impl<'a> SyncState<'a> {
 
         Ok(
             sql_query(
-                "SELECT title, snippet(article_search, 1, '', '', '\u{2026}', ?) AS snippet, slug \
+                "SELECT title, snippet(article_search, 1, '<em>', '</em>', '\u{2026}', ?) AS snippet, slug \
                     FROM article_search \
                     WHERE article_search MATCH ? \
                     ORDER BY rank \

templates/search.html

@@ -13,7 +13,7 @@
 
 <ul class="search-results default-keyboard-focus-control">
     {{#hits}}
-    <li class="search-result"><a data-focusindex="{{.0}}" class="link" href="{{.1.link()}}"><p class="title">{{.1.title}}</p><p class="snippet">{{.1.snippet}}</p></a></li>
+    <li class="search-result"><a data-focusindex="{{.0}}" class="link" href="{{.1.link()}}"><p class="title">{{.1.title}}</p><p class="snippet">{{{.1.snippet}}}</p></a></li>
     {{/hits}}
 </ul>
 {{/hits}}